From e23979df9f92ef0a31f88296fc86a46a3812708e Mon Sep 17 00:00:00 2001 From: LFC <990479+MichaelScofield@users.noreply.github.com> Date: Thu, 10 Apr 2025 10:05:50 +0800 Subject: [PATCH 01/82] chore: un-allow clippy's "readonly_write_lock" (#5862) --- Cargo.toml | 1 - src/log-store/src/raft_engine/backend.rs | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 9c36a76805..38b749e7b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,7 +77,6 @@ clippy.print_stdout = "warn" clippy.print_stderr = "warn" clippy.dbg_macro = "warn" clippy.implicit_clone = "warn" -clippy.readonly_write_lock = "allow" rust.unknown_lints = "deny" rust.unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tokio_unstable)'] } diff --git a/src/log-store/src/raft_engine/backend.rs b/src/log-store/src/raft_engine/backend.rs index 3d41e5298d..8d27994f8b 100644 --- a/src/log-store/src/raft_engine/backend.rs +++ b/src/log-store/src/raft_engine/backend.rs @@ -114,7 +114,13 @@ impl TxnService for RaftEngineBackend { } = txn.into(); let mut succeeded = true; + + // Here we are using the write lock to guard against parallel access inside "txn", and + // outside "get" or "put" etc. It doesn't serve the purpose of mutating some Rust data, so + // the variable is not "mut". Suppress the clippy warning because of this. + #[allow(clippy::readonly_write_lock)] let engine = self.engine.write().unwrap(); + for cmp in compare { let existing_value = engine_get(&engine, &cmp.key)?.map(|kv| kv.value); if !cmp.compare_value(existing_value.as_ref()) { From e052c65a5856226b1d0b72dfc215454720e81999 Mon Sep 17 00:00:00 2001 From: LFC <990479+MichaelScofield@users.noreply.github.com> Date: Thu, 10 Apr 2025 14:30:29 +0800 Subject: [PATCH 02/82] chore: remove repl (#5860) --- Cargo.lock | 1 - src/cli/src/error.rs | 63 +------- src/cli/src/lib.rs | 3 - src/cli/src/repl.rs | 299 -------------------------------------- src/cmd/Cargo.toml | 1 - src/cmd/src/error.rs | 63 +------- src/cmd/tests/cli.rs | 148 ------------------- src/frontend/src/error.rs | 10 +- 8 files changed, 3 insertions(+), 585 deletions(-) delete mode 100644 src/cli/src/repl.rs delete mode 100644 src/cmd/tests/cli.rs diff --git a/Cargo.lock b/Cargo.lock index 188f9ce144..b19e1bb75f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1832,7 +1832,6 @@ dependencies = [ "regex", "reqwest", "rexpect", - "rustyline", "serde", "serde_json", "servers", diff --git a/src/cli/src/error.rs b/src/cli/src/error.rs index be852e7d73..2c18531aaa 100644 --- a/src/cli/src/error.rs +++ b/src/cli/src/error.rs @@ -17,7 +17,6 @@ use std::any::Any; use common_error::ext::{BoxedError, ErrorExt}; use common_error::status_code::StatusCode; use common_macro::stack_trace_debug; -use rustyline::error::ReadlineError; use snafu::{Location, Snafu}; #[derive(Snafu)] @@ -105,52 +104,6 @@ pub enum Error { #[snafu(display("Invalid REPL command: {reason}"))] InvalidReplCommand { reason: String }, - #[snafu(display("Cannot create REPL"))] - ReplCreation { - #[snafu(source)] - error: ReadlineError, - #[snafu(implicit)] - location: Location, - }, - - #[snafu(display("Error reading command"))] - Readline { - #[snafu(source)] - error: ReadlineError, - #[snafu(implicit)] - location: Location, - }, - - #[snafu(display("Failed to request database, sql: {sql}"))] - RequestDatabase { - sql: String, - #[snafu(source)] - source: client::Error, - #[snafu(implicit)] - location: Location, - }, - - #[snafu(display("Failed to collect RecordBatches"))] - CollectRecordBatches { - #[snafu(implicit)] - location: Location, - source: common_recordbatch::error::Error, - }, - - #[snafu(display("Failed to pretty print Recordbatches"))] - PrettyPrintRecordBatches { - #[snafu(implicit)] - location: Location, - source: common_recordbatch::error::Error, - }, - - #[snafu(display("Failed to start Meta client"))] - StartMetaClient { - #[snafu(implicit)] - location: Location, - source: meta_client::error::Error, - }, - #[snafu(display("Failed to parse SQL: {}", sql))] ParseSql { sql: String, @@ -166,13 +119,6 @@ pub enum Error { source: query::error::Error, }, - #[snafu(display("Failed to encode logical plan in substrait"))] - SubstraitEncodeLogicalPlan { - #[snafu(implicit)] - location: Location, - source: substrait::error::Error, - }, - #[snafu(display("Failed to load layered config"))] LoadLayeredConfig { #[snafu(source(from(common_config::error::Error, Box::new)))] @@ -318,17 +264,10 @@ impl ErrorExt for Error { Error::StartProcedureManager { source, .. } | Error::StopProcedureManager { source, .. } => source.status_code(), Error::StartWalOptionsAllocator { source, .. } => source.status_code(), - Error::ReplCreation { .. } | Error::Readline { .. } | Error::HttpQuerySql { .. } => { - StatusCode::Internal - } - Error::RequestDatabase { source, .. } => source.status_code(), - Error::CollectRecordBatches { source, .. } - | Error::PrettyPrintRecordBatches { source, .. } => source.status_code(), - Error::StartMetaClient { source, .. } => source.status_code(), + Error::HttpQuerySql { .. } => StatusCode::Internal, Error::ParseSql { source, .. } | Error::PlanStatement { source, .. } => { source.status_code() } - Error::SubstraitEncodeLogicalPlan { source, .. } => source.status_code(), Error::SerdeJson { .. } | Error::FileIo { .. } diff --git a/src/cli/src/lib.rs b/src/cli/src/lib.rs index 3991f3a666..113e88f1c1 100644 --- a/src/cli/src/lib.rs +++ b/src/cli/src/lib.rs @@ -23,15 +23,12 @@ mod helper; // Wait for https://github.com/GreptimeTeam/greptimedb/issues/2373 mod database; mod import; -#[allow(unused)] -mod repl; use async_trait::async_trait; use clap::Parser; use common_error::ext::BoxedError; pub use database::DatabaseClient; use error::Result; -pub use repl::Repl; pub use crate::bench::BenchTableMetadataCommand; pub use crate::export::ExportCommand; diff --git a/src/cli/src/repl.rs b/src/cli/src/repl.rs deleted file mode 100644 index 8b5e3aa389..0000000000 --- a/src/cli/src/repl.rs +++ /dev/null @@ -1,299 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::path::PathBuf; -use std::sync::Arc; -use std::time::Instant; - -use cache::{ - build_fundamental_cache_registry, with_default_composite_cache_registry, TABLE_CACHE_NAME, - TABLE_ROUTE_CACHE_NAME, -}; -use catalog::information_extension::DistributedInformationExtension; -use catalog::kvbackend::{ - CachedKvBackend, CachedKvBackendBuilder, KvBackendCatalogManager, MetaKvBackend, -}; -use client::{Client, Database, OutputData, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; -use common_base::Plugins; -use common_config::Mode; -use common_error::ext::ErrorExt; -use common_meta::cache::{CacheRegistryBuilder, LayeredCacheRegistryBuilder}; -use common_meta::kv_backend::KvBackendRef; -use common_query::Output; -use common_recordbatch::RecordBatches; -use common_telemetry::debug; -use either::Either; -use meta_client::client::{ClusterKvBackend, MetaClientBuilder}; -use query::datafusion::DatafusionQueryEngine; -use query::parser::QueryLanguageParser; -use query::query_engine::{DefaultSerializer, QueryEngineState}; -use query::QueryEngine; -use rustyline::error::ReadlineError; -use rustyline::Editor; -use session::context::QueryContext; -use snafu::{OptionExt, ResultExt}; -use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan}; - -use crate::cmd::ReplCommand; -use crate::error::{ - CollectRecordBatchesSnafu, ParseSqlSnafu, PlanStatementSnafu, PrettyPrintRecordBatchesSnafu, - ReadlineSnafu, ReplCreationSnafu, RequestDatabaseSnafu, Result, StartMetaClientSnafu, - SubstraitEncodeLogicalPlanSnafu, -}; -use crate::helper::RustylineHelper; -use crate::{error, AttachCommand}; - -/// Captures the state of the repl, gathers commands and executes them one by one -pub struct Repl { - /// Rustyline editor for interacting with user on command line - rl: Editor, - - /// Current prompt - prompt: String, - - /// Client for interacting with GreptimeDB - database: Database, - - query_engine: Option, -} - -#[allow(clippy::print_stdout)] -impl Repl { - fn print_help(&self) { - println!("{}", ReplCommand::help()) - } - - pub(crate) async fn try_new(cmd: &AttachCommand) -> Result { - let mut rl = Editor::new().context(ReplCreationSnafu)?; - - if !cmd.disable_helper { - rl.set_helper(Some(RustylineHelper::default())); - - let history_file = history_file(); - if let Err(e) = rl.load_history(&history_file) { - debug!( - "failed to load history file on {}, error: {e}", - history_file.display() - ); - } - } - - let client = Client::with_urls([&cmd.grpc_addr]); - let database = Database::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, client); - - let query_engine = if let Some(meta_addr) = &cmd.meta_addr { - create_query_engine(meta_addr).await.map(Some)? - } else { - None - }; - - Ok(Self { - rl, - prompt: "> ".to_string(), - database, - query_engine, - }) - } - - /// Parse the next command - fn next_command(&mut self) -> Result { - match self.rl.readline(&self.prompt) { - Ok(ref line) => { - let request = line.trim(); - - let _ = self.rl.add_history_entry(request.to_string()); - - request.try_into() - } - Err(ReadlineError::Eof) | Err(ReadlineError::Interrupted) => Ok(ReplCommand::Exit), - // Some sort of real underlying error - Err(e) => Err(e).context(ReadlineSnafu), - } - } - - /// Read Evaluate Print Loop (interactive command line) for GreptimeDB - /// - /// Inspired / based on repl.rs from InfluxDB IOX - pub(crate) async fn run(&mut self) -> Result<()> { - println!("Ready for commands. (Hint: try 'help')"); - - loop { - match self.next_command()? { - ReplCommand::Help => { - self.print_help(); - } - ReplCommand::UseDatabase { db_name } => { - if self.execute_sql(format!("USE {db_name}")).await { - println!("Using {db_name}"); - self.database.set_schema(&db_name); - self.prompt = format!("[{db_name}] > "); - } - } - ReplCommand::Sql { sql } => { - let _ = self.execute_sql(sql).await; - } - ReplCommand::Exit => { - return Ok(()); - } - } - } - } - - async fn execute_sql(&self, sql: String) -> bool { - self.do_execute_sql(sql) - .await - .map_err(|e| { - let status_code = e.status_code(); - let root_cause = e.output_msg(); - println!("Error: {}({status_code}), {root_cause}", status_code as u32) - }) - .is_ok() - } - - async fn do_execute_sql(&self, sql: String) -> Result<()> { - let start = Instant::now(); - - let output = if let Some(query_engine) = &self.query_engine { - let query_ctx = Arc::new(QueryContext::with( - self.database.catalog(), - self.database.schema(), - )); - - let stmt = QueryLanguageParser::parse_sql(&sql, &query_ctx) - .with_context(|_| ParseSqlSnafu { sql: sql.clone() })?; - - let plan = query_engine - .planner() - .plan(&stmt, query_ctx.clone()) - .await - .context(PlanStatementSnafu)?; - - let plan = query_engine - .optimize(&query_engine.engine_context(query_ctx), &plan) - .context(PlanStatementSnafu)?; - - let plan = DFLogicalSubstraitConvertor {} - .encode(&plan, DefaultSerializer) - .context(SubstraitEncodeLogicalPlanSnafu)?; - - self.database.logical_plan(plan.to_vec()).await - } else { - self.database.sql(&sql).await - } - .context(RequestDatabaseSnafu { sql: &sql })?; - - let either = match output.data { - OutputData::Stream(s) => { - let x = RecordBatches::try_collect(s) - .await - .context(CollectRecordBatchesSnafu)?; - Either::Left(x) - } - OutputData::RecordBatches(x) => Either::Left(x), - OutputData::AffectedRows(rows) => Either::Right(rows), - }; - - let end = Instant::now(); - - match either { - Either::Left(recordbatches) => { - let total_rows: usize = recordbatches.iter().map(|x| x.num_rows()).sum(); - if total_rows > 0 { - println!( - "{}", - recordbatches - .pretty_print() - .context(PrettyPrintRecordBatchesSnafu)? - ); - } - println!("Total Rows: {total_rows}") - } - Either::Right(rows) => println!("Affected Rows: {rows}"), - }; - - println!("Cost {} ms", (end - start).as_millis()); - Ok(()) - } -} - -impl Drop for Repl { - fn drop(&mut self) { - if self.rl.helper().is_some() { - let history_file = history_file(); - if let Err(e) = self.rl.save_history(&history_file) { - debug!( - "failed to save history file on {}, error: {e}", - history_file.display() - ); - } - } - } -} - -/// Return the location of the history file (defaults to $HOME/".greptimedb_cli_history") -fn history_file() -> PathBuf { - let mut buf = match std::env::var("HOME") { - Ok(home) => PathBuf::from(home), - Err(_) => PathBuf::new(), - }; - buf.push(".greptimedb_cli_history"); - buf -} - -async fn create_query_engine(meta_addr: &str) -> Result { - let mut meta_client = MetaClientBuilder::default().enable_store().build(); - meta_client - .start([meta_addr]) - .await - .context(StartMetaClientSnafu)?; - let meta_client = Arc::new(meta_client); - - let cached_meta_backend = Arc::new( - CachedKvBackendBuilder::new(Arc::new(MetaKvBackend::new(meta_client.clone()))).build(), - ); - let layered_cache_builder = LayeredCacheRegistryBuilder::default().add_cache_registry( - CacheRegistryBuilder::default() - .add_cache(cached_meta_backend.clone()) - .build(), - ); - let fundamental_cache_registry = - build_fundamental_cache_registry(Arc::new(MetaKvBackend::new(meta_client.clone()))); - let layered_cache_registry = Arc::new( - with_default_composite_cache_registry( - layered_cache_builder.add_cache_registry(fundamental_cache_registry), - ) - .context(error::BuildCacheRegistrySnafu)? - .build(), - ); - - let information_extension = Arc::new(DistributedInformationExtension::new(meta_client.clone())); - let catalog_manager = KvBackendCatalogManager::new( - information_extension, - cached_meta_backend.clone(), - layered_cache_registry, - None, - ); - let plugins: Plugins = Default::default(); - let state = Arc::new(QueryEngineState::new( - catalog_manager, - None, - None, - None, - None, - false, - plugins.clone(), - )); - - Ok(DatafusionQueryEngine::new(state, plugins)) -} diff --git a/src/cmd/Cargo.toml b/src/cmd/Cargo.toml index c3328fbc8d..b3ffd479a6 100644 --- a/src/cmd/Cargo.toml +++ b/src/cmd/Cargo.toml @@ -68,7 +68,6 @@ query.workspace = true rand.workspace = true regex.workspace = true reqwest.workspace = true -rustyline = "10.1" serde.workspace = true serde_json.workspace = true servers.workspace = true diff --git a/src/cmd/src/error.rs b/src/cmd/src/error.rs index 8697710985..a671290503 100644 --- a/src/cmd/src/error.rs +++ b/src/cmd/src/error.rs @@ -17,7 +17,6 @@ use std::any::Any; use common_error::ext::{BoxedError, ErrorExt}; use common_error::status_code::StatusCode; use common_macro::stack_trace_debug; -use rustyline::error::ReadlineError; use snafu::{Location, Snafu}; #[derive(Snafu)] @@ -181,52 +180,6 @@ pub enum Error { #[snafu(display("Invalid REPL command: {reason}"))] InvalidReplCommand { reason: String }, - #[snafu(display("Cannot create REPL"))] - ReplCreation { - #[snafu(source)] - error: ReadlineError, - #[snafu(implicit)] - location: Location, - }, - - #[snafu(display("Error reading command"))] - Readline { - #[snafu(source)] - error: ReadlineError, - #[snafu(implicit)] - location: Location, - }, - - #[snafu(display("Failed to request database, sql: {sql}"))] - RequestDatabase { - sql: String, - #[snafu(source)] - source: client::Error, - #[snafu(implicit)] - location: Location, - }, - - #[snafu(display("Failed to collect RecordBatches"))] - CollectRecordBatches { - #[snafu(implicit)] - location: Location, - source: common_recordbatch::error::Error, - }, - - #[snafu(display("Failed to pretty print Recordbatches"))] - PrettyPrintRecordBatches { - #[snafu(implicit)] - location: Location, - source: common_recordbatch::error::Error, - }, - - #[snafu(display("Failed to start Meta client"))] - StartMetaClient { - #[snafu(implicit)] - location: Location, - source: meta_client::error::Error, - }, - #[snafu(display("Failed to parse SQL: {}", sql))] ParseSql { sql: String, @@ -242,13 +195,6 @@ pub enum Error { source: query::error::Error, }, - #[snafu(display("Failed to encode logical plan in substrait"))] - SubstraitEncodeLogicalPlan { - #[snafu(implicit)] - location: Location, - source: substrait::error::Error, - }, - #[snafu(display("Failed to load layered config"))] LoadLayeredConfig { #[snafu(source(from(common_config::error::Error, Box::new)))] @@ -395,17 +341,10 @@ impl ErrorExt for Error { | Error::StopProcedureManager { source, .. } => source.status_code(), Error::BuildWalOptionsAllocator { source, .. } | Error::StartWalOptionsAllocator { source, .. } => source.status_code(), - Error::ReplCreation { .. } | Error::Readline { .. } | Error::HttpQuerySql { .. } => { - StatusCode::Internal - } - Error::RequestDatabase { source, .. } => source.status_code(), - Error::CollectRecordBatches { source, .. } - | Error::PrettyPrintRecordBatches { source, .. } => source.status_code(), - Error::StartMetaClient { source, .. } => source.status_code(), + Error::HttpQuerySql { .. } => StatusCode::Internal, Error::ParseSql { source, .. } | Error::PlanStatement { source, .. } => { source.status_code() } - Error::SubstraitEncodeLogicalPlan { source, .. } => source.status_code(), Error::SerdeJson { .. } | Error::FileIo { .. } diff --git a/src/cmd/tests/cli.rs b/src/cmd/tests/cli.rs deleted file mode 100644 index dfea9afc3e..0000000000 --- a/src/cmd/tests/cli.rs +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#[cfg(target_os = "macos")] -mod tests { - use std::path::PathBuf; - use std::process::{Command, Stdio}; - use std::time::Duration; - - use common_test_util::temp_dir::create_temp_dir; - use rexpect::session::PtyReplSession; - - struct Repl { - repl: PtyReplSession, - } - - impl Repl { - fn send_line(&mut self, line: &str) { - let _ = self.repl.send_line(line).unwrap(); - - // read a line to consume the prompt - let _ = self.read_line(); - } - - fn read_line(&mut self) -> String { - self.repl.read_line().unwrap() - } - - fn read_expect(&mut self, expect: &str) { - assert_eq!(self.read_line(), expect); - } - - fn read_contains(&mut self, pat: &str) { - assert!(self.read_line().contains(pat)); - } - } - - // TODO(LFC): Un-ignore this REPL test. - // Ignore this REPL test because some logical plans like create database are not supported yet in Datanode. - #[ignore] - #[test] - fn test_repl() { - let data_home = create_temp_dir("data"); - let wal_dir = create_temp_dir("wal"); - - let mut bin_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - bin_path.push("../../target/debug"); - let bin_path = bin_path.to_str().unwrap(); - - let mut datanode = Command::new("./greptime") - .current_dir(bin_path) - .args([ - "datanode", - "start", - "--rpc-bind-addr=0.0.0.0:4321", - "--node-id=1", - &format!("--data-home={}", data_home.path().display()), - &format!("--wal-dir={}", wal_dir.path().display()), - ]) - .stdout(Stdio::null()) - .spawn() - .unwrap(); - - // wait for Datanode actually started - std::thread::sleep(Duration::from_secs(3)); - - let mut repl_cmd = Command::new("./greptime"); - let _ = repl_cmd.current_dir(bin_path).args([ - "--log-level=off", - "cli", - "attach", - "--grpc-bind-addr=0.0.0.0:4321", - // history commands can sneaky into stdout and mess up our tests, so disable it - "--disable-helper", - ]); - let pty_session = rexpect::session::spawn_command(repl_cmd, Some(5_000)).unwrap(); - let repl = PtyReplSession { - prompt: "> ".to_string(), - pty_session, - quit_command: None, - echo_on: false, - }; - let repl = &mut Repl { repl }; - repl.read_expect("Ready for commands. (Hint: try 'help')"); - - test_create_database(repl); - - test_use_database(repl); - - test_create_table(repl); - - test_insert(repl); - - test_select(repl); - - datanode.kill().unwrap(); - let _ = datanode.wait().unwrap(); - } - - fn test_create_database(repl: &mut Repl) { - repl.send_line("CREATE DATABASE db;"); - repl.read_expect("Affected Rows: 1"); - repl.read_contains("Cost"); - } - - fn test_use_database(repl: &mut Repl) { - repl.send_line("USE db"); - repl.read_expect("Total Rows: 0"); - repl.read_contains("Cost"); - repl.read_expect("Using db"); - } - - fn test_create_table(repl: &mut Repl) { - repl.send_line("CREATE TABLE t(x STRING, ts TIMESTAMP TIME INDEX);"); - repl.read_expect("Affected Rows: 0"); - repl.read_contains("Cost"); - } - - fn test_insert(repl: &mut Repl) { - repl.send_line("INSERT INTO t(x, ts) VALUES ('hello', 1676895812239);"); - repl.read_expect("Affected Rows: 1"); - repl.read_contains("Cost"); - } - - fn test_select(repl: &mut Repl) { - repl.send_line("SELECT * FROM t;"); - - repl.read_expect("+-------+-------------------------+"); - repl.read_expect("| x | ts |"); - repl.read_expect("+-------+-------------------------+"); - repl.read_expect("| hello | 2023-02-20T12:23:32.239 |"); - repl.read_expect("+-------+-------------------------+"); - repl.read_expect("Total Rows: 1"); - - repl.read_contains("Cost"); - } -} diff --git a/src/frontend/src/error.rs b/src/frontend/src/error.rs index 7d599cb0ce..99edbdbc62 100644 --- a/src/frontend/src/error.rs +++ b/src/frontend/src/error.rs @@ -128,13 +128,6 @@ pub enum Error { source: catalog::error::Error, }, - #[snafu(display("Failed to start Meta client"))] - StartMetaClient { - #[snafu(implicit)] - location: Location, - source: meta_client::error::Error, - }, - #[snafu(display("Failed to create heartbeat stream to Metasrv"))] CreateMetaHeartbeatStream { source: meta_client::error::Error, @@ -415,8 +408,7 @@ impl ErrorExt for Error { Error::Catalog { source, .. } => source.status_code(), - Error::StartMetaClient { source, .. } - | Error::CreateMetaHeartbeatStream { source, .. } => source.status_code(), + Error::CreateMetaHeartbeatStream { source, .. } => source.status_code(), Error::PlanStatement { source, .. } | Error::ReadTable { source, .. } From 54ef29f3949f2ceed2bd8f401498a18c0ca9fe1e Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Thu, 10 Apr 2025 14:55:46 +0800 Subject: [PATCH 03/82] feat: add `catalog_manager` to `ProcedureServiceHandler` (#5873) --- Cargo.lock | 1 + src/common/function/Cargo.toml | 1 + src/common/function/src/handlers.rs | 4 ++++ src/common/function/src/state.rs | 5 +++++ src/frontend/src/instance/builder.rs | 1 + src/operator/src/procedure.rs | 16 ++++++++++++++-- 6 files changed, 26 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b19e1bb75f..cba1fa8793 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2014,6 +2014,7 @@ dependencies = [ "arc-swap", "async-trait", "bincode", + "catalog", "chrono", "common-base", "common-catalog", diff --git a/src/common/function/Cargo.toml b/src/common/function/Cargo.toml index 7a4c968a3e..73821a896a 100644 --- a/src/common/function/Cargo.toml +++ b/src/common/function/Cargo.toml @@ -18,6 +18,7 @@ api.workspace = true arc-swap = "1.0" async-trait.workspace = true bincode = "1.3" +catalog.workspace = true chrono.workspace = true common-base.workspace = true common-catalog.workspace = true diff --git a/src/common/function/src/handlers.rs b/src/common/function/src/handlers.rs index 1d994731d5..bcb6ce5460 100644 --- a/src/common/function/src/handlers.rs +++ b/src/common/function/src/handlers.rs @@ -15,6 +15,7 @@ use std::sync::Arc; use async_trait::async_trait; +use catalog::CatalogManagerRef; use common_base::AffectedRows; use common_meta::rpc::procedure::{ AddRegionFollowerRequest, MigrateRegionRequest, ProcedureStateResponse, @@ -72,6 +73,9 @@ pub trait ProcedureServiceHandler: Send + Sync { /// Remove a region follower from a region. async fn remove_region_follower(&self, request: RemoveRegionFollowerRequest) -> Result<()>; + + /// Get the catalog manager + fn catalog_manager(&self) -> &CatalogManagerRef; } /// This flow service handler is only use for flush flow for now. diff --git a/src/common/function/src/state.rs b/src/common/function/src/state.rs index 66f5463fa2..211f7e1438 100644 --- a/src/common/function/src/state.rs +++ b/src/common/function/src/state.rs @@ -34,6 +34,7 @@ impl FunctionState { use api::v1::meta::ProcedureStatus; use async_trait::async_trait; + use catalog::CatalogManagerRef; use common_base::AffectedRows; use common_meta::rpc::procedure::{ AddRegionFollowerRequest, MigrateRegionRequest, ProcedureStateResponse, @@ -80,6 +81,10 @@ impl FunctionState { ) -> Result<()> { Ok(()) } + + fn catalog_manager(&self) -> &CatalogManagerRef { + unimplemented!() + } } #[async_trait] diff --git a/src/frontend/src/instance/builder.rs b/src/frontend/src/instance/builder.rs index 52b2463503..8503999b2c 100644 --- a/src/frontend/src/instance/builder.rs +++ b/src/frontend/src/instance/builder.rs @@ -152,6 +152,7 @@ impl FrontendBuilder { let procedure_service_handler = Arc::new(ProcedureServiceOperator::new( self.procedure_executor.clone(), + self.catalog_manager.clone(), )); let flow_metadata_manager = Arc::new(FlowMetadataManager::new(kv_backend.clone())); diff --git a/src/operator/src/procedure.rs b/src/operator/src/procedure.rs index e2c27c024f..87f805acb1 100644 --- a/src/operator/src/procedure.rs +++ b/src/operator/src/procedure.rs @@ -13,6 +13,7 @@ // limitations under the License. use async_trait::async_trait; +use catalog::CatalogManagerRef; use common_error::ext::BoxedError; use common_function::handlers::ProcedureServiceHandler; use common_meta::ddl::{ExecutorContext, ProcedureExecutorRef}; @@ -28,11 +29,18 @@ use snafu::ResultExt; #[derive(Clone)] pub struct ProcedureServiceOperator { procedure_executor: ProcedureExecutorRef, + catalog_manager: CatalogManagerRef, } impl ProcedureServiceOperator { - pub fn new(procedure_executor: ProcedureExecutorRef) -> Self { - Self { procedure_executor } + pub fn new( + procedure_executor: ProcedureExecutorRef, + catalog_manager: CatalogManagerRef, + ) -> Self { + Self { + procedure_executor, + catalog_manager, + } } } @@ -75,4 +83,8 @@ impl ProcedureServiceHandler for ProcedureServiceOperator { .map_err(BoxedError::new) .context(query_error::ProcedureServiceSnafu) } + + fn catalog_manager(&self) -> &CatalogManagerRef { + &self.catalog_manager + } } From dce5e35d7c3c2908bfa251c5db26abe8caec9f22 Mon Sep 17 00:00:00 2001 From: Zhenchi Date: Thu, 10 Apr 2025 15:32:15 +0800 Subject: [PATCH 04/82] feat: apply terms with fulltext tantivy backend (#5869) * feat: apply terms with fulltext tantivy backend Signed-off-by: Zhenchi * fix test Signed-off-by: Zhenchi * address comments Signed-off-by: Zhenchi --------- Signed-off-by: Zhenchi --- src/index/src/fulltext_index.rs | 41 ++- .../src/fulltext_index/create/bloom_filter.rs | 4 +- .../src/fulltext_index/create/tantivy.rs | 62 ++-- .../src/fulltext_index/search/tantivy.rs | 6 +- src/index/src/fulltext_index/tests.rs | 43 ++- .../src/sst/index/fulltext_index/applier.rs | 36 ++- .../index/fulltext_index/applier/builder.rs | 115 ++++++- .../src/sst/index/fulltext_index/creator.rs | 299 ++++++++++++++---- src/mito2/src/sst/index/puffin_manager.rs | 4 +- src/puffin/src/puffin_manager.rs | 75 +++-- .../fs_puffin_manager/reader.rs | 84 ++--- .../fs_puffin_manager/writer.rs | 10 +- src/puffin/src/puffin_manager/tests.rs | 8 +- 13 files changed, 591 insertions(+), 196 deletions(-) diff --git a/src/index/src/fulltext_index.rs b/src/index/src/fulltext_index.rs index 3a7f58c8ab..4cbbbdf477 100644 --- a/src/index/src/fulltext_index.rs +++ b/src/index/src/fulltext_index.rs @@ -12,18 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. +use puffin::blob_metadata::BlobMetadata; use serde::{Deserialize, Serialize}; - +use snafu::ResultExt; +use tantivy::tokenizer::{LowerCaser, SimpleTokenizer, TextAnalyzer, TokenizerManager}; +use tantivy_jieba::JiebaTokenizer; pub mod create; pub mod error; pub mod search; pub mod tokenizer; +pub const KEY_FULLTEXT_CONFIG: &str = "fulltext_config"; + +use crate::fulltext_index::error::{DeserializeFromJsonSnafu, Result}; + #[cfg(test)] mod tests; /// Configuration for fulltext index. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] pub struct Config { /// Analyzer to use for tokenization. pub analyzer: Analyzer, @@ -33,10 +40,38 @@ pub struct Config { } /// Analyzer to use for tokenization. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] pub enum Analyzer { #[default] English, Chinese, } + +impl Config { + fn build_tantivy_tokenizer(&self) -> TokenizerManager { + let mut builder = match self.analyzer { + Analyzer::English => TextAnalyzer::builder(SimpleTokenizer::default()).dynamic(), + Analyzer::Chinese => TextAnalyzer::builder(JiebaTokenizer {}).dynamic(), + }; + + if !self.case_sensitive { + builder = builder.filter_dynamic(LowerCaser); + } + + let tokenizer = builder.build(); + let tokenizer_manager = TokenizerManager::new(); + tokenizer_manager.register("default", tokenizer); + tokenizer_manager + } + + /// Extracts the fulltext index configuration from the blob metadata. + pub fn from_blob_metadata(metadata: &BlobMetadata) -> Result { + if let Some(config) = metadata.properties.get(KEY_FULLTEXT_CONFIG) { + let config = serde_json::from_str(config).context(DeserializeFromJsonSnafu)?; + return Ok(config); + } + + Ok(Self::default()) + } +} diff --git a/src/index/src/fulltext_index/create/bloom_filter.rs b/src/index/src/fulltext_index/create/bloom_filter.rs index 970f89d65d..127464db71 100644 --- a/src/index/src/fulltext_index/create/bloom_filter.rs +++ b/src/index/src/fulltext_index/create/bloom_filter.rs @@ -30,12 +30,10 @@ use crate::fulltext_index::error::{ SerializeToJsonSnafu, }; use crate::fulltext_index::tokenizer::{Analyzer, ChineseTokenizer, EnglishTokenizer}; -use crate::fulltext_index::Config; +use crate::fulltext_index::{Config, KEY_FULLTEXT_CONFIG}; const PIPE_BUFFER_SIZE_FOR_SENDING_BLOB: usize = 8192; -pub const KEY_FULLTEXT_CONFIG: &str = "fulltext_config"; - /// `BloomFilterFulltextIndexCreator` is for creating a fulltext index using a bloom filter. pub struct BloomFilterFulltextIndexCreator { inner: Option, diff --git a/src/index/src/fulltext_index/create/tantivy.rs b/src/index/src/fulltext_index/create/tantivy.rs index 6b09c1f0fb..274fea596e 100644 --- a/src/index/src/fulltext_index/create/tantivy.rs +++ b/src/index/src/fulltext_index/create/tantivy.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; use std::path::{Path, PathBuf}; use async_trait::async_trait; @@ -21,15 +22,13 @@ use snafu::{OptionExt, ResultExt}; use tantivy::indexer::NoMergePolicy; use tantivy::schema::{Schema, STORED, TEXT}; use tantivy::store::{Compressor, ZstdCompressor}; -use tantivy::tokenizer::{LowerCaser, SimpleTokenizer, TextAnalyzer, TokenizerManager}; use tantivy::{doc, Index, IndexWriter}; -use tantivy_jieba::JiebaTokenizer; use crate::fulltext_index::create::FulltextIndexCreator; use crate::fulltext_index::error::{ - ExternalSnafu, FinishedSnafu, IoSnafu, JoinSnafu, Result, TantivySnafu, + ExternalSnafu, FinishedSnafu, IoSnafu, JoinSnafu, Result, SerializeToJsonSnafu, TantivySnafu, }; -use crate::fulltext_index::{Analyzer, Config}; +use crate::fulltext_index::{Config, KEY_FULLTEXT_CONFIG}; pub const TEXT_FIELD_NAME: &str = "greptime_fulltext_text"; pub const ROWID_FIELD_NAME: &str = "greptime_fulltext_rowid"; @@ -50,6 +49,9 @@ pub struct TantivyFulltextIndexCreator { /// The directory path in filesystem to store the index. path: PathBuf, + + /// The configuration of the fulltext index. + config: Config, } impl TantivyFulltextIndexCreator { @@ -68,7 +70,7 @@ impl TantivyFulltextIndexCreator { let mut index = Index::create_in_dir(&path, schema).context(TantivySnafu)?; index.settings_mut().docstore_compression = Compressor::Zstd(ZstdCompressor::default()); - index.set_tokenizers(Self::build_tokenizer(&config)); + index.set_tokenizers(config.build_tantivy_tokenizer()); let memory_limit = Self::sanitize_memory_limit(memory_limit); @@ -84,25 +86,10 @@ impl TantivyFulltextIndexCreator { rowid_field, max_rowid: 0, path: path.as_ref().to_path_buf(), + config, }) } - fn build_tokenizer(config: &Config) -> TokenizerManager { - let mut builder = match config.analyzer { - Analyzer::English => TextAnalyzer::builder(SimpleTokenizer::default()).dynamic(), - Analyzer::Chinese => TextAnalyzer::builder(JiebaTokenizer {}).dynamic(), - }; - - if !config.case_sensitive { - builder = builder.filter_dynamic(LowerCaser); - } - - let tokenizer = builder.build(); - let tokenizer_manager = TokenizerManager::new(); - tokenizer_manager.register("default", tokenizer); - tokenizer_manager - } - fn sanitize_memory_limit(memory_limit: usize) -> usize { // Port from tantivy::indexer::index_writer::{MEMORY_BUDGET_NUM_BYTES_MIN, MEMORY_BUDGET_NUM_BYTES_MAX} const MARGIN_IN_BYTES: usize = 1_000_000; @@ -137,8 +124,16 @@ impl FulltextIndexCreator for TantivyFulltextIndexCreator { .await .context(JoinSnafu)??; + let property_key = KEY_FULLTEXT_CONFIG.to_string(); + let property_value = serde_json::to_string(&self.config).context(SerializeToJsonSnafu)?; + puffin_writer - .put_dir(blob_key, self.path.clone(), put_options) + .put_dir( + blob_key, + self.path.clone(), + put_options, + HashMap::from([(property_key, property_value)]), + ) .await .map_err(BoxedError::new) .context(ExternalSnafu) @@ -174,6 +169,7 @@ mod tests { use tantivy::TantivyDocument; use super::*; + use crate::fulltext_index::Analyzer; struct MockPuffinWriter; @@ -197,6 +193,7 @@ mod tests { _key: &str, _dir: PathBuf, _options: PutOptions, + _properties: HashMap, ) -> puffin::error::Result { Ok(0) } @@ -226,7 +223,7 @@ mod tests { ("foo", vec![3]), ("bar", vec![4]), ]; - query_and_check(temp_dir.path(), &cases).await; + query_and_check(temp_dir.path(), config, &cases).await; } } @@ -248,9 +245,13 @@ mod tests { ("hello", vec![0u32, 2]), ("world", vec![1, 2]), ("foo", vec![3]), + ("Foo", vec![]), + ("FOO", vec![]), ("bar", vec![]), + ("Bar", vec![4]), + ("BAR", vec![]), ]; - query_and_check(temp_dir.path(), &cases).await; + query_and_check(temp_dir.path(), config, &cases).await; } } @@ -274,7 +275,7 @@ mod tests { ("foo", vec![4]), ("bar", vec![5]), ]; - query_and_check(temp_dir.path(), &cases).await; + query_and_check(temp_dir.path(), config, &cases).await; } } @@ -297,8 +298,12 @@ mod tests { ("世界", vec![1, 2, 3]), ("foo", vec![4]), ("bar", vec![]), + ("Foo", vec![]), + ("FOO", vec![]), + ("Bar", vec![5]), + ("BAR", vec![]), ]; - query_and_check(temp_dir.path(), &cases).await; + query_and_check(temp_dir.path(), config, &cases).await; } } @@ -315,8 +320,9 @@ mod tests { .unwrap(); } - async fn query_and_check(path: &Path, cases: &[(&str, Vec)]) { - let index = Index::open_in_dir(path).unwrap(); + async fn query_and_check(path: &Path, config: Config, cases: &[(&str, Vec)]) { + let mut index = Index::open_in_dir(path).unwrap(); + index.set_tokenizers(config.build_tantivy_tokenizer()); let reader = index.reader().unwrap(); let searcher = reader.searcher(); for (query, expected) in cases { diff --git a/src/index/src/fulltext_index/search/tantivy.rs b/src/index/src/fulltext_index/search/tantivy.rs index 61c87e863f..a55b599d21 100644 --- a/src/index/src/fulltext_index/search/tantivy.rs +++ b/src/index/src/fulltext_index/search/tantivy.rs @@ -29,6 +29,7 @@ use crate::fulltext_index::error::{ Result, TantivyDocNotFoundSnafu, TantivyParserSnafu, TantivySnafu, }; use crate::fulltext_index::search::{FulltextIndexSearcher, RowId}; +use crate::fulltext_index::Config; /// `TantivyFulltextIndexSearcher` is a searcher using Tantivy. pub struct TantivyFulltextIndexSearcher { @@ -42,10 +43,11 @@ pub struct TantivyFulltextIndexSearcher { impl TantivyFulltextIndexSearcher { /// Creates a new `TantivyFulltextIndexSearcher`. - pub fn new(path: impl AsRef) -> Result { + pub fn new(path: impl AsRef, config: Config) -> Result { let now = Instant::now(); - let index = Index::open_in_dir(path.as_ref()).context(TantivySnafu)?; + let mut index = Index::open_in_dir(path.as_ref()).context(TantivySnafu)?; + index.set_tokenizers(config.build_tantivy_tokenizer()); let reader = index .reader_builder() .reload_policy(ReloadPolicy::Manual) diff --git a/src/index/src/fulltext_index/tests.rs b/src/index/src/fulltext_index/tests.rs index d3491a7e9d..a2a87a645a 100644 --- a/src/index/src/fulltext_index/tests.rs +++ b/src/index/src/fulltext_index/tests.rs @@ -19,7 +19,7 @@ use common_test_util::temp_dir::{create_temp_dir, TempDir}; use puffin::puffin_manager::file_accessor::MockFileAccessor; use puffin::puffin_manager::fs_puffin_manager::FsPuffinManager; use puffin::puffin_manager::stager::BoundedStager; -use puffin::puffin_manager::{DirGuard, PuffinManager, PuffinReader, PuffinWriter, PutOptions}; +use puffin::puffin_manager::{PuffinManager, PuffinReader, PuffinWriter, PutOptions}; use crate::fulltext_index::create::{FulltextIndexCreator, TantivyFulltextIndexCreator}; use crate::fulltext_index::search::{FulltextIndexSearcher, RowId, TantivyFulltextIndexSearcher}; @@ -61,8 +61,7 @@ async fn test_search( prefix: &str, config: Config, texts: Vec<&str>, - query: &str, - expected: impl IntoIterator, + query_expected: Vec<(&str, impl IntoIterator)>, ) { let (_staging_dir, stager) = new_bounded_stager(prefix).await; let file_accessor = Arc::new(MockFileAccessor::new(prefix)); @@ -72,14 +71,16 @@ async fn test_search( let blob_key = "fulltext_index".to_string(); let mut writer = puffin_manager.writer(&file_name).await.unwrap(); create_index(prefix, &mut writer, &blob_key, texts, config).await; + writer.finish().await.unwrap(); let reader = puffin_manager.reader(&file_name).await.unwrap(); let index_dir = reader.dir(&blob_key).await.unwrap(); - let searcher = TantivyFulltextIndexSearcher::new(index_dir.path()).unwrap(); - let results = searcher.search(query).await.unwrap(); - - let expected = expected.into_iter().collect::>(); - assert_eq!(results, expected); + let searcher = TantivyFulltextIndexSearcher::new(index_dir.path(), config).unwrap(); + for (query, expected) in query_expected { + let results = searcher.search(query).await.unwrap(); + let expected = expected.into_iter().collect::>(); + assert_eq!(results, expected); + } } #[tokio::test] @@ -91,8 +92,7 @@ async fn test_simple_term() { "This is a sample text containing Barack Obama", "Another document mentioning Barack", ], - "Barack Obama", - [0, 1], + vec![("Barack Obama", [0, 1])], ) .await; } @@ -103,8 +103,7 @@ async fn test_negative_term() { "test_negative_term_", Config::default(), vec!["apple is a fruit", "I like apple", "fruit is healthy"], - "apple -fruit", - [1], + vec![("apple -fruit", [1])], ) .await; } @@ -119,8 +118,7 @@ async fn test_must_term() { "I love apples and fruits", "apple and fruit are good", ], - "+apple +fruit", - [2], + vec![("+apple +fruit", [2])], ) .await; } @@ -131,8 +129,7 @@ async fn test_boolean_operators() { "test_boolean_operators_", Config::default(), vec!["a b c", "a b", "b c", "c"], - "a AND b OR c", - [0, 1, 2, 3], + vec![("a AND b OR c", [0, 1, 2, 3])], ) .await; } @@ -146,8 +143,7 @@ async fn test_phrase_term() { "This is a sample text containing Barack Obama", "Another document mentioning Barack", ], - "\"Barack Obama\"", - [0], + vec![("\"Barack Obama\"", [0])], ) .await; } @@ -161,8 +157,7 @@ async fn test_config_english_analyzer_case_insensitive() { ..Config::default() }, vec!["Banana is a fruit", "I like apple", "Fruit is healthy"], - "banana", - [0], + vec![("banana", [0]), ("Banana", [0]), ("BANANA", [0])], ) .await; } @@ -175,9 +170,8 @@ async fn test_config_english_analyzer_case_sensitive() { case_sensitive: true, ..Config::default() }, - vec!["Banana is a fruit", "I like apple", "Fruit is healthy"], - "banana", - [], + vec!["Banana is a fruit", "I like banana", "Fruit is healthy"], + vec![("banana", [1]), ("Banana", [0])], ) .await; } @@ -191,8 +185,7 @@ async fn test_config_chinese_analyzer() { ..Default::default() }, vec!["苹果是一种水果", "我喜欢苹果", "水果很健康"], - "苹果", - [0, 1], + vec![("苹果", [0, 1])], ) .await; } diff --git a/src/mito2/src/sst/index/fulltext_index/applier.rs b/src/mito2/src/sst/index/fulltext_index/applier.rs index e463bd0ee8..94ceda6891 100644 --- a/src/mito2/src/sst/index/fulltext_index/applier.rs +++ b/src/mito2/src/sst/index/fulltext_index/applier.rs @@ -17,9 +17,10 @@ use std::sync::Arc; use common_telemetry::warn; use index::fulltext_index::search::{FulltextIndexSearcher, RowId, TantivyFulltextIndexSearcher}; +use index::fulltext_index::Config; use object_store::ObjectStore; use puffin::puffin_manager::cache::PuffinMetadataCacheRef; -use puffin::puffin_manager::{BlobWithMetadata, DirGuard, PuffinManager, PuffinReader}; +use puffin::puffin_manager::{GuardWithMetadata, PuffinManager, PuffinReader}; use snafu::ResultExt; use store_api::storage::{ColumnId, RegionId}; @@ -93,7 +94,7 @@ impl FulltextIndexApplier { let mut row_ids: Option> = None; for (column_id, request) in &self.requests { - if request.queries.is_empty() { + if request.queries.is_empty() && request.terms.is_empty() { continue; } @@ -133,15 +134,21 @@ impl FulltextIndexApplier { .dir(file_id, &blob_key, file_size_hint) .await?; - let path = match &dir { - Some(dir) => dir.path(), + let dir = match &dir { + Some(dir) => dir, None => { return Ok(None); } }; - let searcher = TantivyFulltextIndexSearcher::new(path).context(ApplyFulltextIndexSnafu)?; + let config = Config::from_blob_metadata(dir.metadata()).context(ApplyFulltextIndexSnafu)?; + let path = dir.path(); + + let searcher = + TantivyFulltextIndexSearcher::new(path, config).context(ApplyFulltextIndexSnafu)?; let mut row_ids: Option> = None; + + // 1. Apply queries for query in &request.queries { let result = searcher .search(&query.0) @@ -161,6 +168,21 @@ impl FulltextIndexApplier { } } + // 2. Apply terms + let query = request.terms_as_query(config.case_sensitive); + if !query.0.is_empty() { + let result = searcher + .search(&query.0) + .await + .context(ApplyFulltextIndexSnafu)?; + + if let Some(ids) = row_ids.as_mut() { + ids.retain(|id| result.contains(id)); + } else { + row_ids = Some(result); + } + } + Ok(row_ids) } } @@ -217,7 +239,7 @@ impl IndexSource { file_id: FileId, key: &str, file_size_hint: Option, - ) -> Result>> { + ) -> Result>> { let (reader, fallbacked) = self.ensure_reader(file_id, file_size_hint).await?; let res = reader.blob(key).await; match res { @@ -248,7 +270,7 @@ impl IndexSource { file_id: FileId, key: &str, file_size_hint: Option, - ) -> Result> { + ) -> Result>> { let (reader, fallbacked) = self.ensure_reader(file_id, file_size_hint).await?; let res = reader.dir(key).await; match res { diff --git a/src/mito2/src/sst/index/fulltext_index/applier/builder.rs b/src/mito2/src/sst/index/fulltext_index/applier/builder.rs index e5cb6cf765..14f5936a01 100644 --- a/src/mito2/src/sst/index/fulltext_index/applier/builder.rs +++ b/src/mito2/src/sst/index/fulltext_index/applier/builder.rs @@ -30,12 +30,37 @@ use crate::sst::index::puffin_manager::PuffinManagerFactory; /// A request for fulltext index. /// /// It contains all the queries and terms for a column. -#[derive(Default)] +#[derive(Default, Debug)] pub struct FulltextRequest { pub queries: Vec, pub terms: Vec, } +impl FulltextRequest { + /// Convert terms to a query string. + /// + /// For example, if the terms are ["foo", "bar"], the query string will be `r#"+"foo" +"bar""#`. + /// Need to escape the `"` in the term. + /// + /// `skip_lowercased` is used for the situation that lowercased terms are not indexed. + pub fn terms_as_query(&self, skip_lowercased: bool) -> FulltextQuery { + let mut query = String::new(); + for term in &self.terms { + if skip_lowercased && term.col_lowered { + continue; + } + // Escape the `"` in the term. + let escaped_term = term.term.replace("\"", "\\\""); + if query.is_empty() { + query = format!("+\"{escaped_term}\""); + } else { + query.push_str(&format!(" +\"{escaped_term}\"")); + } + } + FulltextQuery(query) + } +} + /// A query to be matched in fulltext index. /// /// `query` is the query to be matched, e.g. "+foo -bar" in `SELECT * FROM t WHERE matches(text, "+foo -bar")`. @@ -543,4 +568,92 @@ mod tests { } ); } + + #[test] + fn test_terms_as_query() { + // Test with empty terms + let request = FulltextRequest::default(); + assert_eq!(request.terms_as_query(false), FulltextQuery(String::new())); + assert_eq!(request.terms_as_query(true), FulltextQuery(String::new())); + + // Test with a single term (not lowercased) + let mut request = FulltextRequest::default(); + request.terms.push(FulltextTerm { + col_lowered: false, + term: "foo".to_string(), + }); + assert_eq!( + request.terms_as_query(false), + FulltextQuery("+\"foo\"".to_string()) + ); + assert_eq!( + request.terms_as_query(true), + FulltextQuery("+\"foo\"".to_string()) + ); + + // Test with a single lowercased term and skip_lowercased=true + let mut request = FulltextRequest::default(); + request.terms.push(FulltextTerm { + col_lowered: true, + term: "foo".to_string(), + }); + assert_eq!( + request.terms_as_query(false), + FulltextQuery("+\"foo\"".to_string()) + ); + assert_eq!(request.terms_as_query(true), FulltextQuery(String::new())); // Should skip lowercased term + + // Test with multiple terms, mix of lowercased and not + let mut request = FulltextRequest::default(); + request.terms.push(FulltextTerm { + col_lowered: false, + term: "foo".to_string(), + }); + request.terms.push(FulltextTerm { + col_lowered: true, + term: "bar".to_string(), + }); + assert_eq!( + request.terms_as_query(false), + FulltextQuery("+\"foo\" +\"bar\"".to_string()) + ); + assert_eq!( + request.terms_as_query(true), + FulltextQuery("+\"foo\"".to_string()) // Only the non-lowercased term + ); + + // Test with term containing quotes that need escaping + let mut request = FulltextRequest::default(); + request.terms.push(FulltextTerm { + col_lowered: false, + term: "foo\"bar".to_string(), + }); + assert_eq!( + request.terms_as_query(false), + FulltextQuery("+\"foo\\\"bar\"".to_string()) + ); + + // Test with a complex mix of terms + let mut request = FulltextRequest::default(); + request.terms.push(FulltextTerm { + col_lowered: false, + term: "foo".to_string(), + }); + request.terms.push(FulltextTerm { + col_lowered: true, + term: "bar\"quoted\"".to_string(), + }); + request.terms.push(FulltextTerm { + col_lowered: false, + term: "baz\\escape".to_string(), + }); + assert_eq!( + request.terms_as_query(false), + FulltextQuery("+\"foo\" +\"bar\\\"quoted\\\"\" +\"baz\\escape\"".to_string()) + ); + assert_eq!( + request.terms_as_query(true), + FulltextQuery("+\"foo\" +\"baz\\escape\"".to_string()) // Skips the lowercased term + ); + } } diff --git a/src/mito2/src/sst/index/fulltext_index/creator.rs b/src/mito2/src/sst/index/fulltext_index/creator.rs index b6eab05bfa..12b83e39d0 100644 --- a/src/mito2/src/sst/index/fulltext_index/creator.rs +++ b/src/mito2/src/sst/index/fulltext_index/creator.rs @@ -376,7 +376,9 @@ mod tests { use crate::access_layer::RegionFilePathFactory; use crate::read::{Batch, BatchColumn}; use crate::sst::file::FileId; - use crate::sst::index::fulltext_index::applier::builder::{FulltextQuery, FulltextRequest}; + use crate::sst::index::fulltext_index::applier::builder::{ + FulltextQuery, FulltextRequest, FulltextTerm, + }; use crate::sst::index::fulltext_index::applier::FulltextIndexApplier; use crate::sst::index::puffin_manager::PuffinManagerFactory; @@ -510,14 +512,25 @@ mod tests { .unwrap() } - async fn build_applier_factory( + /// Applier factory that can handle both queries and terms. + /// + /// It builds a fulltext index with the given data rows, and returns a function + /// that can handle both queries and terms in a single request. + /// + /// The function takes two parameters: + /// - `queries`: A list of (ColumnId, query_string) pairs for fulltext queries + /// - `terms`: A list of (ColumnId, [(bool, String)]) for fulltext terms, where bool indicates if term is lowercased + async fn build_fulltext_applier_factory( prefix: &str, rows: &[( Option<&str>, // text_english_case_sensitive Option<&str>, // text_english_case_insensitive Option<&str>, // text_chinese )], - ) -> impl Fn(Vec<(ColumnId, &str)>) -> BoxFuture<'static, BTreeSet> { + ) -> impl Fn( + Vec<(ColumnId, &str)>, + Vec<(ColumnId, Vec<(bool, &str)>)>, + ) -> BoxFuture<'static, Option>> { let (d, factory) = PuffinManagerFactory::new_for_test_async(prefix).await; let region_dir = "region0".to_string(); let sst_file_id = FileId::random(); @@ -549,74 +562,253 @@ mod tests { let _ = indexer.finish(&mut writer).await.unwrap(); writer.finish().await.unwrap(); - move |queries| { + move |queries: Vec<(ColumnId, &str)>, terms_requests: Vec<(ColumnId, Vec<(bool, &str)>)>| { let _d = &d; - let applier = FulltextIndexApplier::new( - region_dir.clone(), - region_metadata.region_id, - object_store.clone(), - queries + let region_dir = region_dir.clone(); + let object_store = object_store.clone(); + let factory = factory.clone(); + + let mut requests: HashMap = HashMap::new(); + + // Add queries + for (column_id, query) in queries { + requests + .entry(column_id) + .or_default() + .queries + .push(FulltextQuery(query.to_string())); + } + + // Add terms + for (column_id, terms) in terms_requests { + let fulltext_terms = terms .into_iter() - .map(|(a, b)| { - ( - a, - FulltextRequest { - queries: vec![FulltextQuery(b.to_string())], - terms: vec![], - }, - ) + .map(|(col_lowered, term)| FulltextTerm { + col_lowered, + term: term.to_string(), }) - .collect(), - factory.clone(), + .collect::>(); + + requests + .entry(column_id) + .or_default() + .terms + .extend(fulltext_terms); + } + + let applier = FulltextIndexApplier::new( + region_dir, + region_metadata.region_id, + object_store, + requests, + factory, ); - async move { applier.apply(sst_file_id, None).await.unwrap().unwrap() }.boxed() + async move { applier.apply(sst_file_id, None).await.unwrap() }.boxed() } } + fn rows(row_ids: impl IntoIterator) -> BTreeSet { + row_ids.into_iter().collect() + } + #[tokio::test] - async fn test_fulltext_index_basic() { - let applier_factory = build_applier_factory( - "test_fulltext_index_basic_", + async fn test_fulltext_index_basic_case_sensitive() { + let applier_factory = build_fulltext_applier_factory( + "test_fulltext_index_basic_case_sensitive_", &[ - (Some("hello"), None, Some("你好")), - (Some("world"), Some("world"), None), - (None, Some("World"), Some("世界")), - ( - Some("Hello, World"), - Some("Hello, World"), - Some("你好，世界"), - ), + (Some("hello"), None, None), + (Some("world"), None, None), + (None, None, None), + (Some("Hello, World"), None, None), ], ) .await; - let row_ids = applier_factory(vec![(1, "hello")]).await; - assert_eq!(row_ids, vec![0].into_iter().collect()); + let row_ids = applier_factory(vec![(1, "hello")], vec![]).await; + assert_eq!(row_ids, Some(rows([0]))); - let row_ids = applier_factory(vec![(1, "world")]).await; - assert_eq!(row_ids, vec![1].into_iter().collect()); + let row_ids = applier_factory(vec![(1, "world")], vec![]).await; + assert_eq!(row_ids, Some(rows([1]))); - let row_ids = applier_factory(vec![(2, "hello")]).await; - assert_eq!(row_ids, vec![3].into_iter().collect()); + let row_ids = applier_factory(vec![(1, "Hello")], vec![]).await; + assert_eq!(row_ids, Some(rows([3]))); - let row_ids = applier_factory(vec![(2, "world")]).await; - assert_eq!(row_ids, vec![1, 2, 3].into_iter().collect()); + let row_ids = applier_factory(vec![(1, "World")], vec![]).await; + assert_eq!(row_ids, Some(rows([3]))); - let row_ids = applier_factory(vec![(3, "你好")]).await; - assert_eq!(row_ids, vec![0, 3].into_iter().collect()); + let row_ids = applier_factory(vec![], vec![(1, vec![(false, "hello")])]).await; + assert_eq!(row_ids, Some(rows([0]))); - let row_ids = applier_factory(vec![(3, "世界")]).await; - assert_eq!(row_ids, vec![2, 3].into_iter().collect()); + let row_ids = applier_factory(vec![], vec![(1, vec![(true, "hello")])]).await; + assert_eq!(row_ids, None); + + let row_ids = applier_factory(vec![], vec![(1, vec![(false, "world")])]).await; + assert_eq!(row_ids, Some(rows([1]))); + + let row_ids = applier_factory(vec![], vec![(1, vec![(true, "world")])]).await; + assert_eq!(row_ids, None); + + let row_ids = applier_factory(vec![], vec![(1, vec![(false, "Hello")])]).await; + assert_eq!(row_ids, Some(rows([3]))); + + let row_ids = applier_factory(vec![], vec![(1, vec![(true, "Hello")])]).await; + assert_eq!(row_ids, None); + + let row_ids = applier_factory(vec![], vec![(1, vec![(false, "Hello, World")])]).await; + assert_eq!(row_ids, Some(rows([3]))); + + let row_ids = applier_factory(vec![], vec![(1, vec![(true, "Hello, World")])]).await; + assert_eq!(row_ids, None); + } + + #[tokio::test] + async fn test_fulltext_index_basic_case_insensitive() { + let applier_factory = build_fulltext_applier_factory( + "test_fulltext_index_basic_case_insensitive_", + &[ + (None, Some("hello"), None), + (None, None, None), + (None, Some("world"), None), + (None, Some("Hello, World"), None), + ], + ) + .await; + + let row_ids = applier_factory(vec![(2, "hello")], vec![]).await; + assert_eq!(row_ids, Some(rows([0, 3]))); + + let row_ids = applier_factory(vec![(2, "world")], vec![]).await; + assert_eq!(row_ids, Some(rows([2, 3]))); + + let row_ids = applier_factory(vec![(2, "Hello")], vec![]).await; + assert_eq!(row_ids, Some(rows([0, 3]))); + + let row_ids = applier_factory(vec![(2, "World")], vec![]).await; + assert_eq!(row_ids, Some(rows([2, 3]))); + + let row_ids = applier_factory(vec![], vec![(2, vec![(false, "hello")])]).await; + assert_eq!(row_ids, Some(rows([0, 3]))); + + let row_ids = applier_factory(vec![], vec![(2, vec![(true, "hello")])]).await; + assert_eq!(row_ids, Some(rows([0, 3]))); + + let row_ids = applier_factory(vec![], vec![(2, vec![(false, "world")])]).await; + assert_eq!(row_ids, Some(rows([2, 3]))); + + let row_ids = applier_factory(vec![], vec![(2, vec![(true, "world")])]).await; + assert_eq!(row_ids, Some(rows([2, 3]))); + + let row_ids = applier_factory(vec![], vec![(2, vec![(false, "Hello")])]).await; + assert_eq!(row_ids, Some(rows([0, 3]))); + + let row_ids = applier_factory(vec![], vec![(2, vec![(true, "Hello")])]).await; + assert_eq!(row_ids, Some(rows([0, 3]))); + + let row_ids = applier_factory(vec![], vec![(2, vec![(false, "World")])]).await; + assert_eq!(row_ids, Some(rows([2, 3]))); + + let row_ids = applier_factory(vec![], vec![(2, vec![(true, "World")])]).await; + assert_eq!(row_ids, Some(rows([2, 3]))); + } + + #[tokio::test] + async fn test_fulltext_index_basic_chinese() { + let applier_factory = build_fulltext_applier_factory( + "test_fulltext_index_basic_chinese_", + &[ + (None, None, Some("你好")), + (None, None, None), + (None, None, Some("世界")), + (None, None, Some("你好，世界")), + ], + ) + .await; + + let row_ids = applier_factory(vec![(3, "你好")], vec![]).await; + assert_eq!(row_ids, Some(rows([0, 3]))); + + let row_ids = applier_factory(vec![(3, "世界")], vec![]).await; + assert_eq!(row_ids, Some(rows([2, 3]))); + + let row_ids = applier_factory(vec![], vec![(3, vec![(false, "你好")])]).await; + assert_eq!(row_ids, Some(rows([0, 3]))); + + let row_ids = applier_factory(vec![], vec![(3, vec![(false, "世界")])]).await; + assert_eq!(row_ids, Some(rows([2, 3]))); + } + + #[tokio::test] + async fn test_fulltext_index_multi_terms_case_sensitive() { + let applier_factory = build_fulltext_applier_factory( + "test_fulltext_index_multi_terms_case_sensitive_", + &[ + (Some("Hello"), None, None), + (Some("World"), None, None), + (None, None, None), + (Some("Hello, World"), None, None), + ], + ) + .await; + + let row_ids = + applier_factory(vec![], vec![(1, vec![(false, "hello"), (false, "world")])]).await; + assert_eq!(row_ids, Some(rows([]))); + + let row_ids = + applier_factory(vec![], vec![(1, vec![(false, "Hello"), (false, "World")])]).await; + assert_eq!(row_ids, Some(rows([3]))); + + let row_ids = + applier_factory(vec![], vec![(1, vec![(true, "Hello"), (false, "World")])]).await; + assert_eq!(row_ids, Some(rows([1, 3]))); + + let row_ids = + applier_factory(vec![], vec![(1, vec![(false, "Hello"), (true, "World")])]).await; + assert_eq!(row_ids, Some(rows([0, 3]))); + + let row_ids = + applier_factory(vec![], vec![(1, vec![(true, "Hello"), (true, "World")])]).await; + assert_eq!(row_ids, None); + } + + #[tokio::test] + async fn test_fulltext_index_multi_terms_case_insensitive() { + let applier_factory = build_fulltext_applier_factory( + "test_fulltext_index_multi_terms_case_insensitive_", + &[ + (None, Some("hello"), None), + (None, None, None), + (None, Some("world"), None), + (None, Some("Hello, World"), None), + ], + ) + .await; + + let row_ids = + applier_factory(vec![], vec![(2, vec![(false, "hello"), (false, "world")])]).await; + assert_eq!(row_ids, Some(rows([3]))); + + let row_ids = + applier_factory(vec![], vec![(2, vec![(true, "hello"), (false, "world")])]).await; + assert_eq!(row_ids, Some(rows([3]))); + + let row_ids = + applier_factory(vec![], vec![(2, vec![(false, "hello"), (true, "world")])]).await; + assert_eq!(row_ids, Some(rows([3]))); + + let row_ids = + applier_factory(vec![], vec![(2, vec![(true, "hello"), (true, "world")])]).await; + assert_eq!(row_ids, Some(rows([3]))); } #[tokio::test] async fn test_fulltext_index_multi_columns() { - let applier_factory = build_applier_factory( + let applier_factory = build_fulltext_applier_factory( "test_fulltext_index_multi_columns_", &[ - (Some("hello"), None, Some("你好")), - (Some("world"), Some("world"), None), + (Some("Hello"), None, Some("你好")), + (Some("World"), Some("world"), None), (None, Some("World"), Some("世界")), ( Some("Hello, World"), @@ -627,13 +819,14 @@ mod tests { ) .await; - let row_ids = applier_factory(vec![(1, "hello"), (3, "你好")]).await; - assert_eq!(row_ids, vec![0].into_iter().collect()); + let row_ids = applier_factory( + vec![(1, "Hello"), (3, "你好")], + vec![(2, vec![(false, "world")])], + ) + .await; + assert_eq!(row_ids, Some(rows([3]))); - let row_ids = applier_factory(vec![(1, "world"), (3, "世界")]).await; - assert_eq!(row_ids, vec![].into_iter().collect()); - - let row_ids = applier_factory(vec![(2, "world"), (3, "世界")]).await; - assert_eq!(row_ids, vec![2, 3].into_iter().collect()); + let row_ids = applier_factory(vec![(2, "World")], vec![(1, vec![(false, "World")])]).await; + assert_eq!(row_ids, Some(rows([1, 3]))); } } diff --git a/src/mito2/src/sst/index/puffin_manager.rs b/src/mito2/src/sst/index/puffin_manager.rs index 9f288ecd16..cf4bb185a8 100644 --- a/src/mito2/src/sst/index/puffin_manager.rs +++ b/src/mito2/src/sst/index/puffin_manager.rs @@ -174,12 +174,13 @@ impl PuffinFileAccessor for ObjectStorePuffinFileAccessor { #[cfg(test)] mod tests { + use common_base::range_read::RangeReader; use common_test_util::temp_dir::create_temp_dir; use futures::io::Cursor; use object_store::services::Memory; use puffin::blob_metadata::CompressionCodec; - use puffin::puffin_manager::{DirGuard, PuffinManager, PuffinReader, PuffinWriter, PutOptions}; + use puffin::puffin_manager::{PuffinManager, PuffinReader, PuffinWriter, PutOptions}; use super::*; @@ -229,6 +230,7 @@ mod tests { PutOptions { compression: Some(CompressionCodec::Zstd), }, + Default::default(), ) .await .unwrap(); diff --git a/src/puffin/src/puffin_manager.rs b/src/puffin/src/puffin_manager.rs index 1dfec58f5b..9f287128c1 100644 --- a/src/puffin/src/puffin_manager.rs +++ b/src/puffin/src/puffin_manager.rs @@ -65,7 +65,13 @@ pub trait PuffinWriter { /// Returns the number of bytes written. /// /// The specified `dir` should be accessible from the filesystem. - async fn put_dir(&mut self, key: &str, dir: PathBuf, options: PutOptions) -> Result; + async fn put_dir( + &mut self, + key: &str, + dir: PathBuf, + options: PutOptions, + properties: HashMap, + ) -> Result; /// Sets whether the footer should be LZ4 compressed. fn set_footer_lz4_compressed(&mut self, lz4_compressed: bool); @@ -94,15 +100,15 @@ pub trait PuffinReader { /// Reads a blob from the Puffin file. /// - /// The returned `BlobWithMetadata` is used to access the blob data and its metadata. - /// Users should hold the `BlobWithMetadata` until they are done with the blob data. - async fn blob(&self, key: &str) -> Result>; + /// The returned `GuardWithMetadata` is used to access the blob data and its metadata. + /// Users should hold the `GuardWithMetadata` until they are done with the blob data. + async fn blob(&self, key: &str) -> Result>; /// Reads a directory from the Puffin file. /// - /// The returned `DirGuard` is used to access the directory in the filesystem. - /// The caller is responsible for holding the `DirGuard` until they are done with the directory. - async fn dir(&self, key: &str) -> Result; + /// The returned `GuardWithMetadata` is used to access the directory data and its metadata. + /// Users should hold the `GuardWithMetadata` until they are done with the directory data. + async fn dir(&self, key: &str) -> Result>; } /// `BlobGuard` is provided by the `PuffinReader` to access the blob data. @@ -114,32 +120,41 @@ pub trait BlobGuard { async fn reader(&self) -> Result; } -/// `BlobWithMetadata` provides access to the blob data and its metadata. -pub struct BlobWithMetadata { - blob: B, - metadata: BlobMetadata, -} - -impl BlobWithMetadata { - /// Creates a new `BlobWithMetadata` instance. - pub fn new(blob: B, metadata: BlobMetadata) -> Self { - Self { blob, metadata } - } - - /// Returns the reader for the blob data. - pub async fn reader(&self) -> Result { - self.blob.reader().await - } - - /// Returns the metadata of the blob. - pub fn metadata(&self) -> &BlobMetadata { - &self.metadata - } -} - /// `DirGuard` is provided by the `PuffinReader` to access the directory in the filesystem. /// Users should hold the `DirGuard` until they are done with the directory. #[auto_impl::auto_impl(Arc)] pub trait DirGuard { fn path(&self) -> &PathBuf; } + +/// `GuardWithMetadata` provides access to the blob or directory data and its metadata. +pub struct GuardWithMetadata { + guard: G, + metadata: BlobMetadata, +} + +impl GuardWithMetadata { + /// Creates a new `GuardWithMetadata` instance. + pub fn new(guard: G, metadata: BlobMetadata) -> Self { + Self { guard, metadata } + } + + /// Returns the metadata of the directory. + pub fn metadata(&self) -> &BlobMetadata { + &self.metadata + } +} + +impl GuardWithMetadata { + /// Returns the reader for the blob data. + pub async fn reader(&self) -> Result { + self.guard.reader().await + } +} + +impl GuardWithMetadata { + /// Returns the path of the directory. + pub fn path(&self) -> &PathBuf { + self.guard.path() + } +} diff --git a/src/puffin/src/puffin_manager/fs_puffin_manager/reader.rs b/src/puffin/src/puffin_manager/fs_puffin_manager/reader.rs index 5d2033e2e9..2c616578f6 100644 --- a/src/puffin/src/puffin_manager/fs_puffin_manager/reader.rs +++ b/src/puffin/src/puffin_manager/fs_puffin_manager/reader.rs @@ -36,7 +36,7 @@ use crate::puffin_manager::file_accessor::PuffinFileAccessor; use crate::puffin_manager::fs_puffin_manager::dir_meta::DirMetadata; use crate::puffin_manager::fs_puffin_manager::PuffinMetadataCacheRef; use crate::puffin_manager::stager::{BoxWriter, DirWriterProviderRef, Stager}; -use crate::puffin_manager::{BlobGuard, BlobWithMetadata, PuffinReader}; +use crate::puffin_manager::{BlobGuard, GuardWithMetadata, PuffinReader}; /// `FsPuffinReader` is a `PuffinReader` that provides fs readers for puffin files. pub struct FsPuffinReader @@ -96,26 +96,13 @@ where } async fn metadata(&self) -> Result> { - let reader = self.puffin_file_accessor.reader(&self.handle).await?; - let mut file = PuffinFileReader::new(reader); + let mut file = self.puffin_reader().await?; self.get_puffin_file_metadata(&mut file).await } - async fn blob(&self, key: &str) -> Result> { - let mut reader = self.puffin_file_accessor.reader(&self.handle).await?; - if let Some(file_size_hint) = self.file_size_hint { - reader.with_file_size_hint(file_size_hint); - } - let mut file = PuffinFileReader::new(reader); - - let metadata = self.get_puffin_file_metadata(&mut file).await?; - let blob_metadata = metadata - .blobs - .iter() - .find(|m| m.blob_type == key) - .context(BlobNotFoundSnafu { blob: key })? - .clone(); - + async fn blob(&self, key: &str) -> Result> { + let mut file = self.puffin_reader().await?; + let blob_metadata = self.get_blob_metadata(key, &mut file).await?; let blob = if blob_metadata.compression_codec.is_none() { // If the blob is not compressed, we can directly read it from the puffin file. Either::L(RandomReadBlob { @@ -140,28 +127,33 @@ where Either::R(staged_blob) }; - Ok(BlobWithMetadata::new(blob, blob_metadata)) + Ok(GuardWithMetadata::new(blob, blob_metadata)) } - async fn dir(&self, key: &str) -> Result { - self.stager + async fn dir(&self, key: &str) -> Result> { + let mut file = self.puffin_reader().await?; + let blob_metadata = self.get_blob_metadata(key, &mut file).await?; + let dir = self + .stager .get_dir( &self.handle, key, Box::new(|writer_provider| { let accessor = self.puffin_file_accessor.clone(); let handle = self.handle.clone(); - let key = key.to_string(); + let blob_metadata = blob_metadata.clone(); Box::pin(Self::init_dir_to_stager( + file, + blob_metadata, handle, - key, writer_provider, accessor, - self.file_size_hint, )) }), ) - .await + .await?; + + Ok(GuardWithMetadata::new(dir, blob_metadata)) } } @@ -188,6 +180,30 @@ where Ok(metadata) } + async fn get_blob_metadata( + &self, + key: &str, + file: &mut PuffinFileReader, + ) -> Result { + let metadata = self.get_puffin_file_metadata(file).await?; + let blob_metadata = metadata + .blobs + .iter() + .find(|m| m.blob_type == key) + .context(BlobNotFoundSnafu { blob: key })? + .clone(); + + Ok(blob_metadata) + } + + async fn puffin_reader(&self) -> Result> { + let mut reader = self.puffin_file_accessor.reader(&self.handle).await?; + if let Some(file_size_hint) = self.file_size_hint { + reader.with_file_size_hint(file_size_hint); + } + Ok(PuffinFileReader::new(reader)) + } + async fn init_blob_to_stager( reader: PuffinFileReader, blob_metadata: BlobMetadata, @@ -201,26 +217,14 @@ where } async fn init_dir_to_stager( + mut file: PuffinFileReader, + blob_metadata: BlobMetadata, handle: F::FileHandle, - key: String, writer_provider: DirWriterProviderRef, accessor: F, - file_size_hint: Option, ) -> Result { - let mut reader = accessor.reader(&handle).await?; - if let Some(file_size_hint) = file_size_hint { - reader.with_file_size_hint(file_size_hint); - } - let mut file = PuffinFileReader::new(reader); - let puffin_metadata = file.metadata().await?; - let blob_metadata = puffin_metadata - .blobs - .iter() - .find(|m| m.blob_type == key.as_str()) - .context(BlobNotFoundSnafu { blob: key })?; - - let reader = file.blob_reader(blob_metadata)?; + let reader = file.blob_reader(&blob_metadata)?; let meta = reader.metadata().await.context(MetadataSnafu)?; let buf = reader .read(0..meta.content_length) diff --git a/src/puffin/src/puffin_manager/fs_puffin_manager/writer.rs b/src/puffin/src/puffin_manager/fs_puffin_manager/writer.rs index 61d9df52f0..feb7678756 100644 --- a/src/puffin/src/puffin_manager/fs_puffin_manager/writer.rs +++ b/src/puffin/src/puffin_manager/fs_puffin_manager/writer.rs @@ -88,7 +88,13 @@ where Ok(written_bytes) } - async fn put_dir(&mut self, key: &str, dir_path: PathBuf, options: PutOptions) -> Result { + async fn put_dir( + &mut self, + key: &str, + dir_path: PathBuf, + options: PutOptions, + properties: HashMap, + ) -> Result { ensure!( !self.blob_keys.contains(key), DuplicateBlobSnafu { blob: key } @@ -150,7 +156,7 @@ where blob_type: key.to_string(), compressed_data: encoded.as_slice(), compression_codec: None, - properties: Default::default(), + properties, }; written_bytes += self.puffin_file_writer.add_blob(dir_meta_blob).await?; diff --git a/src/puffin/src/puffin_manager/tests.rs b/src/puffin/src/puffin_manager/tests.rs index e2f32e9498..bd3ec9d5a5 100644 --- a/src/puffin/src/puffin_manager/tests.rs +++ b/src/puffin/src/puffin_manager/tests.rs @@ -23,7 +23,7 @@ use crate::blob_metadata::CompressionCodec; use crate::puffin_manager::file_accessor::MockFileAccessor; use crate::puffin_manager::fs_puffin_manager::FsPuffinManager; use crate::puffin_manager::stager::BoundedStager; -use crate::puffin_manager::{DirGuard, PuffinManager, PuffinReader, PuffinWriter, PutOptions}; +use crate::puffin_manager::{PuffinManager, PuffinReader, PuffinWriter, PutOptions}; async fn new_bounded_stager(prefix: &str, capacity: u64) -> (TempDir, Arc>) { let staging_dir = create_temp_dir(prefix); @@ -343,6 +343,7 @@ async fn put_dir( PutOptions { compression: compression_codec, }, + HashMap::from_iter([("test_key".to_string(), "test_value".to_string())]), ) .await .unwrap(); @@ -356,6 +357,11 @@ async fn check_dir( puffin_reader: &impl PuffinReader, ) { let res_dir = puffin_reader.dir(key).await.unwrap(); + let metadata = res_dir.metadata(); + assert_eq!( + metadata.properties, + HashMap::from_iter([("test_key".to_string(), "test_value".to_string())]) + ); for (file_name, raw_data) in files_in_dir { let file_path = if cfg!(windows) { res_dir.path().join(file_name.replace('/', "\\")) From 74d8fd00a4ecb026825cb0f26378d5dff77fd01d Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" <6406592+v0y4g3r@users.noreply.github.com> Date: Thu, 10 Apr 2025 16:07:04 +0800 Subject: [PATCH 05/82] fix: remove metadata region options (#5852) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix/remove-metadata-region-options: ### Add `SKIP_WAL_KEY` Option to Metric Engine - **Enhancements**: - Introduced `SKIP_WAL_KEY` to the metric engine options in `create.rs` and `mito_engine_options.rs`. - Updated test cases in `create.rs` to include `skip_wal` option and ensure it is removed for metadata regions. - **Refactoring**: - Updated `requests.rs` to use `SKIP_WAL_KEY` from `store_api::mito_engine_options`. These changes enhance the metric engine by allowing the option to skip Write-Ahead Logging (WAL) and ensure consistent usage of option keys across modules. * fix/remove-metadata-region-options: Add note for new options in mito_engine_options.rs • Introduce a comment to remind developers to check if new options should be removed in region_options_for_metadata_region within metric_engine::engine::create. * empty --- src/metric-engine/src/engine/create.rs | 14 ++++++++++---- src/store-api/src/mito_engine_options.rs | 3 +++ src/table/src/requests.rs | 2 +- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/metric-engine/src/engine/create.rs b/src/metric-engine/src/engine/create.rs index 856bdb7b72..bfb7737df7 100644 --- a/src/metric-engine/src/engine/create.rs +++ b/src/metric-engine/src/engine/create.rs @@ -34,7 +34,7 @@ use store_api::metric_engine_consts::{ METADATA_SCHEMA_VALUE_COLUMN_INDEX, METADATA_SCHEMA_VALUE_COLUMN_NAME, }; use store_api::mito_engine_options::{ - APPEND_MODE_KEY, MEMTABLE_PARTITION_TREE_PRIMARY_KEY_ENCODING, TTL_KEY, + APPEND_MODE_KEY, MEMTABLE_PARTITION_TREE_PRIMARY_KEY_ENCODING, SKIP_WAL_KEY, TTL_KEY, }; use store_api::region_engine::RegionEngine; use store_api::region_request::{AffectedRows, RegionCreateRequest, RegionRequest}; @@ -549,6 +549,7 @@ pub(crate) fn region_options_for_metadata_region( // Don't allow to set primary key encoding for metadata region. original.remove(MEMTABLE_PARTITION_TREE_PRIMARY_KEY_ENCODING); original.insert(TTL_KEY.to_string(), FOREVER.to_string()); + original.remove(SKIP_WAL_KEY); original } @@ -685,8 +686,12 @@ mod test { #[tokio::test] async fn test_create_request_for_physical_regions() { // original request - let mut ttl_options = HashMap::new(); - ttl_options.insert("ttl".to_string(), "60m".to_string()); + let options: HashMap<_, _> = [ + ("ttl".to_string(), "60m".to_string()), + ("skip_wal".to_string(), "true".to_string()), + ] + .into_iter() + .collect(); let request = RegionCreateRequest { engine: METRIC_ENGINE_NAME.to_string(), column_metadatas: vec![ @@ -710,7 +715,7 @@ mod test { }, ], primary_key: vec![0], - options: ttl_options, + options, region_dir: "/test_dir".to_string(), }; @@ -742,5 +747,6 @@ mod test { metadata_region_request.options.get("ttl").unwrap(), "forever" ); + assert!(!metadata_region_request.options.contains_key("skip_wal")); } } diff --git a/src/store-api/src/mito_engine_options.rs b/src/store-api/src/mito_engine_options.rs index e73060469f..aa6fd8984d 100644 --- a/src/store-api/src/mito_engine_options.rs +++ b/src/store-api/src/mito_engine_options.rs @@ -59,6 +59,9 @@ pub const MEMTABLE_PARTITION_TREE_DATA_FREEZE_THRESHOLD: &str = /// Option key for memtable partition tree fork dictionary bytes. pub const MEMTABLE_PARTITION_TREE_FORK_DICTIONARY_BYTES: &str = "memtable.partition_tree.fork_dictionary_bytes"; +/// Option key for skipping WAL. +pub const SKIP_WAL_KEY: &str = "skip_wal"; +// Note: Adding new options here should also check if this option should be removed in [metric_engine::engine::create::region_options_for_metadata_region]. /// Returns true if the `key` is a valid option key for the mito engine. pub fn is_mito_engine_option_key(key: &str) -> bool { diff --git a/src/table/src/requests.rs b/src/table/src/requests.rs index 5b7ad566c5..75a4ab64d6 100644 --- a/src/table/src/requests.rs +++ b/src/table/src/requests.rs @@ -99,7 +99,7 @@ pub const TTL_KEY: &str = store_api::mito_engine_options::TTL_KEY; pub const STORAGE_KEY: &str = "storage"; pub const COMMENT_KEY: &str = "comment"; pub const AUTO_CREATE_TABLE_KEY: &str = "auto_create_table"; -pub const SKIP_WAL_KEY: &str = "skip_wal"; +pub const SKIP_WAL_KEY: &str = store_api::mito_engine_options::SKIP_WAL_KEY; impl TableOptions { pub fn try_from_iter>( From 382eacdc131ec00d74c17841f0fe65e9ff06d5de Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Thu, 10 Apr 2025 17:19:32 +0800 Subject: [PATCH 06/82] fix: include follower peers in region distribution (#5844) --- src/common/meta/src/rpc/router.rs | 44 +++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/common/meta/src/rpc/router.rs b/src/common/meta/src/rpc/router.rs index 0e700cc6da..03a1e0bfa0 100644 --- a/src/common/meta/src/rpc/router.rs +++ b/src/common/meta/src/rpc/router.rs @@ -32,6 +32,10 @@ use crate::key::RegionDistribution; use crate::peer::Peer; use crate::DatanodeId; +/// Returns the distribution of regions to datanodes. +/// +/// The distribution is a map of datanode id to a list of region ids. +/// The list of region ids is sorted in ascending order. pub fn region_distribution(region_routes: &[RegionRoute]) -> RegionDistribution { let mut regions_id_map = RegionDistribution::new(); for route in region_routes.iter() { @@ -39,6 +43,10 @@ pub fn region_distribution(region_routes: &[RegionRoute]) -> RegionDistribution let region_id = route.region.id.region_number(); regions_id_map.entry(peer.id).or_default().push(region_id); } + for peer in route.follower_peers.iter() { + let region_id = route.region.id.region_number(); + regions_id_map.entry(peer.id).or_default().push(region_id); + } } for (_, regions) in regions_id_map.iter_mut() { // id asc @@ -550,4 +558,40 @@ mod tests { assert_eq!(got, p); } + + #[test] + fn test_region_distribution() { + let region_routes = vec![ + RegionRoute { + region: Region { + id: RegionId::new(1, 1), + name: "r1".to_string(), + partition: None, + attrs: BTreeMap::new(), + }, + leader_peer: Some(Peer::new(1, "a1")), + follower_peers: vec![Peer::new(2, "a2"), Peer::new(3, "a3")], + leader_state: None, + leader_down_since: None, + }, + RegionRoute { + region: Region { + id: RegionId::new(1, 2), + name: "r2".to_string(), + partition: None, + attrs: BTreeMap::new(), + }, + leader_peer: Some(Peer::new(2, "a2")), + follower_peers: vec![Peer::new(1, "a1"), Peer::new(3, "a3")], + leader_state: None, + leader_down_since: None, + }, + ]; + + let distribution = region_distribution(®ion_routes); + assert_eq!(distribution.len(), 3); + assert_eq!(distribution[&1], vec![1, 2]); + assert_eq!(distribution[&2], vec![1, 2]); + assert_eq!(distribution[&3], vec![1, 2]); + } } From 71255b3cbdfa5580b85e4f38453a28a25c12ffa0 Mon Sep 17 00:00:00 2001 From: LFC <990479+MichaelScofield@users.noreply.github.com> Date: Thu, 10 Apr 2025 18:08:45 +0800 Subject: [PATCH 07/82] refactor: avoid empty display in errors (#5858) * refactor: avoid empty display in errors * fix: resolve PR comments --- Cargo.lock | 1 + src/common/error/Cargo.toml | 3 + src/common/error/src/ext.rs | 10 +- src/common/error/tests/ext.rs | 111 +++++++++++++++++++++ src/common/macro/src/stack_trace_debug.rs | 63 +++++++++++- src/common/recordbatch/src/adapter.rs | 2 +- src/common/recordbatch/src/error.rs | 2 +- src/query/src/datafusion.rs | 9 +- src/query/src/datafusion/planner.rs | 6 +- src/query/src/error.rs | 2 +- src/query/src/plan.rs | 5 +- src/query/src/planner.rs | 14 +-- src/query/src/range_select/plan.rs | 37 ++++--- src/query/src/range_select/plan_rewrite.rs | 38 ++----- src/query/src/sql.rs | 8 +- 15 files changed, 227 insertions(+), 84 deletions(-) create mode 100644 src/common/error/tests/ext.rs diff --git a/Cargo.lock b/Cargo.lock index cba1fa8793..da4306f527 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1988,6 +1988,7 @@ dependencies = [ name = "common-error" version = "0.14.0" dependencies = [ + "common-macro", "http 1.1.0", "snafu 0.8.5", "strum 0.27.1", diff --git a/src/common/error/Cargo.toml b/src/common/error/Cargo.toml index 148e2c6633..031f944dbd 100644 --- a/src/common/error/Cargo.toml +++ b/src/common/error/Cargo.toml @@ -12,3 +12,6 @@ http.workspace = true snafu.workspace = true strum.workspace = true tonic.workspace = true + +[dev-dependencies] +common-macro.workspace = true diff --git a/src/common/error/src/ext.rs b/src/common/error/src/ext.rs index 3b4d15a835..3f95c5fe1a 100644 --- a/src/common/error/src/ext.rs +++ b/src/common/error/src/ext.rs @@ -42,7 +42,7 @@ pub trait ErrorExt: StackError { if let Some(external_error) = error.source() { let external_root = external_error.sources().last().unwrap(); - if error.to_string().is_empty() { + if error.transparent() { format!("{external_root}") } else { format!("{error}: {external_root}") @@ -86,6 +86,14 @@ pub trait StackError: std::error::Error { } result } + + /// Indicates whether this error is "transparent", that it delegates its "display" and "source" + /// to the underlying error. Could be useful when you are just wrapping some external error, + /// **AND** can not or would not provide meaningful contextual info. For example, the + /// `DataFusionError`. + fn transparent(&self) -> bool { + false + } } impl StackError for Arc { diff --git a/src/common/error/tests/ext.rs b/src/common/error/tests/ext.rs new file mode 100644 index 0000000000..0a39ed51c6 --- /dev/null +++ b/src/common/error/tests/ext.rs @@ -0,0 +1,111 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::any::Any; + +use common_error::ext::{ErrorExt, PlainError, StackError}; +use common_error::status_code::StatusCode; +use common_macro::stack_trace_debug; +use snafu::{Location, ResultExt, Snafu}; + +#[derive(Snafu)] +#[stack_trace_debug] +enum MyError { + #[snafu(display(r#"A normal error with "display" attribute, message "{}""#, message))] + Normal { + message: String, + #[snafu(source)] + error: PlainError, + #[snafu(implicit)] + location: Location, + }, + + #[snafu(transparent)] + Transparent { + #[snafu(source)] + error: PlainError, + #[snafu(implicit)] + location: Location, + }, +} + +impl ErrorExt for MyError { + fn status_code(&self) -> StatusCode { + StatusCode::Unexpected + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +fn normal_error() -> Result<(), MyError> { + let plain_error = PlainError::new("".to_string(), StatusCode::Unexpected); + Err(plain_error).context(NormalSnafu { message: "blabla" }) +} + +fn transparent_error() -> Result<(), MyError> { + let plain_error = PlainError::new("".to_string(), StatusCode::Unexpected); + Err(plain_error)? +} + +#[test] +fn test_output_msg() { + let result = normal_error(); + assert_eq!( + result.unwrap_err().output_msg(), + r#"A normal error with "display" attribute, message "blabla": "# + ); + + let result = transparent_error(); + assert_eq!(result.unwrap_err().output_msg(), ""); +} + +#[test] +fn test_to_string() { + let result = normal_error(); + assert_eq!( + result.unwrap_err().to_string(), + r#"A normal error with "display" attribute, message "blabla""# + ); + + let result = transparent_error(); + assert_eq!(result.unwrap_err().to_string(), ""); +} + +#[test] +fn test_debug_format() { + let result = normal_error(); + assert_eq!( + format!("{:?}", result.unwrap_err()), + r#"0: A normal error with "display" attribute, message "blabla", at src/common/error/tests/ext.rs:55:22 +1: PlainError { msg: "", status_code: Unexpected }"# + ); + + let result = transparent_error(); + assert_eq!( + format!("{:?}", result.unwrap_err()), + r#"0: , at src/common/error/tests/ext.rs:60:5 +1: PlainError { msg: "", status_code: Unexpected }"# + ); +} + +#[test] +fn test_transparent_flag() { + let result = normal_error(); + assert!(!result.unwrap_err().transparent()); + + let result = transparent_error(); + assert!(result.unwrap_err().transparent()); +} diff --git a/src/common/macro/src/stack_trace_debug.rs b/src/common/macro/src/stack_trace_debug.rs index fbc24260f1..f82f4746d3 100644 --- a/src/common/macro/src/stack_trace_debug.rs +++ b/src/common/macro/src/stack_trace_debug.rs @@ -14,7 +14,7 @@ //! implement `::common_error::ext::StackError` -use proc_macro2::{Span, TokenStream as TokenStream2}; +use proc_macro2::{Literal, Span, TokenStream as TokenStream2, TokenTree}; use quote::{quote, quote_spanned}; use syn::spanned::Spanned; use syn::{parenthesized, Attribute, Ident, ItemEnum, Variant}; @@ -32,6 +32,7 @@ pub fn stack_trace_style_impl(args: TokenStream2, input: TokenStream2) -> TokenS variants.push(variant); } + let transparent_fn = build_transparent_fn(enum_name.clone(), &variants); let debug_fmt_fn = build_debug_fmt_impl(enum_name.clone(), variants.clone()); let next_fn = build_next_impl(enum_name.clone(), variants); let debug_impl = build_debug_impl(enum_name.clone()); @@ -43,6 +44,7 @@ pub fn stack_trace_style_impl(args: TokenStream2, input: TokenStream2) -> TokenS impl ::common_error::ext::StackError for #enum_name { #debug_fmt_fn #next_fn + #transparent_fn } #debug_impl @@ -115,6 +117,7 @@ struct ErrorVariant { has_source: bool, has_external_cause: bool, display: TokenStream2, + transparent: bool, span: Span, cfg_attr: Option, } @@ -140,6 +143,7 @@ impl ErrorVariant { } let mut display = None; + let mut transparent = false; let mut cfg_attr = None; for attr in variant.attrs { if attr.path().is_ident("snafu") { @@ -150,17 +154,29 @@ impl ErrorVariant { let display_ts: TokenStream2 = content.parse()?; display = Some(display_ts); Ok(()) + } else if meta.path.is_ident("transparent") { + display = Some(TokenStream2::from(TokenTree::Literal(Literal::string( + "", + )))); + transparent = true; + Ok(()) } else { Err(meta.error("unrecognized repr")) } }) - .expect("Each error should contains a display attribute"); + .unwrap_or_else(|e| panic!("{e}")); } if attr.path().is_ident("cfg") { cfg_attr = Some(attr); } } + let display = display.unwrap_or_else(|| { + panic!( + r#"Error "{}" must be annotated with attribute "display" or "transparent"."#, + variant.ident, + ) + }); let field_ident = variant .fields @@ -174,7 +190,8 @@ impl ErrorVariant { has_location, has_source, has_external_cause, - display: display.unwrap(), + display, + transparent, span, cfg_attr, } @@ -275,4 +292,44 @@ impl ErrorVariant { } } } + + fn build_transparent_match_arm(&self) -> TokenStream2 { + let cfg = if let Some(cfg) = &self.cfg_attr { + quote_spanned!(cfg.span() => #cfg) + } else { + quote! {} + }; + let name = &self.name; + let fields = &self.fields; + + if self.transparent { + quote_spanned! { + self.span => #cfg #[allow(unused_variables)] #name { #(#fields),* } => { + true + }, + } + } else { + quote_spanned! { + self.span => #cfg #[allow(unused_variables)] #name { #(#fields),* } =>{ + false + } + } + } + } +} + +fn build_transparent_fn(enum_name: Ident, variants: &[ErrorVariant]) -> TokenStream2 { + let match_arms = variants + .iter() + .map(|v| v.build_transparent_match_arm()) + .collect::>(); + + quote! { + fn transparent(&self) -> bool { + use #enum_name::*; + match self { + #(#match_arms)* + } + } + } } diff --git a/src/common/recordbatch/src/adapter.rs b/src/common/recordbatch/src/adapter.rs index d342aa8129..3d27d7120f 100644 --- a/src/common/recordbatch/src/adapter.rs +++ b/src/common/recordbatch/src/adapter.rs @@ -298,7 +298,7 @@ impl Stream for RecordBatchStreamAdapter { match Pin::new(&mut self.stream).poll_next(cx) { Poll::Pending => Poll::Pending, Poll::Ready(Some(df_record_batch)) => { - let df_record_batch = df_record_batch.context(error::PollStreamSnafu)?; + let df_record_batch = df_record_batch?; Poll::Ready(Some(RecordBatch::try_from_df_record_batch( self.schema(), df_record_batch, diff --git a/src/common/recordbatch/src/error.rs b/src/common/recordbatch/src/error.rs index 6a1c61c0a0..dfd85e4aa1 100644 --- a/src/common/recordbatch/src/error.rs +++ b/src/common/recordbatch/src/error.rs @@ -65,7 +65,7 @@ pub enum Error { location: Location, }, - #[snafu(display(""))] + #[snafu(transparent)] PollStream { #[snafu(source)] error: datafusion::error::DataFusionError, diff --git a/src/query/src/datafusion.rs b/src/query/src/datafusion.rs index e0f020cd3a..dba7d0215a 100644 --- a/src/query/src/datafusion.rs +++ b/src/query/src/datafusion.rs @@ -50,9 +50,9 @@ use crate::dataframe::DataFrame; pub use crate::datafusion::planner::DfContextProviderAdapter; use crate::dist_plan::MergeScanLogicalPlan; use crate::error::{ - CatalogSnafu, ConvertSchemaSnafu, CreateRecordBatchSnafu, DataFusionSnafu, - MissingTableMutationHandlerSnafu, MissingTimestampColumnSnafu, QueryExecutionSnafu, Result, - TableMutationSnafu, TableNotFoundSnafu, TableReadOnlySnafu, UnsupportedExprSnafu, + CatalogSnafu, ConvertSchemaSnafu, CreateRecordBatchSnafu, MissingTableMutationHandlerSnafu, + MissingTimestampColumnSnafu, QueryExecutionSnafu, Result, TableMutationSnafu, + TableNotFoundSnafu, TableReadOnlySnafu, UnsupportedExprSnafu, }; use crate::executor::QueryExecutor; use crate::metrics::{OnDone, QUERY_STAGE_ELAPSED}; @@ -308,8 +308,7 @@ impl DatafusionQueryEngine { let physical_plan = state .query_planner() .create_physical_plan(&optimized_plan, state) - .await - .context(DataFusionSnafu)?; + .await?; Ok(physical_plan) } diff --git a/src/query/src/datafusion/planner.rs b/src/query/src/datafusion/planner.rs index 912393690d..0ad531541f 100644 --- a/src/query/src/datafusion/planner.rs +++ b/src/query/src/datafusion/planner.rs @@ -43,7 +43,7 @@ use datafusion_sql::parser::Statement as DfStatement; use session::context::QueryContextRef; use snafu::{Location, ResultExt}; -use crate::error::{CatalogSnafu, DataFusionSnafu, Result}; +use crate::error::{CatalogSnafu, Result}; use crate::query_engine::{DefaultPlanDecoder, QueryEngineState}; pub struct DfContextProviderAdapter { @@ -70,9 +70,7 @@ impl DfContextProviderAdapter { query_ctx: QueryContextRef, ) -> Result { let table_names = if let Some(df_stmt) = df_stmt { - session_state - .resolve_table_references(df_stmt) - .context(DataFusionSnafu)? + session_state.resolve_table_references(df_stmt)? } else { vec![] }; diff --git a/src/query/src/error.rs b/src/query/src/error.rs index 1ebba8de5d..c2a2e960b0 100644 --- a/src/query/src/error.rs +++ b/src/query/src/error.rs @@ -126,7 +126,7 @@ pub enum Error { location: Location, }, - #[snafu(display(""))] + #[snafu(transparent)] DataFusion { #[snafu(source)] error: DataFusionError, diff --git a/src/query/src/plan.rs b/src/query/src/plan.rs index e94c073c70..8d5586607e 100644 --- a/src/query/src/plan.rs +++ b/src/query/src/plan.rs @@ -19,12 +19,11 @@ use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRewriter}; use datafusion_common::TableReference; use datafusion_expr::{BinaryExpr, Expr, Join, LogicalPlan, Operator}; use session::context::QueryContextRef; -use snafu::ResultExt; pub use table::metadata::TableType; use table::table::adapter::DfTableProviderAdapter; use table::table_name::TableName; -use crate::error::{DataFusionSnafu, Result}; +use crate::error::Result; struct TableNamesExtractAndRewriter { pub(crate) table_names: HashSet, @@ -119,7 +118,7 @@ pub fn extract_and_rewrite_full_table_names( query_ctx: QueryContextRef, ) -> Result<(HashSet, LogicalPlan)> { let mut extractor = TableNamesExtractAndRewriter::new(query_ctx); - let plan = plan.rewrite(&mut extractor).context(DataFusionSnafu)?; + let plan = plan.rewrite(&mut extractor)?; Ok((extractor.table_names, plan.data)) } diff --git a/src/query/src/planner.rs b/src/query/src/planner.rs index b0d0063d70..e3ee3904b4 100644 --- a/src/query/src/planner.rs +++ b/src/query/src/planner.rs @@ -31,7 +31,7 @@ use snafu::ResultExt; use sql::ast::Expr as SqlExpr; use sql::statements::statement::Statement; -use crate::error::{DataFusionSnafu, PlanSqlSnafu, QueryPlanSnafu, Result, SqlSnafu}; +use crate::error::{PlanSqlSnafu, QueryPlanSnafu, Result, SqlSnafu}; use crate::log_query::planner::LogQueryPlanner; use crate::parser::QueryStatement; use crate::promql::planner::PromPlanner; @@ -118,8 +118,7 @@ impl DfLogicalPlanner { let context = QueryEngineContext::new(self.session_state.clone(), query_ctx); let plan = self .engine_state - .optimize_by_extension_rules(plan, &context) - .context(DataFusionSnafu)?; + .optimize_by_extension_rules(plan, &context)?; common_telemetry::debug!("Logical planner, optimize result: {plan}"); Ok(plan) @@ -154,9 +153,7 @@ impl DfLogicalPlanner { let sql_to_rel = SqlToRel::new_with_options(&context_provider, parser_options); - sql_to_rel - .sql_to_expr(sql.into(), schema, &mut PlannerContext::new()) - .context(DataFusionSnafu) + Ok(sql_to_rel.sql_to_expr(sql.into(), schema, &mut PlannerContext::new())?) } #[tracing::instrument(skip_all)] @@ -183,10 +180,7 @@ impl DfLogicalPlanner { #[tracing::instrument(skip_all)] fn optimize_logical_plan(&self, plan: LogicalPlan) -> Result { - self.engine_state - .optimize_logical_plan(plan) - .context(DataFusionSnafu) - .map(Into::into) + Ok(self.engine_state.optimize_logical_plan(plan)?) } } diff --git a/src/query/src/range_select/plan.rs b/src/query/src/range_select/plan.rs index eb28aacf1e..f2a25997bc 100644 --- a/src/query/src/range_select/plan.rs +++ b/src/query/src/range_select/plan.rs @@ -55,9 +55,9 @@ use datatypes::arrow::record_batch::RecordBatch; use datatypes::arrow::row::{OwnedRow, RowConverter, SortField}; use futures::{ready, Stream}; use futures_util::StreamExt; -use snafu::{ensure, ResultExt}; +use snafu::ensure; -use crate::error::{DataFusionSnafu, RangeQuerySnafu, Result}; +use crate::error::{RangeQuerySnafu, Result}; type Millisecond = ::Native; @@ -373,25 +373,22 @@ impl RangeSelect { Ok((None, Arc::new(field))) }, ) - .collect::>>() - .context(DataFusionSnafu)?; + .collect::>>()?; // add align_ts - let ts_field = time_index - .to_field(input.schema().as_ref()) - .context(DataFusionSnafu)?; + let ts_field = time_index.to_field(input.schema().as_ref())?; let time_index_name = ts_field.1.name().clone(); fields.push(ts_field); // add by - let by_fields = exprlist_to_fields(&by, &input).context(DataFusionSnafu)?; + let by_fields = exprlist_to_fields(&by, &input)?; fields.extend(by_fields.clone()); - let schema_before_project = Arc::new( - DFSchema::new_with_metadata(fields, input.schema().metadata().clone()) - .context(DataFusionSnafu)?, - ); - let by_schema = Arc::new( - DFSchema::new_with_metadata(by_fields, input.schema().metadata().clone()) - .context(DataFusionSnafu)?, - ); + let schema_before_project = Arc::new(DFSchema::new_with_metadata( + fields, + input.schema().metadata().clone(), + )?); + let by_schema = Arc::new(DFSchema::new_with_metadata( + by_fields, + input.schema().metadata().clone(), + )?); // If the results of project plan can be obtained directly from range plan without any additional // calculations, no project plan is required. We can simply project the final output of the range // plan to produce the final result. @@ -421,10 +418,10 @@ impl RangeSelect { (f.0.cloned(), Arc::new(f.1.clone())) }) .collect(); - Arc::new( - DFSchema::new_with_metadata(project_field, input.schema().metadata().clone()) - .context(DataFusionSnafu)?, - ) + Arc::new(DFSchema::new_with_metadata( + project_field, + input.schema().metadata().clone(), + )?) } else { schema_before_project.clone() }; diff --git a/src/query/src/range_select/plan_rewrite.rs b/src/query/src/range_select/plan_rewrite.rs index ff05a26706..b53e1079b8 100644 --- a/src/query/src/range_select/plan_rewrite.rs +++ b/src/query/src/range_select/plan_rewrite.rs @@ -43,8 +43,7 @@ use snafu::{ensure, OptionExt, ResultExt}; use table::table::adapter::DfTableProviderAdapter; use crate::error::{ - CatalogSnafu, DataFusionSnafu, RangeQuerySnafu, Result, TimeIndexNotFoundSnafu, - UnknownTableSnafu, + CatalogSnafu, RangeQuerySnafu, Result, TimeIndexNotFoundSnafu, UnknownTableSnafu, }; use crate::plan::ExtractExpr; use crate::range_select::plan::{Fill, RangeFn, RangeSelect}; @@ -385,8 +384,7 @@ impl RangePlanRewriter { let new_expr = expr .iter() .map(|expr| expr.clone().rewrite(&mut range_rewriter).map(|x| x.data)) - .collect::>>() - .context(DataFusionSnafu)?; + .collect::>>()?; if range_rewriter.by.is_empty() { range_rewriter.by = default_by; } @@ -408,9 +406,7 @@ impl RangePlanRewriter { } else { let project_plan = LogicalPlanBuilder::from(range_plan) .project(new_expr) - .context(DataFusionSnafu)? - .build() - .context(DataFusionSnafu)?; + .and_then(|x| x.build())?; Ok(Some(project_plan)) } } @@ -436,8 +432,7 @@ impl RangePlanRewriter { } ); LogicalPlanBuilder::from(inputs[0].clone()) - .explain(*verbose, true) - .context(DataFusionSnafu)? + .explain(*verbose, true)? .build() } LogicalPlan::Explain(Explain { verbose, .. }) => { @@ -448,8 +443,7 @@ impl RangePlanRewriter { } ); LogicalPlanBuilder::from(inputs[0].clone()) - .explain(*verbose, false) - .context(DataFusionSnafu)? + .explain(*verbose, false)? .build() } LogicalPlan::Distinct(Distinct::On(DistinctOn { @@ -470,13 +464,11 @@ impl RangePlanRewriter { on_expr.clone(), select_expr.clone(), sort_expr.clone(), - ) - .context(DataFusionSnafu)? + )? .build() } _ => plan.with_new_exprs(plan.expressions_consider_join(), inputs), - } - .context(DataFusionSnafu)?; + }?; Ok(Some(plan)) } else { Ok(None) @@ -606,8 +598,6 @@ fn interval_only_in_expr(expr: &Expr) -> bool { #[cfg(test)] mod test { - use std::error::Error; - use arrow::datatypes::IntervalUnit; use catalog::memory::MemoryCatalogManager; use catalog::RegisterTableRequest; @@ -825,12 +815,7 @@ mod test { /// the right argument is `range_fn(avg(field_0), '5m', 'NULL', '0', '1h')` async fn range_argument_err_1() { let query = r#"SELECT range_fn('5m', avg(field_0), 'NULL', '1', tag_0, '1h') FROM test group by tag_0;"#; - let error = do_query(query) - .await - .unwrap_err() - .source() - .unwrap() - .to_string(); + let error = do_query(query).await.unwrap_err().to_string(); assert_eq!( error, "Error during planning: Illegal argument `Utf8(\"5m\")` in range select query" @@ -840,12 +825,7 @@ mod test { #[tokio::test] async fn range_argument_err_2() { let query = r#"SELECT range_fn(avg(field_0), 5, 'NULL', '1', tag_0, '1h') FROM test group by tag_0;"#; - let error = do_query(query) - .await - .unwrap_err() - .source() - .unwrap() - .to_string(); + let error = do_query(query).await.unwrap_err().to_string(); assert_eq!( error, "Error during planning: Illegal argument `Int64(5)` in range select query" diff --git a/src/query/src/sql.rs b/src/query/src/sql.rs index fbda344427..b62289fb6b 100644 --- a/src/query/src/sql.rs +++ b/src/query/src/sql.rs @@ -301,8 +301,7 @@ async fn query_from_information_schema_table( .state() .clone(), ) - .read_table(view) - .context(error::DataFusionSnafu)?; + .read_table(view)?; let planner = query_engine.planner(); let planner = planner @@ -319,10 +318,7 @@ async fn query_from_information_schema_table( } }; - let stream = dataframe - .execute_stream() - .await - .context(error::DataFusionSnafu)?; + let stream = dataframe.execute_stream().await?; Ok(Output::new_with_stream(Box::pin( RecordBatchStreamAdapter::try_new(stream).context(error::CreateRecordBatchSnafu)?, From 84e2bc52c228e8f552127a9b4a6afdb8081973a8 Mon Sep 17 00:00:00 2001 From: fys <40801205+fengys1996@users.noreply.github.com> Date: Fri, 11 Apr 2025 13:54:28 +0800 Subject: [PATCH 08/82] fix: gRPC connection pool leak (#5876) * fix: gRPC connection pool leak * use .config() instead of .inner.config * cancel the bg task if it is running * fix: cr * add unit test for pool release * Avoid potential data races --- Cargo.lock | 1 + src/common/grpc/Cargo.toml | 1 + src/common/grpc/src/channel_manager.rs | 186 ++++++++++++++++++------- 3 files changed, 140 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index da4306f527..1b108a7546 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2099,6 +2099,7 @@ dependencies = [ "rand 0.9.0", "snafu 0.8.5", "tokio", + "tokio-util", "tonic 0.12.3", "tower 0.5.2", ] diff --git a/src/common/grpc/Cargo.toml b/src/common/grpc/Cargo.toml index d20e751e41..4dadf0571b 100644 --- a/src/common/grpc/Cargo.toml +++ b/src/common/grpc/Cargo.toml @@ -25,6 +25,7 @@ lazy_static.workspace = true prost.workspace = true snafu.workspace = true tokio.workspace = true +tokio-util.workspace = true tonic.workspace = true tower.workspace = true diff --git a/src/common/grpc/src/channel_manager.rs b/src/common/grpc/src/channel_manager.rs index 0127829567..713ad58d81 100644 --- a/src/common/grpc/src/channel_manager.rs +++ b/src/common/grpc/src/channel_manager.rs @@ -22,6 +22,7 @@ use dashmap::mapref::entry::Entry; use dashmap::DashMap; use lazy_static::lazy_static; use snafu::{OptionExt, ResultExt}; +use tokio_util::sync::CancellationToken; use tonic::transport::{ Certificate, Channel as InnerChannel, ClientTlsConfig, Endpoint, Identity, Uri, }; @@ -39,18 +40,48 @@ lazy_static! { static ref ID: AtomicU64 = AtomicU64::new(0); } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub struct ChannelManager { + inner: Arc, +} + +#[derive(Debug)] +struct Inner { id: u64, config: ChannelConfig, client_tls_config: Option, pool: Arc, - channel_recycle_started: Arc, + channel_recycle_started: AtomicBool, + cancel: CancellationToken, } -impl Default for ChannelManager { +impl Default for Inner { fn default() -> Self { - ChannelManager::with_config(ChannelConfig::default()) + Self::with_config(ChannelConfig::default()) + } +} + +impl Drop for Inner { + fn drop(&mut self) { + // Cancel the channel recycle task. + self.cancel.cancel(); + } +} + +impl Inner { + fn with_config(config: ChannelConfig) -> Self { + let id = ID.fetch_add(1, Ordering::Relaxed); + let pool = Arc::new(Pool::default()); + let cancel = CancellationToken::new(); + + Self { + id, + config, + client_tls_config: None, + pool, + channel_recycle_started: AtomicBool::new(false), + cancel, + } } } @@ -60,19 +91,14 @@ impl ChannelManager { } pub fn with_config(config: ChannelConfig) -> Self { - let id = ID.fetch_add(1, Ordering::Relaxed); - let pool = Arc::new(Pool::default()); + let inner = Inner::with_config(config); Self { - id, - config, - client_tls_config: None, - pool, - channel_recycle_started: Arc::new(AtomicBool::new(false)), + inner: Arc::new(inner), } } pub fn with_tls_config(config: ChannelConfig) -> Result { - let mut cm = Self::with_config(config.clone()); + let mut inner = Inner::with_config(config.clone()); // setup tls let path_config = config.client_tls.context(InvalidTlsConfigSnafu { @@ -88,17 +114,23 @@ impl ChannelManager { .context(InvalidConfigFilePathSnafu)?; let client_identity = Identity::from_pem(client_cert, client_key); - cm.client_tls_config = Some( + inner.client_tls_config = Some( ClientTlsConfig::new() .ca_certificate(server_root_ca_cert) .identity(client_identity), ); - Ok(cm) + Ok(Self { + inner: Arc::new(inner), + }) } pub fn config(&self) -> &ChannelConfig { - &self.config + &self.inner.config + } + + fn pool(&self) -> &Arc { + &self.inner.pool } pub fn get(&self, addr: impl AsRef) -> Result { @@ -106,12 +138,12 @@ impl ChannelManager { let addr = addr.as_ref(); // It will acquire the read lock. - if let Some(inner_ch) = self.pool.get(addr) { + if let Some(inner_ch) = self.pool().get(addr) { return Ok(inner_ch); } // It will acquire the write lock. - let entry = match self.pool.entry(addr.to_string()) { + let entry = match self.pool().entry(addr.to_string()) { Entry::Occupied(entry) => { entry.get().increase_access(); entry.into_ref() @@ -150,7 +182,7 @@ impl ChannelManager { access: AtomicUsize::new(1), use_default_connector: false, }; - self.pool.put(addr, channel); + self.pool().put(addr, channel); Ok(inner_channel) } @@ -159,11 +191,11 @@ impl ChannelManager { where F: FnMut(&String, &mut Channel) -> bool, { - self.pool.retain_channel(f); + self.pool().retain_channel(f); } fn build_endpoint(&self, addr: &str) -> Result { - let http_prefix = if self.client_tls_config.is_some() { + let http_prefix = if self.inner.client_tls_config.is_some() { "https" } else { "http" @@ -172,51 +204,52 @@ impl ChannelManager { let mut endpoint = Endpoint::new(format!("{http_prefix}://{addr}")).context(CreateChannelSnafu)?; - if let Some(dur) = self.config.timeout { + if let Some(dur) = self.config().timeout { endpoint = endpoint.timeout(dur); } - if let Some(dur) = self.config.connect_timeout { + if let Some(dur) = self.config().connect_timeout { endpoint = endpoint.connect_timeout(dur); } - if let Some(limit) = self.config.concurrency_limit { + if let Some(limit) = self.config().concurrency_limit { endpoint = endpoint.concurrency_limit(limit); } - if let Some((limit, dur)) = self.config.rate_limit { + if let Some((limit, dur)) = self.config().rate_limit { endpoint = endpoint.rate_limit(limit, dur); } - if let Some(size) = self.config.initial_stream_window_size { + if let Some(size) = self.config().initial_stream_window_size { endpoint = endpoint.initial_stream_window_size(size); } - if let Some(size) = self.config.initial_connection_window_size { + if let Some(size) = self.config().initial_connection_window_size { endpoint = endpoint.initial_connection_window_size(size); } - if let Some(dur) = self.config.http2_keep_alive_interval { + if let Some(dur) = self.config().http2_keep_alive_interval { endpoint = endpoint.http2_keep_alive_interval(dur); } - if let Some(dur) = self.config.http2_keep_alive_timeout { + if let Some(dur) = self.config().http2_keep_alive_timeout { endpoint = endpoint.keep_alive_timeout(dur); } - if let Some(enabled) = self.config.http2_keep_alive_while_idle { + if let Some(enabled) = self.config().http2_keep_alive_while_idle { endpoint = endpoint.keep_alive_while_idle(enabled); } - if let Some(enabled) = self.config.http2_adaptive_window { + if let Some(enabled) = self.config().http2_adaptive_window { endpoint = endpoint.http2_adaptive_window(enabled); } - if let Some(tls_config) = &self.client_tls_config { + if let Some(tls_config) = &self.inner.client_tls_config { endpoint = endpoint .tls_config(tls_config.clone()) .context(CreateChannelSnafu)?; } endpoint = endpoint - .tcp_keepalive(self.config.tcp_keepalive) - .tcp_nodelay(self.config.tcp_nodelay); + .tcp_keepalive(self.config().tcp_keepalive) + .tcp_nodelay(self.config().tcp_nodelay); Ok(endpoint) } fn trigger_channel_recycling(&self) { if self + .inner .channel_recycle_started .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed) .is_err() @@ -224,13 +257,15 @@ impl ChannelManager { return; } - let pool = self.pool.clone(); - let _handle = common_runtime::spawn_global(async { - recycle_channel_in_loop(pool, RECYCLE_CHANNEL_INTERVAL_SECS).await; + let pool = self.pool().clone(); + let cancel = self.inner.cancel.clone(); + let id = self.inner.id; + let _handle = common_runtime::spawn_global(async move { + recycle_channel_in_loop(pool, id, cancel, RECYCLE_CHANNEL_INTERVAL_SECS).await; }); info!( "ChannelManager: {}, channel recycle is started, running in the background!", - self.id + self.inner.id ); } } @@ -443,11 +478,23 @@ impl Pool { } } -async fn recycle_channel_in_loop(pool: Arc, interval_secs: u64) { +async fn recycle_channel_in_loop( + pool: Arc, + id: u64, + cancel: CancellationToken, + interval_secs: u64, +) { let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); loop { - let _ = interval.tick().await; + tokio::select! { + _ = cancel.cancelled() => { + info!("Stop channel recycle, ChannelManager id: {}", id); + break; + }, + _ = interval.tick() => {} + } + pool.retain_channel(|_, c| c.access.swap(0, Ordering::Relaxed) != 0) } } @@ -461,11 +508,7 @@ mod tests { #[should_panic] #[test] fn test_invalid_addr() { - let pool = Arc::new(Pool::default()); - let mgr = ChannelManager { - pool, - ..Default::default() - }; + let mgr = ChannelManager::default(); let addr = "http://test"; let _ = mgr.get(addr).unwrap(); @@ -475,7 +518,9 @@ mod tests { async fn test_access_count() { let mgr = ChannelManager::new(); // Do not start recycle - mgr.channel_recycle_started.store(true, Ordering::Relaxed); + mgr.inner + .channel_recycle_started + .store(true, Ordering::Relaxed); let mgr = Arc::new(mgr); let addr = "test_uri"; @@ -493,12 +538,12 @@ mod tests { join.await.unwrap(); } - assert_eq!(1000, mgr.pool.get_access(addr).unwrap()); + assert_eq!(1000, mgr.pool().get_access(addr).unwrap()); - mgr.pool + mgr.pool() .retain_channel(|_, c| c.access.swap(0, Ordering::Relaxed) != 0); - assert_eq!(0, mgr.pool.get_access(addr).unwrap()); + assert_eq!(0, mgr.pool().get_access(addr).unwrap()); } #[test] @@ -624,4 +669,49 @@ mod tests { true }); } + + #[tokio::test] + async fn test_pool_release_with_channel_recycle() { + let mgr = ChannelManager::new(); + + let pool_holder = mgr.pool().clone(); + + // start channel recycle task + let addr = "test_addr"; + let _ = mgr.get(addr); + + let mgr_clone_1 = mgr.clone(); + let mgr_clone_2 = mgr.clone(); + assert_eq!(3, Arc::strong_count(mgr.pool())); + + drop(mgr_clone_1); + drop(mgr_clone_2); + assert_eq!(3, Arc::strong_count(mgr.pool())); + + drop(mgr); + + // wait for the channel recycle task to finish + tokio::time::sleep(Duration::from_millis(10)).await; + + assert_eq!(1, Arc::strong_count(&pool_holder)); + } + + #[tokio::test] + async fn test_pool_release_without_channel_recycle() { + let mgr = ChannelManager::new(); + + let pool_holder = mgr.pool().clone(); + + let mgr_clone_1 = mgr.clone(); + let mgr_clone_2 = mgr.clone(); + assert_eq!(2, Arc::strong_count(mgr.pool())); + + drop(mgr_clone_1); + drop(mgr_clone_2); + assert_eq!(2, Arc::strong_count(mgr.pool())); + + drop(mgr); + + assert_eq!(1, Arc::strong_count(&pool_holder)); + } } From 5a36fa5e18f979a97ebc3adacbc698852a4bc87a Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Fri, 11 Apr 2025 14:42:41 +0800 Subject: [PATCH 09/82] fix: alway rejects write while downgrading region (#5842) * fix: alway rejects write while downgrading region * chore: apply suggestions from CR --- .../function/src/admin/migrate_region.rs | 5 +- src/common/meta/src/instruction.rs | 10 +- src/datanode/src/heartbeat/handler.rs | 4 +- .../src/heartbeat/handler/downgrade_region.rs | 119 ++++++++++-------- .../src/heartbeat/handler/upgrade_region.rs | 9 ++ src/file-engine/src/engine.rs | 7 +- .../src/procedure/region_migration.rs | 7 ++ .../close_downgraded_region.rs | 5 +- .../downgrade_leader_region.rs | 68 ++-------- .../upgrade_candidate_region.rs | 2 + src/meta-srv/src/procedure/test_util.rs | 1 + src/metric-engine/src/engine.rs | 32 ++++- src/metric-engine/src/engine/catchup.rs | 4 +- src/mito2/src/engine/catchup_test.rs | 13 +- src/mito2/src/engine/set_role_state_test.rs | 19 +-- src/mito2/src/worker.rs | 8 +- src/store-api/src/region_engine.rs | 81 ++++++++++-- src/store-api/src/region_request.rs | 4 + 18 files changed, 244 insertions(+), 154 deletions(-) diff --git a/src/common/function/src/admin/migrate_region.rs b/src/common/function/src/admin/migrate_region.rs index 0a487973d3..b1f79c0c07 100644 --- a/src/common/function/src/admin/migrate_region.rs +++ b/src/common/function/src/admin/migrate_region.rs @@ -25,12 +25,13 @@ use session::context::QueryContextRef; use crate::handlers::ProcedureServiceHandlerRef; use crate::helper::cast_u64; -const DEFAULT_TIMEOUT_SECS: u64 = 30; +/// The default timeout for migrate region procedure. +const DEFAULT_TIMEOUT_SECS: u64 = 300; /// A function to migrate a region from source peer to target peer. /// Returns the submitted procedure id if success. Only available in cluster mode. /// -/// - `migrate_region(region_id, from_peer, to_peer)`, with timeout(30 seconds). +/// - `migrate_region(region_id, from_peer, to_peer)`, with timeout(300 seconds). /// - `migrate_region(region_id, from_peer, to_peer, timeout(secs))`. /// /// The parameters: diff --git a/src/common/meta/src/instruction.rs b/src/common/meta/src/instruction.rs index afdc14dff0..5e00437332 100644 --- a/src/common/meta/src/instruction.rs +++ b/src/common/meta/src/instruction.rs @@ -57,6 +57,8 @@ impl Display for RegionIdent { pub struct DowngradeRegionReply { /// Returns the `last_entry_id` if available. pub last_entry_id: Option, + /// Returns the `metadata_last_entry_id` if available (Only available for metric engine). + pub metadata_last_entry_id: Option, /// Indicates whether the region exists. pub exists: bool, /// Return error if any during the operation. @@ -136,16 +138,14 @@ pub struct DowngradeRegion { /// `None` stands for don't flush before downgrading the region. #[serde(default)] pub flush_timeout: Option, - /// Rejects all write requests after flushing. - pub reject_write: bool, } impl Display for DowngradeRegion { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "DowngradeRegion(region_id={}, flush_timeout={:?}, rejct_write={})", - self.region_id, self.flush_timeout, self.reject_write + "DowngradeRegion(region_id={}, flush_timeout={:?})", + self.region_id, self.flush_timeout, ) } } @@ -157,6 +157,8 @@ pub struct UpgradeRegion { pub region_id: RegionId, /// The `last_entry_id` of old leader region. pub last_entry_id: Option, + /// The `last_entry_id` of old leader metadata region (Only used for metric engine). + pub metadata_last_entry_id: Option, /// The timeout of waiting for a wal replay. /// /// `None` stands for no wait, diff --git a/src/datanode/src/heartbeat/handler.rs b/src/datanode/src/heartbeat/handler.rs index bf93d15128..17950847ed 100644 --- a/src/datanode/src/heartbeat/handler.rs +++ b/src/datanode/src/heartbeat/handler.rs @@ -220,7 +220,6 @@ mod tests { let instruction = Instruction::DowngradeRegion(DowngradeRegion { region_id: RegionId::new(2048, 1), flush_timeout: Some(Duration::from_secs(1)), - reject_write: false, }); assert!(heartbeat_handler .is_acceptable(&heartbeat_env.create_handler_ctx((meta.clone(), instruction)))); @@ -229,6 +228,7 @@ mod tests { let instruction = Instruction::UpgradeRegion(UpgradeRegion { region_id, last_entry_id: None, + metadata_last_entry_id: None, replay_timeout: None, location_id: None, }); @@ -419,7 +419,6 @@ mod tests { let instruction = Instruction::DowngradeRegion(DowngradeRegion { region_id, flush_timeout: Some(Duration::from_secs(1)), - reject_write: false, }); let mut ctx = heartbeat_env.create_handler_ctx((meta, instruction)); @@ -442,7 +441,6 @@ mod tests { let instruction = Instruction::DowngradeRegion(DowngradeRegion { region_id: RegionId::new(2048, 1), flush_timeout: Some(Duration::from_secs(1)), - reject_write: false, }); let mut ctx = heartbeat_env.create_handler_ctx((meta, instruction)); let control = heartbeat_handler.handle(&mut ctx).await.unwrap(); diff --git a/src/datanode/src/heartbeat/handler/downgrade_region.rs b/src/datanode/src/heartbeat/handler/downgrade_region.rs index 216a460921..d82e4e065b 100644 --- a/src/datanode/src/heartbeat/handler/downgrade_region.rs +++ b/src/datanode/src/heartbeat/handler/downgrade_region.rs @@ -14,7 +14,7 @@ use common_meta::instruction::{DowngradeRegion, DowngradeRegionReply, InstructionReply}; use common_telemetry::tracing::info; -use common_telemetry::warn; +use common_telemetry::{error, warn}; use futures_util::future::BoxFuture; use store_api::region_engine::{SetRegionRoleStateResponse, SettableRegionRoleState}; use store_api::region_request::{RegionFlushRequest, RegionRequest}; @@ -33,25 +33,32 @@ impl HandlerContext { .set_region_role_state_gracefully(region_id, SettableRegionRoleState::Follower) .await { - Ok(SetRegionRoleStateResponse::Success { last_entry_id }) => { + Ok(SetRegionRoleStateResponse::Success(success)) => { Some(InstructionReply::DowngradeRegion(DowngradeRegionReply { - last_entry_id, + last_entry_id: success.last_entry_id(), + metadata_last_entry_id: success.metadata_last_entry_id(), exists: true, error: None, })) } Ok(SetRegionRoleStateResponse::NotFound) => { + warn!("Region: {region_id} is not found"); Some(InstructionReply::DowngradeRegion(DowngradeRegionReply { last_entry_id: None, + metadata_last_entry_id: None, exists: false, error: None, })) } - Err(err) => Some(InstructionReply::DowngradeRegion(DowngradeRegionReply { - last_entry_id: None, - exists: true, - error: Some(format!("{err:?}")), - })), + Err(err) => { + error!(err; "Failed to convert region to {}", SettableRegionRoleState::Follower); + Some(InstructionReply::DowngradeRegion(DowngradeRegionReply { + last_entry_id: None, + metadata_last_entry_id: None, + exists: true, + error: Some(format!("{err:?}")), + })) + } } } @@ -60,7 +67,6 @@ impl HandlerContext { DowngradeRegion { region_id, flush_timeout, - reject_write, }: DowngradeRegion, ) -> BoxFuture<'static, Option> { Box::pin(async move { @@ -68,6 +74,7 @@ impl HandlerContext { warn!("Region: {region_id} is not found"); return Some(InstructionReply::DowngradeRegion(DowngradeRegionReply { last_entry_id: None, + metadata_last_entry_id: None, exists: false, error: None, })); @@ -89,33 +96,35 @@ impl HandlerContext { return self.downgrade_to_follower_gracefully(region_id).await; }; - if reject_write { - // Sets region to downgrading, the downgrading region will reject all write requests. - match self - .region_server - .set_region_role_state_gracefully( - region_id, - SettableRegionRoleState::DowngradingLeader, - ) - .await - { - Ok(SetRegionRoleStateResponse::Success { .. }) => {} - Ok(SetRegionRoleStateResponse::NotFound) => { - warn!("Region: {region_id} is not found"); - return Some(InstructionReply::DowngradeRegion(DowngradeRegionReply { - last_entry_id: None, - exists: false, - error: None, - })); - } - Err(err) => { - warn!(err; "Failed to convert region to downgrading leader"); - return Some(InstructionReply::DowngradeRegion(DowngradeRegionReply { - last_entry_id: None, - exists: true, - error: Some(format!("{err:?}")), - })); - } + // Sets region to downgrading, + // the downgrading region will reject all write requests. + // However, the downgrading region will still accept read, flush requests. + match self + .region_server + .set_region_role_state_gracefully( + region_id, + SettableRegionRoleState::DowngradingLeader, + ) + .await + { + Ok(SetRegionRoleStateResponse::Success { .. }) => {} + Ok(SetRegionRoleStateResponse::NotFound) => { + warn!("Region: {region_id} is not found"); + return Some(InstructionReply::DowngradeRegion(DowngradeRegionReply { + last_entry_id: None, + metadata_last_entry_id: None, + exists: false, + error: None, + })); + } + Err(err) => { + error!(err; "Failed to convert region to downgrading leader"); + return Some(InstructionReply::DowngradeRegion(DowngradeRegionReply { + last_entry_id: None, + metadata_last_entry_id: None, + exists: true, + error: Some(format!("{err:?}")), + })); } } @@ -144,20 +153,25 @@ impl HandlerContext { } let mut watcher = register_result.into_watcher(); - let result = self.catchup_tasks.wait(&mut watcher, flush_timeout).await; + let result = self.downgrade_tasks.wait(&mut watcher, flush_timeout).await; match result { WaitResult::Timeout => { Some(InstructionReply::DowngradeRegion(DowngradeRegionReply { last_entry_id: None, + metadata_last_entry_id: None, exists: true, - error: Some(format!("Flush region: {region_id} is timeout")), + error: Some(format!( + "Flush region timeout, region: {region_id}, timeout: {:?}", + flush_timeout + )), })) } WaitResult::Finish(Ok(_)) => self.downgrade_to_follower_gracefully(region_id).await, WaitResult::Finish(Err(err)) => { Some(InstructionReply::DowngradeRegion(DowngradeRegionReply { last_entry_id: None, + metadata_last_entry_id: None, exists: true, error: Some(format!("{err:?}")), })) @@ -174,7 +188,9 @@ mod tests { use common_meta::instruction::{DowngradeRegion, InstructionReply}; use mito2::engine::MITO_ENGINE_NAME; - use store_api::region_engine::{RegionRole, SetRegionRoleStateResponse}; + use store_api::region_engine::{ + RegionRole, SetRegionRoleStateResponse, SetRegionRoleStateSuccess, + }; use store_api::region_request::RegionRequest; use store_api::storage::RegionId; use tokio::time::Instant; @@ -198,7 +214,6 @@ mod tests { .handle_downgrade_region_instruction(DowngradeRegion { region_id, flush_timeout, - reject_write: false, }) .await; assert_matches!(reply, Some(InstructionReply::DowngradeRegion(_))); @@ -227,7 +242,9 @@ mod tests { Ok(0) })); region_engine.handle_set_readonly_gracefully_mock_fn = Some(Box::new(|_| { - Ok(SetRegionRoleStateResponse::success(Some(1024))) + Ok(SetRegionRoleStateResponse::success( + SetRegionRoleStateSuccess::mito(1024), + )) })) }); mock_region_server.register_test_region(region_id, mock_engine); @@ -240,7 +257,6 @@ mod tests { .handle_downgrade_region_instruction(DowngradeRegion { region_id, flush_timeout, - reject_write: false, }) .await; assert_matches!(reply, Some(InstructionReply::DowngradeRegion(_))); @@ -262,7 +278,9 @@ mod tests { region_engine.mock_role = Some(Some(RegionRole::Leader)); region_engine.handle_request_delay = Some(Duration::from_secs(100)); region_engine.handle_set_readonly_gracefully_mock_fn = Some(Box::new(|_| { - Ok(SetRegionRoleStateResponse::success(Some(1024))) + Ok(SetRegionRoleStateResponse::success( + SetRegionRoleStateSuccess::mito(1024), + )) })) }); mock_region_server.register_test_region(region_id, mock_engine); @@ -274,7 +292,6 @@ mod tests { .handle_downgrade_region_instruction(DowngradeRegion { region_id, flush_timeout: Some(flush_timeout), - reject_write: false, }) .await; assert_matches!(reply, Some(InstructionReply::DowngradeRegion(_))); @@ -295,7 +312,9 @@ mod tests { region_engine.mock_role = Some(Some(RegionRole::Leader)); region_engine.handle_request_delay = Some(Duration::from_millis(300)); region_engine.handle_set_readonly_gracefully_mock_fn = Some(Box::new(|_| { - Ok(SetRegionRoleStateResponse::success(Some(1024))) + Ok(SetRegionRoleStateResponse::success( + SetRegionRoleStateSuccess::mito(1024), + )) })) }); mock_region_server.register_test_region(region_id, mock_engine); @@ -312,7 +331,6 @@ mod tests { .handle_downgrade_region_instruction(DowngradeRegion { region_id, flush_timeout, - reject_write: false, }) .await; assert_matches!(reply, Some(InstructionReply::DowngradeRegion(_))); @@ -327,7 +345,6 @@ mod tests { .handle_downgrade_region_instruction(DowngradeRegion { region_id, flush_timeout: Some(Duration::from_millis(500)), - reject_write: false, }) .await; assert_matches!(reply, Some(InstructionReply::DowngradeRegion(_))); @@ -356,7 +373,9 @@ mod tests { .fail() })); region_engine.handle_set_readonly_gracefully_mock_fn = Some(Box::new(|_| { - Ok(SetRegionRoleStateResponse::success(Some(1024))) + Ok(SetRegionRoleStateResponse::success( + SetRegionRoleStateSuccess::mito(1024), + )) })) }); mock_region_server.register_test_region(region_id, mock_engine); @@ -373,7 +392,6 @@ mod tests { .handle_downgrade_region_instruction(DowngradeRegion { region_id, flush_timeout, - reject_write: false, }) .await; assert_matches!(reply, Some(InstructionReply::DowngradeRegion(_))); @@ -388,7 +406,6 @@ mod tests { .handle_downgrade_region_instruction(DowngradeRegion { region_id, flush_timeout: Some(Duration::from_millis(500)), - reject_write: false, }) .await; assert_matches!(reply, Some(InstructionReply::DowngradeRegion(_))); @@ -419,7 +436,6 @@ mod tests { .handle_downgrade_region_instruction(DowngradeRegion { region_id, flush_timeout: None, - reject_write: false, }) .await; assert_matches!(reply, Some(InstructionReply::DowngradeRegion(_))); @@ -451,7 +467,6 @@ mod tests { .handle_downgrade_region_instruction(DowngradeRegion { region_id, flush_timeout: None, - reject_write: false, }) .await; assert_matches!(reply, Some(InstructionReply::DowngradeRegion(_))); diff --git a/src/datanode/src/heartbeat/handler/upgrade_region.rs b/src/datanode/src/heartbeat/handler/upgrade_region.rs index a23ae71a3d..b9a324c197 100644 --- a/src/datanode/src/heartbeat/handler/upgrade_region.rs +++ b/src/datanode/src/heartbeat/handler/upgrade_region.rs @@ -26,6 +26,7 @@ impl HandlerContext { UpgradeRegion { region_id, last_entry_id, + metadata_last_entry_id, replay_timeout, location_id, }: UpgradeRegion, @@ -63,6 +64,7 @@ impl HandlerContext { RegionRequest::Catchup(RegionCatchupRequest { set_writable: true, entry_id: last_entry_id, + metadata_entry_id: metadata_last_entry_id, location_id, }), ) @@ -147,6 +149,7 @@ mod tests { .handle_upgrade_region_instruction(UpgradeRegion { region_id, last_entry_id: None, + metadata_last_entry_id: None, replay_timeout, location_id: None, }) @@ -185,6 +188,7 @@ mod tests { .handle_upgrade_region_instruction(UpgradeRegion { region_id, last_entry_id: None, + metadata_last_entry_id: None, replay_timeout, location_id: None, }) @@ -224,6 +228,7 @@ mod tests { .handle_upgrade_region_instruction(UpgradeRegion { region_id, last_entry_id: None, + metadata_last_entry_id: None, replay_timeout, location_id: None, }) @@ -267,6 +272,7 @@ mod tests { region_id, replay_timeout, last_entry_id: None, + metadata_last_entry_id: None, location_id: None, }) .await; @@ -284,6 +290,7 @@ mod tests { .handle_upgrade_region_instruction(UpgradeRegion { region_id, last_entry_id: None, + metadata_last_entry_id: None, replay_timeout: Some(Duration::from_millis(500)), location_id: None, }) @@ -326,6 +333,7 @@ mod tests { .handle_upgrade_region_instruction(UpgradeRegion { region_id, last_entry_id: None, + metadata_last_entry_id: None, replay_timeout: None, location_id: None, }) @@ -344,6 +352,7 @@ mod tests { .handle_upgrade_region_instruction(UpgradeRegion { region_id, last_entry_id: None, + metadata_last_entry_id: None, replay_timeout: Some(Duration::from_millis(200)), location_id: None, }) diff --git a/src/file-engine/src/engine.rs b/src/file-engine/src/engine.rs index 9e9d1aa405..cf5e5c7576 100644 --- a/src/file-engine/src/engine.rs +++ b/src/file-engine/src/engine.rs @@ -27,7 +27,8 @@ use snafu::{ensure, OptionExt}; use store_api::metadata::RegionMetadataRef; use store_api::region_engine::{ RegionEngine, RegionManifestInfo, RegionRole, RegionScannerRef, RegionStatistic, - SetRegionRoleStateResponse, SettableRegionRoleState, SinglePartitionScanner, + SetRegionRoleStateResponse, SetRegionRoleStateSuccess, SettableRegionRoleState, + SinglePartitionScanner, }; use store_api::region_request::{ AffectedRows, RegionCloseRequest, RegionCreateRequest, RegionDropRequest, RegionOpenRequest, @@ -132,7 +133,9 @@ impl RegionEngine for FileRegionEngine { let exists = self.inner.get_region(region_id).await.is_some(); if exists { - Ok(SetRegionRoleStateResponse::success(None)) + Ok(SetRegionRoleStateResponse::success( + SetRegionRoleStateSuccess::file(), + )) } else { Ok(SetRegionRoleStateResponse::NotFound) } diff --git a/src/meta-srv/src/procedure/region_migration.rs b/src/meta-srv/src/procedure/region_migration.rs index f13939195b..a98320f9e7 100644 --- a/src/meta-srv/src/procedure/region_migration.rs +++ b/src/meta-srv/src/procedure/region_migration.rs @@ -127,6 +127,8 @@ pub struct VolatileContext { leader_region_lease_deadline: Option, /// The last_entry_id of leader region. leader_region_last_entry_id: Option, + /// The last_entry_id of leader metadata region (Only used for metric engine). + leader_region_metadata_last_entry_id: Option, /// Elapsed time of downgrading region and upgrading region. operations_elapsed: Duration, } @@ -148,6 +150,11 @@ impl VolatileContext { pub fn set_last_entry_id(&mut self, last_entry_id: u64) { self.leader_region_last_entry_id = Some(last_entry_id) } + + /// Sets the `leader_region_metadata_last_entry_id`. + pub fn set_metadata_last_entry_id(&mut self, last_entry_id: u64) { + self.leader_region_metadata_last_entry_id = Some(last_entry_id); + } } /// Used to generate new [Context]. diff --git a/src/meta-srv/src/procedure/region_migration/close_downgraded_region.rs b/src/meta-srv/src/procedure/region_migration/close_downgraded_region.rs index 4e09e421d0..94256ba5ec 100644 --- a/src/meta-srv/src/procedure/region_migration/close_downgraded_region.rs +++ b/src/meta-srv/src/procedure/region_migration/close_downgraded_region.rs @@ -16,7 +16,7 @@ use std::any::Any; use std::time::Duration; use api::v1::meta::MailboxMessage; -use common_meta::distributed_time_constants::MAILBOX_RTT_SECS; +use common_meta::distributed_time_constants::REGION_LEASE_SECS; use common_meta::instruction::{Instruction, InstructionReply, SimpleReply}; use common_meta::key::datanode_table::RegionInfo; use common_meta::RegionIdent; @@ -31,7 +31,8 @@ use crate::procedure::region_migration::migration_end::RegionMigrationEnd; use crate::procedure::region_migration::{Context, State}; use crate::service::mailbox::Channel; -const CLOSE_DOWNGRADED_REGION_TIMEOUT: Duration = Duration::from_secs(MAILBOX_RTT_SECS); +/// Uses lease time of a region as the timeout of closing a downgraded region. +const CLOSE_DOWNGRADED_REGION_TIMEOUT: Duration = Duration::from_secs(REGION_LEASE_SECS); #[derive(Debug, Serialize, Deserialize)] pub struct CloseDowngradedRegion; diff --git a/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs b/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs index 41d437d466..02b7216fe7 100644 --- a/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs +++ b/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs @@ -22,10 +22,8 @@ use common_meta::instruction::{ }; use common_procedure::Status; use common_telemetry::{error, info, warn}; -use common_wal::options::WalOptions; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt}; -use store_api::storage::RegionId; use tokio::time::{sleep, Instant}; use crate::error::{self, Result}; @@ -97,32 +95,15 @@ impl DowngradeLeaderRegion { &self, ctx: &Context, flush_timeout: Duration, - reject_write: bool, ) -> Instruction { let pc = &ctx.persistent_ctx; let region_id = pc.region_id; Instruction::DowngradeRegion(DowngradeRegion { region_id, flush_timeout: Some(flush_timeout), - reject_write, }) } - async fn should_reject_write(ctx: &mut Context, region_id: RegionId) -> Result { - let datanode_table_value = ctx.get_from_peer_datanode_table_value().await?; - if let Some(wal_option) = datanode_table_value - .region_info - .region_wal_options - .get(®ion_id.region_number()) - { - let options: WalOptions = serde_json::from_str(wal_option) - .with_context(|_| error::DeserializeFromJsonSnafu { input: wal_option })?; - return Ok(matches!(options, WalOptions::RaftEngine)); - } - - Ok(true) - } - /// Tries to downgrade a leader region. /// /// Retry: @@ -143,9 +124,7 @@ impl DowngradeLeaderRegion { .context(error::ExceededDeadlineSnafu { operation: "Downgrade region", })?; - let reject_write = Self::should_reject_write(ctx, region_id).await?; - let downgrade_instruction = - self.build_downgrade_region_instruction(ctx, operation_timeout, reject_write); + let downgrade_instruction = self.build_downgrade_region_instruction(ctx, operation_timeout); let leader = &ctx.persistent_ctx.from_peer; let msg = MailboxMessage::json_message( @@ -174,6 +153,7 @@ impl DowngradeLeaderRegion { ); let InstructionReply::DowngradeRegion(DowngradeRegionReply { last_entry_id, + metadata_last_entry_id, exists, error, }) = reply @@ -202,9 +182,10 @@ impl DowngradeLeaderRegion { ); } else { info!( - "Region {} leader is downgraded, last_entry_id: {:?}, elapsed: {:?}", + "Region {} leader is downgraded, last_entry_id: {:?}, metadata_last_entry_id: {:?}, elapsed: {:?}", region_id, last_entry_id, + metadata_last_entry_id, now.elapsed() ); } @@ -213,6 +194,11 @@ impl DowngradeLeaderRegion { ctx.volatile_ctx.set_last_entry_id(last_entry_id); } + if let Some(metadata_last_entry_id) = metadata_last_entry_id { + ctx.volatile_ctx + .set_metadata_last_entry_id(metadata_last_entry_id); + } + Ok(()) } Err(error::Error::MailboxTimeout { .. }) => { @@ -276,7 +262,6 @@ mod tests { use common_meta::key::test_utils::new_test_table_info; use common_meta::peer::Peer; use common_meta::rpc::router::{Region, RegionRoute}; - use common_wal::options::KafkaWalOptions; use store_api::storage::RegionId; use tokio::time::Instant; @@ -331,41 +316,6 @@ mod tests { assert!(!err.is_retryable()); } - #[tokio::test] - async fn test_should_reject_writes() { - let persistent_context = new_persistent_context(); - let region_id = persistent_context.region_id; - let env = TestingEnv::new(); - let mut ctx = env.context_factory().new_context(persistent_context); - let wal_options = - HashMap::from([(1, serde_json::to_string(&WalOptions::RaftEngine).unwrap())]); - prepare_table_metadata(&ctx, wal_options).await; - - let reject_write = DowngradeLeaderRegion::should_reject_write(&mut ctx, region_id) - .await - .unwrap(); - assert!(reject_write); - - // Remote WAL - let persistent_context = new_persistent_context(); - let region_id = persistent_context.region_id; - let env = TestingEnv::new(); - let mut ctx = env.context_factory().new_context(persistent_context); - let wal_options = HashMap::from([( - 1, - serde_json::to_string(&WalOptions::Kafka(KafkaWalOptions { - topic: "my_topic".to_string(), - })) - .unwrap(), - )]); - prepare_table_metadata(&ctx, wal_options).await; - - let reject_write = DowngradeLeaderRegion::should_reject_write(&mut ctx, region_id) - .await - .unwrap(); - assert!(!reject_write); - } - #[tokio::test] async fn test_pusher_dropped() { let state = DowngradeLeaderRegion::default(); diff --git a/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs b/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs index 6f42c670dd..552b9d3863 100644 --- a/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs +++ b/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs @@ -76,10 +76,12 @@ impl UpgradeCandidateRegion { let pc = &ctx.persistent_ctx; let region_id = pc.region_id; let last_entry_id = ctx.volatile_ctx.leader_region_last_entry_id; + let metadata_last_entry_id = ctx.volatile_ctx.leader_region_metadata_last_entry_id; Instruction::UpgradeRegion(UpgradeRegion { region_id, last_entry_id, + metadata_last_entry_id, replay_timeout: Some(replay_timeout), location_id: Some(ctx.persistent_ctx.from_peer.id), }) diff --git a/src/meta-srv/src/procedure/test_util.rs b/src/meta-srv/src/procedure/test_util.rs index 61254b133b..34ce23abd4 100644 --- a/src/meta-srv/src/procedure/test_util.rs +++ b/src/meta-srv/src/procedure/test_util.rs @@ -135,6 +135,7 @@ pub fn new_downgrade_region_reply( payload: Some(Payload::Json( serde_json::to_string(&InstructionReply::DowngradeRegion(DowngradeRegionReply { last_entry_id, + metadata_last_entry_id: None, exists: exist, error, })) diff --git a/src/metric-engine/src/engine.rs b/src/metric-engine/src/engine.rs index d1e837d078..74978fda78 100644 --- a/src/metric-engine/src/engine.rs +++ b/src/metric-engine/src/engine.rs @@ -40,7 +40,7 @@ use store_api::metadata::RegionMetadataRef; use store_api::metric_engine_consts::METRIC_ENGINE_NAME; use store_api::region_engine::{ RegionEngine, RegionManifestInfo, RegionRole, RegionScannerRef, RegionStatistic, - SetRegionRoleStateResponse, SettableRegionRoleState, + SetRegionRoleStateResponse, SetRegionRoleStateSuccess, SettableRegionRoleState, }; use store_api::region_request::{BatchRegionDdlRequest, RegionRequest}; use store_api::storage::{RegionId, ScanRequest, SequenceNumber}; @@ -352,17 +352,39 @@ impl RegionEngine for MetricEngine { region_id: RegionId, region_role_state: SettableRegionRoleState, ) -> std::result::Result { - self.inner + let metadata_result = match self + .inner .mito .set_region_role_state_gracefully( utils::to_metadata_region_id(region_id), region_role_state, ) - .await?; - self.inner + .await? + { + SetRegionRoleStateResponse::Success(success) => success, + SetRegionRoleStateResponse::NotFound => { + return Ok(SetRegionRoleStateResponse::NotFound) + } + }; + + let data_result = match self + .inner .mito .set_region_role_state_gracefully(region_id, region_role_state) - .await + .await? + { + SetRegionRoleStateResponse::Success(success) => success, + SetRegionRoleStateResponse::NotFound => { + return Ok(SetRegionRoleStateResponse::NotFound) + } + }; + + Ok(SetRegionRoleStateResponse::success( + SetRegionRoleStateSuccess::metric( + data_result.last_entry_id().unwrap_or_default(), + metadata_result.last_entry_id().unwrap_or_default(), + ), + )) } /// Returns the physical region role. diff --git a/src/metric-engine/src/engine/catchup.rs b/src/metric-engine/src/engine/catchup.rs index 44713f0bc4..d2e92f6e0e 100644 --- a/src/metric-engine/src/engine/catchup.rs +++ b/src/metric-engine/src/engine/catchup.rs @@ -56,7 +56,8 @@ impl MetricEngineInner { metadata_region_id, RegionRequest::Catchup(RegionCatchupRequest { set_writable: req.set_writable, - entry_id: None, + entry_id: req.metadata_entry_id, + metadata_entry_id: None, location_id: req.location_id, }), ) @@ -70,6 +71,7 @@ impl MetricEngineInner { RegionRequest::Catchup(RegionCatchupRequest { set_writable: req.set_writable, entry_id: req.entry_id, + metadata_entry_id: None, location_id: req.location_id, }), ) diff --git a/src/mito2/src/engine/catchup_test.rs b/src/mito2/src/engine/catchup_test.rs index a9de0d6008..8a24fecf3a 100644 --- a/src/mito2/src/engine/catchup_test.rs +++ b/src/mito2/src/engine/catchup_test.rs @@ -35,8 +35,8 @@ use crate::test_util::{ use crate::wal::EntryId; fn get_last_entry_id(resp: SetRegionRoleStateResponse) -> Option { - if let SetRegionRoleStateResponse::Success { last_entry_id } = resp { - last_entry_id + if let SetRegionRoleStateResponse::Success(success) = resp { + success.last_entry_id() } else { unreachable!(); } @@ -118,6 +118,7 @@ async fn test_catchup_with_last_entry_id(factory: Option) { RegionRequest::Catchup(RegionCatchupRequest { set_writable: false, entry_id: last_entry_id, + metadata_entry_id: None, location_id: None, }), ) @@ -150,6 +151,7 @@ async fn test_catchup_with_last_entry_id(factory: Option) { RegionRequest::Catchup(RegionCatchupRequest { set_writable: true, entry_id: last_entry_id, + metadata_entry_id: None, location_id: None, }), ) @@ -237,6 +239,7 @@ async fn test_catchup_with_incorrect_last_entry_id(factory: Option) { RegionRequest::Catchup(RegionCatchupRequest { set_writable: false, entry_id: None, + metadata_entry_id: None, location_id: None, }), ) @@ -353,6 +358,7 @@ async fn test_catchup_without_last_entry_id(factory: Option) { RegionRequest::Catchup(RegionCatchupRequest { set_writable: true, entry_id: None, + metadata_entry_id: None, location_id: None, }), ) @@ -442,6 +448,7 @@ async fn test_catchup_with_manifest_update(factory: Option) { RegionRequest::Catchup(RegionCatchupRequest { set_writable: false, entry_id: None, + metadata_entry_id: None, location_id: None, }), ) @@ -479,6 +486,7 @@ async fn test_catchup_with_manifest_update(factory: Option) { RegionRequest::Catchup(RegionCatchupRequest { set_writable: true, entry_id: None, + metadata_entry_id: None, location_id: None, }), ) @@ -501,6 +509,7 @@ async fn test_catchup_not_exist() { RegionRequest::Catchup(RegionCatchupRequest { set_writable: true, entry_id: None, + metadata_entry_id: None, location_id: None, }), ) diff --git a/src/mito2/src/engine/set_role_state_test.rs b/src/mito2/src/engine/set_role_state_test.rs index 2a4cb9f9ca..93dbc69407 100644 --- a/src/mito2/src/engine/set_role_state_test.rs +++ b/src/mito2/src/engine/set_role_state_test.rs @@ -16,7 +16,8 @@ use api::v1::Rows; use common_error::ext::ErrorExt; use common_error::status_code::StatusCode; use store_api::region_engine::{ - RegionEngine, RegionRole, SetRegionRoleStateResponse, SettableRegionRoleState, + RegionEngine, RegionRole, SetRegionRoleStateResponse, SetRegionRoleStateSuccess, + SettableRegionRoleState, }; use store_api::region_request::{RegionPutRequest, RegionRequest}; use store_api::storage::RegionId; @@ -48,9 +49,7 @@ async fn test_set_role_state_gracefully() { .await .unwrap(); assert_eq!( - SetRegionRoleStateResponse::Success { - last_entry_id: Some(0) - }, + SetRegionRoleStateResponse::success(SetRegionRoleStateSuccess::mito(0)), result ); @@ -60,9 +59,7 @@ async fn test_set_role_state_gracefully() { .await .unwrap(); assert_eq!( - SetRegionRoleStateResponse::Success { - last_entry_id: Some(0) - }, + SetRegionRoleStateResponse::success(SetRegionRoleStateSuccess::mito(0)), result ); @@ -96,9 +93,7 @@ async fn test_set_role_state_gracefully() { .unwrap(); assert_eq!( - SetRegionRoleStateResponse::Success { - last_entry_id: Some(1) - }, + SetRegionRoleStateResponse::success(SetRegionRoleStateSuccess::mito(1)), result ); } @@ -144,9 +139,7 @@ async fn test_write_downgrading_region() { .await .unwrap(); assert_eq!( - SetRegionRoleStateResponse::Success { - last_entry_id: Some(1) - }, + SetRegionRoleStateResponse::success(SetRegionRoleStateSuccess::mito(1)), result ); diff --git a/src/mito2/src/worker.rs b/src/mito2/src/worker.rs index 0eb5abff41..7aee65b651 100644 --- a/src/mito2/src/worker.rs +++ b/src/mito2/src/worker.rs @@ -41,7 +41,9 @@ use prometheus::IntGauge; use rand::{rng, Rng}; use snafu::{ensure, ResultExt}; use store_api::logstore::LogStore; -use store_api::region_engine::{SetRegionRoleStateResponse, SettableRegionRoleState}; +use store_api::region_engine::{ + SetRegionRoleStateResponse, SetRegionRoleStateSuccess, SettableRegionRoleState, +}; use store_api::storage::RegionId; use tokio::sync::mpsc::{Receiver, Sender}; use tokio::sync::{mpsc, oneshot, watch, Mutex}; @@ -931,7 +933,9 @@ impl RegionWorkerLoop { region.set_role_state_gracefully(region_role_state).await; let last_entry_id = region.version_control.current().last_entry_id; - let _ = sender.send(SetRegionRoleStateResponse::success(Some(last_entry_id))); + let _ = sender.send(SetRegionRoleStateResponse::success( + SetRegionRoleStateSuccess::mito(last_entry_id), + )); }); } else { let _ = sender.send(SetRegionRoleStateResponse::NotFound); diff --git a/src/store-api/src/region_engine.rs b/src/store-api/src/region_engine.rs index d4df5216f4..0a38700f1d 100644 --- a/src/store-api/src/region_engine.rs +++ b/src/store-api/src/region_engine.rs @@ -47,6 +47,15 @@ pub enum SettableRegionRoleState { DowngradingLeader, } +impl Display for SettableRegionRoleState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + SettableRegionRoleState::Follower => write!(f, "Follower"), + SettableRegionRoleState::DowngradingLeader => write!(f, "Leader(Downgrading)"), + } + } +} + impl From for RegionRole { fn from(value: SettableRegionRoleState) -> Self { match value { @@ -63,20 +72,78 @@ pub struct SetRegionRoleStateRequest { region_role_state: SettableRegionRoleState, } +/// The success response of setting region role state. +#[derive(Debug, PartialEq, Eq)] +pub enum SetRegionRoleStateSuccess { + File, + Mito { + last_entry_id: entry::Id, + }, + Metric { + last_entry_id: entry::Id, + metadata_last_entry_id: entry::Id, + }, +} + +impl SetRegionRoleStateSuccess { + /// Returns a [SetRegionRoleStateSuccess::File]. + pub fn file() -> Self { + Self::File + } + + /// Returns a [SetRegionRoleStateSuccess::Mito] with the `last_entry_id`. + pub fn mito(last_entry_id: entry::Id) -> Self { + SetRegionRoleStateSuccess::Mito { last_entry_id } + } + + /// Returns a [SetRegionRoleStateSuccess::Metric] with the `last_entry_id` and `metadata_last_entry_id`. + pub fn metric(last_entry_id: entry::Id, metadata_last_entry_id: entry::Id) -> Self { + SetRegionRoleStateSuccess::Metric { + last_entry_id, + metadata_last_entry_id, + } + } +} + +impl SetRegionRoleStateSuccess { + /// Returns the last entry id of the region. + pub fn last_entry_id(&self) -> Option { + match self { + SetRegionRoleStateSuccess::File => None, + SetRegionRoleStateSuccess::Mito { last_entry_id } => Some(*last_entry_id), + SetRegionRoleStateSuccess::Metric { last_entry_id, .. } => Some(*last_entry_id), + } + } + + /// Returns the last entry id of the metadata of the region. + pub fn metadata_last_entry_id(&self) -> Option { + match self { + SetRegionRoleStateSuccess::File => None, + SetRegionRoleStateSuccess::Mito { .. } => None, + SetRegionRoleStateSuccess::Metric { + metadata_last_entry_id, + .. + } => Some(*metadata_last_entry_id), + } + } +} + /// The response of setting region role state. #[derive(Debug, PartialEq, Eq)] pub enum SetRegionRoleStateResponse { - Success { - /// Returns `last_entry_id` of the region if available(e.g., It's not available in file engine). - last_entry_id: Option, - }, + Success(SetRegionRoleStateSuccess), NotFound, } impl SetRegionRoleStateResponse { - /// Returns a [SetRegionRoleStateResponse::Success] with the `last_entry_id`. - pub fn success(last_entry_id: Option) -> Self { - Self::Success { last_entry_id } + /// Returns a [SetRegionRoleStateResponse::Success] with the `File` success. + pub fn success(success: SetRegionRoleStateSuccess) -> Self { + Self::Success(success) + } + + /// Returns true if the response is a [SetRegionRoleStateResponse::NotFound]. + pub fn is_not_found(&self) -> bool { + matches!(self, SetRegionRoleStateResponse::NotFound) } } diff --git a/src/store-api/src/region_request.rs b/src/store-api/src/region_request.rs index d11963987e..638f2ee114 100644 --- a/src/store-api/src/region_request.rs +++ b/src/store-api/src/region_request.rs @@ -1105,6 +1105,10 @@ pub struct RegionCatchupRequest { /// The `entry_id` that was expected to reply to. /// `None` stands replaying to latest. pub entry_id: Option, + /// Used for metrics metadata region. + /// The `entry_id` that was expected to reply to. + /// `None` stands replaying to latest. + pub metadata_entry_id: Option, /// The hint for replaying memtable. pub location_id: Option, } From 5b0c75c85f7847e689860cda60132578c9304fc1 Mon Sep 17 00:00:00 2001 From: liyang Date: Mon, 14 Apr 2025 09:22:40 +0800 Subject: [PATCH 10/82] ci: not push latest image when schedule release (#5883) * ci: delete the scheduled release * do no push latest image when schedule release * check ref type and name * check not schedule --- .github/scripts/create-version.sh | 12 ++++++------ .github/workflows/release.yml | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/scripts/create-version.sh b/.github/scripts/create-version.sh index e87c74cffb..1de37df190 100755 --- a/.github/scripts/create-version.sh +++ b/.github/scripts/create-version.sh @@ -25,7 +25,7 @@ function create_version() { fi # Reuse $NEXT_RELEASE_VERSION to identify whether it's a nightly build. - # It will be like 'nigtly-20230808-7d0d8dc6'. + # It will be like 'nightly-20230808-7d0d8dc6'. if [ "$NEXT_RELEASE_VERSION" = nightly ]; then echo "$NIGHTLY_RELEASE_PREFIX-$(date "+%Y%m%d")-$(git rev-parse --short HEAD)" exit 0 @@ -60,9 +60,9 @@ function create_version() { } # You can run as following examples: -# GITHUB_EVENT_NAME=push NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nigtly GITHUB_REF_NAME=v0.3.0 ./create-version.sh -# GITHUB_EVENT_NAME=workflow_dispatch NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nigtly ./create-version.sh -# GITHUB_EVENT_NAME=schedule NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nigtly ./create-version.sh -# GITHUB_EVENT_NAME=schedule NEXT_RELEASE_VERSION=nightly NIGHTLY_RELEASE_PREFIX=nigtly ./create-version.sh -# GITHUB_EVENT_NAME=workflow_dispatch COMMIT_SHA=f0e7216c4bb6acce9b29a21ec2d683be2e3f984a NEXT_RELEASE_VERSION=dev NIGHTLY_RELEASE_PREFIX=nigtly ./create-version.sh +# GITHUB_EVENT_NAME=push NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nightly GITHUB_REF_NAME=v0.3.0 ./create-version.sh +# GITHUB_EVENT_NAME=workflow_dispatch NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nightly ./create-version.sh +# GITHUB_EVENT_NAME=schedule NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nightly ./create-version.sh +# GITHUB_EVENT_NAME=schedule NEXT_RELEASE_VERSION=nightly NIGHTLY_RELEASE_PREFIX=nightly ./create-version.sh +# GITHUB_EVENT_NAME=workflow_dispatch COMMIT_SHA=f0e7216c4bb6acce9b29a21ec2d683be2e3f984a NEXT_RELEASE_VERSION=dev NIGHTLY_RELEASE_PREFIX=nightly ./create-version.sh create_version diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 52b61320be..fe85a6f2c8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -317,7 +317,7 @@ jobs: image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }} image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }} version: ${{ needs.allocate-runners.outputs.version }} - push-latest-tag: true + push-latest-tag: ${{ github.ref_type == 'tag' && !contains(github.ref_name, 'nightly') && github.event_name != 'schedule' }} - name: Set build image result id: set-build-image-result @@ -364,7 +364,7 @@ jobs: dev-mode: false upload-to-s3: true update-version-info: true - push-latest-tag: true + push-latest-tag: ${{ github.ref_type == 'tag' && !contains(github.ref_name, 'nightly') && github.event_name != 'schedule' }} publish-github-release: name: Create GitHub release and upload artifacts From be837ddc2428cb8ca7a055ffd37b2ad03cc00495 Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Mon, 14 Apr 2025 11:13:46 +0800 Subject: [PATCH 11/82] test: add tests to ensure nested data structure for identity pipeline (#5888) --- tests-integration/tests/http.rs | 40 ++++++++++++++++----------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index b4690cf9e1..ffb74e1b16 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -95,9 +95,9 @@ macro_rules! http_tests { test_pipeline_api, test_test_pipeline_api, test_plain_text_ingestion, - test_identify_pipeline, - test_identify_pipeline_with_flatten, - test_identify_pipeline_with_custom_ts, + test_identity_pipeline, + test_identity_pipeline_with_flatten, + test_identity_pipeline_with_custom_ts, test_pipeline_dispatcher, test_pipeline_suffix_template, @@ -1413,15 +1413,15 @@ transform: guard.remove_all().await; } -pub async fn test_identify_pipeline(store_type: StorageType) { +pub async fn test_identity_pipeline(store_type: StorageType) { common_telemetry::init_default_ut_logging(); let (app, mut guard) = - setup_test_http_app_with_frontend(store_type, "test_identify_pipeline").await; + setup_test_http_app_with_frontend(store_type, "test_identity_pipeline").await; // handshake let client = TestClient::new(app).await; - let body = r#"{"__time__":1453809242,"__topic__":"","__source__":"10.170.***.***","ip":"10.200.**.***","time":"26/Jan/2016:19:54:02 +0800","url":"POST/PutData?Category=YunOsAccountOpLog&AccessKeyId=&Date=Fri%2C%2028%20Jun%202013%2006%3A53%3A30%20GMT&Topic=raw&Signature=HTTP/1.1","status":"200","user-agent":"aliyun-sdk-java"} -{"__time__":1453809242,"__topic__":"","__source__":"10.170.***.***","ip":"10.200.**.***","time":"26/Jan/2016:19:54:02 +0800","url":"POST/PutData?Category=YunOsAccountOpLog&AccessKeyId=&Date=Fri%2C%2028%20Jun%202013%2006%3A53%3A30%20GMT&Topic=raw&Signature=HTTP/1.1","status":"200","user-agent":"aliyun-sdk-java","hasagei":"hasagei","dongdongdong":"guaguagua"}"#; + let body = r#"{"__time__":1453809242,"__topic__":"","__source__":"10.170.***.***","ip":"10.200.**.***","time":"26/Jan/2016:19:54:02 +0800","url":"POST/PutData?Category=YunOsAccountOpLog&AccessKeyId=&Date=Fri%2C%2028%20Jun%202013%2006%3A53%3A30%20GMT&Topic=raw&Signature=HTTP/1.1","status":"200","user-agent":"aliyun-sdk-java", "json_object": {"a":1,"b":2}, "json_array":[1,2,3]} +{"__time__":1453809242,"__topic__":"","__source__":"10.170.***.***","ip":"10.200.**.***","time":"26/Jan/2016:19:54:02 +0800","url":"POST/PutData?Category=YunOsAccountOpLog&AccessKeyId=&Date=Fri%2C%2028%20Jun%202013%2006%3A53%3A30%20GMT&Topic=raw&Signature=HTTP/1.1","status":"200","user-agent":"aliyun-sdk-java","hasagei":"hasagei","dongdongdong":"guaguagua", "json_object": {"a":1,"b":2}, "json_array":[1,2,3]}"#; let res = client .post("/v1/ingest?db=public&table=logs&pipeline_name=greptime_identity") .header("Content-Type", "application/json") @@ -1440,8 +1440,8 @@ pub async fn test_identify_pipeline(store_type: StorageType) { assert_eq!(res.status(), StatusCode::OK); - let line1_expected = r#"[null,"10.170.***.***",1453809242,"","10.200.**.***","200","26/Jan/2016:19:54:02 +0800","POST/PutData?Category=YunOsAccountOpLog&AccessKeyId=&Date=Fri%2C%2028%20Jun%202013%2006%3A53%3A30%20GMT&Topic=raw&Signature=HTTP/1.1","aliyun-sdk-java",null,null]"#; - let line2_expected = r#"[null,"10.170.***.***",1453809242,"","10.200.**.***","200","26/Jan/2016:19:54:02 +0800","POST/PutData?Category=YunOsAccountOpLog&AccessKeyId=&Date=Fri%2C%2028%20Jun%202013%2006%3A53%3A30%20GMT&Topic=raw&Signature=HTTP/1.1","aliyun-sdk-java","guaguagua","hasagei"]"#; + let line1_expected = r#"[null,"10.170.***.***",1453809242,"","10.200.**.***",[1,2,3],{"a":1,"b":2},"200","26/Jan/2016:19:54:02 +0800","POST/PutData?Category=YunOsAccountOpLog&AccessKeyId=&Date=Fri%2C%2028%20Jun%202013%2006%3A53%3A30%20GMT&Topic=raw&Signature=HTTP/1.1","aliyun-sdk-java",null,null]"#; + let line2_expected = r#"[null,"10.170.***.***",1453809242,"","10.200.**.***",[1,2,3],{"a":1,"b":2},"200","26/Jan/2016:19:54:02 +0800","POST/PutData?Category=YunOsAccountOpLog&AccessKeyId=&Date=Fri%2C%2028%20Jun%202013%2006%3A53%3A30%20GMT&Topic=raw&Signature=HTTP/1.1","aliyun-sdk-java","guaguagua","hasagei"]"#; let res = client.get("/v1/sql?sql=select * from logs").send().await; assert_eq!(res.status(), StatusCode::OK); let resp: serde_json::Value = res.json().await; @@ -1464,7 +1464,7 @@ pub async fn test_identify_pipeline(store_type: StorageType) { serde_json::from_str::>(line2_expected).unwrap() ); - let expected = r#"[["greptime_timestamp","TimestampNanosecond","PRI","NO","","TIMESTAMP"],["__source__","String","","YES","","FIELD"],["__time__","Int64","","YES","","FIELD"],["__topic__","String","","YES","","FIELD"],["ip","String","","YES","","FIELD"],["status","String","","YES","","FIELD"],["time","String","","YES","","FIELD"],["url","String","","YES","","FIELD"],["user-agent","String","","YES","","FIELD"],["dongdongdong","String","","YES","","FIELD"],["hasagei","String","","YES","","FIELD"]]"#; + let expected = r#"[["greptime_timestamp","TimestampNanosecond","PRI","NO","","TIMESTAMP"],["__source__","String","","YES","","FIELD"],["__time__","Int64","","YES","","FIELD"],["__topic__","String","","YES","","FIELD"],["ip","String","","YES","","FIELD"],["json_array","Json","","YES","","FIELD"],["json_object","Json","","YES","","FIELD"],["status","String","","YES","","FIELD"],["time","String","","YES","","FIELD"],["url","String","","YES","","FIELD"],["user-agent","String","","YES","","FIELD"],["dongdongdong","String","","YES","","FIELD"],["hasagei","String","","YES","","FIELD"]]"#; validate_data("identity_schema", &client, "desc logs", expected).await; guard.remove_all().await; @@ -1792,10 +1792,10 @@ table_suffix: _${type} guard.remove_all().await; } -pub async fn test_identify_pipeline_with_flatten(store_type: StorageType) { +pub async fn test_identity_pipeline_with_flatten(store_type: StorageType) { common_telemetry::init_default_ut_logging(); let (app, mut guard) = - setup_test_http_app_with_frontend(store_type, "test_identify_pipeline_with_flatten").await; + setup_test_http_app_with_frontend(store_type, "test_identity_pipeline_with_flatten").await; let client = TestClient::new(app).await; let body = r#"{"__time__":1453809242,"__topic__":"","__source__":"10.170.***.***","ip":"10.200.**.***","time":"26/Jan/2016:19:54:02 +0800","url":"POST/PutData?Category=YunOsAccountOpLog&AccessKeyId=&Date=Fri%2C%2028%20Jun%202013%2006%3A53%3A30%20GMT&Topic=raw&Signature=HTTP/1.1","status":"200","user-agent":"aliyun-sdk-java","custom_map":{"value_a":["a","b","c"],"value_b":"b"}}"#; @@ -1822,7 +1822,7 @@ pub async fn test_identify_pipeline_with_flatten(store_type: StorageType) { let expected = r#"[["greptime_timestamp","TimestampNanosecond","PRI","NO","","TIMESTAMP"],["__source__","String","","YES","","FIELD"],["__time__","Int64","","YES","","FIELD"],["__topic__","String","","YES","","FIELD"],["custom_map.value_a","Json","","YES","","FIELD"],["custom_map.value_b","String","","YES","","FIELD"],["ip","String","","YES","","FIELD"],["status","String","","YES","","FIELD"],["time","String","","YES","","FIELD"],["url","String","","YES","","FIELD"],["user-agent","String","","YES","","FIELD"]]"#; validate_data( - "test_identify_pipeline_with_flatten_desc_logs", + "test_identity_pipeline_with_flatten_desc_logs", &client, "desc logs", expected, @@ -1831,7 +1831,7 @@ pub async fn test_identify_pipeline_with_flatten(store_type: StorageType) { let expected = "[[[\"a\",\"b\",\"c\"]]]"; validate_data( - "test_identify_pipeline_with_flatten_select_json", + "test_identity_pipeline_with_flatten_select_json", &client, "select `custom_map.value_a` from logs", expected, @@ -1841,10 +1841,10 @@ pub async fn test_identify_pipeline_with_flatten(store_type: StorageType) { guard.remove_all().await; } -pub async fn test_identify_pipeline_with_custom_ts(store_type: StorageType) { +pub async fn test_identity_pipeline_with_custom_ts(store_type: StorageType) { common_telemetry::init_default_ut_logging(); let (app, mut guard) = - setup_test_http_app_with_frontend(store_type, "test_identify_pipeline_with_custom_ts") + setup_test_http_app_with_frontend(store_type, "test_identity_pipeline_with_custom_ts") .await; let client = TestClient::new(app).await; @@ -1868,7 +1868,7 @@ pub async fn test_identify_pipeline_with_custom_ts(store_type: StorageType) { let expected = r#"[["__time__","TimestampSecond","PRI","NO","","TIMESTAMP"],["__name__","String","","YES","","FIELD"],["__source__","String","","YES","","FIELD"]]"#; validate_data( - "test_identify_pipeline_with_custom_ts_desc_logs", + "test_identity_pipeline_with_custom_ts_desc_logs", &client, "desc logs", expected, @@ -1877,7 +1877,7 @@ pub async fn test_identify_pipeline_with_custom_ts(store_type: StorageType) { let expected = r#"[[1453809242,"hello","10.170.***.***"],[1453809252,null,"10.170.***.***"]]"#; validate_data( - "test_identify_pipeline_with_custom_ts_data", + "test_identity_pipeline_with_custom_ts_data", &client, "select * from logs", expected, @@ -1908,7 +1908,7 @@ pub async fn test_identify_pipeline_with_custom_ts(store_type: StorageType) { let expected = r#"[["__time__","TimestampNanosecond","PRI","NO","","TIMESTAMP"],["__source__","String","","YES","","FIELD"],["__name__","String","","YES","","FIELD"]]"#; validate_data( - "test_identify_pipeline_with_custom_ts_desc_logs", + "test_identity_pipeline_with_custom_ts_desc_logs", &client, "desc logs", expected, @@ -1917,7 +1917,7 @@ pub async fn test_identify_pipeline_with_custom_ts(store_type: StorageType) { let expected = r#"[[1547577721000000000,"10.170.***.***",null],[1547577724000000000,"10.170.***.***","hello"]]"#; validate_data( - "test_identify_pipeline_with_custom_ts_data", + "test_identity_pipeline_with_custom_ts_data", &client, "select * from logs", expected, From 7cd6b0f04bc5e8353897701a59363efd1ec2d1fc Mon Sep 17 00:00:00 2001 From: dennis zhuang Date: Mon, 14 Apr 2025 14:45:24 +0800 Subject: [PATCH 12/82] docs: update readme (#5891) * docs: update readme * chore: format * docs: shorten * chore: title * fix: blank Co-authored-by: Lei, HUANG <6406592+v0y4g3r@users.noreply.github.com> --------- Co-authored-by: Lei, HUANG <6406592+v0y4g3r@users.noreply.github.com> --- README.md | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 07da173117..912183f5bb 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@

-

Unified & Cost-Effective Observerability Database for Metrics, Logs, and Events

+

Unified & Cost-Effective Observability Database for Metrics, Logs, and Events

@@ -62,7 +62,7 @@ ## Introduction -**GreptimeDB** is an open-source unified & cost-effective observerability database for **Metrics**, **Logs**, and **Events** (also **Traces** in plan). You can gain real-time insights from Edge to Cloud at Any Scale. +**GreptimeDB** is an open-source, cloud-native, unified & cost-effective observability database for **Metrics**, **Logs**, and **Traces**. You can gain real-time insights from Edge to Cloud at Any Scale. ## News @@ -70,27 +70,27 @@ ## Why GreptimeDB -Our core developers have been building observerability data platforms for years. Based on our best practices, GreptimeDB was born to give you: +Our core developers have been building observability data platforms for years. Based on our best practices, GreptimeDB was born to give you: -* **Unified Processing of Metrics, Logs, and Events** +* **Unified Processing of Observability Data** - GreptimeDB unifies observerability data processing by treating all data - whether metrics, logs, or events - as timestamped events with context. Users can analyze this data using either [SQL](https://docs.greptime.com/user-guide/query-data/sql) or [PromQL](https://docs.greptime.com/user-guide/query-data/promql) and leverage stream processing ([Flow](https://docs.greptime.com/user-guide/flow-computation/overview)) to enable continuous aggregation. [Read more](https://docs.greptime.com/user-guide/concepts/data-model). + A unified database that treats metrics, logs, and traces as timestamped wide events with context, supporting [SQL](https://docs.greptime.com/user-guide/query-data/sql)/[PromQL](https://docs.greptime.com/user-guide/query-data/promql) queries and [stream processing](https://docs.greptime.com/user-guide/flow-computation/overview) to simplify complex data stacks. + +* **High Performance and Cost-effective** + + Written in Rust, combines a distributed query engine with [rich indexing](https://docs.greptime.com/user-guide/manage-data/data-index) (inverted, fulltext, skip data, and vector) and optimized columnar storage to deliver sub-second responses on petabyte-scale data and high-cost efficiency. * **Cloud-native Distributed Database** Built for [Kubernetes](https://docs.greptime.com/user-guide/deployments/deploy-on-kubernetes/greptimedb-operator-management). GreptimeDB achieves seamless scalability with its [cloud-native architecture](https://docs.greptime.com/user-guide/concepts/architecture) of separated compute and storage, built on object storage (AWS S3, Azure Blob Storage, etc.) while enabling cross-cloud deployment through a unified data access layer. -* **Performance and Cost-effective** +* **Developer-Friendly** - Written in pure Rust for superior performance and reliability. GreptimeDB features a distributed query engine with intelligent indexing to handle high cardinality data efficiently. Its optimized columnar storage achieves 50x cost efficiency on cloud object storage through advanced compression. [Benchmark reports](https://www.greptime.com/blogs/2024-09-09-report-summary). + Access standardized SQL/PromQL interfaces through built-in web dashboard, REST API, and MySQL/PostgreSQL protocols. Supports widely adopted data ingestion [protocols](https://docs.greptime.com/user-guide/protocols/overview) for seamless migration and integration. -* **Cloud-Edge Collaboration** +* **Flexible Deployment Options** - GreptimeDB seamlessly operates across cloud and edge (ARM/Android/Linux), providing consistent APIs and control plane for unified data management and efficient synchronization. [Learn how to run on Android](https://docs.greptime.com/user-guide/deployments/run-on-android/). - -* **Multi-protocol Ingestion, SQL & PromQL Ready** - - Widely adopted database protocols and APIs, including MySQL, PostgreSQL, InfluxDB, OpenTelemetry, Loki and Prometheus, etc. Effortless Adoption & Seamless Migration. [Supported Protocols Overview](https://docs.greptime.com/user-guide/protocols/overview). + Deploy GreptimeDB anywhere from ARM-based edge devices to cloud environments with unified APIs and bandwidth-efficient data synchronization. Query edge and cloud data seamlessly through identical APIs. [Learn how to run on Android](https://docs.greptime.com/user-guide/deployments/run-on-android/). For more detailed info please read [Why GreptimeDB](https://docs.greptime.com/user-guide/concepts/why-greptimedb). @@ -233,3 +233,5 @@ Special thanks to all the contributors who have propelled GreptimeDB forward. Fo - GreptimeDB's query engine is powered by [Apache Arrow DataFusion™](https://arrow.apache.org/datafusion/). - [Apache OpenDAL™](https://opendal.apache.org) gives GreptimeDB a very general and elegant data access abstraction layer. - GreptimeDB's meta service is based on [etcd](https://etcd.io/). + +Known Users \ No newline at end of file From e3675494b492978bd1a941e49b350d7d87353f55 Mon Sep 17 00:00:00 2001 From: Zhenchi Date: Mon, 14 Apr 2025 15:08:59 +0800 Subject: [PATCH 13/82] feat: apply terms with fulltext bloom backend (#5884) * feat: apply terms with fulltext bloom backend Signed-off-by: Zhenchi * perf: preload jieba Signed-off-by: Zhenchi * polish doc Signed-off-by: Zhenchi --------- Signed-off-by: Zhenchi --- Cargo.lock | 1 + src/index/Cargo.toml | 1 + src/index/src/fulltext_index/tokenizer.rs | 9 +- .../src/cache/index/bloom_filter_index.rs | 37 +- src/mito2/src/read/scan_region.rs | 3 +- src/mito2/src/read/scan_util.rs | 1 + .../src/sst/index/bloom_filter/applier.rs | 9 +- .../src/sst/index/fulltext_index/applier.rs | 237 ++++++- .../index/fulltext_index/applier/builder.rs | 13 + .../src/sst/index/fulltext_index/creator.rs | 647 ++++++++++++++++-- src/mito2/src/sst/parquet/reader.rs | 66 +- 11 files changed, 930 insertions(+), 94 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1b108a7546..2ab3200029 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5591,6 +5591,7 @@ dependencies = [ "greptime-proto", "itertools 0.14.0", "jieba-rs", + "lazy_static", "mockall", "pin-project", "prost 0.13.5", diff --git a/src/index/Cargo.toml b/src/index/Cargo.toml index dc6e394ef4..c4b7057895 100644 --- a/src/index/Cargo.toml +++ b/src/index/Cargo.toml @@ -23,6 +23,7 @@ futures.workspace = true greptime-proto.workspace = true itertools.workspace = true jieba-rs = "0.7" +lazy_static.workspace = true mockall.workspace = true pin-project.workspace = true prost.workspace = true diff --git a/src/index/src/fulltext_index/tokenizer.rs b/src/index/src/fulltext_index/tokenizer.rs index 721ffdd3b9..b00e7fda9c 100644 --- a/src/index/src/fulltext_index/tokenizer.rs +++ b/src/index/src/fulltext_index/tokenizer.rs @@ -12,11 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use jieba_rs::Jieba; - use crate::fulltext_index::error::Result; use crate::Bytes; +lazy_static::lazy_static! { + static ref JIEBA: jieba_rs::Jieba = jieba_rs::Jieba::new(); +} + /// `Tokenizer` tokenizes a text into a list of tokens. pub trait Tokenizer: Send { fn tokenize<'a>(&self, text: &'a str) -> Vec<&'a str>; @@ -44,8 +46,7 @@ pub struct ChineseTokenizer; impl Tokenizer for ChineseTokenizer { fn tokenize<'a>(&self, text: &'a str) -> Vec<&'a str> { - let jieba = Jieba::new(); - jieba.cut(text, false) + JIEBA.cut(text, false) } } diff --git a/src/mito2/src/cache/index/bloom_filter_index.rs b/src/mito2/src/cache/index/bloom_filter_index.rs index 097acd6367..61df853573 100644 --- a/src/mito2/src/cache/index/bloom_filter_index.rs +++ b/src/mito2/src/cache/index/bloom_filter_index.rs @@ -29,8 +29,15 @@ use crate::sst::file::FileId; const INDEX_TYPE_BLOOM_FILTER_INDEX: &str = "bloom_filter_index"; +/// Tag for bloom filter index cache. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Tag { + Skipping, + Fulltext, +} + /// Cache for bloom filter index. -pub type BloomFilterIndexCache = IndexCache<(FileId, ColumnId), BloomFilterMeta>; +pub type BloomFilterIndexCache = IndexCache<(FileId, ColumnId, Tag), BloomFilterMeta>; pub type BloomFilterIndexCacheRef = Arc; impl BloomFilterIndexCache { @@ -48,14 +55,20 @@ impl BloomFilterIndexCache { } /// Calculates weight for bloom filter index metadata. -fn bloom_filter_index_metadata_weight(k: &(FileId, ColumnId), _: &Arc) -> u32 { +fn bloom_filter_index_metadata_weight( + k: &(FileId, ColumnId, Tag), + _: &Arc, +) -> u32 { (k.0.as_bytes().len() + std::mem::size_of::() + std::mem::size_of::()) as u32 } /// Calculates weight for bloom filter index content. -fn bloom_filter_index_content_weight((k, _): &((FileId, ColumnId), PageKey), v: &Bytes) -> u32 { +fn bloom_filter_index_content_weight( + (k, _): &((FileId, ColumnId, Tag), PageKey), + v: &Bytes, +) -> u32 { (k.0.as_bytes().len() + std::mem::size_of::() + v.len()) as u32 } @@ -63,6 +76,7 @@ fn bloom_filter_index_content_weight((k, _): &((FileId, ColumnId), PageKey), v: pub struct CachedBloomFilterIndexBlobReader { file_id: FileId, column_id: ColumnId, + tag: Tag, blob_size: u64, inner: R, cache: BloomFilterIndexCacheRef, @@ -73,6 +87,7 @@ impl CachedBloomFilterIndexBlobReader { pub fn new( file_id: FileId, column_id: ColumnId, + tag: Tag, blob_size: u64, inner: R, cache: BloomFilterIndexCacheRef, @@ -80,6 +95,7 @@ impl CachedBloomFilterIndexBlobReader { Self { file_id, column_id, + tag, blob_size, inner, cache, @@ -93,7 +109,7 @@ impl BloomFilterReader for CachedBloomFilterIndexBl let inner = &self.inner; self.cache .get_or_load( - (self.file_id, self.column_id), + (self.file_id, self.column_id, self.tag), self.blob_size, offset, size, @@ -107,7 +123,7 @@ impl BloomFilterReader for CachedBloomFilterIndexBl let fetch = ranges.iter().map(|range| { let inner = &self.inner; self.cache.get_or_load( - (self.file_id, self.column_id), + (self.file_id, self.column_id, self.tag), self.blob_size, range.start, (range.end - range.start) as u32, @@ -123,13 +139,18 @@ impl BloomFilterReader for CachedBloomFilterIndexBl /// Reads the meta information of the bloom filter. async fn metadata(&self) -> Result { - if let Some(cached) = self.cache.get_metadata((self.file_id, self.column_id)) { + if let Some(cached) = self + .cache + .get_metadata((self.file_id, self.column_id, self.tag)) + { CACHE_HIT.with_label_values(&[INDEX_METADATA_TYPE]).inc(); Ok((*cached).clone()) } else { let meta = self.inner.metadata().await?; - self.cache - .put_metadata((self.file_id, self.column_id), Arc::new(meta.clone())); + self.cache.put_metadata( + (self.file_id, self.column_id, self.tag), + Arc::new(meta.clone()), + ); CACHE_MISS.with_label_values(&[INDEX_METADATA_TYPE]).inc(); Ok(meta) } diff --git a/src/mito2/src/read/scan_region.rs b/src/mito2/src/read/scan_region.rs index 855455453f..8b9efb9ae8 100644 --- a/src/mito2/src/read/scan_region.rs +++ b/src/mito2/src/read/scan_region.rs @@ -502,7 +502,7 @@ impl ScanRegion { let file_cache = self.cache_strategy.write_cache().map(|w| w.file_cache()); let puffin_metadata_cache = self.cache_strategy.puffin_metadata_cache().cloned(); - + let bloom_filter_index_cache = self.cache_strategy.bloom_filter_index_cache().cloned(); FulltextIndexApplierBuilder::new( self.access_layer.region_dir().to_string(), self.version.metadata.region_id, @@ -512,6 +512,7 @@ impl ScanRegion { ) .with_file_cache(file_cache) .with_puffin_metadata_cache(puffin_metadata_cache) + .with_bloom_filter_cache(bloom_filter_index_cache) .build(&self.request.filters) .inspect_err(|err| warn!(err; "Failed to build fulltext index applier")) .ok() diff --git a/src/mito2/src/read/scan_util.rs b/src/mito2/src/read/scan_util.rs index 4a211f7117..36899796b0 100644 --- a/src/mito2/src/read/scan_util.rs +++ b/src/mito2/src/read/scan_util.rs @@ -505,6 +505,7 @@ pub(crate) fn scan_file_ranges( // Reports metrics. reader_metrics.observe_rows(read_type); + reader_metrics.filter_metrics.observe(); part_metrics.merge_reader_metrics(&reader_metrics); } } diff --git a/src/mito2/src/sst/index/bloom_filter/applier.rs b/src/mito2/src/sst/index/bloom_filter/applier.rs index afd5cc16cd..fac5db5405 100644 --- a/src/mito2/src/sst/index/bloom_filter/applier.rs +++ b/src/mito2/src/sst/index/bloom_filter/applier.rs @@ -31,7 +31,7 @@ use store_api::storage::{ColumnId, RegionId}; use crate::access_layer::{RegionFilePathFactory, WriteCachePathProvider}; use crate::cache::file_cache::{FileCacheRef, FileType, IndexKey}; use crate::cache::index::bloom_filter_index::{ - BloomFilterIndexCacheRef, CachedBloomFilterIndexBlobReader, + BloomFilterIndexCacheRef, CachedBloomFilterIndexBlobReader, Tag, }; use crate::error::{ ApplyBloomFilterIndexSnafu, Error, MetadataSnafu, PuffinBuildReaderSnafu, PuffinReadBlobSnafu, @@ -165,6 +165,7 @@ impl BloomFilterIndexApplier { let reader = CachedBloomFilterIndexBlobReader::new( file_id, *column_id, + Tag::Skipping, blob_size, BloomFilterReaderImpl::new(blob), bloom_filter_cache.clone(), @@ -308,13 +309,13 @@ impl BloomFilterIndexApplier { ) -> std::result::Result<(), index::bloom_filter::error::Error> { let mut applier = BloomFilterApplier::new(Box::new(reader)).await?; - for (_, output) in output.iter_mut() { + for (_, row_group_output) in output.iter_mut() { // All rows are filtered out, skip the search - if output.is_empty() { + if row_group_output.is_empty() { continue; } - *output = applier.search(predicates, output).await?; + *row_group_output = applier.search(predicates, row_group_output).await?; } Ok(()) diff --git a/src/mito2/src/sst/index/fulltext_index/applier.rs b/src/mito2/src/sst/index/fulltext_index/applier.rs index 94ceda6891..063227a89f 100644 --- a/src/mito2/src/sst/index/fulltext_index/applier.rs +++ b/src/mito2/src/sst/index/fulltext_index/applier.rs @@ -12,10 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::{BTreeSet, HashMap}; +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::iter; +use std::ops::Range; use std::sync::Arc; +use common_base::range_read::RangeReader; use common_telemetry::warn; +use index::bloom_filter::applier::{BloomFilterApplier, InListPredicate}; +use index::bloom_filter::reader::BloomFilterReaderImpl; use index::fulltext_index::search::{FulltextIndexSearcher, RowId, TantivyFulltextIndexSearcher}; use index::fulltext_index::Config; use object_store::ObjectStore; @@ -26,11 +31,17 @@ use store_api::storage::{ColumnId, RegionId}; use crate::access_layer::{RegionFilePathFactory, WriteCachePathProvider}; use crate::cache::file_cache::{FileCacheRef, FileType, IndexKey}; -use crate::error::{ApplyFulltextIndexSnafu, PuffinBuildReaderSnafu, PuffinReadBlobSnafu, Result}; +use crate::cache::index::bloom_filter_index::{ + BloomFilterIndexCacheRef, CachedBloomFilterIndexBlobReader, Tag, +}; +use crate::error::{ + ApplyBloomFilterIndexSnafu, ApplyFulltextIndexSnafu, MetadataSnafu, PuffinBuildReaderSnafu, + PuffinReadBlobSnafu, Result, +}; use crate::metrics::INDEX_APPLY_ELAPSED; use crate::sst::file::FileId; -use crate::sst::index::fulltext_index::applier::builder::FulltextRequest; -use crate::sst::index::fulltext_index::INDEX_BLOB_TYPE_TANTIVY; +use crate::sst::index::fulltext_index::applier::builder::{FulltextRequest, FulltextTerm}; +use crate::sst::index::fulltext_index::{INDEX_BLOB_TYPE_BLOOM, INDEX_BLOB_TYPE_TANTIVY}; use crate::sst::index::puffin_manager::{ PuffinManagerFactory, SstPuffinBlob, SstPuffinDir, SstPuffinReader, }; @@ -45,6 +56,9 @@ pub struct FulltextIndexApplier { /// The source of the index. index_source: IndexSource, + + /// Cache for bloom filter index. + bloom_filter_index_cache: Option, } pub type FulltextIndexApplierRef = Arc; @@ -63,6 +77,7 @@ impl FulltextIndexApplier { Self { requests, index_source, + bloom_filter_index_cache: None, } } @@ -82,13 +97,25 @@ impl FulltextIndexApplier { self } - /// Applies the queries to the fulltext index of the specified SST file. - pub async fn apply( + /// Sets the bloom filter cache. + pub fn with_bloom_filter_cache( + mut self, + bloom_filter_index_cache: Option, + ) -> Self { + self.bloom_filter_index_cache = bloom_filter_index_cache; + self + } +} + +impl FulltextIndexApplier { + /// Applies fine-grained fulltext index to the specified SST file. + /// Returns the row ids that match the queries. + pub async fn apply_fine( &self, file_id: FileId, file_size_hint: Option, ) -> Result>> { - let _timer = INDEX_APPLY_ELAPSED + let timer = INDEX_APPLY_ELAPSED .with_label_values(&[TYPE_FULLTEXT_INDEX]) .start_timer(); @@ -99,7 +126,7 @@ impl FulltextIndexApplier { } let Some(result) = self - .apply_one_column(file_size_hint, file_id, *column_id, request) + .apply_fine_one_column(file_size_hint, file_id, *column_id, request) .await? else { continue; @@ -118,10 +145,13 @@ impl FulltextIndexApplier { } } + if row_ids.is_none() { + timer.stop_and_discard(); + } Ok(row_ids) } - async fn apply_one_column( + async fn apply_fine_one_column( &self, file_size_hint: Option, file_id: FileId, @@ -187,6 +217,195 @@ impl FulltextIndexApplier { } } +impl FulltextIndexApplier { + /// Applies coarse-grained fulltext index to the specified SST file. + /// Returns (row group id -> ranges) that match the queries. + pub async fn apply_coarse( + &self, + file_id: FileId, + file_size_hint: Option, + row_groups: impl Iterator, + ) -> Result>)>>> { + let timer = INDEX_APPLY_ELAPSED + .with_label_values(&[TYPE_FULLTEXT_INDEX]) + .start_timer(); + + let (input, mut output) = Self::init_coarse_output(row_groups); + let mut applied = false; + + for (column_id, request) in &self.requests { + if request.terms.is_empty() { + // only apply terms + continue; + } + + applied |= self + .apply_coarse_one_column( + file_id, + file_size_hint, + *column_id, + &request.terms, + &mut output, + ) + .await?; + } + + if !applied { + timer.stop_and_discard(); + return Ok(None); + } + + Self::adjust_coarse_output(input, &mut output); + Ok(Some(output)) + } + + async fn apply_coarse_one_column( + &self, + file_id: FileId, + file_size_hint: Option, + column_id: ColumnId, + terms: &[FulltextTerm], + output: &mut [(usize, Vec>)], + ) -> Result { + let blob_key = format!("{INDEX_BLOB_TYPE_BLOOM}-{column_id}"); + let Some(reader) = self + .index_source + .blob(file_id, &blob_key, file_size_hint) + .await? + else { + return Ok(false); + }; + let config = + Config::from_blob_metadata(reader.metadata()).context(ApplyFulltextIndexSnafu)?; + + let predicates = Self::terms_to_predicates(terms, &config); + if predicates.is_empty() { + return Ok(false); + } + + let range_reader = reader.reader().await.context(PuffinBuildReaderSnafu)?; + let reader = if let Some(bloom_filter_cache) = &self.bloom_filter_index_cache { + let blob_size = range_reader + .metadata() + .await + .context(MetadataSnafu)? + .content_length; + let reader = CachedBloomFilterIndexBlobReader::new( + file_id, + column_id, + Tag::Fulltext, + blob_size, + BloomFilterReaderImpl::new(range_reader), + bloom_filter_cache.clone(), + ); + Box::new(reader) as _ + } else { + Box::new(BloomFilterReaderImpl::new(range_reader)) as _ + }; + + let mut applier = BloomFilterApplier::new(reader) + .await + .context(ApplyBloomFilterIndexSnafu)?; + for (_, row_group_output) in output.iter_mut() { + // All rows are filtered out, skip the search + if row_group_output.is_empty() { + continue; + } + + *row_group_output = applier + .search(&predicates, row_group_output) + .await + .context(ApplyBloomFilterIndexSnafu)?; + } + + Ok(true) + } + + /// Initializes the coarse output. Must call `adjust_coarse_output` after applying bloom filters. + /// + /// `row_groups` is a list of (row group length, whether to search). + /// + /// Returns (`input`, `output`): + /// * `input` is a list of (row group index to search, row group range based on start of the file). + /// * `output` is a list of (row group index to search, row group ranges based on start of the file). + #[allow(clippy::type_complexity)] + fn init_coarse_output( + row_groups: impl Iterator, + ) -> (Vec<(usize, Range)>, Vec<(usize, Vec>)>) { + // Calculates row groups' ranges based on start of the file. + let mut input = Vec::with_capacity(row_groups.size_hint().0); + let mut start = 0; + for (i, (len, to_search)) in row_groups.enumerate() { + let end = start + len; + if to_search { + input.push((i, start..end)); + } + start = end; + } + + // Initializes output with input ranges, but ranges are based on start of the file not the row group, + // so we need to adjust them later. + let output = input + .iter() + .map(|(i, range)| (*i, vec![range.clone()])) + .collect::>(); + + (input, output) + } + + /// Adjusts the coarse output. Makes the output ranges based on row group start. + fn adjust_coarse_output( + input: Vec<(usize, Range)>, + output: &mut Vec<(usize, Vec>)>, + ) { + // adjust ranges to be based on row group + for ((_, output), (_, input)) in output.iter_mut().zip(input) { + let start = input.start; + for range in output.iter_mut() { + range.start -= start; + range.end -= start; + } + } + output.retain(|(_, ranges)| !ranges.is_empty()); + } + + /// Converts terms to predicates. + /// + /// Split terms by non-alphanumeric characters and convert them to lowercase if case-insensitive. + /// Multiple terms are combined with AND semantics. + fn terms_to_predicates(terms: &[FulltextTerm], config: &Config) -> Vec { + let mut probes = HashSet::new(); + for term in terms { + if config.case_sensitive && term.col_lowered { + // lowercased terms are not indexed + continue; + } + + let ts = term + .term + .split(|c: char| !c.is_alphanumeric()) + .filter(|&t| !t.is_empty()) + .map(|t| { + if !config.case_sensitive { + t.to_lowercase() + } else { + t.to_string() + } + .into_bytes() + }); + + probes.extend(ts); + } + + probes + .into_iter() + .map(|p| InListPredicate { + list: iter::once(p).collect(), + }) + .collect::>() + } +} + /// The source of the index. struct IndexSource { region_dir: String, diff --git a/src/mito2/src/sst/index/fulltext_index/applier/builder.rs b/src/mito2/src/sst/index/fulltext_index/applier/builder.rs index 14f5936a01..3297275f26 100644 --- a/src/mito2/src/sst/index/fulltext_index/applier/builder.rs +++ b/src/mito2/src/sst/index/fulltext_index/applier/builder.rs @@ -23,6 +23,7 @@ use store_api::metadata::RegionMetadata; use store_api::storage::{ColumnId, ConcreteDataType, RegionId}; use crate::cache::file_cache::FileCacheRef; +use crate::cache::index::bloom_filter_index::BloomFilterIndexCacheRef; use crate::error::Result; use crate::sst::index::fulltext_index::applier::FulltextIndexApplier; use crate::sst::index::puffin_manager::PuffinManagerFactory; @@ -86,6 +87,7 @@ pub struct FulltextIndexApplierBuilder<'a> { metadata: &'a RegionMetadata, file_cache: Option, puffin_metadata_cache: Option, + bloom_filter_cache: Option, } impl<'a> FulltextIndexApplierBuilder<'a> { @@ -105,6 +107,7 @@ impl<'a> FulltextIndexApplierBuilder<'a> { metadata, file_cache: None, puffin_metadata_cache: None, + bloom_filter_cache: None, } } @@ -123,6 +126,15 @@ impl<'a> FulltextIndexApplierBuilder<'a> { self } + /// Sets the bloom filter cache to be used by the `FulltextIndexApplier`. + pub fn with_bloom_filter_cache( + mut self, + bloom_filter_cache: Option, + ) -> Self { + self.bloom_filter_cache = bloom_filter_cache; + self + } + /// Builds `SstIndexApplier` from the given expressions. pub fn build(self, exprs: &[Expr]) -> Result> { let mut requests = HashMap::new(); @@ -145,6 +157,7 @@ impl<'a> FulltextIndexApplierBuilder<'a> { ) .with_file_cache(self.file_cache) .with_puffin_metadata_cache(self.puffin_metadata_cache) + .with_bloom_filter_cache(self.bloom_filter_cache) })) } diff --git a/src/mito2/src/sst/index/fulltext_index/creator.rs b/src/mito2/src/sst/index/fulltext_index/creator.rs index 12b83e39d0..1d884ac3a5 100644 --- a/src/mito2/src/sst/index/fulltext_index/creator.rs +++ b/src/mito2/src/sst/index/fulltext_index/creator.rs @@ -360,6 +360,7 @@ mod tests { use std::sync::Arc; use api::v1::SemanticType; + use common_base::BitVec; use datatypes::data_type::DataType; use datatypes::schema::{ColumnSchema, FulltextAnalyzer, FulltextOptions}; use datatypes::vectors::{UInt64Vector, UInt8Vector}; @@ -390,7 +391,7 @@ mod tests { IntermediateManager::init_fs(path).await.unwrap() } - fn mock_region_metadata() -> RegionMetadataRef { + fn mock_region_metadata(backend: FulltextBackend) -> RegionMetadataRef { let mut builder = RegionMetadataBuilder::new(RegionId::new(1, 2)); builder .push_column_metadata(ColumnMetadata { @@ -403,7 +404,7 @@ mod tests { enable: true, analyzer: FulltextAnalyzer::English, case_sensitive: true, - backend: FulltextBackend::Tantivy, + backend: backend.clone(), }) .unwrap(), semantic_type: SemanticType::Field, @@ -419,7 +420,7 @@ mod tests { enable: true, analyzer: FulltextAnalyzer::English, case_sensitive: false, - backend: FulltextBackend::Tantivy, + backend: backend.clone(), }) .unwrap(), semantic_type: SemanticType::Field, @@ -435,7 +436,7 @@ mod tests { enable: true, analyzer: FulltextAnalyzer::Chinese, case_sensitive: false, - backend: FulltextBackend::Tantivy, + backend: backend.clone(), }) .unwrap(), semantic_type: SemanticType::Field, @@ -522,6 +523,7 @@ mod tests { /// - `terms`: A list of (ColumnId, [(bool, String)]) for fulltext terms, where bool indicates if term is lowercased async fn build_fulltext_applier_factory( prefix: &str, + backend: FulltextBackend, rows: &[( Option<&str>, // text_english_case_sensitive Option<&str>, // text_english_case_insensitive @@ -530,12 +532,13 @@ mod tests { ) -> impl Fn( Vec<(ColumnId, &str)>, Vec<(ColumnId, Vec<(bool, &str)>)>, + Option, ) -> BoxFuture<'static, Option>> { let (d, factory) = PuffinManagerFactory::new_for_test_async(prefix).await; let region_dir = "region0".to_string(); let sst_file_id = FileId::random(); let object_store = mock_object_store(); - let region_metadata = mock_region_metadata(); + let region_metadata = mock_region_metadata(backend.clone()); let intm_mgr = new_intm_mgr(d.path().to_string_lossy()).await; let mut indexer = FulltextIndexer::new( @@ -544,7 +547,7 @@ mod tests { &intm_mgr, ®ion_metadata, true, - 8096, + 1, 1024, ) .await @@ -562,7 +565,9 @@ mod tests { let _ = indexer.finish(&mut writer).await.unwrap(); writer.finish().await.unwrap(); - move |queries: Vec<(ColumnId, &str)>, terms_requests: Vec<(ColumnId, Vec<(bool, &str)>)>| { + move |queries: Vec<(ColumnId, &str)>, + terms_requests: Vec<(ColumnId, Vec<(bool, &str)>)>, + coarse_mask: Option| { let _d = &d; let region_dir = region_dir.clone(); let object_store = object_store.clone(); @@ -604,7 +609,29 @@ mod tests { factory, ); - async move { applier.apply(sst_file_id, None).await.unwrap() }.boxed() + let backend = backend.clone(); + async move { + match backend { + FulltextBackend::Tantivy => { + applier.apply_fine(sst_file_id, None).await.unwrap() + } + FulltextBackend::Bloom => { + let coarse_mask = coarse_mask.unwrap_or_default(); + let row_groups = (0..coarse_mask.len()).map(|i| (1, coarse_mask[i])); + // row group id == row id + let resp = applier + .apply_coarse(sst_file_id, None, row_groups) + .await + .unwrap(); + resp.map(|r| { + r.into_iter() + .map(|(row_group_id, _)| row_group_id as RowId) + .collect() + }) + } + } + } + .boxed() } } @@ -613,9 +640,10 @@ mod tests { } #[tokio::test] - async fn test_fulltext_index_basic_case_sensitive() { + async fn test_fulltext_index_basic_case_sensitive_tantivy() { let applier_factory = build_fulltext_applier_factory( - "test_fulltext_index_basic_case_sensitive_", + "test_fulltext_index_basic_case_sensitive_tantivy_", + FulltextBackend::Tantivy, &[ (Some("hello"), None, None), (Some("world"), None, None), @@ -625,47 +653,159 @@ mod tests { ) .await; - let row_ids = applier_factory(vec![(1, "hello")], vec![]).await; + let row_ids = applier_factory(vec![(1, "hello")], vec![], None).await; assert_eq!(row_ids, Some(rows([0]))); - let row_ids = applier_factory(vec![(1, "world")], vec![]).await; + let row_ids = applier_factory(vec![(1, "world")], vec![], None).await; assert_eq!(row_ids, Some(rows([1]))); - let row_ids = applier_factory(vec![(1, "Hello")], vec![]).await; + let row_ids = applier_factory(vec![(1, "Hello")], vec![], None).await; assert_eq!(row_ids, Some(rows([3]))); - let row_ids = applier_factory(vec![(1, "World")], vec![]).await; + let row_ids = applier_factory(vec![(1, "World")], vec![], None).await; assert_eq!(row_ids, Some(rows([3]))); - let row_ids = applier_factory(vec![], vec![(1, vec![(false, "hello")])]).await; + let row_ids = applier_factory(vec![], vec![(1, vec![(false, "hello")])], None).await; assert_eq!(row_ids, Some(rows([0]))); - let row_ids = applier_factory(vec![], vec![(1, vec![(true, "hello")])]).await; + let row_ids = applier_factory(vec![], vec![(1, vec![(true, "hello")])], None).await; assert_eq!(row_ids, None); - let row_ids = applier_factory(vec![], vec![(1, vec![(false, "world")])]).await; + let row_ids = applier_factory(vec![], vec![(1, vec![(false, "world")])], None).await; assert_eq!(row_ids, Some(rows([1]))); - let row_ids = applier_factory(vec![], vec![(1, vec![(true, "world")])]).await; + let row_ids = applier_factory(vec![], vec![(1, vec![(true, "world")])], None).await; assert_eq!(row_ids, None); - let row_ids = applier_factory(vec![], vec![(1, vec![(false, "Hello")])]).await; + let row_ids = applier_factory(vec![], vec![(1, vec![(false, "Hello")])], None).await; assert_eq!(row_ids, Some(rows([3]))); - let row_ids = applier_factory(vec![], vec![(1, vec![(true, "Hello")])]).await; + let row_ids = applier_factory(vec![], vec![(1, vec![(true, "Hello")])], None).await; assert_eq!(row_ids, None); - let row_ids = applier_factory(vec![], vec![(1, vec![(false, "Hello, World")])]).await; + let row_ids = applier_factory(vec![], vec![(1, vec![(false, "Hello, World")])], None).await; assert_eq!(row_ids, Some(rows([3]))); - let row_ids = applier_factory(vec![], vec![(1, vec![(true, "Hello, World")])]).await; + let row_ids = applier_factory(vec![], vec![(1, vec![(true, "Hello, World")])], None).await; assert_eq!(row_ids, None); } #[tokio::test] - async fn test_fulltext_index_basic_case_insensitive() { + async fn test_fulltext_index_basic_case_sensitive_bloom() { let applier_factory = build_fulltext_applier_factory( - "test_fulltext_index_basic_case_insensitive_", + "test_fulltext_index_basic_case_sensitive_bloom_", + FulltextBackend::Bloom, + &[ + (Some("hello"), None, None), + (Some("world"), None, None), + (None, None, None), + (Some("Hello, World"), None, None), + ], + ) + .await; + + let row_ids = applier_factory( + vec![], + vec![(1, vec![(false, "hello")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([0]))); + + let row_ids = applier_factory( + vec![], + vec![(1, vec![(false, "hello")])], + Some(BitVec::from_slice(&[0b1110])), // row 0 is filtered out + ) + .await; + assert_eq!(row_ids, Some(rows([]))); + + let row_ids = applier_factory( + vec![], + vec![(1, vec![(true, "hello")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, None); + + let row_ids = applier_factory( + vec![], + vec![(1, vec![(false, "world")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([1]))); + + let row_ids = applier_factory( + vec![], + vec![(1, vec![(false, "world")])], + Some(BitVec::from_slice(&[0b1101])), // row 1 is filtered out + ) + .await; + assert_eq!(row_ids, Some(rows([]))); + + let row_ids = applier_factory( + vec![], + vec![(1, vec![(true, "world")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, None); + + let row_ids = applier_factory( + vec![], + vec![(1, vec![(false, "Hello")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([3]))); + + let row_ids = applier_factory( + vec![], + vec![(1, vec![(false, "Hello")])], + Some(BitVec::from_slice(&[0b0111])), // row 3 is filtered out + ) + .await; + assert_eq!(row_ids, Some(rows([]))); + + let row_ids = applier_factory( + vec![], + vec![(1, vec![(true, "Hello")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, None); + + let row_ids = applier_factory( + vec![], + vec![(1, vec![(false, "Hello, World")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([3]))); + + let row_ids = applier_factory( + vec![], + vec![(1, vec![(false, "Hello, World")])], + Some(BitVec::from_slice(&[0b0111])), // row 3 is filtered out + ) + .await; + assert_eq!(row_ids, Some(rows([]))); + + let row_ids = applier_factory( + vec![], + vec![(1, vec![(true, "Hello, World")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, None); + } + + #[tokio::test] + async fn test_fulltext_index_basic_case_insensitive_tantivy() { + let applier_factory = build_fulltext_applier_factory( + "test_fulltext_index_basic_case_insensitive_tantivy_", + FulltextBackend::Tantivy, &[ (None, Some("hello"), None), (None, None, None), @@ -675,47 +815,191 @@ mod tests { ) .await; - let row_ids = applier_factory(vec![(2, "hello")], vec![]).await; + let row_ids = applier_factory(vec![(2, "hello")], vec![], None).await; assert_eq!(row_ids, Some(rows([0, 3]))); - let row_ids = applier_factory(vec![(2, "world")], vec![]).await; + let row_ids = applier_factory(vec![(2, "world")], vec![], None).await; assert_eq!(row_ids, Some(rows([2, 3]))); - let row_ids = applier_factory(vec![(2, "Hello")], vec![]).await; + let row_ids = applier_factory(vec![(2, "Hello")], vec![], None).await; assert_eq!(row_ids, Some(rows([0, 3]))); - let row_ids = applier_factory(vec![(2, "World")], vec![]).await; + let row_ids = applier_factory(vec![(2, "World")], vec![], None).await; assert_eq!(row_ids, Some(rows([2, 3]))); - let row_ids = applier_factory(vec![], vec![(2, vec![(false, "hello")])]).await; + let row_ids = applier_factory(vec![], vec![(2, vec![(false, "hello")])], None).await; assert_eq!(row_ids, Some(rows([0, 3]))); - let row_ids = applier_factory(vec![], vec![(2, vec![(true, "hello")])]).await; + let row_ids = applier_factory(vec![], vec![(2, vec![(true, "hello")])], None).await; assert_eq!(row_ids, Some(rows([0, 3]))); - let row_ids = applier_factory(vec![], vec![(2, vec![(false, "world")])]).await; + let row_ids = applier_factory(vec![], vec![(2, vec![(false, "world")])], None).await; assert_eq!(row_ids, Some(rows([2, 3]))); - let row_ids = applier_factory(vec![], vec![(2, vec![(true, "world")])]).await; + let row_ids = applier_factory(vec![], vec![(2, vec![(true, "world")])], None).await; assert_eq!(row_ids, Some(rows([2, 3]))); - let row_ids = applier_factory(vec![], vec![(2, vec![(false, "Hello")])]).await; + let row_ids = applier_factory(vec![], vec![(2, vec![(false, "Hello")])], None).await; assert_eq!(row_ids, Some(rows([0, 3]))); - let row_ids = applier_factory(vec![], vec![(2, vec![(true, "Hello")])]).await; + let row_ids = applier_factory(vec![], vec![(2, vec![(true, "Hello")])], None).await; assert_eq!(row_ids, Some(rows([0, 3]))); - let row_ids = applier_factory(vec![], vec![(2, vec![(false, "World")])]).await; + let row_ids = applier_factory(vec![], vec![(2, vec![(false, "World")])], None).await; assert_eq!(row_ids, Some(rows([2, 3]))); - let row_ids = applier_factory(vec![], vec![(2, vec![(true, "World")])]).await; + let row_ids = applier_factory(vec![], vec![(2, vec![(true, "World")])], None).await; assert_eq!(row_ids, Some(rows([2, 3]))); } #[tokio::test] - async fn test_fulltext_index_basic_chinese() { + async fn test_fulltext_index_basic_case_insensitive_bloom() { let applier_factory = build_fulltext_applier_factory( - "test_fulltext_index_basic_chinese_", + "test_fulltext_index_basic_case_insensitive_bloom_", + FulltextBackend::Bloom, + &[ + (None, Some("hello"), None), + (None, None, None), + (None, Some("world"), None), + (None, Some("Hello, World"), None), + ], + ) + .await; + + let row_ids = applier_factory( + vec![], + vec![(2, vec![(false, "hello")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([0, 3]))); + + let row_ids = applier_factory( + vec![], + vec![(2, vec![(false, "hello")])], + Some(BitVec::from_slice(&[0b1110])), // row 0 is filtered out + ) + .await; + assert_eq!(row_ids, Some(rows([3]))); + + let row_ids = applier_factory( + vec![], + vec![(2, vec![(true, "hello")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([0, 3]))); + + let row_ids = applier_factory( + vec![], + vec![(2, vec![(true, "hello")])], + Some(BitVec::from_slice(&[0b1110])), // row 0 is filtered out + ) + .await; + assert_eq!(row_ids, Some(rows([3]))); + + let row_ids = applier_factory( + vec![], + vec![(2, vec![(false, "world")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([2, 3]))); + + let row_ids = applier_factory( + vec![], + vec![(2, vec![(false, "world")])], + Some(BitVec::from_slice(&[0b1011])), // row 2 is filtered out + ) + .await; + assert_eq!(row_ids, Some(rows([3]))); + + let row_ids = applier_factory( + vec![], + vec![(2, vec![(true, "world")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([2, 3]))); + + let row_ids = applier_factory( + vec![], + vec![(2, vec![(true, "world")])], + Some(BitVec::from_slice(&[0b1011])), // row 2 is filtered out + ) + .await; + assert_eq!(row_ids, Some(rows([3]))); + + let row_ids = applier_factory( + vec![], + vec![(2, vec![(false, "Hello")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([0, 3]))); + + let row_ids = applier_factory( + vec![], + vec![(2, vec![(false, "Hello")])], + Some(BitVec::from_slice(&[0b0111])), // row 3 is filtered out + ) + .await; + assert_eq!(row_ids, Some(rows([0]))); + + let row_ids = applier_factory( + vec![], + vec![(2, vec![(true, "Hello")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([0, 3]))); + + let row_ids = applier_factory( + vec![], + vec![(2, vec![(true, "Hello")])], + Some(BitVec::from_slice(&[0b1110])), // row 0 is filtered out + ) + .await; + assert_eq!(row_ids, Some(rows([3]))); + + let row_ids = applier_factory( + vec![], + vec![(2, vec![(false, "World")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([2, 3]))); + + let row_ids = applier_factory( + vec![], + vec![(2, vec![(false, "World")])], + Some(BitVec::from_slice(&[0b0111])), // row 3 is filtered out + ) + .await; + assert_eq!(row_ids, Some(rows([2]))); + + let row_ids = applier_factory( + vec![], + vec![(2, vec![(true, "World")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([2, 3]))); + + let row_ids = applier_factory( + vec![], + vec![(2, vec![(true, "World")])], + Some(BitVec::from_slice(&[0b1011])), // row 2 is filtered out + ) + .await; + assert_eq!(row_ids, Some(rows([3]))); + } + + #[tokio::test] + async fn test_fulltext_index_basic_chinese_tantivy() { + let applier_factory = build_fulltext_applier_factory( + "test_fulltext_index_basic_chinese_tantivy_", + FulltextBackend::Tantivy, &[ (None, None, Some("你好")), (None, None, None), @@ -725,23 +1009,71 @@ mod tests { ) .await; - let row_ids = applier_factory(vec![(3, "你好")], vec![]).await; + let row_ids = applier_factory(vec![(3, "你好")], vec![], None).await; assert_eq!(row_ids, Some(rows([0, 3]))); - let row_ids = applier_factory(vec![(3, "世界")], vec![]).await; + let row_ids = applier_factory(vec![(3, "世界")], vec![], None).await; assert_eq!(row_ids, Some(rows([2, 3]))); - let row_ids = applier_factory(vec![], vec![(3, vec![(false, "你好")])]).await; + let row_ids = applier_factory(vec![], vec![(3, vec![(false, "你好")])], None).await; assert_eq!(row_ids, Some(rows([0, 3]))); - let row_ids = applier_factory(vec![], vec![(3, vec![(false, "世界")])]).await; + let row_ids = applier_factory(vec![], vec![(3, vec![(false, "世界")])], None).await; assert_eq!(row_ids, Some(rows([2, 3]))); } #[tokio::test] - async fn test_fulltext_index_multi_terms_case_sensitive() { + async fn test_fulltext_index_basic_chinese_bloom() { let applier_factory = build_fulltext_applier_factory( - "test_fulltext_index_multi_terms_case_sensitive_", + "test_fulltext_index_basic_chinese_bloom_", + FulltextBackend::Bloom, + &[ + (None, None, Some("你好")), + (None, None, None), + (None, None, Some("世界")), + (None, None, Some("你好，世界")), + ], + ) + .await; + + let row_ids = applier_factory( + vec![], + vec![(3, vec![(false, "你好")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([0, 3]))); + + let row_ids = applier_factory( + vec![], + vec![(3, vec![(false, "你好")])], + Some(BitVec::from_slice(&[0b1110])), // row 0 is filtered out + ) + .await; + assert_eq!(row_ids, Some(rows([3]))); + + let row_ids = applier_factory( + vec![], + vec![(3, vec![(false, "世界")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([2, 3]))); + + let row_ids = applier_factory( + vec![], + vec![(3, vec![(false, "世界")])], + Some(BitVec::from_slice(&[0b1011])), // row 2 is filtered out + ) + .await; + assert_eq!(row_ids, Some(rows([3]))); + } + + #[tokio::test] + async fn test_fulltext_index_multi_terms_case_sensitive_tantivy() { + let applier_factory = build_fulltext_applier_factory( + "test_fulltext_index_multi_terms_case_sensitive_tantivy_", + FulltextBackend::Tantivy, &[ (Some("Hello"), None, None), (Some("World"), None, None), @@ -751,31 +1083,107 @@ mod tests { ) .await; - let row_ids = - applier_factory(vec![], vec![(1, vec![(false, "hello"), (false, "world")])]).await; + let row_ids = applier_factory( + vec![], + vec![(1, vec![(false, "hello"), (false, "world")])], + None, + ) + .await; assert_eq!(row_ids, Some(rows([]))); - let row_ids = - applier_factory(vec![], vec![(1, vec![(false, "Hello"), (false, "World")])]).await; + let row_ids = applier_factory( + vec![], + vec![(1, vec![(false, "Hello"), (false, "World")])], + None, + ) + .await; assert_eq!(row_ids, Some(rows([3]))); - let row_ids = - applier_factory(vec![], vec![(1, vec![(true, "Hello"), (false, "World")])]).await; + let row_ids = applier_factory( + vec![], + vec![(1, vec![(true, "Hello"), (false, "World")])], + None, + ) + .await; assert_eq!(row_ids, Some(rows([1, 3]))); - let row_ids = - applier_factory(vec![], vec![(1, vec![(false, "Hello"), (true, "World")])]).await; + let row_ids = applier_factory( + vec![], + vec![(1, vec![(false, "Hello"), (true, "World")])], + None, + ) + .await; assert_eq!(row_ids, Some(rows([0, 3]))); - let row_ids = - applier_factory(vec![], vec![(1, vec![(true, "Hello"), (true, "World")])]).await; + let row_ids = applier_factory( + vec![], + vec![(1, vec![(true, "Hello"), (true, "World")])], + None, + ) + .await; assert_eq!(row_ids, None); } #[tokio::test] - async fn test_fulltext_index_multi_terms_case_insensitive() { + async fn test_fulltext_index_multi_terms_case_sensitive_bloom() { let applier_factory = build_fulltext_applier_factory( - "test_fulltext_index_multi_terms_case_insensitive_", + "test_fulltext_index_multi_terms_case_sensitive_bloom_", + FulltextBackend::Bloom, + &[ + (Some("Hello"), None, None), + (Some("World"), None, None), + (None, None, None), + (Some("Hello, World"), None, None), + ], + ) + .await; + + let row_ids = applier_factory( + vec![], + vec![(1, vec![(false, "hello"), (false, "world")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([]))); + + let row_ids = applier_factory( + vec![], + vec![(1, vec![(false, "Hello"), (false, "World")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([3]))); + + let row_ids = applier_factory( + vec![], + vec![(1, vec![(true, "Hello"), (false, "World")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([1, 3]))); + + let row_ids = applier_factory( + vec![], + vec![(1, vec![(false, "Hello"), (true, "World")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([0, 3]))); + + let row_ids = applier_factory( + vec![], + vec![(1, vec![(true, "Hello"), (true, "World")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, None); + } + + #[tokio::test] + async fn test_fulltext_index_multi_terms_case_insensitive_tantivy() { + let applier_factory = build_fulltext_applier_factory( + "test_fulltext_index_multi_terms_case_insensitive_tantivy_", + FulltextBackend::Tantivy, &[ (None, Some("hello"), None), (None, None, None), @@ -785,27 +1193,91 @@ mod tests { ) .await; - let row_ids = - applier_factory(vec![], vec![(2, vec![(false, "hello"), (false, "world")])]).await; + let row_ids = applier_factory( + vec![], + vec![(2, vec![(false, "hello"), (false, "world")])], + None, + ) + .await; assert_eq!(row_ids, Some(rows([3]))); - let row_ids = - applier_factory(vec![], vec![(2, vec![(true, "hello"), (false, "world")])]).await; + let row_ids = applier_factory( + vec![], + vec![(2, vec![(true, "hello"), (false, "world")])], + None, + ) + .await; assert_eq!(row_ids, Some(rows([3]))); - let row_ids = - applier_factory(vec![], vec![(2, vec![(false, "hello"), (true, "world")])]).await; + let row_ids = applier_factory( + vec![], + vec![(2, vec![(false, "hello"), (true, "world")])], + None, + ) + .await; assert_eq!(row_ids, Some(rows([3]))); - let row_ids = - applier_factory(vec![], vec![(2, vec![(true, "hello"), (true, "world")])]).await; + let row_ids = applier_factory( + vec![], + vec![(2, vec![(true, "hello"), (true, "world")])], + None, + ) + .await; assert_eq!(row_ids, Some(rows([3]))); } #[tokio::test] - async fn test_fulltext_index_multi_columns() { + async fn test_fulltext_index_multi_terms_case_insensitive_bloom() { let applier_factory = build_fulltext_applier_factory( - "test_fulltext_index_multi_columns_", + "test_fulltext_index_multi_terms_case_insensitive_bloom_", + FulltextBackend::Bloom, + &[ + (None, Some("hello"), None), + (None, None, None), + (None, Some("world"), None), + (None, Some("Hello, World"), None), + ], + ) + .await; + + let row_ids = applier_factory( + vec![], + vec![(2, vec![(false, "hello"), (false, "world")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([3]))); + + let row_ids = applier_factory( + vec![], + vec![(2, vec![(true, "hello"), (false, "world")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([3]))); + + let row_ids = applier_factory( + vec![], + vec![(2, vec![(false, "hello"), (true, "world")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([3]))); + + let row_ids = applier_factory( + vec![], + vec![(2, vec![(true, "hello"), (true, "world")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([3]))); + } + + #[tokio::test] + async fn test_fulltext_index_multi_columns_tantivy() { + let applier_factory = build_fulltext_applier_factory( + "test_fulltext_index_multi_columns_tantivy_", + FulltextBackend::Tantivy, &[ (Some("Hello"), None, Some("你好")), (Some("World"), Some("world"), None), @@ -822,11 +1294,52 @@ mod tests { let row_ids = applier_factory( vec![(1, "Hello"), (3, "你好")], vec![(2, vec![(false, "world")])], + None, ) .await; assert_eq!(row_ids, Some(rows([3]))); - let row_ids = applier_factory(vec![(2, "World")], vec![(1, vec![(false, "World")])]).await; + let row_ids = + applier_factory(vec![(2, "World")], vec![(1, vec![(false, "World")])], None).await; + assert_eq!(row_ids, Some(rows([1, 3]))); + } + + #[tokio::test] + async fn test_fulltext_index_multi_columns_bloom() { + let applier_factory = build_fulltext_applier_factory( + "test_fulltext_index_multi_columns_bloom_", + FulltextBackend::Bloom, + &[ + (Some("Hello"), None, Some("你好")), + (Some("World"), Some("world"), None), + (None, Some("World"), Some("世界")), + ( + Some("Hello, World"), + Some("Hello, World"), + Some("你好，世界"), + ), + ], + ) + .await; + + let row_ids = applier_factory( + vec![], + vec![ + (1, vec![(false, "Hello")]), + (2, vec![(false, "world")]), + (3, vec![(false, "你好")]), + ], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; + assert_eq!(row_ids, Some(rows([3]))); + + let row_ids = applier_factory( + vec![], + vec![(1, vec![(false, "World")]), (2, vec![(false, "World")])], + Some(BitVec::from_slice(&[0b1111])), + ) + .await; assert_eq!(row_ids, Some(rows([1, 3]))); } } diff --git a/src/mito2/src/sst/parquet/reader.rs b/src/mito2/src/sst/parquet/reader.rs index 069e10344c..5c2ab17591 100644 --- a/src/mito2/src/sst/parquet/reader.rs +++ b/src/mito2/src/sst/parquet/reader.rs @@ -369,6 +369,9 @@ impl ParquetReaderBuilder { self.prune_row_groups_by_bloom_filter(parquet_meta, &mut output, metrics) .await; + self.prune_row_groups_by_fulltext_bloom(parquet_meta, &mut output, metrics) + .await; + output } @@ -389,7 +392,7 @@ impl ParquetReaderBuilder { let file_size_hint = self.file_handle.meta_ref().index_file_size(); let apply_res = match index_applier - .apply(self.file_handle.file_id(), Some(file_size_hint)) + .apply_fine(self.file_handle.file_id(), Some(file_size_hint)) .await { Ok(Some(res)) => res, @@ -631,6 +634,67 @@ impl ParquetReaderBuilder { true } + async fn prune_row_groups_by_fulltext_bloom( + &self, + parquet_meta: &ParquetMetaData, + output: &mut BTreeMap>, + metrics: &mut ReaderFilterMetrics, + ) -> bool { + let Some(index_applier) = &self.fulltext_index_applier else { + return false; + }; + + if !self.file_handle.meta_ref().fulltext_index_available() { + return false; + } + + let file_size_hint = self.file_handle.meta_ref().index_file_size(); + let apply_output = match index_applier + .apply_coarse( + self.file_handle.file_id(), + Some(file_size_hint), + parquet_meta + .row_groups() + .iter() + .enumerate() + .map(|(i, rg)| (rg.num_rows() as usize, output.contains_key(&i))), + ) + .await + { + Ok(Some(apply_output)) => apply_output, + Ok(None) => return false, + Err(err) => { + if cfg!(any(test, feature = "test")) { + panic!( + "Failed to apply fulltext index, region_id: {}, file_id: {}, err: {:?}", + self.file_handle.region_id(), + self.file_handle.file_id(), + err + ); + } else { + warn!( + err; "Failed to apply fulltext index, region_id: {}, file_id: {}", + self.file_handle.region_id(), self.file_handle.file_id() + ); + } + + return false; + } + }; + + Self::prune_row_groups_by_ranges( + parquet_meta, + apply_output + .into_iter() + .map(|(rg, ranges)| (rg, ranges.into_iter())), + output, + &mut metrics.rg_fulltext_filtered, + &mut metrics.rows_fulltext_filtered, + ); + + true + } + /// Prunes row groups by rows. The `rows_in_row_groups` is like a map from row group to /// a list of row ids to keep. fn prune_row_groups_by_rows( From 7ddd7a988855d759c69ab6b6a375556ec162cb46 Mon Sep 17 00:00:00 2001 From: yihong Date: Mon, 14 Apr 2025 15:13:40 +0800 Subject: [PATCH 14/82] fix: flaky test on windows (#5890) Signed-off-by: yihong0618 --- src/common/error/tests/ext.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/common/error/tests/ext.rs b/src/common/error/tests/ext.rs index 0a39ed51c6..26eaf2d5e2 100644 --- a/src/common/error/tests/ext.rs +++ b/src/common/error/tests/ext.rs @@ -87,15 +87,19 @@ fn test_to_string() { #[test] fn test_debug_format() { let result = normal_error(); + let debug_output = format!("{:?}", result.unwrap_err()); + let normalized_output = debug_output.replace('\\', "/"); assert_eq!( - format!("{:?}", result.unwrap_err()), + normalized_output, r#"0: A normal error with "display" attribute, message "blabla", at src/common/error/tests/ext.rs:55:22 1: PlainError { msg: "", status_code: Unexpected }"# ); let result = transparent_error(); + let debug_output = format!("{:?}", result.unwrap_err()); + let normalized_output = debug_output.replace('\\', "/"); assert_eq!( - format!("{:?}", result.unwrap_err()), + normalized_output, r#"0: , at src/common/error/tests/ext.rs:60:5 1: PlainError { msg: "", status_code: Unexpected }"# ); From c522893552e8660e383a04d4d5919e9dbb876fd0 Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Mon, 14 Apr 2025 20:37:31 +0800 Subject: [PATCH 15/82] fix: ensure logical regions are synced during region sync (#5878) * fix: ensure logical regions are synced during region sync * chore: apply suggestions from CR * chore: apply suggestions from CR --- src/datanode/src/region_server.rs | 33 ++- src/datanode/src/tests.rs | 4 +- src/file-engine/src/engine.rs | 6 +- src/metric-engine/src/engine.rs | 42 +-- src/metric-engine/src/engine/alter.rs | 2 +- src/metric-engine/src/engine/create.rs | 13 +- src/metric-engine/src/engine/open.rs | 20 +- .../src/engine/region_metadata.rs | 2 +- src/metric-engine/src/engine/sync.rs | 261 ++++++++++++++++++ src/metric-engine/src/error.rs | 11 +- src/metric-engine/src/metadata_region.rs | 19 +- src/metric-engine/src/test_util.rs | 31 ++- src/mito2/src/engine.rs | 14 +- src/mito2/src/manifest/manager.rs | 7 +- src/mito2/src/manifest/storage.rs | 10 +- src/mito2/src/request.rs | 5 +- src/mito2/src/worker/handle_manifest.rs | 4 +- src/query/src/optimizer/test_util.rs | 4 +- src/store-api/src/region_engine.rs | 58 +++- 19 files changed, 470 insertions(+), 76 deletions(-) create mode 100644 src/metric-engine/src/engine/sync.rs diff --git a/src/datanode/src/region_server.rs b/src/datanode/src/region_server.rs index c05291050c..14ccfa2816 100644 --- a/src/datanode/src/region_server.rs +++ b/src/datanode/src/region_server.rs @@ -308,20 +308,36 @@ impl RegionServer { .with_context(|_| HandleRegionRequestSnafu { region_id }) } + /// Sync region manifest and registers new opened logical regions. pub async fn sync_region_manifest( &self, region_id: RegionId, manifest_info: RegionManifestInfo, ) -> Result<()> { - let engine = self + let engine_with_status = self .inner .region_map .get(®ion_id) .with_context(|| RegionNotFoundSnafu { region_id })?; - engine + + let Some(new_opened_regions) = engine_with_status .sync_region(region_id, manifest_info) .await - .with_context(|_| HandleRegionRequestSnafu { region_id }) + .with_context(|_| HandleRegionRequestSnafu { region_id })? + .new_opened_logical_region_ids() + else { + return Ok(()); + }; + + for region in new_opened_regions { + self.inner.region_map.insert( + region, + RegionEngineWithStatus::Ready(engine_with_status.engine().clone()), + ); + info!("Logical region {} is registered!", region); + } + + Ok(()) } /// Set region role state gracefully. @@ -526,6 +542,15 @@ impl RegionEngineWithStatus { RegionEngineWithStatus::Ready(engine) => engine, } } + + /// Returns [RegionEngineRef] reference. + pub fn engine(&self) -> &RegionEngineRef { + match self { + RegionEngineWithStatus::Registering(engine) => engine, + RegionEngineWithStatus::Deregistering(engine) => engine, + RegionEngineWithStatus::Ready(engine) => engine, + } + } } impl Deref for RegionEngineWithStatus { @@ -1029,7 +1054,7 @@ impl RegionServerInner { for region in logical_regions { self.region_map .insert(region, RegionEngineWithStatus::Ready(engine.clone())); - debug!("Logical region {} is registered!", region); + info!("Logical region {} is registered!", region); } Ok(()) } diff --git a/src/datanode/src/tests.rs b/src/datanode/src/tests.rs index b349024cc9..f182e1c423 100644 --- a/src/datanode/src/tests.rs +++ b/src/datanode/src/tests.rs @@ -33,7 +33,7 @@ use session::context::QueryContextRef; use store_api::metadata::RegionMetadataRef; use store_api::region_engine::{ RegionEngine, RegionManifestInfo, RegionRole, RegionScannerRef, RegionStatistic, - SetRegionRoleStateResponse, SettableRegionRoleState, + SetRegionRoleStateResponse, SettableRegionRoleState, SyncManifestResponse, }; use store_api::region_request::{AffectedRows, RegionRequest}; use store_api::storage::{RegionId, ScanRequest, SequenceNumber}; @@ -250,7 +250,7 @@ impl RegionEngine for MockRegionEngine { &self, _region_id: RegionId, _manifest_info: RegionManifestInfo, - ) -> Result<(), BoxedError> { + ) -> Result { unimplemented!() } diff --git a/src/file-engine/src/engine.rs b/src/file-engine/src/engine.rs index cf5e5c7576..09a373caad 100644 --- a/src/file-engine/src/engine.rs +++ b/src/file-engine/src/engine.rs @@ -28,7 +28,7 @@ use store_api::metadata::RegionMetadataRef; use store_api::region_engine::{ RegionEngine, RegionManifestInfo, RegionRole, RegionScannerRef, RegionStatistic, SetRegionRoleStateResponse, SetRegionRoleStateSuccess, SettableRegionRoleState, - SinglePartitionScanner, + SinglePartitionScanner, SyncManifestResponse, }; use store_api::region_request::{ AffectedRows, RegionCloseRequest, RegionCreateRequest, RegionDropRequest, RegionOpenRequest, @@ -145,9 +145,9 @@ impl RegionEngine for FileRegionEngine { &self, _region_id: RegionId, _manifest_info: RegionManifestInfo, - ) -> Result<(), BoxedError> { + ) -> Result { // File engine doesn't need to sync region manifest. - Ok(()) + Ok(SyncManifestResponse::NotSupported) } fn role(&self, region_id: RegionId) -> Option { diff --git a/src/metric-engine/src/engine.rs b/src/metric-engine/src/engine.rs index 74978fda78..509438b4b2 100644 --- a/src/metric-engine/src/engine.rs +++ b/src/metric-engine/src/engine.rs @@ -24,6 +24,7 @@ mod put; mod read; mod region_metadata; mod state; +mod sync; use std::any::Any; use std::collections::HashMap; @@ -41,6 +42,7 @@ use store_api::metric_engine_consts::METRIC_ENGINE_NAME; use store_api::region_engine::{ RegionEngine, RegionManifestInfo, RegionRole, RegionScannerRef, RegionStatistic, SetRegionRoleStateResponse, SetRegionRoleStateSuccess, SettableRegionRoleState, + SyncManifestResponse, }; use store_api::region_request::{BatchRegionDdlRequest, RegionRequest}; use store_api::storage::{RegionId, ScanRequest, SequenceNumber}; @@ -48,7 +50,7 @@ use store_api::storage::{RegionId, ScanRequest, SequenceNumber}; use self::state::MetricEngineState; use crate::config::EngineConfig; use crate::data_region::DataRegion; -use crate::error::{self, MetricManifestInfoSnafu, Result, UnsupportedRegionRequestSnafu}; +use crate::error::{self, Result, UnsupportedRegionRequestSnafu}; use crate::metadata_region::MetadataRegion; use crate::row_modifier::RowModifier; use crate::utils; @@ -311,40 +313,11 @@ impl RegionEngine for MetricEngine { &self, region_id: RegionId, manifest_info: RegionManifestInfo, - ) -> Result<(), BoxedError> { - if !manifest_info.is_metric() { - return Err(BoxedError::new( - MetricManifestInfoSnafu { region_id }.build(), - )); - } - - let metadata_region_id = utils::to_metadata_region_id(region_id); - // checked by ensure above - let metadata_manifest_version = manifest_info - .metadata_manifest_version() - .unwrap_or_default(); - let metadata_flushed_entry_id = manifest_info - .metadata_flushed_entry_id() - .unwrap_or_default(); - let metadata_region_manifest = - RegionManifestInfo::mito(metadata_manifest_version, metadata_flushed_entry_id); + ) -> Result { self.inner - .mito - .sync_region(metadata_region_id, metadata_region_manifest) - .await?; - - let data_region_id = utils::to_data_region_id(region_id); - let data_manifest_version = manifest_info.data_manifest_version(); - let data_flushed_entry_id = manifest_info.data_flushed_entry_id(); - let data_region_manifest = - RegionManifestInfo::mito(data_manifest_version, data_flushed_entry_id); - - self.inner - .mito - .sync_region(data_region_id, data_region_manifest) - .await?; - - Ok(()) + .sync_region(region_id, manifest_info) + .await + .map_err(BoxedError::new) } async fn set_region_role_state_gracefully( @@ -423,6 +396,7 @@ impl MetricEngine { self.inner.mito.clone() } + /// Returns all logical regions associated with the physical region. pub async fn logical_regions(&self, physical_region_id: RegionId) -> Result> { self.inner .metadata_region diff --git a/src/metric-engine/src/engine/alter.rs b/src/metric-engine/src/engine/alter.rs index 0b23a80bfd..1d82149a7d 100644 --- a/src/metric-engine/src/engine/alter.rs +++ b/src/metric-engine/src/engine/alter.rs @@ -145,7 +145,7 @@ impl MetricEngineInner { let _write_guard = self .metadata_region .write_lock_logical_region(*region_id) - .await; + .await?; write_guards.insert(*region_id, _write_guard); } diff --git a/src/metric-engine/src/engine/create.rs b/src/metric-engine/src/engine/create.rs index bfb7737df7..1ceb20d206 100644 --- a/src/metric-engine/src/engine/create.rs +++ b/src/metric-engine/src/engine/create.rs @@ -279,9 +279,16 @@ impl MetricEngineInner { .add_logical_regions(physical_region_id, true, logical_region_columns) .await?; - let mut state = self.state.write().unwrap(); - state.add_physical_columns(data_region_id, new_add_columns); - state.add_logical_regions(physical_region_id, logical_regions); + { + let mut state = self.state.write().unwrap(); + state.add_physical_columns(data_region_id, new_add_columns); + state.add_logical_regions(physical_region_id, logical_regions.clone()); + } + for logical_region_id in logical_regions { + self.metadata_region + .open_logical_region(logical_region_id) + .await; + } Ok(()) } diff --git a/src/metric-engine/src/engine/open.rs b/src/metric-engine/src/engine/open.rs index eb9f266be2..4b25cf38f2 100644 --- a/src/metric-engine/src/engine/open.rs +++ b/src/metric-engine/src/engine/open.rs @@ -132,12 +132,14 @@ impl MetricEngineInner { /// Includes: /// - Record physical region's column names /// - Record the mapping between logical region id and physical region id + /// + /// Returns new opened logical region ids. pub(crate) async fn recover_states( &self, physical_region_id: RegionId, primary_key_encoding: PrimaryKeyEncoding, physical_region_options: PhysicalRegionOptions, - ) -> Result<()> { + ) -> Result> { // load logical regions and physical column names let logical_regions = self .metadata_region @@ -147,7 +149,6 @@ impl MetricEngineInner { .data_region .physical_columns(physical_region_id) .await?; - let logical_region_num = logical_regions.len(); { let mut state = self.state.write().unwrap(); @@ -168,15 +169,22 @@ impl MetricEngineInner { } } + let mut opened_logical_region_ids = Vec::new(); + // The `recover_states` may be called multiple times, we only count the logical regions + // that are opened for the first time. for logical_region_id in logical_regions { - self.metadata_region + if self + .metadata_region .open_logical_region(logical_region_id) - .await; + .await + { + opened_logical_region_ids.push(logical_region_id); + } } - LOGICAL_REGION_COUNT.add(logical_region_num as i64); + LOGICAL_REGION_COUNT.add(opened_logical_region_ids.len() as i64); - Ok(()) + Ok(opened_logical_region_ids) } } diff --git a/src/metric-engine/src/engine/region_metadata.rs b/src/metric-engine/src/engine/region_metadata.rs index 9f00235e96..f8e0dd8dc3 100644 --- a/src/metric-engine/src/engine/region_metadata.rs +++ b/src/metric-engine/src/engine/region_metadata.rs @@ -46,7 +46,7 @@ impl MetricEngineInner { let _read_guard = self .metadata_region .read_lock_logical_region(logical_region_id) - .await; + .await?; // Load logical and physical columns, and intersect them to get logical column metadata. let logical_column_metadata = self .metadata_region diff --git a/src/metric-engine/src/engine/sync.rs b/src/metric-engine/src/engine/sync.rs new file mode 100644 index 0000000000..fe0d8ef6d0 --- /dev/null +++ b/src/metric-engine/src/engine/sync.rs @@ -0,0 +1,261 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::time::Instant; + +use common_telemetry::info; +use snafu::{ensure, OptionExt, ResultExt}; +use store_api::region_engine::{RegionEngine, RegionManifestInfo, SyncManifestResponse}; +use store_api::storage::RegionId; + +use crate::engine::MetricEngineInner; +use crate::error::{ + MetricManifestInfoSnafu, MitoSyncOperationSnafu, PhysicalRegionNotFoundSnafu, Result, +}; +use crate::utils; + +impl MetricEngineInner { + pub async fn sync_region( + &self, + region_id: RegionId, + manifest_info: RegionManifestInfo, + ) -> Result { + ensure!( + manifest_info.is_metric(), + MetricManifestInfoSnafu { region_id } + ); + + let metadata_region_id = utils::to_metadata_region_id(region_id); + // checked by ensure above + let metadata_manifest_version = manifest_info + .metadata_manifest_version() + .unwrap_or_default(); + let metadata_flushed_entry_id = manifest_info + .metadata_flushed_entry_id() + .unwrap_or_default(); + let metadata_region_manifest = + RegionManifestInfo::mito(metadata_manifest_version, metadata_flushed_entry_id); + let metadata_synced = self + .mito + .sync_region(metadata_region_id, metadata_region_manifest) + .await + .context(MitoSyncOperationSnafu)? + .is_data_synced(); + + let data_region_id = utils::to_data_region_id(region_id); + let data_manifest_version = manifest_info.data_manifest_version(); + let data_flushed_entry_id = manifest_info.data_flushed_entry_id(); + let data_region_manifest = + RegionManifestInfo::mito(data_manifest_version, data_flushed_entry_id); + + let data_synced = self + .mito + .sync_region(data_region_id, data_region_manifest) + .await + .context(MitoSyncOperationSnafu)? + .is_data_synced(); + + if !metadata_synced { + return Ok(SyncManifestResponse::Metric { + metadata_synced, + data_synced, + new_opened_logical_region_ids: vec![], + }); + } + + let now = Instant::now(); + // Recovers the states from the metadata region + // if the metadata manifest version is updated. + let physical_region_options = *self + .state + .read() + .unwrap() + .physical_region_states() + .get(&data_region_id) + .context(PhysicalRegionNotFoundSnafu { + region_id: data_region_id, + })? + .options(); + let primary_key_encoding = self.mito.get_primary_key_encoding(data_region_id).context( + PhysicalRegionNotFoundSnafu { + region_id: data_region_id, + }, + )?; + let new_opened_logical_region_ids = self + .recover_states( + data_region_id, + primary_key_encoding, + physical_region_options, + ) + .await?; + info!( + "Sync metadata region for physical region {}, cost: {:?}, new opened logical region ids: {:?}", + data_region_id, + now.elapsed(), + new_opened_logical_region_ids + ); + + Ok(SyncManifestResponse::Metric { + metadata_synced, + data_synced, + new_opened_logical_region_ids, + }) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use api::v1::SemanticType; + use common_telemetry::info; + use datatypes::data_type::ConcreteDataType; + use datatypes::schema::ColumnSchema; + use store_api::metadata::ColumnMetadata; + use store_api::region_engine::{RegionEngine, RegionManifestInfo}; + use store_api::region_request::{ + AddColumn, AlterKind, RegionAlterRequest, RegionFlushRequest, RegionRequest, + }; + use store_api::storage::RegionId; + + use crate::metadata_region::MetadataRegion; + use crate::test_util::TestEnv; + + #[tokio::test] + async fn test_sync_region_with_new_created_logical_regions() { + common_telemetry::init_default_ut_logging(); + let mut env = TestEnv::with_prefix("sync_with_new_created_logical_regions").await; + env.init_metric_region().await; + + info!("creating follower engine"); + // Create a follower engine. + let (_follower_mito, follower_metric) = env.create_follower_engine().await; + + let physical_region_id = env.default_physical_region_id(); + + // Flushes the physical region + let metric_engine = env.metric(); + metric_engine + .handle_request( + env.default_physical_region_id(), + RegionRequest::Flush(RegionFlushRequest::default()), + ) + .await + .unwrap(); + + let response = follower_metric + .sync_region(physical_region_id, RegionManifestInfo::metric(1, 0, 1, 0)) + .await + .unwrap(); + assert!(response.is_metric()); + let new_opened_logical_region_ids = response.new_opened_logical_region_ids().unwrap(); + assert_eq!(new_opened_logical_region_ids, vec![RegionId::new(3, 2)]); + + // Sync again, no new logical region should be opened + let response = follower_metric + .sync_region(physical_region_id, RegionManifestInfo::metric(1, 0, 1, 0)) + .await + .unwrap(); + assert!(response.is_metric()); + let new_opened_logical_region_ids = response.new_opened_logical_region_ids().unwrap(); + assert!(new_opened_logical_region_ids.is_empty()); + } + + fn test_alter_logical_region_request() -> RegionAlterRequest { + RegionAlterRequest { + kind: AlterKind::AddColumns { + columns: vec![AddColumn { + column_metadata: ColumnMetadata { + column_id: 0, + semantic_type: SemanticType::Tag, + column_schema: ColumnSchema::new( + "tag1", + ConcreteDataType::string_datatype(), + false, + ), + }, + location: None, + }], + }, + } + } + + #[tokio::test] + async fn test_sync_region_alter_alter_logical_region() { + common_telemetry::init_default_ut_logging(); + let mut env = TestEnv::with_prefix("sync_region_alter_alter_logical_region").await; + env.init_metric_region().await; + + info!("creating follower engine"); + let physical_region_id = env.default_physical_region_id(); + // Flushes the physical region + let metric_engine = env.metric(); + metric_engine + .handle_request( + env.default_physical_region_id(), + RegionRequest::Flush(RegionFlushRequest::default()), + ) + .await + .unwrap(); + + // Create a follower engine. + let (follower_mito, follower_metric) = env.create_follower_engine().await; + let metric_engine = env.metric(); + let engine_inner = env.metric().inner; + let region_id = env.default_logical_region_id(); + let request = test_alter_logical_region_request(); + + engine_inner + .alter_logical_regions( + physical_region_id, + vec![(region_id, request)], + &mut HashMap::new(), + ) + .await + .unwrap(); + + // Flushes the physical region + metric_engine + .handle_request( + env.default_physical_region_id(), + RegionRequest::Flush(RegionFlushRequest::default()), + ) + .await + .unwrap(); + + // Sync the follower engine + let response = follower_metric + .sync_region(physical_region_id, RegionManifestInfo::metric(2, 0, 2, 0)) + .await + .unwrap(); + assert!(response.is_metric()); + let new_opened_logical_region_ids = response.new_opened_logical_region_ids().unwrap(); + assert!(new_opened_logical_region_ids.is_empty()); + + let logical_region_id = env.default_logical_region_id(); + let metadata_region = MetadataRegion::new(follower_mito.clone()); + let semantic_type = metadata_region + .column_semantic_type(physical_region_id, logical_region_id, "tag1") + .await + .unwrap() + .unwrap(); + assert_eq!(semantic_type, SemanticType::Tag); + let timestamp_index = metadata_region + .column_semantic_type(physical_region_id, logical_region_id, "greptime_timestamp") + .await + .unwrap() + .unwrap(); + assert_eq!(timestamp_index, SemanticType::Timestamp); + } +} diff --git a/src/metric-engine/src/error.rs b/src/metric-engine/src/error.rs index 8be535ec9f..5f853037e1 100644 --- a/src/metric-engine/src/error.rs +++ b/src/metric-engine/src/error.rs @@ -118,6 +118,7 @@ pub enum Error { #[snafu(implicit)] location: Location, }, + #[snafu(display("Mito delete operation fails"))] MitoDeleteOperation { source: BoxedError, @@ -132,6 +133,13 @@ pub enum Error { location: Location, }, + #[snafu(display("Mito sync operation fails"))] + MitoSyncOperation { + source: BoxedError, + #[snafu(implicit)] + location: Location, + }, + #[snafu(display("Failed to collect record batch stream"))] CollectRecordBatchStream { source: common_recordbatch::error::Error, @@ -311,7 +319,8 @@ impl ErrorExt for Error { | MitoWriteOperation { source, .. } | MitoCatchupOperation { source, .. } | MitoFlushOperation { source, .. } - | MitoDeleteOperation { source, .. } => source.status_code(), + | MitoDeleteOperation { source, .. } + | MitoSyncOperation { source, .. } => source.status_code(), EncodePrimaryKey { source, .. } => source.status_code(), diff --git a/src/metric-engine/src/metadata_region.rs b/src/metric-engine/src/metadata_region.rs index 2b066a0bde..7e7bae095f 100644 --- a/src/metric-engine/src/metadata_region.rs +++ b/src/metric-engine/src/metadata_region.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::hash_map::Entry; use std::collections::HashMap; use std::sync::Arc; @@ -76,11 +77,22 @@ impl MetadataRegion { } } - pub async fn open_logical_region(&self, logical_region_id: RegionId) { - self.logical_region_lock + /// Open a logical region. + /// + /// Returns true if the logical region is opened for the first time. + pub async fn open_logical_region(&self, logical_region_id: RegionId) -> bool { + match self + .logical_region_lock .write() .await - .insert(logical_region_id, Arc::new(RwLock::new(()))); + .entry(logical_region_id) + { + Entry::Occupied(_) => false, + Entry::Vacant(vacant_entry) => { + vacant_entry.insert(Arc::new(RwLock::new(()))); + true + } + } } /// Retrieve a read lock guard of given logical region id. @@ -178,6 +190,7 @@ impl MetadataRegion { Ok(columns) } + /// Return all logical regions associated with the physical region. pub async fn logical_regions(&self, physical_region_id: RegionId) -> Result> { let metadata_region_id = utils::to_metadata_region_id(physical_region_id); diff --git a/src/metric-engine/src/test_util.rs b/src/metric-engine/src/test_util.rs index 284834a029..6bcc002908 100644 --- a/src/metric-engine/src/test_util.rs +++ b/src/metric-engine/src/test_util.rs @@ -16,6 +16,7 @@ use api::v1::value::ValueData; use api::v1::{ColumnDataType, ColumnSchema as PbColumnSchema, Row, SemanticType, Value}; +use common_telemetry::debug; use datatypes::prelude::ConcreteDataType; use datatypes::schema::ColumnSchema; use mito2::config::MitoConfig; @@ -28,7 +29,7 @@ use store_api::metric_engine_consts::{ }; use store_api::region_engine::RegionEngine; use store_api::region_request::{ - AddColumn, AlterKind, RegionAlterRequest, RegionCreateRequest, RegionRequest, + AddColumn, AlterKind, RegionAlterRequest, RegionCreateRequest, RegionOpenRequest, RegionRequest, }; use store_api::storage::{ColumnId, RegionId}; @@ -77,6 +78,34 @@ impl TestEnv { self.metric.clone() } + /// Creates a new follower engine with the same config as the leader engine. + pub async fn create_follower_engine(&mut self) -> (MitoEngine, MetricEngine) { + let mito = self + .mito_env + .create_follower_engine(MitoConfig::default()) + .await; + let metric = MetricEngine::new(mito.clone(), EngineConfig::default()); + + let region_id = self.default_physical_region_id(); + debug!("opening default physical region: {region_id}"); + let physical_region_option = [(PHYSICAL_TABLE_METADATA_KEY.to_string(), String::new())] + .into_iter() + .collect(); + metric + .handle_request( + region_id, + RegionRequest::Open(RegionOpenRequest { + engine: METRIC_ENGINE_NAME.to_string(), + region_dir: self.default_region_dir(), + options: physical_region_option, + skip_wal_replay: true, + }), + ) + .await + .unwrap(); + (mito, metric) + } + /// Create regions in [MetricEngine] under [`default_region_id`] /// and region dir `"test_metric_region"`. /// diff --git a/src/mito2/src/engine.rs b/src/mito2/src/engine.rs index 110f79b875..7b3c7352da 100644 --- a/src/mito2/src/engine.rs +++ b/src/mito2/src/engine.rs @@ -82,7 +82,7 @@ use store_api::manifest::ManifestVersion; use store_api::metadata::RegionMetadataRef; use store_api::region_engine::{ BatchResponses, RegionEngine, RegionManifestInfo, RegionRole, RegionScannerRef, - RegionStatistic, SetRegionRoleStateResponse, SettableRegionRoleState, + RegionStatistic, SetRegionRoleStateResponse, SettableRegionRoleState, SyncManifestResponse, }; use store_api::region_request::{AffectedRows, RegionOpenRequest, RegionRequest}; use store_api::storage::{RegionId, ScanRequest, SequenceNumber}; @@ -496,7 +496,7 @@ impl EngineInner { &self, region_id: RegionId, manifest_info: RegionManifestInfo, - ) -> Result { + ) -> Result<(ManifestVersion, bool)> { ensure!(manifest_info.is_mito(), MitoManifestInfoSnafu); let manifest_version = manifest_info.data_manifest_version(); let (request, receiver) = @@ -631,12 +631,14 @@ impl RegionEngine for MitoEngine { &self, region_id: RegionId, manifest_info: RegionManifestInfo, - ) -> Result<(), BoxedError> { - self.inner + ) -> Result { + let (_, synced) = self + .inner .sync_region(region_id, manifest_info) .await - .map_err(BoxedError::new) - .map(|_| ()) + .map_err(BoxedError::new)?; + + Ok(SyncManifestResponse::Mito { synced }) } fn role(&self, region_id: RegionId) -> Option { diff --git a/src/mito2/src/manifest/manager.rs b/src/mito2/src/manifest/manager.rs index d60f018b4e..2590d7ae6c 100644 --- a/src/mito2/src/manifest/manager.rs +++ b/src/mito2/src/manifest/manager.rs @@ -313,11 +313,12 @@ impl RegionManifestManager { } ); + let region_id = self.manifest.metadata.region_id; // Fetches manifests from the last version strictly. let mut manifests = self .store // Invariant: last_version < target_version. - .fetch_manifests_strict_from(last_version + 1, target_version + 1) + .fetch_manifests_strict_from(last_version + 1, target_version + 1, region_id) .await?; // Case 2: No manifests in range: [current_version+1, target_version+1) @@ -327,7 +328,7 @@ impl RegionManifestManager { // [Current Version]......[Target Version] // [Follower region] if manifests.is_empty() { - debug!( + info!( "Manifests are not strict from {}, region: {}, tries to install the last checkpoint", last_version, self.manifest.metadata.region_id ); @@ -341,7 +342,7 @@ impl RegionManifestManager { manifests = self .store // Invariant: last_version < target_version. - .fetch_manifests_strict_from(last_version + 1, target_version + 1) + .fetch_manifests_strict_from(last_version + 1, target_version + 1, region_id) .await?; } diff --git a/src/mito2/src/manifest/storage.rs b/src/mito2/src/manifest/storage.rs index c0ee01ba60..89e23e2cd4 100644 --- a/src/mito2/src/manifest/storage.rs +++ b/src/mito2/src/manifest/storage.rs @@ -29,6 +29,7 @@ use regex::Regex; use serde::{Deserialize, Serialize}; use snafu::{ensure, ResultExt}; use store_api::manifest::ManifestVersion; +use store_api::storage::RegionId; use tokio::sync::Semaphore; use crate::error::{ @@ -243,12 +244,17 @@ impl ManifestObjectStore { &self, start_version: ManifestVersion, end_version: ManifestVersion, + region_id: RegionId, ) -> Result)>> { let mut manifests = self.fetch_manifests(start_version, end_version).await?; let start_index = manifests.iter().position(|(v, _)| *v == start_version); debug!( - "fetches manifests in range [{},{}), start_index: {:?}", - start_version, end_version, start_index + "Fetches manifests in range [{},{}), start_index: {:?}, region_id: {}, manifests: {:?}", + start_version, + end_version, + start_index, + region_id, + manifests.iter().map(|(v, _)| *v).collect::>() ); if let Some(start_index) = start_index { Ok(manifests.split_off(start_index)) diff --git a/src/mito2/src/request.rs b/src/mito2/src/request.rs index 18ef260abe..33a8f13f07 100644 --- a/src/mito2/src/request.rs +++ b/src/mito2/src/request.rs @@ -692,7 +692,7 @@ impl WorkerRequest { pub(crate) fn new_sync_region_request( region_id: RegionId, manifest_version: ManifestVersion, - ) -> (WorkerRequest, Receiver>) { + ) -> (WorkerRequest, Receiver>) { let (sender, receiver) = oneshot::channel(); ( WorkerRequest::SyncRegion(RegionSyncRequest { @@ -892,7 +892,8 @@ pub(crate) struct RegionEditResult { pub(crate) struct RegionSyncRequest { pub(crate) region_id: RegionId, pub(crate) manifest_version: ManifestVersion, - pub(crate) sender: Sender>, + /// Returns the latest manifest version and a boolean indicating whether new maniefst is installed. + pub(crate) sender: Sender>, } #[cfg(test)] diff --git a/src/mito2/src/worker/handle_manifest.rs b/src/mito2/src/worker/handle_manifest.rs index f1bec95514..4fd0de0d7b 100644 --- a/src/mito2/src/worker/handle_manifest.rs +++ b/src/mito2/src/worker/handle_manifest.rs @@ -136,6 +136,7 @@ impl RegionWorkerLoop { } }; + let original_manifest_version = region.manifest_ctx.manifest_version().await; let manifest = match region .manifest_ctx .install_manifest_to(request.manifest_version) @@ -173,7 +174,8 @@ impl RegionWorkerLoop { .build(); region.version_control.overwrite_current(Arc::new(version)); - let _ = sender.send(Ok(manifest.manifest_version)); + let updated = manifest.manifest_version > original_manifest_version; + let _ = sender.send(Ok((manifest.manifest_version, updated))); } } diff --git a/src/query/src/optimizer/test_util.rs b/src/query/src/optimizer/test_util.rs index 72e4ad093a..2b3b473770 100644 --- a/src/query/src/optimizer/test_util.rs +++ b/src/query/src/optimizer/test_util.rs @@ -29,7 +29,7 @@ use store_api::metadata::{ }; use store_api::region_engine::{ RegionEngine, RegionManifestInfo, RegionRole, RegionScannerRef, RegionStatistic, - SetRegionRoleStateResponse, SettableRegionRoleState, + SetRegionRoleStateResponse, SettableRegionRoleState, SyncManifestResponse, }; use store_api::region_request::RegionRequest; use store_api::storage::{ConcreteDataType, RegionId, ScanRequest, SequenceNumber}; @@ -113,7 +113,7 @@ impl RegionEngine for MetaRegionEngine { &self, _region_id: RegionId, _manifest_info: RegionManifestInfo, - ) -> Result<(), BoxedError> { + ) -> Result { unimplemented!() } diff --git a/src/store-api/src/region_engine.rs b/src/store-api/src/region_engine.rs index 0a38700f1d..8522a2e1ca 100644 --- a/src/store-api/src/region_engine.rs +++ b/src/store-api/src/region_engine.rs @@ -583,6 +583,62 @@ impl RegionStatistic { } } +/// The response of syncing the manifest. +#[derive(Debug)] +pub enum SyncManifestResponse { + NotSupported, + Mito { + /// Indicates if the data region was synced. + synced: bool, + }, + Metric { + /// Indicates if the metadata region was synced. + metadata_synced: bool, + /// Indicates if the data region was synced. + data_synced: bool, + /// The logical regions that were newly opened during the sync operation. + /// This only occurs after the metadata region has been successfully synced. + new_opened_logical_region_ids: Vec, + }, +} + +impl SyncManifestResponse { + /// Returns true if data region is synced. + pub fn is_data_synced(&self) -> bool { + match self { + SyncManifestResponse::NotSupported => false, + SyncManifestResponse::Mito { synced } => *synced, + SyncManifestResponse::Metric { data_synced, .. } => *data_synced, + } + } + + /// Returns true if the engine is supported the sync operation. + pub fn is_supported(&self) -> bool { + matches!(self, SyncManifestResponse::NotSupported) + } + + /// Returns true if the engine is a mito2 engine. + pub fn is_mito(&self) -> bool { + matches!(self, SyncManifestResponse::Mito { .. }) + } + + /// Returns true if the engine is a metric engine. + pub fn is_metric(&self) -> bool { + matches!(self, SyncManifestResponse::Metric { .. }) + } + + /// Returns the new opened logical region ids. + pub fn new_opened_logical_region_ids(self) -> Option> { + match self { + SyncManifestResponse::Metric { + new_opened_logical_region_ids, + .. + } => Some(new_opened_logical_region_ids), + _ => None, + } + } +} + #[async_trait] pub trait RegionEngine: Send + Sync { /// Name of this engine @@ -689,7 +745,7 @@ pub trait RegionEngine: Send + Sync { &self, region_id: RegionId, manifest_info: RegionManifestInfo, - ) -> Result<(), BoxedError>; + ) -> Result; /// Sets region role state gracefully. /// From 747b71bf74b4ba5ead90b8a0278db16d9d63993e Mon Sep 17 00:00:00 2001 From: Ruihang Xia Date: Mon, 14 Apr 2025 21:12:37 +0800 Subject: [PATCH 16/82] feat: add query engine options (#5895) * feat: add query engine options Signed-off-by: Ruihang Xia * update example Signed-off-by: Ruihang Xia --------- Signed-off-by: Ruihang Xia --- config/config.md | 6 ++++ config/datanode.example.toml | 6 ++++ config/frontend.example.toml | 6 ++++ config/standalone.example.toml | 6 ++++ src/datanode/src/config.rs | 3 ++ src/datanode/src/datanode.rs | 1 + src/flow/src/adapter.rs | 3 ++ src/flow/src/server.rs | 1 + src/flow/src/test_utils.rs | 11 ++++++- src/flow/src/transform.rs | 11 ++++++- src/frontend/src/frontend.rs | 3 ++ src/frontend/src/instance/builder.rs | 1 + src/query/src/datafusion.rs | 12 +++++++- src/query/src/lib.rs | 1 + src/query/src/options.rs | 30 +++++++++++++++++++ src/query/src/query_engine.rs | 16 +++++++++- src/query/src/query_engine/context.rs | 2 ++ .../src/query_engine/default_serializer.rs | 11 ++++++- src/query/src/query_engine/state.rs | 6 ++++ src/query/src/range_select/plan_rewrite.rs | 12 +++++++- src/query/src/tests.rs | 12 +++++++- src/query/src/tests/query_engine_test.rs | 23 ++++++++++++-- src/query/src/tests/time_range_filter_test.rs | 13 ++++++-- src/servers/tests/mod.rs | 13 ++++++-- 24 files changed, 195 insertions(+), 14 deletions(-) create mode 100644 src/query/src/options.rs diff --git a/config/config.md b/config/config.md index ba2540f2c6..d0d7582db5 100644 --- a/config/config.md +++ b/config/config.md @@ -96,6 +96,8 @@ | `procedure.max_running_procedures` | Integer | `128` | Max running procedures.
The maximum number of procedures that can be running at the same time.
If the number of running procedures exceeds this limit, the procedure will be rejected. | | `flow` | -- | -- | flow engine options. | | `flow.num_workers` | Integer | `0` | The number of flow worker in flownode.
Not setting(or set to 0) this value will use the number of CPU cores divided by 2. | +| `query` | -- | -- | The query engine options. | +| `query.parallelism` | Integer | `0` | Parallelism of the query engine.
Default to 0, which means the number of CPU cores. | | `storage` | -- | -- | The data storage options. | | `storage.data_home` | String | `./greptimedb_data/` | The working home directory. | | `storage.type` | String | `File` | The storage type used to store the data.
- `File`: the data is stored in the local file system.
- `S3`: the data is stored in the S3 object storage.
- `Gcs`: the data is stored in the Google Cloud Storage.
- `Azblob`: the data is stored in the Azure Blob Storage.
- `Oss`: the data is stored in the Aliyun OSS. | @@ -270,6 +272,8 @@ | `meta_client.metadata_cache_max_capacity` | Integer | `100000` | The configuration about the cache of the metadata. | | `meta_client.metadata_cache_ttl` | String | `10m` | TTL of the metadata cache. | | `meta_client.metadata_cache_tti` | String | `5m` | -- | +| `query` | -- | -- | The query engine options. | +| `query.parallelism` | Integer | `0` | Parallelism of the query engine.
Default to 0, which means the number of CPU cores. | | `datanode` | -- | -- | Datanode options. | | `datanode.client` | -- | -- | Datanode client options. | | `datanode.client.connect_timeout` | String | `10s` | -- | @@ -429,6 +433,8 @@ | `wal.create_index` | Bool | `true` | Whether to enable WAL index creation.
**It's only used when the provider is `kafka`**. | | `wal.dump_index_interval` | String | `60s` | The interval for dumping WAL indexes.
**It's only used when the provider is `kafka`**. | | `wal.overwrite_entry_start_id` | Bool | `false` | Ignore missing entries during read WAL.
**It's only used when the provider is `kafka`**.

This option ensures that when Kafka messages are deleted, the system
can still successfully replay memtable data without throwing an
out-of-range error.
However, enabling this option might lead to unexpected data loss,
as the system will skip over missing entries instead of treating
them as critical errors. | +| `query` | -- | -- | The query engine options. | +| `query.parallelism` | Integer | `0` | Parallelism of the query engine.
Default to 0, which means the number of CPU cores. | | `storage` | -- | -- | The data storage options. | | `storage.data_home` | String | `./greptimedb_data/` | The working home directory. | | `storage.type` | String | `File` | The storage type used to store the data.
- `File`: the data is stored in the local file system.
- `S3`: the data is stored in the S3 object storage.
- `Gcs`: the data is stored in the Google Cloud Storage.
- `Azblob`: the data is stored in the Azure Blob Storage.
- `Oss`: the data is stored in the Aliyun OSS. | diff --git a/config/datanode.example.toml b/config/datanode.example.toml index af6b5571d2..46beb51a23 100644 --- a/config/datanode.example.toml +++ b/config/datanode.example.toml @@ -243,6 +243,12 @@ overwrite_entry_start_id = false # credential = "base64-credential" # endpoint = "https://storage.googleapis.com" +## The query engine options. +[query] +## Parallelism of the query engine. +## Default to 0, which means the number of CPU cores. +parallelism = 0 + ## The data storage options. [storage] ## The working home directory. diff --git a/config/frontend.example.toml b/config/frontend.example.toml index 3d4cd78144..2e3ee4a69d 100644 --- a/config/frontend.example.toml +++ b/config/frontend.example.toml @@ -179,6 +179,12 @@ metadata_cache_ttl = "10m" # TTI of the metadata cache. metadata_cache_tti = "5m" +## The query engine options. +[query] +## Parallelism of the query engine. +## Default to 0, which means the number of CPU cores. +parallelism = 0 + ## Datanode options. [datanode] ## Datanode client options. diff --git a/config/standalone.example.toml b/config/standalone.example.toml index bdef754712..0e72cfcc7e 100644 --- a/config/standalone.example.toml +++ b/config/standalone.example.toml @@ -334,6 +334,12 @@ max_running_procedures = 128 # credential = "base64-credential" # endpoint = "https://storage.googleapis.com" +## The query engine options. +[query] +## Parallelism of the query engine. +## Default to 0, which means the number of CPU cores. +parallelism = 0 + ## The data storage options. [storage] ## The working home directory. diff --git a/src/datanode/src/config.rs b/src/datanode/src/config.rs index 322f337ba3..7d63057a72 100644 --- a/src/datanode/src/config.rs +++ b/src/datanode/src/config.rs @@ -26,6 +26,7 @@ use file_engine::config::EngineConfig as FileEngineConfig; use meta_client::MetaClientOptions; use metric_engine::config::EngineConfig as MetricEngineConfig; use mito2::config::MitoConfig; +use query::options::QueryOptions; use serde::{Deserialize, Serialize}; use servers::export_metrics::ExportMetricsOption; use servers::grpc::GrpcOptions; @@ -375,6 +376,7 @@ pub struct DatanodeOptions { pub enable_telemetry: bool, pub export_metrics: ExportMetricsOption, pub tracing: TracingOptions, + pub query: QueryOptions, /// Deprecated options, please use the new options instead. #[deprecated(note = "Please use `grpc.addr` instead.")] @@ -412,6 +414,7 @@ impl Default for DatanodeOptions { enable_telemetry: true, export_metrics: ExportMetricsOption::default(), tracing: TracingOptions::default(), + query: QueryOptions::default(), // Deprecated options rpc_addr: None, diff --git a/src/datanode/src/datanode.rs b/src/datanode/src/datanode.rs index b32a1668c6..4b1e720032 100644 --- a/src/datanode/src/datanode.rs +++ b/src/datanode/src/datanode.rs @@ -359,6 +359,7 @@ impl DatanodeBuilder { None, false, self.plugins.clone(), + opts.query.clone(), ); let query_engine = query_engine_factory.query_engine(); diff --git a/src/flow/src/adapter.rs b/src/flow/src/adapter.rs index 1dd3e7e40e..8fd62ee2a0 100644 --- a/src/flow/src/adapter.rs +++ b/src/flow/src/adapter.rs @@ -32,6 +32,7 @@ use datatypes::value::Value; use greptime_proto::v1; use itertools::{EitherOrBoth, Itertools}; use meta_client::MetaClientOptions; +use query::options::QueryOptions; use query::QueryEngine; use serde::{Deserialize, Serialize}; use servers::grpc::GrpcOptions; @@ -109,6 +110,7 @@ pub struct FlownodeOptions { pub logging: LoggingOptions, pub tracing: TracingOptions, pub heartbeat: HeartbeatOptions, + pub query: QueryOptions, } impl Default for FlownodeOptions { @@ -122,6 +124,7 @@ impl Default for FlownodeOptions { logging: LoggingOptions::default(), tracing: TracingOptions::default(), heartbeat: HeartbeatOptions::default(), + query: QueryOptions::default(), } } } diff --git a/src/flow/src/server.rs b/src/flow/src/server.rs index f347ac369e..d0038e6ba1 100644 --- a/src/flow/src/server.rs +++ b/src/flow/src/server.rs @@ -332,6 +332,7 @@ impl FlownodeBuilder { None, false, Default::default(), + self.opts.query.clone(), ); let manager = Arc::new( self.build_manager(query_engine_factory.query_engine()) diff --git a/src/flow/src/test_utils.rs b/src/flow/src/test_utils.rs index 4d269a80c0..ecaabae32d 100644 --- a/src/flow/src/test_utils.rs +++ b/src/flow/src/test_utils.rs @@ -23,6 +23,7 @@ use datatypes::timestamp::TimestampMillisecond; use datatypes::vectors::{TimestampMillisecondVectorBuilder, VectorRef}; use itertools::Itertools; use prost::Message; +use query::options::QueryOptions; use query::parser::QueryLanguageParser; use query::query_engine::DefaultSerializer; use query::QueryEngine; @@ -146,7 +147,15 @@ pub fn create_test_query_engine() -> Arc { }; catalog_list.register_table_sync(req_with_ts).unwrap(); - let factory = query::QueryEngineFactory::new(catalog_list, None, None, None, None, false); + let factory = query::QueryEngineFactory::new( + catalog_list, + None, + None, + None, + None, + false, + QueryOptions::default(), + ); let engine = factory.query_engine(); register_function_to_query_engine(&engine); diff --git a/src/flow/src/transform.rs b/src/flow/src/transform.rs index 15da89b21f..04c7f40e68 100644 --- a/src/flow/src/transform.rs +++ b/src/flow/src/transform.rs @@ -171,6 +171,7 @@ mod test { use datatypes::vectors::{TimestampMillisecondVectorBuilder, VectorRef}; use itertools::Itertools; use prost::Message; + use query::options::QueryOptions; use query::parser::QueryLanguageParser; use query::query_engine::DefaultSerializer; use query::QueryEngine; @@ -263,7 +264,15 @@ mod test { }; catalog_list.register_table_sync(req_with_ts).unwrap(); - let factory = query::QueryEngineFactory::new(catalog_list, None, None, None, None, false); + let factory = query::QueryEngineFactory::new( + catalog_list, + None, + None, + None, + None, + false, + QueryOptions::default(), + ); let engine = factory.query_engine(); register_function_to_query_engine(&engine); diff --git a/src/frontend/src/frontend.rs b/src/frontend/src/frontend.rs index 983550d0e7..ba795730c4 100644 --- a/src/frontend/src/frontend.rs +++ b/src/frontend/src/frontend.rs @@ -19,6 +19,7 @@ use common_config::config::Configurable; use common_options::datanode::DatanodeClientOptions; use common_telemetry::logging::{LoggingOptions, TracingOptions}; use meta_client::MetaClientOptions; +use query::options::QueryOptions; use serde::{Deserialize, Serialize}; use servers::export_metrics::{ExportMetricsOption, ExportMetricsTask}; use servers::grpc::GrpcOptions; @@ -58,6 +59,7 @@ pub struct FrontendOptions { pub user_provider: Option, pub export_metrics: ExportMetricsOption, pub tracing: TracingOptions, + pub query: QueryOptions, pub max_in_flight_write_bytes: Option, } @@ -82,6 +84,7 @@ impl Default for FrontendOptions { user_provider: None, export_metrics: ExportMetricsOption::default(), tracing: TracingOptions::default(), + query: QueryOptions::default(), max_in_flight_write_bytes: None, } } diff --git a/src/frontend/src/instance/builder.rs b/src/frontend/src/instance/builder.rs index 8503999b2c..ffbfeabca1 100644 --- a/src/frontend/src/instance/builder.rs +++ b/src/frontend/src/instance/builder.rs @@ -166,6 +166,7 @@ impl FrontendBuilder { Some(Arc::new(flow_service)), true, plugins.clone(), + self.options.query.clone(), ) .query_engine(); diff --git a/src/query/src/datafusion.rs b/src/query/src/datafusion.rs index dba7d0215a..db4207fd8a 100644 --- a/src/query/src/datafusion.rs +++ b/src/query/src/datafusion.rs @@ -567,6 +567,7 @@ mod tests { use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME}; use super::*; + use crate::options::QueryOptions; use crate::parser::QueryLanguageParser; use crate::query_engine::{QueryEngineFactory, QueryEngineRef}; @@ -581,7 +582,16 @@ mod tests { }; catalog_manager.register_table_sync(req).unwrap(); - QueryEngineFactory::new(catalog_manager, None, None, None, None, false).query_engine() + QueryEngineFactory::new( + catalog_manager, + None, + None, + None, + None, + false, + QueryOptions::default(), + ) + .query_engine() } #[tokio::test] diff --git a/src/query/src/lib.rs b/src/query/src/lib.rs index 6e1fbfae0a..26fbfb27cd 100644 --- a/src/query/src/lib.rs +++ b/src/query/src/lib.rs @@ -29,6 +29,7 @@ pub mod executor; pub mod log_query; pub mod metrics; mod optimizer; +pub mod options; pub mod parser; mod part_sort; pub mod physical_wrapper; diff --git a/src/query/src/options.rs b/src/query/src/options.rs new file mode 100644 index 0000000000..441e9f161f --- /dev/null +++ b/src/query/src/options.rs @@ -0,0 +1,30 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use serde::{Deserialize, Serialize}; + +/// Query engine config +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(default)] +pub struct QueryOptions { + /// Parallelism of query engine. Default to 0, which implies the number of logical CPUs. + pub parallelism: usize, +} + +#[allow(clippy::derivable_impls)] +impl Default for QueryOptions { + fn default() -> Self { + Self { parallelism: 0 } + } +} diff --git a/src/query/src/query_engine.rs b/src/query/src/query_engine.rs index c4e8aee7d1..8b0c091054 100644 --- a/src/query/src/query_engine.rs +++ b/src/query/src/query_engine.rs @@ -38,6 +38,7 @@ use table::TableRef; use crate::dataframe::DataFrame; use crate::datafusion::DatafusionQueryEngine; use crate::error::Result; +use crate::options::QueryOptions; use crate::planner::LogicalPlanner; pub use crate::query_engine::context::QueryEngineContext; pub use crate::query_engine::state::QueryEngineState; @@ -106,6 +107,7 @@ impl QueryEngineFactory { procedure_service_handler: Option, flow_service_handler: Option, with_dist_planner: bool, + options: QueryOptions, ) -> Self { Self::new_with_plugins( catalog_manager, @@ -115,9 +117,11 @@ impl QueryEngineFactory { flow_service_handler, with_dist_planner, Default::default(), + options, ) } + #[allow(clippy::too_many_arguments)] pub fn new_with_plugins( catalog_manager: CatalogManagerRef, region_query_handler: Option, @@ -126,6 +130,7 @@ impl QueryEngineFactory { flow_service_handler: Option, with_dist_planner: bool, plugins: Plugins, + options: QueryOptions, ) -> Self { let state = Arc::new(QueryEngineState::new( catalog_manager, @@ -135,6 +140,7 @@ impl QueryEngineFactory { flow_service_handler, with_dist_planner, plugins.clone(), + options, )); let query_engine = Arc::new(DatafusionQueryEngine::new(state, plugins)); register_functions(&query_engine); @@ -166,7 +172,15 @@ mod tests { #[test] fn test_query_engine_factory() { let catalog_list = catalog::memory::new_memory_catalog_manager().unwrap(); - let factory = QueryEngineFactory::new(catalog_list, None, None, None, None, false); + let factory = QueryEngineFactory::new( + catalog_list, + None, + None, + None, + None, + false, + QueryOptions::default(), + ); let engine = factory.query_engine(); diff --git a/src/query/src/query_engine/context.rs b/src/query/src/query_engine/context.rs index d8c110d2f2..df20a70a42 100644 --- a/src/query/src/query_engine/context.rs +++ b/src/query/src/query_engine/context.rs @@ -75,6 +75,7 @@ impl QueryEngineContext { use common_base::Plugins; use session::context::QueryContext; + use crate::options::QueryOptions; use crate::query_engine::QueryEngineState; let state = Arc::new(QueryEngineState::new( @@ -85,6 +86,7 @@ impl QueryEngineContext { None, false, Plugins::default(), + QueryOptions::default(), )); QueryEngineContext::new(state.session_state(), QueryContext::arc()) diff --git a/src/query/src/query_engine/default_serializer.rs b/src/query/src/query_engine/default_serializer.rs index 23d6789866..c3feed1d55 100644 --- a/src/query/src/query_engine/default_serializer.rs +++ b/src/query/src/query_engine/default_serializer.rs @@ -159,6 +159,7 @@ mod tests { use super::*; use crate::dummy_catalog::DummyCatalogList; use crate::optimizer::test_util::mock_table_provider; + use crate::options::QueryOptions; use crate::QueryEngineFactory; fn mock_plan(schema: SchemaRef) -> LogicalPlan { @@ -177,7 +178,15 @@ mod tests { #[tokio::test] async fn test_serializer_decode_plan() { let catalog_list = catalog::memory::new_memory_catalog_manager().unwrap(); - let factory = QueryEngineFactory::new(catalog_list, None, None, None, None, false); + let factory = QueryEngineFactory::new( + catalog_list, + None, + None, + None, + None, + false, + QueryOptions::default(), + ); let engine = factory.query_engine(); diff --git a/src/query/src/query_engine/state.rs b/src/query/src/query_engine/state.rs index 812fc2c2af..75e1ed84a7 100644 --- a/src/query/src/query_engine/state.rs +++ b/src/query/src/query_engine/state.rs @@ -54,6 +54,7 @@ use crate::optimizer::string_normalization::StringNormalizationRule; use crate::optimizer::type_conversion::TypeConversionRule; use crate::optimizer::windowed_sort::WindowedSortPhysicalRule; use crate::optimizer::ExtensionAnalyzerRule; +use crate::options::QueryOptions as QueryOptionsNew; use crate::query_engine::options::QueryOptions; use crate::query_engine::DefaultSerializer; use crate::range_select::planner::RangeSelectPlanner; @@ -81,6 +82,7 @@ impl fmt::Debug for QueryEngineState { } impl QueryEngineState { + #[allow(clippy::too_many_arguments)] pub fn new( catalog_list: CatalogManagerRef, region_query_handler: Option, @@ -89,9 +91,13 @@ impl QueryEngineState { flow_service_handler: Option, with_dist_planner: bool, plugins: Plugins, + options: QueryOptionsNew, ) -> Self { let runtime_env = Arc::new(RuntimeEnv::default()); let mut session_config = SessionConfig::new().with_create_default_catalog_and_schema(false); + if options.parallelism > 0 { + session_config = session_config.with_target_partitions(options.parallelism); + } // todo(hl): This serves as a workaround for https://github.com/GreptimeTeam/greptimedb/issues/5659 // and we can add that check back once we upgrade datafusion. diff --git a/src/query/src/range_select/plan_rewrite.rs b/src/query/src/range_select/plan_rewrite.rs index b53e1079b8..5e0f223663 100644 --- a/src/query/src/range_select/plan_rewrite.rs +++ b/src/query/src/range_select/plan_rewrite.rs @@ -611,6 +611,7 @@ mod test { use table::test_util::EmptyTable; use super::*; + use crate::options::QueryOptions; use crate::parser::QueryLanguageParser; use crate::{QueryEngineFactory, QueryEngineRef}; @@ -663,7 +664,16 @@ mod test { table, }) .is_ok()); - QueryEngineFactory::new(catalog_list, None, None, None, None, false).query_engine() + QueryEngineFactory::new( + catalog_list, + None, + None, + None, + None, + false, + QueryOptions::default(), + ) + .query_engine() } async fn do_query(sql: &str) -> Result { diff --git a/src/query/src/tests.rs b/src/query/src/tests.rs index f2f2e40bf3..7c004e5229 100644 --- a/src/query/src/tests.rs +++ b/src/query/src/tests.rs @@ -18,6 +18,7 @@ use common_recordbatch::{util, RecordBatch}; use session::context::QueryContext; use table::TableRef; +use crate::options::QueryOptions; use crate::parser::QueryLanguageParser; use crate::{QueryEngineFactory, QueryEngineRef}; @@ -46,5 +47,14 @@ async fn exec_selection(engine: QueryEngineRef, sql: &str) -> Vec { pub fn new_query_engine_with_table(table: TableRef) -> QueryEngineRef { let catalog_manager = MemoryCatalogManager::new_with_table(table); - QueryEngineFactory::new(catalog_manager, None, None, None, None, false).query_engine() + QueryEngineFactory::new( + catalog_manager, + None, + None, + None, + None, + false, + QueryOptions::default(), + ) + .query_engine() } diff --git a/src/query/src/tests/query_engine_test.rs b/src/query/src/tests/query_engine_test.rs index 0f3f817703..07bac1363a 100644 --- a/src/query/src/tests/query_engine_test.rs +++ b/src/query/src/tests/query_engine_test.rs @@ -33,6 +33,7 @@ use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME}; use table::test_util::MemTable; use crate::error::{QueryExecutionSnafu, Result}; +use crate::options::QueryOptions as QueryOptionsNew; use crate::parser::QueryLanguageParser; use crate::query_engine::options::QueryOptions; use crate::query_engine::QueryEngineFactory; @@ -43,7 +44,15 @@ async fn test_datafusion_query_engine() -> Result<()> { let catalog_list = catalog::memory::new_memory_catalog_manager() .map_err(BoxedError::new) .context(QueryExecutionSnafu)?; - let factory = QueryEngineFactory::new(catalog_list, None, None, None, None, false); + let factory = QueryEngineFactory::new( + catalog_list, + None, + None, + None, + None, + false, + QueryOptionsNew::default(), + ); let engine = factory.query_engine(); let column_schemas = vec![ColumnSchema::new( @@ -122,8 +131,16 @@ async fn test_query_validate() -> Result<()> { disallow_cross_catalog_query: true, }); - let factory = - QueryEngineFactory::new_with_plugins(catalog_list, None, None, None, None, false, plugins); + let factory = QueryEngineFactory::new_with_plugins( + catalog_list, + None, + None, + None, + None, + false, + plugins, + QueryOptionsNew::default(), + ); let engine = factory.query_engine(); let stmt = diff --git a/src/query/src/tests/time_range_filter_test.rs b/src/query/src/tests/time_range_filter_test.rs index e141c99fa5..84bdd8cb18 100644 --- a/src/query/src/tests/time_range_filter_test.rs +++ b/src/query/src/tests/time_range_filter_test.rs @@ -33,6 +33,7 @@ use table::predicate::build_time_range_predicate; use table::test_util::MemTable; use table::{Table, TableRef}; +use crate::options::QueryOptions; use crate::tests::exec_selection; use crate::{QueryEngineFactory, QueryEngineRef}; @@ -102,8 +103,16 @@ fn create_test_engine() -> TimeRangeTester { }; let _ = catalog_manager.register_table_sync(req).unwrap(); - let engine = - QueryEngineFactory::new(catalog_manager, None, None, None, None, false).query_engine(); + let engine = QueryEngineFactory::new( + catalog_manager, + None, + None, + None, + None, + false, + QueryOptions::default(), + ) + .query_engine(); TimeRangeTester { engine, filter } } diff --git a/src/servers/tests/mod.rs b/src/servers/tests/mod.rs index aa07980240..13c78a293f 100644 --- a/src/servers/tests/mod.rs +++ b/src/servers/tests/mod.rs @@ -21,6 +21,7 @@ use catalog::memory::MemoryCatalogManager; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_query::Output; use datafusion_expr::LogicalPlan; +use query::options::QueryOptions; use query::parser::{PromQuery, QueryLanguageParser, QueryStatement}; use query::query_engine::DescribeResult; use query::{QueryEngineFactory, QueryEngineRef}; @@ -158,8 +159,16 @@ impl GrpcQueryHandler for DummyInstance { fn create_testing_instance(table: TableRef) -> DummyInstance { let catalog_manager = MemoryCatalogManager::new_with_table(table); - let query_engine = - QueryEngineFactory::new(catalog_manager, None, None, None, None, false).query_engine(); + let query_engine = QueryEngineFactory::new( + catalog_manager, + None, + None, + None, + None, + false, + QueryOptions::default(), + ) + .query_engine(); DummyInstance::new(query_engine) } From 6a50d719207088c9d77f634cac7b966057e5dc01 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" <6406592+v0y4g3r@users.noreply.github.com> Date: Mon, 14 Apr 2025 21:15:56 +0800 Subject: [PATCH 17/82] fix: memtable panic (#5894) * fix: memtable panic * fix: ci --- src/mito2/src/memtable/time_series.rs | 196 ++++++++++++++++++++++---- 1 file changed, 171 insertions(+), 25 deletions(-) diff --git a/src/mito2/src/memtable/time_series.rs b/src/mito2/src/memtable/time_series.rs index 82758a542b..44bce1ec74 100644 --- a/src/mito2/src/memtable/time_series.rs +++ b/src/mito2/src/memtable/time_series.rs @@ -161,18 +161,15 @@ impl TimeSeriesMemtable { let primary_key_encoded = self.row_codec.encode(kv.primary_keys())?; - let (series, series_allocated) = self.series_set.get_or_add_series(primary_key_encoded); - stats.key_bytes += series_allocated; + let (key_allocated, value_allocated) = + self.series_set.push_to_series(primary_key_encoded, &kv); + stats.key_bytes += key_allocated; + stats.value_bytes += value_allocated; // safety: timestamp of kv must be both present and a valid timestamp value. let ts = kv.timestamp().as_timestamp().unwrap().unwrap().value(); stats.min_ts = stats.min_ts.min(ts); stats.max_ts = stats.max_ts.max(ts); - - let mut guard = series.write().unwrap(); - let size = guard.push(kv.timestamp(), kv.sequence(), kv.op_type(), kv.fields()); - stats.value_bytes += size; - Ok(()) } } @@ -368,25 +365,46 @@ impl SeriesSet { } impl SeriesSet { - /// Returns the series for given primary key, or create a new series if not already exist, - /// along with the allocated memory footprint for primary keys. - fn get_or_add_series(&self, primary_key: Vec) -> (Arc>, usize) { + /// Push [KeyValue] to SeriesSet with given primary key and return key/value allocated memory size. + fn push_to_series(&self, primary_key: Vec, kv: &KeyValue) -> (usize, usize) { if let Some(series) = self.series.read().unwrap().get(&primary_key) { - return (series.clone(), 0); + let value_allocated = series.write().unwrap().push( + kv.timestamp(), + kv.sequence(), + kv.op_type(), + kv.fields(), + ); + return (0, value_allocated); }; - let s = Arc::new(RwLock::new(Series::new(&self.region_metadata))); + let mut indices = self.series.write().unwrap(); match indices.entry(primary_key) { Entry::Vacant(v) => { let key_len = v.key().len(); - v.insert(s.clone()); - (s, key_len) + let mut series = Series::new(&self.region_metadata); + let value_allocated = + series.push(kv.timestamp(), kv.sequence(), kv.op_type(), kv.fields()); + v.insert(Arc::new(RwLock::new(series))); + (key_len, value_allocated) } // safety: series must exist at given index. - Entry::Occupied(v) => (v.get().clone(), 0), + Entry::Occupied(v) => { + let value_allocated = v.get().write().unwrap().push( + kv.timestamp(), + kv.sequence(), + kv.op_type(), + kv.fields(), + ); + (0, value_allocated) + } } } + #[cfg(test)] + fn get_series(&self, primary_key: &[u8]) -> Option>> { + self.series.read().unwrap().get(primary_key).cloned() + } + /// Iterates all series in [SeriesSet]. fn iter_series( &self, @@ -948,7 +966,7 @@ mod tests { use api::helper::ColumnDataTypeWrapper; use api::v1::value::ValueData; - use api::v1::{Row, Rows, SemanticType}; + use api::v1::{Mutation, Row, Rows, SemanticType}; use common_time::Timestamp; use datatypes::prelude::{ConcreteDataType, ScalarVector}; use datatypes::schema::ColumnSchema; @@ -959,6 +977,7 @@ mod tests { use super::*; use crate::row_converter::SortField; + use crate::test_util::column_metadata_to_column_schema; fn schema_for_test() -> RegionMetadataRef { let mut builder = RegionMetadataBuilder::new(RegionId::new(123, 456)); @@ -1242,18 +1261,54 @@ mod tests { let mut handles = Vec::with_capacity(concurrency); for i in 0..concurrency { let set = set.clone(); + let schema = schema.clone(); + let column_schemas = schema + .column_metadatas + .iter() + .map(column_metadata_to_column_schema) + .collect::>(); let handle = std::thread::spawn(move || { for j in i * 100..(i + 1) * 100 { let pk = j % pk_num; let primary_key = format!("pk-{}", pk).as_bytes().to_vec(); - let (series, _) = set.get_or_add_series(primary_key); - let mut guard = series.write().unwrap(); - guard.push( - ts_value_ref(j as i64), - j as u64, - OpType::Put, - field_value_ref(j as i64, j as f64), - ); + + let kvs = KeyValues::new( + &schema, + Mutation { + op_type: OpType::Put as i32, + sequence: j as u64, + rows: Some(Rows { + schema: column_schemas.clone(), + rows: vec![Row { + values: vec![ + api::v1::Value { + value_data: Some(ValueData::StringValue(format!( + "{}", + j + ))), + }, + api::v1::Value { + value_data: Some(ValueData::I64Value(j as i64)), + }, + api::v1::Value { + value_data: Some(ValueData::TimestampMillisecondValue( + j as i64, + )), + }, + api::v1::Value { + value_data: Some(ValueData::I64Value(j as i64)), + }, + api::v1::Value { + value_data: Some(ValueData::F64Value(j as f64)), + }, + ], + }], + }), + write_hint: None, + }, + ) + .unwrap(); + set.push_to_series(primary_key, &kvs.iter().next().unwrap()); } }); handles.push(handle); @@ -1269,7 +1324,7 @@ mod tests { for i in 0..pk_num { let pk = format!("pk-{}", i).as_bytes().to_vec(); - let (series, _) = set.get_or_add_series(pk); + let series = set.get_series(&pk).unwrap(); let mut guard = series.write().unwrap(); let values = guard.compact(&schema).unwrap(); timestamps.extend(values.sequence.iter_data().map(|v| v.unwrap() as i64)); @@ -1385,4 +1440,95 @@ mod tests { } assert_eq!((0..100i64).collect::>(), v0_all); } + + #[test] + fn test_memtable_concurrent_write_read() { + common_telemetry::init_default_ut_logging(); + let schema = schema_for_test(); + let memtable = Arc::new(TimeSeriesMemtable::new( + schema.clone(), + 42, + None, + true, + MergeMode::LastRow, + )); + + // Number of writer threads + let num_writers = 10; + // Number of reader threads + let num_readers = 5; + // Number of series per writer + let series_per_writer = 100; + // Number of rows per series + let rows_per_series = 10; + // Total number of series + let total_series = num_writers * series_per_writer; + + // Create a barrier to synchronize the start of all threads + let barrier = Arc::new(std::sync::Barrier::new(num_writers + num_readers + 1)); + + // Spawn writer threads + let mut writer_handles = Vec::with_capacity(num_writers); + for writer_id in 0..num_writers { + let memtable = memtable.clone(); + let schema = schema.clone(); + let barrier = barrier.clone(); + + let handle = std::thread::spawn(move || { + // Wait for all threads to be ready + barrier.wait(); + + // Create and write series + for series_id in 0..series_per_writer { + let series_key = format!("writer-{}-series-{}", writer_id, series_id); + let kvs = + build_key_values(&schema, series_key, series_id as i64, rows_per_series); + memtable.write(&kvs).unwrap(); + } + }); + + writer_handles.push(handle); + } + + // Spawn reader threads + let mut reader_handles = Vec::with_capacity(num_readers); + for _ in 0..num_readers { + let memtable = memtable.clone(); + let barrier = barrier.clone(); + + let handle = std::thread::spawn(move || { + barrier.wait(); + + for _ in 0..10 { + let iter = memtable.iter(None, None, None).unwrap(); + for batch_result in iter { + let _ = batch_result.unwrap(); + } + } + }); + + reader_handles.push(handle); + } + + barrier.wait(); + + for handle in writer_handles { + handle.join().unwrap(); + } + for handle in reader_handles { + handle.join().unwrap(); + } + + let iter = memtable.iter(None, None, None).unwrap(); + let mut series_count = 0; + let mut row_count = 0; + + for batch_result in iter { + let batch = batch_result.unwrap(); + series_count += 1; + row_count += batch.num_rows(); + } + assert_eq!(total_series, series_count); + assert_eq!(total_series * rows_per_series, row_count); + } } From 8d485e9be0da767db281af758de4859a20ffab0e Mon Sep 17 00:00:00 2001 From: Zhenchi Date: Tue, 15 Apr 2025 14:36:06 +0800 Subject: [PATCH 18/82] feat: support altering fulltext backend (#5896) * feat: add `greptime_index_type` to `information_schema.key_column_usage` Signed-off-by: Zhenchi * fix: show create Signed-off-by: Zhenchi --------- Signed-off-by: Zhenchi --- .../information_schema/key_column_usage.rs | 69 +++++++-- .../information_schema/table_constraints.rs | 6 +- src/datatypes/src/schema/column_schema.rs | 2 +- src/query/src/sql.rs | 34 +--- src/query/src/sql/show_create_table.rs | 10 +- src/store-api/src/metadata.rs | 16 +- src/table/src/metadata.rs | 16 +- tests-integration/tests/http.rs | 2 +- .../alter/change_col_fulltext_options.result | 146 +++++++++++++----- .../alter/change_col_fulltext_options.sql | 14 ++ .../common/create/create_with_fulltext.result | 74 ++++----- .../standalone/common/show/show_create.result | 28 ++-- .../standalone/common/show/show_index.result | 36 ++--- .../common/system/information_schema.result | 32 ++-- 14 files changed, 294 insertions(+), 191 deletions(-) diff --git a/src/catalog/src/system_schema/information_schema/key_column_usage.rs b/src/catalog/src/system_schema/information_schema/key_column_usage.rs index 9f08839303..ffcd5eaaa5 100644 --- a/src/catalog/src/system_schema/information_schema/key_column_usage.rs +++ b/src/catalog/src/system_schema/information_schema/key_column_usage.rs @@ -24,7 +24,7 @@ use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatch use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream; use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream; use datatypes::prelude::{ConcreteDataType, MutableVector, ScalarVectorBuilder, VectorRef}; -use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; +use datatypes::schema::{ColumnSchema, FulltextBackend, Schema, SchemaRef}; use datatypes::value::Value; use datatypes::vectors::{ConstantVector, StringVector, StringVectorBuilder, UInt32VectorBuilder}; use futures_util::TryStreamExt; @@ -47,20 +47,38 @@ pub const TABLE_SCHEMA: &str = "table_schema"; pub const TABLE_NAME: &str = "table_name"; pub const COLUMN_NAME: &str = "column_name"; pub const ORDINAL_POSITION: &str = "ordinal_position"; +/// The type of the index. +pub const GREPTIME_INDEX_TYPE: &str = "greptime_index_type"; const INIT_CAPACITY: usize = 42; -/// Primary key constraint name -pub(crate) const PRI_CONSTRAINT_NAME: &str = "PRIMARY"; /// Time index constraint name -pub(crate) const TIME_INDEX_CONSTRAINT_NAME: &str = "TIME INDEX"; +pub(crate) const CONSTRAINT_NAME_TIME_INDEX: &str = "TIME INDEX"; + +/// Primary key constraint name +pub(crate) const CONSTRAINT_NAME_PRI: &str = "PRIMARY"; +/// Primary key index type +pub(crate) const INDEX_TYPE_PRI: &str = "greptime-primary-key-v1"; + /// Inverted index constraint name -pub(crate) const INVERTED_INDEX_CONSTRAINT_NAME: &str = "INVERTED INDEX"; +pub(crate) const CONSTRAINT_NAME_INVERTED_INDEX: &str = "INVERTED INDEX"; +/// Inverted index type +pub(crate) const INDEX_TYPE_INVERTED_INDEX: &str = "greptime-inverted-index-v1"; + /// Fulltext index constraint name -pub(crate) const FULLTEXT_INDEX_CONSTRAINT_NAME: &str = "FULLTEXT INDEX"; +pub(crate) const CONSTRAINT_NAME_FULLTEXT_INDEX: &str = "FULLTEXT INDEX"; +/// Fulltext index v1 type +pub(crate) const INDEX_TYPE_FULLTEXT_TANTIVY: &str = "greptime-fulltext-index-v1"; +/// Fulltext index bloom type +pub(crate) const INDEX_TYPE_FULLTEXT_BLOOM: &str = "greptime-fulltext-index-bloom"; + /// Skipping index constraint name -pub(crate) const SKIPPING_INDEX_CONSTRAINT_NAME: &str = "SKIPPING INDEX"; +pub(crate) const CONSTRAINT_NAME_SKIPPING_INDEX: &str = "SKIPPING INDEX"; +/// Skipping index type +pub(crate) const INDEX_TYPE_SKIPPING_INDEX: &str = "greptime-bloom-filter-v1"; /// The virtual table implementation for `information_schema.KEY_COLUMN_USAGE`. +/// +/// Provides an extra column `greptime_index_type` for the index type of the key column. #[derive(Debug)] pub(super) struct InformationSchemaKeyColumnUsage { schema: SchemaRef, @@ -120,6 +138,11 @@ impl InformationSchemaKeyColumnUsage { ConcreteDataType::string_datatype(), true, ), + ColumnSchema::new( + GREPTIME_INDEX_TYPE, + ConcreteDataType::string_datatype(), + true, + ), ])) } @@ -184,6 +207,7 @@ struct InformationSchemaKeyColumnUsageBuilder { column_name: StringVectorBuilder, ordinal_position: UInt32VectorBuilder, position_in_unique_constraint: UInt32VectorBuilder, + greptime_index_type: StringVectorBuilder, } impl InformationSchemaKeyColumnUsageBuilder { @@ -206,6 +230,7 @@ impl InformationSchemaKeyColumnUsageBuilder { column_name: StringVectorBuilder::with_capacity(INIT_CAPACITY), ordinal_position: UInt32VectorBuilder::with_capacity(INIT_CAPACITY), position_in_unique_constraint: UInt32VectorBuilder::with_capacity(INIT_CAPACITY), + greptime_index_type: StringVectorBuilder::with_capacity(INIT_CAPACITY), } } @@ -229,34 +254,47 @@ impl InformationSchemaKeyColumnUsageBuilder { for (idx, column) in schema.column_schemas().iter().enumerate() { let mut constraints = vec![]; + let mut greptime_index_type = vec![]; if column.is_time_index() { self.add_key_column_usage( &predicates, &schema_name, - TIME_INDEX_CONSTRAINT_NAME, + CONSTRAINT_NAME_TIME_INDEX, &catalog_name, &schema_name, table_name, &column.name, 1, //always 1 for time index + "", ); } // TODO(dimbtp): foreign key constraint not supported yet if keys.contains(&idx) { - constraints.push(PRI_CONSTRAINT_NAME); + constraints.push(CONSTRAINT_NAME_PRI); + greptime_index_type.push(INDEX_TYPE_PRI); } if column.is_inverted_indexed() { - constraints.push(INVERTED_INDEX_CONSTRAINT_NAME); + constraints.push(CONSTRAINT_NAME_INVERTED_INDEX); + greptime_index_type.push(INDEX_TYPE_INVERTED_INDEX); } - if column.is_fulltext_indexed() { - constraints.push(FULLTEXT_INDEX_CONSTRAINT_NAME); + if let Ok(Some(options)) = column.fulltext_options() { + if options.enable { + constraints.push(CONSTRAINT_NAME_FULLTEXT_INDEX); + let index_type = match options.backend { + FulltextBackend::Bloom => INDEX_TYPE_FULLTEXT_BLOOM, + FulltextBackend::Tantivy => INDEX_TYPE_FULLTEXT_TANTIVY, + }; + greptime_index_type.push(index_type); + } } if column.is_skipping_indexed() { - constraints.push(SKIPPING_INDEX_CONSTRAINT_NAME); + constraints.push(CONSTRAINT_NAME_SKIPPING_INDEX); + greptime_index_type.push(INDEX_TYPE_SKIPPING_INDEX); } if !constraints.is_empty() { let aggregated_constraints = constraints.join(", "); + let aggregated_index_types = greptime_index_type.join(", "); self.add_key_column_usage( &predicates, &schema_name, @@ -266,6 +304,7 @@ impl InformationSchemaKeyColumnUsageBuilder { table_name, &column.name, idx as u32 + 1, + &aggregated_index_types, ); } } @@ -288,6 +327,7 @@ impl InformationSchemaKeyColumnUsageBuilder { table_name: &str, column_name: &str, ordinal_position: u32, + index_types: &str, ) { let row = [ (CONSTRAINT_SCHEMA, &Value::from(constraint_schema)), @@ -297,6 +337,7 @@ impl InformationSchemaKeyColumnUsageBuilder { (TABLE_NAME, &Value::from(table_name)), (COLUMN_NAME, &Value::from(column_name)), (ORDINAL_POSITION, &Value::from(ordinal_position)), + (GREPTIME_INDEX_TYPE, &Value::from(index_types)), ]; if !predicates.eval(&row) { @@ -313,6 +354,7 @@ impl InformationSchemaKeyColumnUsageBuilder { self.column_name.push(Some(column_name)); self.ordinal_position.push(Some(ordinal_position)); self.position_in_unique_constraint.push(None); + self.greptime_index_type.push(Some(index_types)); } fn finish(&mut self) -> Result { @@ -336,6 +378,7 @@ impl InformationSchemaKeyColumnUsageBuilder { null_string_vector.clone(), null_string_vector.clone(), null_string_vector, + Arc::new(self.greptime_index_type.finish()), ]; RecordBatch::new(self.schema.clone(), columns).context(CreateRecordBatchSnafu) } diff --git a/src/catalog/src/system_schema/information_schema/table_constraints.rs b/src/catalog/src/system_schema/information_schema/table_constraints.rs index a1f9d899f4..77ac93632f 100644 --- a/src/catalog/src/system_schema/information_schema/table_constraints.rs +++ b/src/catalog/src/system_schema/information_schema/table_constraints.rs @@ -36,7 +36,7 @@ use crate::error::{ CreateRecordBatchSnafu, InternalSnafu, Result, UpgradeWeakCatalogManagerRefSnafu, }; use crate::information_schema::key_column_usage::{ - PRI_CONSTRAINT_NAME, TIME_INDEX_CONSTRAINT_NAME, + CONSTRAINT_NAME_PRI, CONSTRAINT_NAME_TIME_INDEX, }; use crate::information_schema::Predicates; use crate::system_schema::information_schema::{InformationTable, TABLE_CONSTRAINTS}; @@ -188,7 +188,7 @@ impl InformationSchemaTableConstraintsBuilder { self.add_table_constraint( &predicates, &schema_name, - TIME_INDEX_CONSTRAINT_NAME, + CONSTRAINT_NAME_TIME_INDEX, &schema_name, &table.table_info().name, TIME_INDEX_CONSTRAINT_TYPE, @@ -199,7 +199,7 @@ impl InformationSchemaTableConstraintsBuilder { self.add_table_constraint( &predicates, &schema_name, - PRI_CONSTRAINT_NAME, + CONSTRAINT_NAME_PRI, &schema_name, &table.table_info().name, PRI_KEY_CONSTRAINT_TYPE, diff --git a/src/datatypes/src/schema/column_schema.rs b/src/datatypes/src/schema/column_schema.rs index 9a975c4008..376c9e6de0 100644 --- a/src/datatypes/src/schema/column_schema.rs +++ b/src/datatypes/src/schema/column_schema.rs @@ -537,8 +537,8 @@ impl fmt::Display for FulltextOptions { #[serde(rename_all = "kebab-case")] pub enum FulltextBackend { #[default] + Bloom, Tantivy, - Bloom, // TODO(zhongzc): when bloom is ready, use it as default } impl fmt::Display for FulltextBackend { diff --git a/src/query/src/sql.rs b/src/query/src/sql.rs index b62289fb6b..8f823fe809 100644 --- a/src/query/src/sql.rs +++ b/src/query/src/sql.rs @@ -40,7 +40,7 @@ use common_recordbatch::RecordBatches; use common_time::timezone::get_timezone; use common_time::Timestamp; use datafusion::common::ScalarValue; -use datafusion::prelude::{concat_ws, SessionContext}; +use datafusion::prelude::SessionContext; use datafusion_expr::expr::WildcardOptions; use datafusion_expr::{case, col, lit, Expr, SortExpr}; use datatypes::prelude::*; @@ -399,23 +399,6 @@ pub async fn show_index( query_ctx.current_schema() }; - let primary_key_expr = case(col("constraint_name").like(lit("%PRIMARY%"))) - .when(lit(true), lit("greptime-primary-key-v1")) - .otherwise(null()) - .context(error::PlanSqlSnafu)?; - let inverted_index_expr = case(col("constraint_name").like(lit("%INVERTED INDEX%"))) - .when(lit(true), lit("greptime-inverted-index-v1")) - .otherwise(null()) - .context(error::PlanSqlSnafu)?; - let fulltext_index_expr = case(col("constraint_name").like(lit("%FULLTEXT INDEX%"))) - .when(lit(true), lit("greptime-fulltext-index-v1")) - .otherwise(null()) - .context(error::PlanSqlSnafu)?; - let skipping_index_expr = case(col("constraint_name").like(lit("%SKIPPING INDEX%"))) - .when(lit(true), lit("greptime-bloom-filter-v1")) - .otherwise(null()) - .context(error::PlanSqlSnafu)?; - let select = vec![ // 1 as `Non_unique`: contain duplicates lit(1).alias(INDEX_NONT_UNIQUE_COLUMN), @@ -433,16 +416,6 @@ pub async fn show_index( .otherwise(lit(YES_STR)) .context(error::PlanSqlSnafu)? .alias(COLUMN_NULLABLE_COLUMN), - concat_ws( - lit(", "), - vec![ - primary_key_expr, - inverted_index_expr, - fulltext_index_expr, - skipping_index_expr, - ], - ) - .alias(INDEX_INDEX_TYPE_COLUMN), lit("").alias(COLUMN_COMMENT_COLUMN), lit("").alias(INDEX_COMMENT_COLUMN), lit(YES_STR).alias(INDEX_VISIBLE_COLUMN), @@ -467,7 +440,10 @@ pub async fn show_index( (INDEX_SUB_PART_COLUMN, INDEX_SUB_PART_COLUMN), (INDEX_PACKED_COLUMN, INDEX_PACKED_COLUMN), (COLUMN_NULLABLE_COLUMN, COLUMN_NULLABLE_COLUMN), - (INDEX_INDEX_TYPE_COLUMN, INDEX_INDEX_TYPE_COLUMN), + ( + key_column_usage::GREPTIME_INDEX_TYPE, + INDEX_INDEX_TYPE_COLUMN, + ), (COLUMN_COMMENT_COLUMN, COLUMN_COMMENT_COLUMN), (INDEX_COMMENT_COLUMN, INDEX_COMMENT_COLUMN), (INDEX_VISIBLE_COLUMN, INDEX_VISIBLE_COLUMN), diff --git a/src/query/src/sql/show_create_table.rs b/src/query/src/sql/show_create_table.rs index 3eebfbc03e..bc004f514e 100644 --- a/src/query/src/sql/show_create_table.rs +++ b/src/query/src/sql/show_create_table.rs @@ -19,8 +19,8 @@ use std::collections::HashMap; use common_meta::SchemaOptions; use datatypes::schema::{ ColumnDefaultConstraint, ColumnSchema, SchemaRef, COLUMN_FULLTEXT_OPT_KEY_ANALYZER, - COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE, COLUMN_SKIPPING_INDEX_OPT_KEY_GRANULARITY, - COLUMN_SKIPPING_INDEX_OPT_KEY_TYPE, COMMENT_KEY, + COLUMN_FULLTEXT_OPT_KEY_BACKEND, COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE, + COLUMN_SKIPPING_INDEX_OPT_KEY_GRANULARITY, COLUMN_SKIPPING_INDEX_OPT_KEY_TYPE, COMMENT_KEY, }; use snafu::ResultExt; use sql::ast::{ColumnDef, ColumnOption, ColumnOptionDef, Expr, Ident, ObjectName}; @@ -113,6 +113,10 @@ fn create_column(column_schema: &ColumnSchema, quote_style: char) -> Result, ) -> Result<()> { if let Some(current_options) = current_options { - ensure!( - !current_options.enable, - InvalidColumnOptionSnafu { - column_name, - msg: "FULLTEXT index already enabled".to_string(), - } - ); - ensure!( current_options.analyzer == options.analyzer && current_options.case_sensitive == options.case_sensitive, diff --git a/src/table/src/metadata.rs b/src/table/src/metadata.rs index a457afe107..4235588ff0 100644 --- a/src/table/src/metadata.rs +++ b/src/table/src/metadata.rs @@ -1149,6 +1149,14 @@ impl TryFrom for TableInfo { } } +/// Set column fulltext options if it passed the validation. +/// +/// Options allowed to modify: +/// * backend +/// +/// Options not allowed to modify: +/// * analyzer +/// * case_sensitive fn set_column_fulltext_options( column_schema: &mut ColumnSchema, column_name: &str, @@ -1156,14 +1164,6 @@ fn set_column_fulltext_options( current_options: Option, ) -> Result<()> { if let Some(current_options) = current_options { - ensure!( - !current_options.enable, - error::InvalidColumnOptionSnafu { - column_name, - msg: "FULLTEXT index already enabled", - } - ); - ensure!( current_options.analyzer == options.analyzer && current_options.case_sensitive == options.case_sensitive, diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index ffb74e1b16..6eb4d10562 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -1371,7 +1371,7 @@ transform: assert_eq!(res.status(), StatusCode::OK); // 3. check schema - let expected_schema = "[[\"logs1\",\"CREATE TABLE IF NOT EXISTS \\\"logs1\\\" (\\n \\\"id1\\\" INT NULL INVERTED INDEX,\\n \\\"id2\\\" INT NULL INVERTED INDEX,\\n \\\"logger\\\" STRING NULL,\\n \\\"type\\\" STRING NULL SKIPPING INDEX WITH(granularity = '10240', type = 'BLOOM'),\\n \\\"log\\\" STRING NULL FULLTEXT INDEX WITH(analyzer = 'English', case_sensitive = 'false'),\\n \\\"time\\\" TIMESTAMP(9) NOT NULL,\\n TIME INDEX (\\\"time\\\"),\\n PRIMARY KEY (\\\"type\\\", \\\"log\\\")\\n)\\n\\nENGINE=mito\\nWITH(\\n append_mode = 'true'\\n)\"]]"; + let expected_schema = "[[\"logs1\",\"CREATE TABLE IF NOT EXISTS \\\"logs1\\\" (\\n \\\"id1\\\" INT NULL INVERTED INDEX,\\n \\\"id2\\\" INT NULL INVERTED INDEX,\\n \\\"logger\\\" STRING NULL,\\n \\\"type\\\" STRING NULL SKIPPING INDEX WITH(granularity = '10240', type = 'BLOOM'),\\n \\\"log\\\" STRING NULL FULLTEXT INDEX WITH(analyzer = 'English', backend = 'bloom', case_sensitive = 'false'),\\n \\\"time\\\" TIMESTAMP(9) NOT NULL,\\n TIME INDEX (\\\"time\\\"),\\n PRIMARY KEY (\\\"type\\\", \\\"log\\\")\\n)\\n\\nENGINE=mito\\nWITH(\\n append_mode = 'true'\\n)\"]]"; validate_data( "pipeline_schema", &client, diff --git a/tests/cases/standalone/common/alter/change_col_fulltext_options.result b/tests/cases/standalone/common/alter/change_col_fulltext_options.result index ee400593cc..13202ae12c 100644 --- a/tests/cases/standalone/common/alter/change_col_fulltext_options.result +++ b/tests/cases/standalone/common/alter/change_col_fulltext_options.result @@ -79,29 +79,29 @@ SELECT * FROM test WHERE MATCHES(message, 'hello') ORDER BY message; -- SQLNESS ARG restart=true SHOW CREATE TABLE test; -+-------+---------------------------------------------------------------------------------------------+ -| Table | Create Table | -+-------+---------------------------------------------------------------------------------------------+ -| test | CREATE TABLE IF NOT EXISTS "test" ( | -| | "message" STRING NULL FULLTEXT INDEX WITH(analyzer = 'Chinese', case_sensitive = 'true'), | -| | "time" TIMESTAMP(3) NOT NULL, | -| | TIME INDEX ("time") | -| | ) | -| | | -| | ENGINE=mito | -| | WITH( | -| | append_mode = 'true' | -| | ) | -+-------+---------------------------------------------------------------------------------------------+ ++-------+----------------------------------------------------------------------------------------------------------------+ +| Table | Create Table | ++-------+----------------------------------------------------------------------------------------------------------------+ +| test | CREATE TABLE IF NOT EXISTS "test" ( | +| | "message" STRING NULL FULLTEXT INDEX WITH(analyzer = 'Chinese', backend = 'bloom', case_sensitive = 'true'), | +| | "time" TIMESTAMP(3) NOT NULL, | +| | TIME INDEX ("time") | +| | ) | +| | | +| | ENGINE=mito | +| | WITH( | +| | append_mode = 'true' | +| | ) | ++-------+----------------------------------------------------------------------------------------------------------------+ SHOW INDEX FROM test; -+-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+----------------------------+---------+---------------+---------+------------+ -| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | -+-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+----------------------------+---------+---------------+---------+------------+ -| test | 1 | FULLTEXT INDEX | 1 | message | A | | | | YES | greptime-fulltext-index-v1 | | | YES | | -| test | 1 | TIME INDEX | 1 | time | A | | | | NO | | | | YES | | -+-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+----------------------------+---------+---------------+---------+------------+ ++-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+-------------------------------+---------+---------------+---------+------------+ +| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | ++-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+-------------------------------+---------+---------------+---------+------------+ +| test | 1 | FULLTEXT INDEX | 1 | message | A | | | | YES | greptime-fulltext-index-bloom | | | YES | | +| test | 1 | TIME INDEX | 1 | time | A | | | | NO | | | | YES | | ++-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+-------------------------------+---------+---------------+---------+------------+ ALTER TABLE test MODIFY COLUMN message UNSET FULLTEXT INDEX; @@ -138,33 +138,33 @@ Affected Rows: 0 SHOW CREATE TABLE test; -+-------+---------------------------------------------------------------------------------------------+ -| Table | Create Table | -+-------+---------------------------------------------------------------------------------------------+ -| test | CREATE TABLE IF NOT EXISTS "test" ( | -| | "message" STRING NULL FULLTEXT INDEX WITH(analyzer = 'Chinese', case_sensitive = 'true'), | -| | "time" TIMESTAMP(3) NOT NULL, | -| | TIME INDEX ("time") | -| | ) | -| | | -| | ENGINE=mito | -| | WITH( | -| | append_mode = 'true' | -| | ) | -+-------+---------------------------------------------------------------------------------------------+ ++-------+----------------------------------------------------------------------------------------------------------------+ +| Table | Create Table | ++-------+----------------------------------------------------------------------------------------------------------------+ +| test | CREATE TABLE IF NOT EXISTS "test" ( | +| | "message" STRING NULL FULLTEXT INDEX WITH(analyzer = 'Chinese', backend = 'bloom', case_sensitive = 'true'), | +| | "time" TIMESTAMP(3) NOT NULL, | +| | TIME INDEX ("time") | +| | ) | +| | | +| | ENGINE=mito | +| | WITH( | +| | append_mode = 'true' | +| | ) | ++-------+----------------------------------------------------------------------------------------------------------------+ SHOW INDEX FROM test; -+-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+----------------------------+---------+---------------+---------+------------+ -| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | -+-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+----------------------------+---------+---------------+---------+------------+ -| test | 1 | FULLTEXT INDEX | 1 | message | A | | | | YES | greptime-fulltext-index-v1 | | | YES | | -| test | 1 | TIME INDEX | 1 | time | A | | | | NO | | | | YES | | -+-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+----------------------------+---------+---------------+---------+------------+ ++-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+-------------------------------+---------+---------------+---------+------------+ +| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | ++-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+-------------------------------+---------+---------------+---------+------------+ +| test | 1 | FULLTEXT INDEX | 1 | message | A | | | | YES | greptime-fulltext-index-bloom | | | YES | | +| test | 1 | TIME INDEX | 1 | time | A | | | | NO | | | | YES | | ++-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+-------------------------------+---------+---------------+---------+------------+ ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'Chinese', case_sensitive = 'false'); -Error: 1004(InvalidArguments), Invalid column option, column name: message, error: FULLTEXT index already enabled +Error: 1004(InvalidArguments), Invalid column option, column name: message, error: Cannot change analyzer or case_sensitive if FULLTEXT index is set before. Previous analyzer: Chinese, previous case_sensitive: true ALTER TABLE test MODIFY COLUMN message UNSET FULLTEXT INDEX; @@ -195,6 +195,66 @@ SHOW INDEX FROM test; | test | 1 | TIME INDEX | 1 | time | A | | | | NO | | | | YES | | +-------+------------+------------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+---------+------------+ +ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'Chinese', case_sensitive = 'true', backend = 'bloom'); + +Affected Rows: 0 + +SHOW CREATE TABLE test; + ++-------+----------------------------------------------------------------------------------------------------------------+ +| Table | Create Table | ++-------+----------------------------------------------------------------------------------------------------------------+ +| test | CREATE TABLE IF NOT EXISTS "test" ( | +| | "message" STRING NULL FULLTEXT INDEX WITH(analyzer = 'Chinese', backend = 'bloom', case_sensitive = 'true'), | +| | "time" TIMESTAMP(3) NOT NULL, | +| | TIME INDEX ("time") | +| | ) | +| | | +| | ENGINE=mito | +| | WITH( | +| | append_mode = 'true' | +| | ) | ++-------+----------------------------------------------------------------------------------------------------------------+ + +SHOW INDEX FROM test; + ++-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+-------------------------------+---------+---------------+---------+------------+ +| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | ++-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+-------------------------------+---------+---------------+---------+------------+ +| test | 1 | FULLTEXT INDEX | 1 | message | A | | | | YES | greptime-fulltext-index-bloom | | | YES | | +| test | 1 | TIME INDEX | 1 | time | A | | | | NO | | | | YES | | ++-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+-------------------------------+---------+---------------+---------+------------+ + +ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'Chinese', case_sensitive = 'true', backend = 'tantivy'); + +Affected Rows: 0 + +SHOW CREATE TABLE test; + ++-------+------------------------------------------------------------------------------------------------------------------+ +| Table | Create Table | ++-------+------------------------------------------------------------------------------------------------------------------+ +| test | CREATE TABLE IF NOT EXISTS "test" ( | +| | "message" STRING NULL FULLTEXT INDEX WITH(analyzer = 'Chinese', backend = 'tantivy', case_sensitive = 'true'), | +| | "time" TIMESTAMP(3) NOT NULL, | +| | TIME INDEX ("time") | +| | ) | +| | | +| | ENGINE=mito | +| | WITH( | +| | append_mode = 'true' | +| | ) | ++-------+------------------------------------------------------------------------------------------------------------------+ + +SHOW INDEX FROM test; + ++-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+----------------------------+---------+---------------+---------+------------+ +| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | ++-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+----------------------------+---------+---------------+---------+------------+ +| test | 1 | FULLTEXT INDEX | 1 | message | A | | | | YES | greptime-fulltext-index-v1 | | | YES | | +| test | 1 | TIME INDEX | 1 | time | A | | | | NO | | | | YES | | ++-------+------------+----------------+--------------+-------------+-----------+-------------+----------+--------+------+----------------------------+---------+---------------+---------+------------+ + ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'Chinglish', case_sensitive = 'false'); Error: 1002(Unexpected), Invalid fulltext option: Chinglish, expected: 'English' | 'Chinese' @@ -211,6 +271,10 @@ ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'Engli Error: 1004(InvalidArguments), Invalid column option, column name: message, error: Cannot change analyzer or case_sensitive if FULLTEXT index is set before. Previous analyzer: Chinese, previous case_sensitive: true +ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(backend = 'xor'); + +Error: 1002(Unexpected), Invalid fulltext option: xor, expected: 'bloom' | 'tantivy' + DROP TABLE test; Affected Rows: 0 diff --git a/tests/cases/standalone/common/alter/change_col_fulltext_options.sql b/tests/cases/standalone/common/alter/change_col_fulltext_options.sql index b5ead6e610..df56e8179e 100644 --- a/tests/cases/standalone/common/alter/change_col_fulltext_options.sql +++ b/tests/cases/standalone/common/alter/change_col_fulltext_options.sql @@ -51,6 +51,18 @@ SHOW CREATE TABLE test; SHOW INDEX FROM test; +ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'Chinese', case_sensitive = 'true', backend = 'bloom'); + +SHOW CREATE TABLE test; + +SHOW INDEX FROM test; + +ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'Chinese', case_sensitive = 'true', backend = 'tantivy'); + +SHOW CREATE TABLE test; + +SHOW INDEX FROM test; + ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'Chinglish', case_sensitive = 'false'); ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'Chinese', case_sensitive = 'no'); @@ -59,4 +71,6 @@ ALTER TABLE test MODIFY COLUMN time SET FULLTEXT INDEX WITH(analyzer = 'Chinese' ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(analyzer = 'English', case_sensitive = 'true'); +ALTER TABLE test MODIFY COLUMN message SET FULLTEXT INDEX WITH(backend = 'xor'); + DROP TABLE test; diff --git a/tests/cases/standalone/common/create/create_with_fulltext.result b/tests/cases/standalone/common/create/create_with_fulltext.result index 3ab0435780..d5ae9ee2dd 100644 --- a/tests/cases/standalone/common/create/create_with_fulltext.result +++ b/tests/cases/standalone/common/create/create_with_fulltext.result @@ -7,18 +7,18 @@ Affected Rows: 0 SHOW CREATE TABLE log; -+-------+------------------------------------------------------------------------------------------+ -| Table | Create Table | -+-------+------------------------------------------------------------------------------------------+ -| log | CREATE TABLE IF NOT EXISTS "log" ( | -| | "ts" TIMESTAMP(3) NOT NULL, | -| | "msg" STRING NULL FULLTEXT INDEX WITH(analyzer = 'English', case_sensitive = 'false'), | -| | TIME INDEX ("ts") | -| | ) | -| | | -| | ENGINE=mito | -| | | -+-------+------------------------------------------------------------------------------------------+ ++-------+-------------------------------------------------------------------------------------------------------------+ +| Table | Create Table | ++-------+-------------------------------------------------------------------------------------------------------------+ +| log | CREATE TABLE IF NOT EXISTS "log" ( | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | "msg" STRING NULL FULLTEXT INDEX WITH(analyzer = 'English', backend = 'bloom', case_sensitive = 'false'), | +| | TIME INDEX ("ts") | +| | ) | +| | | +| | ENGINE=mito | +| | | ++-------+-------------------------------------------------------------------------------------------------------------+ DROP TABLE log; @@ -33,18 +33,18 @@ Affected Rows: 0 SHOW CREATE TABLE log_with_opts; -+---------------+-----------------------------------------------------------------------------------------+ -| Table | Create Table | -+---------------+-----------------------------------------------------------------------------------------+ -| log_with_opts | CREATE TABLE IF NOT EXISTS "log_with_opts" ( | -| | "ts" TIMESTAMP(3) NOT NULL, | -| | "msg" STRING NULL FULLTEXT INDEX WITH(analyzer = 'English', case_sensitive = 'true'), | -| | TIME INDEX ("ts") | -| | ) | -| | | -| | ENGINE=mito | -| | | -+---------------+-----------------------------------------------------------------------------------------+ ++---------------+------------------------------------------------------------------------------------------------------------+ +| Table | Create Table | ++---------------+------------------------------------------------------------------------------------------------------------+ +| log_with_opts | CREATE TABLE IF NOT EXISTS "log_with_opts" ( | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | "msg" STRING NULL FULLTEXT INDEX WITH(analyzer = 'English', backend = 'bloom', case_sensitive = 'true'), | +| | TIME INDEX ("ts") | +| | ) | +| | | +| | ENGINE=mito | +| | | ++---------------+------------------------------------------------------------------------------------------------------------+ DROP TABLE log_with_opts; @@ -60,19 +60,19 @@ Affected Rows: 0 SHOW CREATE TABLE log_multi_fulltext_cols; -+-------------------------+-------------------------------------------------------------------------------------------+ -| Table | Create Table | -+-------------------------+-------------------------------------------------------------------------------------------+ -| log_multi_fulltext_cols | CREATE TABLE IF NOT EXISTS "log_multi_fulltext_cols" ( | -| | "ts" TIMESTAMP(3) NOT NULL, | -| | "msg" STRING NULL FULLTEXT INDEX WITH(analyzer = 'English', case_sensitive = 'false'), | -| | "msg2" STRING NULL FULLTEXT INDEX WITH(analyzer = 'English', case_sensitive = 'false'), | -| | TIME INDEX ("ts") | -| | ) | -| | | -| | ENGINE=mito | -| | | -+-------------------------+-------------------------------------------------------------------------------------------+ ++-------------------------+--------------------------------------------------------------------------------------------------------------+ +| Table | Create Table | ++-------------------------+--------------------------------------------------------------------------------------------------------------+ +| log_multi_fulltext_cols | CREATE TABLE IF NOT EXISTS "log_multi_fulltext_cols" ( | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | "msg" STRING NULL FULLTEXT INDEX WITH(analyzer = 'English', backend = 'bloom', case_sensitive = 'false'), | +| | "msg2" STRING NULL FULLTEXT INDEX WITH(analyzer = 'English', backend = 'bloom', case_sensitive = 'false'), | +| | TIME INDEX ("ts") | +| | ) | +| | | +| | ENGINE=mito | +| | | ++-------------------------+--------------------------------------------------------------------------------------------------------------+ DROP TABLE log_multi_fulltext_cols; diff --git a/tests/cases/standalone/common/show/show_create.result b/tests/cases/standalone/common/show/show_create.result index ddbdd4179a..5d7019265a 100644 --- a/tests/cases/standalone/common/show/show_create.result +++ b/tests/cases/standalone/common/show/show_create.result @@ -373,20 +373,20 @@ Affected Rows: 0 show create table test_column_constrain_composite_indexes; -+-----------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Table | Create Table | -+-----------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| test_column_constrain_composite_indexes | CREATE TABLE IF NOT EXISTS "test_column_constrain_composite_indexes" ( | -| | "id" INT NULL SKIPPING INDEX WITH(granularity = '10240', type = 'BLOOM') INVERTED INDEX, | -| | "host" STRING NULL FULLTEXT INDEX WITH(analyzer = 'English', case_sensitive = 'false') SKIPPING INDEX WITH(granularity = '10240', type = 'BLOOM') INVERTED INDEX, | -| | "ts" TIMESTAMP(3) NOT NULL, | -| | TIME INDEX ("ts"), | -| | PRIMARY KEY ("host") | -| | ) | -| | | -| | ENGINE=mito | -| | | -+-----------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Table | Create Table | ++-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| test_column_constrain_composite_indexes | CREATE TABLE IF NOT EXISTS "test_column_constrain_composite_indexes" ( | +| | "id" INT NULL SKIPPING INDEX WITH(granularity = '10240', type = 'BLOOM') INVERTED INDEX, | +| | "host" STRING NULL FULLTEXT INDEX WITH(analyzer = 'English', backend = 'bloom', case_sensitive = 'false') SKIPPING INDEX WITH(granularity = '10240', type = 'BLOOM') INVERTED INDEX, | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | TIME INDEX ("ts"), | +| | PRIMARY KEY ("host") | +| | ) | +| | | +| | ENGINE=mito | +| | | ++-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ drop table test_column_constrain_composite_indexes; diff --git a/tests/cases/standalone/common/show/show_index.result b/tests/cases/standalone/common/show/show_index.result index 0376443746..80010b5331 100644 --- a/tests/cases/standalone/common/show/show_index.result +++ b/tests/cases/standalone/common/show/show_index.result @@ -80,27 +80,27 @@ SHOW INDEX FROM test_no_inverted_index; SHOW INDEX FROM system_metrics; -+----------------+------------+-----------------------------------------+--------------+-------------+-----------+-------------+----------+--------+------+---------------------------------------------------------------------------------+---------+---------------+---------+------------+ -| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | -+----------------+------------+-----------------------------------------+--------------+-------------+-----------+-------------+----------+--------+------+---------------------------------------------------------------------------------+---------+---------------+---------+------------+ -| system_metrics | 1 | FULLTEXT INDEX | 7 | desc2 | A | | | | YES | greptime-fulltext-index-v1 | | | YES | | -| system_metrics | 1 | FULLTEXT INDEX | 8 | desc3 | A | | | | YES | greptime-fulltext-index-v1 | | | YES | | -| system_metrics | 1 | PRIMARY | 1 | host | A | | | | YES | greptime-primary-key-v1 | | | YES | | -| system_metrics | 1 | PRIMARY, INVERTED INDEX, FULLTEXT INDEX | 2 | idc | A | | | | YES | greptime-primary-key-v1, greptime-inverted-index-v1, greptime-fulltext-index-v1 | | | YES | | -| system_metrics | 1 | TIME INDEX | 1 | ts | A | | | | NO | | | | YES | | -+----------------+------------+-----------------------------------------+--------------+-------------+-----------+-------------+----------+--------+------+---------------------------------------------------------------------------------+---------+---------------+---------+------------+ ++----------------+------------+-----------------------------------------+--------------+-------------+-----------+-------------+----------+--------+------+------------------------------------------------------------------------------------+---------+---------------+---------+------------+ +| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | ++----------------+------------+-----------------------------------------+--------------+-------------+-----------+-------------+----------+--------+------+------------------------------------------------------------------------------------+---------+---------------+---------+------------+ +| system_metrics | 1 | FULLTEXT INDEX | 7 | desc2 | A | | | | YES | greptime-fulltext-index-bloom | | | YES | | +| system_metrics | 1 | FULLTEXT INDEX | 8 | desc3 | A | | | | YES | greptime-fulltext-index-bloom | | | YES | | +| system_metrics | 1 | PRIMARY | 1 | host | A | | | | YES | greptime-primary-key-v1 | | | YES | | +| system_metrics | 1 | PRIMARY, INVERTED INDEX, FULLTEXT INDEX | 2 | idc | A | | | | YES | greptime-primary-key-v1, greptime-inverted-index-v1, greptime-fulltext-index-bloom | | | YES | | +| system_metrics | 1 | TIME INDEX | 1 | ts | A | | | | NO | | | | YES | | ++----------------+------------+-----------------------------------------+--------------+-------------+-----------+-------------+----------+--------+------+------------------------------------------------------------------------------------+---------+---------------+---------+------------+ SHOW INDEX FROM system_metrics in public; -+----------------+------------+-----------------------------------------+--------------+-------------+-----------+-------------+----------+--------+------+---------------------------------------------------------------------------------+---------+---------------+---------+------------+ -| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | -+----------------+------------+-----------------------------------------+--------------+-------------+-----------+-------------+----------+--------+------+---------------------------------------------------------------------------------+---------+---------------+---------+------------+ -| system_metrics | 1 | FULLTEXT INDEX | 7 | desc2 | A | | | | YES | greptime-fulltext-index-v1 | | | YES | | -| system_metrics | 1 | FULLTEXT INDEX | 8 | desc3 | A | | | | YES | greptime-fulltext-index-v1 | | | YES | | -| system_metrics | 1 | PRIMARY | 1 | host | A | | | | YES | greptime-primary-key-v1 | | | YES | | -| system_metrics | 1 | PRIMARY, INVERTED INDEX, FULLTEXT INDEX | 2 | idc | A | | | | YES | greptime-primary-key-v1, greptime-inverted-index-v1, greptime-fulltext-index-v1 | | | YES | | -| system_metrics | 1 | TIME INDEX | 1 | ts | A | | | | NO | | | | YES | | -+----------------+------------+-----------------------------------------+--------------+-------------+-----------+-------------+----------+--------+------+---------------------------------------------------------------------------------+---------+---------------+---------+------------+ ++----------------+------------+-----------------------------------------+--------------+-------------+-----------+-------------+----------+--------+------+------------------------------------------------------------------------------------+---------+---------------+---------+------------+ +| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment | Visible | Expression | ++----------------+------------+-----------------------------------------+--------------+-------------+-----------+-------------+----------+--------+------+------------------------------------------------------------------------------------+---------+---------------+---------+------------+ +| system_metrics | 1 | FULLTEXT INDEX | 7 | desc2 | A | | | | YES | greptime-fulltext-index-bloom | | | YES | | +| system_metrics | 1 | FULLTEXT INDEX | 8 | desc3 | A | | | | YES | greptime-fulltext-index-bloom | | | YES | | +| system_metrics | 1 | PRIMARY | 1 | host | A | | | | YES | greptime-primary-key-v1 | | | YES | | +| system_metrics | 1 | PRIMARY, INVERTED INDEX, FULLTEXT INDEX | 2 | idc | A | | | | YES | greptime-primary-key-v1, greptime-inverted-index-v1, greptime-fulltext-index-bloom | | | YES | | +| system_metrics | 1 | TIME INDEX | 1 | ts | A | | | | NO | | | | YES | | ++----------------+------------+-----------------------------------------+--------------+-------------+-----------+-------------+----------+--------+------+------------------------------------------------------------------------------------+---------+---------------+---------+------------+ SHOW INDEX FROM system_metrics like '%util%'; diff --git a/tests/cases/standalone/common/system/information_schema.result b/tests/cases/standalone/common/system/information_schema.result index bfcf39b69e..a19e7944c0 100644 --- a/tests/cases/standalone/common/system/information_schema.result +++ b/tests/cases/standalone/common/system/information_schema.result @@ -208,6 +208,7 @@ select * from information_schema.columns order by table_schema, table_name, colu | greptime | information_schema | key_column_usage | constraint_catalog | 1 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | | greptime | information_schema | key_column_usage | constraint_name | 3 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | | greptime | information_schema | key_column_usage | constraint_schema | 2 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | +| greptime | information_schema | key_column_usage | greptime_index_type | 14 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | Yes | string | | | | greptime | information_schema | key_column_usage | ordinal_position | 9 | | | 10 | 0 | | | | | | select,insert | | UInt32 | int unsigned | FIELD | | No | int unsigned | | | | greptime | information_schema | key_column_usage | position_in_unique_constraint | 10 | | | 10 | 0 | | | | | | select,insert | | UInt32 | int unsigned | FIELD | | Yes | int unsigned | | | | greptime | information_schema | key_column_usage | real_table_catalog | 5 | 2147483647 | 2147483647 | | | | utf8 | utf8_bin | | | select,insert | | String | string | FIELD | | No | string | | | @@ -593,11 +594,11 @@ select * from KEY_COLUMN_USAGE where CONSTRAINT_NAME = 'TIME INDEX'; select * from KEY_COLUMN_USAGE where CONSTRAINT_NAME != 'TIME INDEX'; -+--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ -| constraint_catalog | constraint_schema | constraint_name | table_catalog | real_table_catalog | table_schema | table_name | column_name | ordinal_position | position_in_unique_constraint | referenced_table_schema | referenced_table_name | referenced_column_name | -+--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ -| def | public | PRIMARY | def | greptime | public | numbers | number | 1 | | | | | -+--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+-------------------------+ +| constraint_catalog | constraint_schema | constraint_name | table_catalog | real_table_catalog | table_schema | table_name | column_name | ordinal_position | position_in_unique_constraint | referenced_table_schema | referenced_table_name | referenced_column_name | greptime_index_type | ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+-------------------------+ +| def | public | PRIMARY | def | greptime | public | numbers | number | 1 | | | | | greptime-primary-key-v1 | ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+-------------------------+ select * from KEY_COLUMN_USAGE where CONSTRAINT_NAME LIKE '%INDEX'; @@ -606,11 +607,11 @@ select * from KEY_COLUMN_USAGE where CONSTRAINT_NAME LIKE '%INDEX'; select * from KEY_COLUMN_USAGE where CONSTRAINT_NAME NOT LIKE '%INDEX'; -+--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ -| constraint_catalog | constraint_schema | constraint_name | table_catalog | real_table_catalog | table_schema | table_name | column_name | ordinal_position | position_in_unique_constraint | referenced_table_schema | referenced_table_name | referenced_column_name | -+--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ -| def | public | PRIMARY | def | greptime | public | numbers | number | 1 | | | | | -+--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+-------------------------+ +| constraint_catalog | constraint_schema | constraint_name | table_catalog | real_table_catalog | table_schema | table_name | column_name | ordinal_position | position_in_unique_constraint | referenced_table_schema | referenced_table_name | referenced_column_name | greptime_index_type | ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+-------------------------+ +| def | public | PRIMARY | def | greptime | public | numbers | number | 1 | | | | | greptime-primary-key-v1 | ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+-------------------------+ select * from KEY_COLUMN_USAGE where CONSTRAINT_NAME == 'TIME INDEX' AND CONSTRAINT_SCHEMA != 'my_db'; @@ -688,15 +689,16 @@ desc table key_column_usage; | referenced_table_schema | String | | YES | | FIELD | | referenced_table_name | String | | YES | | FIELD | | referenced_column_name | String | | YES | | FIELD | +| greptime_index_type | String | | YES | | FIELD | +-------------------------------+--------+-----+------+---------+---------------+ select * from key_column_usage; -+--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ -| constraint_catalog | constraint_schema | constraint_name | table_catalog | real_table_catalog | table_schema | table_name | column_name | ordinal_position | position_in_unique_constraint | referenced_table_schema | referenced_table_name | referenced_column_name | -+--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ -| def | public | PRIMARY | def | greptime | public | numbers | number | 1 | | | | | -+--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+ ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+-------------------------+ +| constraint_catalog | constraint_schema | constraint_name | table_catalog | real_table_catalog | table_schema | table_name | column_name | ordinal_position | position_in_unique_constraint | referenced_table_schema | referenced_table_name | referenced_column_name | greptime_index_type | ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+-------------------------+ +| def | public | PRIMARY | def | greptime | public | numbers | number | 1 | | | | | greptime-primary-key-v1 | ++--------------------+-------------------+-----------------+---------------+--------------------+--------------+------------+-------------+------------------+-------------------------------+-------------------------+-----------------------+------------------------+-------------------------+ -- tables not implemented DESC TABLE COLUMN_PRIVILEGES; From 96fbce1797c3fd0ab166c4f14f25028ecb2b1630 Mon Sep 17 00:00:00 2001 From: Ruihang Xia Date: Tue, 15 Apr 2025 14:45:00 +0800 Subject: [PATCH 19/82] feat: report per-region metrics on region server (#5893) * feat: report per-region metrics on region server Signed-off-by: Ruihang Xia * rename Change to Ingest Signed-off-by: Ruihang Xia --------- Signed-off-by: Ruihang Xia --- src/datanode/src/metrics.rs | 12 +++++++-- src/datanode/src/region_server.rs | 45 ++++++++++++++++++------------- 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/src/datanode/src/metrics.rs b/src/datanode/src/metrics.rs index d11e8af9fe..12ac482826 100644 --- a/src/datanode/src/metrics.rs +++ b/src/datanode/src/metrics.rs @@ -20,13 +20,21 @@ pub const REGION_REQUEST_TYPE: &str = "datanode_region_request_type"; pub const REGION_ROLE: &str = "region_role"; pub const REGION_ID: &str = "region_id"; +pub const RESULT_TYPE: &str = "result"; lazy_static! { /// The elapsed time of handling a request in the region_server. pub static ref HANDLE_REGION_REQUEST_ELAPSED: HistogramVec = register_histogram_vec!( "greptime_datanode_handle_region_request_elapsed", "datanode handle region request elapsed", - &[REGION_REQUEST_TYPE] + &[REGION_ID, REGION_REQUEST_TYPE] + ) + .unwrap(); + /// The number of rows in region request received by region server, labeled with request type. + pub static ref REGION_CHANGED_ROW_COUNT: IntCounterVec = register_int_counter_vec!( + "greptime_datanode_region_changed_row_count", + "datanode region changed row count", + &[REGION_ID, REGION_REQUEST_TYPE] ) .unwrap(); /// The elapsed time since the last received heartbeat. @@ -64,7 +72,7 @@ lazy_static! { pub static ref HEARTBEAT_RECV_COUNT: IntCounterVec = register_int_counter_vec!( "greptime_datanode_heartbeat_recv_count", "datanode heartbeat received", - &["result"] + &[RESULT_TYPE] ) .unwrap(); } diff --git a/src/datanode/src/region_server.rs b/src/datanode/src/region_server.rs index 14ccfa2816..bff28c109b 100644 --- a/src/datanode/src/region_server.rs +++ b/src/datanode/src/region_server.rs @@ -690,18 +690,20 @@ impl RegionServerInner { }, None => return Ok(CurrentEngine::EarlyReturn(0)), }, - RegionChange::None | RegionChange::Catchup => match current_region_status { - Some(status) => match status.clone() { - RegionEngineWithStatus::Registering(_) => { - return error::RegionNotReadySnafu { region_id }.fail() - } - RegionEngineWithStatus::Deregistering(_) => { - return error::RegionNotFoundSnafu { region_id }.fail() - } - RegionEngineWithStatus::Ready(engine) => engine, - }, - None => return error::RegionNotFoundSnafu { region_id }.fail(), - }, + RegionChange::None | RegionChange::Catchup | RegionChange::Ingest => { + match current_region_status { + Some(status) => match status.clone() { + RegionEngineWithStatus::Registering(_) => { + return error::RegionNotReadySnafu { region_id }.fail() + } + RegionEngineWithStatus::Deregistering(_) => { + return error::RegionNotFoundSnafu { region_id }.fail() + } + RegionEngineWithStatus::Ready(engine) => engine, + }, + None => return error::RegionNotFoundSnafu { region_id }.fail(), + } + } }; Ok(CurrentEngine::Engine(engine)) @@ -885,8 +887,9 @@ impl RegionServerInner { request: RegionRequest, ) -> Result { let request_type = request.request_type(); + let region_id_str = region_id.to_string(); let _timer = crate::metrics::HANDLE_REGION_REQUEST_ELAPSED - .with_label_values(&[request_type]) + .with_label_values(&[®ion_id_str, request_type]) .start_timer(); let region_change = match &request { @@ -899,9 +902,8 @@ impl RegionServerInner { RegionChange::Register(attribute) } RegionRequest::Close(_) | RegionRequest::Drop(_) => RegionChange::Deregisters, - RegionRequest::Put(_) - | RegionRequest::Delete(_) - | RegionRequest::Alter(_) + RegionRequest::Put(_) | RegionRequest::Delete(_) => RegionChange::Ingest, + RegionRequest::Alter(_) | RegionRequest::Flush(_) | RegionRequest::Compact(_) | RegionRequest::Truncate(_) => RegionChange::None, @@ -922,6 +924,12 @@ impl RegionServerInner { .with_context(|_| HandleRegionRequestSnafu { region_id }) { Ok(result) => { + // Update metrics + if matches!(region_change, RegionChange::Ingest) { + crate::metrics::REGION_CHANGED_ROW_COUNT + .with_label_values(&[®ion_id_str, request_type]) + .inc_by(result.affected_rows as u64); + } // Sets corresponding region status to ready. self.set_region_status_ready(region_id, engine, region_change) .await?; @@ -968,7 +976,7 @@ impl RegionServerInner { region_change: RegionChange, ) { match region_change { - RegionChange::None => {} + RegionChange::None | RegionChange::Ingest => {} RegionChange::Register(_) => { self.region_map.remove(®ion_id); } @@ -988,7 +996,7 @@ impl RegionServerInner { ) -> Result<()> { let engine_type = engine.name(); match region_change { - RegionChange::None => {} + RegionChange::None | RegionChange::Ingest => {} RegionChange::Register(attribute) => { info!( "Region {region_id} is registered to engine {}", @@ -1129,6 +1137,7 @@ enum RegionChange { Register(RegionAttribute), Deregisters, Catchup, + Ingest, } fn is_metric_engine(engine: &str) -> bool { From 2189631efd54fa0ead07aaf9a4865b9ef371ba80 Mon Sep 17 00:00:00 2001 From: Zhenchi Date: Tue, 15 Apr 2025 14:45:56 +0800 Subject: [PATCH 20/82] feat: optimize `matches_term` with constant term pre-compilation (#5886) * feat: precompile finder for `matches_term` Signed-off-by: Zhenchi * fix sqlness Signed-off-by: Zhenchi --------- Signed-off-by: Zhenchi --- src/query/src/optimizer.rs | 1 + src/query/src/optimizer/constant_term.rs | 454 ++++++++++++++++++ src/query/src/query_engine/state.rs | 4 + .../common/tql-explain-analyze/explain.result | 1 + 4 files changed, 460 insertions(+) create mode 100644 src/query/src/optimizer/constant_term.rs diff --git a/src/query/src/optimizer.rs b/src/query/src/optimizer.rs index c98ad0c634..e6596e923a 100644 --- a/src/query/src/optimizer.rs +++ b/src/query/src/optimizer.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod constant_term; pub mod count_wildcard; pub mod parallelize_scan; pub mod pass_distribution; diff --git a/src/query/src/optimizer/constant_term.rs b/src/query/src/optimizer/constant_term.rs new file mode 100644 index 0000000000..60e5b76d9d --- /dev/null +++ b/src/query/src/optimizer/constant_term.rs @@ -0,0 +1,454 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fmt; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; + +use arrow::array::{AsArray, BooleanArray}; +use common_function::scalars::matches_term::MatchesTermFinder; +use datafusion::config::ConfigOptions; +use datafusion::error::Result as DfResult; +use datafusion::physical_optimizer::PhysicalOptimizerRule; +use datafusion::physical_plan::filter::FilterExec; +use datafusion::physical_plan::ExecutionPlan; +use datafusion_common::tree_node::{Transformed, TreeNode}; +use datafusion_common::ScalarValue; +use datafusion_expr::ColumnarValue; +use datafusion_physical_expr::expressions::Literal; +use datafusion_physical_expr::{PhysicalExpr, ScalarFunctionExpr}; + +/// A physical expression that uses a pre-compiled term finder for the `matches_term` function. +/// +/// This expression optimizes the `matches_term` function by pre-compiling the term +/// when the term is a constant value. This avoids recompiling the term for each row +/// during execution. +#[derive(Debug)] +pub struct PreCompiledMatchesTermExpr { + /// The text column expression to search in + text: Arc, + /// The constant term to search for + term: String, + /// The pre-compiled term finder + finder: MatchesTermFinder, +} + +impl fmt::Display for PreCompiledMatchesTermExpr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "MatchesConstTerm({}, \"{}\")", self.text, self.term) + } +} + +impl Hash for PreCompiledMatchesTermExpr { + fn hash(&self, state: &mut H) { + self.text.hash(state); + self.term.hash(state); + } +} + +impl PartialEq for PreCompiledMatchesTermExpr { + fn eq(&self, other: &Self) -> bool { + self.text.eq(&other.text) && self.term.eq(&other.term) + } +} + +impl Eq for PreCompiledMatchesTermExpr {} + +impl PhysicalExpr for PreCompiledMatchesTermExpr { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn data_type( + &self, + _input_schema: &arrow_schema::Schema, + ) -> datafusion::error::Result { + Ok(arrow_schema::DataType::Boolean) + } + + fn nullable(&self, input_schema: &arrow_schema::Schema) -> datafusion::error::Result { + self.text.nullable(input_schema) + } + + fn evaluate( + &self, + batch: &common_recordbatch::DfRecordBatch, + ) -> datafusion::error::Result { + let num_rows = batch.num_rows(); + + let text_value = self.text.evaluate(batch)?; + let array = text_value.into_array(num_rows)?; + let str_array = array.as_string::(); + + let mut result = BooleanArray::builder(num_rows); + for text in str_array { + match text { + Some(text) => { + result.append_value(self.finder.find(text)); + } + None => { + result.append_null(); + } + } + } + + Ok(ColumnarValue::Array(Arc::new(result.finish()))) + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.text] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> datafusion::error::Result> { + Ok(Arc::new(PreCompiledMatchesTermExpr { + text: children[0].clone(), + term: self.term.clone(), + finder: self.finder.clone(), + })) + } +} + +/// Optimizer rule that pre-compiles constant term in `matches_term` function. +/// +/// This optimizer looks for `matches_term` function calls where the second argument +/// (the term to match) is a constant value. When found, it replaces the function +/// call with a specialized `PreCompiledMatchesTermExpr` that uses a pre-compiled +/// term finder. +/// +/// Example: +/// ```sql +/// -- Before optimization: +/// matches_term(text_column, 'constant_term') +/// +/// -- After optimization: +/// PreCompiledMatchesTermExpr(text_column, 'constant_term') +/// ``` +/// +/// This optimization improves performance by: +/// 1. Pre-compiling the term once instead of for each row +/// 2. Using a specialized expression that avoids function call overhead +#[derive(Debug)] +pub struct MatchesConstantTermOptimizer; + +impl PhysicalOptimizerRule for MatchesConstantTermOptimizer { + fn optimize( + &self, + plan: Arc, + _config: &ConfigOptions, + ) -> DfResult> { + let res = plan + .transform_down(&|plan: Arc| { + if let Some(filter) = plan.as_any().downcast_ref::() { + let pred = filter.predicate().clone(); + let new_pred = pred.transform_down(&|expr: Arc| { + if let Some(func) = expr.as_any().downcast_ref::() { + if !func.name().eq_ignore_ascii_case("matches_term") { + return Ok(Transformed::no(expr)); + } + let args = func.args(); + if args.len() != 2 { + return Ok(Transformed::no(expr)); + } + + if let Some(lit) = args[1].as_any().downcast_ref::() { + if let ScalarValue::Utf8(Some(term)) = lit.value() { + let finder = MatchesTermFinder::new(term); + let expr = PreCompiledMatchesTermExpr { + text: args[0].clone(), + term: term.to_string(), + finder, + }; + + return Ok(Transformed::yes(Arc::new(expr))); + } + } + } + + Ok(Transformed::no(expr)) + })?; + + if new_pred.transformed { + let exec = FilterExec::try_new(new_pred.data, filter.input().clone())? + .with_default_selectivity(filter.default_selectivity())? + .with_projection(filter.projection().cloned())?; + return Ok(Transformed::yes(Arc::new(exec) as _)); + } + } + + Ok(Transformed::no(plan)) + })? + .data; + + Ok(res) + } + + fn name(&self) -> &str { + "MatchesConstantTerm" + } + + fn schema_check(&self) -> bool { + false + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow::array::{ArrayRef, StringArray}; + use arrow::datatypes::{DataType, Field, Schema}; + use arrow::record_batch::RecordBatch; + use catalog::memory::MemoryCatalogManager; + use catalog::RegisterTableRequest; + use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; + use common_function::scalars::matches_term::MatchesTermFunction; + use common_function::scalars::udf::create_udf; + use common_function::state::FunctionState; + use datafusion::physical_optimizer::PhysicalOptimizerRule; + use datafusion::physical_plan::filter::FilterExec; + use datafusion::physical_plan::get_plan_string; + use datafusion::physical_plan::memory::MemoryExec; + use datafusion_common::{Column, DFSchema, ScalarValue}; + use datafusion_expr::expr::ScalarFunction; + use datafusion_expr::{Expr, ScalarUDF}; + use datafusion_physical_expr::{create_physical_expr, ScalarFunctionExpr}; + use datatypes::prelude::ConcreteDataType; + use datatypes::schema::ColumnSchema; + use session::context::QueryContext; + use table::metadata::{TableInfoBuilder, TableMetaBuilder}; + use table::test_util::EmptyTable; + + use super::*; + use crate::parser::QueryLanguageParser; + use crate::{QueryEngineFactory, QueryEngineRef}; + + fn create_test_batch() -> RecordBatch { + let schema = Schema::new(vec![Field::new("text", DataType::Utf8, true)]); + + let text_array = StringArray::from(vec![ + Some("hello world"), + Some("greeting"), + Some("hello there"), + None, + ]); + + RecordBatch::try_new(Arc::new(schema), vec![Arc::new(text_array) as ArrayRef]).unwrap() + } + + fn create_test_engine() -> QueryEngineRef { + let table_name = "test".to_string(); + let columns = vec![ + ColumnSchema::new( + "text".to_string(), + ConcreteDataType::string_datatype(), + false, + ), + ColumnSchema::new( + "timestamp".to_string(), + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ) + .with_time_index(true), + ]; + + let schema = Arc::new(datatypes::schema::Schema::new(columns)); + let table_meta = TableMetaBuilder::empty() + .schema(schema) + .primary_key_indices(vec![]) + .value_indices(vec![0]) + .next_column_id(2) + .build() + .unwrap(); + let table_info = TableInfoBuilder::default() + .name(&table_name) + .meta(table_meta) + .build() + .unwrap(); + let table = EmptyTable::from_table_info(&table_info); + let catalog_list = MemoryCatalogManager::with_default_setup(); + assert!(catalog_list + .register_table_sync(RegisterTableRequest { + catalog: DEFAULT_CATALOG_NAME.to_string(), + schema: DEFAULT_SCHEMA_NAME.to_string(), + table_name, + table_id: 1024, + table, + }) + .is_ok()); + QueryEngineFactory::new( + catalog_list, + None, + None, + None, + None, + false, + Default::default(), + ) + .query_engine() + } + + fn matches_term_udf() -> Arc { + Arc::new(create_udf( + Arc::new(MatchesTermFunction), + QueryContext::arc(), + Arc::new(FunctionState::default()), + )) + } + + #[test] + fn test_matches_term_optimization() { + let batch = create_test_batch(); + + // Create a predicate with a constant pattern + let predicate = create_physical_expr( + &Expr::ScalarFunction(ScalarFunction::new_udf( + matches_term_udf(), + vec![ + Expr::Column(Column::from_name("text")), + Expr::Literal(ScalarValue::Utf8(Some("hello".to_string()))), + ], + )), + &DFSchema::try_from(batch.schema().clone()).unwrap(), + &Default::default(), + ) + .unwrap(); + + let input = + Arc::new(MemoryExec::try_new(&[vec![batch.clone()]], batch.schema(), None).unwrap()); + let filter = FilterExec::try_new(predicate, input).unwrap(); + + // Apply the optimizer + let optimizer = MatchesConstantTermOptimizer; + let optimized_plan = optimizer + .optimize(Arc::new(filter), &Default::default()) + .unwrap(); + + let optimized_filter = optimized_plan + .as_any() + .downcast_ref::() + .unwrap(); + let predicate = optimized_filter.predicate(); + + // The predicate should be a PreCompiledMatchesTermExpr + assert!( + std::any::TypeId::of::() == predicate.as_any().type_id() + ); + } + + #[test] + fn test_matches_term_no_optimization() { + let batch = create_test_batch(); + + // Create a predicate with a non-constant pattern + let predicate = create_physical_expr( + &Expr::ScalarFunction(ScalarFunction::new_udf( + matches_term_udf(), + vec![ + Expr::Column(Column::from_name("text")), + Expr::Column(Column::from_name("text")), + ], + )), + &DFSchema::try_from(batch.schema().clone()).unwrap(), + &Default::default(), + ) + .unwrap(); + + let input = + Arc::new(MemoryExec::try_new(&[vec![batch.clone()]], batch.schema(), None).unwrap()); + let filter = FilterExec::try_new(predicate, input).unwrap(); + + let optimizer = MatchesConstantTermOptimizer; + let optimized_plan = optimizer + .optimize(Arc::new(filter), &Default::default()) + .unwrap(); + + let optimized_filter = optimized_plan + .as_any() + .downcast_ref::() + .unwrap(); + let predicate = optimized_filter.predicate(); + + // The predicate should still be a ScalarFunctionExpr + assert!(std::any::TypeId::of::() == predicate.as_any().type_id()); + } + + #[tokio::test] + async fn test_matches_term_optimization_from_sql() { + let sql = "WITH base AS ( + SELECT text, timestamp FROM test + WHERE MATCHES_TERM(text, 'hello') + AND timestamp > '2025-01-01 00:00:00' + ), + subquery1 AS ( + SELECT * FROM base + WHERE MATCHES_TERM(text, 'world') + ), + subquery2 AS ( + SELECT * FROM test + WHERE MATCHES_TERM(text, 'greeting') + AND timestamp < '2025-01-02 00:00:00' + ), + union_result AS ( + SELECT * FROM subquery1 + UNION ALL + SELECT * FROM subquery2 + ), + joined_data AS ( + SELECT a.text, a.timestamp, b.text as other_text + FROM union_result a + JOIN test b ON a.timestamp = b.timestamp + WHERE MATCHES_TERM(a.text, 'there') + ) + SELECT text, other_text + FROM joined_data + WHERE MATCHES_TERM(text, '42') + AND MATCHES_TERM(other_text, 'foo')"; + + let query_ctx = QueryContext::arc(); + + let stmt = QueryLanguageParser::parse_sql(sql, &query_ctx).unwrap(); + let engine = create_test_engine(); + let logical_plan = engine + .planner() + .plan(&stmt, query_ctx.clone()) + .await + .unwrap(); + + let engine_ctx = engine.engine_context(query_ctx); + let state = engine_ctx.state(); + + let analyzed_plan = state + .analyzer() + .execute_and_check(logical_plan.clone(), state.config_options(), |_, _| {}) + .unwrap(); + + let optimized_plan = state + .optimizer() + .optimize(analyzed_plan, state, |_, _| {}) + .unwrap(); + + let physical_plan = state + .query_planner() + .create_physical_plan(&optimized_plan, state) + .await + .unwrap(); + + let plan_str = get_plan_string(&physical_plan).join("\n"); + assert!(plan_str.contains("MatchesConstTerm")); + assert!(!plan_str.contains("matches_term")) + } +} diff --git a/src/query/src/query_engine/state.rs b/src/query/src/query_engine/state.rs index 75e1ed84a7..03f3a2a13d 100644 --- a/src/query/src/query_engine/state.rs +++ b/src/query/src/query_engine/state.rs @@ -45,6 +45,7 @@ use table::table::adapter::DfTableProviderAdapter; use table::TableRef; use crate::dist_plan::{DistExtensionPlanner, DistPlannerAnalyzer, MergeSortExtensionPlanner}; +use crate::optimizer::constant_term::MatchesConstantTermOptimizer; use crate::optimizer::count_wildcard::CountWildcardToTimeIndexRule; use crate::optimizer::parallelize_scan::ParallelizeScan; use crate::optimizer::pass_distribution::PassDistribution; @@ -143,6 +144,9 @@ impl QueryEngineState { physical_optimizer .rules .push(Arc::new(WindowedSortPhysicalRule)); + physical_optimizer + .rules + .push(Arc::new(MatchesConstantTermOptimizer)); // Add rule to remove duplicate nodes generated by other rules. Run this in the last. physical_optimizer.rules.push(Arc::new(RemoveDuplicate)); // Place SanityCheckPlan at the end of the list to ensure that it runs after all other rules. diff --git a/tests/cases/standalone/common/tql-explain-analyze/explain.result b/tests/cases/standalone/common/tql-explain-analyze/explain.result index e1bbaa89e3..8b4952ed3d 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/explain.result +++ b/tests/cases/standalone/common/tql-explain-analyze/explain.result @@ -167,6 +167,7 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test; | physical_plan after ProjectionPushdown_| SAME TEXT AS ABOVE_| | physical_plan after LimitPushdown_| SAME TEXT AS ABOVE_| | physical_plan after WindowedSortRule_| SAME TEXT AS ABOVE_| +| physical_plan after MatchesConstantTerm_| SAME TEXT AS ABOVE_| | physical_plan after RemoveDuplicateRule_| SAME TEXT AS ABOVE_| | physical_plan after SanityCheckPlan_| SAME TEXT AS ABOVE_| | physical_plan_| PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_| From 7b13376239abf0ed491e98e334469bb2d6ad1f88 Mon Sep 17 00:00:00 2001 From: zyy17 Date: Tue, 15 Apr 2025 14:46:31 +0800 Subject: [PATCH 21/82] refactor: add `partition_rules_for_uuid()` (#5743) * refactor: add partition_rules_for_uuid() * refactor: support up to 65536 partitions for partition_rules_for_uuid() --- Cargo.lock | 1 + src/operator/src/error.rs | 11 +- src/operator/src/insert.rs | 7 +- src/sql/Cargo.toml | 1 + src/sql/src/error.rs | 11 ++ src/sql/src/partition.rs | 268 ++++++++++++++++++++++---------- tests-integration/tests/http.rs | 2 +- 7 files changed, 213 insertions(+), 88 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2ab3200029..f1bf4eb3d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11011,6 +11011,7 @@ dependencies = [ "sqlparser_derive 0.1.1", "store-api", "table", + "uuid", ] [[package]] diff --git a/src/operator/src/error.rs b/src/operator/src/error.rs index c0c102ceda..900a3b4310 100644 --- a/src/operator/src/error.rs +++ b/src/operator/src/error.rs @@ -799,6 +799,14 @@ pub enum Error { #[snafu(implicit)] location: Location, }, + + #[snafu(display("Failed to create partition rules"))] + CreatePartitionRules { + #[snafu(source)] + source: sql::error::Error, + #[snafu(implicit)] + location: Location, + }, } pub type Result = std::result::Result; @@ -840,7 +848,8 @@ impl ErrorExt for Error { | Error::PhysicalExpr { .. } | Error::InvalidJsonFormat { .. } | Error::CursorNotFound { .. } - | Error::CursorExists { .. } => StatusCode::InvalidArguments, + | Error::CursorExists { .. } + | Error::CreatePartitionRules { .. } => StatusCode::InvalidArguments, Error::TableAlreadyExists { .. } | Error::ViewAlreadyExists { .. } => { StatusCode::TableAlreadyExists diff --git a/src/operator/src/insert.rs b/src/operator/src/insert.rs index b2742685fb..2f7c30ccd0 100644 --- a/src/operator/src/insert.rs +++ b/src/operator/src/insert.rs @@ -63,8 +63,8 @@ use table::table_reference::TableReference; use table::TableRef; use crate::error::{ - CatalogSnafu, ColumnOptionsSnafu, FindRegionLeaderSnafu, InvalidInsertRequestSnafu, - JoinTaskSnafu, RequestInsertsSnafu, Result, TableNotFoundSnafu, + CatalogSnafu, ColumnOptionsSnafu, CreatePartitionRulesSnafu, FindRegionLeaderSnafu, + InvalidInsertRequestSnafu, JoinTaskSnafu, RequestInsertsSnafu, Result, TableNotFoundSnafu, }; use crate::expr_helper; use crate::region_req_factory::RegionRequestFactory; @@ -591,7 +591,8 @@ impl Inserter { } else { // prebuilt partition rules for uuid data: see the function // for more information - let partitions = partition_rule_for_hexstring(TRACE_ID_COLUMN); + let partitions = partition_rule_for_hexstring(TRACE_ID_COLUMN) + .context(CreatePartitionRulesSnafu)?; // add skip index to // - trace_id: when searching by trace id // - parent_span_id: when searching root span diff --git a/src/sql/Cargo.toml b/src/sql/Cargo.toml index 3cb81d6dd4..812fe42709 100644 --- a/src/sql/Cargo.toml +++ b/src/sql/Cargo.toml @@ -37,6 +37,7 @@ sqlparser.workspace = true sqlparser_derive = "0.1" store-api.workspace = true table.workspace = true +uuid.workspace = true [dev-dependencies] common-datasource.workspace = true diff --git a/src/sql/src/error.rs b/src/sql/src/error.rs index e7253d6c46..e07efdbe6c 100644 --- a/src/sql/src/error.rs +++ b/src/sql/src/error.rs @@ -345,6 +345,16 @@ pub enum Error { #[snafu(implicit)] location: Location, }, + + #[snafu(display( + "Invalid partition number: {}, should be in range [2, 65536]", + partition_num + ))] + InvalidPartitionNumber { + partition_num: u32, + #[snafu(implicit)] + location: Location, + }, } impl ErrorExt for Error { @@ -380,6 +390,7 @@ impl ErrorExt for Error { | Simplification { .. } | InvalidInterval { .. } | InvalidUnaryOp { .. } + | InvalidPartitionNumber { .. } | UnsupportedUnaryOp { .. } => StatusCode::InvalidArguments, SerializeColumnDefaultConstraint { source, .. } => source.status_code(), diff --git a/src/sql/src/partition.rs b/src/sql/src/partition.rs index 4979bf702f..a1fd8e642e 100644 --- a/src/sql/src/partition.rs +++ b/src/sql/src/partition.rs @@ -12,10 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. +use snafu::ensure; use sqlparser::ast::{BinaryOperator, Expr, Ident, Value}; +use crate::error::{InvalidPartitionNumberSnafu, Result}; use crate::statements::create::Partitions; +/// The default number of partitions for OpenTelemetry traces. +const DEFAULT_PARTITION_NUM_FOR_TRACES: u32 = 16; + +/// The maximum number of partitions for OpenTelemetry traces. +const MAX_PARTITION_NUM_FOR_TRACES: u32 = 65536; + macro_rules! between_string { ($col: expr, $left_incl: expr, $right_excl: expr) => { Expr::BinaryOp { @@ -38,98 +46,105 @@ macro_rules! between_string { }; } -macro_rules! or { - ($left: expr, $right: expr) => { - Expr::BinaryOp { - op: BinaryOperator::Or, - left: Box::new($left), - right: Box::new($right), - } - }; +pub fn partition_rule_for_hexstring(ident: &str) -> Result { + Ok(Partitions { + column_list: vec![Ident::new(ident)], + exprs: partition_rules_for_uuid(DEFAULT_PARTITION_NUM_FOR_TRACES, ident)?, + }) } -pub fn partition_rule_for_hexstring(ident: &str) -> Partitions { - let ident = Ident::new(ident); - let ident_expr = Expr::Identifier(ident.clone()); +// partition_rules_for_uuid can creates partition rules up to 65536 partitions. +fn partition_rules_for_uuid(partition_num: u32, ident: &str) -> Result> { + ensure!( + partition_num.is_power_of_two() && (2..=65536).contains(&partition_num), + InvalidPartitionNumberSnafu { partition_num } + ); - // rules are like: - // - // "trace_id < '1'", - // "trace_id >= '1' AND trace_id < '2'", - // "trace_id >= '2' AND trace_id < '3'", - // "trace_id >= '3' AND trace_id < '4'", - // "trace_id >= '4' AND trace_id < '5'", - // "trace_id >= '5' AND trace_id < '6'", - // "trace_id >= '6' AND trace_id < '7'", - // "trace_id >= '7' AND trace_id < '8'", - // "trace_id >= '8' AND trace_id < '9'", - // "trace_id >= '9' AND trace_id < 'A'", - // "trace_id >= 'A' AND trace_id < 'B' OR trace_id >= 'a' AND trace_id < 'b'", - // "trace_id >= 'B' AND trace_id < 'C' OR trace_id >= 'b' AND trace_id < 'c'", - // "trace_id >= 'C' AND trace_id < 'D' OR trace_id >= 'c' AND trace_id < 'd'", - // "trace_id >= 'D' AND trace_id < 'E' OR trace_id >= 'd' AND trace_id < 'e'", - // "trace_id >= 'E' AND trace_id < 'F' OR trace_id >= 'e' AND trace_id < 'f'", - // "trace_id >= 'F' AND trace_id < 'a' OR trace_id >= 'f'", - let rules = vec![ - Expr::BinaryOp { - left: Box::new(ident_expr.clone()), - op: BinaryOperator::Lt, - right: Box::new(Expr::Value(Value::SingleQuotedString("1".to_string()))), - }, - // [left, right) - between_string!(ident_expr, "1", "2"), - between_string!(ident_expr, "2", "3"), - between_string!(ident_expr, "3", "4"), - between_string!(ident_expr, "4", "5"), - between_string!(ident_expr, "5", "6"), - between_string!(ident_expr, "6", "7"), - between_string!(ident_expr, "7", "8"), - between_string!(ident_expr, "8", "9"), - between_string!(ident_expr, "9", "A"), - or!( - between_string!(ident_expr, "A", "B"), - between_string!(ident_expr, "a", "b") - ), - or!( - between_string!(ident_expr, "B", "C"), - between_string!(ident_expr, "b", "c") - ), - or!( - between_string!(ident_expr, "C", "D"), - between_string!(ident_expr, "c", "d") - ), - or!( - between_string!(ident_expr, "D", "E"), - between_string!(ident_expr, "d", "e") - ), - or!( - between_string!(ident_expr, "E", "F"), - between_string!(ident_expr, "e", "f") - ), - or!( - between_string!(ident_expr, "F", "a"), - Expr::BinaryOp { + let ident_expr = Expr::Identifier(Ident::new(ident).clone()); + + let (total_partitions, hex_length) = { + match partition_num { + 2..=16 => (16, 1), + 17..=256 => (256, 2), + 257..=4096 => (4096, 3), + 4097..=MAX_PARTITION_NUM_FOR_TRACES => (MAX_PARTITION_NUM_FOR_TRACES, 4), + _ => unreachable!(), + } + }; + + let partition_size = total_partitions / partition_num; + let remainder = total_partitions % partition_num; + + let mut rules = Vec::new(); + let mut current_boundary = 0; + for i in 0..partition_num { + let mut size = partition_size; + if i < remainder { + size += 1; + } + let start = current_boundary; + let end = current_boundary + size; + + if i == 0 { + // Create the leftmost rule, for example: trace_id < '1'. + rules.push(Expr::BinaryOp { + left: Box::new(ident_expr.clone()), + op: BinaryOperator::Lt, + right: Box::new(Expr::Value(Value::SingleQuotedString(format!( + "{:0hex_length$x}", + end + )))), + }); + } else if i == partition_num - 1 { + // Create the rightmost rule, for example: trace_id >= 'f'. + rules.push(Expr::BinaryOp { left: Box::new(ident_expr.clone()), op: BinaryOperator::GtEq, - right: Box::new(Expr::Value(Value::SingleQuotedString("f".to_string()))), - } - ), - ]; + right: Box::new(Expr::Value(Value::SingleQuotedString(format!( + "{:0hex_length$x}", + start + )))), + }); + } else { + // Create the middle rules, for example: trace_id >= '1' AND trace_id < '2'. + rules.push(between_string!( + ident_expr, + format!("{:0hex_length$x}", start), + format!("{:0hex_length$x}", end) + )); + } - Partitions { - column_list: vec![ident], - exprs: rules, + current_boundary = end; } + + Ok(rules) } #[cfg(test)] mod tests { + use std::collections::HashMap; + use sqlparser::ast::Expr; use sqlparser::dialect::GenericDialect; use sqlparser::parser::Parser; + use uuid::Uuid; use super::*; + #[test] + fn test_partition_rules_for_uuid() { + // NOTE: We only test a subset of partitions to keep the test execution time reasonable. + // As the number of partitions increases, we need to increase the number of test samples to ensure uniform distribution. + assert!(check_distribution(2, 10_000)); // 2^1 + assert!(check_distribution(4, 10_000)); // 2^2 + assert!(check_distribution(8, 10_000)); // 2^3 + assert!(check_distribution(16, 10_000)); // 2^4 + assert!(check_distribution(32, 10_000)); // 2^5 + assert!(check_distribution(64, 100_000)); // 2^6 + assert!(check_distribution(128, 100_000)); // 2^7 + assert!(check_distribution(256, 100_000)); // 2^8 + } + #[test] fn test_rules() { let expr = vec![ @@ -142,13 +157,13 @@ mod tests { "trace_id >= '6' AND trace_id < '7'", "trace_id >= '7' AND trace_id < '8'", "trace_id >= '8' AND trace_id < '9'", - "trace_id >= '9' AND trace_id < 'A'", - "trace_id >= 'A' AND trace_id < 'B' OR trace_id >= 'a' AND trace_id < 'b'", - "trace_id >= 'B' AND trace_id < 'C' OR trace_id >= 'b' AND trace_id < 'c'", - "trace_id >= 'C' AND trace_id < 'D' OR trace_id >= 'c' AND trace_id < 'd'", - "trace_id >= 'D' AND trace_id < 'E' OR trace_id >= 'd' AND trace_id < 'e'", - "trace_id >= 'E' AND trace_id < 'F' OR trace_id >= 'e' AND trace_id < 'f'", - "trace_id >= 'F' AND trace_id < 'a' OR trace_id >= 'f'", + "trace_id >= '9' AND trace_id < 'a'", + "trace_id >= 'a' AND trace_id < 'b'", + "trace_id >= 'b' AND trace_id < 'c'", + "trace_id >= 'c' AND trace_id < 'd'", + "trace_id >= 'd' AND trace_id < 'e'", + "trace_id >= 'e' AND trace_id < 'f'", + "trace_id >= 'f'", ]; let dialect = GenericDialect {}; @@ -160,6 +175,93 @@ mod tests { }) .collect::>(); - assert_eq!(results, partition_rule_for_hexstring("trace_id").exprs); + assert_eq!( + results, + partition_rule_for_hexstring("trace_id").unwrap().exprs + ); + } + + fn check_distribution(test_partition: u32, test_uuid_num: usize) -> bool { + // Generate test_uuid_num random uuids. + let uuids = (0..test_uuid_num) + .map(|_| Uuid::new_v4().to_string().replace("-", "").to_lowercase()) + .collect::>(); + + // Generate the partition rules. + let rules = partition_rules_for_uuid(test_partition, "test_trace_id").unwrap(); + + // Collect the number of partitions for each uuid. + let mut stats = HashMap::new(); + for uuid in uuids { + let partition = allocate_partition_for_uuid(uuid.clone(), &rules); + // Count the number of uuids in each partition. + *stats.entry(partition).or_insert(0) += 1; + } + + // Check if the partition distribution is uniform. + let expected_ratio = 100.0 / test_partition as f64; + + // tolerance is the allowed deviation from the expected ratio. + let tolerance = 100.0 / test_partition as f64 * 0.30; + + // For each partition, its ratio should be as close as possible to the expected ratio. + for (_, count) in stats { + let ratio = (count as f64 / test_uuid_num as f64) * 100.0; + if (ratio - expected_ratio).abs() >= tolerance { + return false; + } + } + + true + } + + fn allocate_partition_for_uuid(uuid: String, rules: &[Expr]) -> usize { + for (i, rule) in rules.iter().enumerate() { + if let Expr::BinaryOp { left, op: _, right } = rule { + if i == 0 { + // Hit the leftmost rule. + if let Expr::Value(Value::SingleQuotedString(leftmost)) = *right.clone() { + if uuid < leftmost { + return i; + } + } + } else if i == rules.len() - 1 { + // Hit the rightmost rule. + if let Expr::Value(Value::SingleQuotedString(rightmost)) = *right.clone() { + if uuid >= rightmost { + return i; + } + } + } else { + // Hit the middle rules. + if let Expr::BinaryOp { + left: _, + op: _, + right: inner_right, + } = *left.clone() + { + if let Expr::Value(Value::SingleQuotedString(lower)) = *inner_right.clone() + { + if let Expr::BinaryOp { + left: _, + op: _, + right: inner_right, + } = *right.clone() + { + if let Expr::Value(Value::SingleQuotedString(upper)) = + *inner_right.clone() + { + if uuid >= lower && uuid < upper { + return i; + } + } + } + } + } + } + } + } + + panic!("No partition found for uuid: {}, rules: {:?}", uuid, rules); } } diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index 6eb4d10562..d20fe50241 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -2488,7 +2488,7 @@ pub async fn test_otlp_traces_v1(store_type: StorageType) { let expected = r#"[[1736480942444376000,1736480942444499000,123000,null,"c05d7a4ec8e1f231f02ed6e8da8655b4","d24f921c75f68e23","SPAN_KIND_CLIENT","lets-go","STATUS_CODE_UNSET","","","telemetrygen","","telemetrygen","1.2.3.4","telemetrygen-server",[],[]],[1736480942444376000,1736480942444499000,123000,"d24f921c75f68e23","c05d7a4ec8e1f231f02ed6e8da8655b4","9630f2916e2f7909","SPAN_KIND_SERVER","okey-dokey-0","STATUS_CODE_UNSET","","","telemetrygen","","telemetrygen","1.2.3.4","telemetrygen-client",[],[]],[1736480942444589000,1736480942444712000,123000,null,"cc9e0991a2e63d274984bd44ee669203","eba7be77e3558179","SPAN_KIND_CLIENT","lets-go","STATUS_CODE_UNSET","","","telemetrygen","","telemetrygen","1.2.3.4","telemetrygen-server",[],[]],[1736480942444589000,1736480942444712000,123000,"eba7be77e3558179","cc9e0991a2e63d274984bd44ee669203","8f847259b0f6e1ab","SPAN_KIND_SERVER","okey-dokey-0","STATUS_CODE_UNSET","","","telemetrygen","","telemetrygen","1.2.3.4","telemetrygen-client",[],[]]]"#; validate_data("otlp_traces", &client, "select * from mytable;", expected).await; - let expected_ddl = r#"[["mytable","CREATE TABLE IF NOT EXISTS \"mytable\" (\n \"timestamp\" TIMESTAMP(9) NOT NULL,\n \"timestamp_end\" TIMESTAMP(9) NULL,\n \"duration_nano\" BIGINT UNSIGNED NULL,\n \"parent_span_id\" STRING NULL SKIPPING INDEX WITH(granularity = '10240', type = 'BLOOM'),\n \"trace_id\" STRING NULL SKIPPING INDEX WITH(granularity = '10240', type = 'BLOOM'),\n \"span_id\" STRING NULL,\n \"span_kind\" STRING NULL,\n \"span_name\" STRING NULL,\n \"span_status_code\" STRING NULL,\n \"span_status_message\" STRING NULL,\n \"trace_state\" STRING NULL,\n \"scope_name\" STRING NULL,\n \"scope_version\" STRING NULL,\n \"service_name\" STRING NULL SKIPPING INDEX WITH(granularity = '10240', type = 'BLOOM'),\n \"span_attributes.net.peer.ip\" STRING NULL,\n \"span_attributes.peer.service\" STRING NULL,\n \"span_events\" JSON NULL,\n \"span_links\" JSON NULL,\n TIME INDEX (\"timestamp\"),\n PRIMARY KEY (\"service_name\")\n)\nPARTITION ON COLUMNS (\"trace_id\") (\n trace_id < '1',\n trace_id >= '1' AND trace_id < '2',\n trace_id >= '2' AND trace_id < '3',\n trace_id >= '3' AND trace_id < '4',\n trace_id >= '4' AND trace_id < '5',\n trace_id >= '5' AND trace_id < '6',\n trace_id >= '6' AND trace_id < '7',\n trace_id >= '7' AND trace_id < '8',\n trace_id >= '8' AND trace_id < '9',\n trace_id >= '9' AND trace_id < 'A',\n trace_id >= 'A' AND trace_id < 'B' OR trace_id >= 'a' AND trace_id < 'b',\n trace_id >= 'B' AND trace_id < 'C' OR trace_id >= 'b' AND trace_id < 'c',\n trace_id >= 'C' AND trace_id < 'D' OR trace_id >= 'c' AND trace_id < 'd',\n trace_id >= 'D' AND trace_id < 'E' OR trace_id >= 'd' AND trace_id < 'e',\n trace_id >= 'E' AND trace_id < 'F' OR trace_id >= 'e' AND trace_id < 'f',\n trace_id >= 'F' AND trace_id < 'a' OR trace_id >= 'f'\n)\nENGINE=mito\nWITH(\n append_mode = 'true',\n table_data_model = 'greptime_trace_v1'\n)"]]"#; + let expected_ddl = r#"[["mytable","CREATE TABLE IF NOT EXISTS \"mytable\" (\n \"timestamp\" TIMESTAMP(9) NOT NULL,\n \"timestamp_end\" TIMESTAMP(9) NULL,\n \"duration_nano\" BIGINT UNSIGNED NULL,\n \"parent_span_id\" STRING NULL SKIPPING INDEX WITH(granularity = '10240', type = 'BLOOM'),\n \"trace_id\" STRING NULL SKIPPING INDEX WITH(granularity = '10240', type = 'BLOOM'),\n \"span_id\" STRING NULL,\n \"span_kind\" STRING NULL,\n \"span_name\" STRING NULL,\n \"span_status_code\" STRING NULL,\n \"span_status_message\" STRING NULL,\n \"trace_state\" STRING NULL,\n \"scope_name\" STRING NULL,\n \"scope_version\" STRING NULL,\n \"service_name\" STRING NULL SKIPPING INDEX WITH(granularity = '10240', type = 'BLOOM'),\n \"span_attributes.net.peer.ip\" STRING NULL,\n \"span_attributes.peer.service\" STRING NULL,\n \"span_events\" JSON NULL,\n \"span_links\" JSON NULL,\n TIME INDEX (\"timestamp\"),\n PRIMARY KEY (\"service_name\")\n)\nPARTITION ON COLUMNS (\"trace_id\") (\n trace_id < '1',\n trace_id >= 'f',\n trace_id >= '1' AND trace_id < '2',\n trace_id >= '2' AND trace_id < '3',\n trace_id >= '3' AND trace_id < '4',\n trace_id >= '4' AND trace_id < '5',\n trace_id >= '5' AND trace_id < '6',\n trace_id >= '6' AND trace_id < '7',\n trace_id >= '7' AND trace_id < '8',\n trace_id >= '8' AND trace_id < '9',\n trace_id >= '9' AND trace_id < 'a',\n trace_id >= 'a' AND trace_id < 'b',\n trace_id >= 'b' AND trace_id < 'c',\n trace_id >= 'c' AND trace_id < 'd',\n trace_id >= 'd' AND trace_id < 'e',\n trace_id >= 'e' AND trace_id < 'f'\n)\nENGINE=mito\nWITH(\n append_mode = 'true',\n table_data_model = 'greptime_trace_v1'\n)"]]"#; validate_data( "otlp_traces", &client, From 032df4c5330a2b94515d38764c56c7da4fe4ee77 Mon Sep 17 00:00:00 2001 From: discord9 <55937128+discord9@users.noreply.github.com> Date: Tue, 15 Apr 2025 15:03:12 +0800 Subject: [PATCH 22/82] feat(flow): dual engine (#5881) * feat: partial use batch mode(WIP) * feat: add flow engine trait * refactor: more trait method * dual engine * feat: dual engine * refactor: flow map cache * chore: per review * chore: per review --- Cargo.toml | 5 + src/cmd/src/flownode.rs | 6 +- src/cmd/src/standalone.rs | 8 +- src/common/meta/src/ddl/create_flow.rs | 2 +- src/flow/src/adapter.rs | 53 +-- src/flow/src/adapter/flownode_impl.rs | 437 +++++++++++++++++++++---- src/flow/src/batching_mode.rs | 4 +- src/flow/src/batching_mode/engine.rs | 48 ++- src/flow/src/batching_mode/state.rs | 3 +- src/flow/src/batching_mode/task.rs | 4 +- src/flow/src/engine.rs | 57 ++++ src/flow/src/error.rs | 2 +- src/flow/src/lib.rs | 5 +- src/flow/src/server.rs | 10 +- src/meta-client/src/client.rs | 1 + tests-integration/src/standalone.rs | 5 +- 16 files changed, 534 insertions(+), 116 deletions(-) create mode 100644 src/flow/src/engine.rs diff --git a/Cargo.toml b/Cargo.toml index 38b749e7b0..f3bd54a661 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -307,3 +307,8 @@ strip = true [profile.dev.package.tests-fuzz] debug = false strip = true + +[profile.dev] +opt-level = 1 +[profile.dev.package."*"] +opt-level = 3 diff --git a/src/cmd/src/flownode.rs b/src/cmd/src/flownode.rs index fc23d37c23..a7b530e558 100644 --- a/src/cmd/src/flownode.rs +++ b/src/cmd/src/flownode.rs @@ -32,7 +32,9 @@ use common_meta::key::TableMetadataManager; use common_telemetry::info; use common_telemetry::logging::TracingOptions; use common_version::{short_version, version}; -use flow::{FlownodeBuilder, FlownodeInstance, FlownodeServiceBuilder, FrontendInvoker}; +use flow::{ + FlownodeBuilder, FlownodeInstance, FlownodeServiceBuilder, FrontendClient, FrontendInvoker, +}; use meta_client::{MetaClientOptions, MetaClientType}; use snafu::{ensure, OptionExt, ResultExt}; use tracing_appender::non_blocking::WorkerGuard; @@ -313,12 +315,14 @@ impl StartCommand { ); let flow_metadata_manager = Arc::new(FlowMetadataManager::new(cached_meta_backend.clone())); + let frontend_client = FrontendClient::from_meta_client(meta_client.clone()); let flownode_builder = FlownodeBuilder::new( opts.clone(), Plugins::new(), table_metadata_manager, catalog_manager.clone(), flow_metadata_manager, + Arc::new(frontend_client), ) .with_heartbeat_task(heartbeat_task); diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index 4504927cc8..3177a2446f 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -57,7 +57,7 @@ use datanode::region_server::RegionServer; use file_engine::config::EngineConfig as FileEngineConfig; use flow::{ FlowConfig, FlowWorkerManager, FlownodeBuilder, FlownodeInstance, FlownodeOptions, - FrontendInvoker, + FrontendClient, FrontendInvoker, }; use frontend::frontend::{Frontend, FrontendOptions}; use frontend::instance::builder::FrontendBuilder; @@ -523,12 +523,18 @@ impl StartCommand { flow: opts.flow.clone(), ..Default::default() }; + + // TODO(discord9): for standalone not use grpc, but just somehow get a handler to frontend grpc client without + // actually make a connection + let fe_server_addr = fe_opts.grpc.bind_addr.clone(); + let frontend_client = FrontendClient::from_static_grpc_addr(fe_server_addr); let flow_builder = FlownodeBuilder::new( flownode_options, plugins.clone(), table_metadata_manager.clone(), catalog_manager.clone(), flow_metadata_manager.clone(), + Arc::new(frontend_client), ); let flownode = flow_builder .build() diff --git a/src/common/meta/src/ddl/create_flow.rs b/src/common/meta/src/ddl/create_flow.rs index 4e7d661c1d..8b1c0354d4 100644 --- a/src/common/meta/src/ddl/create_flow.rs +++ b/src/common/meta/src/ddl/create_flow.rs @@ -324,7 +324,7 @@ pub enum CreateFlowState { } /// The type of flow. -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum FlowType { /// The flow is a batching task. Batching, diff --git a/src/flow/src/adapter.rs b/src/flow/src/adapter.rs index 8fd62ee2a0..516254ae55 100644 --- a/src/flow/src/adapter.rs +++ b/src/flow/src/adapter.rs @@ -16,7 +16,7 @@ //! and communicating with other parts of the database #![warn(unused_imports)] -use std::collections::{BTreeMap, HashMap}; +use std::collections::BTreeMap; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime}; @@ -56,6 +56,7 @@ use crate::error::{EvalSnafu, ExternalSnafu, InternalSnafu, InvalidQuerySnafu, U use crate::expr::Batch; use crate::metrics::{METRIC_FLOW_INSERT_ELAPSED, METRIC_FLOW_ROWS, METRIC_FLOW_RUN_INTERVAL_MS}; use crate::repr::{self, DiffRow, RelationDesc, Row, BATCH_SIZE}; +use crate::{CreateFlowArgs, FlowId, TableName}; mod flownode_impl; mod parse_expr; @@ -78,11 +79,6 @@ pub const AUTO_CREATED_PLACEHOLDER_TS_COL: &str = "__ts_placeholder"; pub const AUTO_CREATED_UPDATE_AT_TS_COL: &str = "update_at"; -// TODO(discord9): refactor common types for flow to a separate module -/// FlowId is a unique identifier for a flow task -pub type FlowId = u64; -pub type TableName = [String; 3]; - /// Flow config that exists both in standalone&distributed mode #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] #[serde(default)] @@ -731,25 +727,10 @@ impl FlowWorkerManager { } } -/// The arguments to create a flow in [`FlowWorkerManager`]. -#[derive(Debug, Clone)] -pub struct CreateFlowArgs { - pub flow_id: FlowId, - pub sink_table_name: TableName, - pub source_table_ids: Vec, - pub create_if_not_exists: bool, - pub or_replace: bool, - pub expire_after: Option, - pub comment: Option, - pub sql: String, - pub flow_options: HashMap, - pub query_ctx: Option, -} - /// Create&Remove flow impl FlowWorkerManager { /// remove a flow by it's id - pub async fn remove_flow(&self, flow_id: FlowId) -> Result<(), Error> { + pub async fn remove_flow_inner(&self, flow_id: FlowId) -> Result<(), Error> { for handle in self.worker_handles.iter() { if handle.contains_flow(flow_id).await? { handle.remove_flow(flow_id).await?; @@ -766,7 +747,7 @@ impl FlowWorkerManager { /// 1. parse query into typed plan(and optional parse expire_after expr) /// 2. render source/sink with output table id and used input table id #[allow(clippy::too_many_arguments)] - pub async fn create_flow(&self, args: CreateFlowArgs) -> Result, Error> { + pub async fn create_flow_inner(&self, args: CreateFlowArgs) -> Result, Error> { let CreateFlowArgs { flow_id, sink_table_name, @@ -905,6 +886,32 @@ impl FlowWorkerManager { info!("Successfully create flow with id={}", flow_id); Ok(Some(flow_id)) } + + pub async fn flush_flow_inner(&self, flow_id: FlowId) -> Result { + debug!("Starting to flush flow_id={:?}", flow_id); + // lock to make sure writes before flush are written to flow + // and immediately drop to prevent following writes to be blocked + drop(self.flush_lock.write().await); + let flushed_input_rows = self.node_context.read().await.flush_all_sender().await?; + let rows_send = self.run_available(true).await?; + let row = self.send_writeback_requests().await?; + debug!( + "Done to flush flow_id={:?} with {} input rows flushed, {} rows sended and {} output rows flushed", + flow_id, flushed_input_rows, rows_send, row + ); + Ok(row) + } + + pub async fn flow_exist_inner(&self, flow_id: FlowId) -> Result { + let mut exist = false; + for handle in self.worker_handles.iter() { + if handle.contains_flow(flow_id).await? { + exist = true; + break; + } + } + Ok(exist) + } } /// FlowTickManager is a manager for flow tick, which trakc flow execution progress diff --git a/src/flow/src/adapter/flownode_impl.rs b/src/flow/src/adapter/flownode_impl.rs index 1daec77fbd..b7d218ef21 100644 --- a/src/flow/src/adapter/flownode_impl.rs +++ b/src/flow/src/adapter/flownode_impl.rs @@ -13,40 +13,228 @@ // limitations under the License. //! impl `FlowNode` trait for FlowNodeManager so standalone can call them -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; use api::v1::flow::{ flow_request, CreateRequest, DropRequest, FlowRequest, FlowResponse, FlushFlow, }; use api::v1::region::InsertRequests; use common_error::ext::BoxedError; -use common_meta::error::{ExternalSnafu, Result, UnexpectedSnafu}; -use common_telemetry::{debug, trace}; +use common_meta::ddl::create_flow::FlowType; +use common_meta::error::{Result as MetaResult, UnexpectedSnafu}; +use common_runtime::JoinHandle; +use common_telemetry::{trace, warn}; use datatypes::value::Value; use itertools::Itertools; use snafu::{IntoError, OptionExt, ResultExt}; -use store_api::storage::RegionId; +use store_api::storage::{RegionId, TableId}; use crate::adapter::{CreateFlowArgs, FlowWorkerManager}; -use crate::error::{CreateFlowSnafu, InsertIntoFlowSnafu, InternalSnafu}; +use crate::batching_mode::engine::BatchingEngine; +use crate::engine::FlowEngine; +use crate::error::{CreateFlowSnafu, FlowNotFoundSnafu, InsertIntoFlowSnafu, InternalSnafu}; use crate::metrics::METRIC_FLOW_TASK_COUNT; use crate::repr::{self, DiffRow}; +use crate::{Error, FlowId}; -/// return a function to convert `crate::error::Error` to `common_meta::error::Error` -fn to_meta_err( - location: snafu::Location, -) -> impl FnOnce(crate::error::Error) -> common_meta::error::Error { - move |err: crate::error::Error| -> common_meta::error::Error { - common_meta::error::Error::External { - location, - source: BoxedError::new(err), +/// Manage both streaming and batching mode engine +/// +/// including create/drop/flush flow +/// and redirect insert requests to the appropriate engine +pub struct FlowDualEngine { + streaming_engine: Arc, + batching_engine: Arc, + /// helper struct for faster query flow by table id or vice versa + src_table2flow: std::sync::RwLock, +} + +struct SrcTableToFlow { + /// mapping of table ids to flow ids for streaming mode + stream: HashMap>, + /// mapping of table ids to flow ids for batching mode + batch: HashMap>, + /// mapping of flow ids to (flow type, source table ids) + flow_infos: HashMap)>, +} + +impl SrcTableToFlow { + fn in_stream(&self, table_id: TableId) -> bool { + self.stream.contains_key(&table_id) + } + fn in_batch(&self, table_id: TableId) -> bool { + self.batch.contains_key(&table_id) + } + fn add_flow(&mut self, flow_id: FlowId, flow_type: FlowType, src_table_ids: Vec) { + let mapping = match flow_type { + FlowType::Streaming => &mut self.stream, + FlowType::Batching => &mut self.batch, + }; + + for src_table in src_table_ids.clone() { + mapping + .entry(src_table) + .and_modify(|flows| { + flows.insert(flow_id); + }) + .or_insert_with(|| { + let mut set = HashSet::new(); + set.insert(flow_id); + set + }); } + self.flow_infos.insert(flow_id, (flow_type, src_table_ids)); + } + + fn remove_flow(&mut self, flow_id: FlowId) { + let mapping = match self.get_flow_type(flow_id) { + Some(FlowType::Streaming) => &mut self.stream, + Some(FlowType::Batching) => &mut self.batch, + None => return, + }; + if let Some((_, src_table_ids)) = self.flow_infos.remove(&flow_id) { + for src_table in src_table_ids { + if let Some(flows) = mapping.get_mut(&src_table) { + flows.remove(&flow_id); + } + } + } + } + + fn get_flow_type(&self, flow_id: FlowId) -> Option { + self.flow_infos + .get(&flow_id) + .map(|(flow_type, _)| flow_type) + .cloned() + } +} + +impl FlowEngine for FlowDualEngine { + async fn create_flow(&self, args: CreateFlowArgs) -> Result, Error> { + let flow_type = args + .flow_options + .get(FlowType::FLOW_TYPE_KEY) + .map(|s| s.as_str()); + + let flow_type = match flow_type { + Some(FlowType::BATCHING) => FlowType::Batching, + Some(FlowType::STREAMING) => FlowType::Streaming, + None => FlowType::Batching, + Some(flow_type) => { + return InternalSnafu { + reason: format!("Invalid flow type: {}", flow_type), + } + .fail() + } + }; + + let flow_id = args.flow_id; + let src_table_ids = args.source_table_ids.clone(); + + let res = match flow_type { + FlowType::Batching => self.batching_engine.create_flow(args).await, + FlowType::Streaming => self.streaming_engine.create_flow(args).await, + }?; + + self.src_table2flow + .write() + .unwrap() + .add_flow(flow_id, flow_type, src_table_ids); + + Ok(res) + } + + async fn remove_flow(&self, flow_id: FlowId) -> Result<(), Error> { + let flow_type = self.src_table2flow.read().unwrap().get_flow_type(flow_id); + match flow_type { + Some(FlowType::Batching) => self.batching_engine.remove_flow(flow_id).await, + Some(FlowType::Streaming) => self.streaming_engine.remove_flow(flow_id).await, + None => FlowNotFoundSnafu { id: flow_id }.fail(), + }?; + // remove mapping + self.src_table2flow.write().unwrap().remove_flow(flow_id); + Ok(()) + } + + async fn flush_flow(&self, flow_id: FlowId) -> Result { + let flow_type = self.src_table2flow.read().unwrap().get_flow_type(flow_id); + match flow_type { + Some(FlowType::Batching) => self.batching_engine.flush_flow(flow_id).await, + Some(FlowType::Streaming) => self.streaming_engine.flush_flow(flow_id).await, + None => FlowNotFoundSnafu { id: flow_id }.fail(), + } + } + + async fn flow_exist(&self, flow_id: FlowId) -> Result { + let flow_type = self.src_table2flow.read().unwrap().get_flow_type(flow_id); + // not using `flow_type.is_some()` to make sure the flow is actually exist in the underlying engine + match flow_type { + Some(FlowType::Batching) => self.batching_engine.flow_exist(flow_id).await, + Some(FlowType::Streaming) => self.streaming_engine.flow_exist(flow_id).await, + None => Ok(false), + } + } + + async fn handle_flow_inserts( + &self, + request: api::v1::region::InsertRequests, + ) -> Result<(), Error> { + // TODO(discord9): make as little clone as possible + let mut to_stream_engine = Vec::with_capacity(request.requests.len()); + let mut to_batch_engine = request.requests; + + { + let src_table2flow = self.src_table2flow.read().unwrap(); + to_batch_engine.retain(|req| { + let region_id = RegionId::from(req.region_id); + let table_id = region_id.table_id(); + let is_in_stream = src_table2flow.in_stream(table_id); + let is_in_batch = src_table2flow.in_batch(table_id); + if is_in_stream { + to_stream_engine.push(req.clone()); + } + if is_in_batch { + return true; + } + if !is_in_batch && !is_in_stream { + // TODO(discord9): also put to centralized logging for flow once it implemented + warn!("Table {} is not any flow's source table", table_id) + } + false + }); + // drop(src_table2flow); + // can't use drop due to https://github.com/rust-lang/rust/pull/128846 + } + + let streaming_engine = self.streaming_engine.clone(); + let stream_handler: JoinHandle> = + common_runtime::spawn_global(async move { + streaming_engine + .handle_flow_inserts(api::v1::region::InsertRequests { + requests: to_stream_engine, + }) + .await?; + Ok(()) + }); + self.batching_engine + .handle_flow_inserts(api::v1::region::InsertRequests { + requests: to_batch_engine, + }) + .await?; + stream_handler.await.map_err(|e| { + crate::error::UnexpectedSnafu { + reason: format!("JoinError when handle inserts for flow stream engine: {e:?}"), + } + .build() + })??; + + Ok(()) } } #[async_trait::async_trait] -impl common_meta::node_manager::Flownode for FlowWorkerManager { - async fn handle(&self, request: FlowRequest) -> Result { +impl common_meta::node_manager::Flownode for FlowDualEngine { + async fn handle(&self, request: FlowRequest) -> MetaResult { let query_ctx = request .header .and_then(|h| h.query_context) @@ -109,31 +297,10 @@ impl common_meta::node_manager::Flownode for FlowWorkerManager { Some(flow_request::Body::Flush(FlushFlow { flow_id: Some(flow_id), })) => { - // TODO(discord9): impl individual flush - debug!("Starting to flush flow_id={:?}", flow_id); - // lock to make sure writes before flush are written to flow - // and immediately drop to prevent following writes to be blocked - drop(self.flush_lock.write().await); - let flushed_input_rows = self - .node_context - .read() - .await - .flush_all_sender() - .await - .map_err(to_meta_err(snafu::location!()))?; - let rows_send = self - .run_available(true) - .await - .map_err(to_meta_err(snafu::location!()))?; let row = self - .send_writeback_requests() + .flush_flow(flow_id.id as u64) .await .map_err(to_meta_err(snafu::location!()))?; - - debug!( - "Done to flush flow_id={:?} with {} input rows flushed, {} rows sended and {} output rows flushed", - flow_id, flushed_input_rows, rows_send, row - ); Ok(FlowResponse { affected_flows: vec![flow_id], affected_rows: row as u64, @@ -151,7 +318,167 @@ impl common_meta::node_manager::Flownode for FlowWorkerManager { } } - async fn handle_inserts(&self, request: InsertRequests) -> Result { + async fn handle_inserts(&self, request: InsertRequests) -> MetaResult { + FlowEngine::handle_flow_inserts(self, request) + .await + .map(|_| Default::default()) + .map_err(to_meta_err(snafu::location!())) + } +} + +/// return a function to convert `crate::error::Error` to `common_meta::error::Error` +fn to_meta_err( + location: snafu::Location, +) -> impl FnOnce(crate::error::Error) -> common_meta::error::Error { + move |err: crate::error::Error| -> common_meta::error::Error { + common_meta::error::Error::External { + location, + source: BoxedError::new(err), + } + } +} + +#[async_trait::async_trait] +impl common_meta::node_manager::Flownode for FlowWorkerManager { + async fn handle(&self, request: FlowRequest) -> MetaResult { + let query_ctx = request + .header + .and_then(|h| h.query_context) + .map(|ctx| ctx.into()); + match request.body { + Some(flow_request::Body::Create(CreateRequest { + flow_id: Some(task_id), + source_table_ids, + sink_table_name: Some(sink_table_name), + create_if_not_exists, + expire_after, + comment, + sql, + flow_options, + or_replace, + })) => { + let source_table_ids = source_table_ids.into_iter().map(|id| id.id).collect_vec(); + let sink_table_name = [ + sink_table_name.catalog_name, + sink_table_name.schema_name, + sink_table_name.table_name, + ]; + let expire_after = expire_after.map(|e| e.value); + let args = CreateFlowArgs { + flow_id: task_id.id as u64, + sink_table_name, + source_table_ids, + create_if_not_exists, + or_replace, + expire_after, + comment: Some(comment), + sql: sql.clone(), + flow_options, + query_ctx, + }; + let ret = self + .create_flow(args) + .await + .map_err(BoxedError::new) + .with_context(|_| CreateFlowSnafu { sql: sql.clone() }) + .map_err(to_meta_err(snafu::location!()))?; + METRIC_FLOW_TASK_COUNT.inc(); + Ok(FlowResponse { + affected_flows: ret + .map(|id| greptime_proto::v1::FlowId { id: id as u32 }) + .into_iter() + .collect_vec(), + ..Default::default() + }) + } + Some(flow_request::Body::Drop(DropRequest { + flow_id: Some(flow_id), + })) => { + self.remove_flow(flow_id.id as u64) + .await + .map_err(to_meta_err(snafu::location!()))?; + METRIC_FLOW_TASK_COUNT.dec(); + Ok(Default::default()) + } + Some(flow_request::Body::Flush(FlushFlow { + flow_id: Some(flow_id), + })) => { + let row = self + .flush_flow_inner(flow_id.id as u64) + .await + .map_err(to_meta_err(snafu::location!()))?; + Ok(FlowResponse { + affected_flows: vec![flow_id], + affected_rows: row as u64, + ..Default::default() + }) + } + None => UnexpectedSnafu { + err_msg: "Missing request body", + } + .fail(), + _ => UnexpectedSnafu { + err_msg: "Invalid request body.", + } + .fail(), + } + } + + async fn handle_inserts(&self, request: InsertRequests) -> MetaResult { + self.handle_inserts_inner(request) + .await + .map(|_| Default::default()) + .map_err(to_meta_err(snafu::location!())) + } +} + +impl FlowEngine for FlowWorkerManager { + async fn create_flow(&self, args: CreateFlowArgs) -> Result, Error> { + self.create_flow_inner(args).await + } + + async fn remove_flow(&self, flow_id: FlowId) -> Result<(), Error> { + self.remove_flow_inner(flow_id).await + } + + async fn flush_flow(&self, flow_id: FlowId) -> Result { + self.flush_flow_inner(flow_id).await + } + + async fn flow_exist(&self, flow_id: FlowId) -> Result { + self.flow_exist_inner(flow_id).await + } + + async fn handle_flow_inserts( + &self, + request: api::v1::region::InsertRequests, + ) -> Result<(), Error> { + self.handle_inserts_inner(request).await + } +} + +/// Simple helper enum for fetching value from row with default value +#[derive(Debug, Clone)] +enum FetchFromRow { + Idx(usize), + Default(Value), +} + +impl FetchFromRow { + /// Panic if idx is out of bound + fn fetch(&self, row: &repr::Row) -> Value { + match self { + FetchFromRow::Idx(idx) => row.get(*idx).unwrap().clone(), + FetchFromRow::Default(v) => v.clone(), + } + } +} + +impl FlowWorkerManager { + async fn handle_inserts_inner( + &self, + request: InsertRequests, + ) -> std::result::Result<(), Error> { // using try_read to ensure two things: // 1. flush wouldn't happen until inserts before it is inserted // 2. inserts happening concurrently with flush wouldn't be block by flush @@ -172,11 +499,7 @@ impl common_meta::node_manager::Flownode for FlowWorkerManager { let ctx = self.node_context.read().await; // TODO(discord9): also check schema version so that altered table can be reported - let table_schema = ctx - .table_source - .table_from_id(&table_id) - .await - .map_err(to_meta_err(snafu::location!()))?; + let table_schema = ctx.table_source.table_from_id(&table_id).await?; let default_vals = table_schema .default_values .iter() @@ -210,9 +533,9 @@ impl common_meta::node_manager::Flownode for FlowWorkerManager { None => InternalSnafu { reason: format!("Expect column {idx} of table id={table_id} to have name in table schema, found None"), } - .fail().map_err(BoxedError::new).context(ExternalSnafu), + .fail(), }) - .collect::>>()?; + .collect::, _>>()?; let name_to_col = HashMap::<_, _>::from_iter( insert_schema .iter() @@ -229,8 +552,8 @@ impl common_meta::node_manager::Flownode for FlowWorkerManager { .copied() .map(FetchFromRow::Idx) .or_else(|| col_default_val.clone().map(FetchFromRow::Default)) - .with_context(|| UnexpectedSnafu { - err_msg: format!( + .with_context(|| crate::error::UnexpectedSnafu { + reason: format!( "Column not found: {}, default_value: {:?}", col_name, col_default_val ), @@ -272,27 +595,9 @@ impl common_meta::node_manager::Flownode for FlowWorkerManager { } .into_error(err); common_telemetry::error!(err; "Failed to handle write request"); - let err = to_meta_err(snafu::location!())(err); return Err(err); } } - Ok(Default::default()) - } -} - -/// Simple helper enum for fetching value from row with default value -#[derive(Debug, Clone)] -enum FetchFromRow { - Idx(usize), - Default(Value), -} - -impl FetchFromRow { - /// Panic if idx is out of bound - fn fetch(&self, row: &repr::Row) -> Value { - match self { - FetchFromRow::Idx(idx) => row.get(*idx).unwrap().clone(), - FetchFromRow::Default(v) => v.clone(), - } + Ok(()) } } diff --git a/src/flow/src/batching_mode.rs b/src/flow/src/batching_mode.rs index 138a44b633..152ad5781c 100644 --- a/src/flow/src/batching_mode.rs +++ b/src/flow/src/batching_mode.rs @@ -16,8 +16,8 @@ use std::time::Duration; -mod engine; -mod frontend_client; +pub(crate) mod engine; +pub(crate) mod frontend_client; mod state; mod task; mod time_window; diff --git a/src/flow/src/batching_mode/engine.rs b/src/flow/src/batching_mode/engine.rs index 72ab7042d2..c53107f695 100644 --- a/src/flow/src/batching_mode/engine.rs +++ b/src/flow/src/batching_mode/engine.rs @@ -12,10 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! Batching mode engine + use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; -use api::v1::flow::FlowResponse; use common_error::ext::BoxedError; use common_meta::ddl::create_flow::FlowType; use common_meta::key::flow::FlowMetadataManagerRef; @@ -30,13 +31,13 @@ use store_api::storage::RegionId; use table::metadata::TableId; use tokio::sync::{oneshot, RwLock}; -use crate::adapter::{CreateFlowArgs, FlowId, TableName}; use crate::batching_mode::frontend_client::FrontendClient; use crate::batching_mode::task::BatchingTask; use crate::batching_mode::time_window::{find_time_window_expr, TimeWindowExpr}; use crate::batching_mode::utils::sql_to_df_plan; +use crate::engine::FlowEngine; use crate::error::{ExternalSnafu, FlowAlreadyExistSnafu, TableNotFoundMetaSnafu, UnexpectedSnafu}; -use crate::Error; +use crate::{CreateFlowArgs, Error, FlowId, TableName}; /// Batching mode Engine, responsible for driving all the batching mode tasks /// @@ -67,10 +68,10 @@ impl BatchingEngine { } } - pub async fn handle_inserts( + pub async fn handle_inserts_inner( &self, request: api::v1::region::InsertRequests, - ) -> Result { + ) -> Result<(), Error> { let table_info_mgr = self.table_meta.table_info_manager(); let mut group_by_table_id: HashMap> = HashMap::new(); @@ -170,7 +171,7 @@ impl BatchingEngine { } drop(tasks); - Ok(Default::default()) + Ok(()) } } @@ -191,7 +192,7 @@ async fn get_table_name( } impl BatchingEngine { - pub async fn create_flow(&self, args: CreateFlowArgs) -> Result, Error> { + pub async fn create_flow_inner(&self, args: CreateFlowArgs) -> Result, Error> { let CreateFlowArgs { flow_id, sink_table_name, @@ -308,7 +309,7 @@ impl BatchingEngine { Ok(Some(flow_id)) } - pub async fn remove_flow(&self, flow_id: FlowId) -> Result<(), Error> { + pub async fn remove_flow_inner(&self, flow_id: FlowId) -> Result<(), Error> { if self.tasks.write().await.remove(&flow_id).is_none() { warn!("Flow {flow_id} not found in tasks") } @@ -324,19 +325,42 @@ impl BatchingEngine { Ok(()) } - pub async fn flush_flow(&self, flow_id: FlowId) -> Result<(), Error> { + pub async fn flush_flow_inner(&self, flow_id: FlowId) -> Result { let task = self.tasks.read().await.get(&flow_id).cloned(); let task = task.with_context(|| UnexpectedSnafu { reason: format!("Can't found task for flow {flow_id}"), })?; - task.gen_exec_once(&self.query_engine, &self.frontend_client) + let res = task + .gen_exec_once(&self.query_engine, &self.frontend_client) .await?; - Ok(()) + let affected_rows = res.map(|(r, _)| r).unwrap_or_default() as usize; + Ok(affected_rows) } /// Determine if the batching mode flow task exists with given flow id - pub async fn flow_exist(&self, flow_id: FlowId) -> bool { + pub async fn flow_exist_inner(&self, flow_id: FlowId) -> bool { self.tasks.read().await.contains_key(&flow_id) } } + +impl FlowEngine for BatchingEngine { + async fn create_flow(&self, args: CreateFlowArgs) -> Result, Error> { + self.create_flow_inner(args).await + } + async fn remove_flow(&self, flow_id: FlowId) -> Result<(), Error> { + self.remove_flow_inner(flow_id).await + } + async fn flush_flow(&self, flow_id: FlowId) -> Result { + self.flush_flow_inner(flow_id).await + } + async fn flow_exist(&self, flow_id: FlowId) -> Result { + Ok(self.flow_exist_inner(flow_id).await) + } + async fn handle_flow_inserts( + &self, + request: api::v1::region::InsertRequests, + ) -> Result<(), Error> { + self.handle_inserts_inner(request).await + } +} diff --git a/src/flow/src/batching_mode/state.rs b/src/flow/src/batching_mode/state.rs index a406dae798..3a9802713c 100644 --- a/src/flow/src/batching_mode/state.rs +++ b/src/flow/src/batching_mode/state.rs @@ -26,11 +26,10 @@ use snafu::ResultExt; use tokio::sync::oneshot; use tokio::time::Instant; -use crate::adapter::FlowId; use crate::batching_mode::task::BatchingTask; use crate::batching_mode::MIN_REFRESH_DURATION; use crate::error::{DatatypesSnafu, InternalSnafu, TimeSnafu}; -use crate::Error; +use crate::{Error, FlowId}; /// The state of the [`BatchingTask`]. #[derive(Debug)] diff --git a/src/flow/src/batching_mode/task.rs b/src/flow/src/batching_mode/task.rs index 44312509d4..f4280f54bd 100644 --- a/src/flow/src/batching_mode/task.rs +++ b/src/flow/src/batching_mode/task.rs @@ -43,7 +43,7 @@ use tokio::sync::oneshot; use tokio::sync::oneshot::error::TryRecvError; use tokio::time::Instant; -use crate::adapter::{FlowId, AUTO_CREATED_PLACEHOLDER_TS_COL, AUTO_CREATED_UPDATE_AT_TS_COL}; +use crate::adapter::{AUTO_CREATED_PLACEHOLDER_TS_COL, AUTO_CREATED_UPDATE_AT_TS_COL}; use crate::batching_mode::frontend_client::FrontendClient; use crate::batching_mode::state::TaskState; use crate::batching_mode::time_window::TimeWindowExpr; @@ -60,7 +60,7 @@ use crate::error::{ use crate::metrics::{ METRIC_FLOW_BATCHING_ENGINE_QUERY_TIME, METRIC_FLOW_BATCHING_ENGINE_SLOW_QUERY, }; -use crate::Error; +use crate::{Error, FlowId}; /// The task's config, immutable once created #[derive(Clone)] diff --git a/src/flow/src/engine.rs b/src/flow/src/engine.rs new file mode 100644 index 0000000000..33da5252d7 --- /dev/null +++ b/src/flow/src/engine.rs @@ -0,0 +1,57 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Define a trait for flow engine, which is used by both streaming engine and batch engine + +use std::collections::HashMap; + +use session::context::QueryContext; +use table::metadata::TableId; + +use crate::Error; +// TODO(discord9): refactor common types for flow to a separate module +/// FlowId is a unique identifier for a flow task +pub type FlowId = u64; +pub type TableName = [String; 3]; + +/// The arguments to create a flow +#[derive(Debug, Clone)] +pub struct CreateFlowArgs { + pub flow_id: FlowId, + pub sink_table_name: TableName, + pub source_table_ids: Vec, + pub create_if_not_exists: bool, + pub or_replace: bool, + pub expire_after: Option, + pub comment: Option, + pub sql: String, + pub flow_options: HashMap, + pub query_ctx: Option, +} + +pub trait FlowEngine { + /// Create a flow using the provided arguments, return previous flow id if exists and is replaced + async fn create_flow(&self, args: CreateFlowArgs) -> Result, Error>; + /// Remove a flow by its ID + async fn remove_flow(&self, flow_id: FlowId) -> Result<(), Error>; + /// Flush the flow, return the number of rows flushed + async fn flush_flow(&self, flow_id: FlowId) -> Result; + /// Check if the flow exists + async fn flow_exist(&self, flow_id: FlowId) -> Result; + /// Handle the insert requests for the flow + async fn handle_flow_inserts( + &self, + request: api::v1::region::InsertRequests, + ) -> Result<(), Error>; +} diff --git a/src/flow/src/error.rs b/src/flow/src/error.rs index 2488b0a677..1741f8cb1b 100644 --- a/src/flow/src/error.rs +++ b/src/flow/src/error.rs @@ -25,8 +25,8 @@ use common_telemetry::common_error::status_code::StatusCode; use snafu::{Location, ResultExt, Snafu}; use tonic::metadata::MetadataMap; -use crate::adapter::FlowId; use crate::expr::EvalError; +use crate::FlowId; /// This error is used to represent all possible errors that can occur in the flow module. #[derive(Snafu)] diff --git a/src/flow/src/lib.rs b/src/flow/src/lib.rs index 8ec464730b..5dc3c67491 100644 --- a/src/flow/src/lib.rs +++ b/src/flow/src/lib.rs @@ -26,9 +26,10 @@ // allow unused for now because it should be use later mod adapter; -mod batching_mode; +pub(crate) mod batching_mode; mod compute; mod df_optimizer; +pub(crate) mod engine; pub mod error; mod expr; pub mod heartbeat; @@ -43,6 +44,8 @@ mod utils; mod test_utils; pub use adapter::{FlowConfig, FlowWorkerManager, FlowWorkerManagerRef, FlownodeOptions}; +pub use batching_mode::frontend_client::FrontendClient; +pub(crate) use engine::{CreateFlowArgs, FlowId, TableName}; pub use error::{Error, Result}; pub use server::{ FlownodeBuilder, FlownodeInstance, FlownodeServer, FlownodeServiceBuilder, FrontendInvoker, diff --git a/src/flow/src/server.rs b/src/flow/src/server.rs index d0038e6ba1..53712ffb67 100644 --- a/src/flow/src/server.rs +++ b/src/flow/src/server.rs @@ -50,7 +50,7 @@ use tonic::codec::CompressionEncoding; use tonic::transport::server::TcpIncoming; use tonic::{Request, Response, Status}; -use crate::adapter::{create_worker, CreateFlowArgs, FlowWorkerManagerRef}; +use crate::adapter::{create_worker, FlowWorkerManagerRef}; use crate::error::{ to_status_with_last_err, CacheRequiredSnafu, CreateFlowSnafu, ExternalSnafu, FlowNotFoundSnafu, ListFlowsSnafu, ParseAddrSnafu, ShutdownServerSnafu, StartServerSnafu, UnexpectedSnafu, @@ -59,12 +59,13 @@ use crate::heartbeat::HeartbeatTask; use crate::metrics::{METRIC_FLOW_PROCESSING_TIME, METRIC_FLOW_ROWS}; use crate::transform::register_function_to_query_engine; use crate::utils::{SizeReportSender, StateReportHandler}; -use crate::{Error, FlowWorkerManager, FlownodeOptions}; +use crate::{CreateFlowArgs, Error, FlowWorkerManager, FlownodeOptions, FrontendClient}; pub const FLOW_NODE_SERVER_NAME: &str = "FLOW_NODE_SERVER"; /// wrapping flow node manager to avoid orphan rule with Arc<...> #[derive(Clone)] pub struct FlowService { + /// TODO(discord9): replace with dual engine pub manager: FlowWorkerManagerRef, } @@ -290,6 +291,7 @@ pub struct FlownodeBuilder { heartbeat_task: Option, /// receive a oneshot sender to send state size report state_report_handler: Option, + frontend_client: Arc, } impl FlownodeBuilder { @@ -300,6 +302,7 @@ impl FlownodeBuilder { table_meta: TableMetadataManagerRef, catalog_manager: CatalogManagerRef, flow_metadata_manager: FlowMetadataManagerRef, + frontend_client: Arc, ) -> Self { Self { opts, @@ -309,6 +312,7 @@ impl FlownodeBuilder { flow_metadata_manager, heartbeat_task: None, state_report_handler: None, + frontend_client, } } @@ -432,7 +436,7 @@ impl FlownodeBuilder { ), }; manager - .create_flow(args) + .create_flow_inner(args) .await .map_err(BoxedError::new) .with_context(|_| CreateFlowSnafu { diff --git a/src/meta-client/src/client.rs b/src/meta-client/src/client.rs index 3829ae2273..e022717ea1 100644 --- a/src/meta-client/src/client.rs +++ b/src/meta-client/src/client.rs @@ -117,6 +117,7 @@ impl MetaClientBuilder { .enable_store() .enable_heartbeat() .enable_procedure() + .enable_access_cluster_info() } pub fn enable_heartbeat(self) -> Self { diff --git a/tests-integration/src/standalone.rs b/tests-integration/src/standalone.rs index 2d6c9bcf97..b85c848c88 100644 --- a/tests-integration/src/standalone.rs +++ b/tests-integration/src/standalone.rs @@ -41,7 +41,7 @@ use common_procedure::options::ProcedureConfig; use common_procedure::ProcedureManagerRef; use common_wal::config::{DatanodeWalConfig, MetasrvWalConfig}; use datanode::datanode::DatanodeBuilder; -use flow::FlownodeBuilder; +use flow::{FlownodeBuilder, FrontendClient}; use frontend::frontend::Frontend; use frontend::instance::builder::FrontendBuilder; use frontend::instance::{Instance, StandaloneDatanodeManager}; @@ -174,12 +174,15 @@ impl GreptimeDbStandaloneBuilder { Some(procedure_manager.clone()), ); + let fe_server_addr = opts.frontend_options().grpc.bind_addr.clone(); + let frontend_client = FrontendClient::from_static_grpc_addr(fe_server_addr); let flow_builder = FlownodeBuilder::new( Default::default(), plugins.clone(), table_metadata_manager.clone(), catalog_manager.clone(), flow_metadata_manager.clone(), + Arc::new(frontend_client), ); let flownode = Arc::new(flow_builder.build().await.unwrap()); From 6700c0762d4937160bfa392d221dd320268eb8d7 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" <6406592+v0y4g3r@users.noreply.github.com> Date: Tue, 15 Apr 2025 18:42:07 +0800 Subject: [PATCH 23/82] feat: Column-wise partition rule implementation (#5804) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wip: naive impl * feat/column-partition: ### Add support for DataFusion physical expressions - **`Cargo.lock` & `Cargo.toml`**: Added `datafusion-physical-expr` as a dependency to support physical expression creation. - **`expr.rs`**: Implemented conversion methods `try_as_logical_expr` and `try_as_physical_expr` for `Operand` and `PartitionExpr` to facilitate logical and physical expression handling. - **`multi_dim.rs`**: Enhanced `MultiDimPartitionRule` to utilize physical expressions for partitioning logic, including new methods for evaluating record batches. - **Tests**: Added unit tests for logical and physical expression conversions and partitioning logic in `expr.rs` and `multi_dim.rs`. * feat/column-partition: ### Refactor and Enhance Partition Handling - **Refactor Partition Parsing Logic**: Moved partition parsing logic from `src/operator/src/statement/ddl.rs` to a new utility module `src/partition/src/utils.rs`. This includes functions like `parse_partitions`, `find_partition_bounds`, and `convert_one_expr`. - **Error Handling Improvements**: Added new error variants `ColumnNotFound`, `InvalidPartitionRule`, and `ParseSqlValue` in `src/partition/src/error.rs` to improve error reporting for partition-related operations. - **Dependency Updates**: Updated `Cargo.lock` and `Cargo.toml` to include new dependencies `common-time` and `session`. - **Code Cleanup**: Removed redundant partition parsing functions from `src/operator/src/error.rs` and `src/operator/src/statement/ddl.rs`. * feat/column-partition: ## Refactor and Enhance SQL and Table Handling - **Refactor Column Definitions and Error Handling** - Made `FULLTEXT_GRPC_KEY`, `INVERTED_INDEX_GRPC_KEY`, and `SKIPPING_INDEX_GRPC_KEY` public in `column_def.rs`. - Removed `IllegalPrimaryKeysDef` error from `error.rs` and moved it to `sql/src/error.rs`. - Updated error handling in `fill_impure_default.rs` and `expr_helper.rs`. - **Enhance SQL Utility Functions** - Moved and refactored functions like `create_to_expr`, `find_primary_keys`, and `validate_create_expr` to `sql/src/util.rs`. - Added new utility functions for SQL parsing and validation in `sql/src/util.rs`. - **Improve Partition Handling** - Added `parse_partition_columns_and_exprs` function in `partition/src/utils.rs`. - Updated partition rule tests in `partition/src/multi_dim.rs` to use SQL-based partitioning. - **Simplify Table Name Handling** - Re-exported `table_idents_to_full_name` from `sql::util` in `session/src/table_name.rs`. - **Test Enhancements** - Updated tests in `partition/src/multi_dim.rs` to use SQL for partition rule creation. * feat/column-partition: **Add Benchmarking and Enhance Partitioning Logic** - **Benchmarking**: Introduced a new benchmark for `split_record_batch` in `bench_split_record_batch.rs` using `criterion` and `rand` as development dependencies in `Cargo.toml`. - **Partitioning Logic**: Enhanced `MultiDimPartitionRule` in `multi_dim.rs` to include a default region for unmatched partition expressions and optimized the `split_record_batch` method. - **Refactoring**: Moved `sql_to_partition_rule` function to a public scope for reuse in `multi_dim.rs`. - **Testing**: Added new test module `test_split_record_batch` to validate the partitioning logic. * Revert "feat/column-partition: ### Refactor and Enhance Partition Handling" This reverts commit 183fa19f * fix: revert refctoring parse_partition * revert some refactor * feat/column-partition: ### Enhance Partitioning and Error Handling - **Benchmark Enhancements**: Added new benchmark `bench_split_record_batch_vs_row` in `bench_split_record_batch.rs` to compare row and column-based splitting. - **Error Handling Improvements**: Introduced new error variants in `error.rs` for better error reporting related to record batch evaluation and arrow kernel computation. - **Expression Handling**: Updated `expr.rs` to improve error context when converting schemas and creating physical expressions. - **Partition Rule Enhancements**: Made `row_at` and `record_batch_to_cols` methods public in `multi_dim.rs` and improved error handling for physical expression evaluation and boolean operations. * feat/column-partition: ### Add `eq` Method and Optimize Expression Caching - **`expr.rs`**: Added a new `eq` method to the `Operand` struct for equality comparisons. - **`multi_dim.rs`**: Introduced a caching mechanism for physical expressions using `RwLock` to improve performance in `MultiDimPartitionRule`. - **`lib.rs`**: Enabled the `let_chains` feature for more concise code. - **`multi_dim.rs` Tests**: Enhanced test coverage with new test cases for multi-dimensional partitioning, including random record batch generation and default region handling. * feat/column-partition: ### Add `split_record_batch` Method to `PartitionRule` Trait - **Files Modified**: - `src/partition/src/multi_dim.rs` - `src/partition/src/partition.rs` - `src/partition/src/splitter.rs` Added a new method `split_record_batch` to the `PartitionRule` trait, allowing record batches to be split into multiple regions based on partition values. Implemented this method in `MultiDimPartitionRule` and provided unimplemented stubs in test modules. ### Dependency Update - **File Modified**: - `src/operator/src/expr_helper.rs` Removed unused import `ColumnDataType` and `Timezone` from the test module. ### Miscellaneous - **File Modified**: - `src/partition/Cargo.toml` No functional changes; only minor formatting adjustments. * chore: add license header * chore: remove useless fules * feat/column-partition: Add support for handling unsupported partition expression values - **`error.rs`**: Introduced a new error variant `UnsupportedPartitionExprValue` to handle unsupported partition expression values, and updated `ErrorExt` to map this error to `StatusCode::InvalidArguments`. - **`expr.rs`**: Modified the `Operand` implementation to return the new error when encountering unsupported partition expression values. - **`multi_dim.rs`**: Added a fast path to optimize the selection process when all rows are selected. * feat/column-partition: Add validation for expression and region length in MultiDimPartitionRule constructor • Ensure the lengths of exprs and regions match to prevent mismatches. • Introduce error handling for length discrepancies with a descriptive error message. * chore: add debug log * feat/column-partition: Removed the validation check for matching lengths between exprs and regions in MultiDimPartitionRule constructor, simplifying the initialization process. * fix: unit tests --- Cargo.lock | 3 + src/partition/Cargo.toml | 9 + .../benches/bench_split_record_batch.rs | 226 +++++++++++++ src/partition/src/error.rs | 62 ++++ src/partition/src/expr.rs | 117 ++++++- src/partition/src/lib.rs | 2 +- src/partition/src/multi_dim.rs | 307 +++++++++++++++++- src/partition/src/partition.rs | 9 + src/partition/src/splitter.rs | 23 +- 9 files changed, 753 insertions(+), 5 deletions(-) create mode 100644 src/partition/benches/bench_split_record_batch.rs diff --git a/Cargo.lock b/Cargo.lock index f1bf4eb3d0..5182582e82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8138,10 +8138,13 @@ dependencies = [ "common-macro", "common-meta", "common-query", + "criterion 0.5.1", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr", "datatypes", "itertools 0.14.0", + "rand 0.8.5", "serde", "serde_json", "session", diff --git a/src/partition/Cargo.toml b/src/partition/Cargo.toml index ebb7d68f8d..6a0904f8f2 100644 --- a/src/partition/Cargo.toml +++ b/src/partition/Cargo.toml @@ -16,6 +16,7 @@ common-meta.workspace = true common-query.workspace = true datafusion-common.workspace = true datafusion-expr.workspace = true +datafusion-physical-expr.workspace = true datatypes.workspace = true itertools.workspace = true serde.workspace = true @@ -26,3 +27,11 @@ sql.workspace = true sqlparser.workspace = true store-api.workspace = true table.workspace = true + +[dev-dependencies] +criterion = "0.5" +rand = "0.8" + +[[bench]] +name = "bench_split_record_batch" +harness = false diff --git a/src/partition/benches/bench_split_record_batch.rs b/src/partition/benches/bench_split_record_batch.rs new file mode 100644 index 0000000000..f6c1bd69d4 --- /dev/null +++ b/src/partition/benches/bench_split_record_batch.rs @@ -0,0 +1,226 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; +use std::vec; + +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use datatypes::arrow::array::{ArrayRef, Int32Array, StringArray, TimestampMillisecondArray}; +use datatypes::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; +use datatypes::arrow::record_batch::RecordBatch; +use datatypes::value::Value; +use partition::expr::{col, Operand}; +use partition::multi_dim::MultiDimPartitionRule; +use partition::PartitionRule; +use rand::Rng; +use store_api::storage::RegionNumber; + +fn table_schema() -> Arc { + Arc::new(Schema::new(vec![ + Field::new("a0", DataType::Int32, false), + Field::new("a1", DataType::Utf8, false), + Field::new("a2", DataType::Int32, false), + Field::new( + "ts", + DataType::Timestamp(TimeUnit::Millisecond, None), + false, + ), + ])) +} + +fn create_test_rule(num_columns: usize) -> MultiDimPartitionRule { + let (columns, exprs) = match num_columns { + 1 => { + let exprs = vec![ + col("a0").lt(Value::Int32(50)), + col("a0").gt_eq(Value::Int32(50)), + ]; + (vec!["a0".to_string()], exprs) + } + 2 => { + let exprs = vec![ + col("a0") + .lt(Value::Int32(50)) + .and(col("a1").lt(Value::String("server50".into()))), + col("a0") + .lt(Value::Int32(50)) + .and(col("a1").gt_eq(Value::String("server50".into()))), + col("a0") + .gt_eq(Value::Int32(50)) + .and(col("a1").lt(Value::String("server50".into()))), + col("a0") + .gt_eq(Value::Int32(50)) + .and(col("a1").gt_eq(Value::String("server50".into()))), + ]; + (vec!["a0".to_string(), "a1".to_string()], exprs) + } + 3 => { + let expr = vec![ + col("a0") + .lt(Value::Int32(50)) + .and(col("a1").lt(Value::String("server50".into()))) + .and(col("a2").lt(Value::Int32(50))), + col("a0") + .lt(Operand::Value(Value::Int32(50))) + .and(col("a1").lt(Value::String("server50".into()))) + .and(col("a2").gt_eq(Value::Int32(50))), + col("a0") + .lt(Value::Int32(50)) + .and(col("a1").gt_eq(Value::String("server50".into()))) + .and(col("a2").lt(Value::Int32(50))), + col("a0") + .lt(Value::Int32(50)) + .and(col("a1").gt_eq(Value::String("server50".into()))) + .and(col("a2").gt_eq(Value::Int32(50))), + col("a0") + .gt_eq(Value::Int32(50)) + .and(col("a1").lt(Value::String("server50".into()))) + .and(col("a2").lt(Value::Int32(50))), + col("a0") + .gt_eq(Operand::Value(Value::Int32(50))) + .and(col("a1").lt(Value::String("server50".into()))) + .and(col("a2").gt_eq(Value::Int32(50))), + col("a0") + .gt_eq(Value::Int32(50)) + .and(col("a1").gt_eq(Value::String("server50".into()))) + .and(col("a2").lt(Value::Int32(50))), + col("a0") + .gt_eq(Value::Int32(50)) + .and(col("a1").gt_eq(Value::String("server50".into()))) + .and(col("a2").gt_eq(Value::Int32(50))), + ]; + + ( + vec!["a0".to_string(), "a1".to_string(), "a2".to_string()], + expr, + ) + } + _ => { + panic!("invalid number of columns, only 1-3 are supported"); + } + }; + + let regions = (0..exprs.len()).map(|v| v as u32).collect(); + MultiDimPartitionRule::try_new(columns, regions, exprs).unwrap() +} + +fn create_test_batch(size: usize) -> RecordBatch { + let mut rng = rand::thread_rng(); + + let schema = table_schema(); + let arrays: Vec = (0..3) + .map(|col_idx| { + if col_idx % 2 == 0 { + // Integer columns (a0, a2) + Arc::new(Int32Array::from_iter_values( + (0..size).map(|_| rng.gen_range(0..100)), + )) as ArrayRef + } else { + // String columns (a1) + let values: Vec = (0..size) + .map(|_| { + let server_id: i32 = rng.gen_range(0..100); + format!("server{}", server_id) + }) + .collect(); + Arc::new(StringArray::from(values)) as ArrayRef + } + }) + .chain(std::iter::once({ + // Timestamp column (ts) + Arc::new(TimestampMillisecondArray::from_iter_values( + (0..size).map(|idx| idx as i64), + )) as ArrayRef + })) + .collect(); + RecordBatch::try_new(schema, arrays).unwrap() +} + +fn bench_split_record_batch_naive_vs_optimized(c: &mut Criterion) { + let mut group = c.benchmark_group("split_record_batch"); + + for num_columns in [1, 2, 3].iter() { + for num_rows in [100, 1000, 10000, 100000].iter() { + let rule = create_test_rule(*num_columns); + let batch = create_test_batch(*num_rows); + + group.bench_function(format!("naive_{}_{}", num_columns, num_rows), |b| { + b.iter(|| { + black_box(rule.split_record_batch_naive(black_box(&batch))).unwrap(); + }); + }); + group.bench_function(format!("optimized_{}_{}", num_columns, num_rows), |b| { + b.iter(|| { + black_box(rule.split_record_batch(black_box(&batch))).unwrap(); + }); + }); + } + } + + group.finish(); +} + +fn record_batch_to_rows( + rule: &MultiDimPartitionRule, + record_batch: &RecordBatch, +) -> Vec> { + let num_rows = record_batch.num_rows(); + let vectors = rule.record_batch_to_cols(record_batch).unwrap(); + let mut res = Vec::with_capacity(num_rows); + let mut current_row = vec![Value::Null; vectors.len()]; + + for row in 0..num_rows { + rule.row_at(&vectors, row, &mut current_row).unwrap(); + res.push(current_row.clone()); + } + res +} + +fn find_all_regions(rule: &MultiDimPartitionRule, rows: &[Vec]) -> Vec { + rows.iter() + .map(|row| rule.find_region(row).unwrap()) + .collect() +} + +fn bench_split_record_batch_vs_row(c: &mut Criterion) { + let mut group = c.benchmark_group("bench_split_record_batch_vs_row"); + + for num_columns in [1, 2, 3].iter() { + for num_rows in [100, 1000, 10000, 100000].iter() { + let rule = create_test_rule(*num_columns); + let batch = create_test_batch(*num_rows); + let rows = record_batch_to_rows(&rule, &batch); + + group.bench_function(format!("split_by_row_{}_{}", num_columns, num_rows), |b| { + b.iter(|| { + black_box(find_all_regions(&rule, &rows)); + }); + }); + group.bench_function(format!("split_by_col_{}_{}", num_columns, num_rows), |b| { + b.iter(|| { + black_box(rule.split_record_batch(black_box(&batch))).unwrap(); + }); + }); + } + } + + group.finish(); +} + +criterion_group!( + benches, + bench_split_record_batch_naive_vs_optimized, + bench_split_record_batch_vs_row +); +criterion_main!(benches); diff --git a/src/partition/src/error.rs b/src/partition/src/error.rs index 2487fa0974..2194583f40 100644 --- a/src/partition/src/error.rs +++ b/src/partition/src/error.rs @@ -18,6 +18,8 @@ use common_error::ext::ErrorExt; use common_error::status_code::StatusCode; use common_macro::stack_trace_debug; use datafusion_common::ScalarValue; +use datatypes::arrow; +use datatypes::prelude::Value; use snafu::{Location, Snafu}; use store_api::storage::RegionId; use table::metadata::TableId; @@ -173,6 +175,59 @@ pub enum Error { #[snafu(implicit)] location: Location, }, + + #[snafu(display("Failed to convert to vector"))] + ConvertToVector { + source: datatypes::error::Error, + #[snafu(implicit)] + location: Location, + }, + + #[snafu(display("Failed to evaluate record batch"))] + EvaluateRecordBatch { + #[snafu(source)] + error: datafusion_common::error::DataFusionError, + #[snafu(implicit)] + location: Location, + }, + + #[snafu(display("Failed to compute arrow kernel"))] + ComputeArrowKernel { + #[snafu(source)] + error: arrow::error::ArrowError, + #[snafu(implicit)] + location: Location, + }, + + #[snafu(display("Unexpected evaluation result column type: {}", data_type))] + UnexpectedColumnType { + data_type: arrow::datatypes::DataType, + #[snafu(implicit)] + location: Location, + }, + + #[snafu(display("Failed to convert to DataFusion's Schema"))] + ToDFSchema { + #[snafu(source)] + error: datafusion_common::error::DataFusionError, + #[snafu(implicit)] + location: Location, + }, + + #[snafu(display("Failed to create physical expression"))] + CreatePhysicalExpr { + #[snafu(source)] + error: datafusion_common::error::DataFusionError, + #[snafu(implicit)] + location: Location, + }, + + #[snafu(display("Partition expr value is not supported: {:?}", value))] + UnsupportedPartitionExprValue { + value: Value, + #[snafu(implicit)] + location: Location, + }, } impl ErrorExt for Error { @@ -201,6 +256,13 @@ impl ErrorExt for Error { Error::TableRouteNotFound { .. } => StatusCode::TableNotFound, Error::TableRouteManager { source, .. } => source.status_code(), Error::UnexpectedLogicalRouteTable { source, .. } => source.status_code(), + Error::ConvertToVector { source, .. } => source.status_code(), + Error::EvaluateRecordBatch { .. } => StatusCode::Internal, + Error::ComputeArrowKernel { .. } => StatusCode::Internal, + Error::UnexpectedColumnType { .. } => StatusCode::Internal, + Error::ToDFSchema { .. } => StatusCode::Internal, + Error::CreatePhysicalExpr { .. } => StatusCode::Internal, + Error::UnsupportedPartitionExprValue { .. } => StatusCode::InvalidArguments, } } diff --git a/src/partition/src/expr.rs b/src/partition/src/expr.rs index bec9543e72..b758d6dcba 100644 --- a/src/partition/src/expr.rs +++ b/src/partition/src/expr.rs @@ -13,12 +13,23 @@ // limitations under the License. use std::fmt::{Debug, Display, Formatter}; +use std::sync::Arc; -use datatypes::value::Value; +use datafusion_common::{ScalarValue, ToDFSchema}; +use datafusion_expr::execution_props::ExecutionProps; +use datafusion_expr::Expr; +use datafusion_physical_expr::{create_physical_expr, PhysicalExpr}; +use datatypes::arrow; +use datatypes::value::{ + duration_to_scalar_value, time_to_scalar_value, timestamp_to_scalar_value, Value, +}; use serde::{Deserialize, Serialize}; +use snafu::ResultExt; use sql::statements::value_to_sql_value; use sqlparser::ast::{BinaryOperator as ParserBinaryOperator, Expr as ParserExpr, Ident}; +use crate::error; + /// Struct for partition expression. This can be converted back to sqlparser's [Expr]. /// by [`Self::to_parser_expr`]. /// @@ -37,6 +48,75 @@ pub enum Operand { Expr(PartitionExpr), } +pub fn col(column_name: impl Into) -> Operand { + Operand::Column(column_name.into()) +} + +impl From for Operand { + fn from(value: Value) -> Self { + Operand::Value(value) + } +} + +impl Operand { + pub fn try_as_logical_expr(&self) -> error::Result { + match self { + Self::Column(c) => Ok(datafusion_expr::col(c)), + Self::Value(v) => { + let scalar_value = match v { + Value::Boolean(v) => ScalarValue::Boolean(Some(*v)), + Value::UInt8(v) => ScalarValue::UInt8(Some(*v)), + Value::UInt16(v) => ScalarValue::UInt16(Some(*v)), + Value::UInt32(v) => ScalarValue::UInt32(Some(*v)), + Value::UInt64(v) => ScalarValue::UInt64(Some(*v)), + Value::Int8(v) => ScalarValue::Int8(Some(*v)), + Value::Int16(v) => ScalarValue::Int16(Some(*v)), + Value::Int32(v) => ScalarValue::Int32(Some(*v)), + Value::Int64(v) => ScalarValue::Int64(Some(*v)), + Value::Float32(v) => ScalarValue::Float32(Some(v.0)), + Value::Float64(v) => ScalarValue::Float64(Some(v.0)), + Value::String(v) => ScalarValue::Utf8(Some(v.as_utf8().to_string())), + Value::Binary(v) => ScalarValue::Binary(Some(v.to_vec())), + Value::Date(v) => ScalarValue::Date32(Some(v.val())), + Value::Null => ScalarValue::Null, + Value::Timestamp(t) => timestamp_to_scalar_value(t.unit(), Some(t.value())), + Value::Time(t) => time_to_scalar_value(*t.unit(), Some(t.value())).unwrap(), + Value::IntervalYearMonth(v) => ScalarValue::IntervalYearMonth(Some(v.to_i32())), + Value::IntervalDayTime(v) => ScalarValue::IntervalDayTime(Some((*v).into())), + Value::IntervalMonthDayNano(v) => { + ScalarValue::IntervalMonthDayNano(Some((*v).into())) + } + Value::Duration(d) => duration_to_scalar_value(d.unit(), Some(d.value())), + Value::Decimal128(d) => { + let (v, p, s) = d.to_scalar_value(); + ScalarValue::Decimal128(v, p, s) + } + other => { + return error::UnsupportedPartitionExprValueSnafu { + value: other.clone(), + } + .fail() + } + }; + Ok(datafusion_expr::lit(scalar_value)) + } + Self::Expr(e) => e.try_as_logical_expr(), + } + } + + pub fn lt(self, rhs: impl Into) -> PartitionExpr { + PartitionExpr::new(self, RestrictedOp::Lt, rhs.into()) + } + + pub fn gt_eq(self, rhs: impl Into) -> PartitionExpr { + PartitionExpr::new(self, RestrictedOp::GtEq, rhs.into()) + } + + pub fn eq(self, rhs: impl Into) -> PartitionExpr { + PartitionExpr::new(self, RestrictedOp::Eq, rhs.into()) + } +} + impl Display for Operand { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { @@ -140,6 +220,41 @@ impl PartitionExpr { right: Box::new(rhs), } } + + pub fn try_as_logical_expr(&self) -> error::Result { + let lhs = self.lhs.try_as_logical_expr()?; + let rhs = self.rhs.try_as_logical_expr()?; + + let expr = match &self.op { + RestrictedOp::And => datafusion_expr::and(lhs, rhs), + RestrictedOp::Or => datafusion_expr::or(lhs, rhs), + RestrictedOp::Gt => lhs.gt(rhs), + RestrictedOp::GtEq => lhs.gt_eq(rhs), + RestrictedOp::Lt => lhs.lt(rhs), + RestrictedOp::LtEq => lhs.lt_eq(rhs), + RestrictedOp::Eq => lhs.eq(rhs), + RestrictedOp::NotEq => lhs.not_eq(rhs), + }; + Ok(expr) + } + + pub fn try_as_physical_expr( + &self, + schema: &arrow::datatypes::SchemaRef, + ) -> error::Result> { + let df_schema = schema + .clone() + .to_dfschema_ref() + .context(error::ToDFSchemaSnafu)?; + let execution_props = &ExecutionProps::default(); + let expr = self.try_as_logical_expr()?; + create_physical_expr(&expr, &df_schema, execution_props) + .context(error::CreatePhysicalExprSnafu) + } + + pub fn and(self, rhs: PartitionExpr) -> PartitionExpr { + PartitionExpr::new(Operand::Expr(self), RestrictedOp::And, Operand::Expr(rhs)) + } } impl Display for PartitionExpr { diff --git a/src/partition/src/lib.rs b/src/partition/src/lib.rs index b1843a1093..bc56edc584 100644 --- a/src/partition/src/lib.rs +++ b/src/partition/src/lib.rs @@ -13,7 +13,7 @@ // limitations under the License. #![feature(assert_matches)] - +#![feature(let_chains)] //! Structs and traits for partitioning rule. pub mod error; diff --git a/src/partition/src/multi_dim.rs b/src/partition/src/multi_dim.rs index f47d71f98b..551fb6a8de 100644 --- a/src/partition/src/multi_dim.rs +++ b/src/partition/src/multi_dim.rs @@ -15,10 +15,18 @@ use std::any::Any; use std::cmp::Ordering; use std::collections::HashMap; +use std::sync::{Arc, RwLock}; +use datafusion_expr::ColumnarValue; +use datafusion_physical_expr::PhysicalExpr; +use datatypes::arrow; +use datatypes::arrow::array::{BooleanArray, BooleanBufferBuilder, RecordBatch}; +use datatypes::arrow::buffer::BooleanBuffer; +use datatypes::arrow::datatypes::Schema; use datatypes::prelude::Value; +use datatypes::vectors::{Helper, VectorRef}; use serde::{Deserialize, Serialize}; -use snafu::{ensure, OptionExt}; +use snafu::{ensure, OptionExt, ResultExt}; use store_api::storage::RegionNumber; use crate::error::{ @@ -28,6 +36,11 @@ use crate::error::{ use crate::expr::{Operand, PartitionExpr, RestrictedOp}; use crate::PartitionRule; +/// The default region number when no partition exprs are matched. +const DEFAULT_REGION: RegionNumber = 0; + +type PhysicalExprCache = Option<(Vec>, Arc)>; + /// Multi-Dimiension partition rule. RFC [here](https://github.com/GreptimeTeam/greptimedb/blob/main/docs/rfcs/2024-02-21-multi-dimension-partition-rule/rfc.md) /// /// This partition rule is defined by a set of simple expressions on the partition @@ -44,6 +57,9 @@ pub struct MultiDimPartitionRule { regions: Vec, /// Partition expressions. exprs: Vec, + /// Cache of physical expressions. + #[serde(skip)] + physical_expr_cache: RwLock, } impl MultiDimPartitionRule { @@ -63,6 +79,7 @@ impl MultiDimPartitionRule { name_to_index, regions, exprs, + physical_expr_cache: RwLock::new(None), }; let mut checker = RuleChecker::new(&rule); @@ -87,7 +104,7 @@ impl MultiDimPartitionRule { } // return the default region number - Ok(0) + Ok(DEFAULT_REGION) } fn evaluate_expr(&self, expr: &PartitionExpr, values: &[Value]) -> Result { @@ -134,6 +151,133 @@ impl MultiDimPartitionRule { Ok(result) } + + pub fn row_at(&self, cols: &[VectorRef], index: usize, row: &mut [Value]) -> Result<()> { + for (col_idx, col) in cols.iter().enumerate() { + row[col_idx] = col.get(index); + } + Ok(()) + } + + pub fn record_batch_to_cols(&self, record_batch: &RecordBatch) -> Result> { + self.partition_columns + .iter() + .map(|col_name| { + record_batch + .column_by_name(col_name) + .context(error::UndefinedColumnSnafu { column: col_name }) + .and_then(|array| { + Helper::try_into_vector(array).context(error::ConvertToVectorSnafu) + }) + }) + .collect::>>() + } + + pub fn split_record_batch_naive( + &self, + record_batch: &RecordBatch, + ) -> Result> { + let num_rows = record_batch.num_rows(); + + let mut result = self + .regions + .iter() + .map(|region| { + let mut builder = BooleanBufferBuilder::new(num_rows); + builder.append_n(num_rows, false); + (*region, builder) + }) + .collect::>(); + + let cols = self.record_batch_to_cols(record_batch)?; + let mut current_row = vec![Value::Null; self.partition_columns.len()]; + for row_idx in 0..num_rows { + self.row_at(&cols, row_idx, &mut current_row)?; + let current_region = self.find_region(¤t_row)?; + let region_mask = result + .get_mut(¤t_region) + .unwrap_or_else(|| panic!("Region {} must be initialized", current_region)); + region_mask.set_bit(row_idx, true); + } + + Ok(result + .into_iter() + .map(|(region, mut mask)| (region, BooleanArray::new(mask.finish(), None))) + .collect()) + } + + pub fn split_record_batch( + &self, + record_batch: &RecordBatch, + ) -> Result> { + let num_rows = record_batch.num_rows(); + let physical_exprs = { + let cache_read_guard = self.physical_expr_cache.read().unwrap(); + if let Some((cached_exprs, schema)) = cache_read_guard.as_ref() + && schema == record_batch.schema_ref() + { + cached_exprs.clone() + } else { + drop(cache_read_guard); // Release the read lock before acquiring write lock + + let schema = record_batch.schema(); + let new_cache = self + .exprs + .iter() + .map(|e| e.try_as_physical_expr(&schema)) + .collect::>>()?; + + let mut cache_write_guard = self.physical_expr_cache.write().unwrap(); + cache_write_guard.replace((new_cache.clone(), schema)); + new_cache + } + }; + + let mut result: HashMap = physical_exprs + .iter() + .zip(self.regions.iter()) + .map(|(expr, region_num)| { + let ColumnarValue::Array(column) = expr + .evaluate(record_batch) + .context(error::EvaluateRecordBatchSnafu)? + else { + unreachable!("Expected an array") + }; + Ok(( + *region_num, + column + .as_any() + .downcast_ref::() + .with_context(|| error::UnexpectedColumnTypeSnafu { + data_type: column.data_type().clone(), + })? + .clone(), + )) + }) + .collect::>()?; + + let mut selected = BooleanArray::new(BooleanBuffer::new_unset(num_rows), None); + for region_selection in result.values() { + selected = arrow::compute::kernels::boolean::or(&selected, region_selection) + .context(error::ComputeArrowKernelSnafu)?; + } + + // fast path: all rows are selected + if selected.true_count() == num_rows { + return Ok(result); + } + + // find unselected rows and assign to default region + let unselected = arrow::compute::kernels::boolean::not(&selected) + .context(error::ComputeArrowKernelSnafu)?; + let default_region_selection = result + .entry(DEFAULT_REGION) + .or_insert_with(|| unselected.clone()); + *default_region_selection = + arrow::compute::kernels::boolean::or(default_region_selection, &unselected) + .context(error::ComputeArrowKernelSnafu)?; + Ok(result) + } } impl PartitionRule for MultiDimPartitionRule { @@ -148,6 +292,13 @@ impl PartitionRule for MultiDimPartitionRule { fn find_region(&self, values: &[Value]) -> Result { self.find_region(values) } + + fn split_record_batch( + &self, + record_batch: &RecordBatch, + ) -> Result> { + self.split_record_batch(record_batch) + } } /// Helper for [RuleChecker] @@ -633,3 +784,155 @@ mod tests { assert!(rule.is_err()); } } + +#[cfg(test)] +mod test_split_record_batch { + use std::sync::Arc; + + use datatypes::arrow::array::{Int64Array, StringArray}; + use datatypes::arrow::datatypes::{DataType, Field, Schema}; + use datatypes::arrow::record_batch::RecordBatch; + use rand::Rng; + + use super::*; + use crate::expr::col; + + fn test_schema() -> Arc { + Arc::new(Schema::new(vec![ + Field::new("host", DataType::Utf8, false), + Field::new("value", DataType::Int64, false), + ])) + } + + fn generate_random_record_batch(num_rows: usize) -> RecordBatch { + let schema = test_schema(); + let mut rng = rand::thread_rng(); + let mut host_array = Vec::with_capacity(num_rows); + let mut value_array = Vec::with_capacity(num_rows); + for _ in 0..num_rows { + host_array.push(format!("server{}", rng.gen_range(0..20))); + value_array.push(rng.gen_range(0..20)); + } + let host_array = StringArray::from(host_array); + let value_array = Int64Array::from(value_array); + RecordBatch::try_new(schema, vec![Arc::new(host_array), Arc::new(value_array)]).unwrap() + } + + #[test] + fn test_split_record_batch_by_one_column() { + // Create a simple MultiDimPartitionRule + let rule = MultiDimPartitionRule::try_new( + vec!["host".to_string(), "value".to_string()], + vec![0, 1], + vec![ + col("host").lt(Value::String("server1".into())), + col("host").gt_eq(Value::String("server1".into())), + ], + ) + .unwrap(); + + let batch = generate_random_record_batch(1000); + // Split the batch + let result = rule.split_record_batch(&batch).unwrap(); + let expected = rule.split_record_batch_naive(&batch).unwrap(); + assert_eq!(result.len(), expected.len()); + for (region, value) in &result { + assert_eq!( + value, + expected.get(region).unwrap(), + "failed on region: {}", + region + ); + } + } + + #[test] + fn test_split_record_batch_empty() { + // Create a simple MultiDimPartitionRule + let rule = MultiDimPartitionRule::try_new( + vec!["host".to_string()], + vec![1], + vec![PartitionExpr::new( + Operand::Column("host".to_string()), + RestrictedOp::Eq, + Operand::Value(Value::String("server1".into())), + )], + ) + .unwrap(); + + let schema = test_schema(); + let host_array = StringArray::from(Vec::<&str>::new()); + let value_array = Int64Array::from(Vec::::new()); + let batch = RecordBatch::try_new(schema, vec![Arc::new(host_array), Arc::new(value_array)]) + .unwrap(); + + let result = rule.split_record_batch(&batch).unwrap(); + assert_eq!(result.len(), 1); + } + + #[test] + fn test_split_record_batch_by_two_columns() { + let rule = MultiDimPartitionRule::try_new( + vec!["host".to_string(), "value".to_string()], + vec![0, 1, 2, 3], + vec![ + col("host") + .lt(Value::String("server10".into())) + .and(col("value").lt(Value::Int64(10))), + col("host") + .lt(Value::String("server10".into())) + .and(col("value").gt_eq(Value::Int64(10))), + col("host") + .gt_eq(Value::String("server10".into())) + .and(col("value").lt(Value::Int64(10))), + col("host") + .gt_eq(Value::String("server10".into())) + .and(col("value").gt_eq(Value::Int64(10))), + ], + ) + .unwrap(); + + let batch = generate_random_record_batch(1000); + let result = rule.split_record_batch(&batch).unwrap(); + let expected = rule.split_record_batch_naive(&batch).unwrap(); + assert_eq!(result.len(), expected.len()); + for (region, value) in &result { + assert_eq!(value, expected.get(region).unwrap()); + } + } + + #[test] + fn test_default_region() { + let rule = MultiDimPartitionRule::try_new( + vec!["host".to_string(), "value".to_string()], + vec![0, 1, 2, 3], + vec![ + col("host") + .lt(Value::String("server10".into())) + .and(col("value").eq(Value::Int64(10))), + col("host") + .lt(Value::String("server10".into())) + .and(col("value").eq(Value::Int64(20))), + col("host") + .gt_eq(Value::String("server10".into())) + .and(col("value").eq(Value::Int64(10))), + col("host") + .gt_eq(Value::String("server10".into())) + .and(col("value").eq(Value::Int64(20))), + ], + ) + .unwrap(); + + let schema = test_schema(); + let host_array = StringArray::from(vec!["server1", "server1", "server1", "server100"]); + let value_array = Int64Array::from(vec![10, 20, 30, 10]); + let batch = RecordBatch::try_new(schema, vec![Arc::new(host_array), Arc::new(value_array)]) + .unwrap(); + let result = rule.split_record_batch(&batch).unwrap(); + let expected = rule.split_record_batch_naive(&batch).unwrap(); + assert_eq!(result.len(), expected.len()); + for (region, value) in &result { + assert_eq!(value, expected.get(region).unwrap()); + } + } +} diff --git a/src/partition/src/partition.rs b/src/partition/src/partition.rs index ac965034c6..a190d33eca 100644 --- a/src/partition/src/partition.rs +++ b/src/partition/src/partition.rs @@ -13,11 +13,13 @@ // limitations under the License. use std::any::Any; +use std::collections::HashMap; use std::fmt::{Debug, Display, Formatter}; use std::sync::Arc; use common_meta::rpc::router::Partition as MetaPartition; use datafusion_expr::Operator; +use datatypes::arrow::array::{BooleanArray, RecordBatch}; use datatypes::prelude::Value; use itertools::Itertools; use serde::{Deserialize, Serialize}; @@ -37,6 +39,13 @@ pub trait PartitionRule: Sync + Send { /// /// Note that the `values` should have the same length as the `partition_columns`. fn find_region(&self, values: &[Value]) -> Result; + + /// Split the record batch into multiple regions by the partition values. + /// The result is a map from region number to a boolean array, where the boolean array is true for the rows that match the partition values. + fn split_record_batch( + &self, + record_batch: &RecordBatch, + ) -> Result>; } /// The right bound(exclusive) of partition range. diff --git a/src/partition/src/splitter.rs b/src/partition/src/splitter.rs index f62210a6b5..87c04a4942 100644 --- a/src/partition/src/splitter.rs +++ b/src/partition/src/splitter.rs @@ -136,6 +136,7 @@ mod tests { use api::v1::value::ValueData; use api::v1::{ColumnDataType, SemanticType}; + use datatypes::arrow::array::BooleanArray; use serde::{Deserialize, Serialize}; use super::*; @@ -209,6 +210,13 @@ mod tests { Ok(val.parse::().unwrap() % 2) } + + fn split_record_batch( + &self, + _record_batch: &datatypes::arrow::array::RecordBatch, + ) -> Result> { + unimplemented!() + } } #[derive(Debug, Serialize, Deserialize)] @@ -232,6 +240,13 @@ mod tests { Ok(val) } + + fn split_record_batch( + &self, + _record_batch: &datatypes::arrow::array::RecordBatch, + ) -> Result> { + unimplemented!() + } } #[derive(Debug, Serialize, Deserialize)] @@ -249,8 +264,14 @@ mod tests { fn find_region(&self, _values: &[Value]) -> Result { Ok(0) } - } + fn split_record_batch( + &self, + _record_batch: &datatypes::arrow::array::RecordBatch, + ) -> Result> { + unimplemented!() + } + } #[test] fn test_writer_splitter() { let rows = mock_rows(); From dcf1a486f68d91ef1f9308109206291f92959698 Mon Sep 17 00:00:00 2001 From: Ruihang Xia Date: Tue, 15 Apr 2025 19:05:17 +0800 Subject: [PATCH 24/82] feat: support `@@` (AtAt) operator for term matching (#5902) * update dep and sqlness case Signed-off-by: Ruihang Xia * implement transcribe rule Signed-off-by: Ruihang Xia * more tests Signed-off-by: Ruihang Xia * update sqlness result Signed-off-by: Ruihang Xia --------- Signed-off-by: Ruihang Xia --- Cargo.lock | 48 +-- Cargo.toml | 18 +- src/query/src/optimizer.rs | 1 + src/query/src/optimizer/transcribe_atat.rs | 230 +++++++++++++ src/query/src/query_engine/state.rs | 2 + .../cases/standalone/common/expr/atat.result | 315 ++++++++++++++++++ tests/cases/standalone/common/expr/atat.sql | 144 ++++++++ .../common/tql-explain-analyze/explain.result | 1 + 8 files changed, 726 insertions(+), 33 deletions(-) create mode 100644 src/query/src/optimizer/transcribe_atat.rs create mode 100644 tests/cases/standalone/common/expr/atat.result create mode 100644 tests/cases/standalone/common/expr/atat.sql diff --git a/Cargo.lock b/Cargo.lock index 5182582e82..2035b5090c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2897,7 +2897,7 @@ checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" [[package]] name = "datafusion" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "arrow", "arrow-array", @@ -2948,7 +2948,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "arrow", "async-trait", @@ -2968,7 +2968,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "arrow", "arrow-schema", @@ -2991,7 +2991,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "ahash 0.8.11", "arrow", @@ -3016,7 +3016,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "log", "tokio", @@ -3025,12 +3025,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" [[package]] name = "datafusion-execution" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "arrow", "dashmap", @@ -3048,7 +3048,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "arrow", "chrono", @@ -3068,7 +3068,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "arrow", "datafusion-common", @@ -3079,7 +3079,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "arrow", "arrow-buffer", @@ -3108,7 +3108,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "ahash 0.8.11", "arrow", @@ -3129,7 +3129,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "ahash 0.8.11", "arrow", @@ -3141,7 +3141,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "arrow", "arrow-array", @@ -3163,7 +3163,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "arrow", "async-trait", @@ -3178,7 +3178,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "datafusion-common", "datafusion-doc", @@ -3194,7 +3194,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -3203,7 +3203,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "datafusion-expr", "quote", @@ -3213,7 +3213,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "arrow", "chrono", @@ -3231,7 +3231,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "ahash 0.8.11", "arrow", @@ -3254,7 +3254,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "ahash 0.8.11", "arrow", @@ -3267,7 +3267,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "arrow", "arrow-schema", @@ -3288,7 +3288,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "ahash 0.8.11", "arrow", @@ -3318,7 +3318,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "arrow", "arrow-array", @@ -3336,7 +3336,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=8ebed674dd71f8a466f658626877944cd16a4375#8ebed674dd71f8a466f658626877944cd16a4375" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "async-recursion", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index f3bd54a661..05835d88f3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -112,15 +112,15 @@ clap = { version = "4.4", features = ["derive"] } config = "0.13.0" crossbeam-utils = "0.8" dashmap = "6.1" -datafusion = { git = "https://github.com/apache/datafusion.git", rev = "8ebed674dd71f8a466f658626877944cd16a4375" } -datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "8ebed674dd71f8a466f658626877944cd16a4375" } -datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "8ebed674dd71f8a466f658626877944cd16a4375" } -datafusion-functions = { git = "https://github.com/apache/datafusion.git", rev = "8ebed674dd71f8a466f658626877944cd16a4375" } -datafusion-optimizer = { git = "https://github.com/apache/datafusion.git", rev = "8ebed674dd71f8a466f658626877944cd16a4375" } -datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git", rev = "8ebed674dd71f8a466f658626877944cd16a4375" } -datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git", rev = "8ebed674dd71f8a466f658626877944cd16a4375" } -datafusion-sql = { git = "https://github.com/apache/datafusion.git", rev = "8ebed674dd71f8a466f658626877944cd16a4375" } -datafusion-substrait = { git = "https://github.com/apache/datafusion.git", rev = "8ebed674dd71f8a466f658626877944cd16a4375" } +datafusion = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "5bbedc6704162afb03478f56ffb629405a4e1220" } +datafusion-common = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "5bbedc6704162afb03478f56ffb629405a4e1220" } +datafusion-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "5bbedc6704162afb03478f56ffb629405a4e1220" } +datafusion-functions = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "5bbedc6704162afb03478f56ffb629405a4e1220" } +datafusion-optimizer = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "5bbedc6704162afb03478f56ffb629405a4e1220" } +datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "5bbedc6704162afb03478f56ffb629405a4e1220" } +datafusion-physical-plan = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "5bbedc6704162afb03478f56ffb629405a4e1220" } +datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "5bbedc6704162afb03478f56ffb629405a4e1220" } +datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "5bbedc6704162afb03478f56ffb629405a4e1220" } deadpool = "0.12" deadpool-postgres = "0.14" derive_builder = "0.20" diff --git a/src/query/src/optimizer.rs b/src/query/src/optimizer.rs index e6596e923a..52a33029e2 100644 --- a/src/query/src/optimizer.rs +++ b/src/query/src/optimizer.rs @@ -21,6 +21,7 @@ pub mod scan_hint; pub mod string_normalization; #[cfg(test)] pub(crate) mod test_util; +pub mod transcribe_atat; pub mod type_conversion; pub mod windowed_sort; diff --git a/src/query/src/optimizer/transcribe_atat.rs b/src/query/src/optimizer/transcribe_atat.rs new file mode 100644 index 0000000000..3292f19f08 --- /dev/null +++ b/src/query/src/optimizer/transcribe_atat.rs @@ -0,0 +1,230 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use common_function::scalars::matches_term::MatchesTermFunction; +use common_function::scalars::udf::create_udf; +use common_function::state::FunctionState; +use datafusion::config::ConfigOptions; +use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRewriter}; +use datafusion_common::Result; +use datafusion_expr::expr::ScalarFunction; +use datafusion_expr::{Expr, LogicalPlan}; +use datafusion_optimizer::analyzer::AnalyzerRule; +use session::context::QueryContext; + +use crate::plan::ExtractExpr; + +/// TranscribeAtatRule is an analyzer rule that transcribes `@@` operator +/// to `matches_term` function. +/// +/// Example: +/// ```sql +/// SELECT matches_term('cat!', 'cat') as result; +/// +/// SELECT matches_term(`log_message`, '/start') as `matches_start` FROM t; +/// ``` +/// +/// to +/// +/// ```sql +/// SELECT 'cat!' @@ 'cat' as result; +/// +/// SELECT `log_message` @@ '/start' as `matches_start` FROM t; +/// ``` +#[derive(Debug)] +pub struct TranscribeAtatRule; + +impl AnalyzerRule for TranscribeAtatRule { + fn analyze(&self, plan: LogicalPlan, _config: &ConfigOptions) -> Result { + plan.transform(Self::do_analyze).map(|x| x.data) + } + + fn name(&self) -> &str { + "TranscribeAtatRule" + } +} + +impl TranscribeAtatRule { + fn do_analyze(plan: LogicalPlan) -> Result> { + let mut rewriter = TranscribeAtatRewriter::default(); + let new_expr = plan + .expressions_consider_join() + .into_iter() + .map(|e| e.rewrite(&mut rewriter).map(|x| x.data)) + .collect::>>()?; + + if rewriter.transcribed { + let inputs = plan.inputs().into_iter().cloned().collect::>(); + plan.with_new_exprs(new_expr, inputs).map(Transformed::yes) + } else { + Ok(Transformed::no(plan)) + } + } +} + +#[derive(Default)] +struct TranscribeAtatRewriter { + transcribed: bool, +} + +impl TreeNodeRewriter for TranscribeAtatRewriter { + type Node = Expr; + + fn f_up(&mut self, expr: Expr) -> Result> { + if let Expr::BinaryExpr(binary_expr) = &expr + && matches!(binary_expr.op, datafusion_expr::Operator::AtAt) + { + self.transcribed = true; + let scalar_udf = create_udf( + Arc::new(MatchesTermFunction), + QueryContext::arc(), + Arc::new(FunctionState::default()), + ); + let exprs = vec![ + binary_expr.left.as_ref().clone(), + binary_expr.right.as_ref().clone(), + ]; + Ok(Transformed::yes(Expr::ScalarFunction( + ScalarFunction::new_udf(Arc::new(scalar_udf), exprs), + ))) + } else { + Ok(Transformed::no(expr)) + } + } +} +#[cfg(test)] +mod tests { + + use arrow_schema::SchemaRef; + use datafusion::datasource::{provider_as_source, MemTable}; + use datafusion::logical_expr::{col, lit, LogicalPlan, LogicalPlanBuilder}; + use datafusion_expr::{BinaryExpr, Operator}; + use datatypes::arrow::datatypes::{DataType, Field, Schema}; + + use super::*; + + fn optimize(plan: LogicalPlan) -> Result { + TranscribeAtatRule.analyze(plan, &ConfigOptions::default()) + } + + fn prepare_test_plan_builder() -> LogicalPlanBuilder { + let schema = Schema::new(vec![ + Field::new("a", DataType::Utf8, false), + Field::new("b", DataType::Utf8, false), + ]); + let table = MemTable::try_new(SchemaRef::from(schema), vec![]).unwrap(); + LogicalPlanBuilder::scan("t", provider_as_source(Arc::new(table)), None).unwrap() + } + + #[test] + fn test_multiple_atat() { + let plan = prepare_test_plan_builder() + .filter( + Expr::BinaryExpr(BinaryExpr { + left: Box::new(col("a")), + op: Operator::AtAt, + right: Box::new(lit("foo")), + }) + .and(Expr::BinaryExpr(BinaryExpr { + left: Box::new(col("b")), + op: Operator::AtAt, + right: Box::new(lit("bar")), + })), + ) + .unwrap() + .project(vec![ + Expr::BinaryExpr(BinaryExpr { + left: Box::new(col("a")), + op: Operator::AtAt, + right: Box::new(col("b")), + }), + col("b"), + ]) + .unwrap() + .build() + .unwrap(); + + let expected = r#"Projection: matches_term(t.a, t.b), t.b + Filter: matches_term(t.a, Utf8("foo")) AND matches_term(t.b, Utf8("bar")) + TableScan: t"#; + + let optimized_plan = optimize(plan).unwrap(); + let formatted = optimized_plan.to_string(); + + assert_eq!(formatted, expected); + } + + #[test] + fn test_nested_atat() { + let plan = prepare_test_plan_builder() + .filter( + Expr::BinaryExpr(BinaryExpr { + left: Box::new(col("a")), + op: Operator::AtAt, + right: Box::new(lit("foo")), + }) + .and( + Expr::BinaryExpr(BinaryExpr { + left: Box::new(col("b")), + op: Operator::AtAt, + right: Box::new(lit("bar")), + }) + .or(Expr::BinaryExpr(BinaryExpr { + left: Box::new( + // Nested case in function argument + Expr::BinaryExpr(BinaryExpr { + left: Box::new(col("a")), + op: Operator::AtAt, + right: Box::new(lit("nested")), + }), + ), + op: Operator::Eq, + right: Box::new(lit(true)), + })), + ), + ) + .unwrap() + .project(vec![ + col("a"), + // Complex nested expression with multiple @@ operators + Expr::BinaryExpr(BinaryExpr { + left: Box::new(Expr::BinaryExpr(BinaryExpr { + left: Box::new(col("a")), + op: Operator::AtAt, + right: Box::new(lit("foo")), + })), + op: Operator::And, + right: Box::new(Expr::BinaryExpr(BinaryExpr { + left: Box::new(col("b")), + op: Operator::AtAt, + right: Box::new(lit("bar")), + })), + }), + ]) + .unwrap() + .build() + .unwrap(); + + let expected = r#"Projection: t.a, matches_term(t.a, Utf8("foo")) AND matches_term(t.b, Utf8("bar")) + Filter: matches_term(t.a, Utf8("foo")) AND (matches_term(t.b, Utf8("bar")) OR matches_term(t.a, Utf8("nested")) = Boolean(true)) + TableScan: t"#; + + let optimized_plan = optimize(plan).unwrap(); + let formatted = optimized_plan.to_string(); + + assert_eq!(formatted, expected); + } +} diff --git a/src/query/src/query_engine/state.rs b/src/query/src/query_engine/state.rs index 03f3a2a13d..d55ab471f9 100644 --- a/src/query/src/query_engine/state.rs +++ b/src/query/src/query_engine/state.rs @@ -52,6 +52,7 @@ use crate::optimizer::pass_distribution::PassDistribution; use crate::optimizer::remove_duplicate::RemoveDuplicate; use crate::optimizer::scan_hint::ScanHintRule; use crate::optimizer::string_normalization::StringNormalizationRule; +use crate::optimizer::transcribe_atat::TranscribeAtatRule; use crate::optimizer::type_conversion::TypeConversionRule; use crate::optimizer::windowed_sort::WindowedSortPhysicalRule; use crate::optimizer::ExtensionAnalyzerRule; @@ -115,6 +116,7 @@ impl QueryEngineState { // Apply the datafusion rules let mut analyzer = Analyzer::new(); + analyzer.rules.insert(0, Arc::new(TranscribeAtatRule)); analyzer.rules.insert(0, Arc::new(StringNormalizationRule)); // Use our custom rule instead to optimize the count(*) query diff --git a/tests/cases/standalone/common/expr/atat.result b/tests/cases/standalone/common/expr/atat.result new file mode 100644 index 0000000000..6beec6347a --- /dev/null +++ b/tests/cases/standalone/common/expr/atat.result @@ -0,0 +1,315 @@ +-- Derived from matches_term cases +-- Test basic term matching +-- Expect: true +SELECT 'cat!' @@ 'cat' as result; + ++--------+ +| result | ++--------+ +| true | ++--------+ + +-- Test phrase matching with spaces +-- Expect: true +SELECT 'warning:hello world!' @@ 'hello world' as result; + ++--------+ +| result | ++--------+ +| true | ++--------+ + +-- Test numbers in term +SELECT 'v1.0!' @@ 'v1.0' as result; + ++--------+ +| result | ++--------+ +| true | ++--------+ + +-- Test case sensitivity +-- Expect: true +SELECT 'Cat' @@ 'Cat' as result; + ++--------+ +| result | ++--------+ +| true | ++--------+ + +-- Expect: false +SELECT 'cat' @@ 'Cat' as result; + ++--------+ +| result | ++--------+ +| false | ++--------+ + +-- Test empty string handling +-- Expect: true +SELECT '' @@ '' as result; + ++--------+ +| result | ++--------+ +| true | ++--------+ + +-- Expect: false +SELECT 'any' @@ '' as result; + ++--------+ +| result | ++--------+ +| false | ++--------+ + +-- Expect: false +SELECT '' @@ 'any' as result; + ++--------+ +| result | ++--------+ +| false | ++--------+ + +-- Test partial matches (should fail) +-- Expect: false +SELECT 'category' @@ 'cat' as result; + ++--------+ +| result | ++--------+ +| false | ++--------+ + +-- Expect: false +SELECT 'rebooted' @@ 'boot' as result; + ++--------+ +| result | ++--------+ +| false | ++--------+ + +-- Test adjacent alphanumeric characters +SELECT 'cat5' @@ 'cat' as result; + ++--------+ +| result | ++--------+ +| false | ++--------+ + +SELECT 'dogcat' @@ 'dog' as result; + ++--------+ +| result | ++--------+ +| false | ++--------+ + +-- Test leading non-alphanumeric +-- Expect: true +SELECT 'dog/cat' @@ '/cat' as result; + ++--------+ +| result | ++--------+ +| true | ++--------+ + +-- Expect: true +SELECT 'dog/cat' @@ 'dog/' as result; + ++--------+ +| result | ++--------+ +| true | ++--------+ + +-- Expect: true +SELECT 'dog/cat' @@ 'dog/cat' as result; + ++--------+ +| result | ++--------+ +| true | ++--------+ + +-- Test unicode characters +-- Expect: true +SELECT 'café>' @@ 'café' as result; + ++--------+ +| result | ++--------+ +| true | ++--------+ + +-- Expect: true +SELECT 'русский!' @@ 'русский' as result; + ++--------+ +| result | ++--------+ +| true | ++--------+ + +-- Test complete word matching +CREATE TABLE logs ( + `id` TIMESTAMP TIME INDEX, + `log_message` STRING +); + +Affected Rows: 0 + +INSERT INTO logs VALUES + (1, 'An error occurred!'), + (2, 'Critical error: system failure'), + (3, 'error-prone'), + (4, 'errors'), + (5, 'error123'), + (6, 'errorLogs'), + (7, 'Version v1.0 released'), + (8, 'v1.0!'), + (9, 'v1.0a'), + (10, 'v1.0beta'), + (11, 'GET /app/start'), + (12, 'Command: /start-prosess'), + (13, 'Command: /start'), + (14, 'start'), + (15, 'start/stop'), + (16, 'Alert: system failure detected'), + (17, 'system failure!'), + (18, 'system-failure'), + (19, 'system failure2023'), + (20, 'critical error: system failure'), + (21, 'critical failure detected'), + (22, 'critical issue'), + (23, 'failure imminent'), + (24, 'Warning: high temperature'), + (25, 'WARNING: system overload'), + (26, 'warned'), + (27, 'warnings'); + +Affected Rows: 27 + +-- Test complete word matching for 'error' +-- Expect: +-- 1|An error occurred!|true +-- 2|Critical error: system failure|true +-- 3|error-prone|true +-- 4|errors|false +-- 5|error123|false +-- 6|errorLogs|false +SELECT `id`, `log_message`, `log_message` @@ 'error' as `matches_error` FROM logs WHERE `id` <= 6 ORDER BY `id`; + ++-------------------------+--------------------------------+---------------+ +| id | log_message | matches_error | ++-------------------------+--------------------------------+---------------+ +| 1970-01-01T00:00:00.001 | An error occurred! | true | +| 1970-01-01T00:00:00.002 | Critical error: system failure | true | +| 1970-01-01T00:00:00.003 | error-prone | true | +| 1970-01-01T00:00:00.004 | errors | false | +| 1970-01-01T00:00:00.005 | error123 | false | +| 1970-01-01T00:00:00.006 | errorLogs | false | ++-------------------------+--------------------------------+---------------+ + +-- Test complete word matching for 'v1.0' +-- Expect: +-- 7|Version v1.0 released|true +-- 8|v1.0!|true +-- 9|v1.0a|false +-- 10|v1.0beta|false +SELECT `id`, `log_message`, `log_message` @@ 'v1.0' as `matches_version` FROM logs WHERE `id` BETWEEN 7 AND 10 ORDER BY `id`; + ++-------------------------+-----------------------+-----------------+ +| id | log_message | matches_version | ++-------------------------+-----------------------+-----------------+ +| 1970-01-01T00:00:00.007 | Version v1.0 released | true | +| 1970-01-01T00:00:00.008 | v1.0! | true | +| 1970-01-01T00:00:00.009 | v1.0a | false | +| 1970-01-01T00:00:00.010 | v1.0beta | false | ++-------------------------+-----------------------+-----------------+ + +-- Test complete word matching for '/start' +-- Expect: +-- 11|GET /app/start|true +-- 12|Command: /start-prosess|true +-- 13|Command: /start|true +-- 14|start|false +-- 15|start/stop|false +SELECT `id`, `log_message`, `log_message` @@ '/start' as `matches_start` FROM logs WHERE `id` BETWEEN 11 AND 15 ORDER BY `id`; + ++-------------------------+-------------------------+---------------+ +| id | log_message | matches_start | ++-------------------------+-------------------------+---------------+ +| 1970-01-01T00:00:00.011 | GET /app/start | true | +| 1970-01-01T00:00:00.012 | Command: /start-prosess | true | +| 1970-01-01T00:00:00.013 | Command: /start | true | +| 1970-01-01T00:00:00.014 | start | false | +| 1970-01-01T00:00:00.015 | start/stop | false | ++-------------------------+-------------------------+---------------+ + +-- Test phrase matching for 'system failure' +-- Expect: +-- 16|Alert: system failure detected|true +-- 17|system failure!|true +-- 18|system-failure|false +-- 19|system failure2023|false +SELECT `id`, `log_message`, `log_message` @@ 'system failure' as `matches_phrase` FROM logs WHERE `id` BETWEEN 16 AND 19 ORDER BY `id`; + ++-------------------------+--------------------------------+----------------+ +| id | log_message | matches_phrase | ++-------------------------+--------------------------------+----------------+ +| 1970-01-01T00:00:00.016 | Alert: system failure detected | true | +| 1970-01-01T00:00:00.017 | system failure! | true | +| 1970-01-01T00:00:00.018 | system-failure | false | +| 1970-01-01T00:00:00.019 | system failure2023 | false | ++-------------------------+--------------------------------+----------------+ + +-- Test multi-word matching using AND +-- Expect: +-- 20|critical error: system failure|true|true|true +-- 21|critical failure detected|true|true|true +-- 22|critical issue|true|false|false +-- 23|failure imminent|false|true|false +SELECT `id`, `log_message`, + `log_message` @@ 'critical' as `matches_critical`, + `log_message` @@ 'failure' as `matches_failure`, + `log_message` @@ 'critical' AND `log_message` @@ 'failure' as `matches_both` +FROM logs WHERE `id` BETWEEN 20 AND 23 ORDER BY `id`; + ++-------------------------+--------------------------------+------------------+-----------------+--------------+ +| id | log_message | matches_critical | matches_failure | matches_both | ++-------------------------+--------------------------------+------------------+-----------------+--------------+ +| 1970-01-01T00:00:00.020 | critical error: system failure | true | true | true | +| 1970-01-01T00:00:00.021 | critical failure detected | true | true | true | +| 1970-01-01T00:00:00.022 | critical issue | true | false | false | +| 1970-01-01T00:00:00.023 | failure imminent | false | true | false | ++-------------------------+--------------------------------+------------------+-----------------+--------------+ + +-- Test case-insensitive matching using lower() +-- Expect: +-- 24|Warning: high temperature|true +-- 25|WARNING: system overload|true +-- 26|warned|false +-- 27|warnings|false +SELECT `id`, `log_message`, lower(`log_message`) @@ 'warning' as `matches_warning` FROM logs WHERE `id` >= 24 ORDER BY `id`; + ++-------------------------+---------------------------+-----------------+ +| id | log_message | matches_warning | ++-------------------------+---------------------------+-----------------+ +| 1970-01-01T00:00:00.024 | Warning: high temperature | true | +| 1970-01-01T00:00:00.025 | WARNING: system overload | true | +| 1970-01-01T00:00:00.026 | warned | false | +| 1970-01-01T00:00:00.027 | warnings | false | ++-------------------------+---------------------------+-----------------+ + +DROP TABLE logs; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/expr/atat.sql b/tests/cases/standalone/common/expr/atat.sql new file mode 100644 index 0000000000..da32dcf0bf --- /dev/null +++ b/tests/cases/standalone/common/expr/atat.sql @@ -0,0 +1,144 @@ +-- Derived from matches_term cases + +-- Test basic term matching +-- Expect: true +SELECT 'cat!' @@ 'cat' as result; + +-- Test phrase matching with spaces +-- Expect: true +SELECT 'warning:hello world!' @@ 'hello world' as result; + +-- Test numbers in term +SELECT 'v1.0!' @@ 'v1.0' as result; + +-- Test case sensitivity +-- Expect: true +SELECT 'Cat' @@ 'Cat' as result; +-- Expect: false +SELECT 'cat' @@ 'Cat' as result; + +-- Test empty string handling +-- Expect: true +SELECT '' @@ '' as result; +-- Expect: false +SELECT 'any' @@ '' as result; +-- Expect: false +SELECT '' @@ 'any' as result; + +-- Test partial matches (should fail) +-- Expect: false +SELECT 'category' @@ 'cat' as result; +-- Expect: false +SELECT 'rebooted' @@ 'boot' as result; + +-- Test adjacent alphanumeric characters +SELECT 'cat5' @@ 'cat' as result; +SELECT 'dogcat' @@ 'dog' as result; + +-- Test leading non-alphanumeric +-- Expect: true +SELECT 'dog/cat' @@ '/cat' as result; +-- Expect: true +SELECT 'dog/cat' @@ 'dog/' as result; +-- Expect: true +SELECT 'dog/cat' @@ 'dog/cat' as result; + +-- Test unicode characters +-- Expect: true +SELECT 'café>' @@ 'café' as result; +-- Expect: true +SELECT 'русский!' @@ 'русский' as result; + +-- Test complete word matching +CREATE TABLE logs ( + `id` TIMESTAMP TIME INDEX, + `log_message` STRING +); + +INSERT INTO logs VALUES + (1, 'An error occurred!'), + (2, 'Critical error: system failure'), + (3, 'error-prone'), + (4, 'errors'), + (5, 'error123'), + (6, 'errorLogs'), + (7, 'Version v1.0 released'), + (8, 'v1.0!'), + (9, 'v1.0a'), + (10, 'v1.0beta'), + (11, 'GET /app/start'), + (12, 'Command: /start-prosess'), + (13, 'Command: /start'), + (14, 'start'), + (15, 'start/stop'), + (16, 'Alert: system failure detected'), + (17, 'system failure!'), + (18, 'system-failure'), + (19, 'system failure2023'), + (20, 'critical error: system failure'), + (21, 'critical failure detected'), + (22, 'critical issue'), + (23, 'failure imminent'), + (24, 'Warning: high temperature'), + (25, 'WARNING: system overload'), + (26, 'warned'), + (27, 'warnings'); + +-- Test complete word matching for 'error' +-- Expect: +-- 1|An error occurred!|true +-- 2|Critical error: system failure|true +-- 3|error-prone|true +-- 4|errors|false +-- 5|error123|false +-- 6|errorLogs|false +SELECT `id`, `log_message`, `log_message` @@ 'error' as `matches_error` FROM logs WHERE `id` <= 6 ORDER BY `id`; + + +-- Test complete word matching for 'v1.0' +-- Expect: +-- 7|Version v1.0 released|true +-- 8|v1.0!|true +-- 9|v1.0a|false +-- 10|v1.0beta|false +SELECT `id`, `log_message`, `log_message` @@ 'v1.0' as `matches_version` FROM logs WHERE `id` BETWEEN 7 AND 10 ORDER BY `id`; + +-- Test complete word matching for '/start' +-- Expect: +-- 11|GET /app/start|true +-- 12|Command: /start-prosess|true +-- 13|Command: /start|true +-- 14|start|false +-- 15|start/stop|false +SELECT `id`, `log_message`, `log_message` @@ '/start' as `matches_start` FROM logs WHERE `id` BETWEEN 11 AND 15 ORDER BY `id`; + +-- Test phrase matching for 'system failure' +-- Expect: +-- 16|Alert: system failure detected|true +-- 17|system failure!|true +-- 18|system-failure|false +-- 19|system failure2023|false +SELECT `id`, `log_message`, `log_message` @@ 'system failure' as `matches_phrase` FROM logs WHERE `id` BETWEEN 16 AND 19 ORDER BY `id`; + + +-- Test multi-word matching using AND +-- Expect: +-- 20|critical error: system failure|true|true|true +-- 21|critical failure detected|true|true|true +-- 22|critical issue|true|false|false +-- 23|failure imminent|false|true|false +SELECT `id`, `log_message`, + `log_message` @@ 'critical' as `matches_critical`, + `log_message` @@ 'failure' as `matches_failure`, + `log_message` @@ 'critical' AND `log_message` @@ 'failure' as `matches_both` +FROM logs WHERE `id` BETWEEN 20 AND 23 ORDER BY `id`; + +-- Test case-insensitive matching using lower() +-- Expect: +-- 24|Warning: high temperature|true +-- 25|WARNING: system overload|true +-- 26|warned|false +-- 27|warnings|false +SELECT `id`, `log_message`, lower(`log_message`) @@ 'warning' as `matches_warning` FROM logs WHERE `id` >= 24 ORDER BY `id`; + +DROP TABLE logs; diff --git a/tests/cases/standalone/common/tql-explain-analyze/explain.result b/tests/cases/standalone/common/tql-explain-analyze/explain.result index 8b4952ed3d..200ec5c814 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/explain.result +++ b/tests/cases/standalone/common/tql-explain-analyze/explain.result @@ -80,6 +80,7 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test; |_|_TableScan: test_| | logical_plan after count_wildcard_to_time_index_rule_| SAME TEXT AS ABOVE_| | logical_plan after StringNormalizationRule_| SAME TEXT AS ABOVE_| +| logical_plan after TranscribeAtatRule_| SAME TEXT AS ABOVE_| | logical_plan after inline_table_scan_| SAME TEXT AS ABOVE_| | logical_plan after expand_wildcard_rule_| SAME TEXT AS ABOVE_| | logical_plan after resolve_grouping_function_| SAME TEXT AS ABOVE_| From 799c7cbfa97721ca6e3e16d29df65b95bb453b58 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" <6406592+v0y4g3r@users.noreply.github.com> Date: Tue, 15 Apr 2025 22:11:50 +0800 Subject: [PATCH 25/82] feat(mito): bulk insert request handling on datanode (#5831) * wip: implement basic request handling * feat/bulk-insert: ### Add Error Handling and Enhance Bulk Insert Functionality - **Error Handling**: Introduced a new error variant `ConvertDataType` in `error.rs` to handle conversion failures from `ConcreteDataType` to `ColumnDataType`. - **Bulk Insert Enhancements**: - Updated `WorkerRequest::BulkInserts` in `request.rs` to include metadata and sender. - Implemented `handle_bulk_inserts` in `worker.rs` to process bulk insert requests with region metadata. - Added functions `region_metadata_to_column_schema` and `record_batch_to_rows` in `handle_bulk_insert.rs` for schema conversion and row processing. - **API Changes**: Modified `RegionBulkInsertsRequest` in `region_request.rs` to include `region_id`. Files affected: `error.rs`, `request.rs`, `worker.rs`, `handle_bulk_insert.rs`, `region_request.rs`. * feat/bulk-insert: **Enhance Error Handling and Add Unit Tests** - Improved error handling in `record_batch_to_rows` function within `handle_bulk_insert.rs` by returning `Result` and handling errors with `context`. - Added unit tests for `region_metadata_to_column_schema` and `record_batch_to_rows` functions in `handle_bulk_insert.rs` to ensure correct functionality and error handling. * chore: update proto version * feat/bulk-insert: - **Refactor Error Handling**: Updated error handling in `error.rs` by modifying the `ConvertDataType` error handling. - **Improve Logging and Error Reporting**: Enhanced logging and error reporting in `worker.rs` by adding error messages for missing region metadata. - **Add New Error Type**: Introduced `DecodeArrowIpc` error in `metadata.rs` to handle Arrow IPC decoding failures. - **Handle Arrow IPC Decoding**: Updated `region_request.rs` to handle Arrow IPC decoding errors using the new `DecodeArrowIpc` error type. * chore: update proto version * feat/bulk-insert: Refactor `handle_bulk_insert.rs` to simplify row construction - Removed the mutable `current_row` vector and refactored `row_at` function to return a new vector directly. - Updated `record_batch_to_rows` to utilize the refactored `row_at` function for constructing rows. * feat/bulk-insert: ### Commit Summary **Enhancements in Region Server Request Handling** - Updated `region_server.rs` to include `RegionRequest::BulkInserts(_)` in the `RegionChange::Ingest` category, improving the handling of bulk insert operations. - Refined the categorization of region requests to ensure accurate mapping to `RegionChange` actions. --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/datanode/src/region_server.rs | 4 +- src/metric-engine/src/engine.rs | 4 + src/mito2/src/error.rs | 12 + src/mito2/src/request.rs | 18 +- src/mito2/src/worker.rs | 25 ++- src/mito2/src/worker/handle_bulk_insert.rs | 247 +++++++++++++++++++++ src/mito2/src/worker/handle_write.rs | 2 +- src/store-api/src/lib.rs | 4 +- src/store-api/src/metadata.rs | 9 + src/store-api/src/region_request.rs | 76 ++++++- 12 files changed, 387 insertions(+), 18 deletions(-) create mode 100644 src/mito2/src/worker/handle_bulk_insert.rs diff --git a/Cargo.lock b/Cargo.lock index 2035b5090c..62d14792ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4722,7 +4722,7 @@ dependencies = [ [[package]] name = "greptime-proto" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=dd4a1996982534636734674db66e44464b0c0d83#dd4a1996982534636734674db66e44464b0c0d83" +source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=583daa3fbbbe39c90b7b92d13646bc3291d9c941#583daa3fbbbe39c90b7b92d13646bc3291d9c941" dependencies = [ "prost 0.13.5", "serde", diff --git a/Cargo.toml b/Cargo.toml index 05835d88f3..6a0e59c0a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -129,7 +129,7 @@ etcd-client = "0.14" fst = "0.4.7" futures = "0.3" futures-util = "0.3" -greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "dd4a1996982534636734674db66e44464b0c0d83" } +greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "583daa3fbbbe39c90b7b92d13646bc3291d9c941" } hex = "0.4" http = "1" humantime = "2.1" diff --git a/src/datanode/src/region_server.rs b/src/datanode/src/region_server.rs index bff28c109b..072c1682cc 100644 --- a/src/datanode/src/region_server.rs +++ b/src/datanode/src/region_server.rs @@ -902,7 +902,9 @@ impl RegionServerInner { RegionChange::Register(attribute) } RegionRequest::Close(_) | RegionRequest::Drop(_) => RegionChange::Deregisters, - RegionRequest::Put(_) | RegionRequest::Delete(_) => RegionChange::Ingest, + RegionRequest::Put(_) | RegionRequest::Delete(_) | RegionRequest::BulkInserts(_) => { + RegionChange::Ingest + } RegionRequest::Alter(_) | RegionRequest::Flush(_) | RegionRequest::Compact(_) diff --git a/src/metric-engine/src/engine.rs b/src/metric-engine/src/engine.rs index 509438b4b2..71cb843ab1 100644 --- a/src/metric-engine/src/engine.rs +++ b/src/metric-engine/src/engine.rs @@ -221,6 +221,10 @@ impl RegionEngine for MetricEngine { } } RegionRequest::Catchup(req) => self.inner.catchup_region(region_id, req).await, + RegionRequest::BulkInserts(_) => { + // todo(hl): find a way to support bulk inserts in metric engine. + UnsupportedRegionRequestSnafu { request }.fail() + } }; result.map_err(BoxedError::new).map(|rows| RegionResponse { diff --git a/src/mito2/src/error.rs b/src/mito2/src/error.rs index 9c9c78f07e..0ac04e8b3e 100644 --- a/src/mito2/src/error.rs +++ b/src/mito2/src/error.rs @@ -1021,6 +1021,17 @@ pub enum Error { #[snafu(implicit)] location: Location, }, + + #[snafu(display( + "Failed to convert ConcreteDataType to ColumnDataType: {:?}", + data_type + ))] + ConvertDataType { + data_type: ConcreteDataType, + source: api::error::Error, + #[snafu(implicit)] + location: Location, + }, } pub type Result = std::result::Result; @@ -1172,6 +1183,7 @@ impl ErrorExt for Error { ManualCompactionOverride {} => StatusCode::Cancelled, IncompatibleWalProviderChange { .. } => StatusCode::InvalidArguments, + ConvertDataType { .. } => StatusCode::Internal, } } diff --git a/src/mito2/src/request.rs b/src/mito2/src/request.rs index 33a8f13f07..5331ba6fdc 100644 --- a/src/mito2/src/request.rs +++ b/src/mito2/src/request.rs @@ -35,9 +35,9 @@ use store_api::manifest::ManifestVersion; use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataRef}; use store_api::region_engine::{SetRegionRoleStateResponse, SettableRegionRoleState}; use store_api::region_request::{ - AffectedRows, RegionAlterRequest, RegionCatchupRequest, RegionCloseRequest, - RegionCompactRequest, RegionCreateRequest, RegionFlushRequest, RegionOpenRequest, - RegionRequest, RegionTruncateRequest, + AffectedRows, RegionAlterRequest, RegionBulkInsertsRequest, RegionCatchupRequest, + RegionCloseRequest, RegionCompactRequest, RegionCreateRequest, RegionFlushRequest, + RegionOpenRequest, RegionRequest, RegionTruncateRequest, }; use store_api::storage::{RegionId, SequenceNumber}; use tokio::sync::oneshot::{self, Receiver, Sender}; @@ -569,6 +569,13 @@ pub(crate) enum WorkerRequest { /// Keep the manifest of a region up to date. SyncRegion(RegionSyncRequest), + + /// Bulk inserts request and region metadata. + BulkInserts { + metadata: Option, + request: RegionBulkInsertsRequest, + sender: OptionOutputTx, + }, } impl WorkerRequest { @@ -668,6 +675,11 @@ impl WorkerRequest { sender: sender.into(), request: DdlRequest::Catchup(v), }), + RegionRequest::BulkInserts(region_bulk_inserts_request) => WorkerRequest::BulkInserts { + metadata: region_metadata, + sender: sender.into(), + request: region_bulk_inserts_request, + }, }; Ok((worker_request, receiver)) diff --git a/src/mito2/src/worker.rs b/src/mito2/src/worker.rs index 7aee65b651..33edb186d4 100644 --- a/src/mito2/src/worker.rs +++ b/src/mito2/src/worker.rs @@ -15,6 +15,7 @@ //! Structs and utilities for writing regions. mod handle_alter; +mod handle_bulk_insert; mod handle_catchup; mod handle_close; mod handle_compaction; @@ -25,6 +26,7 @@ mod handle_manifest; mod handle_open; mod handle_truncate; mod handle_write; + use std::collections::HashMap; use std::path::Path; use std::sync::atomic::{AtomicBool, Ordering}; @@ -52,6 +54,7 @@ use crate::cache::write_cache::{WriteCache, WriteCacheRef}; use crate::cache::{CacheManager, CacheManagerRef}; use crate::compaction::CompactionScheduler; use crate::config::MitoConfig; +use crate::error; use crate::error::{CreateDirSnafu, JoinSnafu, Result, WorkerStoppedSnafu}; use crate::flush::{FlushScheduler, WriteBufferManagerImpl, WriteBufferManagerRef}; use crate::memtable::MemtableBuilderProvider; @@ -820,16 +823,30 @@ impl RegionWorkerLoop { WorkerRequest::EditRegion(request) => { self.handle_region_edit(request).await; } - // We receive a stop signal, but we still want to process remaining - // requests. The worker thread will then check the running flag and - // then exit. WorkerRequest::Stop => { debug_assert!(!self.running.load(Ordering::Relaxed)); } - WorkerRequest::SyncRegion(req) => { self.handle_region_sync(req).await; } + WorkerRequest::BulkInserts { + metadata, + request, + sender, + } => { + if let Some(region_metadata) = metadata { + self.handle_bulk_inserts(request, region_metadata, write_requests, sender) + .await; + } else { + error!("Cannot find region metadata for {}", request.region_id); + sender.send( + error::RegionNotFoundSnafu { + region_id: request.region_id, + } + .fail(), + ); + } + } } } diff --git a/src/mito2/src/worker/handle_bulk_insert.rs b/src/mito2/src/worker/handle_bulk_insert.rs new file mode 100644 index 0000000000..9f2a745835 --- /dev/null +++ b/src/mito2/src/worker/handle_bulk_insert.rs @@ -0,0 +1,247 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Handles bulk insert requests. + +use api::helper::{value_to_grpc_value, ColumnDataTypeWrapper}; +use api::v1::{ColumnSchema, OpType, Row, Rows}; +use common_recordbatch::DfRecordBatch; +use datatypes::prelude::VectorRef; +use datatypes::vectors::Helper; +use snafu::ResultExt; +use store_api::logstore::LogStore; +use store_api::metadata::RegionMetadataRef; +use store_api::region_request::{BulkInsertPayload, RegionBulkInsertsRequest}; + +use crate::error; +use crate::request::{OptionOutputTx, SenderWriteRequest, WriteRequest}; +use crate::worker::RegionWorkerLoop; + +impl RegionWorkerLoop { + pub(crate) async fn handle_bulk_inserts( + &mut self, + request: RegionBulkInsertsRequest, + region_metadata: RegionMetadataRef, + pending_write_requests: &mut Vec, + sender: OptionOutputTx, + ) { + let schema = match region_metadata_to_column_schema(®ion_metadata) { + Ok(schema) => schema, + Err(e) => { + sender.send(Err(e)); + return; + } + }; + let mut pending_tasks = Vec::with_capacity(request.payloads.len()); + for req in request.payloads { + match req { + BulkInsertPayload::ArrowIpc(df_record_batch) => { + let rows = match record_batch_to_rows(®ion_metadata, &df_record_batch) { + Ok(rows) => rows, + Err(e) => { + sender.send(Err(e)); + return; + } + }; + + let write_request = match WriteRequest::new( + region_metadata.region_id, + OpType::Put, + Rows { + schema: schema.clone(), + rows, + }, + Some(region_metadata.clone()), + ) { + Ok(write_request) => write_request, + Err(e) => { + sender.send(Err(e)); + return; + } + }; + + let (tx, rx) = tokio::sync::oneshot::channel(); + let sender = OptionOutputTx::from(tx); + let req = SenderWriteRequest { + sender, + request: write_request, + }; + pending_tasks.push(rx); + pending_write_requests.push(req); + } + } + } + + common_runtime::spawn_global(async move { + let results = match futures::future::try_join_all(pending_tasks).await { + Ok(results) => results, + Err(e) => { + sender.send(Err(e).context(error::RecvSnafu)); + return; + } + }; + let result1 = match results.into_iter().collect::>>() { + Ok(results) => Ok(results.into_iter().sum()), + Err(e) => Err(e), + }; + sender.send(result1); + }); + } +} + +fn region_metadata_to_column_schema( + region_meta: &RegionMetadataRef, +) -> error::Result> { + region_meta + .column_metadatas + .iter() + .map(|c| { + let wrapper = ColumnDataTypeWrapper::try_from(c.column_schema.data_type.clone()) + .with_context(|_| error::ConvertDataTypeSnafu { + data_type: c.column_schema.data_type.clone(), + })?; + + Ok(ColumnSchema { + column_name: c.column_schema.name.clone(), + datatype: wrapper.datatype() as i32, + semantic_type: c.semantic_type as i32, + ..Default::default() + }) + }) + .collect::>() +} + +/// Convert [DfRecordBatch] to gRPC rows. +fn record_batch_to_rows( + region_metadata: &RegionMetadataRef, + rb: &DfRecordBatch, +) -> error::Result> { + let num_rows = rb.num_rows(); + let mut rows = Vec::with_capacity(num_rows); + if num_rows == 0 { + return Ok(rows); + } + let vectors: Vec> = region_metadata + .column_metadatas + .iter() + .map(|c| { + rb.column_by_name(&c.column_schema.name) + .map(|column| Helper::try_into_vector(column).context(error::ConvertVectorSnafu)) + .transpose() + }) + .collect::>()?; + + for row_idx in 0..num_rows { + let row = Row { + values: row_at(&vectors, row_idx), + }; + rows.push(row); + } + Ok(rows) +} + +fn row_at(vectors: &[Option], row_idx: usize) -> Vec { + let mut row = Vec::with_capacity(vectors.len()); + for a in vectors { + let value = if let Some(a) = a { + value_to_grpc_value(a.get(row_idx)) + } else { + api::v1::Value { value_data: None } + }; + row.push(value) + } + row +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use api::v1::SemanticType; + use datatypes::arrow::array::{Int64Array, TimestampMillisecondArray}; + + use super::*; + use crate::test_util::meta_util::TestRegionMetadataBuilder; + + fn build_record_batch(num_rows: usize) -> DfRecordBatch { + let region_metadata = Arc::new(TestRegionMetadataBuilder::default().build()); + let schema = region_metadata.schema.arrow_schema().clone(); + let values = (0..num_rows).map(|v| v as i64).collect::>(); + let ts_array = Arc::new(TimestampMillisecondArray::from_iter_values(values.clone())); + let k0_array = Arc::new(Int64Array::from_iter_values(values.clone())); + let v0_array = Arc::new(Int64Array::from_iter_values(values)); + DfRecordBatch::try_new(schema, vec![ts_array, k0_array, v0_array]).unwrap() + } + + #[test] + fn test_region_metadata_to_column_schema() { + let region_metadata = Arc::new(TestRegionMetadataBuilder::default().build()); + let result = region_metadata_to_column_schema(®ion_metadata).unwrap(); + assert_eq!(result.len(), 3); + + assert_eq!(result[0].column_name, "ts"); + assert_eq!(result[0].semantic_type, SemanticType::Timestamp as i32); + + assert_eq!(result[1].column_name, "k0"); + assert_eq!(result[1].semantic_type, SemanticType::Tag as i32); + + assert_eq!(result[2].column_name, "v0"); + assert_eq!(result[2].semantic_type, SemanticType::Field as i32); + } + + #[test] + fn test_record_batch_to_rows() { + // Create record batch + let region_metadata = Arc::new(TestRegionMetadataBuilder::default().build()); + let record_batch = build_record_batch(10); + let rows = record_batch_to_rows(®ion_metadata, &record_batch).unwrap(); + + assert_eq!(rows.len(), 10); + assert_eq!(rows[0].values.len(), 3); + + for (row_idx, row) in rows.iter().enumerate().take(10) { + assert_eq!( + row.values[0].value_data.as_ref().unwrap(), + &api::v1::value::ValueData::TimestampMillisecondValue(row_idx as i64) + ); + } + } + + #[test] + fn test_record_batch_to_rows_schema_mismatch() { + let region_metadata = Arc::new(TestRegionMetadataBuilder::default().num_fields(2).build()); + let record_batch = build_record_batch(1); + + let rows = record_batch_to_rows(®ion_metadata, &record_batch).unwrap(); + assert_eq!(rows.len(), 1); + + // Check first row + let row1 = &rows[0]; + assert_eq!(row1.values.len(), 4); + assert_eq!( + row1.values[0].value_data.as_ref().unwrap(), + &api::v1::value::ValueData::TimestampMillisecondValue(0) + ); + assert_eq!( + row1.values[1].value_data.as_ref().unwrap(), + &api::v1::value::ValueData::I64Value(0) + ); + assert_eq!( + row1.values[2].value_data.as_ref().unwrap(), + &api::v1::value::ValueData::I64Value(0) + ); + + assert!(row1.values[3].value_data.is_none()); + } +} diff --git a/src/mito2/src/worker/handle_write.rs b/src/mito2/src/worker/handle_write.rs index a5dafcdd3a..b6e8783e1e 100644 --- a/src/mito2/src/worker/handle_write.rs +++ b/src/mito2/src/worker/handle_write.rs @@ -299,7 +299,7 @@ impl RegionWorkerLoop { } /// Returns true if the engine needs to reject some write requests. - fn should_reject_write(&self) -> bool { + pub(crate) fn should_reject_write(&self) -> bool { // If memory usage reaches high threshold (we should also consider stalled requests) returns true. self.write_buffer_manager.memory_usage() + self.stalled_requests.estimated_size >= self.config.global_write_buffer_reject_size.as_bytes() as usize diff --git a/src/store-api/src/lib.rs b/src/store-api/src/lib.rs index cd0416dc29..c6ee28f5d3 100644 --- a/src/store-api/src/lib.rs +++ b/src/store-api/src/lib.rs @@ -1,4 +1,3 @@ -#![feature(let_chains)] // Copyright 2023 Greptime Team // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,6 +14,9 @@ //! Storage related APIs +#![feature(let_chains)] +#![feature(iterator_try_collect)] + pub mod codec; pub mod data_source; pub mod logstore; diff --git a/src/store-api/src/metadata.rs b/src/store-api/src/metadata.rs index 2c06a8a36a..bb622ac7f8 100644 --- a/src/store-api/src/metadata.rs +++ b/src/store-api/src/metadata.rs @@ -27,6 +27,7 @@ use api::v1::SemanticType; use common_error::ext::ErrorExt; use common_error::status_code::StatusCode; use common_macro::stack_trace_debug; +use datatypes::arrow; use datatypes::arrow::datatypes::FieldRef; use datatypes::schema::{ColumnSchema, FulltextOptions, Schema, SchemaRef, SkippingIndexOptions}; use serde::de::Error; @@ -957,6 +958,14 @@ pub enum MetadataError { #[snafu(implicit)] location: Location, }, + + #[snafu(display("Failed to decode arrow ipc record batches"))] + DecodeArrowIpc { + #[snafu(source)] + error: arrow::error::ArrowError, + #[snafu(implicit)] + location: Location, + }, } impl ErrorExt for MetadataError { diff --git a/src/store-api/src/region_request.rs b/src/store-api/src/region_request.rs index 638f2ee114..08ad2ac559 100644 --- a/src/store-api/src/region_request.rs +++ b/src/store-api/src/region_request.rs @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::hash_map::Entry; use std::collections::HashMap; use std::fmt::{self, Display}; +use std::io::Cursor; use api::helper::ColumnDataTypeWrapper; use api::v1::add_column_location::LocationType; @@ -21,16 +23,19 @@ use api::v1::column_def::{ as_fulltext_option_analyzer, as_fulltext_option_backend, as_skipping_index_type, }; use api::v1::region::{ - alter_request, compact_request, region_request, AlterRequest, AlterRequests, CloseRequest, - CompactRequest, CreateRequest, CreateRequests, DeleteRequests, DropRequest, DropRequests, - FlushRequest, InsertRequests, OpenRequest, TruncateRequest, + alter_request, compact_request, region_request, AlterRequest, AlterRequests, + BulkInsertRequests, CloseRequest, CompactRequest, CreateRequest, CreateRequests, + DeleteRequests, DropRequest, DropRequests, FlushRequest, InsertRequests, OpenRequest, + TruncateRequest, }; use api::v1::{ self, set_index, Analyzer, FulltextBackend as PbFulltextBackend, Option as PbOption, Rows, SemanticType, SkippingIndexType as PbSkippingIndexType, WriteHint, }; pub use common_base::AffectedRows; +use common_recordbatch::DfRecordBatch; use common_time::TimeToLive; +use datatypes::arrow::ipc::reader::FileReader; use datatypes::prelude::ConcreteDataType; use datatypes::schema::{FulltextOptions, SkippingIndexOptions}; use serde::{Deserialize, Serialize}; @@ -39,9 +44,9 @@ use strum::{AsRefStr, IntoStaticStr}; use crate::logstore::entry; use crate::metadata::{ - ColumnMetadata, DecodeProtoSnafu, InvalidRawRegionRequestSnafu, InvalidRegionRequestSnafu, - InvalidSetRegionOptionRequestSnafu, InvalidUnsetRegionOptionRequestSnafu, MetadataError, - RegionMetadata, Result, + ColumnMetadata, DecodeArrowIpcSnafu, DecodeProtoSnafu, InvalidRawRegionRequestSnafu, + InvalidRegionRequestSnafu, InvalidSetRegionOptionRequestSnafu, + InvalidUnsetRegionOptionRequestSnafu, MetadataError, RegionMetadata, Result, }; use crate::metric_engine_consts::PHYSICAL_TABLE_METADATA_KEY; use crate::mito_engine_options::{ @@ -126,6 +131,7 @@ pub enum RegionRequest { Compact(RegionCompactRequest), Truncate(RegionTruncateRequest), Catchup(RegionCatchupRequest), + BulkInserts(RegionBulkInsertsRequest), } impl RegionRequest { @@ -146,6 +152,7 @@ impl RegionRequest { region_request::Body::Creates(creates) => make_region_creates(creates), region_request::Body::Drops(drops) => make_region_drops(drops), region_request::Body::Alters(alters) => make_region_alters(alters), + region_request::Body::BulkInserts(bulk) => make_region_bulk_inserts(bulk), } } @@ -315,6 +322,51 @@ fn make_region_truncate(truncate: TruncateRequest) -> Result Result> { + let mut region_requests: HashMap> = + HashMap::with_capacity(requests.requests.len()); + + for req in requests.requests { + let region_id = req.region_id; + match req.payload_type() { + api::v1::region::BulkInsertType::ArrowIpc => { + // todo(hl): use StreamReader instead + let reader = FileReader::try_new(Cursor::new(req.payload), None) + .context(DecodeArrowIpcSnafu)?; + let record_batches = reader + .map(|b| b.map(BulkInsertPayload::ArrowIpc)) + .try_collect::>() + .context(DecodeArrowIpcSnafu)?; + match region_requests.entry(region_id) { + Entry::Occupied(mut e) => { + e.get_mut().extend(record_batches); + } + Entry::Vacant(e) => { + e.insert(record_batches); + } + } + } + } + } + + let result = region_requests + .into_iter() + .map(|(region_id, payloads)| { + ( + region_id.into(), + RegionRequest::BulkInserts(RegionBulkInsertsRequest { + region_id: region_id.into(), + payloads, + }), + ) + }) + .collect::>(); + Ok(result) +} + /// Request to put data into a region. #[derive(Debug)] pub struct RegionPutRequest { @@ -1119,6 +1171,17 @@ pub struct RegionSequencesRequest { pub region_ids: Vec, } +#[derive(Debug, Clone)] +pub struct RegionBulkInsertsRequest { + pub region_id: RegionId, + pub payloads: Vec, +} + +#[derive(Debug, Clone)] +pub enum BulkInsertPayload { + ArrowIpc(DfRecordBatch), +} + impl fmt::Display for RegionRequest { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -1133,6 +1196,7 @@ impl fmt::Display for RegionRequest { RegionRequest::Compact(_) => write!(f, "Compact"), RegionRequest::Truncate(_) => write!(f, "Truncate"), RegionRequest::Catchup(_) => write!(f, "Catchup"), + RegionRequest::BulkInserts(_) => write!(f, "BulkInserts"), } } } From 0fb9e1995eba9e0f648b60fb7bbf1956b6c48fde Mon Sep 17 00:00:00 2001 From: Ruihang Xia Date: Tue, 15 Apr 2025 23:00:30 +0800 Subject: [PATCH 26/82] fix: preserve timestamp precision of irate (#5904) Signed-off-by: Ruihang Xia --- src/promql/src/functions/idelta.rs | 4 +- .../standalone/common/tql/operator.result | 53 +++++++++++++++++++ .../cases/standalone/common/tql/operator.sql | 35 ++++++++++++ 3 files changed, 89 insertions(+), 3 deletions(-) diff --git a/src/promql/src/functions/idelta.rs b/src/promql/src/functions/idelta.rs index 0e581d2d2c..c5d1897a3e 100644 --- a/src/promql/src/functions/idelta.rs +++ b/src/promql/src/functions/idelta.rs @@ -138,9 +138,7 @@ impl IDelta { } // else is rate - // TODO(ruihang): "divide 1000" converts the timestamp from millisecond to second. - // it should consider other percisions. - let sampled_interval = (timestamps[len - 1] - timestamps[len - 2]) / 1000; + let sampled_interval = (timestamps[len - 1] - timestamps[len - 2]) as f64 / 1000.0; let last_value = values[len - 1]; let prev_value = values[len - 2]; let result_value = if last_value < prev_value { diff --git a/tests/cases/standalone/common/tql/operator.result b/tests/cases/standalone/common/tql/operator.result index d3b85ac445..6ad97ae88a 100644 --- a/tests/cases/standalone/common/tql/operator.result +++ b/tests/cases/standalone/common/tql/operator.result @@ -78,3 +78,56 @@ drop table trignan; Affected Rows: 0 +-- About irate. Related to issue https://github.com/GreptimeTeam/greptimedb/issues/5880 +CREATE TABLE t( + greptime_timestamp TIMESTAMP(9) TIME INDEX, + greptime_value DOUBLE +); + +Affected Rows: 0 + +INSERT INTO t(greptime_timestamp, greptime_value) +VALUES + ('2025-04-01T00:00:00.5Z', 1), + ('2025-04-01T00:00:01Z', 2), + ('2025-04-01T00:00:01.5Z', 3), + ('2025-04-01T00:00:02Z', 4), + ('2025-04-01T00:00:02.5Z', 5), + ('2025-04-01T00:00:03Z', 6), + ('2025-04-01T00:00:03.5Z', 7), + ('2025-04-01T00:00:04Z', 8), + ('2025-04-01T00:00:04.5Z', 9), + ('2025-04-01T00:00:05Z', 10), + ('2025-04-01T00:00:05.5Z', 11), + ('2025-04-01T00:00:06Z', 12), + ('2025-04-01T00:00:06.5Z', 13), + ('2025-04-01T00:00:07Z', 14), + ('2025-04-01T00:00:07.5Z', 15), + ('2025-04-01T00:00:08Z', 16), + ('2025-04-01T00:00:08.5Z', 17), + ('2025-04-01T00:00:09Z', 18), + ('2025-04-01T00:00:09.5Z', 19), + ('2025-04-01T00:00:10Z', 20); + +Affected Rows: 20 + +tql eval (1743465600.5, 1743465610, '1s') irate(t[2s]); + ++-------------------------+-----------------------------------------------------+ +| greptime_timestamp | prom_irate(greptime_timestamp_range,greptime_value) | ++-------------------------+-----------------------------------------------------+ +| 2025-04-01T00:00:01.500 | 2.0 | +| 2025-04-01T00:00:02.500 | 2.0 | +| 2025-04-01T00:00:03.500 | 2.0 | +| 2025-04-01T00:00:04.500 | 2.0 | +| 2025-04-01T00:00:05.500 | 2.0 | +| 2025-04-01T00:00:06.500 | 2.0 | +| 2025-04-01T00:00:07.500 | 2.0 | +| 2025-04-01T00:00:08.500 | 2.0 | +| 2025-04-01T00:00:09.500 | 2.0 | ++-------------------------+-----------------------------------------------------+ + +drop table t; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/tql/operator.sql b/tests/cases/standalone/common/tql/operator.sql index f6c342696b..3a6c2c527b 100644 --- a/tests/cases/standalone/common/tql/operator.sql +++ b/tests/cases/standalone/common/tql/operator.sql @@ -39,3 +39,38 @@ drop table trigx; drop table trigy; drop table trignan; + + +-- About irate. Related to issue https://github.com/GreptimeTeam/greptimedb/issues/5880 +CREATE TABLE t( + greptime_timestamp TIMESTAMP(9) TIME INDEX, + greptime_value DOUBLE +); + +INSERT INTO t(greptime_timestamp, greptime_value) +VALUES + ('2025-04-01T00:00:00.5Z', 1), + ('2025-04-01T00:00:01Z', 2), + ('2025-04-01T00:00:01.5Z', 3), + ('2025-04-01T00:00:02Z', 4), + ('2025-04-01T00:00:02.5Z', 5), + ('2025-04-01T00:00:03Z', 6), + ('2025-04-01T00:00:03.5Z', 7), + ('2025-04-01T00:00:04Z', 8), + ('2025-04-01T00:00:04.5Z', 9), + ('2025-04-01T00:00:05Z', 10), + ('2025-04-01T00:00:05.5Z', 11), + ('2025-04-01T00:00:06Z', 12), + ('2025-04-01T00:00:06.5Z', 13), + ('2025-04-01T00:00:07Z', 14), + ('2025-04-01T00:00:07.5Z', 15), + ('2025-04-01T00:00:08Z', 16), + ('2025-04-01T00:00:08.5Z', 17), + ('2025-04-01T00:00:09Z', 18), + ('2025-04-01T00:00:09.5Z', 19), + ('2025-04-01T00:00:10Z', 20); + +tql eval (1743465600.5, 1743465610, '1s') irate(t[2s]); + +drop table t; + From 55c9a0de427e0c37cc01a8e5681073a3438073a5 Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Wed, 16 Apr 2025 02:48:42 +0800 Subject: [PATCH 27/82] chore: upgrade opendal to 0.52 (#5857) * chore: upgrade opendal to 0.52 * chore: ugprade object_store_opendal to 0.50 * Update Cargo.toml Co-authored-by: dennis zhuang --------- Co-authored-by: dennis zhuang --- Cargo.lock | 41 ++++++++++++++++--- Cargo.toml | 2 +- src/common/procedure/src/store/state_store.rs | 1 + src/object-store/Cargo.toml | 2 +- 4 files changed, 38 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 62d14792ad..d8e5f15cdf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1691,7 +1691,7 @@ dependencies = [ "humantime", "meta-client", "nu-ansi-term", - "opendal", + "opendal 0.51.2", "query", "rand 0.9.0", "reqwest", @@ -7588,7 +7588,7 @@ dependencies = [ "lazy_static", "md5", "moka", - "opendal", + "opendal 0.52.0", "prometheus", "tokio", "uuid", @@ -7617,9 +7617,9 @@ dependencies = [ [[package]] name = "object_store_opendal" -version = "0.49.1" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b18eb960885330175ec89daa991b0bc9050dc7f259b31a887fbfbb297312f83" +checksum = "59db4de9230e630346e3fd821e2b8d1fd03ddd509c32c964836588e82241762a" dependencies = [ "async-trait", "bytes", @@ -7627,7 +7627,7 @@ dependencies = [ "futures", "futures-util", "object_store", - "opendal", + "opendal 0.52.0", "pin-project", "tokio", ] @@ -7658,6 +7658,35 @@ name = "opendal" version = "0.51.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b1063ea459fa9e94584115743b06330f437902dd1d9f692b863ef1875a20548" +dependencies = [ + "anyhow", + "async-trait", + "backon", + "base64 0.22.1", + "bytes", + "chrono", + "crc32c", + "futures", + "getrandom 0.2.15", + "http 1.1.0", + "log", + "md-5", + "once_cell", + "percent-encoding", + "quick-xml 0.36.2", + "reqsign", + "reqwest", + "serde", + "serde_json", + "tokio", + "uuid", +] + +[[package]] +name = "opendal" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55c840b5a6ad96106d6c0612fabb8f35a5ace826e0474fc55ebda33042b8d33" dependencies = [ "anyhow", "async-trait", @@ -8934,7 +8963,7 @@ dependencies = [ "log", "multimap", "once_cell", - "petgraph 0.6.5", + "petgraph 0.7.1", "prettyplease", "prost 0.13.5", "prost-types 0.13.5", diff --git a/Cargo.toml b/Cargo.toml index 6a0e59c0a1..919b4f8d44 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -147,7 +147,7 @@ moka = "0.12" nalgebra = "0.33" notify = "8.0" num_cpus = "1.16" -object_store_opendal = "0.49.0" +object_store_opendal = "0.50" once_cell = "1.18" opentelemetry-proto = { version = "0.27", features = [ "gen-tonic", diff --git a/src/common/procedure/src/store/state_store.rs b/src/common/procedure/src/store/state_store.rs index a43d7da86f..d98ce06a1e 100644 --- a/src/common/procedure/src/store/state_store.rs +++ b/src/common/procedure/src/store/state_store.rs @@ -137,6 +137,7 @@ impl StateStore for ObjectStateStore { )) }) .context(PutStateSnafu { key }) + .map(|_| ()) } async fn walk_top_down(&self, path: &str) -> Result { diff --git a/src/object-store/Cargo.toml b/src/object-store/Cargo.toml index 8aab3af382..f90ea42d61 100644 --- a/src/object-store/Cargo.toml +++ b/src/object-store/Cargo.toml @@ -17,7 +17,7 @@ futures.workspace = true lazy_static.workspace = true md5 = "0.7" moka = { workspace = true, features = ["future"] } -opendal = { version = "0.51.1", features = [ +opendal = { version = "0.52", features = [ "layers-tracing", "layers-prometheus", "services-azblob", From 7274ceba30ffb56bef378a1de93057aa0b0e3c14 Mon Sep 17 00:00:00 2001 From: Lin Yihai Date: Wed, 16 Apr 2025 18:17:20 +0800 Subject: [PATCH 28/82] feat: Add query pipeline http api (#5819) * feat(pipeline): add query pipeline http api. * chore(pipeline): rename get pipepile method * refactor(pipeline): Also insert string piple into cache after inserting into table. --------- Co-authored-by: shuiyisong <113876041+shuiyisong@users.noreply.github.com> --- Cargo.lock | 1 + src/frontend/src/instance/log_handler.rs | 13 +++++ src/pipeline/src/manager/pipeline_operator.rs | 24 +++++++++ src/pipeline/src/manager/table.rs | 48 ++++++++++++++++-- src/servers/src/http.rs | 8 +++ src/servers/src/http/event.rs | 37 ++++++++++++++ .../src/http/result/greptime_manage_resp.rs | 16 +++++- src/servers/src/query_handler.rs | 9 ++++ tests-integration/Cargo.toml | 1 + tests-integration/tests/http.rs | 50 +++++++++++++++++++ 10 files changed, 201 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d8e5f15cdf..b133daa143 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12087,6 +12087,7 @@ dependencies = [ "tower 0.5.2", "url", "uuid", + "yaml-rust", "zstd 0.13.2", ] diff --git a/src/frontend/src/instance/log_handler.rs b/src/frontend/src/instance/log_handler.rs index 029a40e980..179d5e098f 100644 --- a/src/frontend/src/instance/log_handler.rs +++ b/src/frontend/src/instance/log_handler.rs @@ -19,6 +19,7 @@ use async_trait::async_trait; use auth::{PermissionChecker, PermissionCheckerRef, PermissionReq}; use client::Output; use common_error::ext::BoxedError; +use datatypes::timestamp::TimestampNanosecond; use pipeline::pipeline_operator::PipelineOperator; use pipeline::{Pipeline, PipelineInfo, PipelineVersion}; use servers::error::{ @@ -103,6 +104,18 @@ impl PipelineHandler for Instance { fn build_pipeline(&self, pipeline: &str) -> ServerResult { PipelineOperator::build_pipeline(pipeline).context(PipelineSnafu) } + + async fn get_pipeline_str( + &self, + name: &str, + version: PipelineVersion, + query_ctx: QueryContextRef, + ) -> ServerResult<(String, TimestampNanosecond)> { + self.pipeline_operator + .get_pipeline_str(name, version, query_ctx) + .await + .context(PipelineSnafu) + } } impl Instance { diff --git a/src/pipeline/src/manager/pipeline_operator.rs b/src/pipeline/src/manager/pipeline_operator.rs index f98bab6b6c..c5d11574b7 100644 --- a/src/pipeline/src/manager/pipeline_operator.rs +++ b/src/pipeline/src/manager/pipeline_operator.rs @@ -20,6 +20,7 @@ use api::v1::CreateTableExpr; use catalog::{CatalogManagerRef, RegisterSystemTableRequest}; use common_catalog::consts::{default_engine, DEFAULT_PRIVATE_SCHEMA_NAME}; use common_telemetry::info; +use datatypes::timestamp::TimestampNanosecond; use futures::FutureExt; use operator::insert::InserterRef; use operator::statement::StatementExecutorRef; @@ -198,6 +199,29 @@ impl PipelineOperator { .await } + /// Get a original pipeline by name. + pub async fn get_pipeline_str( + &self, + name: &str, + version: PipelineVersion, + query_ctx: QueryContextRef, + ) -> Result<(String, TimestampNanosecond)> { + let schema = query_ctx.current_schema(); + self.create_pipeline_table_if_not_exists(query_ctx.clone()) + .await?; + + let timer = Instant::now(); + self.get_pipeline_table_from_cache(query_ctx.current_catalog()) + .context(PipelineTableNotFoundSnafu)? + .get_pipeline_str(&schema, name, version) + .inspect(|re| { + METRIC_PIPELINE_RETRIEVE_HISTOGRAM + .with_label_values(&[&re.is_ok().to_string()]) + .observe(timer.elapsed().as_secs_f64()) + }) + .await + } + /// Insert a pipeline into the pipeline table. pub async fn insert_pipeline( &self, diff --git a/src/pipeline/src/manager/table.rs b/src/pipeline/src/manager/table.rs index f01742d7c2..649bab9f6e 100644 --- a/src/pipeline/src/manager/table.rs +++ b/src/pipeline/src/manager/table.rs @@ -69,6 +69,7 @@ pub struct PipelineTable { table: TableRef, query_engine: QueryEngineRef, pipelines: Cache>, + original_pipelines: Cache, } impl PipelineTable { @@ -88,6 +89,10 @@ impl PipelineTable { .max_capacity(PIPELINES_CACHE_SIZE) .time_to_live(PIPELINES_CACHE_TTL) .build(), + original_pipelines: Cache::builder() + .max_capacity(PIPELINES_CACHE_SIZE) + .time_to_live(PIPELINES_CACHE_TTL) + .build(), } } @@ -273,10 +278,7 @@ impl PipelineTable { return Ok(pipeline); } - let pipeline = self - .find_pipeline(schema, name, version) - .await? - .context(PipelineNotFoundSnafu { name, version })?; + let pipeline = self.get_pipeline_str(schema, name, version).await?; let compiled_pipeline = Arc::new(Self::compile_pipeline(&pipeline.0)?); self.pipelines.insert( @@ -286,6 +288,31 @@ impl PipelineTable { Ok(compiled_pipeline) } + /// Get a original pipeline by name. + /// If the pipeline is not in the cache, it will be get from table and compiled and inserted into the cache. + pub async fn get_pipeline_str( + &self, + schema: &str, + name: &str, + version: PipelineVersion, + ) -> Result<(String, TimestampNanosecond)> { + if let Some(pipeline) = self + .original_pipelines + .get(&generate_pipeline_cache_key(schema, name, version)) + { + return Ok(pipeline); + } + let pipeline = self + .find_pipeline(schema, name, version) + .await? + .context(PipelineNotFoundSnafu { name, version })?; + self.original_pipelines.insert( + generate_pipeline_cache_key(schema, name, version), + pipeline.clone(), + ); + Ok(pipeline) + } + /// Insert a pipeline into the pipeline table and compile it. /// The compiled pipeline will be inserted into the cache. pub async fn insert_and_compile( @@ -310,6 +337,15 @@ impl PipelineTable { generate_pipeline_cache_key(schema, name, Some(TimestampNanosecond(version))), compiled_pipeline.clone(), ); + + self.original_pipelines.insert( + generate_pipeline_cache_key(schema, name, None), + (pipeline.to_owned(), TimestampNanosecond(version)), + ); + self.original_pipelines.insert( + generate_pipeline_cache_key(schema, name, Some(TimestampNanosecond(version))), + (pipeline.to_owned(), TimestampNanosecond(version)), + ); } Ok((version, compiled_pipeline)) @@ -392,6 +428,10 @@ impl PipelineTable { .remove(&generate_pipeline_cache_key(schema, name, version)); self.pipelines .remove(&generate_pipeline_cache_key(schema, name, None)); + self.original_pipelines + .remove(&generate_pipeline_cache_key(schema, name, version)); + self.original_pipelines + .remove(&generate_pipeline_cache_key(schema, name, None)); Ok(Some(())) } diff --git a/src/servers/src/http.rs b/src/servers/src/http.rs index 07f589f127..07d1c7ea24 100644 --- a/src/servers/src/http.rs +++ b/src/servers/src/http.rs @@ -928,6 +928,10 @@ impl HttpServer { fn route_log_deprecated(log_state: LogState) -> Router { Router::new() .route("/logs", routing::post(event::log_ingester)) + .route( + "/pipelines/{pipeline_name}", + routing::get(event::query_pipeline), + ) .route( "/pipelines/{pipeline_name}", routing::post(event::add_pipeline), @@ -947,6 +951,10 @@ impl HttpServer { fn route_pipelines(log_state: LogState) -> Router { Router::new() .route("/ingest", routing::post(event::log_ingester)) + .route( + "/pipelines/{pipeline_name}", + routing::get(event::query_pipeline), + ) .route( "/pipelines/{pipeline_name}", routing::post(event::add_pipeline), diff --git a/src/servers/src/http/event.rs b/src/servers/src/http/event.rs index bee41b8053..cfc0694ae0 100644 --- a/src/servers/src/http/event.rs +++ b/src/servers/src/http/event.rs @@ -147,6 +147,41 @@ where } } +#[axum_macros::debug_handler] +pub async fn query_pipeline( + State(state): State, + Extension(mut query_ctx): Extension, + Query(query_params): Query, + Path(pipeline_name): Path, +) -> Result { + let start = Instant::now(); + let handler = state.log_handler; + ensure!( + !pipeline_name.is_empty(), + InvalidParameterSnafu { + reason: "pipeline_name is required in path", + } + ); + + let version = to_pipeline_version(query_params.version.as_deref()).context(PipelineSnafu)?; + + query_ctx.set_channel(Channel::Http); + let query_ctx = Arc::new(query_ctx); + + let (pipeline, pipeline_version) = handler + .get_pipeline_str(&pipeline_name, version, query_ctx) + .await?; + + Ok(GreptimedbManageResponse::from_pipeline( + pipeline_name, + query_params + .version + .unwrap_or(pipeline_version.0.to_iso8601_string()), + start.elapsed().as_millis() as u64, + Some(pipeline), + )) +} + #[axum_macros::debug_handler] pub async fn add_pipeline( State(state): State, @@ -189,6 +224,7 @@ pub async fn add_pipeline( pipeline_name, pipeline.0.to_timezone_aware_string(None), start.elapsed().as_millis() as u64, + None, ) }) .map_err(|e| { @@ -231,6 +267,7 @@ pub async fn delete_pipeline( pipeline_name, version_str, start.elapsed().as_millis() as u64, + None, ) } else { GreptimedbManageResponse::from_pipelines(vec![], start.elapsed().as_millis() as u64) diff --git a/src/servers/src/http/result/greptime_manage_resp.rs b/src/servers/src/http/result/greptime_manage_resp.rs index a46df298b4..3db07028b3 100644 --- a/src/servers/src/http/result/greptime_manage_resp.rs +++ b/src/servers/src/http/result/greptime_manage_resp.rs @@ -30,10 +30,19 @@ pub struct GreptimedbManageResponse { } impl GreptimedbManageResponse { - pub fn from_pipeline(name: String, version: String, execution_time_ms: u64) -> Self { + pub fn from_pipeline( + name: String, + version: String, + execution_time_ms: u64, + pipeline: Option, + ) -> Self { GreptimedbManageResponse { manage_result: ManageResult::Pipelines { - pipelines: vec![PipelineOutput { name, version }], + pipelines: vec![PipelineOutput { + name, + version, + pipeline, + }], }, execution_time_ms, } @@ -68,6 +77,8 @@ pub enum ManageResult { pub struct PipelineOutput { name: String, version: String, + #[serde(skip_serializing_if = "Option::is_none")] + pipeline: Option, } impl IntoResponse for GreptimedbManageResponse { @@ -109,6 +120,7 @@ mod tests { pipelines: vec![PipelineOutput { name: "test_name".to_string(), version: "test_version".to_string(), + pipeline: None, }], }, execution_time_ms: 42, diff --git a/src/servers/src/query_handler.rs b/src/servers/src/query_handler.rs index b4e734fca2..30fd360455 100644 --- a/src/servers/src/query_handler.rs +++ b/src/servers/src/query_handler.rs @@ -34,6 +34,7 @@ use api::v1::RowInsertRequests; use async_trait::async_trait; use catalog::CatalogManager; use common_query::Output; +use datatypes::timestamp::TimestampNanosecond; use headers::HeaderValue; use log_query::LogQuery; use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; @@ -165,6 +166,14 @@ pub trait PipelineHandler { //// Build a pipeline from a string. fn build_pipeline(&self, pipeline: &str) -> Result; + + /// Get a original pipeline by name. + async fn get_pipeline_str( + &self, + name: &str, + version: PipelineVersion, + query_ctx: QueryContextRef, + ) -> Result<(String, TimestampNanosecond)>; } /// Handle log query requests. diff --git a/tests-integration/Cargo.toml b/tests-integration/Cargo.toml index 0ce38049ca..f2f538f528 100644 --- a/tests-integration/Cargo.toml +++ b/tests-integration/Cargo.toml @@ -100,3 +100,4 @@ session = { workspace = true, features = ["testing"] } store-api.workspace = true tokio-postgres = { workspace = true } url = "2.3" +yaml-rust = "0.4" diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index d20fe50241..8ef27cdbf0 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -50,6 +50,7 @@ use tests_integration::test_util::{ setup_test_http_app_with_frontend_and_user_provider, setup_test_prom_app_with_frontend, StorageType, }; +use yaml_rust::YamlLoader; #[macro_export] macro_rules! http_test { @@ -1348,6 +1349,55 @@ transform: .as_str() .unwrap() .to_string(); + let encoded_ver_str: String = + url::form_urlencoded::byte_serialize(version_str.as_bytes()).collect(); + + // get pipeline + let res = client + .get(format!("/v1/pipelines/test?version={}", encoded_ver_str).as_str()) + .send() + .await; + + assert_eq!(res.status(), StatusCode::OK); + + let content = res.text().await; + let content = serde_json::from_str(&content); + let content: Value = content.unwrap(); + let pipeline_yaml = content + .get("pipelines") + .unwrap() + .as_array() + .unwrap() + .first() + .unwrap() + .get("pipeline") + .unwrap() + .as_str() + .unwrap(); + let docs = YamlLoader::load_from_str(pipeline_yaml).unwrap(); + let body_yaml = YamlLoader::load_from_str(body).unwrap(); + assert_eq!(docs, body_yaml); + + // Do not specify version, get the latest version + let res = client.get("/v1/pipelines/test").send().await; + assert_eq!(res.status(), StatusCode::OK); + + let content = res.text().await; + let content = serde_json::from_str(&content); + let content: Value = content.unwrap(); + let pipeline_yaml = content + .get("pipelines") + .unwrap() + .as_array() + .unwrap() + .first() + .unwrap() + .get("pipeline") + .unwrap() + .as_str() + .unwrap(); + let docs = YamlLoader::load_from_str(pipeline_yaml).unwrap(); + assert_eq!(docs, body_yaml); // 2. write data let data_body = r#" From fdab5d198ee84523fb6c5837d87f7d26f631c89a Mon Sep 17 00:00:00 2001 From: shuiyisong <113876041+shuiyisong@users.noreply.github.com> Date: Wed, 16 Apr 2025 18:37:07 +0800 Subject: [PATCH 29/82] feat: add json parse processor (#5910) * feat: add json parse processor * chore: support parse to arr --- src/pipeline/src/error.rs | 16 +- src/pipeline/src/etl/processor.rs | 6 + src/pipeline/src/etl/processor/json_parse.rs | 147 +++++++++++++++ .../src/etl/processor/simple_extract.rs | 2 +- src/pipeline/src/etl/value.rs | 8 +- src/pipeline/src/etl/value/array.rs | 13 ++ src/pipeline/tests/common.rs | 1 + src/pipeline/tests/json_parse.rs | 178 ++++++++++++++++++ src/pipeline/tests/simple_extract.rs | 2 +- src/servers/src/http/loki.rs | 2 +- 10 files changed, 361 insertions(+), 14 deletions(-) create mode 100644 src/pipeline/src/etl/processor/json_parse.rs create mode 100644 src/pipeline/tests/json_parse.rs diff --git a/src/pipeline/src/error.rs b/src/pipeline/src/error.rs index b590e6847d..099d2b5100 100644 --- a/src/pipeline/src/error.rs +++ b/src/pipeline/src/error.rs @@ -517,12 +517,7 @@ pub enum Error { #[snafu(implicit)] location: Location, }, - #[snafu(display("Unsupported number type: {value}"))] - ValueUnsupportedNumberType { - value: serde_json::Number, - #[snafu(implicit)] - location: Location, - }, + #[snafu(display("Unsupported yaml type: {value:?}"))] ValueUnsupportedYamlType { value: yaml_rust::Yaml, @@ -574,6 +569,13 @@ pub enum Error { #[snafu(implicit)] location: Location, }, + #[snafu(display("Failed to parse json"))] + JsonParse { + #[snafu(source)] + error: serde_json::Error, + #[snafu(implicit)] + location: Location, + }, #[snafu(display("Column datatype mismatch. For column: {column}, expected datatype: {expected}, actual datatype: {actual}"))] IdentifyPipelineColumnTypeMismatch { column: String, @@ -808,7 +810,6 @@ impl ErrorExt for Error { | ValueParseFloat { .. } | ValueParseBoolean { .. } | ValueDefaultValueUnsupported { .. } - | ValueUnsupportedNumberType { .. } | ValueUnsupportedYamlType { .. } | ValueYamlKeyMustBeString { .. } | YamlLoad { .. } @@ -818,6 +819,7 @@ impl ErrorExt for Error { | UnsupportedIndexType { .. } | UnsupportedNumberType { .. } | IdentifyPipelineColumnTypeMismatch { .. } + | JsonParse { .. } | JsonPathParse { .. } | JsonPathParseResultIndex { .. } | FieldRequiredForDispatcher diff --git a/src/pipeline/src/etl/processor.rs b/src/pipeline/src/etl/processor.rs index 240b477efd..aa10fd9e78 100644 --- a/src/pipeline/src/etl/processor.rs +++ b/src/pipeline/src/etl/processor.rs @@ -21,6 +21,7 @@ pub mod dissect; pub mod epoch; pub mod gsub; pub mod join; +pub mod json_parse; pub mod json_path; pub mod letter; pub mod regex; @@ -50,6 +51,7 @@ use crate::error::{ ProcessorMustBeMapSnafu, ProcessorMustHaveStringKeySnafu, Result, UnsupportedProcessorSnafu, }; use crate::etl::field::{Field, Fields}; +use crate::etl::processor::json_parse::JsonParseProcessor; use crate::etl::processor::simple_extract::SimpleExtractProcessor; use crate::etl::PipelineMap; @@ -98,6 +100,7 @@ pub enum ProcessorKind { Epoch(EpochProcessor), Date(DateProcessor), JsonPath(JsonPathProcessor), + JsonParse(JsonParseProcessor), SimpleJsonPath(SimpleExtractProcessor), Decolorize(DecolorizeProcessor), Digest(DigestProcessor), @@ -179,6 +182,9 @@ fn parse_processor(doc: &yaml_rust::Yaml) -> Result { simple_extract::PROCESSOR_SIMPLE_EXTRACT => { ProcessorKind::SimpleJsonPath(SimpleExtractProcessor::try_from(value)?) } + json_parse::PROCESSOR_JSON_PARSE => { + ProcessorKind::JsonParse(JsonParseProcessor::try_from(value)?) + } _ => return UnsupportedProcessorSnafu { processor: str_key }.fail(), }; diff --git a/src/pipeline/src/etl/processor/json_parse.rs b/src/pipeline/src/etl/processor/json_parse.rs new file mode 100644 index 0000000000..7361ee9151 --- /dev/null +++ b/src/pipeline/src/etl/processor/json_parse.rs @@ -0,0 +1,147 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use snafu::{OptionExt as _, ResultExt}; + +use crate::error::{ + Error, FieldMustBeTypeSnafu, JsonParseSnafu, KeyMustBeStringSnafu, ProcessorMissingFieldSnafu, + ProcessorUnsupportedValueSnafu, Result, +}; +use crate::etl::field::Fields; +use crate::etl::processor::{ + yaml_bool, yaml_new_field, yaml_new_fields, FIELDS_NAME, FIELD_NAME, IGNORE_MISSING_NAME, +}; +use crate::{json_to_map, PipelineMap, Processor, Value}; + +pub(crate) const PROCESSOR_JSON_PARSE: &str = "json_parse"; + +#[derive(Debug, Default)] +pub struct JsonParseProcessor { + fields: Fields, + ignore_missing: bool, +} + +impl TryFrom<&yaml_rust::yaml::Hash> for JsonParseProcessor { + type Error = Error; + + fn try_from(value: &yaml_rust::yaml::Hash) -> std::result::Result { + let mut fields = Fields::default(); + let mut ignore_missing = false; + + for (k, v) in value.iter() { + let key = k + .as_str() + .with_context(|| KeyMustBeStringSnafu { k: k.clone() })?; + match key { + FIELD_NAME => { + fields = Fields::one(yaml_new_field(v, FIELD_NAME)?); + } + FIELDS_NAME => { + fields = yaml_new_fields(v, FIELDS_NAME)?; + } + IGNORE_MISSING_NAME => { + ignore_missing = yaml_bool(v, IGNORE_MISSING_NAME)?; + } + _ => {} + } + } + + let processor = JsonParseProcessor { + fields, + ignore_missing, + }; + + Ok(processor) + } +} + +impl JsonParseProcessor { + fn process_field(&self, val: &Value) -> Result { + let Some(json_str) = val.as_str() else { + return FieldMustBeTypeSnafu { + field: val.to_str_type(), + ty: "string", + } + .fail(); + }; + let parsed: serde_json::Value = serde_json::from_str(json_str).context(JsonParseSnafu)?; + match parsed { + serde_json::Value::Object(_) => Ok(Value::Map(json_to_map(parsed)?.into())), + serde_json::Value::Array(arr) => Ok(Value::Array(arr.try_into()?)), + _ => ProcessorUnsupportedValueSnafu { + processor: self.kind(), + val: val.to_str_type(), + } + .fail(), + } + } +} + +impl Processor for JsonParseProcessor { + fn kind(&self) -> &str { + PROCESSOR_JSON_PARSE + } + + fn ignore_missing(&self) -> bool { + self.ignore_missing + } + + fn exec_mut(&self, val: &mut PipelineMap) -> Result<()> { + for field in self.fields.iter() { + let index = field.input_field(); + match val.get(index) { + Some(v) => { + let processed = self.process_field(v)?; + let output_index = field.target_or_input_field(); + val.insert(output_index.to_string(), processed); + } + None => { + if !self.ignore_missing { + return ProcessorMissingFieldSnafu { + processor: self.kind(), + field: field.input_field(), + } + .fail(); + } + } + } + } + Ok(()) + } +} + +#[cfg(test)] +mod test { + + #[test] + fn test_json_parse() { + use super::*; + use crate::Value; + + let processor = JsonParseProcessor { + ..Default::default() + }; + + let result = processor + .process_field(&Value::String(r#"{"hello": "world"}"#.to_string())) + .unwrap(); + + let expected = Value::Map(crate::Map::one( + "hello".to_string(), + Value::String("world".to_string()), + )); + + assert_eq!(result, expected); + } +} diff --git a/src/pipeline/src/etl/processor/simple_extract.rs b/src/pipeline/src/etl/processor/simple_extract.rs index ebac88f937..6015fc4591 100644 --- a/src/pipeline/src/etl/processor/simple_extract.rs +++ b/src/pipeline/src/etl/processor/simple_extract.rs @@ -126,7 +126,7 @@ impl Processor for SimpleExtractProcessor { mod test { #[test] - fn test_json_path() { + fn test_simple_extract() { use super::*; use crate::{Map, Value}; diff --git a/src/pipeline/src/etl/value.rs b/src/pipeline/src/etl/value.rs index 5f44b264b3..51ac76c4a5 100644 --- a/src/pipeline/src/etl/value.rs +++ b/src/pipeline/src/etl/value.rs @@ -29,9 +29,9 @@ use snafu::{OptionExt, ResultExt}; pub use time::Timestamp; use crate::error::{ - Error, Result, ValueDefaultValueUnsupportedSnafu, ValueInvalidResolutionSnafu, - ValueParseBooleanSnafu, ValueParseFloatSnafu, ValueParseIntSnafu, ValueParseTypeSnafu, - ValueUnsupportedNumberTypeSnafu, ValueUnsupportedYamlTypeSnafu, ValueYamlKeyMustBeStringSnafu, + Error, Result, UnsupportedNumberTypeSnafu, ValueDefaultValueUnsupportedSnafu, + ValueInvalidResolutionSnafu, ValueParseBooleanSnafu, ValueParseFloatSnafu, ValueParseIntSnafu, + ValueParseTypeSnafu, ValueUnsupportedYamlTypeSnafu, ValueYamlKeyMustBeStringSnafu, }; use crate::etl::PipelineMap; @@ -413,7 +413,7 @@ impl TryFrom for Value { } else if let Some(v) = v.as_f64() { Ok(Value::Float64(v)) } else { - ValueUnsupportedNumberTypeSnafu { value: v }.fail() + UnsupportedNumberTypeSnafu { value: v }.fail() } } serde_json::Value::String(v) => Ok(Value::String(v)), diff --git a/src/pipeline/src/etl/value/array.rs b/src/pipeline/src/etl/value/array.rs index 4cba167215..0658d502df 100644 --- a/src/pipeline/src/etl/value/array.rs +++ b/src/pipeline/src/etl/value/array.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::error::{Error, Result}; use crate::etl::value::Value; #[derive(Debug, Clone, PartialEq, Default)] @@ -66,3 +67,15 @@ impl From> for Array { Array { values } } } + +impl TryFrom> for Array { + type Error = Error; + + fn try_from(value: Vec) -> Result { + let values = value + .into_iter() + .map(|v| v.try_into()) + .collect::>>()?; + Ok(Array { values }) + } +} diff --git a/src/pipeline/tests/common.rs b/src/pipeline/tests/common.rs index 0361f4e41c..781f70f0d5 100644 --- a/src/pipeline/tests/common.rs +++ b/src/pipeline/tests/common.rs @@ -56,6 +56,7 @@ pub fn parse_and_exec(input_str: &str, pipeline_yaml: &str) -> Rows { } /// test util function to create column schema +#[allow(dead_code)] pub fn make_column_schema( column_name: String, datatype: ColumnDataType, diff --git a/src/pipeline/tests/json_parse.rs b/src/pipeline/tests/json_parse.rs new file mode 100644 index 0000000000..bd43d9754c --- /dev/null +++ b/src/pipeline/tests/json_parse.rs @@ -0,0 +1,178 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod common; + +use std::borrow::Cow; + +use api::v1::value::ValueData; +use api::v1::ColumnDataType; + +const INPUT_VALUE_OBJ: &str = r#" +[ + { + "commit": "{\"commitTime\": \"1573840000.000\", \"commitAuthor\": \"test\"}" + } +] +"#; + +const INPUT_VALUE_ARR: &str = r#" +[ + { + "commit": "[\"test1\", \"test2\"]" + } +] +"#; + +#[test] +fn test_json_parse_inplace() { + let pipeline_yaml = r#" +--- +processors: + - json_parse: + field: commit + +transform: + - field: commit + type: json +"#; + + let output = common::parse_and_exec(INPUT_VALUE_OBJ, pipeline_yaml); + + // check schema + assert_eq!(output.schema[0].column_name, "commit"); + let type_id: i32 = ColumnDataType::Binary.into(); + assert_eq!(output.schema[0].datatype, type_id); + + // check value + let ValueData::BinaryValue(json_value) = output.rows[0].values[0].value_data.clone().unwrap() + else { + panic!("expect binary value"); + }; + let v = jsonb::from_slice(&json_value).unwrap(); + + let mut expected = jsonb::Object::new(); + expected.insert( + "commitTime".to_string(), + jsonb::Value::String(Cow::Borrowed("1573840000.000")), + ); + expected.insert( + "commitAuthor".to_string(), + jsonb::Value::String(Cow::Borrowed("test")), + ); + assert_eq!(v, jsonb::Value::Object(expected)); +} + +#[test] +fn test_json_parse_new_var() { + let pipeline_yaml = r#" +--- +processors: + - json_parse: + field: commit, commit_json + +transform: + - field: commit_json + type: json +"#; + + let output = common::parse_and_exec(INPUT_VALUE_OBJ, pipeline_yaml); + + // check schema + assert_eq!(output.schema[0].column_name, "commit_json"); + let type_id: i32 = ColumnDataType::Binary.into(); + assert_eq!(output.schema[0].datatype, type_id); + + // check value + let ValueData::BinaryValue(json_value) = output.rows[0].values[0].value_data.clone().unwrap() + else { + panic!("expect binary value"); + }; + let v = jsonb::from_slice(&json_value).unwrap(); + + let mut expected = jsonb::Object::new(); + expected.insert( + "commitTime".to_string(), + jsonb::Value::String(Cow::Borrowed("1573840000.000")), + ); + expected.insert( + "commitAuthor".to_string(), + jsonb::Value::String(Cow::Borrowed("test")), + ); + assert_eq!(v, jsonb::Value::Object(expected)); +} + +#[test] +fn test_json_parse_with_simple_extractor() { + let pipeline_yaml = r#" +--- +processors: + - json_parse: + field: commit, commit_json + - simple_extract: + field: commit_json, commit_author + key: "commitAuthor" + + +transform: + - field: commit_author + type: string +"#; + + let output = common::parse_and_exec(INPUT_VALUE_OBJ, pipeline_yaml); + + // check schema + assert_eq!(output.schema[0].column_name, "commit_author"); + let type_id: i32 = ColumnDataType::String.into(); + assert_eq!(output.schema[0].datatype, type_id); + + assert_eq!( + output.rows[0].values[0].value_data, + Some(ValueData::StringValue("test".to_string())) + ); +} + +#[test] +fn test_json_parse_array() { + let pipeline_yaml = r#" +--- +processors: + - json_parse: + field: commit + +transform: + - field: commit + type: json +"#; + + let output = common::parse_and_exec(INPUT_VALUE_ARR, pipeline_yaml); + + // check schema + assert_eq!(output.schema[0].column_name, "commit"); + let type_id: i32 = ColumnDataType::Binary.into(); + assert_eq!(output.schema[0].datatype, type_id); + + // check value + let ValueData::BinaryValue(json_value) = output.rows[0].values[0].value_data.clone().unwrap() + else { + panic!("expect binary value"); + }; + let v = jsonb::from_slice(&json_value).unwrap(); + + let expected = jsonb::Value::Array(vec![ + jsonb::Value::String(Cow::Borrowed("test1")), + jsonb::Value::String(Cow::Borrowed("test2")), + ]); + assert_eq!(v, expected); +} diff --git a/src/pipeline/tests/simple_extract.rs b/src/pipeline/tests/simple_extract.rs index 5e989f0365..ee2fbcbcae 100644 --- a/src/pipeline/tests/simple_extract.rs +++ b/src/pipeline/tests/simple_extract.rs @@ -34,7 +34,7 @@ lazy_static! { } #[test] -fn test_gsub() { +fn test_simple_extract() { let input_value_str = r#" [ { diff --git a/src/servers/src/http/loki.rs b/src/servers/src/http/loki.rs index ac7afe6d45..f010b10c5d 100644 --- a/src/servers/src/http/loki.rs +++ b/src/servers/src/http/loki.rs @@ -345,7 +345,7 @@ pub fn parse_loki_labels(labels: &str) -> Result> { while !labels.is_empty() { // parse key - let first_index = labels.find("=").context(InvalidLokiLabelsSnafu { + let first_index = labels.find("=").with_context(|| InvalidLokiLabelsSnafu { msg: format!("missing `=` near: {}", labels), })?; let key = &labels[..first_index]; From d27b9fc3a1f74450edfef7fc7ecb102a3436e7ca Mon Sep 17 00:00:00 2001 From: LFC <990479+MichaelScofield@users.noreply.github.com> Date: Thu, 17 Apr 2025 11:46:19 +0800 Subject: [PATCH 30/82] feat: implement Arrow Flight "DoPut" in Frontend (#5836) * feat: implement Arrow Flight "DoPut" in Frontend * support auth for "do_put" * set request_id in DoPut requests and responses * set "db" in request header --- Cargo.lock | 4 + src/client/Cargo.toml | 2 + src/client/src/database.rs | 76 ++++++- src/client/src/error.rs | 13 +- src/client/src/lib.rs | 2 +- src/common/grpc/Cargo.toml | 2 + src/common/grpc/src/error.rs | 11 +- src/common/grpc/src/flight.rs | 2 + src/common/grpc/src/flight/do_put.rs | 93 +++++++++ src/frontend/src/instance/grpc.rs | 36 +++- src/servers/src/grpc/flight.rs | 193 ++++++++++++++++- src/servers/src/grpc/greptime_handler.rs | 107 +++++++++- src/servers/src/http.rs | 15 -- src/servers/src/query_handler/grpc.rs | 22 ++ src/servers/tests/http/influxdb_test.rs | 15 -- src/servers/tests/http/opentsdb_test.rs | 15 -- src/servers/tests/http/prom_store_test.rs | 15 -- src/servers/tests/mod.rs | 14 +- src/table/src/error.rs | 6 +- src/table/src/table_name.rs | 38 ++++ tests-integration/src/cluster.rs | 39 ++-- tests-integration/src/grpc.rs | 56 ++--- tests-integration/src/grpc/flight.rs | 242 ++++++++++++++++++++++ tests-integration/src/test_util.rs | 36 +++- tests-integration/src/tests/test_util.rs | 9 +- tests-integration/tests/grpc.rs | 66 +++--- 26 files changed, 944 insertions(+), 185 deletions(-) create mode 100644 src/common/grpc/src/flight/do_put.rs create mode 100644 tests-integration/src/grpc/flight.rs diff --git a/Cargo.lock b/Cargo.lock index b133daa143..cb9ad31c15 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1718,6 +1718,7 @@ dependencies = [ "arrow-flight", "async-stream", "async-trait", + "base64 0.22.1", "common-catalog", "common-error", "common-grpc", @@ -1728,6 +1729,7 @@ dependencies = [ "common-recordbatch", "common-telemetry", "enum_dispatch", + "futures", "futures-util", "lazy_static", "moka", @@ -2097,6 +2099,8 @@ dependencies = [ "lazy_static", "prost 0.13.5", "rand 0.9.0", + "serde", + "serde_json", "snafu 0.8.5", "tokio", "tokio-util", diff --git a/src/client/Cargo.toml b/src/client/Cargo.toml index f8702fe6ac..99d0c97806 100644 --- a/src/client/Cargo.toml +++ b/src/client/Cargo.toml @@ -16,6 +16,7 @@ arc-swap = "1.6" arrow-flight.workspace = true async-stream.workspace = true async-trait.workspace = true +base64.workspace = true common-catalog.workspace = true common-error.workspace = true common-grpc.workspace = true @@ -25,6 +26,7 @@ common-query.workspace = true common-recordbatch.workspace = true common-telemetry.workspace = true enum_dispatch = "0.3" +futures.workspace = true futures-util.workspace = true lazy_static.workspace = true moka = { workspace = true, features = ["future"] } diff --git a/src/client/src/database.rs b/src/client/src/database.rs index 2479240562..c9dc9b08e5 100644 --- a/src/client/src/database.rs +++ b/src/client/src/database.rs @@ -12,36 +12,49 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::pin::Pin; +use std::str::FromStr; + use api::v1::auth_header::AuthScheme; use api::v1::ddl_request::Expr as DdlExpr; use api::v1::greptime_database_client::GreptimeDatabaseClient; use api::v1::greptime_request::Request; use api::v1::query_request::Query; use api::v1::{ - AlterTableExpr, AuthHeader, CreateTableExpr, DdlRequest, GreptimeRequest, InsertRequests, - QueryRequest, RequestHeader, + AlterTableExpr, AuthHeader, Basic, CreateTableExpr, DdlRequest, GreptimeRequest, + InsertRequests, QueryRequest, RequestHeader, }; -use arrow_flight::Ticket; +use arrow_flight::{FlightData, Ticket}; use async_stream::stream; +use base64::prelude::BASE64_STANDARD; +use base64::Engine; +use common_catalog::build_db_string; +use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_error::ext::{BoxedError, ErrorExt}; +use common_grpc::flight::do_put::DoPutResponse; use common_grpc::flight::{FlightDecoder, FlightMessage}; use common_query::Output; use common_recordbatch::error::ExternalSnafu; use common_recordbatch::RecordBatchStreamWrapper; use common_telemetry::error; use common_telemetry::tracing_context::W3cTrace; -use futures_util::StreamExt; +use futures::future; +use futures_util::{Stream, StreamExt, TryStreamExt}; use prost::Message; use snafu::{ensure, ResultExt}; -use tonic::metadata::AsciiMetadataKey; +use tonic::metadata::{AsciiMetadataKey, MetadataValue}; use tonic::transport::Channel; use crate::error::{ ConvertFlightDataSnafu, Error, FlightGetSnafu, IllegalFlightMessagesSnafu, InvalidAsciiSnafu, - ServerSnafu, + InvalidTonicMetadataValueSnafu, ServerSnafu, }; use crate::{from_grpc_response, Client, Result}; +type FlightDataStream = Pin + Send>>; + +type DoPutResponseStream = Pin>>>; + #[derive(Clone, Debug, Default)] pub struct Database { // The "catalog" and "schema" to be used in processing the requests at the server side. @@ -108,16 +121,24 @@ impl Database { self.catalog = catalog.into(); } - pub fn catalog(&self) -> &String { - &self.catalog + fn catalog_or_default(&self) -> &str { + if self.catalog.is_empty() { + DEFAULT_CATALOG_NAME + } else { + &self.catalog + } } pub fn set_schema(&mut self, schema: impl Into) { self.schema = schema.into(); } - pub fn schema(&self) -> &String { - &self.schema + fn schema_or_default(&self) -> &str { + if self.schema.is_empty() { + DEFAULT_SCHEMA_NAME + } else { + &self.schema + } } pub fn set_timezone(&mut self, timezone: impl Into) { @@ -310,6 +331,41 @@ impl Database { } } } + + /// Ingest a stream of [RecordBatch]es that belong to a table, using Arrow Flight's "`DoPut`" + /// method. The return value is also a stream, produces [DoPutResponse]s. + pub async fn do_put(&self, stream: FlightDataStream) -> Result { + let mut request = tonic::Request::new(stream); + + if let Some(AuthHeader { + auth_scheme: Some(AuthScheme::Basic(Basic { username, password })), + }) = &self.ctx.auth_header + { + let encoded = BASE64_STANDARD.encode(format!("{username}:{password}")); + let value = + MetadataValue::from_str(&encoded).context(InvalidTonicMetadataValueSnafu)?; + request.metadata_mut().insert("x-greptime-auth", value); + } + + let db_to_put = if !self.dbname.is_empty() { + &self.dbname + } else { + &build_db_string(self.catalog_or_default(), self.schema_or_default()) + }; + request.metadata_mut().insert( + "x-greptime-db-name", + MetadataValue::from_str(db_to_put).context(InvalidTonicMetadataValueSnafu)?, + ); + + let mut client = self.client.make_flight_client()?; + let response = client.mut_inner().do_put(request).await?; + let response = response + .into_inner() + .map_err(Into::into) + .and_then(|x| future::ready(DoPutResponse::try_from(x).context(ConvertFlightDataSnafu))) + .boxed(); + Ok(response) + } } #[derive(Default, Debug, Clone)] diff --git a/src/client/src/error.rs b/src/client/src/error.rs index ed0c1b5ccb..3f680b1427 100644 --- a/src/client/src/error.rs +++ b/src/client/src/error.rs @@ -19,6 +19,7 @@ use common_error::status_code::{convert_tonic_code_to_status_code, StatusCode}; use common_error::{GREPTIME_DB_HEADER_ERROR_CODE, GREPTIME_DB_HEADER_ERROR_MSG}; use common_macro::stack_trace_debug; use snafu::{location, Location, Snafu}; +use tonic::metadata::errors::InvalidMetadataValue; use tonic::{Code, Status}; #[derive(Snafu)] @@ -115,6 +116,14 @@ pub enum Error { #[snafu(implicit)] location: Location, }, + + #[snafu(display("Invalid Tonic metadata value"))] + InvalidTonicMetadataValue { + #[snafu(source)] + error: InvalidMetadataValue, + #[snafu(implicit)] + location: Location, + }, } pub type Result = std::result::Result; @@ -135,7 +144,9 @@ impl ErrorExt for Error { | Error::CreateTlsChannel { source, .. } => source.status_code(), Error::IllegalGrpcClientState { .. } => StatusCode::Unexpected, - Error::InvalidAscii { .. } => StatusCode::InvalidArguments, + Error::InvalidAscii { .. } | Error::InvalidTonicMetadataValue { .. } => { + StatusCode::InvalidArguments + } } } diff --git a/src/client/src/lib.rs b/src/client/src/lib.rs index 125c185a5a..7078e71795 100644 --- a/src/client/src/lib.rs +++ b/src/client/src/lib.rs @@ -16,7 +16,7 @@ mod client; pub mod client_manager; -mod database; +pub mod database; pub mod error; pub mod flow; pub mod load_balance; diff --git a/src/common/grpc/Cargo.toml b/src/common/grpc/Cargo.toml index 4dadf0571b..f15d0761d1 100644 --- a/src/common/grpc/Cargo.toml +++ b/src/common/grpc/Cargo.toml @@ -23,6 +23,8 @@ flatbuffers = "24" hyper.workspace = true lazy_static.workspace = true prost.workspace = true +serde.workspace = true +serde_json.workspace = true snafu.workspace = true tokio.workspace = true tokio-util.workspace = true diff --git a/src/common/grpc/src/error.rs b/src/common/grpc/src/error.rs index d0ca7d970c..af194f2501 100644 --- a/src/common/grpc/src/error.rs +++ b/src/common/grpc/src/error.rs @@ -97,6 +97,14 @@ pub enum Error { #[snafu(display("Not supported: {}", feat))] NotSupported { feat: String }, + + #[snafu(display("Failed to serde Json"))] + SerdeJson { + #[snafu(source)] + error: serde_json::error::Error, + #[snafu(implicit)] + location: Location, + }, } impl ErrorExt for Error { @@ -110,7 +118,8 @@ impl ErrorExt for Error { Error::CreateChannel { .. } | Error::Conversion { .. } - | Error::DecodeFlightData { .. } => StatusCode::Internal, + | Error::DecodeFlightData { .. } + | Error::SerdeJson { .. } => StatusCode::Internal, Error::CreateRecordBatch { source, .. } => source.status_code(), Error::ConvertArrowSchema { source, .. } => source.status_code(), diff --git a/src/common/grpc/src/flight.rs b/src/common/grpc/src/flight.rs index 26f3676ce1..872897ccbf 100644 --- a/src/common/grpc/src/flight.rs +++ b/src/common/grpc/src/flight.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod do_put; + use std::collections::HashMap; use std::sync::Arc; diff --git a/src/common/grpc/src/flight/do_put.rs b/src/common/grpc/src/flight/do_put.rs new file mode 100644 index 0000000000..15011fc74b --- /dev/null +++ b/src/common/grpc/src/flight/do_put.rs @@ -0,0 +1,93 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use arrow_flight::PutResult; +use common_base::AffectedRows; +use serde::{Deserialize, Serialize}; +use snafu::ResultExt; + +use crate::error::{Error, SerdeJsonSnafu}; + +/// The metadata for "DoPut" requests and responses. +/// +/// Currently, there's only a "request_id", for coordinating requests and responses in the streams. +/// Client can set a unique request id in this metadata, and the server will return the same id in +/// the corresponding response. In doing so, a client can know how to do with its pending requests. +#[derive(Serialize, Deserialize)] +pub struct DoPutMetadata { + request_id: i64, +} + +impl DoPutMetadata { + pub fn new(request_id: i64) -> Self { + Self { request_id } + } + + pub fn request_id(&self) -> i64 { + self.request_id + } +} + +/// The response in the "DoPut" returned stream. +#[derive(Serialize, Deserialize)] +pub struct DoPutResponse { + /// The same "request_id" in the request; see the [DoPutMetadata]. + request_id: i64, + /// The successfully ingested rows number. + affected_rows: AffectedRows, +} + +impl DoPutResponse { + pub fn new(request_id: i64, affected_rows: AffectedRows) -> Self { + Self { + request_id, + affected_rows, + } + } + + pub fn request_id(&self) -> i64 { + self.request_id + } + + pub fn affected_rows(&self) -> AffectedRows { + self.affected_rows + } +} + +impl TryFrom for DoPutResponse { + type Error = Error; + + fn try_from(value: PutResult) -> Result { + serde_json::from_slice(&value.app_metadata).context(SerdeJsonSnafu) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_serde_do_put_metadata() { + let serialized = r#"{"request_id":42}"#; + let metadata = serde_json::from_str::(serialized).unwrap(); + assert_eq!(metadata.request_id(), 42); + } + + #[test] + fn test_serde_do_put_response() { + let x = DoPutResponse::new(42, 88); + let serialized = serde_json::to_string(&x).unwrap(); + assert_eq!(serialized, r#"{"request_id":42,"affected_rows":88}"#); + } +} diff --git a/src/frontend/src/instance/grpc.rs b/src/frontend/src/instance/grpc.rs index decd713555..915d884d7e 100644 --- a/src/frontend/src/instance/grpc.rs +++ b/src/frontend/src/instance/grpc.rs @@ -18,12 +18,13 @@ use api::v1::query_request::Query; use api::v1::{DeleteRequests, DropFlowExpr, InsertRequests, RowDeleteRequests, RowInsertRequests}; use async_trait::async_trait; use auth::{PermissionChecker, PermissionCheckerRef, PermissionReq}; +use common_base::AffectedRows; use common_query::Output; use common_telemetry::tracing::{self}; use datafusion::execution::SessionStateBuilder; use query::parser::PromQuery; use servers::interceptor::{GrpcQueryInterceptor, GrpcQueryInterceptorRef}; -use servers::query_handler::grpc::GrpcQueryHandler; +use servers::query_handler::grpc::{GrpcQueryHandler, RawRecordBatch}; use servers::query_handler::sql::SqlQueryHandler; use session::context::QueryContextRef; use snafu::{ensure, OptionExt, ResultExt}; @@ -31,8 +32,9 @@ use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan}; use table::table_name::TableName; use crate::error::{ - Error, InFlightWriteBytesExceededSnafu, IncompleteGrpcRequestSnafu, NotSupportedSnafu, - PermissionSnafu, Result, SubstraitDecodeLogicalPlanSnafu, TableOperationSnafu, + CatalogSnafu, Error, InFlightWriteBytesExceededSnafu, IncompleteGrpcRequestSnafu, + NotSupportedSnafu, PermissionSnafu, Result, SubstraitDecodeLogicalPlanSnafu, + TableNotFoundSnafu, TableOperationSnafu, }; use crate::instance::{attach_timer, Instance}; use crate::metrics::{ @@ -203,6 +205,34 @@ impl GrpcQueryHandler for Instance { let output = interceptor.post_execute(output, ctx)?; Ok(output) } + + async fn put_record_batch( + &self, + table: &TableName, + record_batch: RawRecordBatch, + ) -> Result { + let _table = self + .catalog_manager() + .table( + &table.catalog_name, + &table.schema_name, + &table.table_name, + None, + ) + .await + .context(CatalogSnafu)? + .with_context(|| TableNotFoundSnafu { + table_name: table.to_string(), + })?; + + // TODO(LFC): Implement it. + common_telemetry::debug!( + "calling put_record_batch with table: {:?} and record_batch size: {}", + table, + record_batch.len() + ); + Ok(record_batch.len()) + } } fn fill_catalog_and_schema_from_context(ddl_expr: &mut DdlExpr, ctx: &QueryContextRef) { diff --git a/src/servers/src/grpc/flight.rs b/src/servers/src/grpc/flight.rs index 76a6cc00ce..648cfff377 100644 --- a/src/servers/src/grpc/flight.rs +++ b/src/servers/src/grpc/flight.rs @@ -16,6 +16,7 @@ mod stream; use std::pin::Pin; use std::sync::Arc; +use std::task::{Context, Poll}; use api::v1::GreptimeRequest; use arrow_flight::flight_service_server::FlightService; @@ -24,21 +25,33 @@ use arrow_flight::{ HandshakeRequest, HandshakeResponse, PollInfo, PutResult, SchemaResult, Ticket, }; use async_trait::async_trait; +use bytes::Bytes; +use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; +use common_catalog::parse_catalog_and_schema_from_db_string; +use common_grpc::flight::do_put::{DoPutMetadata, DoPutResponse}; use common_grpc::flight::{FlightEncoder, FlightMessage}; use common_query::{Output, OutputData}; use common_telemetry::tracing::info_span; use common_telemetry::tracing_context::{FutureExt, TracingContext}; -use futures::Stream; +use futures::{future, ready, Stream}; +use futures_util::{StreamExt, TryStreamExt}; use prost::Message; -use snafu::ResultExt; +use snafu::{ensure, ResultExt}; +use table::table_name::TableName; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; use tonic::{Request, Response, Status, Streaming}; use crate::error; +use crate::error::{InvalidParameterSnafu, ParseJsonSnafu, Result, ToJsonSnafu}; pub use crate::grpc::flight::stream::FlightRecordBatchStream; use crate::grpc::greptime_handler::{get_request_type, GreptimeRequestHandler}; use crate::grpc::TonicResult; +use crate::http::header::constants::GREPTIME_DB_HEADER_NAME; +use crate::http::AUTHORIZATION_HEADER; +use crate::query_handler::grpc::RawRecordBatch; -pub type TonicStream = Pin> + Send + Sync + 'static>>; +pub type TonicStream = Pin> + Send + 'static>>; /// A subset of [FlightService] #[async_trait] @@ -47,6 +60,14 @@ pub trait FlightCraft: Send + Sync + 'static { &self, request: Request, ) -> TonicResult>>; + + async fn do_put( + &self, + request: Request>, + ) -> TonicResult>> { + let _ = request; + Err(Status::unimplemented("Not yet implemented")) + } } pub type FlightCraftRef = Arc; @@ -67,6 +88,13 @@ impl FlightCraft for FlightCraftRef { ) -> TonicResult>> { (**self).do_get(request).await } + + async fn do_put( + &self, + request: Request>, + ) -> TonicResult>> { + self.as_ref().do_put(request).await + } } #[async_trait] @@ -120,9 +148,9 @@ impl FlightService for FlightCraftWrapper { async fn do_put( &self, - _: Request>, + request: Request>, ) -> TonicResult> { - Err(Status::unimplemented("Not yet implemented")) + self.0.do_put(request).await } type DoExchangeStream = TonicStream; @@ -168,13 +196,164 @@ impl FlightCraft for GreptimeRequestHandler { ); async { let output = self.handle_request(request, Default::default()).await?; - let stream: Pin> + Send + Sync>> = - to_flight_data_stream(output, TracingContext::from_current_span()); + let stream = to_flight_data_stream(output, TracingContext::from_current_span()); Ok(Response::new(stream)) } .trace(span) .await } + + async fn do_put( + &self, + request: Request>, + ) -> TonicResult>> { + let (headers, _, stream) = request.into_parts(); + + let header = |key: &str| -> TonicResult> { + let Some(v) = headers.get(key) else { + return Ok(None); + }; + let Ok(v) = std::str::from_utf8(v.as_bytes()) else { + return Err(InvalidParameterSnafu { + reason: "expect valid UTF-8 value", + } + .build() + .into()); + }; + Ok(Some(v)) + }; + + let username_and_password = header(AUTHORIZATION_HEADER)?; + let db = header(GREPTIME_DB_HEADER_NAME)?; + if !self.validate_auth(username_and_password, db).await? { + return Err(Status::unauthenticated("auth failed")); + } + + const MAX_PENDING_RESPONSES: usize = 32; + let (tx, rx) = mpsc::channel::>(MAX_PENDING_RESPONSES); + + let stream = PutRecordBatchRequestStream { + flight_data_stream: stream, + state: PutRecordBatchRequestStreamState::Init(db.map(ToString::to_string)), + }; + self.put_record_batches(stream, tx).await; + + let response = ReceiverStream::new(rx) + .and_then(|response| { + future::ready({ + serde_json::to_vec(&response) + .context(ToJsonSnafu) + .map(|x| PutResult { + app_metadata: Bytes::from(x), + }) + .map_err(Into::into) + }) + }) + .boxed(); + Ok(Response::new(response)) + } +} + +pub(crate) struct PutRecordBatchRequest { + pub(crate) table_name: TableName, + pub(crate) request_id: i64, + pub(crate) record_batch: RawRecordBatch, +} + +impl PutRecordBatchRequest { + fn try_new(table_name: TableName, flight_data: FlightData) -> Result { + let metadata: DoPutMetadata = + serde_json::from_slice(&flight_data.app_metadata).context(ParseJsonSnafu)?; + Ok(Self { + table_name, + request_id: metadata.request_id(), + record_batch: flight_data.data_body, + }) + } +} + +pub(crate) struct PutRecordBatchRequestStream { + flight_data_stream: Streaming, + state: PutRecordBatchRequestStreamState, +} + +enum PutRecordBatchRequestStreamState { + Init(Option), + Started(TableName), +} + +impl Stream for PutRecordBatchRequestStream { + type Item = TonicResult; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + fn extract_table_name(mut descriptor: FlightDescriptor) -> Result { + ensure!( + descriptor.r#type == arrow_flight::flight_descriptor::DescriptorType::Path as i32, + InvalidParameterSnafu { + reason: "expect FlightDescriptor::type == 'Path' only", + } + ); + ensure!( + descriptor.path.len() == 1, + InvalidParameterSnafu { + reason: "expect FlightDescriptor::path has only one table name", + } + ); + Ok(descriptor.path.remove(0)) + } + + let poll = ready!(self.flight_data_stream.poll_next_unpin(cx)); + + let result = match &mut self.state { + PutRecordBatchRequestStreamState::Init(db) => match poll { + Some(Ok(mut flight_data)) => { + let flight_descriptor = flight_data.flight_descriptor.take(); + let result = if let Some(descriptor) = flight_descriptor { + let table_name = extract_table_name(descriptor).map(|x| { + let (catalog, schema) = if let Some(db) = db { + parse_catalog_and_schema_from_db_string(db) + } else { + ( + DEFAULT_CATALOG_NAME.to_string(), + DEFAULT_SCHEMA_NAME.to_string(), + ) + }; + TableName::new(catalog, schema, x) + }); + let table_name = match table_name { + Ok(table_name) => table_name, + Err(e) => return Poll::Ready(Some(Err(e.into()))), + }; + + let request = + PutRecordBatchRequest::try_new(table_name.clone(), flight_data); + let request = match request { + Ok(request) => request, + Err(e) => return Poll::Ready(Some(Err(e.into()))), + }; + + self.state = PutRecordBatchRequestStreamState::Started(table_name); + + Ok(request) + } else { + Err(Status::failed_precondition( + "table to put is not found in flight descriptor", + )) + }; + Some(result) + } + Some(Err(e)) => Some(Err(e)), + None => None, + }, + PutRecordBatchRequestStreamState::Started(table_name) => poll.map(|x| { + x.and_then(|flight_data| { + PutRecordBatchRequest::try_new(table_name.clone(), flight_data) + .map_err(Into::into) + }) + }), + }; + Poll::Ready(result) + } } fn to_flight_data_stream( diff --git a/src/servers/src/grpc/greptime_handler.rs b/src/servers/src/grpc/greptime_handler.rs index b032ffc847..73e1a768c8 100644 --- a/src/servers/src/grpc/greptime_handler.rs +++ b/src/servers/src/grpc/greptime_handler.rs @@ -18,23 +18,33 @@ use std::time::Instant; use api::helper::request_type; use api::v1::auth_header::AuthScheme; -use api::v1::{Basic, GreptimeRequest, RequestHeader}; +use api::v1::{AuthHeader, Basic, GreptimeRequest, RequestHeader}; use auth::{Identity, Password, UserInfoRef, UserProviderRef}; +use base64::prelude::BASE64_STANDARD; +use base64::Engine; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_catalog::parse_catalog_and_schema_from_db_string; use common_error::ext::ErrorExt; use common_error::status_code::StatusCode; +use common_grpc::flight::do_put::DoPutResponse; use common_query::Output; use common_runtime::runtime::RuntimeTrait; use common_runtime::Runtime; use common_telemetry::tracing_context::{FutureExt, TracingContext}; -use common_telemetry::{debug, error, tracing}; +use common_telemetry::{debug, error, tracing, warn}; use common_time::timezone::parse_timezone; -use session::context::{QueryContextBuilder, QueryContextRef}; +use futures_util::StreamExt; +use session::context::{QueryContext, QueryContextBuilder, QueryContextRef}; use snafu::{OptionExt, ResultExt}; +use tokio::sync::mpsc; use crate::error::Error::UnsupportedAuthScheme; -use crate::error::{AuthSnafu, InvalidQuerySnafu, JoinTaskSnafu, NotFoundAuthHeaderSnafu, Result}; +use crate::error::{ + AuthSnafu, InvalidAuthHeaderInvalidUtf8ValueSnafu, InvalidBase64ValueSnafu, InvalidQuerySnafu, + JoinTaskSnafu, NotFoundAuthHeaderSnafu, Result, +}; +use crate::grpc::flight::{PutRecordBatchRequest, PutRecordBatchRequestStream}; +use crate::grpc::TonicResult; use crate::metrics::{METRIC_AUTH_FAILURE, METRIC_SERVER_GRPC_DB_REQUEST_TIMER}; use crate::query_handler::grpc::ServerGrpcQueryHandlerRef; @@ -118,6 +128,95 @@ impl GreptimeRequestHandler { None => result_future.await, } } + + pub(crate) async fn put_record_batches( + &self, + mut stream: PutRecordBatchRequestStream, + result_sender: mpsc::Sender>, + ) { + let handler = self.handler.clone(); + let runtime = self + .runtime + .clone() + .unwrap_or_else(common_runtime::global_runtime); + runtime.spawn(async move { + while let Some(request) = stream.next().await { + let request = match request { + Ok(request) => request, + Err(e) => { + let _ = result_sender.try_send(Err(e)); + break; + } + }; + + let PutRecordBatchRequest { + table_name, + request_id, + record_batch, + } = request; + let result = handler.put_record_batch(&table_name, record_batch).await; + let result = result + .map(|x| DoPutResponse::new(request_id, x)) + .map_err(Into::into); + if result_sender.try_send(result).is_err() { + warn!(r#""DoPut" client maybe unreachable, abort handling its message"#); + break; + } + } + }); + } + + pub(crate) async fn validate_auth( + &self, + username_and_password: Option<&str>, + db: Option<&str>, + ) -> Result { + if self.user_provider.is_none() { + return Ok(true); + } + + let username_and_password = username_and_password.context(NotFoundAuthHeaderSnafu)?; + let username_and_password = BASE64_STANDARD + .decode(username_and_password) + .context(InvalidBase64ValueSnafu) + .and_then(|x| String::from_utf8(x).context(InvalidAuthHeaderInvalidUtf8ValueSnafu))?; + + let mut split = username_and_password.splitn(2, ':'); + let (username, password) = match (split.next(), split.next()) { + (Some(username), Some(password)) => (username, password), + (Some(username), None) => (username, ""), + (None, None) => return Ok(false), + _ => unreachable!(), // because this iterator won't yield Some after None + }; + + let (catalog, schema) = if let Some(db) = db { + parse_catalog_and_schema_from_db_string(db) + } else { + ( + DEFAULT_CATALOG_NAME.to_string(), + DEFAULT_SCHEMA_NAME.to_string(), + ) + }; + let header = RequestHeader { + authorization: Some(AuthHeader { + auth_scheme: Some(AuthScheme::Basic(Basic { + username: username.to_string(), + password: password.to_string(), + })), + }), + catalog, + schema, + ..Default::default() + }; + + Ok(auth( + self.user_provider.clone(), + Some(&header), + &QueryContext::arc(), + ) + .await + .is_ok()) + } } pub fn get_request_type(request: &GreptimeRequest) -> &'static str { diff --git a/src/servers/src/http.rs b/src/servers/src/http.rs index 07d1c7ea24..bff38d980f 100644 --- a/src/servers/src/http.rs +++ b/src/servers/src/http.rs @@ -1169,7 +1169,6 @@ mod test { use std::io::Cursor; use std::sync::Arc; - use api::v1::greptime_request::Request; use arrow_ipc::reader::FileReader; use arrow_schema::DataType; use axum::handler::Handler; @@ -1191,26 +1190,12 @@ mod test { use super::*; use crate::error::Error; use crate::http::test_helpers::TestClient; - use crate::query_handler::grpc::GrpcQueryHandler; use crate::query_handler::sql::{ServerSqlQueryHandlerAdapter, SqlQueryHandler}; struct DummyInstance { _tx: mpsc::Sender<(String, Vec)>, } - #[async_trait] - impl GrpcQueryHandler for DummyInstance { - type Error = Error; - - async fn do_query( - &self, - _query: Request, - _ctx: QueryContextRef, - ) -> std::result::Result { - unimplemented!() - } - } - #[async_trait] impl SqlQueryHandler for DummyInstance { type Error = Error; diff --git a/src/servers/src/query_handler/grpc.rs b/src/servers/src/query_handler/grpc.rs index 01464012d6..7af5c9935a 100644 --- a/src/servers/src/query_handler/grpc.rs +++ b/src/servers/src/query_handler/grpc.rs @@ -16,16 +16,20 @@ use std::sync::Arc; use api::v1::greptime_request::Request; use async_trait::async_trait; +use common_base::AffectedRows; use common_error::ext::{BoxedError, ErrorExt}; use common_query::Output; use session::context::QueryContextRef; use snafu::ResultExt; +use table::table_name::TableName; use crate::error::{self, Result}; pub type GrpcQueryHandlerRef = Arc + Send + Sync>; pub type ServerGrpcQueryHandlerRef = GrpcQueryHandlerRef; +pub type RawRecordBatch = bytes::Bytes; + #[async_trait] pub trait GrpcQueryHandler { type Error: ErrorExt; @@ -35,6 +39,12 @@ pub trait GrpcQueryHandler { query: Request, ctx: QueryContextRef, ) -> std::result::Result; + + async fn put_record_batch( + &self, + table: &TableName, + record_batch: RawRecordBatch, + ) -> std::result::Result; } pub struct ServerGrpcQueryHandlerAdapter(GrpcQueryHandlerRef); @@ -59,4 +69,16 @@ where .map_err(BoxedError::new) .context(error::ExecuteGrpcQuerySnafu) } + + async fn put_record_batch( + &self, + table: &TableName, + record_batch: RawRecordBatch, + ) -> Result { + self.0 + .put_record_batch(table, record_batch) + .await + .map_err(BoxedError::new) + .context(error::ExecuteGrpcRequestSnafu) + } } diff --git a/src/servers/tests/http/influxdb_test.rs b/src/servers/tests/http/influxdb_test.rs index 1a251763ae..93932252fb 100644 --- a/src/servers/tests/http/influxdb_test.rs +++ b/src/servers/tests/http/influxdb_test.rs @@ -14,7 +14,6 @@ use std::sync::Arc; -use api::v1::greptime_request::Request; use api::v1::RowInsertRequests; use async_trait::async_trait; use auth::tests::{DatabaseAuthInfo, MockUserProvider}; @@ -29,7 +28,6 @@ use servers::http::header::constants::GREPTIME_DB_HEADER_NAME; use servers::http::test_helpers::TestClient; use servers::http::{HttpOptions, HttpServerBuilder}; use servers::influxdb::InfluxdbRequest; -use servers::query_handler::grpc::GrpcQueryHandler; use servers::query_handler::sql::SqlQueryHandler; use servers::query_handler::InfluxdbLineProtocolHandler; use session::context::QueryContextRef; @@ -39,19 +37,6 @@ struct DummyInstance { tx: Arc>, } -#[async_trait] -impl GrpcQueryHandler for DummyInstance { - type Error = Error; - - async fn do_query( - &self, - _query: Request, - _ctx: QueryContextRef, - ) -> std::result::Result { - unimplemented!() - } -} - #[async_trait] impl InfluxdbLineProtocolHandler for DummyInstance { async fn exec(&self, request: InfluxdbRequest, ctx: QueryContextRef) -> Result { diff --git a/src/servers/tests/http/opentsdb_test.rs b/src/servers/tests/http/opentsdb_test.rs index 358af19dc8..6ac835e72d 100644 --- a/src/servers/tests/http/opentsdb_test.rs +++ b/src/servers/tests/http/opentsdb_test.rs @@ -14,7 +14,6 @@ use std::sync::Arc; -use api::v1::greptime_request::Request; use async_trait::async_trait; use axum::Router; use common_query::Output; @@ -26,7 +25,6 @@ use servers::error::{self, Result}; use servers::http::test_helpers::TestClient; use servers::http::{HttpOptions, HttpServerBuilder}; use servers::opentsdb::codec::DataPoint; -use servers::query_handler::grpc::GrpcQueryHandler; use servers::query_handler::sql::SqlQueryHandler; use servers::query_handler::OpentsdbProtocolHandler; use session::context::QueryContextRef; @@ -36,19 +34,6 @@ struct DummyInstance { tx: mpsc::Sender, } -#[async_trait] -impl GrpcQueryHandler for DummyInstance { - type Error = crate::Error; - - async fn do_query( - &self, - _query: Request, - _ctx: QueryContextRef, - ) -> std::result::Result { - unimplemented!() - } -} - #[async_trait] impl OpentsdbProtocolHandler for DummyInstance { async fn exec(&self, data_points: Vec, _ctx: QueryContextRef) -> Result { diff --git a/src/servers/tests/http/prom_store_test.rs b/src/servers/tests/http/prom_store_test.rs index 77a06db079..c8c5671b8c 100644 --- a/src/servers/tests/http/prom_store_test.rs +++ b/src/servers/tests/http/prom_store_test.rs @@ -17,7 +17,6 @@ use std::sync::Arc; use api::prom_store::remote::{ LabelMatcher, Query, QueryResult, ReadRequest, ReadResponse, WriteRequest, }; -use api::v1::greptime_request::Request; use api::v1::RowInsertRequests; use async_trait::async_trait; use axum::Router; @@ -33,7 +32,6 @@ use servers::http::test_helpers::TestClient; use servers::http::{HttpOptions, HttpServerBuilder}; use servers::prom_store; use servers::prom_store::{snappy_compress, Metrics}; -use servers::query_handler::grpc::GrpcQueryHandler; use servers::query_handler::sql::SqlQueryHandler; use servers::query_handler::{PromStoreProtocolHandler, PromStoreResponse}; use session::context::QueryContextRef; @@ -43,19 +41,6 @@ struct DummyInstance { tx: mpsc::Sender<(String, Vec)>, } -#[async_trait] -impl GrpcQueryHandler for DummyInstance { - type Error = Error; - - async fn do_query( - &self, - _query: Request, - _ctx: QueryContextRef, - ) -> std::result::Result { - unimplemented!() - } -} - #[async_trait] impl PromStoreProtocolHandler for DummyInstance { async fn write( diff --git a/src/servers/tests/mod.rs b/src/servers/tests/mod.rs index 13c78a293f..43aeb362fa 100644 --- a/src/servers/tests/mod.rs +++ b/src/servers/tests/mod.rs @@ -18,6 +18,7 @@ use api::v1::greptime_request::Request; use api::v1::query_request::Query; use async_trait::async_trait; use catalog::memory::MemoryCatalogManager; +use common_base::AffectedRows; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_query::Output; use datafusion_expr::LogicalPlan; @@ -26,11 +27,12 @@ use query::parser::{PromQuery, QueryLanguageParser, QueryStatement}; use query::query_engine::DescribeResult; use query::{QueryEngineFactory, QueryEngineRef}; use servers::error::{Error, NotSupportedSnafu, Result}; -use servers::query_handler::grpc::{GrpcQueryHandler, ServerGrpcQueryHandlerRef}; +use servers::query_handler::grpc::{GrpcQueryHandler, RawRecordBatch, ServerGrpcQueryHandlerRef}; use servers::query_handler::sql::{ServerSqlQueryHandlerRef, SqlQueryHandler}; use session::context::QueryContextRef; use snafu::ensure; use sql::statements::statement::Statement; +use table::table_name::TableName; use table::TableRef; mod grpc; @@ -155,6 +157,16 @@ impl GrpcQueryHandler for DummyInstance { }; Ok(output) } + + async fn put_record_batch( + &self, + table: &TableName, + record_batch: RawRecordBatch, + ) -> std::result::Result { + let _ = table; + let _ = record_batch; + unimplemented!() + } } fn create_testing_instance(table: TableRef) -> DummyInstance { diff --git a/src/table/src/error.rs b/src/table/src/error.rs index ef08ebc4a1..6cd79fd61c 100644 --- a/src/table/src/error.rs +++ b/src/table/src/error.rs @@ -172,6 +172,9 @@ pub enum Error { #[snafu(implicit)] location: Location, }, + + #[snafu(display("Invalid table name: '{s}'"))] + InvalidTableName { s: String }, } impl ErrorExt for Error { @@ -197,7 +200,8 @@ impl ErrorExt for Error { Error::MissingTimeIndexColumn { .. } => StatusCode::IllegalState, Error::InvalidTableOptionValue { .. } | Error::SetSkippingOptions { .. } - | Error::UnsetSkippingOptions { .. } => StatusCode::InvalidArguments, + | Error::UnsetSkippingOptions { .. } + | Error::InvalidTableName { .. } => StatusCode::InvalidArguments, } } diff --git a/src/table/src/table_name.rs b/src/table/src/table_name.rs index f999e013f2..d2d1c1e48b 100644 --- a/src/table/src/table_name.rs +++ b/src/table/src/table_name.rs @@ -15,8 +15,12 @@ use std::fmt::{Display, Formatter}; use api::v1::TableName as PbTableName; +use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use serde::{Deserialize, Serialize}; +use snafu::ensure; +use crate::error; +use crate::error::InvalidTableNameSnafu; use crate::table_reference::TableReference; #[derive(Debug, Clone, Hash, Eq, PartialEq, Deserialize, Serialize)] @@ -83,3 +87,37 @@ impl From> for TableName { Self::new(table_ref.catalog, table_ref.schema, table_ref.table) } } + +impl TryFrom> for TableName { + type Error = error::Error; + + fn try_from(v: Vec) -> Result { + ensure!( + !v.is_empty() && v.len() <= 3, + InvalidTableNameSnafu { + s: format!("{v:?}") + } + ); + let mut v = v.into_iter(); + match (v.next(), v.next(), v.next()) { + (Some(catalog_name), Some(schema_name), Some(table_name)) => Ok(Self { + catalog_name, + schema_name, + table_name, + }), + (Some(schema_name), Some(table_name), None) => Ok(Self { + catalog_name: DEFAULT_CATALOG_NAME.to_string(), + schema_name, + table_name, + }), + (Some(table_name), None, None) => Ok(Self { + catalog_name: DEFAULT_CATALOG_NAME.to_string(), + schema_name: DEFAULT_SCHEMA_NAME.to_string(), + table_name, + }), + // Unreachable because it's ensured that "v" is not empty, + // and its iterator will not yield `Some` after `None`. + _ => unreachable!(), + } + } +} diff --git a/tests-integration/src/cluster.rs b/tests-integration/src/cluster.rs index c1159e18d5..836c3a5483 100644 --- a/tests-integration/src/cluster.rs +++ b/tests-integration/src/cluster.rs @@ -67,13 +67,12 @@ use tower::service_fn; use uuid::Uuid; use crate::test_util::{ - self, create_datanode_opts, create_tmp_dir_and_datanode_opts, FileDirGuard, StorageGuard, - StorageType, PEER_PLACEHOLDER_ADDR, + self, create_datanode_opts, create_tmp_dir_and_datanode_opts, FileDirGuard, StorageType, + TestGuard, PEER_PLACEHOLDER_ADDR, }; pub struct GreptimeDbCluster { - pub storage_guards: Vec, - pub dir_guards: Vec, + pub guards: Vec, pub datanode_options: Vec, pub datanode_instances: HashMap, @@ -177,8 +176,7 @@ impl GreptimeDbClusterBuilder { pub async fn build_with( &self, datanode_options: Vec, - storage_guards: Vec, - dir_guards: Vec, + guards: Vec, ) -> GreptimeDbCluster { let datanodes = datanode_options.len(); let channel_config = ChannelConfig::new().timeout(Duration::from_secs(20)); @@ -224,8 +222,7 @@ impl GreptimeDbClusterBuilder { GreptimeDbCluster { datanode_options, - storage_guards, - dir_guards, + guards, datanode_instances, kv_backend: self.kv_backend.clone(), metasrv: metasrv.metasrv, @@ -235,19 +232,16 @@ impl GreptimeDbClusterBuilder { pub async fn build(&self) -> GreptimeDbCluster { let datanodes = self.datanodes.unwrap_or(4); - let (datanode_options, storage_guards, dir_guards) = - self.build_datanode_options_and_guards(datanodes).await; - self.build_with(datanode_options, storage_guards, dir_guards) - .await + let (datanode_options, guards) = self.build_datanode_options_and_guards(datanodes).await; + self.build_with(datanode_options, guards).await } async fn build_datanode_options_and_guards( &self, datanodes: u32, - ) -> (Vec, Vec, Vec) { + ) -> (Vec, Vec) { let mut options = Vec::with_capacity(datanodes as usize); - let mut storage_guards = Vec::with_capacity(datanodes as usize); - let mut dir_guards = Vec::with_capacity(datanodes as usize); + let mut guards = Vec::with_capacity(datanodes as usize); for i in 0..datanodes { let datanode_id = i as u64 + 1; @@ -257,7 +251,10 @@ impl GreptimeDbClusterBuilder { } else { let home_tmp_dir = create_temp_dir(&format!("gt_home_{}", &self.cluster_name)); let home_dir = home_tmp_dir.path().to_str().unwrap().to_string(); - dir_guards.push(FileDirGuard::new(home_tmp_dir)); + guards.push(TestGuard { + home_guard: FileDirGuard::new(home_tmp_dir), + storage_guards: Vec::new(), + }); home_dir }; @@ -275,9 +272,7 @@ impl GreptimeDbClusterBuilder { &format!("{}-dn-{}", self.cluster_name, datanode_id), self.datanode_wal_config.clone(), ); - - storage_guards.push(guard.storage_guards); - dir_guards.push(guard.home_guard); + guards.push(guard); opts }; @@ -285,11 +280,7 @@ impl GreptimeDbClusterBuilder { options.push(opts); } - ( - options, - storage_guards.into_iter().flatten().collect(), - dir_guards, - ) + (options, guards) } async fn build_datanodes_with_options( diff --git a/tests-integration/src/grpc.rs b/tests-integration/src/grpc.rs index 501c41d0c8..d09bbc3761 100644 --- a/tests-integration/src/grpc.rs +++ b/tests-integration/src/grpc.rs @@ -12,6 +12,33 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod flight; + +use api::v1::greptime_request::Request; +use api::v1::query_request::Query; +use api::v1::QueryRequest; +use common_query::OutputData; +use common_recordbatch::RecordBatches; +use frontend::instance::Instance; +use servers::query_handler::grpc::GrpcQueryHandler; +use session::context::QueryContext; + +#[allow(unused)] +async fn query_and_expect(instance: &Instance, sql: &str, expected: &str) { + let request = Request::Query(QueryRequest { + query: Some(Query::Sql(sql.to_string())), + }); + let output = GrpcQueryHandler::do_query(instance, request, QueryContext::arc()) + .await + .unwrap(); + let OutputData::Stream(stream) = output.data else { + unreachable!() + }; + let recordbatches = RecordBatches::try_collect(stream).await.unwrap(); + let actual = recordbatches.pretty_print().unwrap(); + assert_eq!(actual, expected, "actual: {}", actual); +} + #[cfg(test)] mod test { use std::collections::HashMap; @@ -41,6 +68,7 @@ mod test { use store_api::storage::RegionId; use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan}; + use super::*; use crate::standalone::GreptimeDbStandaloneBuilder; use crate::tests; use crate::tests::MockDistributedInstance; @@ -219,24 +247,14 @@ mod test { let output = query(instance, request).await; assert!(matches!(output.data, OutputData::AffectedRows(1))); - let request = Request::Query(QueryRequest { - query: Some(Query::Sql( - "SELECT ts, a, b FROM database_created_through_grpc.table_created_through_grpc" - .to_string(), - )), - }); - let output = query(instance, request).await; - let OutputData::Stream(stream) = output.data else { - unreachable!() - }; - let recordbatches = RecordBatches::try_collect(stream).await.unwrap(); + let sql = "SELECT ts, a, b FROM database_created_through_grpc.table_created_through_grpc"; let expected = "\ +---------------------+---+---+ | ts | a | b | +---------------------+---+---+ | 2023-01-04T07:14:26 | s | 1 | +---------------------+---+---+"; - assert_eq!(recordbatches.pretty_print().unwrap(), expected); + query_and_expect(instance, sql, expected).await; let request = Request::Ddl(DdlRequest { expr: Some(DdlExpr::DropTable(DropTableExpr { @@ -323,24 +341,14 @@ mod test { let output = query(instance, request).await; assert!(matches!(output.data, OutputData::AffectedRows(1))); - let request = Request::Query(QueryRequest { - query: Some(Query::Sql( - "SELECT ts, a, b FROM database_created_through_grpc.table_created_through_grpc" - .to_string(), - )), - }); - let output = query(instance, request).await; - let OutputData::Stream(stream) = output.data else { - unreachable!() - }; - let recordbatches = RecordBatches::try_collect(stream).await.unwrap(); + let sql = "SELECT ts, a, b FROM database_created_through_grpc.table_created_through_grpc"; let expected = "\ +---------------------+---+---+ | ts | a | b | +---------------------+---+---+ | 2023-01-04T07:14:26 | s | 1 | +---------------------+---+---+"; - assert_eq!(recordbatches.pretty_print().unwrap(), expected); + query_and_expect(instance, sql, expected).await; let request = Request::Ddl(DdlRequest { expr: Some(DdlExpr::DropTable(DropTableExpr { diff --git a/tests-integration/src/grpc/flight.rs b/tests-integration/src/grpc/flight.rs new file mode 100644 index 0000000000..e97165f16c --- /dev/null +++ b/tests-integration/src/grpc/flight.rs @@ -0,0 +1,242 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#[cfg(test)] +mod test { + use std::net::SocketAddr; + use std::sync::Arc; + use std::time::Duration; + + use api::v1::auth_header::AuthScheme; + use api::v1::{Basic, ColumnDataType, ColumnDef, CreateTableExpr, SemanticType}; + use arrow_flight::FlightDescriptor; + use auth::user_provider_from_option; + use client::{Client, Database}; + use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; + use common_grpc::flight::do_put::DoPutMetadata; + use common_grpc::flight::{FlightEncoder, FlightMessage}; + use common_query::OutputData; + use common_recordbatch::RecordBatch; + use datatypes::prelude::{ConcreteDataType, ScalarVector, VectorRef}; + use datatypes::schema::{ColumnSchema, Schema}; + use datatypes::vectors::{Int32Vector, StringVector, TimestampMillisecondVector}; + use futures_util::StreamExt; + use itertools::Itertools; + use servers::grpc::builder::GrpcServerBuilder; + use servers::grpc::greptime_handler::GreptimeRequestHandler; + use servers::grpc::GrpcServerConfig; + use servers::query_handler::grpc::ServerGrpcQueryHandlerAdapter; + use servers::server::Server; + + use crate::cluster::GreptimeDbClusterBuilder; + use crate::grpc::query_and_expect; + use crate::test_util::{setup_grpc_server, StorageType}; + use crate::tests::test_util::MockInstance; + + #[tokio::test(flavor = "multi_thread")] + async fn test_standalone_flight_do_put() { + common_telemetry::init_default_ut_logging(); + + let (addr, db, _server) = + setup_grpc_server(StorageType::File, "test_standalone_flight_do_put").await; + + let client = Client::with_urls(vec![addr]); + let client = Database::new_with_dbname("greptime-public", client); + + create_table(&client).await; + + let record_batches = create_record_batches(1); + test_put_record_batches(&client, record_batches).await; + + let sql = "select ts, a, b from foo order by ts"; + let expected = "\ +++ +++"; + query_and_expect(db.frontend().as_ref(), sql, expected).await; + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_distributed_flight_do_put() { + common_telemetry::init_default_ut_logging(); + + let db = GreptimeDbClusterBuilder::new("test_distributed_flight_do_put") + .await + .build() + .await; + + let runtime = common_runtime::global_runtime().clone(); + let greptime_request_handler = GreptimeRequestHandler::new( + ServerGrpcQueryHandlerAdapter::arc(db.frontend.instance.clone()), + user_provider_from_option( + &"static_user_provider:cmd:greptime_user=greptime_pwd".to_string(), + ) + .ok(), + Some(runtime.clone()), + ); + let grpc_server = GrpcServerBuilder::new(GrpcServerConfig::default(), runtime) + .flight_handler(Arc::new(greptime_request_handler)) + .build(); + let addr = grpc_server + .start("127.0.0.1:0".parse::().unwrap()) + .await + .unwrap() + .to_string(); + + // wait for GRPC server to start + tokio::time::sleep(Duration::from_secs(1)).await; + + let client = Client::with_urls(vec![addr]); + let mut client = Database::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, client); + client.set_auth(AuthScheme::Basic(Basic { + username: "greptime_user".to_string(), + password: "greptime_pwd".to_string(), + })); + + create_table(&client).await; + + let record_batches = create_record_batches(1); + test_put_record_batches(&client, record_batches).await; + + let sql = "select ts, a, b from foo order by ts"; + let expected = "\ +++ +++"; + query_and_expect(db.fe_instance().as_ref(), sql, expected).await; + } + + async fn test_put_record_batches(client: &Database, record_batches: Vec) { + let requests_count = record_batches.len(); + + let stream = tokio_stream::iter(record_batches) + .enumerate() + .map(|(i, x)| { + let mut encoder = FlightEncoder::default(); + let message = FlightMessage::Recordbatch(x); + let mut data = encoder.encode(message); + + let metadata = DoPutMetadata::new(i as i64); + data.app_metadata = serde_json::to_vec(&metadata).unwrap().into(); + + // first message in "DoPut" stream should carry table name in flight descriptor + if i == 0 { + data.flight_descriptor = Some(FlightDescriptor { + r#type: arrow_flight::flight_descriptor::DescriptorType::Path as i32, + path: vec!["foo".to_string()], + ..Default::default() + }); + } + data + }) + .boxed(); + + let response_stream = client.do_put(stream).await.unwrap(); + + let responses = response_stream.collect::>().await; + let responses_count = responses.len(); + for (i, response) in responses.into_iter().enumerate() { + assert!(response.is_ok(), "{}", response.err().unwrap()); + let response = response.unwrap(); + assert_eq!(response.request_id(), i as i64); + assert_eq!(response.affected_rows(), 448); + } + assert_eq!(requests_count, responses_count); + } + + fn create_record_batches(start: i64) -> Vec { + let schema = Arc::new(Schema::new(vec![ + ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ) + .with_time_index(true), + ColumnSchema::new("a", ConcreteDataType::int32_datatype(), false), + ColumnSchema::new("b", ConcreteDataType::string_datatype(), true), + ])); + + let mut record_batches = Vec::with_capacity(3); + for chunk in &(start..start + 9).chunks(3) { + let vs = chunk.collect_vec(); + let x1 = vs[0]; + let x2 = vs[1]; + let x3 = vs[2]; + + record_batches.push( + RecordBatch::new( + schema.clone(), + vec![ + Arc::new(TimestampMillisecondVector::from_vec(vec![x1, x2, x3])) + as VectorRef, + Arc::new(Int32Vector::from_vec(vec![ + -x1 as i32, -x2 as i32, -x3 as i32, + ])), + Arc::new(StringVector::from_vec(vec![ + format!("s{x1}"), + format!("s{x2}"), + format!("s{x3}"), + ])), + ], + ) + .unwrap(), + ); + } + record_batches + } + + async fn create_table(client: &Database) { + // create table foo ( + // ts timestamp time index, + // a int primary key, + // b string, + // ) + let output = client + .create(CreateTableExpr { + schema_name: "public".to_string(), + table_name: "foo".to_string(), + column_defs: vec![ + ColumnDef { + name: "ts".to_string(), + data_type: ColumnDataType::TimestampMillisecond as i32, + semantic_type: SemanticType::Timestamp as i32, + is_nullable: false, + ..Default::default() + }, + ColumnDef { + name: "a".to_string(), + data_type: ColumnDataType::Int32 as i32, + semantic_type: SemanticType::Tag as i32, + is_nullable: false, + ..Default::default() + }, + ColumnDef { + name: "b".to_string(), + data_type: ColumnDataType::String as i32, + semantic_type: SemanticType::Field as i32, + is_nullable: true, + ..Default::default() + }, + ], + time_index: "ts".to_string(), + primary_keys: vec!["a".to_string()], + engine: "mito".to_string(), + ..Default::default() + }) + .await + .unwrap(); + let OutputData::AffectedRows(affected_rows) = output.data else { + unreachable!() + }; + assert_eq!(affected_rows, 0); + } +} diff --git a/tests-integration/src/test_util.rs b/tests-integration/src/test_util.rs index 4b31b941b6..b842c9412c 100644 --- a/tests-integration/src/test_util.rs +++ b/tests-integration/src/test_util.rs @@ -299,6 +299,34 @@ impl TestGuard { } } +impl Drop for TestGuard { + fn drop(&mut self) { + let (tx, rx) = std::sync::mpsc::channel(); + + let guards = std::mem::take(&mut self.storage_guards); + common_runtime::spawn_global(async move { + let mut errors = vec![]; + for guard in guards { + if let TempDirGuard::S3(guard) + | TempDirGuard::Oss(guard) + | TempDirGuard::Azblob(guard) + | TempDirGuard::Gcs(guard) = guard.0 + { + if let Err(e) = guard.remove_all().await { + errors.push(e); + } + } + } + if errors.is_empty() { + tx.send(Ok(())).unwrap(); + } else { + tx.send(Err(errors)).unwrap(); + } + }); + rx.recv().unwrap().unwrap_or_else(|e| panic!("{:?}", e)); + } +} + pub fn create_tmp_dir_and_datanode_opts( default_store_type: StorageType, store_provider_types: Vec, @@ -504,7 +532,7 @@ pub async fn setup_test_prom_app_with_frontend( pub async fn setup_grpc_server( store_type: StorageType, name: &str, -) -> (String, TestGuard, Arc) { +) -> (String, GreptimeDbStandalone, Arc) { setup_grpc_server_with(store_type, name, None, None).await } @@ -512,7 +540,7 @@ pub async fn setup_grpc_server_with_user_provider( store_type: StorageType, name: &str, user_provider: Option, -) -> (String, TestGuard, Arc) { +) -> (String, GreptimeDbStandalone, Arc) { setup_grpc_server_with(store_type, name, user_provider, None).await } @@ -521,7 +549,7 @@ pub async fn setup_grpc_server_with( name: &str, user_provider: Option, grpc_config: Option, -) -> (String, TestGuard, Arc) { +) -> (String, GreptimeDbStandalone, Arc) { let instance = setup_standalone_instance(name, store_type).await; let runtime: Runtime = RuntimeBuilder::default() @@ -560,7 +588,7 @@ pub async fn setup_grpc_server_with( // wait for GRPC server to start tokio::time::sleep(Duration::from_secs(1)).await; - (fe_grpc_addr, instance.guard, fe_grpc_server) + (fe_grpc_addr, instance, fe_grpc_server) } pub async fn setup_mysql_server( diff --git a/tests-integration/src/tests/test_util.rs b/tests-integration/src/tests/test_util.rs index 605ed2c178..d8df68afaa 100644 --- a/tests-integration/src/tests/test_util.rs +++ b/tests-integration/src/tests/test_util.rs @@ -126,17 +126,12 @@ impl MockInstanceBuilder { unreachable!() }; let GreptimeDbCluster { - storage_guards, - dir_guards, + guards, datanode_options, .. } = instance; - MockInstanceImpl::Distributed( - builder - .build_with(datanode_options, storage_guards, dir_guards) - .await, - ) + MockInstanceImpl::Distributed(builder.build_with(datanode_options, guards).await) } } } diff --git a/tests-integration/tests/grpc.rs b/tests-integration/tests/grpc.rs index 11db34acb8..0a7fffa82d 100644 --- a/tests-integration/tests/grpc.rs +++ b/tests-integration/tests/grpc.rs @@ -90,8 +90,7 @@ macro_rules! grpc_tests { } pub async fn test_invalid_dbname(store_type: StorageType) { - let (addr, mut guard, fe_grpc_server) = - setup_grpc_server(store_type, "auto_create_table").await; + let (addr, _db, fe_grpc_server) = setup_grpc_server(store_type, "test_invalid_dbname").await; let grpc_client = Client::with_urls(vec![addr]); let db = Database::new_with_dbname("tom", grpc_client); @@ -115,12 +114,10 @@ pub async fn test_invalid_dbname(store_type: StorageType) { assert!(result.is_err()); let _ = fe_grpc_server.shutdown().await; - guard.remove_all().await; } pub async fn test_dbname(store_type: StorageType) { - let (addr, mut guard, fe_grpc_server) = - setup_grpc_server(store_type, "auto_create_table").await; + let (addr, _db, fe_grpc_server) = setup_grpc_server(store_type, "test_dbname").await; let grpc_client = Client::with_urls(vec![addr]); let db = Database::new_with_dbname( @@ -129,7 +126,6 @@ pub async fn test_dbname(store_type: StorageType) { ); insert_and_assert(&db).await; let _ = fe_grpc_server.shutdown().await; - guard.remove_all().await; } pub async fn test_grpc_message_size_ok(store_type: StorageType) { @@ -138,8 +134,8 @@ pub async fn test_grpc_message_size_ok(store_type: StorageType) { max_send_message_size: 1024, ..Default::default() }; - let (addr, mut guard, fe_grpc_server) = - setup_grpc_server_with(store_type, "auto_create_table", None, Some(config)).await; + let (addr, _db, fe_grpc_server) = + setup_grpc_server_with(store_type, "test_grpc_message_size_ok", None, Some(config)).await; let grpc_client = Client::with_urls(vec![addr]); let db = Database::new_with_dbname( @@ -148,7 +144,6 @@ pub async fn test_grpc_message_size_ok(store_type: StorageType) { ); db.sql("show tables;").await.unwrap(); let _ = fe_grpc_server.shutdown().await; - guard.remove_all().await; } pub async fn test_grpc_zstd_compression(store_type: StorageType) { @@ -158,8 +153,8 @@ pub async fn test_grpc_zstd_compression(store_type: StorageType) { max_send_message_size: 1024, ..Default::default() }; - let (addr, mut guard, fe_grpc_server) = - setup_grpc_server_with(store_type, "auto_create_table", None, Some(config)).await; + let (addr, _db, fe_grpc_server) = + setup_grpc_server_with(store_type, "test_grpc_zstd_compression", None, Some(config)).await; let grpc_client = Client::with_urls(vec![addr]); let db = Database::new_with_dbname( @@ -168,7 +163,6 @@ pub async fn test_grpc_zstd_compression(store_type: StorageType) { ); db.sql("show tables;").await.unwrap(); let _ = fe_grpc_server.shutdown().await; - guard.remove_all().await; } pub async fn test_grpc_message_size_limit_send(store_type: StorageType) { @@ -177,8 +171,13 @@ pub async fn test_grpc_message_size_limit_send(store_type: StorageType) { max_send_message_size: 50, ..Default::default() }; - let (addr, mut guard, fe_grpc_server) = - setup_grpc_server_with(store_type, "auto_create_table", None, Some(config)).await; + let (addr, _db, fe_grpc_server) = setup_grpc_server_with( + store_type, + "test_grpc_message_size_limit_send", + None, + Some(config), + ) + .await; let grpc_client = Client::with_urls(vec![addr]); let db = Database::new_with_dbname( @@ -188,7 +187,6 @@ pub async fn test_grpc_message_size_limit_send(store_type: StorageType) { let err_msg = db.sql("show tables;").await.unwrap_err().to_string(); assert!(err_msg.contains("message length too large"), "{}", err_msg); let _ = fe_grpc_server.shutdown().await; - guard.remove_all().await; } pub async fn test_grpc_message_size_limit_recv(store_type: StorageType) { @@ -197,8 +195,13 @@ pub async fn test_grpc_message_size_limit_recv(store_type: StorageType) { max_send_message_size: 1024, ..Default::default() }; - let (addr, mut guard, fe_grpc_server) = - setup_grpc_server_with(store_type, "auto_create_table", None, Some(config)).await; + let (addr, _db, fe_grpc_server) = setup_grpc_server_with( + store_type, + "test_grpc_message_size_limit_recv", + None, + Some(config), + ) + .await; let grpc_client = Client::with_urls(vec![addr]); let db = Database::new_with_dbname( @@ -212,7 +215,6 @@ pub async fn test_grpc_message_size_limit_recv(store_type: StorageType) { err_msg ); let _ = fe_grpc_server.shutdown().await; - guard.remove_all().await; } pub async fn test_grpc_auth(store_type: StorageType) { @@ -220,7 +222,7 @@ pub async fn test_grpc_auth(store_type: StorageType) { &"static_user_provider:cmd:greptime_user=greptime_pwd".to_string(), ) .unwrap(); - let (addr, mut guard, fe_grpc_server) = + let (addr, _db, fe_grpc_server) = setup_grpc_server_with_user_provider(store_type, "auto_create_table", Some(user_provider)) .await; @@ -265,29 +267,25 @@ pub async fn test_grpc_auth(store_type: StorageType) { assert!(re.is_ok()); let _ = fe_grpc_server.shutdown().await; - guard.remove_all().await; } pub async fn test_auto_create_table(store_type: StorageType) { - let (addr, mut guard, fe_grpc_server) = - setup_grpc_server(store_type, "auto_create_table").await; + let (addr, _db, fe_grpc_server) = setup_grpc_server(store_type, "test_auto_create_table").await; let grpc_client = Client::with_urls(vec![addr]); let db = Database::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, grpc_client); insert_and_assert(&db).await; let _ = fe_grpc_server.shutdown().await; - guard.remove_all().await; } pub async fn test_auto_create_table_with_hints(store_type: StorageType) { - let (addr, mut guard, fe_grpc_server) = + let (addr, _db, fe_grpc_server) = setup_grpc_server(store_type, "auto_create_table_with_hints").await; let grpc_client = Client::with_urls(vec![addr]); let db = Database::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, grpc_client); insert_with_hints_and_assert(&db).await; let _ = fe_grpc_server.shutdown().await; - guard.remove_all().await; } fn expect_data() -> (Column, Column, Column, Column) { @@ -348,8 +346,7 @@ fn expect_data() -> (Column, Column, Column, Column) { pub async fn test_insert_and_select(store_type: StorageType) { common_telemetry::init_default_ut_logging(); - let (addr, mut guard, fe_grpc_server) = - setup_grpc_server(store_type, "insert_and_select").await; + let (addr, _db, fe_grpc_server) = setup_grpc_server(store_type, "test_insert_and_select").await; let grpc_client = Client::with_urls(vec![addr]); let db = Database::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, grpc_client); @@ -388,7 +385,6 @@ pub async fn test_insert_and_select(store_type: StorageType) { insert_and_assert(&db).await; let _ = fe_grpc_server.shutdown().await; - guard.remove_all().await; } async fn insert_with_hints_and_assert(db: &Database) { @@ -591,21 +587,20 @@ fn testing_create_expr() -> CreateTableExpr { } pub async fn test_health_check(store_type: StorageType) { - let (addr, mut guard, fe_grpc_server) = - setup_grpc_server(store_type, "auto_create_table").await; + let (addr, _db, fe_grpc_server) = setup_grpc_server(store_type, "test_health_check").await; let grpc_client = Client::with_urls(vec![addr]); grpc_client.health_check().await.unwrap(); let _ = fe_grpc_server.shutdown().await; - guard.remove_all().await; } pub async fn test_prom_gateway_query(store_type: StorageType) { common_telemetry::init_default_ut_logging(); // prepare connection - let (addr, mut guard, fe_grpc_server) = setup_grpc_server(store_type, "prom_gateway").await; + let (addr, _db, fe_grpc_server) = + setup_grpc_server(store_type, "test_prom_gateway_query").await; let grpc_client = Client::with_urls(vec![addr]); let db = Database::new( DEFAULT_CATALOG_NAME, @@ -772,7 +767,6 @@ pub async fn test_prom_gateway_query(store_type: StorageType) { // clean up let _ = fe_grpc_server.shutdown().await; - guard.remove_all().await; } pub async fn test_grpc_timezone(store_type: StorageType) { @@ -781,7 +775,7 @@ pub async fn test_grpc_timezone(store_type: StorageType) { max_send_message_size: 1024, ..Default::default() }; - let (addr, mut guard, fe_grpc_server) = + let (addr, _db, fe_grpc_server) = setup_grpc_server_with(store_type, "auto_create_table", None, Some(config)).await; let grpc_client = Client::with_urls(vec![addr]); @@ -824,7 +818,6 @@ pub async fn test_grpc_timezone(store_type: StorageType) { +-----------+" ); let _ = fe_grpc_server.shutdown().await; - guard.remove_all().await; } async fn to_batch(output: Output) -> String { @@ -856,7 +849,7 @@ pub async fn test_grpc_tls_config(store_type: StorageType) { max_send_message_size: 1024, tls, }; - let (addr, mut guard, fe_grpc_server) = + let (addr, _db, fe_grpc_server) = setup_grpc_server_with(store_type, "tls_create_table", None, Some(config)).await; let mut client_tls = ClientTlsOption { @@ -902,5 +895,4 @@ pub async fn test_grpc_tls_config(store_type: StorageType) { } let _ = fe_grpc_server.shutdown().await; - guard.remove_all().await; } From c58217ccec7831ae777dce7ccaa2c1c187f7e3ef Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" <6406592+v0y4g3r@users.noreply.github.com> Date: Thu, 17 Apr 2025 11:58:36 +0800 Subject: [PATCH 31/82] fix: support duration to interval conversion in PostgreSQL protocol (#5913) * fix/pg-timestamp-diff: ### Add Support for `Duration` Type in PostgreSQL Encoding - **Enhanced `encode_value` Functionality**: Updated `src/servers/src/postgres/types.rs` to support encoding of `Value::Duration` using `PgInterval`. - **Implemented `Duration` Conversion**: Added conversion logic from `Duration` to `PgInterval` in `src/servers/src/postgres/types/interval.rs`. - **Added Unit Tests**: Introduced tests for `Duration` to `PgInterval` conversion in `src/servers/src/postgres/types/interval.rs`. - **Updated SQL Test Cases**: Modified `tests/cases/standalone/common/types/timestamp/timestamp.sql` and `timestamp.result` to include tests for timestamp subtraction using PostgreSQL protocol. * fix: overflow * fix/pg-timestamp-diff: Update `timestamp.sql` to ensure newline consistency - Modified `timestamp.sql` to add a newline at the end of the file for consistency. * fix/pg-timestamp-diff: ### Add Documentation for Month Approximation in Interval Calculation - **File Modified**: `src/servers/src/postgres/types/interval.rs` - **Key Change**: Added a comment explaining the approximation of one month as 30.44 days in the interval calculations. --- src/servers/src/error.rs | 5 + src/servers/src/postgres/types.rs | 27 ++-- src/servers/src/postgres/types/interval.rs | 147 +++++++++++++++++- .../common/types/timestamp/timestamp.result | 9 ++ .../common/types/timestamp/timestamp.sql | 3 + 5 files changed, 176 insertions(+), 15 deletions(-) diff --git a/src/servers/src/error.rs b/src/servers/src/error.rs index dd3516d890..bfb36c32ca 100644 --- a/src/servers/src/error.rs +++ b/src/servers/src/error.rs @@ -25,6 +25,7 @@ use common_error::ext::{BoxedError, ErrorExt}; use common_error::status_code::StatusCode; use common_macro::stack_trace_debug; use common_telemetry::{error, warn}; +use common_time::Duration; use datafusion::error::DataFusionError; use datatypes::prelude::ConcreteDataType; use headers::ContentType; @@ -615,6 +616,9 @@ pub enum Error { #[snafu(implicit)] location: Location, }, + + #[snafu(display("Overflow while casting `{:?}` to Interval", val))] + DurationOverflow { val: Duration }, } pub type Result = std::result::Result; @@ -734,6 +738,7 @@ impl ErrorExt for Error { ConvertSqlValue { source, .. } => source.status_code(), InFlightWriteBytesExceeded { .. } => StatusCode::RateLimited, + DurationOverflow { .. } => StatusCode::InvalidArguments, } } diff --git a/src/servers/src/postgres/types.rs b/src/servers/src/postgres/types.rs index a8daf880c6..68bdb9d2e9 100644 --- a/src/servers/src/postgres/types.rs +++ b/src/servers/src/postgres/types.rs @@ -421,13 +421,13 @@ pub(super) fn encode_value( Value::IntervalDayTime(v) => builder.encode_field(&PgInterval::from(*v)), Value::IntervalMonthDayNano(v) => builder.encode_field(&PgInterval::from(*v)), Value::Decimal128(v) => builder.encode_field(&v.to_string()), + Value::Duration(d) => match PgInterval::try_from(*d) { + Ok(i) => builder.encode_field(&i), + Err(e) => Err(PgWireError::ApiError(Box::new(Error::Internal { + err_msg: e.to_string(), + }))), + }, Value::List(values) => encode_array(query_ctx, values, builder), - Value::Duration(_) => Err(PgWireError::ApiError(Box::new(Error::Internal { - err_msg: format!( - "cannot write value {:?} in postgres protocol: unimplemented", - &value - ), - }))), } } @@ -466,8 +466,8 @@ pub(super) fn type_gt_to_pg(origin: &ConcreteDataType) -> Result { &ConcreteDataType::Interval(_) => Ok(Type::INTERVAL_ARRAY), &ConcreteDataType::Decimal128(_) => Ok(Type::NUMERIC_ARRAY), &ConcreteDataType::Json(_) => Ok(Type::JSON_ARRAY), - &ConcreteDataType::Duration(_) - | &ConcreteDataType::Dictionary(_) + &ConcreteDataType::Duration(_) => Ok(Type::INTERVAL_ARRAY), + &ConcreteDataType::Dictionary(_) | &ConcreteDataType::Vector(_) | &ConcreteDataType::List(_) => server_error::UnsupportedDataTypeSnafu { data_type: origin, @@ -475,13 +475,12 @@ pub(super) fn type_gt_to_pg(origin: &ConcreteDataType) -> Result { } .fail(), }, - &ConcreteDataType::Duration(_) | &ConcreteDataType::Dictionary(_) => { - server_error::UnsupportedDataTypeSnafu { - data_type: origin, - reason: "not implemented", - } - .fail() + &ConcreteDataType::Dictionary(_) => server_error::UnsupportedDataTypeSnafu { + data_type: origin, + reason: "not implemented", } + .fail(), + &ConcreteDataType::Duration(_) => Ok(Type::INTERVAL), } } diff --git a/src/servers/src/postgres/types/interval.rs b/src/servers/src/postgres/types/interval.rs index c15833f86c..ec9bfe912b 100644 --- a/src/servers/src/postgres/types/interval.rs +++ b/src/servers/src/postgres/types/interval.rs @@ -16,10 +16,19 @@ use std::fmt::Display; use bytes::{Buf, BufMut}; use common_time::interval::IntervalFormat; -use common_time::{IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth}; +use common_time::timestamp::TimeUnit; +use common_time::{Duration, IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth}; use pgwire::types::ToSqlText; use postgres_types::{to_sql_checked, FromSql, IsNull, ToSql, Type}; +use crate::error; + +/// On average one month has 30.44 day, which is a common approximation. +const SECONDS_PER_MONTH: i64 = 24 * 6 * 6 * 3044; +const SECONDS_PER_DAY: i64 = 24 * 60 * 60; +const MILLISECONDS_PER_MONTH: i64 = SECONDS_PER_MONTH * 1000; +const MILLISECONDS_PER_DAY: i64 = SECONDS_PER_DAY * 1000; + #[derive(Debug, Clone, Copy, Default)] pub struct PgInterval { pub(crate) months: i32, @@ -57,6 +66,62 @@ impl From for PgInterval { } } +impl TryFrom for PgInterval { + type Error = error::Error; + + fn try_from(duration: Duration) -> error::Result { + let value = duration.value(); + let unit = duration.unit(); + + // Convert the duration to microseconds + match unit { + TimeUnit::Second => { + let months = i32::try_from(value / SECONDS_PER_MONTH) + .map_err(|_| error::DurationOverflowSnafu { val: duration }.build())?; + let days = + i32::try_from((value - (months as i64) * SECONDS_PER_MONTH) / SECONDS_PER_DAY) + .map_err(|_| error::DurationOverflowSnafu { val: duration }.build())?; + let microseconds = + (value - (months as i64) * SECONDS_PER_MONTH - (days as i64) * SECONDS_PER_DAY) + .checked_mul(1_000_000) + .ok_or(error::DurationOverflowSnafu { val: duration }.build())?; + + Ok(Self { + months, + days, + microseconds, + }) + } + TimeUnit::Millisecond => { + let months = i32::try_from(value / MILLISECONDS_PER_MONTH) + .map_err(|_| error::DurationOverflowSnafu { val: duration }.build())?; + let days = i32::try_from( + (value - (months as i64) * MILLISECONDS_PER_MONTH) / MILLISECONDS_PER_DAY, + ) + .map_err(|_| error::DurationOverflowSnafu { val: duration }.build())?; + let microseconds = ((value - (months as i64) * MILLISECONDS_PER_MONTH) + - (days as i64) * MILLISECONDS_PER_DAY) + * 1_000; + Ok(Self { + months, + days, + microseconds, + }) + } + TimeUnit::Microsecond => Ok(Self { + months: 0, + days: 0, + microseconds: value, + }), + TimeUnit::Nanosecond => Ok(Self { + months: 0, + days: 0, + microseconds: value / 1000, + }), + } + } +} + impl From for IntervalMonthDayNano { fn from(interval: PgInterval) -> Self { IntervalMonthDayNano::new( @@ -149,3 +214,83 @@ impl ToSqlText for PgInterval { Ok(IsNull::No) } } + +#[cfg(test)] +mod tests { + use common_time::timestamp::TimeUnit; + use common_time::Duration; + + use super::*; + + #[test] + fn test_duration_to_pg_interval() { + // Test with seconds + let duration = Duration::new(86400, TimeUnit::Second); // 1 day + let interval = PgInterval::try_from(duration).unwrap(); + assert_eq!(interval.months, 0); + assert_eq!(interval.days, 1); + assert_eq!(interval.microseconds, 0); + + // Test with milliseconds + let duration = Duration::new(86400000, TimeUnit::Millisecond); // 1 day + let interval = PgInterval::try_from(duration).unwrap(); + assert_eq!(interval.months, 0); + assert_eq!(interval.days, 1); + assert_eq!(interval.microseconds, 0); + + // Test with microseconds + let duration = Duration::new(86400000000, TimeUnit::Microsecond); // 1 day + let interval = PgInterval::try_from(duration).unwrap(); + assert_eq!(interval.months, 0); + assert_eq!(interval.days, 0); + assert_eq!(interval.microseconds, 86400000000); + + // Test with nanoseconds + let duration = Duration::new(86400000000000, TimeUnit::Nanosecond); // 1 day + let interval = PgInterval::try_from(duration).unwrap(); + assert_eq!(interval.months, 0); + assert_eq!(interval.days, 0); + assert_eq!(interval.microseconds, 86400000000); + + // Test with partial day + let duration = Duration::new(43200, TimeUnit::Second); // 12 hours + let interval = PgInterval::try_from(duration).unwrap(); + assert_eq!(interval.months, 0); + assert_eq!(interval.days, 0); + assert_eq!(interval.microseconds, 43_200_000_000); // 12 hours in microseconds + + // Test with negative duration + let duration = Duration::new(-86400, TimeUnit::Second); // -1 day + let interval = PgInterval::try_from(duration).unwrap(); + assert_eq!(interval.months, 0); + assert_eq!(interval.days, -1); + assert_eq!(interval.microseconds, 0); + + // Test with multiple days + let duration = Duration::new(259200, TimeUnit::Second); // 3 days + let interval = PgInterval::try_from(duration).unwrap(); + assert_eq!(interval.months, 0); + assert_eq!(interval.days, 3); + assert_eq!(interval.microseconds, 0); + + // Test with small duration (less than a day) + let duration = Duration::new(3600, TimeUnit::Second); // 1 hour + let interval = PgInterval::try_from(duration).unwrap(); + assert_eq!(interval.months, 0); + assert_eq!(interval.days, 0); + assert_eq!(interval.microseconds, 3600000000); // 1 hour in microseconds + + // Test with very small duration + let duration = Duration::new(1, TimeUnit::Microsecond); // 1 microsecond + let interval = PgInterval::try_from(duration).unwrap(); + assert_eq!(interval.months, 0); + assert_eq!(interval.days, 0); + assert_eq!(interval.microseconds, 1); + + let duration = Duration::new(i64::MAX, TimeUnit::Second); + assert!(PgInterval::try_from(duration).is_err()); + + let duration = Duration::new(i64::MAX, TimeUnit::Millisecond); + assert!(PgInterval::try_from(duration).is_err()); + } +} diff --git a/tests/cases/standalone/common/types/timestamp/timestamp.result b/tests/cases/standalone/common/types/timestamp/timestamp.result index 51bc812814..c0d1c79250 100644 --- a/tests/cases/standalone/common/types/timestamp/timestamp.result +++ b/tests/cases/standalone/common/types/timestamp/timestamp.result @@ -192,3 +192,12 @@ DROP TABLE timestamp; Affected Rows: 0 +-- SQLNESS PROTOCOL POSTGRES +SELECT '2025-04-14 20:42:19.021000'::TIMESTAMP - '2025-04-14 20:42:18.103000'::TIMESTAMP; + ++---------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| arrow_cast(Utf8("2025-04-14 20:42:19.021000"),Utf8("Timestamp(Millisecond, None)")) - arrow_cast(Utf8("2025-04-14 20:42:18.103000"),Utf8("Timestamp(Millisecond, None)")) | ++---------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| 00:00:00.918000 | ++---------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + diff --git a/tests/cases/standalone/common/types/timestamp/timestamp.sql b/tests/cases/standalone/common/types/timestamp/timestamp.sql index e2924bc2e0..070032810b 100644 --- a/tests/cases/standalone/common/types/timestamp/timestamp.sql +++ b/tests/cases/standalone/common/types/timestamp/timestamp.sql @@ -55,3 +55,6 @@ SELECT TIMESTAMP '-8-01-01 00:00:01.5'::VARCHAR; SELECT TIMESTAMP '100000-01-01 00:00:01.5'::VARCHAR; DROP TABLE timestamp; + +-- SQLNESS PROTOCOL POSTGRES +SELECT '2025-04-14 20:42:19.021000'::TIMESTAMP - '2025-04-14 20:42:18.103000'::TIMESTAMP; From 0f252c4d2411f155d26ca2d457907782e983fb3a Mon Sep 17 00:00:00 2001 From: yihong Date: Thu, 17 Apr 2025 15:34:13 +0800 Subject: [PATCH 32/82] fix: oom for sqlness test in container (#5917) Signed-off-by: yihong0618 --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index a0e4b590aa..c40b7f67cb 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,10 @@ ifneq ($(strip $(BUILD_JOBS)),) NEXTEST_OPTS += --build-jobs=${BUILD_JOBS} endif +ifneq ($(strip $(BUILD_JOBS)),) + SQLNESS_OPTS += --jobs ${BUILD_JOBS} +endif + ifneq ($(strip $(CARGO_PROFILE)),) CARGO_BUILD_OPTS += --profile ${CARGO_PROFILE} endif From e4556ce12b3b7d4863380262c4e830e7ad6dc977 Mon Sep 17 00:00:00 2001 From: Ruihang Xia Date: Thu, 17 Apr 2025 22:01:21 +0800 Subject: [PATCH 33/82] fix: label values potential panic (#5921) Signed-off-by: Ruihang Xia --- src/frontend/src/instance/promql.rs | 8 ++------ tests-integration/src/test_util.rs | 9 +++++++++ tests-integration/tests/http.rs | 8 ++++++++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/frontend/src/instance/promql.rs b/src/frontend/src/instance/promql.rs index 1dca064982..f3f276bdd9 100644 --- a/src/frontend/src/instance/promql.rs +++ b/src/frontend/src/instance/promql.rs @@ -161,15 +161,11 @@ impl Instance { let mut results = Vec::with_capacity(batches.iter().map(|b| b.num_rows()).sum()); for batch in batches { - // Only one column the results, ensured by `prometheus::label_values_matchers_to_plan`. + // Only one column in results, ensured by `prometheus::label_values_matchers_to_plan`. let names = batch.column(0); for i in 0..names.len() { - let Value::String(name) = names.get(i) else { - unreachable!(); - }; - - results.push(name.into_string()); + results.push(names.get(i).to_string()); } } diff --git a/tests-integration/src/test_util.rs b/tests-integration/src/test_util.rs index b842c9412c..ee68b8d715 100644 --- a/tests-integration/src/test_util.rs +++ b/tests-integration/src/test_util.rs @@ -503,6 +503,9 @@ pub async fn setup_test_prom_app_with_frontend( let sql = "INSERT INTO demo_metrics(idc, val, ts) VALUES ('idc1', 1.1, 0), ('idc2', 2.1, 600000)"; run_sql(sql, &instance).await; + // insert a row with empty label + let sql = "INSERT INTO demo_metrics(val, ts) VALUES (1.1, 0)"; + run_sql(sql, &instance).await; // build physical table let sql = "CREATE TABLE phy2 (ts timestamp(9) time index, val double, host string primary key) engine=metric with ('physical_metric_table' = '')"; @@ -512,6 +515,12 @@ pub async fn setup_test_prom_app_with_frontend( let sql = "INSERT INTO demo_metrics_with_nanos(idc, val, ts) VALUES ('idc1', 1.1, 0)"; run_sql(sql, &instance).await; + // a mito table with non-prometheus compatible values + let sql = "CREATE TABLE mito (ts timestamp(9) time index, val double, host bigint primary key) engine=mito"; + run_sql(sql, &instance).await; + let sql = "INSERT INTO mito(host, val, ts) VALUES (1, 1.1, 0)"; + run_sql(sql, &instance).await; + let http_opts = HttpOptions { addr: format!("127.0.0.1:{}", ports::get_port()), ..Default::default() diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index 8ef27cdbf0..9e369e654d 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -765,6 +765,13 @@ pub async fn test_prom_http_api(store_type: StorageType) { assert!(prom_resp.error.is_none()); assert!(prom_resp.error_type.is_none()); + // query non-string value + let res = client + .get("/v1/prometheus/api/v1/label/host/values?match[]=mito") + .send() + .await; + assert_eq!(res.status(), StatusCode::OK); + // query `__name__` without match[] // create a physical table and a logical table let res = client @@ -794,6 +801,7 @@ pub async fn test_prom_http_api(store_type: StorageType) { "demo_metrics".to_string(), "demo_metrics_with_nanos".to_string(), "logic_table".to_string(), + "mito".to_string(), "numbers".to_string() ]) ); From cc1b297831c0929ae393c2b90aeea516c06fa815 Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Fri, 18 Apr 2025 11:36:35 +0800 Subject: [PATCH 34/82] fix: avoid double schema projection in file format readers (#5918) --- src/operator/src/statement/copy_table_from.rs | 32 +++++++++----- .../common/copy/copy_from_fs_csv.result | 44 +++++++++++++++++++ .../common/copy/copy_from_fs_csv.sql | 16 +++++++ .../common/copy/copy_from_fs_json.result | 44 +++++++++++++++++++ .../common/copy/copy_from_fs_json.sql | 16 +++++++ .../common/copy/copy_from_fs_parquet.result | 44 +++++++++++++++++++ .../common/copy/copy_from_fs_parquet.sql | 16 +++++++ 7 files changed, 200 insertions(+), 12 deletions(-) diff --git a/src/operator/src/statement/copy_table_from.rs b/src/operator/src/statement/copy_table_from.rs index 2d9fbef6b9..505d74c3d7 100644 --- a/src/operator/src/statement/copy_table_from.rs +++ b/src/operator/src/statement/copy_table_from.rs @@ -237,7 +237,7 @@ impl StatementExecutor { path, schema, } => { - let projected_schema = Arc::new( + let output_schema = Arc::new( compat_schema .project(&projection) .context(error::ProjectSchemaSnafu)?, @@ -255,17 +255,23 @@ impl StatementExecutor { )), None, )); - + let projected_file_schema = Arc::new( + schema + .project(&projection) + .context(error::ProjectSchemaSnafu)?, + ); let stream = self .build_file_stream( CsvOpener::new(csv_config, format.compression_type.into()), path, - schema.clone(), + projected_file_schema, ) .await?; Ok(Box::pin( - RecordBatchStreamTypeAdapter::new(projected_schema, stream, Some(projection)) + // The projection is already applied in the CSV reader when we created the stream, + // so we pass None here to avoid double projection which would cause schema mismatch errors. + RecordBatchStreamTypeAdapter::new(output_schema, stream, None) .with_filter(filters) .context(error::PhysicalExprSnafu)?, )) @@ -280,7 +286,7 @@ impl StatementExecutor { .project(&projection) .context(error::ProjectSchemaSnafu)?, ); - let projected_schema = Arc::new( + let output_schema = Arc::new( compat_schema .project(&projection) .context(error::ProjectSchemaSnafu)?, @@ -290,17 +296,19 @@ impl StatementExecutor { .build_file_stream( JsonOpener::new( DEFAULT_BATCH_SIZE, - projected_file_schema, + projected_file_schema.clone(), format.compression_type.into(), Arc::new(store), ), path, - schema.clone(), + projected_file_schema, ) .await?; Ok(Box::pin( - RecordBatchStreamTypeAdapter::new(projected_schema, stream, Some(projection)) + // The projection is already applied in the JSON reader when we created the stream, + // so we pass None here to avoid double projection which would cause schema mismatch errors. + RecordBatchStreamTypeAdapter::new(output_schema, stream, None) .with_filter(filters) .context(error::PhysicalExprSnafu)?, )) @@ -325,13 +333,13 @@ impl StatementExecutor { .build() .context(error::BuildParquetRecordBatchStreamSnafu)?; - let projected_schema = Arc::new( + let output_schema = Arc::new( compat_schema .project(&projection) .context(error::ProjectSchemaSnafu)?, ); Ok(Box::pin( - RecordBatchStreamTypeAdapter::new(projected_schema, stream, Some(projection)) + RecordBatchStreamTypeAdapter::new(output_schema, stream, Some(projection)) .with_filter(filters) .context(error::PhysicalExprSnafu)?, )) @@ -352,14 +360,14 @@ impl StatementExecutor { .await .context(error::ReadOrcSnafu)?; - let projected_schema = Arc::new( + let output_schema = Arc::new( compat_schema .project(&projection) .context(error::ProjectSchemaSnafu)?, ); Ok(Box::pin( - RecordBatchStreamTypeAdapter::new(projected_schema, stream, Some(projection)) + RecordBatchStreamTypeAdapter::new(output_schema, stream, Some(projection)) .with_filter(filters) .context(error::PhysicalExprSnafu)?, )) diff --git a/tests/cases/standalone/common/copy/copy_from_fs_csv.result b/tests/cases/standalone/common/copy/copy_from_fs_csv.result index f412677a32..d1838a5079 100644 --- a/tests/cases/standalone/common/copy/copy_from_fs_csv.result +++ b/tests/cases/standalone/common/copy/copy_from_fs_csv.result @@ -66,6 +66,42 @@ select * from with_pattern order by ts; | host3 | 99.9 | 444.4 | 2024-07-27T10:47:43 | +-------+------+--------+---------------------+ +CREATE TABLE demo_with_external_column(host string, cpu double, memory double, ts timestamp time index, external_column string default 'default_value'); + +Affected Rows: 0 + +Copy demo_with_external_column FROM '${SQLNESS_HOME}/demo/export/csv/demo.csv' WITH (format='csv'); + +Affected Rows: 3 + +select * from demo_with_external_column order by ts; + ++-------+------+--------+---------------------+-----------------+ +| host | cpu | memory | ts | external_column | ++-------+------+--------+---------------------+-----------------+ +| host1 | 66.6 | 1024.0 | 2022-06-15T07:02:37 | default_value | +| host2 | 88.8 | 333.3 | 2022-06-15T07:02:38 | default_value | +| host3 | 99.9 | 444.4 | 2024-07-27T10:47:43 | default_value | ++-------+------+--------+---------------------+-----------------+ + +CREATE TABLE demo_with_less_columns(host string, memory double, ts timestamp time index); + +Affected Rows: 0 + +Copy demo_with_less_columns FROM '${SQLNESS_HOME}/demo/export/csv/demo.csv' WITH (format='csv'); + +Affected Rows: 3 + +select * from demo_with_less_columns order by ts; + ++-------+--------+---------------------+ +| host | memory | ts | ++-------+--------+---------------------+ +| host1 | 1024.0 | 2022-06-15T07:02:37 | +| host2 | 333.3 | 2022-06-15T07:02:38 | +| host3 | 444.4 | 2024-07-27T10:47:43 | ++-------+--------+---------------------+ + drop table demo; Affected Rows: 0 @@ -82,3 +118,11 @@ drop table with_pattern; Affected Rows: 0 +drop table demo_with_external_column; + +Affected Rows: 0 + +drop table demo_with_less_columns; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/copy/copy_from_fs_csv.sql b/tests/cases/standalone/common/copy/copy_from_fs_csv.sql index f2c6ccf5bd..8c4e9b3aa9 100644 --- a/tests/cases/standalone/common/copy/copy_from_fs_csv.sql +++ b/tests/cases/standalone/common/copy/copy_from_fs_csv.sql @@ -27,6 +27,18 @@ Copy with_pattern FROM '${SQLNESS_HOME}/demo/export/csv/' WITH (pattern = 'demo. select * from with_pattern order by ts; +CREATE TABLE demo_with_external_column(host string, cpu double, memory double, ts timestamp time index, external_column string default 'default_value'); + +Copy demo_with_external_column FROM '${SQLNESS_HOME}/demo/export/csv/demo.csv' WITH (format='csv'); + +select * from demo_with_external_column order by ts; + +CREATE TABLE demo_with_less_columns(host string, memory double, ts timestamp time index); + +Copy demo_with_less_columns FROM '${SQLNESS_HOME}/demo/export/csv/demo.csv' WITH (format='csv'); + +select * from demo_with_less_columns order by ts; + drop table demo; drop table with_filename; @@ -34,3 +46,7 @@ drop table with_filename; drop table with_path; drop table with_pattern; + +drop table demo_with_external_column; + +drop table demo_with_less_columns; diff --git a/tests/cases/standalone/common/copy/copy_from_fs_json.result b/tests/cases/standalone/common/copy/copy_from_fs_json.result index e1e3a810d5..153fac1e87 100644 --- a/tests/cases/standalone/common/copy/copy_from_fs_json.result +++ b/tests/cases/standalone/common/copy/copy_from_fs_json.result @@ -66,6 +66,42 @@ select * from with_pattern order by ts; | host2 | 88.8 | 333.3 | 2022-06-15T07:02:38 | +-------+------+--------+---------------------+ +CREATE TABLE demo_with_external_column(host string, cpu double, memory double, ts timestamp time index, external_column string default 'default_value'); + +Affected Rows: 0 + +Copy demo_with_external_column FROM '${SQLNESS_HOME}/demo/export/json/demo.json' WITH (format='json'); + +Affected Rows: 3 + +select * from demo_with_external_column order by ts; + ++-------+------+--------+---------------------+-----------------+ +| host | cpu | memory | ts | external_column | ++-------+------+--------+---------------------+-----------------+ +| host1 | 66.6 | 1024.0 | 2022-06-15T07:02:37 | default_value | +| host2 | 88.8 | 333.3 | 2022-06-15T07:02:38 | default_value | +| host3 | 99.9 | 444.4 | 2024-07-27T10:47:43 | default_value | ++-------+------+--------+---------------------+-----------------+ + +CREATE TABLE demo_with_less_columns(host string, memory double, ts timestamp time index); + +Affected Rows: 0 + +Copy demo_with_less_columns FROM '${SQLNESS_HOME}/demo/export/json/demo.json' WITH (format='json'); + +Affected Rows: 3 + +select * from demo_with_less_columns order by ts; + ++-------+--------+---------------------+ +| host | memory | ts | ++-------+--------+---------------------+ +| host1 | 1024.0 | 2022-06-15T07:02:37 | +| host2 | 333.3 | 2022-06-15T07:02:38 | +| host3 | 444.4 | 2024-07-27T10:47:43 | ++-------+--------+---------------------+ + drop table demo; Affected Rows: 0 @@ -82,3 +118,11 @@ drop table with_pattern; Affected Rows: 0 +drop table demo_with_external_column; + +Affected Rows: 0 + +drop table demo_with_less_columns; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/copy/copy_from_fs_json.sql b/tests/cases/standalone/common/copy/copy_from_fs_json.sql index 55e8a55ebf..b032332262 100644 --- a/tests/cases/standalone/common/copy/copy_from_fs_json.sql +++ b/tests/cases/standalone/common/copy/copy_from_fs_json.sql @@ -27,6 +27,18 @@ Copy with_pattern FROM '${SQLNESS_HOME}/demo/export/json/' WITH (pattern = 'demo select * from with_pattern order by ts; +CREATE TABLE demo_with_external_column(host string, cpu double, memory double, ts timestamp time index, external_column string default 'default_value'); + +Copy demo_with_external_column FROM '${SQLNESS_HOME}/demo/export/json/demo.json' WITH (format='json'); + +select * from demo_with_external_column order by ts; + +CREATE TABLE demo_with_less_columns(host string, memory double, ts timestamp time index); + +Copy demo_with_less_columns FROM '${SQLNESS_HOME}/demo/export/json/demo.json' WITH (format='json'); + +select * from demo_with_less_columns order by ts; + drop table demo; drop table with_filename; @@ -34,3 +46,7 @@ drop table with_filename; drop table with_path; drop table with_pattern; + +drop table demo_with_external_column; + +drop table demo_with_less_columns; diff --git a/tests/cases/standalone/common/copy/copy_from_fs_parquet.result b/tests/cases/standalone/common/copy/copy_from_fs_parquet.result index f31aebd7b1..ac7fbcddba 100644 --- a/tests/cases/standalone/common/copy/copy_from_fs_parquet.result +++ b/tests/cases/standalone/common/copy/copy_from_fs_parquet.result @@ -123,6 +123,42 @@ Copy with_limit_rows_segment FROM '${SQLNESS_HOME}/demo/export/parquet_files/' L Error: 2000(InvalidSyntax), Unexpected token while parsing SQL statement, expected: 'the number of maximum rows', found: ;: sql parser error: Expected: literal int, found: hello at Line: 1, Column: 86 +CREATE TABLE demo_with_external_column(host string, cpu double, memory double, ts timestamp time index, external_column string default 'default_value'); + +Affected Rows: 0 + +Copy demo_with_external_column FROM '${SQLNESS_HOME}/demo/export/parquet_files/demo.parquet'; + +Affected Rows: 3 + +select * from demo_with_external_column order by ts; + ++-------+-------+--------+---------------------+-----------------+ +| host | cpu | memory | ts | external_column | ++-------+-------+--------+---------------------+-----------------+ +| host1 | 66.6 | 1024.0 | 2022-06-15T07:02:37 | default_value | +| host2 | 88.8 | 333.3 | 2022-06-15T07:02:38 | default_value | +| host3 | 111.1 | 444.4 | 2024-07-27T10:47:43 | default_value | ++-------+-------+--------+---------------------+-----------------+ + +CREATE TABLE demo_with_less_columns(host string, memory double, ts timestamp time index); + +Affected Rows: 0 + +Copy demo_with_less_columns FROM '${SQLNESS_HOME}/demo/export/parquet_files/demo.parquet'; + +Affected Rows: 3 + +select * from demo_with_less_columns order by ts; + ++-------+--------+---------------------+ +| host | memory | ts | ++-------+--------+---------------------+ +| host1 | 1024.0 | 2022-06-15T07:02:37 | +| host2 | 333.3 | 2022-06-15T07:02:38 | +| host3 | 444.4 | 2024-07-27T10:47:43 | ++-------+--------+---------------------+ + drop table demo; Affected Rows: 0 @@ -151,3 +187,11 @@ drop table with_limit_rows_segment; Affected Rows: 0 +drop table demo_with_external_column; + +Affected Rows: 0 + +drop table demo_with_less_columns; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/copy/copy_from_fs_parquet.sql b/tests/cases/standalone/common/copy/copy_from_fs_parquet.sql index 0db5e119d8..13e15bd91e 100644 --- a/tests/cases/standalone/common/copy/copy_from_fs_parquet.sql +++ b/tests/cases/standalone/common/copy/copy_from_fs_parquet.sql @@ -52,6 +52,18 @@ select count(*) from with_limit_rows_segment; Copy with_limit_rows_segment FROM '${SQLNESS_HOME}/demo/export/parquet_files/' LIMIT hello; +CREATE TABLE demo_with_external_column(host string, cpu double, memory double, ts timestamp time index, external_column string default 'default_value'); + +Copy demo_with_external_column FROM '${SQLNESS_HOME}/demo/export/parquet_files/demo.parquet'; + +select * from demo_with_external_column order by ts; + +CREATE TABLE demo_with_less_columns(host string, memory double, ts timestamp time index); + +Copy demo_with_less_columns FROM '${SQLNESS_HOME}/demo/export/parquet_files/demo.parquet'; + +select * from demo_with_less_columns order by ts; + drop table demo; drop table demo_2; @@ -65,3 +77,7 @@ drop table with_pattern; drop table without_limit_rows; drop table with_limit_rows_segment; + +drop table demo_with_external_column; + +drop table demo_with_less_columns; From 4d38d8aa1e66fac85a3222e7afa40647dc54d39f Mon Sep 17 00:00:00 2001 From: jeremyhi Date: Fri, 18 Apr 2025 15:22:12 +0800 Subject: [PATCH 35/82] chore: add heartbeat metrics (#5929) --- src/common/meta/src/datanode.rs | 37 +++++++++++++++++++ .../src/handler/extract_stat_handler.rs | 3 ++ src/meta-srv/src/metrics.rs | 12 +++--- 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/src/common/meta/src/datanode.rs b/src/common/meta/src/datanode.rs index ed1957cbd7..2f45e6bdb9 100644 --- a/src/common/meta/src/datanode.rs +++ b/src/common/meta/src/datanode.rs @@ -142,6 +142,43 @@ impl Stat { self.wcus = self.region_stats.iter().map(|s| s.wcus).sum(); self.region_num = self.region_stats.len() as u64; } + + pub fn memory_size(&self) -> usize { + // timestamp_millis, rcus, wcus + std::mem::size_of::() * 3 + + // id, region_num, node_epoch + std::mem::size_of::() * 3 + + // addr + std::mem::size_of::() + self.addr.capacity() + + // region_stats + self.region_stats.iter().map(|s| s.memory_size()).sum::() + } +} + +impl RegionStat { + pub fn memory_size(&self) -> usize { + // role + std::mem::size_of::() + + // id + std::mem::size_of::() + + // rcus, wcus, approximate_bytes, num_rows + std::mem::size_of::() * 4 + + // memtable_size, manifest_size, sst_size, index_size + std::mem::size_of::() * 4 + + // engine + std::mem::size_of::() + self.engine.capacity() + + // region_manifest + self.region_manifest.memory_size() + } +} + +impl RegionManifestInfo { + pub fn memory_size(&self) -> usize { + match self { + RegionManifestInfo::Mito { .. } => std::mem::size_of::() * 2, + RegionManifestInfo::Metric { .. } => std::mem::size_of::() * 4, + } + } } impl TryFrom<&HeartbeatRequest> for Stat { diff --git a/src/meta-srv/src/handler/extract_stat_handler.rs b/src/meta-srv/src/handler/extract_stat_handler.rs index b31f41e4ef..ab8334ac3d 100644 --- a/src/meta-srv/src/handler/extract_stat_handler.rs +++ b/src/meta-srv/src/handler/extract_stat_handler.rs @@ -19,6 +19,7 @@ use common_telemetry::{info, warn}; use crate::error::Result; use crate::handler::{HandleControl, HeartbeatAccumulator, HeartbeatHandler}; use crate::metasrv::Context; +use crate::metrics::{METRIC_META_HEARTBEAT_RATE, METRIC_META_HEARTBEAT_STAT_MEMORY_SIZE}; pub struct ExtractStatHandler; @@ -42,6 +43,8 @@ impl HeartbeatHandler for ExtractStatHandler { match Stat::try_from(req) { Ok(stat) => { + METRIC_META_HEARTBEAT_RATE.inc(); + METRIC_META_HEARTBEAT_STAT_MEMORY_SIZE.observe(stat.memory_size() as f64); let _ = acc.stat.insert(stat); } Err(Some(header)) => { diff --git a/src/meta-srv/src/metrics.rs b/src/meta-srv/src/metrics.rs index 9160aa1e1d..c5e2d3df4d 100644 --- a/src/meta-srv/src/metrics.rs +++ b/src/meta-srv/src/metrics.rs @@ -60,10 +60,10 @@ lazy_static! { /// The migration fail counter. pub static ref METRIC_META_REGION_MIGRATION_FAIL: IntCounter = register_int_counter!("greptime_meta_region_migration_fail", "meta region migration fail").unwrap(); - /// The add region follower execute histogram. - pub static ref METRIC_META_ADD_REGION_FOLLOWER_EXECUTE: HistogramVec = - register_histogram_vec!("greptime_meta_add_region_follower_execute", "meta add region follower execute", &["state"]).unwrap(); - /// The remove region follower execute histogram. - pub static ref METRIC_META_REMOVE_REGION_FOLLOWER_EXECUTE: HistogramVec = - register_histogram_vec!("greptime_meta_remove_region_follower_execute", "meta remove region follower execute", &["state"]).unwrap(); + // The heartbeat stat memory size histogram. + pub static ref METRIC_META_HEARTBEAT_STAT_MEMORY_SIZE: Histogram = + register_histogram!("greptime_meta_heartbeat_stat_memory_size", "meta heartbeat stat memory size").unwrap(); + // The heartbeat rate counter. + pub static ref METRIC_META_HEARTBEAT_RATE: IntCounter = + register_int_counter!("greptime_meta_heartbeat_rate", "meta heartbeat arrival rate").unwrap(); } From 5287b8792563ec1c869a883729a6b7b12cc25062 Mon Sep 17 00:00:00 2001 From: discord9 <55937128+discord9@users.noreply.github.com> Date: Fri, 18 Apr 2025 17:28:05 +0800 Subject: [PATCH 36/82] docs: memory profile scripts (#5922) * docs: memory profile scripts * chore: typo * chore: comment * Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * chore: newline eof --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docs/how-to/how-to-profile-memory.md | 2 +- .../memory-profile-scripts/scripts/README.md | 52 +++++++++++ .../memory-profile-scripts/scripts/dump.sh | 78 ++++++++++++++++ .../scripts/find_compiled_jeprof.sh | 15 ++++ .../scripts/gen_flamegraph.sh | 89 +++++++++++++++++++ .../scripts/gen_from_collapse.sh | 44 +++++++++ .../scripts/get_flamegraph_tool.sh | 6 ++ 7 files changed, 285 insertions(+), 1 deletion(-) create mode 100644 docs/how-to/memory-profile-scripts/scripts/README.md create mode 100755 docs/how-to/memory-profile-scripts/scripts/dump.sh create mode 100755 docs/how-to/memory-profile-scripts/scripts/find_compiled_jeprof.sh create mode 100755 docs/how-to/memory-profile-scripts/scripts/gen_flamegraph.sh create mode 100755 docs/how-to/memory-profile-scripts/scripts/gen_from_collapse.sh create mode 100755 docs/how-to/memory-profile-scripts/scripts/get_flamegraph_tool.sh diff --git a/docs/how-to/how-to-profile-memory.md b/docs/how-to/how-to-profile-memory.md index 06a063acca..83343b9b2f 100644 --- a/docs/how-to/how-to-profile-memory.md +++ b/docs/how-to/how-to-profile-memory.md @@ -1,6 +1,6 @@ # Profile memory usage of GreptimeDB -This crate provides an easy approach to dump memory profiling info. +This crate provides an easy approach to dump memory profiling info. A set of ready to use scripts is provided in [docs/how-to/memory-profile-scripts](docs/how-to/memory-profile-scripts). ## Prerequisites ### jemalloc diff --git a/docs/how-to/memory-profile-scripts/scripts/README.md b/docs/how-to/memory-profile-scripts/scripts/README.md new file mode 100644 index 0000000000..3ac1cd90fa --- /dev/null +++ b/docs/how-to/memory-profile-scripts/scripts/README.md @@ -0,0 +1,52 @@ +# Memory Analysis Process +This section will guide you through the process of analyzing memory usage for greptimedb. + +1. Get the `jeprof` tool script, see the next section("Getting the `jeprof` tool") for details. + +2. After starting `greptimedb`(with env var `MALLOC_CONF=prof:true`), execute the `dump.sh` script with the PID of the `greptimedb` process as an argument. This continuously monitors memory usage and captures profiles when exceeding thresholds (e.g. +20MB within 10 minutes). Outputs `greptime-{timestamp}.gprof` files. + +3. With 2-3 gprof files, run `gen_flamegraph.sh` in the same environment to generate flame graphs showing memory allocation call stacks. + +4. **NOTE:** The `gen_flamegraph.sh` script requires `jeprof` and optionally `flamegraph.pl` to be in the current directory. If needed to gen flamegraph now, run the `get_flamegraph_tool.sh` script, which downloads the flame graph generation tool `flamegraph.pl` to the current directory. + The usage of `gen_flamegraph.sh` is: + + `Usage: ./gen_flamegraph.sh ` + where `` is the path to the greptimedb binary, `` is the directory containing the gprof files(the directory `dump.sh` is dumping profiles to). + Example call: `./gen_flamegraph.sh ./greptime .` + + Generating the flame graph might take a few minutes. The generated flame graphs are located in the `/flamegraphs` directory. Or if no `flamegraph.pl` is found, it will only contain `.collapse` files which is also fine. +5. You can send the generated flame graphs(the entire folder of `/flamegraphs`) to developers for further analysis. + + +## Getting the `jeprof` tool +there are three ways to get `jeprof`, list in here from simple to complex, using any one of those methods is ok, as long as it's the same environment as the `greptimedb` will be running on: +1. If you are compiling greptimedb from source, then `jeprof` is already produced during compilation. After running `cargo build`, execute `find_compiled_jeprof.sh`. This will copy `jeprof` to the current directory. +2. Or, if you have the Rust toolchain installed locally, simply follow these commands: +```bash +cargo new get_jeprof +cd get_jeprof +``` +Then add this line to `Cargo.toml`: +```toml +[dependencies] +tikv-jemalloc-ctl = { version = "0.6", features = ["use_std", "stats"] } +``` +then run: +```bash +cargo build +``` +after that the `jeprof` tool is produced. Now run `find_compiled_jeprof.sh` in current directory, it will copy the `jeprof` tool to the current directory. + +3. compile jemalloc from source +you can first clone this repo, and checkout to this commit: +```bash +git clone https://github.com/tikv/jemalloc.git +cd jemalloc +git checkout e13ca993e8ccb9ba9847cc330696e02839f328f7 +``` +then run: +```bash +./configure +make +``` +and `jeprof` is in `.bin/` directory. Copy it to the current directory. diff --git a/docs/how-to/memory-profile-scripts/scripts/dump.sh b/docs/how-to/memory-profile-scripts/scripts/dump.sh new file mode 100755 index 0000000000..d84bee75a5 --- /dev/null +++ b/docs/how-to/memory-profile-scripts/scripts/dump.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# Monitors greptime process memory usage every 10 minutes +# Triggers memory profile capture via `curl -X POST localhost:4000/debug/prof/mem > greptime-{timestamp}.gprof` +# when memory increases by more than 20MB since last check +# Generated profiles can be analyzed using flame graphs as described in `how-to-profile-memory.md` +# (jeprof is compiled with the database - see documentation) +# Alternative: Share binaries + profiles for analysis (Docker images preferred) + +# Threshold in Kilobytes (20 MB) +threshold_kb=$((20 * 1024)) +sleep_interval=$((10 * 60)) + +# Variable to store the last measured memory usage in KB +last_mem_kb=0 + +echo "Starting memory monitoring for 'greptime' process..." + +while true; do + + # Check if PID is provided as an argument + if [ -z "$1" ]; then + echo "$(date): PID must be provided as a command-line argument." + exit 1 + fi + + pid="$1" + + # Validate that the PID is a number + if ! [[ "$pid" =~ ^[0-9]+$ ]]; then + echo "$(date): Invalid PID: '$pid'. PID must be a number." + exit 1 + fi + + # Get the current Resident Set Size (RSS) in Kilobytes + current_mem_kb=$(ps -o rss= -p "$pid") + + # Check if ps command was successful and returned a number + if ! [[ "$current_mem_kb" =~ ^[0-9]+$ ]]; then + echo "$(date): Failed to get memory usage for PID $pid. Skipping check." + # Keep last_mem_kb to avoid false positives if the process briefly becomes unreadable. + continue + fi + + echo "$(date): Current memory usage for PID $pid: ${current_mem_kb} KB" + + # Compare with the last measurement + # if it's the first run, also do a baseline dump just to make sure we can dump + + diff_kb=$((current_mem_kb - last_mem_kb)) + echo "$(date): Memory usage change since last check: ${diff_kb} KB" + + if [ "$diff_kb" -gt "$threshold_kb" ]; then + echo "$(date): Memory increase (${diff_kb} KB) exceeded threshold (${threshold_kb} KB). Dumping profile..." + timestamp=$(date +%Y%m%d%H%M%S) + profile_file="greptime-${timestamp}.gprof" + # Execute curl and capture output to file + if curl -sf -X POST localhost:4000/debug/prof/mem > "$profile_file"; then + echo "$(date): Memory profile saved to $profile_file" + else + echo "$(date): Failed to dump memory profile (curl exit code: $?)." + # Remove the potentially empty/failed profile file + rm -f "$profile_file" + fi + else + echo "$(date): Memory increase (${diff_kb} KB) is within the threshold (${threshold_kb} KB)." + fi + + + # Update the last memory usage + last_mem_kb=$current_mem_kb + + # Wait for 5 minutes + echo "$(date): Sleeping for $sleep_interval seconds..." + sleep $sleep_interval +done + +echo "Memory monitoring script stopped." # This line might not be reached in normal operation diff --git a/docs/how-to/memory-profile-scripts/scripts/find_compiled_jeprof.sh b/docs/how-to/memory-profile-scripts/scripts/find_compiled_jeprof.sh new file mode 100755 index 0000000000..b59488d1b4 --- /dev/null +++ b/docs/how-to/memory-profile-scripts/scripts/find_compiled_jeprof.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Locates compiled jeprof binary (memory analysis tool) after cargo build +# Copies it to current directory from target/ build directories + +JPROF_PATH=$(find . -name 'jeprof' -print -quit) +if [ -n "$JPROF_PATH" ]; then + echo "Found jeprof at $JPROF_PATH" + cp "$JPROF_PATH" . + chmod +x jeprof + echo "Copied jeprof to current directory and made it executable." +else + echo "jeprof not found" + exit 1 +fi diff --git a/docs/how-to/memory-profile-scripts/scripts/gen_flamegraph.sh b/docs/how-to/memory-profile-scripts/scripts/gen_flamegraph.sh new file mode 100755 index 0000000000..454d3da8ae --- /dev/null +++ b/docs/how-to/memory-profile-scripts/scripts/gen_flamegraph.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +# Generate flame graphs from a series of `.gprof` files +# First argument: Path to the binary executable +# Second argument: Path to directory containing gprof files +# Requires `jeprof` and `flamegraph.pl` in current directory +# What this script essentially does is: +# ./jeprof --collapse | ./flamegraph.pl > +# For differential analysis between consecutive profiles: +# ./jeprof --base --collapse | ./flamegraph.pl > + +set -e # Exit immediately if a command exits with a non-zero status. + +# Check for required tools +if [ ! -f "./jeprof" ]; then + echo "Error: jeprof not found in the current directory." + exit 1 +fi + +if [ ! -f "./flamegraph.pl" ]; then + echo "Error: flamegraph.pl not found in the current directory." + exit 1 +fi + +# Check arguments +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +BINARY_PATH=$1 +GPROF_DIR=$2 +OUTPUT_DIR="${GPROF_DIR}/flamegraphs" # Store outputs in a subdirectory + +if [ ! -f "$BINARY_PATH" ]; then + echo "Error: Binary file not found at $BINARY_PATH" + exit 1 +fi + +if [ ! -d "$GPROF_DIR" ]; then + echo "Error: gprof directory not found at $GPROF_DIR" + exit 1 +fi + +mkdir -p "$OUTPUT_DIR" +echo "Generating flamegraphs in $OUTPUT_DIR" + +# Find and sort gprof files +# Use find + sort -V for natural sort of version numbers if present in filenames +# Use null-terminated strings for safety with find/xargs/sort +mapfile -d $'\0' gprof_files < <(find "$GPROF_DIR" -maxdepth 1 -name '*.gprof' -print0 | sort -zV) + +if [ ${#gprof_files[@]} -eq 0 ]; then + echo "No .gprof files found in $GPROF_DIR" + exit 0 +fi + +prev_gprof="" + +# Generate flamegraphs +for gprof_file in "${gprof_files[@]}"; do + # Skip empty entries if any + if [ -z "$gprof_file" ]; then + continue + fi + + filename=$(basename "$gprof_file" .gprof) + output_collapse="${OUTPUT_DIR}/${filename}.collapse" + output_svg="${OUTPUT_DIR}/${filename}.svg" + echo "Generating collapse file for $gprof_file -> $output_collapse" + ./jeprof "$BINARY_PATH" "$gprof_file" --collapse > "$output_collapse" + echo "Generating flamegraph for $gprof_file -> $output_svg" + ./flamegraph.pl "$output_collapse" > "$output_svg" || true + + # Generate diff flamegraph if not the first file + if [ -n "$prev_gprof" ]; then + prev_filename=$(basename "$prev_gprof" .gprof) + diff_output_collapse="${OUTPUT_DIR}/${prev_filename}_vs_${filename}_diff.collapse" + diff_output_svg="${OUTPUT_DIR}/${prev_filename}_vs_${filename}_diff.svg" + echo "Generating diff collapse file for $prev_gprof vs $gprof_file -> $diff_output_collapse" + ./jeprof "$BINARY_PATH" --base "$prev_gprof" "$gprof_file" --collapse > "$diff_output_collapse" + echo "Generating diff flamegraph for $prev_gprof vs $gprof_file -> $diff_output_svg" + ./flamegraph.pl "$diff_output_collapse" > "$diff_output_svg" || true + fi + + prev_gprof="$gprof_file" +done + +echo "Flamegraph generation complete." diff --git a/docs/how-to/memory-profile-scripts/scripts/gen_from_collapse.sh b/docs/how-to/memory-profile-scripts/scripts/gen_from_collapse.sh new file mode 100755 index 0000000000..0546ede38e --- /dev/null +++ b/docs/how-to/memory-profile-scripts/scripts/gen_from_collapse.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Generate flame graphs from .collapse files +# Argument: Path to directory containing collapse files +# Requires `flamegraph.pl` in current directory + +# Check if flamegraph.pl exists +if [ ! -f "./flamegraph.pl" ]; then + echo "Error: flamegraph.pl not found in the current directory." + exit 1 +fi + +# Check if directory argument is provided +if [ -z "$1" ]; then + echo "Usage: $0 " + exit 1 +fi + +COLLAPSE_DIR=$1 + +# Check if the provided argument is a directory +if [ ! -d "$COLLAPSE_DIR" ]; then + echo "Error: '$COLLAPSE_DIR' is not a valid directory." + exit 1 +fi + +echo "Generating flame graphs from collapse files in '$COLLAPSE_DIR'..." + +# Find and process each .collapse file +find "$COLLAPSE_DIR" -maxdepth 1 -name "*.collapse" -print0 | while IFS= read -r -d $'\0' collapse_file; do + if [ -f "$collapse_file" ]; then + # Construct the output SVG filename + svg_file="${collapse_file%.collapse}.svg" + echo "Generating $svg_file from $collapse_file..." + ./flamegraph.pl "$collapse_file" > "$svg_file" + if [ $? -ne 0 ]; then + echo "Error generating flame graph for $collapse_file" + else + echo "Successfully generated $svg_file" + fi + fi +done + +echo "Flame graph generation complete." diff --git a/docs/how-to/memory-profile-scripts/scripts/get_flamegraph_tool.sh b/docs/how-to/memory-profile-scripts/scripts/get_flamegraph_tool.sh new file mode 100755 index 0000000000..d299d71698 --- /dev/null +++ b/docs/how-to/memory-profile-scripts/scripts/get_flamegraph_tool.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# Download flamegraph.pl to current directory - this is the flame graph generation tool script + +curl https://raw.githubusercontent.com/brendangregg/FlameGraph/master/flamegraph.pl > ./flamegraph.pl +chmod +x ./flamegraph.pl From a5c443f734af33c9466de73810db02748c4f68d4 Mon Sep 17 00:00:00 2001 From: Yingwen Date: Fri, 18 Apr 2025 17:36:28 +0800 Subject: [PATCH 37/82] perf: keep compiled regex in SimpleFilterEvaluator to avoid re-compiling (#5919) * feat: cache regex in evaluator * chore: fix warnings * chore: add reference * refactor: address CR comments * Add negative to state * Don't create the evaluator if the regex is invalid * test: add test for maybe_build_regex --- Cargo.lock | 1 + src/common/recordbatch/Cargo.toml | 1 + src/common/recordbatch/src/error.rs | 2 +- src/common/recordbatch/src/filter.rs | 202 ++++++++++++++++++++++++--- 4 files changed, 187 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cb9ad31c15..539216920d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2315,6 +2315,7 @@ dependencies = [ "datatypes", "futures", "pin-project", + "regex", "serde", "serde_json", "snafu 0.8.5", diff --git a/src/common/recordbatch/Cargo.toml b/src/common/recordbatch/Cargo.toml index 5c3d9fa550..bb8ba13907 100644 --- a/src/common/recordbatch/Cargo.toml +++ b/src/common/recordbatch/Cargo.toml @@ -17,6 +17,7 @@ datafusion-common.workspace = true datatypes.workspace = true futures.workspace = true pin-project.workspace = true +regex.workspace = true serde.workspace = true serde_json.workspace = true snafu.workspace = true diff --git a/src/common/recordbatch/src/error.rs b/src/common/recordbatch/src/error.rs index dfd85e4aa1..e0b66eb903 100644 --- a/src/common/recordbatch/src/error.rs +++ b/src/common/recordbatch/src/error.rs @@ -23,7 +23,7 @@ use datatypes::prelude::ConcreteDataType; use datatypes::schema::SchemaRef; use snafu::{Location, Snafu}; -pub type Result = std::result::Result; +pub type Result = std::result::Result; #[derive(Snafu)] #[snafu(visibility(pub))] diff --git a/src/common/recordbatch/src/filter.rs b/src/common/recordbatch/src/filter.rs index b16ee401c0..2c509396ff 100644 --- a/src/common/recordbatch/src/filter.rs +++ b/src/common/recordbatch/src/filter.rs @@ -24,12 +24,16 @@ use datafusion_common::arrow::buffer::BooleanBuffer; use datafusion_common::arrow::compute::kernels::cmp; use datafusion_common::cast::{as_boolean_array, as_null_array, as_string_array}; use datafusion_common::{internal_err, DataFusionError, ScalarValue}; -use datatypes::arrow::array::{Array, BooleanArray, RecordBatch}; +use datatypes::arrow::array::{ + Array, ArrayAccessor, ArrayData, BooleanArray, BooleanBufferBuilder, RecordBatch, + StringArrayType, +}; use datatypes::arrow::compute::filter_record_batch; +use datatypes::arrow::datatypes::DataType; use datatypes::arrow::error::ArrowError; -use datatypes::compute::kernels::regexp; use datatypes::compute::or_kleene; use datatypes::vectors::VectorRef; +use regex::Regex; use snafu::ResultExt; use crate::error::{ArrowComputeSnafu, Result, ToArrowScalarSnafu, UnsupportedOperationSnafu}; @@ -53,6 +57,12 @@ pub struct SimpleFilterEvaluator { op: Operator, /// Only used when the operator is `Or`-chain. literal_list: Vec>, + /// Pre-compiled regex. + /// Only used when the operator is regex operators. + /// If the regex is empty, it is also `None`. + regex: Option, + /// Whether the regex is negative. + regex_negative: bool, } impl SimpleFilterEvaluator { @@ -76,6 +86,8 @@ impl SimpleFilterEvaluator { literal: val.to_scalar().ok()?, op, literal_list: vec![], + regex: None, + regex_negative: false, }) } @@ -121,6 +133,8 @@ impl SimpleFilterEvaluator { literal: placeholder_literal, op: Operator::Or, literal_list: list, + regex: None, + regex_negative: false, }); } _ => return None, @@ -138,12 +152,15 @@ impl SimpleFilterEvaluator { _ => return None, }; + let (regex, regex_negative) = Self::maybe_build_regex(op, rhs).ok()?; let literal = rhs.to_scalar().ok()?; Some(Self { column_name: lhs.name.clone(), literal, op, literal_list: vec![], + regex, + regex_negative, }) } _ => None, @@ -179,10 +196,10 @@ impl SimpleFilterEvaluator { Operator::LtEq => cmp::lt_eq(input, &self.literal), Operator::Gt => cmp::gt(input, &self.literal), Operator::GtEq => cmp::gt_eq(input, &self.literal), - Operator::RegexMatch => self.regex_match(input, false, false), - Operator::RegexIMatch => self.regex_match(input, true, false), - Operator::RegexNotMatch => self.regex_match(input, false, true), - Operator::RegexNotIMatch => self.regex_match(input, true, true), + Operator::RegexMatch => self.regex_match(input), + Operator::RegexIMatch => self.regex_match(input), + Operator::RegexNotMatch => self.regex_match(input), + Operator::RegexNotIMatch => self.regex_match(input), Operator::Or => { // OR operator stands for OR-chained EQs (or INLIST in other words) let mut result: BooleanArray = vec![false; input_len].into(); @@ -204,23 +221,54 @@ impl SimpleFilterEvaluator { .map(|array| array.values().clone()) } - fn regex_match( - &self, - input: &impl Datum, - ignore_case: bool, - negative: bool, - ) -> std::result::Result { + /// Builds a regex pattern from a scalar value and operator. + /// Returns the `(regex, negative)` and if successful. + /// + /// Returns `Err` if + /// - the value is not a string + /// - the regex pattern is invalid + /// + /// The regex is `None` if + /// - the operator is not a regex operator + /// - the pattern is empty + fn maybe_build_regex( + operator: Operator, + value: &ScalarValue, + ) -> Result<(Option, bool), ArrowError> { + let (ignore_case, negative) = match operator { + Operator::RegexMatch => (false, false), + Operator::RegexIMatch => (true, false), + Operator::RegexNotMatch => (false, true), + Operator::RegexNotIMatch => (true, true), + _ => return Ok((None, false)), + }; let flag = if ignore_case { Some("i") } else { None }; + let regex = value + .try_as_str() + .ok_or_else(|| ArrowError::CastError(format!("Cannot cast {:?} to str", value)))? + .ok_or_else(|| ArrowError::CastError("Regex should not be null".to_string()))?; + let pattern = match flag { + Some(flag) => format!("(?{flag}){regex}"), + None => regex.to_string(), + }; + if pattern.is_empty() { + Ok((None, negative)) + } else { + Regex::new(pattern.as_str()) + .map_err(|e| { + ArrowError::ComputeError(format!("Regular expression did not compile: {e:?}")) + }) + .map(|regex| (Some(regex), negative)) + } + } + + fn regex_match(&self, input: &impl Datum) -> std::result::Result { let array = input.get().0; let string_array = as_string_array(array).map_err(|_| { ArrowError::CastError(format!("Cannot cast {:?} to StringArray", array)) })?; - let literal_array = self.literal.clone().into_inner(); - let regex_array = as_string_array(&literal_array).map_err(|_| { - ArrowError::CastError(format!("Cannot cast {:?} to StringArray", literal_array)) - })?; - let mut result = regexp::regexp_is_match_scalar(string_array, regex_array.value(0), flag)?; - if negative { + let mut result = regexp_is_match_scalar(string_array, self.regex.as_ref())?; + if self.regex_negative { result = datatypes::compute::not(&result)?; } Ok(result) @@ -254,6 +302,44 @@ pub fn batch_filter( }) } +/// The same as arrow [regexp_is_match_scalar()](datatypes::compute::kernels::regexp::regexp_is_match_scalar()) +/// with pre-compiled regex. +/// See for the implementation details. +pub fn regexp_is_match_scalar<'a, S>( + array: &'a S, + regex: Option<&Regex>, +) -> Result +where + &'a S: StringArrayType<'a>, +{ + let null_bit_buffer = array.nulls().map(|x| x.inner().sliced()); + let mut result = BooleanBufferBuilder::new(array.len()); + + if let Some(re) = regex { + for i in 0..array.len() { + let value = array.value(i); + result.append(re.is_match(value)); + } + } else { + result.append_n(array.len(), true); + } + + let buffer = result.into(); + let data = unsafe { + ArrayData::new_unchecked( + DataType::Boolean, + array.len(), + None, + null_bit_buffer, + 0, + vec![buffer], + vec![], + ) + }; + + Ok(BooleanArray::from(data)) +} + #[cfg(test)] mod test { @@ -446,4 +532,84 @@ mod test { .unwrap(); assert_eq!(col_filtered, &expected); } + + #[test] + fn test_maybe_build_regex() { + // Test case for RegexMatch (case sensitive, non-negative) + let (regex, negative) = SimpleFilterEvaluator::maybe_build_regex( + Operator::RegexMatch, + &ScalarValue::Utf8(Some("a.*b".to_string())), + ) + .unwrap(); + assert!(regex.is_some()); + assert!(!negative); + assert!(regex.unwrap().is_match("axxb")); + + // Test case for RegexIMatch (case insensitive, non-negative) + let (regex, negative) = SimpleFilterEvaluator::maybe_build_regex( + Operator::RegexIMatch, + &ScalarValue::Utf8(Some("a.*b".to_string())), + ) + .unwrap(); + assert!(regex.is_some()); + assert!(!negative); + assert!(regex.unwrap().is_match("AxxB")); + + // Test case for RegexNotMatch (case sensitive, negative) + let (regex, negative) = SimpleFilterEvaluator::maybe_build_regex( + Operator::RegexNotMatch, + &ScalarValue::Utf8(Some("a.*b".to_string())), + ) + .unwrap(); + assert!(regex.is_some()); + assert!(negative); + + // Test case for RegexNotIMatch (case insensitive, negative) + let (regex, negative) = SimpleFilterEvaluator::maybe_build_regex( + Operator::RegexNotIMatch, + &ScalarValue::Utf8(Some("a.*b".to_string())), + ) + .unwrap(); + assert!(regex.is_some()); + assert!(negative); + + // Test with empty regex pattern + let (regex, negative) = SimpleFilterEvaluator::maybe_build_regex( + Operator::RegexMatch, + &ScalarValue::Utf8(Some("".to_string())), + ) + .unwrap(); + assert!(regex.is_none()); + assert!(!negative); + + // Test with non-regex operator + let (regex, negative) = SimpleFilterEvaluator::maybe_build_regex( + Operator::Eq, + &ScalarValue::Utf8(Some("a.*b".to_string())), + ) + .unwrap(); + assert!(regex.is_none()); + assert!(!negative); + + // Test with invalid regex pattern + let result = SimpleFilterEvaluator::maybe_build_regex( + Operator::RegexMatch, + &ScalarValue::Utf8(Some("a(b".to_string())), + ); + assert!(result.is_err()); + + // Test with non-string value + let result = SimpleFilterEvaluator::maybe_build_regex( + Operator::RegexMatch, + &ScalarValue::Int64(Some(123)), + ); + assert!(result.is_err()); + + // Test with null value + let result = SimpleFilterEvaluator::maybe_build_regex( + Operator::RegexMatch, + &ScalarValue::Utf8(None), + ); + assert!(result.is_err()); + } } From 115e5a03a8e8438b4079e51712244293cf886649 Mon Sep 17 00:00:00 2001 From: Ruihang Xia Date: Fri, 18 Apr 2025 18:13:45 +0800 Subject: [PATCH 38/82] fix: anchor regex string to fully match in promql (#5920) * fix: anchor regex string to fully match in promql Signed-off-by: Ruihang Xia * fix format Signed-off-by: Ruihang Xia * update sqlness result Signed-off-by: Ruihang Xia * update test result Signed-off-by: Ruihang Xia * update test result again Signed-off-by: Ruihang Xia --------- Signed-off-by: Ruihang Xia --- Cargo.lock | 4 +- Cargo.toml | 2 +- src/query/src/promql/planner.rs | 27 +++++++----- .../standalone/common/promql/regex.result | 2 +- .../cases/standalone/common/promql/regex.sql | 2 +- .../common/select/tql_filter.result | 44 +++++++++++++++++++ .../standalone/common/select/tql_filter.sql | 20 +++++++++ 7 files changed, 85 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 539216920d..618cc09880 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8892,9 +8892,9 @@ dependencies = [ [[package]] name = "promql-parser" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c6b1429bdd199d53bd58b745075c1652efedbe2746e5d4f0d56d3184dda48ec" +checksum = "60d851f6523a8215e2fbf86b6cef4548433f8b76092e9ffb607105de52ae63fd" dependencies = [ "cfgrammar", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 919b4f8d44..a6cd8cdb81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -161,7 +161,7 @@ parquet = { version = "54.2", default-features = false, features = ["arrow", "as paste = "1.0" pin-project = "1.0" prometheus = { version = "0.13.3", features = ["process"] } -promql-parser = { version = "0.5", features = ["ser"] } +promql-parser = { version = "0.5.1", features = ["ser"] } prost = "0.13" raft-engine = { version = "0.4.1", default-features = false } rand = "0.9" diff --git a/src/query/src/promql/planner.rs b/src/query/src/promql/planner.rs index c0b7bfb098..f81d2052dc 100644 --- a/src/query/src/promql/planner.rs +++ b/src/query/src/promql/planner.rs @@ -1165,13 +1165,17 @@ impl PromPlanner { DfExpr::BinaryExpr(BinaryExpr { left: Box::new(col), op: Operator::RegexMatch, - right: Box::new(lit), + right: Box::new(DfExpr::Literal(ScalarValue::Utf8(Some( + re.as_str().to_string(), + )))), }) } - MatchOp::NotRe(_) => DfExpr::BinaryExpr(BinaryExpr { + MatchOp::NotRe(re) => DfExpr::BinaryExpr(BinaryExpr { left: Box::new(col), op: Operator::RegexNotMatch, - right: Box::new(lit), + right: Box::new(DfExpr::Literal(ScalarValue::Utf8(Some( + re.as_str().to_string(), + )))), }), }; exprs.push(expr); @@ -4237,7 +4241,8 @@ mod test { interval: Duration::from_secs(5), lookback_delta: Duration::from_secs(1), }; - let case = r#"sum(prometheus_tsdb_head_series{tag_1=~"(10\\.0\\.160\\.237:8080|10\\.0\\.160\\.237:9090)"})"#; + let case = + r#"sum(prometheus_tsdb_head_series{tag_1=~"(10.0.160.237:8080|10.0.160.237:9090)"})"#; let prom_expr = parser::parse(case).unwrap(); eval_stmt.expr = prom_expr; @@ -4258,7 +4263,7 @@ mod test { \n PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[timestamp] [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N]\ \n PromSeriesDivide: tags=[\"tag_0\", \"tag_1\", \"tag_2\"] [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N]\ \n Sort: prometheus_tsdb_head_series.tag_0 ASC NULLS FIRST, prometheus_tsdb_head_series.tag_1 ASC NULLS FIRST, prometheus_tsdb_head_series.tag_2 ASC NULLS FIRST, prometheus_tsdb_head_series.timestamp ASC NULLS FIRST [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N]\ - \n Filter: prometheus_tsdb_head_series.tag_1 ~ Utf8(\"(10\\.0\\.160\\.237:8080|10\\.0\\.160\\.237:9090)\") AND prometheus_tsdb_head_series.timestamp >= TimestampMillisecond(-1000, None) AND prometheus_tsdb_head_series.timestamp <= TimestampMillisecond(100001000, None) [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N]\ + \n Filter: prometheus_tsdb_head_series.tag_1 ~ Utf8(\"^(10.0.160.237:8080|10.0.160.237:9090)$\") AND prometheus_tsdb_head_series.timestamp >= TimestampMillisecond(-1000, None) AND prometheus_tsdb_head_series.timestamp <= TimestampMillisecond(100001000, None) [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N]\ \n TableScan: prometheus_tsdb_head_series [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N]"; assert_eq!(plan.display_indent_schema().to_string(), expected); } @@ -4274,7 +4279,7 @@ mod test { interval: Duration::from_secs(5), lookback_delta: Duration::from_secs(1), }; - let case = r#"topk(10, sum(prometheus_tsdb_head_series{ip=~"(10\\.0\\.160\\.237:8080|10\\.0\\.160\\.237:9090)"}) by (ip))"#; + let case = r#"topk(10, sum(prometheus_tsdb_head_series{ip=~"(10.0.160.237:8080|10.0.160.237:9090)"}) by (ip))"#; let prom_expr = parser::parse(case).unwrap(); eval_stmt.expr = prom_expr; @@ -4305,7 +4310,7 @@ mod test { \n PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[greptime_timestamp] [ip:Utf8, greptime_timestamp:Timestamp(Millisecond, None), greptime_value:Float64;N]\ \n PromSeriesDivide: tags=[\"ip\"] [ip:Utf8, greptime_timestamp:Timestamp(Millisecond, None), greptime_value:Float64;N]\ \n Sort: prometheus_tsdb_head_series.ip ASC NULLS FIRST, prometheus_tsdb_head_series.greptime_timestamp ASC NULLS FIRST [ip:Utf8, greptime_timestamp:Timestamp(Millisecond, None), greptime_value:Float64;N]\ - \n Filter: prometheus_tsdb_head_series.ip ~ Utf8(\"(10\\.0\\.160\\.237:8080|10\\.0\\.160\\.237:9090)\") AND prometheus_tsdb_head_series.greptime_timestamp >= TimestampMillisecond(-1000, None) AND prometheus_tsdb_head_series.greptime_timestamp <= TimestampMillisecond(100001000, None) [ip:Utf8, greptime_timestamp:Timestamp(Millisecond, None), greptime_value:Float64;N]\ + \n Filter: prometheus_tsdb_head_series.ip ~ Utf8(\"^(10.0.160.237:8080|10.0.160.237:9090)$\") AND prometheus_tsdb_head_series.greptime_timestamp >= TimestampMillisecond(-1000, None) AND prometheus_tsdb_head_series.greptime_timestamp <= TimestampMillisecond(100001000, None) [ip:Utf8, greptime_timestamp:Timestamp(Millisecond, None), greptime_value:Float64;N]\ \n TableScan: prometheus_tsdb_head_series [ip:Utf8, greptime_timestamp:Timestamp(Millisecond, None), greptime_value:Float64;N]"; assert_eq!(plan.display_indent_schema().to_string(), expected); @@ -4322,7 +4327,7 @@ mod test { interval: Duration::from_secs(5), lookback_delta: Duration::from_secs(1), }; - let case = r#"count_values('series', prometheus_tsdb_head_series{ip=~"(10\\.0\\.160\\.237:8080|10\\.0\\.160\\.237:9090)"}) by (ip)"#; + let case = r#"count_values('series', prometheus_tsdb_head_series{ip=~"(10.0.160.237:8080|10.0.160.237:9090)"}) by (ip)"#; let prom_expr = parser::parse(case).unwrap(); eval_stmt.expr = prom_expr; @@ -4351,7 +4356,7 @@ mod test { \n PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[greptime_timestamp] [ip:Utf8, greptime_timestamp:Timestamp(Millisecond, None), greptime_value:Float64;N]\ \n PromSeriesDivide: tags=[\"ip\"] [ip:Utf8, greptime_timestamp:Timestamp(Millisecond, None), greptime_value:Float64;N]\ \n Sort: prometheus_tsdb_head_series.ip ASC NULLS FIRST, prometheus_tsdb_head_series.greptime_timestamp ASC NULLS FIRST [ip:Utf8, greptime_timestamp:Timestamp(Millisecond, None), greptime_value:Float64;N]\ - \n Filter: prometheus_tsdb_head_series.ip ~ Utf8(\"(10\\.0\\.160\\.237:8080|10\\.0\\.160\\.237:9090)\") AND prometheus_tsdb_head_series.greptime_timestamp >= TimestampMillisecond(-1000, None) AND prometheus_tsdb_head_series.greptime_timestamp <= TimestampMillisecond(100001000, None) [ip:Utf8, greptime_timestamp:Timestamp(Millisecond, None), greptime_value:Float64;N]\ + \n Filter: prometheus_tsdb_head_series.ip ~ Utf8(\"^(10.0.160.237:8080|10.0.160.237:9090)$\") AND prometheus_tsdb_head_series.greptime_timestamp >= TimestampMillisecond(-1000, None) AND prometheus_tsdb_head_series.greptime_timestamp <= TimestampMillisecond(100001000, None) [ip:Utf8, greptime_timestamp:Timestamp(Millisecond, None), greptime_value:Float64;N]\ \n TableScan: prometheus_tsdb_head_series [ip:Utf8, greptime_timestamp:Timestamp(Millisecond, None), greptime_value:Float64;N]"; assert_eq!(plan.display_indent_schema().to_string(), expected); @@ -4368,7 +4373,7 @@ mod test { interval: Duration::from_secs(5), lookback_delta: Duration::from_secs(1), }; - let case = r#"quantile(0.3, sum(prometheus_tsdb_head_series{ip=~"(10\\.0\\.160\\.237:8080|10\\.0\\.160\\.237:9090)"}) by (ip))"#; + let case = r#"quantile(0.3, sum(prometheus_tsdb_head_series{ip=~"(10.0.160.237:8080|10.0.160.237:9090)"}) by (ip))"#; let prom_expr = parser::parse(case).unwrap(); eval_stmt.expr = prom_expr; @@ -4397,7 +4402,7 @@ mod test { \n PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[greptime_timestamp] [ip:Utf8, greptime_timestamp:Timestamp(Millisecond, None), greptime_value:Float64;N]\ \n PromSeriesDivide: tags=[\"ip\"] [ip:Utf8, greptime_timestamp:Timestamp(Millisecond, None), greptime_value:Float64;N]\ \n Sort: prometheus_tsdb_head_series.ip ASC NULLS FIRST, prometheus_tsdb_head_series.greptime_timestamp ASC NULLS FIRST [ip:Utf8, greptime_timestamp:Timestamp(Millisecond, None), greptime_value:Float64;N]\ - \n Filter: prometheus_tsdb_head_series.ip ~ Utf8(\"(10\\.0\\.160\\.237:8080|10\\.0\\.160\\.237:9090)\") AND prometheus_tsdb_head_series.greptime_timestamp >= TimestampMillisecond(-1000, None) AND prometheus_tsdb_head_series.greptime_timestamp <= TimestampMillisecond(100001000, None) [ip:Utf8, greptime_timestamp:Timestamp(Millisecond, None), greptime_value:Float64;N]\ + \n Filter: prometheus_tsdb_head_series.ip ~ Utf8(\"^(10.0.160.237:8080|10.0.160.237:9090)$\") AND prometheus_tsdb_head_series.greptime_timestamp >= TimestampMillisecond(-1000, None) AND prometheus_tsdb_head_series.greptime_timestamp <= TimestampMillisecond(100001000, None) [ip:Utf8, greptime_timestamp:Timestamp(Millisecond, None), greptime_value:Float64;N]\ \n TableScan: prometheus_tsdb_head_series [ip:Utf8, greptime_timestamp:Timestamp(Millisecond, None), greptime_value:Float64;N]"; assert_eq!(plan.display_indent_schema().to_string(), expected); diff --git a/tests/cases/standalone/common/promql/regex.result b/tests/cases/standalone/common/promql/regex.result index ec4c74d204..71e9abed34 100644 --- a/tests/cases/standalone/common/promql/regex.result +++ b/tests/cases/standalone/common/promql/regex.result @@ -22,7 +22,7 @@ SELECT * FROM test; | 1970-01-01T00:00:00 | 10.0.160.237:8081 | 1 | +---------------------+-------------------+-----+ -TQL EVAL (0, 100, '15s') test{host=~"(10\\.0\\.160\\.237:8080|10\\.0\\.160\\.237:9090)"}; +TQL EVAL (0, 100, '15s') test{host=~"(10.0.160.237:8080|10.0.160.237:9090)"}; +---------------------+-------------------+-----+ | ts | host | val | diff --git a/tests/cases/standalone/common/promql/regex.sql b/tests/cases/standalone/common/promql/regex.sql index 71b13a6eb6..1324730177 100644 --- a/tests/cases/standalone/common/promql/regex.sql +++ b/tests/cases/standalone/common/promql/regex.sql @@ -11,6 +11,6 @@ INSERT INTO TABLE test VALUES SELECT * FROM test; -TQL EVAL (0, 100, '15s') test{host=~"(10\\.0\\.160\\.237:8080|10\\.0\\.160\\.237:9090)"}; +TQL EVAL (0, 100, '15s') test{host=~"(10.0.160.237:8080|10.0.160.237:9090)"}; DROP TABLE test; diff --git a/tests/cases/standalone/common/select/tql_filter.result b/tests/cases/standalone/common/select/tql_filter.result index 9e41d5ed9b..5c10d77a52 100644 --- a/tests/cases/standalone/common/select/tql_filter.result +++ b/tests/cases/standalone/common/select/tql_filter.result @@ -70,3 +70,47 @@ drop table t1; Affected Rows: 0 +create table t2 (a string primary key, b timestamp time index, c double); + +Affected Rows: 0 + +INSERT INTO TABLE t2 VALUES + ('10.0.160.237:8080', 0, 1), + ('10.0.160.237:8081', 0, 1), + ('20.0.10.237:8081', 0, 1), + ('abcx', 0, 1), + ('xabc', 0, 1); + +Affected Rows: 5 + +-- SQLNESS SORT_RESULT 3 1 +tql eval (0, 0, '1s') t2{a=~"10"}; + +++ +++ + +-- SQLNESS SORT_RESULT 3 1 +tql eval (0, 0, '1s') t2{a=~"10.*"}; + ++-------------------+---------------------+-----+ +| a | b | c | ++-------------------+---------------------+-----+ +| 10.0.160.237:8080 | 1970-01-01T00:00:00 | 1.0 | +| 10.0.160.237:8081 | 1970-01-01T00:00:00 | 1.0 | ++-------------------+---------------------+-----+ + +-- SQLNESS SORT_RESULT 3 1 +tql eval (0, 0, '1s') t2{a=~".*10.*"}; + ++-------------------+---------------------+-----+ +| a | b | c | ++-------------------+---------------------+-----+ +| 10.0.160.237:8080 | 1970-01-01T00:00:00 | 1.0 | +| 10.0.160.237:8081 | 1970-01-01T00:00:00 | 1.0 | +| 20.0.10.237:8081 | 1970-01-01T00:00:00 | 1.0 | ++-------------------+---------------------+-----+ + +drop table t2; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/select/tql_filter.sql b/tests/cases/standalone/common/select/tql_filter.sql index c37512f362..de0179443e 100644 --- a/tests/cases/standalone/common/select/tql_filter.sql +++ b/tests/cases/standalone/common/select/tql_filter.sql @@ -27,3 +27,23 @@ tql analyze (1, 3, '1s') t1{ a =~ ".*" }; tql analyze (1, 3, '1s') t1{ a =~ "a.*" }; drop table t1; + +create table t2 (a string primary key, b timestamp time index, c double); + +INSERT INTO TABLE t2 VALUES + ('10.0.160.237:8080', 0, 1), + ('10.0.160.237:8081', 0, 1), + ('20.0.10.237:8081', 0, 1), + ('abcx', 0, 1), + ('xabc', 0, 1); + +-- SQLNESS SORT_RESULT 3 1 +tql eval (0, 0, '1s') t2{a=~"10"}; + +-- SQLNESS SORT_RESULT 3 1 +tql eval (0, 0, '1s') t2{a=~"10.*"}; + +-- SQLNESS SORT_RESULT 3 1 +tql eval (0, 0, '1s') t2{a=~".*10.*"}; + +drop table t2; From b8c6f1c8ed9daed6f3a552ea1a87b7b38c23b22b Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Fri, 18 Apr 2025 18:21:35 +0800 Subject: [PATCH 39/82] feat: sync region followers after altering regions (#5901) * feat: close follower regions after dropping leader regions * chore: upgrade greptime-proto * feat: sync region followers after alter region operations * test: add tests * chore: apply suggestions from CR * chore: apply suggestions from CR --- Cargo.lock | 2 +- Cargo.toml | 2 +- .../meta/src/ddl/alter_logical_tables.rs | 44 ++++- src/common/meta/src/ddl/alter_table.rs | 33 +++- .../meta/src/ddl/create_logical_tables.rs | 40 ++++- .../meta/src/ddl/drop_table/executor.rs | 57 ++++++- .../meta/src/ddl/test_util/create_table.rs | 5 +- .../src/ddl/test_util/datanode_handler.rs | 33 +++- .../src/ddl/tests/alter_logical_tables.rs | 93 ++++++++++- src/common/meta/src/ddl/tests/alter_table.rs | 150 ++++++++++++++--- .../src/ddl/tests/create_logical_tables.rs | 87 +++++++++- src/common/meta/src/ddl/tests/drop_table.rs | 30 +++- src/common/meta/src/ddl/utils.rs | 155 ++++++++++++++++-- src/common/meta/src/rpc/router.rs | 24 +++ src/datanode/src/region_server.rs | 117 +++++++++---- src/metric-engine/src/engine.rs | 26 +-- src/metric-engine/src/engine/alter.rs | 8 +- src/metric-engine/src/engine/create.rs | 11 +- src/metric-engine/src/error.rs | 11 +- src/metric-engine/src/utils.rs | 75 ++++++++- src/mito2/src/engine.rs | 40 ++++- src/store-api/src/metadata.rs | 7 + src/store-api/src/metric_engine_consts.rs | 4 + src/store-api/src/region_engine.rs | 10 ++ src/store-api/src/region_request.rs | 6 +- 25 files changed, 922 insertions(+), 148 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 618cc09880..984978a1c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4727,7 +4727,7 @@ dependencies = [ [[package]] name = "greptime-proto" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=583daa3fbbbe39c90b7b92d13646bc3291d9c941#583daa3fbbbe39c90b7b92d13646bc3291d9c941" +source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=b6d9cffd43c4e6358805a798f17e03e232994b82#b6d9cffd43c4e6358805a798f17e03e232994b82" dependencies = [ "prost 0.13.5", "serde", diff --git a/Cargo.toml b/Cargo.toml index a6cd8cdb81..5e95df1487 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -129,7 +129,7 @@ etcd-client = "0.14" fst = "0.4.7" futures = "0.3" futures-util = "0.3" -greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "583daa3fbbbe39c90b7b92d13646bc3291d9c941" } +greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "b6d9cffd43c4e6358805a798f17e03e232994b82" } hex = "0.4" http = "1" humantime = "2.1" diff --git a/src/common/meta/src/ddl/alter_logical_tables.rs b/src/common/meta/src/ddl/alter_logical_tables.rs index ea741accf3..74706a8ddb 100644 --- a/src/common/meta/src/ddl/alter_logical_tables.rs +++ b/src/common/meta/src/ddl/alter_logical_tables.rs @@ -18,10 +18,12 @@ mod region_request; mod table_cache_keys; mod update_metadata; +use api::region::RegionResponse; use async_trait::async_trait; +use common_catalog::format_full_table_name; use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu}; use common_procedure::{Context, LockKey, Procedure, Status}; -use common_telemetry::{info, warn}; +use common_telemetry::{error, info, warn}; use futures_util::future; use serde::{Deserialize, Serialize}; use snafu::{ensure, ResultExt}; @@ -30,7 +32,7 @@ use store_api::metric_engine_consts::ALTER_PHYSICAL_EXTENSION_KEY; use strum::AsRefStr; use table::metadata::TableId; -use crate::ddl::utils::add_peer_context_if_needed; +use crate::ddl::utils::{add_peer_context_if_needed, sync_follower_regions}; use crate::ddl::DdlContext; use crate::error::{DecodeJsonSnafu, Error, MetadataCorruptionSnafu, Result}; use crate::key::table_info::TableInfoValue; @@ -39,7 +41,7 @@ use crate::key::DeserializedValueWithBytes; use crate::lock_key::{CatalogLock, SchemaLock, TableLock}; use crate::metrics; use crate::rpc::ddl::AlterTableTask; -use crate::rpc::router::find_leaders; +use crate::rpc::router::{find_leaders, RegionRoute}; pub struct AlterLogicalTablesProcedure { pub context: DdlContext, @@ -125,14 +127,20 @@ impl AlterLogicalTablesProcedure { }); } - // Collects responses from datanodes. - let phy_raw_schemas = future::join_all(alter_region_tasks) + let mut results = future::join_all(alter_region_tasks) .await .into_iter() - .map(|res| res.map(|mut res| res.extensions.remove(ALTER_PHYSICAL_EXTENSION_KEY))) .collect::>>()?; + // Collects responses from datanodes. + let phy_raw_schemas = results + .iter_mut() + .map(|res| res.extensions.remove(ALTER_PHYSICAL_EXTENSION_KEY)) + .collect::>(); + if phy_raw_schemas.is_empty() { + self.submit_sync_region_requests(results, &physical_table_route.region_routes) + .await; self.data.state = AlterTablesState::UpdateMetadata; return Ok(Status::executing(true)); } @@ -155,10 +163,34 @@ impl AlterLogicalTablesProcedure { warn!("altering logical table result doesn't contains extension key `{ALTER_PHYSICAL_EXTENSION_KEY}`,leaving the physical table's schema unchanged"); } + self.submit_sync_region_requests(results, &physical_table_route.region_routes) + .await; self.data.state = AlterTablesState::UpdateMetadata; Ok(Status::executing(true)) } + async fn submit_sync_region_requests( + &self, + results: Vec, + region_routes: &[RegionRoute], + ) { + let table_info = &self.data.physical_table_info.as_ref().unwrap().table_info; + if let Err(err) = sync_follower_regions( + &self.context, + self.data.physical_table_id, + results, + region_routes, + table_info.meta.engine.as_str(), + ) + .await + { + error!(err; "Failed to sync regions for table {}, table_id: {}", + format_full_table_name(&table_info.catalog_name, &table_info.schema_name, &table_info.name), + self.data.physical_table_id + ); + } + } + pub(crate) async fn on_update_metadata(&mut self) -> Result { self.update_physical_table_metadata().await?; self.update_logical_tables_metadata().await?; diff --git a/src/common/meta/src/ddl/alter_table.rs b/src/common/meta/src/ddl/alter_table.rs index 89406e6a96..ee7b15509c 100644 --- a/src/common/meta/src/ddl/alter_table.rs +++ b/src/common/meta/src/ddl/alter_table.rs @@ -19,6 +19,7 @@ mod update_metadata; use std::vec; +use api::region::RegionResponse; use api::v1::alter_table_expr::Kind; use api::v1::RenameTable; use async_trait::async_trait; @@ -29,7 +30,7 @@ use common_procedure::{ PoisonKeys, Procedure, ProcedureId, Status, StringKey, }; use common_telemetry::{debug, error, info}; -use futures::future; +use futures::future::{self}; use serde::{Deserialize, Serialize}; use snafu::{ensure, ResultExt}; use store_api::storage::RegionId; @@ -38,7 +39,9 @@ use table::metadata::{RawTableInfo, TableId, TableInfo}; use table::table_reference::TableReference; use crate::cache_invalidator::Context; -use crate::ddl::utils::{add_peer_context_if_needed, handle_multiple_results, MultipleResults}; +use crate::ddl::utils::{ + add_peer_context_if_needed, handle_multiple_results, sync_follower_regions, MultipleResults, +}; use crate::ddl::DdlContext; use crate::error::{AbortProcedureSnafu, Error, NoLeaderSnafu, PutPoisonSnafu, Result}; use crate::instruction::CacheIdent; @@ -48,7 +51,7 @@ use crate::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock}; use crate::metrics; use crate::poison_key::table_poison_key; use crate::rpc::ddl::AlterTableTask; -use crate::rpc::router::{find_leader_regions, find_leaders, region_distribution}; +use crate::rpc::router::{find_leader_regions, find_leaders, region_distribution, RegionRoute}; /// The alter table procedure pub struct AlterTableProcedure { @@ -194,7 +197,9 @@ impl AlterTableProcedure { // Just returns the error, and wait for the next try. Err(error) } - MultipleResults::Ok => { + MultipleResults::Ok(results) => { + self.submit_sync_region_requests(results, &physical_table_route.region_routes) + .await; self.data.state = AlterTableState::UpdateMetadata; Ok(Status::executing_with_clean_poisons(true)) } @@ -211,6 +216,26 @@ impl AlterTableProcedure { } } + async fn submit_sync_region_requests( + &mut self, + results: Vec, + region_routes: &[RegionRoute], + ) { + // Safety: filled in `prepare` step. + let table_info = self.data.table_info().unwrap(); + if let Err(err) = sync_follower_regions( + &self.context, + self.data.table_id(), + results, + region_routes, + table_info.meta.engine.as_str(), + ) + .await + { + error!(err; "Failed to sync regions for table {}, table_id: {}", self.data.table_ref(), self.data.table_id()); + } + } + /// Update table metadata. pub(crate) async fn on_update_metadata(&mut self) -> Result { let table_id = self.data.table_id(); diff --git a/src/common/meta/src/ddl/create_logical_tables.rs b/src/common/meta/src/ddl/create_logical_tables.rs index 59882ec491..628f17d398 100644 --- a/src/common/meta/src/ddl/create_logical_tables.rs +++ b/src/common/meta/src/ddl/create_logical_tables.rs @@ -17,12 +17,14 @@ mod metadata; mod region_request; mod update_metadata; +use api::region::RegionResponse; use api::v1::CreateTableExpr; use async_trait::async_trait; +use common_catalog::consts::METRIC_ENGINE; use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu}; use common_procedure::{Context as ProcedureContext, LockKey, Procedure, Status}; -use common_telemetry::{debug, warn}; -use futures_util::future::join_all; +use common_telemetry::{debug, error, warn}; +use futures::future; use serde::{Deserialize, Serialize}; use snafu::{ensure, ResultExt}; use store_api::metadata::ColumnMetadata; @@ -31,7 +33,7 @@ use store_api::storage::{RegionId, RegionNumber}; use strum::AsRefStr; use table::metadata::{RawTableInfo, TableId}; -use crate::ddl::utils::{add_peer_context_if_needed, handle_retry_error}; +use crate::ddl::utils::{add_peer_context_if_needed, handle_retry_error, sync_follower_regions}; use crate::ddl::DdlContext; use crate::error::{DecodeJsonSnafu, MetadataCorruptionSnafu, Result}; use crate::key::table_route::TableRouteValue; @@ -156,14 +158,20 @@ impl CreateLogicalTablesProcedure { }); } - // Collects response from datanodes. - let phy_raw_schemas = join_all(create_region_tasks) + let mut results = future::join_all(create_region_tasks) .await .into_iter() - .map(|res| res.map(|mut res| res.extensions.remove(ALTER_PHYSICAL_EXTENSION_KEY))) .collect::>>()?; + // Collects response from datanodes. + let phy_raw_schemas = results + .iter_mut() + .map(|res| res.extensions.remove(ALTER_PHYSICAL_EXTENSION_KEY)) + .collect::>(); + if phy_raw_schemas.is_empty() { + self.submit_sync_region_requests(results, region_routes) + .await; self.data.state = CreateTablesState::CreateMetadata; return Ok(Status::executing(false)); } @@ -186,10 +194,30 @@ impl CreateLogicalTablesProcedure { warn!("creating logical table result doesn't contains extension key `{ALTER_PHYSICAL_EXTENSION_KEY}`,leaving the physical table's schema unchanged"); } + self.submit_sync_region_requests(results, region_routes) + .await; self.data.state = CreateTablesState::CreateMetadata; Ok(Status::executing(true)) } + + async fn submit_sync_region_requests( + &self, + results: Vec, + region_routes: &[RegionRoute], + ) { + if let Err(err) = sync_follower_regions( + &self.context, + self.data.physical_table_id, + results, + region_routes, + METRIC_ENGINE, + ) + .await + { + error!(err; "Failed to sync regions for physical table_id: {}",self.data.physical_table_id); + } + } } #[async_trait] diff --git a/src/common/meta/src/ddl/drop_table/executor.rs b/src/common/meta/src/ddl/drop_table/executor.rs index 1204629a1e..7aae31b13a 100644 --- a/src/common/meta/src/ddl/drop_table/executor.rs +++ b/src/common/meta/src/ddl/drop_table/executor.rs @@ -15,12 +15,13 @@ use std::collections::HashMap; use api::v1::region::{ - region_request, DropRequest as PbDropRegionRequest, RegionRequest, RegionRequestHeader, + region_request, CloseRequest as PbCloseRegionRequest, DropRequest as PbDropRegionRequest, + RegionRequest, RegionRequestHeader, }; use common_error::ext::ErrorExt; use common_error::status_code::StatusCode; -use common_telemetry::debug; use common_telemetry::tracing_context::TracingContext; +use common_telemetry::{debug, error}; use common_wal::options::WalOptions; use futures::future::join_all; use snafu::ensure; @@ -36,7 +37,8 @@ use crate::instruction::CacheIdent; use crate::key::table_name::TableNameKey; use crate::key::table_route::TableRouteValue; use crate::rpc::router::{ - find_leader_regions, find_leaders, operating_leader_regions, RegionRoute, + find_follower_regions, find_followers, find_leader_regions, find_leaders, + operating_leader_regions, RegionRoute, }; /// [Control] indicated to the caller whether to go to the next step. @@ -210,10 +212,10 @@ impl DropTableExecutor { region_routes: &[RegionRoute], fast_path: bool, ) -> Result<()> { + // Drops leader regions on datanodes. let leaders = find_leaders(region_routes); let mut drop_region_tasks = Vec::with_capacity(leaders.len()); let table_id = self.table_id; - for datanode in leaders { let requester = ctx.node_manager.datanode(&datanode).await; let regions = find_leader_regions(region_routes, &datanode); @@ -252,6 +254,53 @@ impl DropTableExecutor { .into_iter() .collect::>>()?; + // Drops follower regions on datanodes. + let followers = find_followers(region_routes); + let mut close_region_tasks = Vec::with_capacity(followers.len()); + for datanode in followers { + let requester = ctx.node_manager.datanode(&datanode).await; + let regions = find_follower_regions(region_routes, &datanode); + let region_ids = regions + .iter() + .map(|region_number| RegionId::new(table_id, *region_number)) + .collect::>(); + + for region_id in region_ids { + debug!("Closing region {region_id} on Datanode {datanode:?}"); + let request = RegionRequest { + header: Some(RegionRequestHeader { + tracing_context: TracingContext::from_current_span().to_w3c(), + ..Default::default() + }), + body: Some(region_request::Body::Close(PbCloseRegionRequest { + region_id: region_id.as_u64(), + })), + }; + + let datanode = datanode.clone(); + let requester = requester.clone(); + close_region_tasks.push(async move { + if let Err(err) = requester.handle(request).await { + if err.status_code() != StatusCode::RegionNotFound { + return Err(add_peer_context_if_needed(datanode)(err)); + } + } + Ok(()) + }); + } + } + + // Failure to close follower regions is not critical. + // When a leader region is dropped, follower regions will be unable to renew their leases via metasrv. + // Eventually, these follower regions will be automatically closed by the region livekeeper. + if let Err(err) = join_all(close_region_tasks) + .await + .into_iter() + .collect::>>() + { + error!(err; "Failed to close follower regions on datanodes, table_id: {}", table_id); + } + // Deletes the leader region from registry. let region_ids = operating_leader_regions(region_routes); ctx.leader_region_registry diff --git a/src/common/meta/src/ddl/test_util/create_table.rs b/src/common/meta/src/ddl/test_util/create_table.rs index 12896fbf91..9d99bbf5c6 100644 --- a/src/common/meta/src/ddl/test_util/create_table.rs +++ b/src/common/meta/src/ddl/test_util/create_table.rs @@ -18,7 +18,9 @@ use api::v1::column_def::try_as_column_schema; use api::v1::meta::Partition; use api::v1::{ColumnDataType, ColumnDef, CreateTableExpr, SemanticType}; use chrono::DateTime; -use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MITO2_ENGINE}; +use common_catalog::consts::{ + DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, MITO2_ENGINE, MITO_ENGINE, +}; use datatypes::schema::RawSchema; use derive_builder::Builder; use store_api::storage::TableId; @@ -164,6 +166,7 @@ pub fn test_create_table_task(name: &str, table_id: TableId) -> CreateTableTask .time_index("ts") .primary_keys(["host".into()]) .table_name(name) + .engine(MITO_ENGINE) .build() .unwrap() .into(); diff --git a/src/common/meta/src/ddl/test_util/datanode_handler.rs b/src/common/meta/src/ddl/test_util/datanode_handler.rs index 7f02d9cc5a..bed78724a5 100644 --- a/src/common/meta/src/ddl/test_util/datanode_handler.rs +++ b/src/common/meta/src/ddl/test_util/datanode_handler.rs @@ -45,14 +45,41 @@ impl MockDatanodeHandler for () { } #[derive(Clone)] -pub struct DatanodeWatcher(pub mpsc::Sender<(Peer, RegionRequest)>); +pub struct DatanodeWatcher { + sender: mpsc::Sender<(Peer, RegionRequest)>, + handler: Option Result>, +} + +impl DatanodeWatcher { + pub fn new(sender: mpsc::Sender<(Peer, RegionRequest)>) -> Self { + Self { + sender, + handler: None, + } + } + + pub fn with_handler( + mut self, + user_handler: fn(Peer, RegionRequest) -> Result, + ) -> Self { + self.handler = Some(user_handler); + self + } +} #[async_trait::async_trait] impl MockDatanodeHandler for DatanodeWatcher { async fn handle(&self, peer: &Peer, request: RegionRequest) -> Result { debug!("Returning Ok(0) for request: {request:?}, peer: {peer:?}"); - self.0.send((peer.clone(), request)).await.unwrap(); - Ok(RegionResponse::new(0)) + self.sender + .send((peer.clone(), request.clone())) + .await + .unwrap(); + if let Some(handler) = self.handler { + handler(peer.clone(), request) + } else { + Ok(RegionResponse::new(0)) + } } async fn handle_query( diff --git a/src/common/meta/src/ddl/tests/alter_logical_tables.rs b/src/common/meta/src/ddl/tests/alter_logical_tables.rs index 1c22cdf6f4..01ab8e513c 100644 --- a/src/common/meta/src/ddl/tests/alter_logical_tables.rs +++ b/src/common/meta/src/ddl/tests/alter_logical_tables.rs @@ -15,19 +15,33 @@ use std::assert_matches::assert_matches; use std::sync::Arc; +use api::region::RegionResponse; +use api::v1::meta::Peer; +use api::v1::region::sync_request::ManifestInfo; +use api::v1::region::{region_request, MetricManifestInfo, RegionRequest, SyncRequest}; use api::v1::{ColumnDataType, SemanticType}; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_procedure::{Procedure, ProcedureId, Status}; use common_procedure_test::MockContextProvider; +use store_api::metric_engine_consts::MANIFEST_INFO_EXTENSION_KEY; +use store_api::region_engine::RegionManifestInfo; +use store_api::storage::RegionId; +use tokio::sync::mpsc; use crate::ddl::alter_logical_tables::AlterLogicalTablesProcedure; use crate::ddl::test_util::alter_table::TestAlterTableExprBuilder; use crate::ddl::test_util::columns::TestColumnDefBuilder; -use crate::ddl::test_util::datanode_handler::NaiveDatanodeHandler; -use crate::ddl::test_util::{create_logical_table, create_physical_table}; +use crate::ddl::test_util::datanode_handler::{DatanodeWatcher, NaiveDatanodeHandler}; +use crate::ddl::test_util::{ + create_logical_table, create_physical_table, create_physical_table_metadata, + test_create_physical_table_task, +}; use crate::error::Error::{AlterLogicalTablesInvalidArguments, TableNotFound}; +use crate::error::Result; use crate::key::table_name::TableNameKey; +use crate::key::table_route::{PhysicalTableRouteValue, TableRouteValue}; use crate::rpc::ddl::AlterTableTask; +use crate::rpc::router::{Region, RegionRoute}; use crate::test_util::{new_ddl_context, MockDatanodeManager}; fn make_alter_logical_table_add_column_task( @@ -407,3 +421,78 @@ async fn test_on_part_duplicate_alter_request() { ] ); } + +fn alters_request_handler(_peer: Peer, request: RegionRequest) -> Result { + if let region_request::Body::Alters(_) = request.body.unwrap() { + let mut response = RegionResponse::new(0); + // Default region id for physical table. + let region_id = RegionId::new(1000, 1); + response.extensions.insert( + MANIFEST_INFO_EXTENSION_KEY.to_string(), + RegionManifestInfo::encode_list(&[(region_id, RegionManifestInfo::metric(1, 0, 2, 0))]) + .unwrap(), + ); + return Ok(response); + } + + Ok(RegionResponse::new(0)) +} + +#[tokio::test] +async fn test_on_submit_alter_region_request() { + common_telemetry::init_default_ut_logging(); + let (tx, mut rx) = mpsc::channel(8); + let handler = DatanodeWatcher::new(tx).with_handler(alters_request_handler); + let node_manager = Arc::new(MockDatanodeManager::new(handler)); + let ddl_context = new_ddl_context(node_manager); + + let mut create_physical_table_task = test_create_physical_table_task("phy"); + let phy_id = 1000u32; + let region_routes = vec![RegionRoute { + region: Region::new_test(RegionId::new(phy_id, 1)), + leader_peer: Some(Peer::empty(1)), + follower_peers: vec![Peer::empty(5)], + leader_state: None, + leader_down_since: None, + }]; + create_physical_table_task.set_table_id(phy_id); + create_physical_table_metadata( + &ddl_context, + create_physical_table_task.table_info.clone(), + TableRouteValue::Physical(PhysicalTableRouteValue::new(region_routes)), + ) + .await; + create_logical_table(ddl_context.clone(), phy_id, "table1").await; + create_logical_table(ddl_context.clone(), phy_id, "table2").await; + + let tasks = vec![ + make_alter_logical_table_add_column_task(None, "table1", vec!["new_col".to_string()]), + make_alter_logical_table_add_column_task(None, "table2", vec!["mew_col".to_string()]), + ]; + + let mut procedure = AlterLogicalTablesProcedure::new(tasks, phy_id, ddl_context); + procedure.on_prepare().await.unwrap(); + procedure.on_submit_alter_region_requests().await.unwrap(); + let mut results = Vec::new(); + for _ in 0..2 { + let result = rx.try_recv().unwrap(); + results.push(result); + } + rx.try_recv().unwrap_err(); + let (peer, request) = results.remove(0); + assert_eq!(peer.id, 1); + assert_matches!(request.body.unwrap(), region_request::Body::Alters(_)); + let (peer, request) = results.remove(0); + assert_eq!(peer.id, 5); + assert_matches!( + request.body.unwrap(), + region_request::Body::Sync(SyncRequest { + manifest_info: Some(ManifestInfo::MetricManifestInfo(MetricManifestInfo { + data_manifest_version: 1, + metadata_manifest_version: 2, + .. + })), + .. + }) + ); +} diff --git a/src/common/meta/src/ddl/tests/alter_table.rs b/src/common/meta/src/ddl/tests/alter_table.rs index c8d0450b90..6f24870e6e 100644 --- a/src/common/meta/src/ddl/tests/alter_table.rs +++ b/src/common/meta/src/ddl/tests/alter_table.rs @@ -16,7 +16,9 @@ use std::assert_matches::assert_matches; use std::collections::HashMap; use std::sync::Arc; +use api::region::RegionResponse; use api::v1::alter_table_expr::Kind; +use api::v1::region::sync_request::ManifestInfo; use api::v1::region::{region_request, RegionRequest}; use api::v1::{ AddColumn, AddColumns, AlterTableExpr, ColumnDataType, ColumnDef as PbColumnDef, DropColumn, @@ -28,6 +30,8 @@ use common_error::status_code::StatusCode; use common_procedure::store::poison_store::PoisonStore; use common_procedure::{ProcedureId, Status}; use common_procedure_test::MockContextProvider; +use store_api::metric_engine_consts::MANIFEST_INFO_EXTENSION_KEY; +use store_api::region_engine::RegionManifestInfo; use store_api::storage::RegionId; use table::requests::TTL_KEY; use tokio::sync::mpsc::{self}; @@ -39,7 +43,7 @@ use crate::ddl::test_util::datanode_handler::{ AllFailureDatanodeHandler, DatanodeWatcher, PartialSuccessDatanodeHandler, RequestOutdatedErrorDatanodeHandler, }; -use crate::error::Error; +use crate::error::{Error, Result}; use crate::key::datanode_table::DatanodeTableKey; use crate::key::table_name::TableNameKey; use crate::key::table_route::TableRouteValue; @@ -120,10 +124,71 @@ async fn test_on_prepare_table_not_exists_err() { assert_matches!(err.status_code(), StatusCode::TableNotFound); } +fn test_alter_table_task(table_name: &str) -> AlterTableTask { + AlterTableTask { + alter_table: AlterTableExpr { + catalog_name: DEFAULT_CATALOG_NAME.to_string(), + schema_name: DEFAULT_SCHEMA_NAME.to_string(), + table_name: table_name.to_string(), + kind: Some(Kind::DropColumns(DropColumns { + drop_columns: vec![DropColumn { + name: "cpu".to_string(), + }], + })), + }, + } +} + +fn assert_alter_request( + peer: Peer, + request: RegionRequest, + expected_peer_id: u64, + expected_region_id: RegionId, +) { + assert_eq!(peer.id, expected_peer_id); + let Some(region_request::Body::Alter(req)) = request.body else { + unreachable!(); + }; + assert_eq!(req.region_id, expected_region_id); +} + +fn assert_sync_request( + peer: Peer, + request: RegionRequest, + expected_peer_id: u64, + expected_region_id: RegionId, + expected_manifest_version: u64, +) { + assert_eq!(peer.id, expected_peer_id); + let Some(region_request::Body::Sync(req)) = request.body else { + unreachable!(); + }; + let Some(ManifestInfo::MitoManifestInfo(info)) = req.manifest_info else { + unreachable!(); + }; + assert_eq!(info.data_manifest_version, expected_manifest_version); + assert_eq!(req.region_id, expected_region_id); +} + +fn alter_request_handler(_peer: Peer, request: RegionRequest) -> Result { + if let region_request::Body::Alter(req) = request.body.unwrap() { + let mut response = RegionResponse::new(0); + let region_id = RegionId::from(req.region_id); + response.extensions.insert( + MANIFEST_INFO_EXTENSION_KEY.to_string(), + RegionManifestInfo::encode_list(&[(region_id, RegionManifestInfo::mito(1, 1))]) + .unwrap(), + ); + return Ok(response); + } + + Ok(RegionResponse::new(0)) +} + #[tokio::test] async fn test_on_submit_alter_request() { let (tx, mut rx) = mpsc::channel(8); - let datanode_handler = DatanodeWatcher(tx); + let datanode_handler = DatanodeWatcher::new(tx).with_handler(alter_request_handler); let node_manager = Arc::new(MockDatanodeManager::new(datanode_handler)); let ddl_context = new_ddl_context(node_manager); let table_id = 1024; @@ -140,18 +205,7 @@ async fn test_on_submit_alter_request() { .await .unwrap(); - let alter_table_task = AlterTableTask { - alter_table: AlterTableExpr { - catalog_name: DEFAULT_CATALOG_NAME.to_string(), - schema_name: DEFAULT_SCHEMA_NAME.to_string(), - table_name: table_name.to_string(), - kind: Some(Kind::DropColumns(DropColumns { - drop_columns: vec![DropColumn { - name: "cpu".to_string(), - }], - })), - }, - }; + let alter_table_task = test_alter_table_task(table_name); let procedure_id = ProcedureId::random(); let provider = Arc::new(MockContextProvider::default()); let mut procedure = @@ -162,30 +216,72 @@ async fn test_on_submit_alter_request() { .await .unwrap(); - let check = |peer: Peer, - request: RegionRequest, - expected_peer_id: u64, - expected_region_id: RegionId| { - assert_eq!(peer.id, expected_peer_id); - let Some(region_request::Body::Alter(req)) = request.body else { - unreachable!(); - }; - assert_eq!(req.region_id, expected_region_id); - }; + let mut results = Vec::new(); + for _ in 0..5 { + let result = rx.try_recv().unwrap(); + results.push(result); + } + rx.try_recv().unwrap_err(); + results.sort_unstable_by(|(a, _), (b, _)| a.id.cmp(&b.id)); + + let (peer, request) = results.remove(0); + assert_alter_request(peer, request, 1, RegionId::new(table_id, 1)); + let (peer, request) = results.remove(0); + assert_alter_request(peer, request, 2, RegionId::new(table_id, 2)); + let (peer, request) = results.remove(0); + assert_alter_request(peer, request, 3, RegionId::new(table_id, 3)); + let (peer, request) = results.remove(0); + assert_sync_request(peer, request, 4, RegionId::new(table_id, 2), 1); + let (peer, request) = results.remove(0); + assert_sync_request(peer, request, 5, RegionId::new(table_id, 1), 1); +} + +#[tokio::test] +async fn test_on_submit_alter_request_without_sync_request() { + let (tx, mut rx) = mpsc::channel(8); + // without use `alter_request_handler`, so no sync request will be sent. + let datanode_handler = DatanodeWatcher::new(tx); + let node_manager = Arc::new(MockDatanodeManager::new(datanode_handler)); + let ddl_context = new_ddl_context(node_manager); + let table_id = 1024; + let table_name = "foo"; + let task = test_create_table_task(table_name, table_id); + // Puts a value to table name key. + ddl_context + .table_metadata_manager + .create_table_metadata( + task.table_info.clone(), + prepare_table_route(table_id), + HashMap::new(), + ) + .await + .unwrap(); + + let alter_table_task = test_alter_table_task(table_name); + let procedure_id = ProcedureId::random(); + let provider = Arc::new(MockContextProvider::default()); + let mut procedure = + AlterTableProcedure::new(table_id, alter_table_task, ddl_context.clone()).unwrap(); + procedure.on_prepare().await.unwrap(); + procedure + .submit_alter_region_requests(procedure_id, provider.as_ref()) + .await + .unwrap(); let mut results = Vec::new(); for _ in 0..3 { let result = rx.try_recv().unwrap(); results.push(result); } + rx.try_recv().unwrap_err(); results.sort_unstable_by(|(a, _), (b, _)| a.id.cmp(&b.id)); let (peer, request) = results.remove(0); - check(peer, request, 1, RegionId::new(table_id, 1)); + assert_alter_request(peer, request, 1, RegionId::new(table_id, 1)); let (peer, request) = results.remove(0); - check(peer, request, 2, RegionId::new(table_id, 2)); + assert_alter_request(peer, request, 2, RegionId::new(table_id, 2)); let (peer, request) = results.remove(0); - check(peer, request, 3, RegionId::new(table_id, 3)); + assert_alter_request(peer, request, 3, RegionId::new(table_id, 3)); } #[tokio::test] diff --git a/src/common/meta/src/ddl/tests/create_logical_tables.rs b/src/common/meta/src/ddl/tests/create_logical_tables.rs index fb5518d463..af0af4ccdb 100644 --- a/src/common/meta/src/ddl/tests/create_logical_tables.rs +++ b/src/common/meta/src/ddl/tests/create_logical_tables.rs @@ -15,20 +15,28 @@ use std::assert_matches::assert_matches; use std::sync::Arc; +use api::region::RegionResponse; +use api::v1::meta::Peer; +use api::v1::region::sync_request::ManifestInfo; +use api::v1::region::{region_request, MetricManifestInfo, RegionRequest, SyncRequest}; use common_error::ext::ErrorExt; use common_error::status_code::StatusCode; use common_procedure::{Context as ProcedureContext, Procedure, ProcedureId, Status}; use common_procedure_test::MockContextProvider; +use store_api::metric_engine_consts::MANIFEST_INFO_EXTENSION_KEY; +use store_api::region_engine::RegionManifestInfo; use store_api::storage::RegionId; +use tokio::sync::mpsc; use crate::ddl::create_logical_tables::CreateLogicalTablesProcedure; -use crate::ddl::test_util::datanode_handler::NaiveDatanodeHandler; +use crate::ddl::test_util::datanode_handler::{DatanodeWatcher, NaiveDatanodeHandler}; use crate::ddl::test_util::{ create_physical_table_metadata, test_create_logical_table_task, test_create_physical_table_task, }; use crate::ddl::TableMetadata; -use crate::error::Error; -use crate::key::table_route::TableRouteValue; +use crate::error::{Error, Result}; +use crate::key::table_route::{PhysicalTableRouteValue, TableRouteValue}; +use crate::rpc::router::{Region, RegionRoute}; use crate::test_util::{new_ddl_context, MockDatanodeManager}; #[tokio::test] @@ -390,3 +398,76 @@ async fn test_on_create_metadata_err() { let error = procedure.execute(&ctx).await.unwrap_err(); assert!(!error.is_retry_later()); } + +fn creates_request_handler(_peer: Peer, request: RegionRequest) -> Result { + if let region_request::Body::Creates(_) = request.body.unwrap() { + let mut response = RegionResponse::new(0); + // Default region id for physical table. + let region_id = RegionId::new(1024, 1); + response.extensions.insert( + MANIFEST_INFO_EXTENSION_KEY.to_string(), + RegionManifestInfo::encode_list(&[(region_id, RegionManifestInfo::metric(1, 0, 2, 0))]) + .unwrap(), + ); + return Ok(response); + } + + Ok(RegionResponse::new(0)) +} + +#[tokio::test] +async fn test_on_submit_create_request() { + common_telemetry::init_default_ut_logging(); + let (tx, mut rx) = mpsc::channel(8); + let handler = DatanodeWatcher::new(tx).with_handler(creates_request_handler); + let node_manager = Arc::new(MockDatanodeManager::new(handler)); + let ddl_context = new_ddl_context(node_manager); + let mut create_physical_table_task = test_create_physical_table_task("phy_table"); + let table_id = 1024u32; + let region_routes = vec![RegionRoute { + region: Region::new_test(RegionId::new(table_id, 1)), + leader_peer: Some(Peer::empty(1)), + follower_peers: vec![Peer::empty(5)], + leader_state: None, + leader_down_since: None, + }]; + create_physical_table_task.set_table_id(table_id); + create_physical_table_metadata( + &ddl_context, + create_physical_table_task.table_info.clone(), + TableRouteValue::Physical(PhysicalTableRouteValue::new(region_routes)), + ) + .await; + let physical_table_id = table_id; + let task = test_create_logical_table_task("foo"); + let yet_another_task = test_create_logical_table_task("bar"); + let mut procedure = CreateLogicalTablesProcedure::new( + vec![task, yet_another_task], + physical_table_id, + ddl_context, + ); + procedure.on_prepare().await.unwrap(); + procedure.on_datanode_create_regions().await.unwrap(); + let mut results = Vec::new(); + for _ in 0..2 { + let result = rx.try_recv().unwrap(); + results.push(result); + } + rx.try_recv().unwrap_err(); + let (peer, request) = results.remove(0); + assert_eq!(peer.id, 1); + assert_matches!(request.body.unwrap(), region_request::Body::Creates(_)); + let (peer, request) = results.remove(0); + assert_eq!(peer.id, 5); + assert_matches!( + request.body.unwrap(), + region_request::Body::Sync(SyncRequest { + manifest_info: Some(ManifestInfo::MetricManifestInfo(MetricManifestInfo { + data_manifest_version: 1, + metadata_manifest_version: 2, + .. + })), + .. + }) + ); +} diff --git a/src/common/meta/src/ddl/tests/drop_table.rs b/src/common/meta/src/ddl/tests/drop_table.rs index 3e09f65422..9983e19ec5 100644 --- a/src/common/meta/src/ddl/tests/drop_table.rs +++ b/src/common/meta/src/ddl/tests/drop_table.rs @@ -100,7 +100,7 @@ async fn test_on_prepare_table() { #[tokio::test] async fn test_on_datanode_drop_regions() { let (tx, mut rx) = mpsc::channel(8); - let datanode_handler = DatanodeWatcher(tx); + let datanode_handler = DatanodeWatcher::new(tx); let node_manager = Arc::new(MockDatanodeManager::new(datanode_handler)); let ddl_context = new_ddl_context(node_manager); let table_id = 1024; @@ -148,27 +148,39 @@ async fn test_on_datanode_drop_regions() { let check = |peer: Peer, request: RegionRequest, expected_peer_id: u64, - expected_region_id: RegionId| { + expected_region_id: RegionId, + follower: bool| { assert_eq!(peer.id, expected_peer_id); - let Some(region_request::Body::Drop(req)) = request.body else { - unreachable!(); + if follower { + let Some(region_request::Body::Close(req)) = request.body else { + unreachable!(); + }; + assert_eq!(req.region_id, expected_region_id); + } else { + let Some(region_request::Body::Drop(req)) = request.body else { + unreachable!(); + }; + assert_eq!(req.region_id, expected_region_id); }; - assert_eq!(req.region_id, expected_region_id); }; let mut results = Vec::new(); - for _ in 0..3 { + for _ in 0..5 { let result = rx.try_recv().unwrap(); results.push(result); } results.sort_unstable_by(|(a, _), (b, _)| a.id.cmp(&b.id)); let (peer, request) = results.remove(0); - check(peer, request, 1, RegionId::new(table_id, 1)); + check(peer, request, 1, RegionId::new(table_id, 1), false); let (peer, request) = results.remove(0); - check(peer, request, 2, RegionId::new(table_id, 2)); + check(peer, request, 2, RegionId::new(table_id, 2), false); let (peer, request) = results.remove(0); - check(peer, request, 3, RegionId::new(table_id, 3)); + check(peer, request, 3, RegionId::new(table_id, 3), false); + let (peer, request) = results.remove(0); + check(peer, request, 4, RegionId::new(table_id, 2), true); + let (peer, request) = results.remove(0); + check(peer, request, 5, RegionId::new(table_id, 1), true); } #[tokio::test] diff --git a/src/common/meta/src/ddl/utils.rs b/src/common/meta/src/ddl/utils.rs index 2e909946e0..8c48d9f419 100644 --- a/src/common/meta/src/ddl/utils.rs +++ b/src/common/meta/src/ddl/utils.rs @@ -15,27 +15,37 @@ use std::collections::HashMap; use std::fmt::Debug; -use common_catalog::consts::METRIC_ENGINE; +use api::region::RegionResponse; +use api::v1::region::sync_request::ManifestInfo; +use api::v1::region::{ + region_request, MetricManifestInfo, MitoManifestInfo, RegionRequest, RegionRequestHeader, + SyncRequest, +}; +use common_catalog::consts::{METRIC_ENGINE, MITO_ENGINE}; use common_error::ext::BoxedError; use common_procedure::error::Error as ProcedureError; -use common_telemetry::{error, warn}; +use common_telemetry::tracing_context::TracingContext; +use common_telemetry::{error, info, warn}; use common_wal::options::WalOptions; +use futures::future::join_all; use snafu::{ensure, OptionExt, ResultExt}; -use store_api::metric_engine_consts::LOGICAL_TABLE_METADATA_KEY; -use store_api::storage::RegionNumber; +use store_api::metric_engine_consts::{LOGICAL_TABLE_METADATA_KEY, MANIFEST_INFO_EXTENSION_KEY}; +use store_api::region_engine::RegionManifestInfo; +use store_api::storage::{RegionId, RegionNumber}; use table::metadata::TableId; use table::table_reference::TableReference; -use crate::ddl::DetectingRegion; +use crate::ddl::{DdlContext, DetectingRegion}; use crate::error::{ - Error, OperateDatanodeSnafu, ParseWalOptionsSnafu, Result, TableNotFoundSnafu, UnsupportedSnafu, + self, Error, OperateDatanodeSnafu, ParseWalOptionsSnafu, Result, TableNotFoundSnafu, + UnsupportedSnafu, }; use crate::key::datanode_table::DatanodeTableValue; use crate::key::table_name::TableNameKey; use crate::key::TableMetadataManagerRef; use crate::peer::Peer; use crate::rpc::ddl::CreateTableTask; -use crate::rpc::router::RegionRoute; +use crate::rpc::router::{find_follower_regions, find_followers, RegionRoute}; /// Adds [Peer] context if the error is unretryable. pub fn add_peer_context_if_needed(datanode: Peer) -> impl FnOnce(Error) -> Error { @@ -192,8 +202,8 @@ pub fn extract_region_wal_options( /// - PartialNonRetryable: if any operation is non retryable, the result is non retryable. /// - AllRetryable: all operations are retryable. /// - AllNonRetryable: all operations are not retryable. -pub enum MultipleResults { - Ok, +pub enum MultipleResults { + Ok(Vec), PartialRetryable(Error), PartialNonRetryable(Error), AllRetryable(Error), @@ -205,9 +215,9 @@ pub enum MultipleResults { /// For partial success, we need to check if the errors are retryable. /// If all the errors are retryable, we return a retryable error. /// Otherwise, we return the first error. -pub fn handle_multiple_results(results: Vec>) -> MultipleResults { +pub fn handle_multiple_results(results: Vec>) -> MultipleResults { if results.is_empty() { - return MultipleResults::Ok; + return MultipleResults::Ok(Vec::new()); } let num_results = results.len(); let mut retryable_results = Vec::new(); @@ -216,7 +226,7 @@ pub fn handle_multiple_results(results: Vec>) -> MultipleRes for result in results { match result { - Ok(_) => ok_results.push(result), + Ok(value) => ok_results.push(value), Err(err) => { if err.is_retry_later() { retryable_results.push(err); @@ -243,7 +253,7 @@ pub fn handle_multiple_results(results: Vec>) -> MultipleRes } return MultipleResults::AllNonRetryable(non_retryable_results.into_iter().next().unwrap()); } else if ok_results.len() == num_results { - return MultipleResults::Ok; + return MultipleResults::Ok(ok_results); } else if !retryable_results.is_empty() && !ok_results.is_empty() && non_retryable_results.is_empty() @@ -264,6 +274,125 @@ pub fn handle_multiple_results(results: Vec>) -> MultipleRes MultipleResults::PartialNonRetryable(non_retryable_results.into_iter().next().unwrap()) } +/// Parses manifest infos from extensions. +pub fn parse_manifest_infos_from_extensions( + extensions: &HashMap>, +) -> Result> { + let data_manifest_version = + extensions + .get(MANIFEST_INFO_EXTENSION_KEY) + .context(error::UnexpectedSnafu { + err_msg: "manifest info extension not found", + })?; + let data_manifest_version = + RegionManifestInfo::decode_list(data_manifest_version).context(error::SerdeJsonSnafu {})?; + Ok(data_manifest_version) +} + +/// Sync follower regions on datanodes. +pub async fn sync_follower_regions( + context: &DdlContext, + table_id: TableId, + results: Vec, + region_routes: &[RegionRoute], + engine: &str, +) -> Result<()> { + if engine != MITO_ENGINE && engine != METRIC_ENGINE { + info!( + "Skip submitting sync region requests for table_id: {}, engine: {}", + table_id, engine + ); + return Ok(()); + } + + let results = results + .into_iter() + .map(|response| parse_manifest_infos_from_extensions(&response.extensions)) + .collect::>>()? + .into_iter() + .flatten() + .collect::>(); + + let is_mito_engine = engine == MITO_ENGINE; + + let followers = find_followers(region_routes); + if followers.is_empty() { + return Ok(()); + } + let mut sync_region_tasks = Vec::with_capacity(followers.len()); + for datanode in followers { + let requester = context.node_manager.datanode(&datanode).await; + let regions = find_follower_regions(region_routes, &datanode); + for region in regions { + let region_id = RegionId::new(table_id, region); + let manifest_info = if is_mito_engine { + let region_manifest_info = + results.get(®ion_id).context(error::UnexpectedSnafu { + err_msg: format!("No manifest info found for region {}", region_id), + })?; + ensure!( + region_manifest_info.is_mito(), + error::UnexpectedSnafu { + err_msg: format!("Region {} is not a mito region", region_id) + } + ); + ManifestInfo::MitoManifestInfo(MitoManifestInfo { + data_manifest_version: region_manifest_info.data_manifest_version(), + }) + } else { + let region_manifest_info = + results.get(®ion_id).context(error::UnexpectedSnafu { + err_msg: format!("No manifest info found for region {}", region_id), + })?; + ensure!( + region_manifest_info.is_metric(), + error::UnexpectedSnafu { + err_msg: format!("Region {} is not a metric region", region_id) + } + ); + ManifestInfo::MetricManifestInfo(MetricManifestInfo { + data_manifest_version: region_manifest_info.data_manifest_version(), + metadata_manifest_version: region_manifest_info + .metadata_manifest_version() + .unwrap_or_default(), + }) + }; + let request = RegionRequest { + header: Some(RegionRequestHeader { + tracing_context: TracingContext::from_current_span().to_w3c(), + ..Default::default() + }), + body: Some(region_request::Body::Sync(SyncRequest { + region_id: region_id.as_u64(), + manifest_info: Some(manifest_info), + })), + }; + + let datanode = datanode.clone(); + let requester = requester.clone(); + sync_region_tasks.push(async move { + requester + .handle(request) + .await + .map_err(add_peer_context_if_needed(datanode)) + }); + } + } + + // Failure to sync region is not critical. + // We try our best to sync the regions. + if let Err(err) = join_all(sync_region_tasks) + .await + .into_iter() + .collect::>>() + { + error!(err; "Failed to sync follower regions on datanodes, table_id: {}", table_id); + } + info!("Sync follower regions on datanodes, table_id: {}", table_id); + + Ok(()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/common/meta/src/rpc/router.rs b/src/common/meta/src/rpc/router.rs index 03a1e0bfa0..7ddd104c61 100644 --- a/src/common/meta/src/rpc/router.rs +++ b/src/common/meta/src/rpc/router.rs @@ -62,6 +62,7 @@ pub struct TableRoute { region_leaders: HashMap>, } +/// Returns the leader peers of the table. pub fn find_leaders(region_routes: &[RegionRoute]) -> HashSet { region_routes .iter() @@ -70,6 +71,15 @@ pub fn find_leaders(region_routes: &[RegionRoute]) -> HashSet { .collect() } +/// Returns the followers of the table. +pub fn find_followers(region_routes: &[RegionRoute]) -> HashSet { + region_routes + .iter() + .flat_map(|x| &x.follower_peers) + .cloned() + .collect() +} + /// Returns the operating leader regions with corresponding [DatanodeId]. pub fn operating_leader_regions(region_routes: &[RegionRoute]) -> Vec<(RegionId, DatanodeId)> { region_routes @@ -108,6 +118,7 @@ pub fn find_region_leader( .cloned() } +/// Returns the region numbers of the leader regions on the target datanode. pub fn find_leader_regions(region_routes: &[RegionRoute], datanode: &Peer) -> Vec { region_routes .iter() @@ -122,6 +133,19 @@ pub fn find_leader_regions(region_routes: &[RegionRoute], datanode: &Peer) -> Ve .collect() } +/// Returns the region numbers of the follower regions on the target datanode. +pub fn find_follower_regions(region_routes: &[RegionRoute], datanode: &Peer) -> Vec { + region_routes + .iter() + .filter_map(|x| { + if x.follower_peers.contains(datanode) { + return Some(x.region.id.region_number()); + } + None + }) + .collect() +} + impl TableRoute { pub fn new(table: Table, region_routes: Vec) -> Self { let region_leaders = region_routes diff --git a/src/datanode/src/region_server.rs b/src/datanode/src/region_server.rs index 072c1682cc..9bbb899bf1 100644 --- a/src/datanode/src/region_server.rs +++ b/src/datanode/src/region_server.rs @@ -19,7 +19,8 @@ use std::sync::{Arc, RwLock}; use std::time::Duration; use api::region::RegionResponse; -use api::v1::region::{region_request, RegionResponse as RegionResponseV1}; +use api::v1::region::sync_request::ManifestInfo; +use api::v1::region::{region_request, RegionResponse as RegionResponseV1, SyncRequest}; use api::v1::{ResponseHeader, Status}; use arrow_flight::{FlightData, Ticket}; use async_trait::async_trait; @@ -308,38 +309,6 @@ impl RegionServer { .with_context(|_| HandleRegionRequestSnafu { region_id }) } - /// Sync region manifest and registers new opened logical regions. - pub async fn sync_region_manifest( - &self, - region_id: RegionId, - manifest_info: RegionManifestInfo, - ) -> Result<()> { - let engine_with_status = self - .inner - .region_map - .get(®ion_id) - .with_context(|| RegionNotFoundSnafu { region_id })?; - - let Some(new_opened_regions) = engine_with_status - .sync_region(region_id, manifest_info) - .await - .with_context(|_| HandleRegionRequestSnafu { region_id })? - .new_opened_logical_region_ids() - else { - return Ok(()); - }; - - for region in new_opened_regions { - self.inner.region_map.insert( - region, - RegionEngineWithStatus::Ready(engine_with_status.engine().clone()), - ); - info!("Logical region {} is registered!", region); - } - - Ok(()) - } - /// Set region role state gracefully. /// /// For [SettableRegionRoleState::Follower]: @@ -468,6 +437,52 @@ impl RegionServer { extensions, }) } + + async fn handle_sync_region_request(&self, request: &SyncRequest) -> Result { + let region_id = RegionId::from_u64(request.region_id); + let manifest_info = request + .manifest_info + .context(error::MissingRequiredFieldSnafu { + name: "manifest_info", + })?; + + let manifest_info = match manifest_info { + ManifestInfo::MitoManifestInfo(info) => { + RegionManifestInfo::mito(info.data_manifest_version, 0) + } + ManifestInfo::MetricManifestInfo(info) => RegionManifestInfo::metric( + info.data_manifest_version, + 0, + info.metadata_manifest_version, + 0, + ), + }; + + let tracing_context = TracingContext::from_current_span(); + let span = tracing_context.attach(info_span!("RegionServer::handle_sync_region_request")); + + self.sync_region(region_id, manifest_info) + .trace(span) + .await + .map(|_| RegionResponse::new(AffectedRows::default())) + } + + /// Sync region manifest and registers new opened logical regions. + pub async fn sync_region( + &self, + region_id: RegionId, + manifest_info: RegionManifestInfo, + ) -> Result<()> { + let engine_with_status = self + .inner + .region_map + .get(®ion_id) + .with_context(|| RegionNotFoundSnafu { region_id })?; + + self.inner + .handle_sync_region(engine_with_status.engine(), region_id, manifest_info) + .await + } } #[async_trait] @@ -480,6 +495,9 @@ impl RegionServerHandler for RegionServer { region_request::Body::Inserts(_) | region_request::Body::Deletes(_) => { self.handle_requests_in_parallel(request).await } + region_request::Body::Sync(sync_request) => { + self.handle_sync_region_request(sync_request).await + } _ => self.handle_requests_in_serial(request).await, } .map_err(BoxedError::new) @@ -861,8 +879,8 @@ impl RegionServerInner { match result { Ok(result) => { - for (region_id, region_change) in region_changes { - self.set_region_status_ready(region_id, engine.clone(), region_change) + for (region_id, region_change) in ®ion_changes { + self.set_region_status_ready(*region_id, engine.clone(), *region_change) .await?; } @@ -933,8 +951,9 @@ impl RegionServerInner { .inc_by(result.affected_rows as u64); } // Sets corresponding region status to ready. - self.set_region_status_ready(region_id, engine, region_change) + self.set_region_status_ready(region_id, engine.clone(), region_change) .await?; + Ok(RegionResponse { affected_rows: result.affected_rows, extensions: result.extensions, @@ -948,6 +967,32 @@ impl RegionServerInner { } } + /// Handles the sync region request. + pub async fn handle_sync_region( + &self, + engine: &RegionEngineRef, + region_id: RegionId, + manifest_info: RegionManifestInfo, + ) -> Result<()> { + let Some(new_opened_regions) = engine + .sync_region(region_id, manifest_info) + .await + .with_context(|_| HandleRegionRequestSnafu { region_id })? + .new_opened_logical_region_ids() + else { + warn!("No new opened logical regions"); + return Ok(()); + }; + + for region in new_opened_regions { + self.region_map + .insert(region, RegionEngineWithStatus::Ready(engine.clone())); + info!("Logical region {} is registered!", region); + } + + Ok(()) + } + fn set_region_status_not_ready( &self, region_id: RegionId, diff --git a/src/metric-engine/src/engine.rs b/src/metric-engine/src/engine.rs index 71cb843ab1..28709264b3 100644 --- a/src/metric-engine/src/engine.rs +++ b/src/metric-engine/src/engine.rs @@ -53,7 +53,7 @@ use crate::data_region::DataRegion; use crate::error::{self, Result, UnsupportedRegionRequestSnafu}; use crate::metadata_region::MetadataRegion; use crate::row_modifier::RowModifier; -use crate::utils; +use crate::utils::{self, get_region_statistic}; #[cfg_attr(doc, aquamarine::aquamarine)] /// # Metric Engine @@ -264,29 +264,7 @@ impl RegionEngine for MetricEngine { /// Note: Returns `None` if it's a logical region. fn region_statistic(&self, region_id: RegionId) -> Option { if self.inner.is_physical_region(region_id) { - let metadata_region_id = utils::to_metadata_region_id(region_id); - let data_region_id = utils::to_data_region_id(region_id); - - let metadata_stat = self.inner.mito.region_statistic(metadata_region_id); - let data_stat = self.inner.mito.region_statistic(data_region_id); - - match (metadata_stat, data_stat) { - (Some(metadata_stat), Some(data_stat)) => Some(RegionStatistic { - num_rows: metadata_stat.num_rows + data_stat.num_rows, - memtable_size: metadata_stat.memtable_size + data_stat.memtable_size, - wal_size: metadata_stat.wal_size + data_stat.wal_size, - manifest_size: metadata_stat.manifest_size + data_stat.manifest_size, - sst_size: metadata_stat.sst_size + data_stat.sst_size, - index_size: metadata_stat.index_size + data_stat.index_size, - manifest: RegionManifestInfo::Metric { - data_flushed_entry_id: data_stat.manifest.data_flushed_entry_id(), - data_manifest_version: data_stat.manifest.data_manifest_version(), - metadata_flushed_entry_id: metadata_stat.manifest.data_flushed_entry_id(), - metadata_manifest_version: metadata_stat.manifest.data_manifest_version(), - }, - }), - _ => None, - } + get_region_statistic(&self.inner.mito, region_id) } else { None } diff --git a/src/metric-engine/src/engine/alter.rs b/src/metric-engine/src/engine/alter.rs index 1d82149a7d..22ef54cab8 100644 --- a/src/metric-engine/src/engine/alter.rs +++ b/src/metric-engine/src/engine/alter.rs @@ -30,7 +30,7 @@ use crate::error::{ LogicalRegionNotFoundSnafu, PhysicalRegionNotFoundSnafu, Result, SerializeColumnMetadataSnafu, UnexpectedRequestSnafu, }; -use crate::utils::to_data_region_id; +use crate::utils::{append_manifest_info, encode_manifest_info_to_extensions, to_data_region_id}; impl MetricEngineInner { pub async fn alter_regions( @@ -63,11 +63,15 @@ impl MetricEngineInner { .unwrap() .get_physical_region_id(region_id) .with_context(|| LogicalRegionNotFoundSnafu { region_id })?; + let mut manifest_infos = Vec::with_capacity(1); self.alter_logical_regions(physical_region_id, requests, extension_return_value) .await?; + append_manifest_info(&self.mito, region_id, &mut manifest_infos); + encode_manifest_info_to_extensions(&manifest_infos, extension_return_value)?; } else { let grouped_requests = self.group_logical_region_requests_by_physical_region_id(requests)?; + let mut manifest_infos = Vec::with_capacity(grouped_requests.len()); for (physical_region_id, requests) in grouped_requests { self.alter_logical_regions( physical_region_id, @@ -75,7 +79,9 @@ impl MetricEngineInner { extension_return_value, ) .await?; + append_manifest_info(&self.mito, physical_region_id, &mut manifest_infos); } + encode_manifest_info_to_extensions(&manifest_infos, extension_return_value)?; } } Ok(0) diff --git a/src/metric-engine/src/engine/create.rs b/src/metric-engine/src/engine/create.rs index 1ceb20d206..78d2293302 100644 --- a/src/metric-engine/src/engine/create.rs +++ b/src/metric-engine/src/engine/create.rs @@ -51,7 +51,10 @@ use crate::error::{ Result, SerializeColumnMetadataSnafu, UnexpectedRequestSnafu, }; use crate::metrics::PHYSICAL_REGION_COUNT; -use crate::utils::{self, to_data_region_id, to_metadata_region_id}; +use crate::utils::{ + self, append_manifest_info, encode_manifest_info_to_extensions, to_data_region_id, + to_metadata_region_id, +}; const DEFAULT_TABLE_ID_SKIPPING_INDEX_GRANULARITY: u32 = 1024; @@ -88,11 +91,15 @@ impl MetricEngineInner { if requests.len() == 1 { let request = &requests.first().unwrap().1; let physical_region_id = parse_physical_region_id(request)?; + let mut manifest_infos = Vec::with_capacity(1); self.create_logical_regions(physical_region_id, requests, extension_return_value) .await?; + append_manifest_info(&self.mito, physical_region_id, &mut manifest_infos); + encode_manifest_info_to_extensions(&manifest_infos, extension_return_value)?; } else { let grouped_requests = group_create_logical_region_requests_by_physical_region_id(requests)?; + let mut manifest_infos = Vec::with_capacity(grouped_requests.len()); for (physical_region_id, requests) in grouped_requests { self.create_logical_regions( physical_region_id, @@ -100,7 +107,9 @@ impl MetricEngineInner { extension_return_value, ) .await?; + append_manifest_info(&self.mito, physical_region_id, &mut manifest_infos); } + encode_manifest_info_to_extensions(&manifest_infos, extension_return_value)?; } } else { return MissingRegionOptionSnafu {}.fail(); diff --git a/src/metric-engine/src/error.rs b/src/metric-engine/src/error.rs index 5f853037e1..015e9d9f98 100644 --- a/src/metric-engine/src/error.rs +++ b/src/metric-engine/src/error.rs @@ -67,6 +67,14 @@ pub enum Error { location: Location, }, + #[snafu(display("Failed to serialize region manifest info"))] + SerializeRegionManifestInfo { + #[snafu(source)] + error: serde_json::Error, + #[snafu(implicit)] + location: Location, + }, + #[snafu(display("Failed to decode base64 column value"))] DecodeColumnValue { #[snafu(source)] @@ -304,7 +312,8 @@ impl ErrorExt for Error { | DecodeColumnValue { .. } | ParseRegionId { .. } | InvalidMetadata { .. } - | SetSkippingIndexOption { .. } => StatusCode::Unexpected, + | SetSkippingIndexOption { .. } + | SerializeRegionManifestInfo { .. } => StatusCode::Unexpected, PhysicalRegionNotFound { .. } | LogicalRegionNotFound { .. } => { StatusCode::RegionNotFound diff --git a/src/metric-engine/src/utils.rs b/src/metric-engine/src/utils.rs index 183ba2eaa3..ad0695f54a 100644 --- a/src/metric-engine/src/utils.rs +++ b/src/metric-engine/src/utils.rs @@ -12,9 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -use store_api::metric_engine_consts::{METRIC_DATA_REGION_GROUP, METRIC_METADATA_REGION_GROUP}; +use std::collections::HashMap; + +use common_telemetry::{info, warn}; +use mito2::engine::MitoEngine; +use snafu::ResultExt; +use store_api::metric_engine_consts::{ + MANIFEST_INFO_EXTENSION_KEY, METRIC_DATA_REGION_GROUP, METRIC_METADATA_REGION_GROUP, +}; +use store_api::region_engine::{RegionEngine, RegionManifestInfo, RegionStatistic}; use store_api::storage::RegionId; +use crate::error::{Result, SerializeRegionManifestInfoSnafu}; + /// Change the given [RegionId]'s region group to [METRIC_METADATA_REGION_GROUP]. pub fn to_metadata_region_id(region_id: RegionId) -> RegionId { let table_id = region_id.table_id(); @@ -29,6 +39,69 @@ pub fn to_data_region_id(region_id: RegionId) -> RegionId { RegionId::with_group_and_seq(table_id, METRIC_DATA_REGION_GROUP, region_sequence) } +/// Get the region statistic of the given [RegionId]. +pub fn get_region_statistic(mito: &MitoEngine, region_id: RegionId) -> Option { + let metadata_region_id = to_metadata_region_id(region_id); + let data_region_id = to_data_region_id(region_id); + + let metadata_stat = mito.region_statistic(metadata_region_id); + let data_stat = mito.region_statistic(data_region_id); + + match (&metadata_stat, &data_stat) { + (Some(metadata_stat), Some(data_stat)) => Some(RegionStatistic { + num_rows: metadata_stat.num_rows + data_stat.num_rows, + memtable_size: metadata_stat.memtable_size + data_stat.memtable_size, + wal_size: metadata_stat.wal_size + data_stat.wal_size, + manifest_size: metadata_stat.manifest_size + data_stat.manifest_size, + sst_size: metadata_stat.sst_size + data_stat.sst_size, + index_size: metadata_stat.index_size + data_stat.index_size, + manifest: RegionManifestInfo::Metric { + data_flushed_entry_id: data_stat.manifest.data_flushed_entry_id(), + data_manifest_version: data_stat.manifest.data_manifest_version(), + metadata_flushed_entry_id: metadata_stat.manifest.data_flushed_entry_id(), + metadata_manifest_version: metadata_stat.manifest.data_manifest_version(), + }, + }), + _ => { + warn!( + "Failed to get region statistic for region {}, metadata_stat: {:?}, data_stat: {:?}", + region_id, metadata_stat, data_stat + ); + None + } + } +} + +/// Appends the given [RegionId]'s manifest info to the given list. +pub(crate) fn append_manifest_info( + mito: &MitoEngine, + region_id: RegionId, + manifest_infos: &mut Vec<(RegionId, RegionManifestInfo)>, +) { + if let Some(statistic) = get_region_statistic(mito, region_id) { + manifest_infos.push((region_id, statistic.manifest)); + } +} + +/// Encodes the given list of ([RegionId], [RegionManifestInfo]) to extensions(key: MANIFEST_INFO_EXTENSION_KEY). +pub(crate) fn encode_manifest_info_to_extensions( + manifest_infos: &[(RegionId, RegionManifestInfo)], + extensions: &mut HashMap>, +) -> Result<()> { + extensions.insert( + MANIFEST_INFO_EXTENSION_KEY.to_string(), + RegionManifestInfo::encode_list(manifest_infos) + .context(SerializeRegionManifestInfoSnafu)?, + ); + for (region_id, manifest_info) in manifest_infos { + info!( + "Added manifest info: {:?} to extensions, region_id: {:?}", + manifest_info, region_id + ); + } + Ok(()) +} + #[cfg(test)] mod tests { diff --git a/src/mito2/src/engine.rs b/src/mito2/src/engine.rs index 7b3c7352da..fbedef3c35 100644 --- a/src/mito2/src/engine.rs +++ b/src/mito2/src/engine.rs @@ -70,7 +70,7 @@ use common_base::Plugins; use common_error::ext::BoxedError; use common_meta::key::SchemaMetadataManagerRef; use common_recordbatch::SendableRecordBatchStream; -use common_telemetry::tracing; +use common_telemetry::{info, tracing}; use common_wal::options::{WalOptions, WAL_OPTIONS_KEY}; use futures::future::{join_all, try_join_all}; use object_store::manager::ObjectStoreManagerRef; @@ -80,6 +80,7 @@ use store_api::logstore::provider::Provider; use store_api::logstore::LogStore; use store_api::manifest::ManifestVersion; use store_api::metadata::RegionMetadataRef; +use store_api::metric_engine_consts::MANIFEST_INFO_EXTENSION_KEY; use store_api::region_engine::{ BatchResponses, RegionEngine, RegionManifestInfo, RegionRole, RegionScannerRef, RegionStatistic, SetRegionRoleStateResponse, SettableRegionRoleState, SyncManifestResponse, @@ -224,6 +225,24 @@ impl MitoEngine { pub(crate) fn get_region(&self, id: RegionId) -> Option { self.inner.workers.get_region(id) } + + fn encode_manifest_info_to_extensions( + region_id: &RegionId, + manifest_info: RegionManifestInfo, + extensions: &mut HashMap>, + ) -> Result<()> { + let region_manifest_info = vec![(*region_id, manifest_info)]; + + extensions.insert( + MANIFEST_INFO_EXTENSION_KEY.to_string(), + RegionManifestInfo::encode_list(®ion_manifest_info).context(SerdeJsonSnafu)?, + ); + info!( + "Added manifest info: {:?} to extensions, region_id: {:?}", + region_manifest_info, region_id + ); + Ok(()) + } } /// Check whether the region edit is valid. Only adding files to region is considered valid now. @@ -557,11 +576,26 @@ impl RegionEngine for MitoEngine { .with_label_values(&[request.request_type()]) .start_timer(); - self.inner + let is_alter = matches!(request, RegionRequest::Alter(_)); + let mut response = self + .inner .handle_request(region_id, request) .await .map(RegionResponse::new) - .map_err(BoxedError::new) + .map_err(BoxedError::new)?; + + if is_alter { + if let Some(statistic) = self.region_statistic(region_id) { + Self::encode_manifest_info_to_extensions( + ®ion_id, + statistic.manifest, + &mut response.extensions, + ) + .map_err(BoxedError::new)?; + } + } + + Ok(response) } #[tracing::instrument(skip_all)] diff --git a/src/store-api/src/metadata.rs b/src/store-api/src/metadata.rs index bb622ac7f8..426e3f69ca 100644 --- a/src/store-api/src/metadata.rs +++ b/src/store-api/src/metadata.rs @@ -966,6 +966,13 @@ pub enum MetadataError { #[snafu(implicit)] location: Location, }, + + #[snafu(display("Unexpected: {}", reason))] + Unexpected { + reason: String, + #[snafu(implicit)] + location: Location, + }, } impl ErrorExt for MetadataError { diff --git a/src/store-api/src/metric_engine_consts.rs b/src/store-api/src/metric_engine_consts.rs index af2a523e70..b2994b0b00 100644 --- a/src/store-api/src/metric_engine_consts.rs +++ b/src/store-api/src/metric_engine_consts.rs @@ -73,6 +73,10 @@ pub const LOGICAL_TABLE_METADATA_KEY: &str = "on_physical_table"; /// Represent a list of column metadata that are added to physical table. pub const ALTER_PHYSICAL_EXTENSION_KEY: &str = "ALTER_PHYSICAL"; +/// HashMap key to be used in the region server's extension response. +/// Represent the manifest info of a region. +pub const MANIFEST_INFO_EXTENSION_KEY: &str = "MANIFEST_INFO"; + /// Returns true if it's a internal column of the metric engine. pub fn is_metric_engine_internal_column(name: &str) -> bool { name == DATA_SCHEMA_TABLE_ID_COLUMN_NAME || name == DATA_SCHEMA_TSID_COLUMN_NAME diff --git a/src/store-api/src/region_engine.rs b/src/store-api/src/region_engine.rs index 8522a2e1ca..a2db2d9f52 100644 --- a/src/store-api/src/region_engine.rs +++ b/src/store-api/src/region_engine.rs @@ -549,6 +549,16 @@ impl RegionManifestInfo { } => Some(*metadata_flushed_entry_id), } } + + /// Encodes a list of ([RegionId], [RegionManifestInfo]) to a byte array. + pub fn encode_list(manifest_infos: &[(RegionId, Self)]) -> serde_json::Result> { + serde_json::to_vec(manifest_infos) + } + + /// Decodes a list of ([RegionId], [RegionManifestInfo]) from a byte array. + pub fn decode_list(value: &[u8]) -> serde_json::Result> { + serde_json::from_slice(value) + } } impl Default for RegionManifestInfo { diff --git a/src/store-api/src/region_request.rs b/src/store-api/src/region_request.rs index 08ad2ac559..5a6e1289c6 100644 --- a/src/store-api/src/region_request.rs +++ b/src/store-api/src/region_request.rs @@ -46,7 +46,7 @@ use crate::logstore::entry; use crate::metadata::{ ColumnMetadata, DecodeArrowIpcSnafu, DecodeProtoSnafu, InvalidRawRegionRequestSnafu, InvalidRegionRequestSnafu, InvalidSetRegionOptionRequestSnafu, - InvalidUnsetRegionOptionRequestSnafu, MetadataError, RegionMetadata, Result, + InvalidUnsetRegionOptionRequestSnafu, MetadataError, RegionMetadata, Result, UnexpectedSnafu, }; use crate::metric_engine_consts::PHYSICAL_TABLE_METADATA_KEY; use crate::mito_engine_options::{ @@ -153,6 +153,10 @@ impl RegionRequest { region_request::Body::Drops(drops) => make_region_drops(drops), region_request::Body::Alters(alters) => make_region_alters(alters), region_request::Body::BulkInserts(bulk) => make_region_bulk_inserts(bulk), + region_request::Body::Sync(_) => UnexpectedSnafu { + reason: "Sync request should be handled separately by RegionServer", + } + .fail(), } } From a9065f5319fcae0882722c023973b618d2977a36 Mon Sep 17 00:00:00 2001 From: discord9 <55937128+discord9@users.noreply.github.com> Date: Fri, 18 Apr 2025 19:04:19 +0800 Subject: [PATCH 40/82] chore: rm dev opt level 3 (#5932) remove accidentally added dev profile opt level 3 for depend --- Cargo.toml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5e95df1487..3753c457f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -307,8 +307,3 @@ strip = true [profile.dev.package.tests-fuzz] debug = false strip = true - -[profile.dev] -opt-level = 1 -[profile.dev.package."*"] -opt-level = 3 From 1e394af583366a6499dc662a355a0f373d060a7d Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Fri, 18 Apr 2025 19:13:01 +0800 Subject: [PATCH 41/82] feat: prevent migrating a leader region to a peer that already has a region follower (#5923) * feat: prevent migrating a leader region to a peer that already has a region follower * chore: refine err msg --- src/meta-srv/Cargo.toml | 1 + .../src/procedure/region_migration/manager.rs | 62 +++++++++++++++++-- 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/src/meta-srv/Cargo.toml b/src/meta-srv/Cargo.toml index 15830a4f05..0e0f3e5cde 100644 --- a/src/meta-srv/Cargo.toml +++ b/src/meta-srv/Cargo.toml @@ -83,6 +83,7 @@ chrono.workspace = true client = { workspace = true, features = ["testing"] } common-meta = { workspace = true, features = ["testing"] } common-procedure-test.workspace = true +common-wal = { workspace = true, features = ["testing"] } session.workspace = true tracing = "0.1" tracing-subscriber.workspace = true diff --git a/src/meta-srv/src/procedure/region_migration/manager.rs b/src/meta-srv/src/procedure/region_migration/manager.rs index e2345559d0..417881f549 100644 --- a/src/meta-srv/src/procedure/region_migration/manager.rs +++ b/src/meta-srv/src/procedure/region_migration/manager.rs @@ -268,13 +268,35 @@ impl RegionMigrationManager { ensure!( leader_peer.id == task.from_peer.id, error::InvalidArgumentsSnafu { - err_msg: "Invalid region migration `from_peer` argument" + err_msg: format!( + "Region's leader peer({}) is not the `from_peer`({}), region: {}", + leader_peer.id, task.from_peer.id, task.region_id + ), } ); Ok(()) } + /// Throws an error if `to_peer` is already has a region follower. + fn verify_region_follower_peers( + &self, + region_route: &RegionRoute, + task: &RegionMigrationProcedureTask, + ) -> Result<()> { + ensure!( + !region_route.follower_peers.contains(&task.to_peer), + error::InvalidArgumentsSnafu { + err_msg: format!( + "The `to_peer`({}) is already has a region follower, region: {}", + task.to_peer.id, task.region_id + ), + }, + ); + + Ok(()) + } + /// Submits a new region migration procedure. pub async fn submit_procedure( &self, @@ -308,7 +330,7 @@ impl RegionMigrationManager { } self.verify_region_leader_peer(®ion_route, &task)?; - + self.verify_region_follower_peers(®ion_route, &task)?; let table_info = self.retrieve_table_info(region_id).await?; let TableName { catalog_name, @@ -486,9 +508,39 @@ mod test { let err = manager.submit_procedure(task).await.unwrap_err(); assert_matches!(err, error::Error::InvalidArguments { .. }); - assert!(err - .to_string() - .contains("Invalid region migration `from_peer` argument")); + assert_eq!(err.to_string(), "Invalid arguments: Region's leader peer(3) is not the `from_peer`(1), region: 4398046511105(1024, 1)"); + } + + #[tokio::test] + async fn test_submit_procedure_region_follower_on_to_peer() { + let env = TestingEnv::new(); + let context_factory = env.context_factory(); + let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory); + let region_id = RegionId::new(1024, 1); + let task = RegionMigrationProcedureTask { + region_id, + from_peer: Peer::empty(3), + to_peer: Peer::empty(2), + timeout: Duration::from_millis(1000), + }; + + let table_info = new_test_table_info(1024, vec![1]).into(); + let region_routes = vec![RegionRoute { + region: Region::new_test(region_id), + leader_peer: Some(Peer::empty(3)), + follower_peers: vec![Peer::empty(2)], + ..Default::default() + }]; + + env.create_physical_table_metadata(table_info, region_routes) + .await; + + let err = manager.submit_procedure(task).await.unwrap_err(); + assert_matches!(err, error::Error::InvalidArguments { .. }); + assert_eq!( + err.to_string(), + "Invalid arguments: The `to_peer`(2) is already has a region follower, region: 4398046511105(1024, 1)" + ); } #[tokio::test] From 41814bb49f4e568dc6ac1acb2a00a8a967dca033 Mon Sep 17 00:00:00 2001 From: Yuhan Wang Date: Fri, 18 Apr 2025 20:10:47 +0800 Subject: [PATCH 42/82] feat: introduce `high_watermark` for remote wal logstore (#5877) * feat: introduce high_watermark_since_flush * test: add unit test for high watermark * refactor: submit a request instead * fix: send reply before submit request * fix: no need to update twice * feat: update high watermark in background periodically * test: update unit tests * fix: update high watermark periodically * test: update unit tests * chore: apply review comments * chore: rename * chore: apply review comments * chore: clean up * chore: apply review comments --- Cargo.lock | 1 + src/common/meta/src/region_registry.rs | 1 - src/log-store/Cargo.toml | 1 + src/log-store/src/kafka.rs | 3 + src/log-store/src/kafka/client_manager.rs | 66 ++------ .../src/kafka/high_watermark_manager.rs | 131 +++++++++++++++ src/log-store/src/kafka/log_store.rs | 61 ++++++- src/log-store/src/kafka/producer.rs | 32 ++-- src/log-store/src/kafka/test_util.rs | 88 ++++++++++ src/log-store/src/kafka/worker.rs | 8 + .../src/kafka/worker/update_high_watermark.rs | 59 +++++++ src/log-store/src/raft_engine/log_store.rs | 12 ++ src/metric-engine/src/utils.rs | 2 + src/mito2/src/engine/flush_test.rs | 70 +++++++- src/mito2/src/region.rs | 16 +- src/mito2/src/region/opener.rs | 2 + src/mito2/src/wal/raw_entry_reader.rs | 4 + src/mito2/src/worker/handle_flush.rs | 150 +++++++++++------- src/store-api/src/logstore.rs | 3 + src/store-api/src/region_engine.rs | 7 + 20 files changed, 589 insertions(+), 128 deletions(-) create mode 100644 src/log-store/src/kafka/high_watermark_manager.rs create mode 100644 src/log-store/src/kafka/test_util.rs create mode 100644 src/log-store/src/kafka/worker/update_high_watermark.rs diff --git a/Cargo.lock b/Cargo.lock index 984978a1c4..520b33c929 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6416,6 +6416,7 @@ dependencies = [ "common-test-util", "common-time", "common-wal", + "dashmap", "delta-encoding", "derive_builder 0.20.1", "futures", diff --git a/src/common/meta/src/region_registry.rs b/src/common/meta/src/region_registry.rs index 76fb271f52..344e0fef5b 100644 --- a/src/common/meta/src/region_registry.rs +++ b/src/common/meta/src/region_registry.rs @@ -97,7 +97,6 @@ impl LeaderRegionManifestInfo { } /// Returns the minimum flushed entry id of the leader region. - /// It is used to determine the minimum flushed entry id that can be pruned in remote wal. pub fn min_flushed_entry_id(&self) -> u64 { match self { LeaderRegionManifestInfo::Mito { diff --git a/src/log-store/Cargo.toml b/src/log-store/Cargo.toml index 31571afe3e..354dee6102 100644 --- a/src/log-store/Cargo.toml +++ b/src/log-store/Cargo.toml @@ -26,6 +26,7 @@ common-runtime.workspace = true common-telemetry.workspace = true common-time.workspace = true common-wal.workspace = true +dashmap.workspace = true delta-encoding = "0.4" derive_builder.workspace = true futures.workspace = true diff --git a/src/log-store/src/kafka.rs b/src/log-store/src/kafka.rs index 1d765cf90b..452c88164a 100644 --- a/src/log-store/src/kafka.rs +++ b/src/log-store/src/kafka.rs @@ -14,9 +14,12 @@ pub(crate) mod client_manager; pub(crate) mod consumer; +mod high_watermark_manager; pub(crate) mod index; pub mod log_store; pub(crate) mod producer; +#[cfg(test)] +pub(crate) mod test_util; pub(crate) mod util; pub(crate) mod worker; diff --git a/src/log-store/src/kafka/client_manager.rs b/src/log-store/src/kafka/client_manager.rs index 9317cc938b..65599cb09e 100644 --- a/src/log-store/src/kafka/client_manager.rs +++ b/src/log-store/src/kafka/client_manager.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use common_wal::config::kafka::common::DEFAULT_BACKOFF_CONFIG; use common_wal::config::kafka::DatanodeKafkaConfig; +use dashmap::DashMap; use rskafka::client::partition::{Compression, PartitionClient, UnknownTopicHandling}; use rskafka::client::ClientBuilder; use snafu::ResultExt; @@ -64,6 +65,9 @@ pub(crate) struct ClientManager { flush_batch_size: usize, compression: Compression, + + /// High watermark for each topic. + high_watermark: Arc, u64>>, } impl ClientManager { @@ -71,6 +75,7 @@ impl ClientManager { pub(crate) async fn try_new( config: &DatanodeKafkaConfig, global_index_collector: Option, + high_watermark: Arc, u64>>, ) -> Result { // Sets backoff config for the top-level kafka client and all clients constructed by it. let broker_endpoints = common_wal::resolve_to_ipv4(&config.connection.broker_endpoints) @@ -96,6 +101,7 @@ impl ClientManager { flush_batch_size: config.max_batch_bytes.as_bytes() as usize, compression: Compression::Lz4, global_index_collector, + high_watermark, }) } @@ -111,6 +117,7 @@ impl ClientManager { .write() .await .insert(provider.clone(), client.clone()); + self.high_watermark.insert(provider.clone(), 0); Ok(client) } } @@ -159,6 +166,7 @@ impl ClientManager { self.compression, self.flush_batch_size, index_collector, + self.high_watermark.clone(), )); Ok(Client { client, producer }) @@ -167,66 +175,20 @@ impl ClientManager { pub(crate) fn global_index_collector(&self) -> Option<&GlobalIndexCollector> { self.global_index_collector.as_ref() } + + #[cfg(test)] + pub(crate) fn high_watermark(&self) -> &Arc, u64>> { + &self.high_watermark + } } #[cfg(test)] mod tests { - use common_wal::config::kafka::common::KafkaConnectionConfig; use common_wal::test_util::run_test_with_kafka_wal; use tokio::sync::Barrier; use super::*; - - /// Creates `num_topics` number of topics each will be decorated by the given decorator. - pub async fn create_topics( - num_topics: usize, - decorator: F, - broker_endpoints: &[String], - ) -> Vec - where - F: Fn(usize) -> String, - { - assert!(!broker_endpoints.is_empty()); - let client = ClientBuilder::new(broker_endpoints.to_vec()) - .build() - .await - .unwrap(); - let ctrl_client = client.controller_client().unwrap(); - let (topics, tasks): (Vec<_>, Vec<_>) = (0..num_topics) - .map(|i| { - let topic = decorator(i); - let task = ctrl_client.create_topic(topic.clone(), 1, 1, 500); - (topic, task) - }) - .unzip(); - futures::future::try_join_all(tasks).await.unwrap(); - topics - } - - /// Prepares for a test in that a collection of topics and a client manager are created. - async fn prepare( - test_name: &str, - num_topics: usize, - broker_endpoints: Vec, - ) -> (ClientManager, Vec) { - let topics = create_topics( - num_topics, - |i| format!("{test_name}_{}_{}", i, uuid::Uuid::new_v4()), - &broker_endpoints, - ) - .await; - - let config = DatanodeKafkaConfig { - connection: KafkaConnectionConfig { - broker_endpoints, - ..Default::default() - }, - ..Default::default() - }; - let manager = ClientManager::try_new(&config, None).await.unwrap(); - - (manager, topics) - } + use crate::kafka::test_util::prepare; /// Sends `get_or_insert` requests sequentially to the client manager, and checks if it could handle them correctly. #[tokio::test] diff --git a/src/log-store/src/kafka/high_watermark_manager.rs b/src/log-store/src/kafka/high_watermark_manager.rs new file mode 100644 index 0000000000..8a4c2a1252 --- /dev/null +++ b/src/log-store/src/kafka/high_watermark_manager.rs @@ -0,0 +1,131 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; +use std::time::Duration; + +use common_telemetry::error; +use dashmap::DashMap; +use store_api::logstore::provider::KafkaProvider; +use tokio::time::{interval, MissedTickBehavior}; + +use crate::error::Result; +use crate::kafka::client_manager::ClientManagerRef; + +/// HighWatermarkManager is responsible for periodically updating the high watermark +/// (latest existing record offset) for each Kafka topic. +pub(crate) struct HighWatermarkManager { + /// Interval to update high watermark. + update_interval: Duration, + /// The high watermark for each topic. + high_watermark: Arc, u64>>, + /// Client manager to send requests. + client_manager: ClientManagerRef, +} + +impl HighWatermarkManager { + pub(crate) fn new( + update_interval: Duration, + high_watermark: Arc, u64>>, + client_manager: ClientManagerRef, + ) -> Self { + Self { + update_interval, + high_watermark, + client_manager, + } + } + + /// Starts the high watermark manager as a background task + /// + /// This spawns a task that periodically queries Kafka for the latest + /// high watermark values for all registered topics and updates the shared map. + pub(crate) async fn run(self) { + common_runtime::spawn_global(async move { + let mut interval = interval(self.update_interval); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + interval.tick().await; + if let Err(e) = self.try_update().await { + error!(e; "Failed to update high watermark"); + } + } + }); + } + + /// Attempts to update the high watermark for all registered topics + /// + /// Iterates through all topics in the high watermark map, obtains a producer + /// for each topic, and requests an update of the high watermark value. + pub(crate) async fn try_update(&self) -> Result<()> { + for iterator_element in self.high_watermark.iter() { + let producer = self + .client_manager + .get_or_insert(iterator_element.key()) + .await? + .producer() + .clone(); + producer.update_high_watermark().await?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use common_wal::test_util::run_test_with_kafka_wal; + use store_api::storage::RegionId; + + use super::*; + use crate::kafka::test_util::{prepare, record}; + + #[tokio::test] + async fn test_try_update_high_watermark() { + run_test_with_kafka_wal(|broker_endpoints| { + Box::pin(async { + let (manager, topics) = + prepare("test_try_update_high_watermark", 1, broker_endpoints).await; + let manager = Arc::new(manager); + let high_watermark_manager = HighWatermarkManager::new( + Duration::from_millis(100), + manager.high_watermark().clone(), + manager.clone(), + ); + let high_watermark = high_watermark_manager.high_watermark.clone(); + high_watermark_manager.run().await; + + let topic = topics[0].clone(); + let provider = Arc::new(KafkaProvider::new(topic.to_string())); + let producer = manager + .get_or_insert(&provider) + .await + .unwrap() + .producer() + .clone(); + + tokio::time::sleep(Duration::from_millis(150)).await; + let current_high_watermark = *high_watermark.get(&provider).unwrap(); + assert_eq!(current_high_watermark, 0); + + let record = vec![record()]; + let region = RegionId::new(1, 1); + producer.produce(region, record.clone()).await.unwrap(); + tokio::time::sleep(Duration::from_millis(150)).await; + let current_high_watermark = *high_watermark.get(&provider).unwrap(); + assert_eq!(current_high_watermark, record.len() as u64); + }) + }) + .await + } +} diff --git a/src/log-store/src/kafka/log_store.rs b/src/log-store/src/kafka/log_store.rs index 3cc728a998..66ad613bbc 100644 --- a/src/log-store/src/kafka/log_store.rs +++ b/src/log-store/src/kafka/log_store.rs @@ -18,6 +18,7 @@ use std::time::Duration; use common_telemetry::{debug, warn}; use common_wal::config::kafka::DatanodeKafkaConfig; +use dashmap::DashMap; use futures::future::try_join_all; use futures_util::StreamExt; use rskafka::client::partition::OffsetAt; @@ -32,6 +33,7 @@ use store_api::storage::RegionId; use crate::error::{self, ConsumeRecordSnafu, Error, GetOffsetSnafu, InvalidProviderSnafu, Result}; use crate::kafka::client_manager::{ClientManager, ClientManagerRef}; use crate::kafka::consumer::{ConsumerBuilder, RecordsBuffer}; +use crate::kafka::high_watermark_manager::HighWatermarkManager; use crate::kafka::index::{ build_region_wal_index_iterator, GlobalIndexCollector, MIN_BATCH_WINDOW_SIZE, }; @@ -41,6 +43,8 @@ use crate::kafka::util::record::{ }; use crate::metrics; +const DEFAULT_HIGH_WATERMARK_UPDATE_INTERVAL: Duration = Duration::from_secs(60); + /// A log store backed by Kafka. #[derive(Debug)] pub struct KafkaLogStore { @@ -52,6 +56,18 @@ pub struct KafkaLogStore { consumer_wait_timeout: Duration, /// Ignore missing entries during read WAL. overwrite_entry_start_id: bool, + /// High watermark for all topics. + /// + /// Represents the offset of the last record in each topic. This is used to track + /// the latest available data in Kafka topics. + /// + /// The high watermark is updated in two ways: + /// - Automatically when the producer successfully commits data to Kafka + /// - Periodically by the [HighWatermarkManager](crate::kafka::high_watermark_manager::HighWatermarkManager). + /// + /// This shared map allows multiple components to access the latest high watermark + /// information without needing to query Kafka directly. + high_watermark: Arc, u64>>, } impl KafkaLogStore { @@ -60,14 +76,23 @@ impl KafkaLogStore { config: &DatanodeKafkaConfig, global_index_collector: Option, ) -> Result { - let client_manager = - Arc::new(ClientManager::try_new(config, global_index_collector).await?); + let high_watermark = Arc::new(DashMap::new()); + let client_manager = Arc::new( + ClientManager::try_new(config, global_index_collector, high_watermark.clone()).await?, + ); + let high_watermark_manager = HighWatermarkManager::new( + DEFAULT_HIGH_WATERMARK_UPDATE_INTERVAL, + high_watermark.clone(), + client_manager.clone(), + ); + high_watermark_manager.run().await; Ok(Self { client_manager, max_batch_bytes: config.max_batch_bytes.as_bytes() as usize, consumer_wait_timeout: config.consumer_wait_timeout, overwrite_entry_start_id: config.overwrite_entry_start_id, + high_watermark, }) } } @@ -158,6 +183,7 @@ impl LogStore for KafkaLogStore { .collect::>(); let mut region_grouped_records: HashMap)> = HashMap::with_capacity(region_ids.len()); + let mut region_to_provider = HashMap::with_capacity(region_ids.len()); for entry in entries { let provider = entry.provider().as_kafka_provider().with_context(|| { error::InvalidProviderSnafu { @@ -165,6 +191,7 @@ impl LogStore for KafkaLogStore { actual: entry.provider().type_name(), } })?; + region_to_provider.insert(entry.region_id(), provider.clone()); let region_id = entry.region_id(); match region_grouped_records.entry(region_id) { std::collections::hash_map::Entry::Occupied(mut slot) => { @@ -199,6 +226,13 @@ impl LogStore for KafkaLogStore { )) .await?; + // Updates the high watermark offset of the last record in the topic. + for (region_id, offset) in ®ion_grouped_max_offset { + // Safety: `region_id` is always valid. + let provider = region_to_provider.get(region_id).unwrap(); + self.high_watermark.insert(provider.clone(), *offset); + } + Ok(AppendBatchResponse { last_entry_ids: region_grouped_max_offset.into_iter().collect(), }) @@ -383,6 +417,25 @@ impl LogStore for KafkaLogStore { Ok(()) } + /// Returns the highest entry id of the specified topic in remote WAL. + fn high_watermark(&self, provider: &Provider) -> Result { + let provider = provider + .as_kafka_provider() + .with_context(|| InvalidProviderSnafu { + expected: KafkaProvider::type_name(), + actual: provider.type_name(), + })?; + + let high_watermark = self + .high_watermark + .get(provider) + .as_deref() + .copied() + .unwrap_or(0); + + Ok(high_watermark) + } + /// Stops components of the logstore. async fn stop(&self) -> Result<()> { Ok(()) @@ -567,6 +620,8 @@ mod tests { .for_each(|entry| entry.set_entry_id(0)); assert_eq!(expected_entries, actual_entries); } + let high_wathermark = logstore.high_watermark(&provider).unwrap(); + assert_eq!(high_wathermark, 99); } #[tokio::test] @@ -640,5 +695,7 @@ mod tests { .for_each(|entry| entry.set_entry_id(0)); assert_eq!(expected_entries, actual_entries); } + let high_wathermark = logstore.high_watermark(&provider).unwrap(); + assert_eq!(high_wathermark, (data_size_kb as u64 / 8 + 1) * 20 * 5 - 1); } } diff --git a/src/log-store/src/kafka/producer.rs b/src/log-store/src/kafka/producer.rs index 910465ba61..6f89c75e7a 100644 --- a/src/log-store/src/kafka/producer.rs +++ b/src/log-store/src/kafka/producer.rs @@ -15,6 +15,7 @@ use std::sync::Arc; use common_telemetry::warn; +use dashmap::DashMap; use rskafka::client::partition::{Compression, OffsetAt, PartitionClient}; use rskafka::record::Record; use store_api::logstore::provider::KafkaProvider; @@ -56,6 +57,7 @@ impl OrderedBatchProducer { compression: Compression, max_batch_bytes: usize, index_collector: Box, + high_watermark: Arc, u64>>, ) -> Self { let mut worker = BackgroundProducerWorker { provider, @@ -65,6 +67,7 @@ impl OrderedBatchProducer { request_batch_size: REQUEST_BATCH_SIZE, max_batch_bytes, index_collector, + high_watermark, }; tokio::spawn(async move { worker.run().await }); Self { sender: tx } @@ -90,6 +93,21 @@ impl OrderedBatchProducer { Ok(handle) } + + /// Sends an [WorkerRequest::UpdateHighWatermark] request to the producer. + /// This is used to update the high watermark for the topic. + pub(crate) async fn update_high_watermark(&self) -> Result<()> { + if self + .sender + .send(WorkerRequest::UpdateHighWatermark) + .await + .is_err() + { + warn!("OrderedBatchProducer is already exited"); + return error::OrderedBatchProducerStoppedSnafu {}.fail(); + } + Ok(()) + } } #[async_trait::async_trait] @@ -135,7 +153,6 @@ mod tests { use std::sync::{Arc, Mutex}; use std::time::Duration; - use chrono::{TimeZone, Utc}; use common_base::readable_size::ReadableSize; use common_telemetry::debug; use futures::stream::FuturesUnordered; @@ -149,6 +166,7 @@ mod tests { use super::*; use crate::kafka::index::NoopCollector; use crate::kafka::producer::OrderedBatchProducer; + use crate::kafka::test_util::record; #[derive(Debug)] struct MockClient { @@ -196,15 +214,6 @@ mod tests { } } - fn record() -> Record { - Record { - key: Some(vec![0; 4]), - value: Some(vec![0; 6]), - headers: Default::default(), - timestamp: Utc.timestamp_millis_opt(320).unwrap(), - } - } - #[tokio::test] async fn test_producer() { common_telemetry::init_default_ut_logging(); @@ -224,6 +233,7 @@ mod tests { Compression::NoCompression, ReadableSize((record.approximate_size() * 2) as u64).as_bytes() as usize, Box::new(NoopCollector), + Arc::new(DashMap::new()), ); let region_id = RegionId::new(1, 1); @@ -272,6 +282,7 @@ mod tests { Compression::NoCompression, ReadableSize((record.approximate_size() * 2) as u64).as_bytes() as usize, Box::new(NoopCollector), + Arc::new(DashMap::new()), ); let region_id = RegionId::new(1, 1); @@ -324,6 +335,7 @@ mod tests { Compression::NoCompression, ReadableSize((record.approximate_size() * 2) as u64).as_bytes() as usize, Box::new(NoopCollector), + Arc::new(DashMap::new()), ); let region_id = RegionId::new(1, 1); diff --git a/src/log-store/src/kafka/test_util.rs b/src/log-store/src/kafka/test_util.rs new file mode 100644 index 0000000000..c83e0ef00e --- /dev/null +++ b/src/log-store/src/kafka/test_util.rs @@ -0,0 +1,88 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use chrono::{TimeZone, Utc}; +use common_wal::config::kafka::common::KafkaConnectionConfig; +use common_wal::config::kafka::DatanodeKafkaConfig; +use dashmap::DashMap; +use rskafka::client::ClientBuilder; +use rskafka::record::Record; + +use crate::kafka::client_manager::ClientManager; + +/// Creates `num_topics` number of topics each will be decorated by the given decorator. +pub(crate) async fn create_topics( + num_topics: usize, + decorator: F, + broker_endpoints: &[String], +) -> Vec +where + F: Fn(usize) -> String, +{ + assert!(!broker_endpoints.is_empty()); + let client = ClientBuilder::new(broker_endpoints.to_vec()) + .build() + .await + .unwrap(); + let ctrl_client = client.controller_client().unwrap(); + let (topics, tasks): (Vec<_>, Vec<_>) = (0..num_topics) + .map(|i| { + let topic = decorator(i); + let task = ctrl_client.create_topic(topic.clone(), 1, 1, 500); + (topic, task) + }) + .unzip(); + futures::future::try_join_all(tasks).await.unwrap(); + topics +} + +/// Prepares for a test in that a collection of topics and a client manager are created. +pub(crate) async fn prepare( + test_name: &str, + num_topics: usize, + broker_endpoints: Vec, +) -> (ClientManager, Vec) { + let topics = create_topics( + num_topics, + |i| format!("{test_name}_{}_{}", i, uuid::Uuid::new_v4()), + &broker_endpoints, + ) + .await; + + let config = DatanodeKafkaConfig { + connection: KafkaConnectionConfig { + broker_endpoints, + ..Default::default() + }, + ..Default::default() + }; + let high_watermark = Arc::new(DashMap::new()); + let manager = ClientManager::try_new(&config, None, high_watermark) + .await + .unwrap(); + + (manager, topics) +} + +/// Generate a record to produce. +pub fn record() -> Record { + Record { + key: Some(vec![0; 4]), + value: Some(vec![0; 6]), + headers: Default::default(), + timestamp: Utc.timestamp_millis_opt(320).unwrap(), + } +} diff --git a/src/log-store/src/kafka/worker.rs b/src/log-store/src/kafka/worker.rs index b05351d172..372d8c5567 100644 --- a/src/log-store/src/kafka/worker.rs +++ b/src/log-store/src/kafka/worker.rs @@ -15,10 +15,12 @@ pub(crate) mod dump_index; pub(crate) mod flush; pub(crate) mod produce; +pub(crate) mod update_high_watermark; use std::sync::Arc; use common_telemetry::debug; +use dashmap::DashMap; use futures::future::try_join_all; use rskafka::client::partition::Compression; use rskafka::record::Record; @@ -37,6 +39,7 @@ pub(crate) enum WorkerRequest { Produce(ProduceRequest), TruncateIndex(TruncateIndexRequest), DumpIndex(DumpIndexRequest), + UpdateHighWatermark, } impl WorkerRequest { @@ -157,6 +160,8 @@ pub(crate) struct BackgroundProducerWorker { pub(crate) max_batch_bytes: usize, /// Collecting ids of WAL entries. pub(crate) index_collector: Box, + /// High watermark for all topics. + pub(crate) high_watermark: Arc, u64>>, } impl BackgroundProducerWorker { @@ -194,6 +199,9 @@ impl BackgroundProducerWorker { entry_id, }) => self.index_collector.truncate(region_id, entry_id), WorkerRequest::DumpIndex(req) => self.dump_index(req).await, + WorkerRequest::UpdateHighWatermark => { + self.update_high_watermark().await; + } } } diff --git a/src/log-store/src/kafka/worker/update_high_watermark.rs b/src/log-store/src/kafka/worker/update_high_watermark.rs new file mode 100644 index 0000000000..8404086418 --- /dev/null +++ b/src/log-store/src/kafka/worker/update_high_watermark.rs @@ -0,0 +1,59 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use common_telemetry::{debug, error}; +use rskafka::client::partition::OffsetAt; +use snafu::ResultExt; + +use crate::error; +use crate::kafka::worker::BackgroundProducerWorker; + +impl BackgroundProducerWorker { + /// Updates the high watermark for the topic. + /// + /// This function retrieves the latest offset from Kafka and updates the high watermark + /// in the shared map. + pub async fn update_high_watermark(&mut self) { + match self + .client + .get_offset(OffsetAt::Latest) + .await + .context(error::GetOffsetSnafu { + topic: &self.provider.topic, + }) { + Ok(offset) => match self.high_watermark.entry(self.provider.clone()) { + dashmap::Entry::Occupied(mut occupied_entry) => { + let offset = offset as u64; + if *occupied_entry.get() != offset { + occupied_entry.insert(offset); + debug!( + "Updated high watermark for topic {} to {}", + self.provider.topic, offset + ); + } + } + dashmap::Entry::Vacant(vacant_entry) => { + vacant_entry.insert(offset as u64); + debug!( + "Inserted high watermark for topic {} to {}", + self.provider.topic, offset + ); + } + }, + Err(err) => { + error!(err; "Failed to get offset for topic {}", self.provider.topic); + } + } + } +} diff --git a/src/log-store/src/raft_engine/log_store.rs b/src/log-store/src/raft_engine/log_store.rs index c7df8be66c..a060c9a976 100644 --- a/src/log-store/src/raft_engine/log_store.rs +++ b/src/log-store/src/raft_engine/log_store.rs @@ -483,6 +483,18 @@ impl LogStore for RaftEngineLogStore { ); Ok(()) } + + fn high_watermark(&self, provider: &Provider) -> Result { + let ns = provider + .as_raft_engine_provider() + .with_context(|| InvalidProviderSnafu { + expected: RaftEngineProvider::type_name(), + actual: provider.type_name(), + })?; + let namespace_id = ns.id; + let last_index = self.engine.last_index(namespace_id).unwrap_or(0); + Ok(last_index) + } } #[derive(Debug, Clone)] diff --git a/src/metric-engine/src/utils.rs b/src/metric-engine/src/utils.rs index ad0695f54a..0f28a6365e 100644 --- a/src/metric-engine/src/utils.rs +++ b/src/metric-engine/src/utils.rs @@ -61,6 +61,8 @@ pub fn get_region_statistic(mito: &MitoEngine, region_id: RegionId) -> Option { warn!( diff --git a/src/mito2/src/engine/flush_test.rs b/src/mito2/src/engine/flush_test.rs index 25bba7e085..1d836da733 100644 --- a/src/mito2/src/engine/flush_test.rs +++ b/src/mito2/src/engine/flush_test.rs @@ -25,7 +25,7 @@ use common_wal::options::WAL_OPTIONS_KEY; use rstest::rstest; use rstest_reuse::{self, apply}; use store_api::region_engine::RegionEngine; -use store_api::region_request::RegionRequest; +use store_api::region_request::{RegionFlushRequest, RegionRequest}; use store_api::storage::{RegionId, ScanRequest}; use crate::config::MitoConfig; @@ -33,8 +33,8 @@ use crate::engine::listener::{FlushListener, StallListener}; use crate::test_util::{ build_rows, build_rows_for_key, flush_region, kafka_log_store_factory, multiple_log_store_factories, prepare_test_for_kafka_log_store, put_rows, - raft_engine_log_store_factory, reopen_region, rows_schema, CreateRequestBuilder, - LogStoreFactory, MockWriteBufferManager, TestEnv, + raft_engine_log_store_factory, reopen_region, rows_schema, single_kafka_log_store_factory, + CreateRequestBuilder, LogStoreFactory, MockWriteBufferManager, TestEnv, }; use crate::time_provider::TimeProvider; use crate::worker::MAX_INITIAL_CHECK_DELAY_SECS; @@ -544,3 +544,67 @@ async fn test_flush_workers() { +-------+---------+---------------------+"; assert_eq!(expected, batches.pretty_print().unwrap()); } + +#[apply(single_kafka_log_store_factory)] +async fn test_update_topic_latest_entry_id(factory: Option) { + common_telemetry::init_default_ut_logging(); + let Some(factory) = factory else { + return; + }; + let write_buffer_manager = Arc::new(MockWriteBufferManager::default()); + let listener = Arc::new(FlushListener::default()); + + let mut env = TestEnv::new().with_log_store_factory(factory.clone()); + let engine = env + .create_engine_with( + MitoConfig::default(), + Some(write_buffer_manager.clone()), + Some(listener.clone()), + ) + .await; + let region_id = RegionId::new(1, 1); + env.get_schema_metadata_manager() + .register_region_table_info( + region_id.table_id(), + "test_table", + "test_catalog", + "test_schema", + None, + env.get_kv_backend(), + ) + .await; + + let topic = prepare_test_for_kafka_log_store(&factory).await; + let request = CreateRequestBuilder::new() + .kafka_topic(topic.clone()) + .build(); + let column_schemas = rows_schema(&request); + engine + .handle_request(region_id, RegionRequest::Create(request.clone())) + .await + .unwrap(); + + let region = engine.get_region(region_id).unwrap(); + assert_eq!(region.topic_latest_entry_id.load(Ordering::Relaxed), 0); + + let rows = Rows { + schema: column_schemas.clone(), + rows: build_rows_for_key("a", 0, 2, 0), + }; + put_rows(&engine, region_id, rows.clone()).await; + + let request = RegionFlushRequest::default(); + engine + .handle_request(region_id, RegionRequest::Flush(request.clone())) + .await + .unwrap(); + // Wait until flush is finished. + listener.wait().await; + assert_eq!(region.topic_latest_entry_id.load(Ordering::Relaxed), 0); + + engine + .handle_request(region_id, RegionRequest::Flush(request.clone())) + .await + .unwrap(); + assert_eq!(region.topic_latest_entry_id.load(Ordering::Relaxed), 1); +} diff --git a/src/mito2/src/region.rs b/src/mito2/src/region.rs index 191300a076..4ad3ea8e2d 100644 --- a/src/mito2/src/region.rs +++ b/src/mito2/src/region.rs @@ -119,6 +119,16 @@ pub(crate) struct MitoRegion { last_compaction_millis: AtomicI64, /// Provider to get current time. time_provider: TimeProviderRef, + /// The topic's latest entry id since the region's last flushing. + /// **Only used for remote WAL pruning.** + /// + /// The value will be updated to the high watermark of the topic + /// if region receives a flush request or schedules a periodic flush task + /// and the region's memtable is empty. + /// + /// There are no WAL entries in range [flushed_entry_id, topic_latest_entry_id] for current region, + /// which means these WAL entries maybe able to be pruned up to `topic_latest_entry_id`. + pub(crate) topic_latest_entry_id: AtomicU64, /// Memtable builder for the region. pub(crate) memtable_builder: MemtableBuilderRef, /// manifest stats @@ -287,12 +297,14 @@ impl MitoRegion { let sst_usage = version.ssts.sst_usage(); let index_usage = version.ssts.index_usage(); + let flushed_entry_id = version.flushed_entry_id; let wal_usage = self.estimated_wal_usage(memtable_usage); let manifest_usage = self.stats.total_manifest_size(); let num_rows = version.ssts.num_rows() + version.memtables.num_rows(); let manifest_version = self.stats.manifest_version(); - let flushed_entry_id = version.flushed_entry_id; + + let topic_latest_entry_id = self.topic_latest_entry_id.load(Ordering::Relaxed); RegionStatistic { num_rows, @@ -305,6 +317,8 @@ impl MitoRegion { manifest_version, flushed_entry_id, }, + data_topic_latest_entry_id: topic_latest_entry_id, + metadata_topic_latest_entry_id: topic_latest_entry_id, } } diff --git a/src/mito2/src/region/opener.rs b/src/mito2/src/region/opener.rs index b7c4803b54..5a1bf86c18 100644 --- a/src/mito2/src/region/opener.rs +++ b/src/mito2/src/region/opener.rs @@ -274,6 +274,7 @@ impl RegionOpener { last_flush_millis: AtomicI64::new(now), last_compaction_millis: AtomicI64::new(now), time_provider: self.time_provider.clone(), + topic_latest_entry_id: AtomicU64::new(0), memtable_builder, stats: self.stats, }) @@ -452,6 +453,7 @@ impl RegionOpener { last_flush_millis: AtomicI64::new(now), last_compaction_millis: AtomicI64::new(now), time_provider: self.time_provider.clone(), + topic_latest_entry_id: AtomicU64::new(0), memtable_builder, stats: self.stats.clone(), }; diff --git a/src/mito2/src/wal/raw_entry_reader.rs b/src/mito2/src/wal/raw_entry_reader.rs index 85a0c945b9..9a86c9ebfc 100644 --- a/src/mito2/src/wal/raw_entry_reader.rs +++ b/src/mito2/src/wal/raw_entry_reader.rs @@ -196,6 +196,10 @@ mod tests { ) -> Result { unreachable!() } + + fn high_watermark(&self, _provider: &Provider) -> Result { + unreachable!() + } } #[tokio::test] diff --git a/src/mito2/src/worker/handle_flush.rs b/src/mito2/src/worker/handle_flush.rs index e846809a5d..dc5a58d794 100644 --- a/src/mito2/src/worker/handle_flush.rs +++ b/src/mito2/src/worker/handle_flush.rs @@ -14,6 +14,7 @@ //! Handling flush related requests. +use std::sync::atomic::Ordering; use std::sync::Arc; use common_telemetry::{error, info}; @@ -29,34 +30,6 @@ use crate::request::{FlushFailed, FlushFinished, OnFailure, OptionOutputTx}; use crate::worker::RegionWorkerLoop; impl RegionWorkerLoop { - /// Handles manual flush request. - pub(crate) async fn handle_flush_request( - &mut self, - region_id: RegionId, - request: RegionFlushRequest, - mut sender: OptionOutputTx, - ) { - let Some(region) = self.regions.flushable_region_or(region_id, &mut sender) else { - return; - }; - - let reason = if region.is_downgrading() { - FlushReason::Downgrading - } else { - FlushReason::Manual - }; - - let mut task = - self.new_flush_task(®ion, reason, request.row_group_size, self.config.clone()); - task.push_sender(sender); - if let Err(e) = - self.flush_scheduler - .schedule_flush(region.region_id, ®ion.version_control, task) - { - error!(e; "Failed to schedule flush task for region {}", region.region_id); - } - } - /// On region flush job failed. pub(crate) async fn handle_flush_failed(&mut self, region_id: RegionId, request: FlushFailed) { self.flush_scheduler.on_flush_failed(region_id, request.err); @@ -129,37 +102,6 @@ impl RegionWorkerLoop { Ok(()) } - /// Flushes regions periodically. - pub(crate) fn flush_periodically(&mut self) -> Result<()> { - let regions = self.regions.list_regions(); - let now = self.time_provider.current_time_millis(); - let min_last_flush_time = now - self.config.auto_flush_interval.as_millis() as i64; - - for region in ®ions { - if self.flush_scheduler.is_flush_requested(region.region_id) || !region.is_writable() { - // Already flushing or not writable. - continue; - } - - if region.last_flush_millis() < min_last_flush_time { - // If flush time of this region is earlier than `min_last_flush_time`, we can flush this region. - let task = self.new_flush_task( - region, - FlushReason::Periodically, - None, - self.config.clone(), - ); - self.flush_scheduler.schedule_flush( - region.region_id, - ®ion.version_control, - task, - )?; - } - } - - Ok(()) - } - /// Creates a flush task with specific `reason` for the `region`. pub(crate) fn new_flush_task( &self, @@ -185,6 +127,75 @@ impl RegionWorkerLoop { } impl RegionWorkerLoop { + /// Handles manual flush request. + pub(crate) async fn handle_flush_request( + &mut self, + region_id: RegionId, + request: RegionFlushRequest, + mut sender: OptionOutputTx, + ) { + let Some(region) = self.regions.flushable_region_or(region_id, &mut sender) else { + return; + }; + // `update_topic_latest_entry_id` updates `topic_latest_entry_id` when memtables are empty. + // But the flush is skipped if memtables are empty. Thus should update the `topic_latest_entry_id` + // when handling flush request instead of in `schedule_flush` or `flush_finished`. + self.update_topic_latest_entry_id(®ion); + info!( + "Region {} flush request, high watermark: {}", + region_id, + region.topic_latest_entry_id.load(Ordering::Relaxed) + ); + + let reason = if region.is_downgrading() { + FlushReason::Downgrading + } else { + FlushReason::Manual + }; + + let mut task = + self.new_flush_task(®ion, reason, request.row_group_size, self.config.clone()); + task.push_sender(sender); + if let Err(e) = + self.flush_scheduler + .schedule_flush(region.region_id, ®ion.version_control, task) + { + error!(e; "Failed to schedule flush task for region {}", region.region_id); + } + } + + /// Flushes regions periodically. + pub(crate) fn flush_periodically(&mut self) -> Result<()> { + let regions = self.regions.list_regions(); + let now = self.time_provider.current_time_millis(); + let min_last_flush_time = now - self.config.auto_flush_interval.as_millis() as i64; + + for region in ®ions { + if self.flush_scheduler.is_flush_requested(region.region_id) || !region.is_writable() { + // Already flushing or not writable. + continue; + } + self.update_topic_latest_entry_id(region); + + if region.last_flush_millis() < min_last_flush_time { + // If flush time of this region is earlier than `min_last_flush_time`, we can flush this region. + let task = self.new_flush_task( + region, + FlushReason::Periodically, + None, + self.config.clone(), + ); + self.flush_scheduler.schedule_flush( + region.region_id, + ®ion.version_control, + task, + )?; + } + } + + Ok(()) + } + /// On region flush job finished. pub(crate) async fn handle_flush_finished( &mut self, @@ -247,4 +258,25 @@ impl RegionWorkerLoop { self.listener.on_flush_success(region_id); } + + /// Updates the latest entry id since flush of the region. + /// **This is only used for remote WAL pruning.** + pub(crate) fn update_topic_latest_entry_id(&mut self, region: &MitoRegionRef) { + if region.provider.is_remote_wal() && region.version().memtables.is_empty() { + let high_watermark = self + .wal + .store() + .high_watermark(®ion.provider) + .unwrap_or(0); + if high_watermark != 0 { + region + .topic_latest_entry_id + .store(high_watermark, Ordering::Relaxed); + } + info!( + "Region {} high watermark updated to {}", + region.region_id, high_watermark + ); + } + } } diff --git a/src/store-api/src/logstore.rs b/src/store-api/src/logstore.rs index 86a2263398..573fab469e 100644 --- a/src/store-api/src/logstore.rs +++ b/src/store-api/src/logstore.rs @@ -94,6 +94,9 @@ pub trait LogStore: Send + Sync + 'static + std::fmt::Debug { region_id: RegionId, provider: &Provider, ) -> Result; + + /// Returns the highest existing entry id in the log store. + fn high_watermark(&self, provider: &Provider) -> Result; } /// The response of an `append` operation. diff --git a/src/store-api/src/region_engine.rs b/src/store-api/src/region_engine.rs index a2db2d9f52..5f6069d961 100644 --- a/src/store-api/src/region_engine.rs +++ b/src/store-api/src/region_engine.rs @@ -451,6 +451,13 @@ pub struct RegionStatistic { /// The details of the region. #[serde(default)] pub manifest: RegionManifestInfo, + /// The latest entry id of the region's remote WAL since last flush. + /// For metric engine, there're two latest entry ids, one for data and one for metadata. + /// TODO(weny): remove this two fields and use single instead. + #[serde(default)] + pub data_topic_latest_entry_id: u64, + #[serde(default)] + pub metadata_topic_latest_entry_id: u64, } /// The manifest info of a region. From e817a65d755d04ab76b92d626a6b9bc1ef82db56 Mon Sep 17 00:00:00 2001 From: Yuhan Wang Date: Sat, 19 Apr 2025 00:02:33 +0800 Subject: [PATCH 43/82] feat: enable submitting wal prune procedure periodically (#5867) * feat: enable submitting wal prune procedure periodically * chore: fix and add options * test: add unit test * test: fix unit test * test: enable active_wal_pruning in test * test: update default config * chore: update config name * refactor: use semaphore to control the number of prune process * refactor: use split client for wal prune manager and topic creator * chore: add configs * chore: apply review comments * fix: use tracker properly * fix: use guard to track semaphore * test: update unit tests * chore: update config name * chore: use prunable_entry_id * refactor: semaphore to only limit the process of submitting * chore: remove legacy sort * chore: better configs * fix: update config.md * chore: respect fmt * test: update unit tests * chore: use interval_at * fix: fix unit test * test: fix unit test * test: fix unit test * chore: apply review comments * docs: update config docs --- .../with-remote-wal.yaml | 4 +- config/config.md | 3 + config/metasrv.example.toml | 16 + src/cmd/src/standalone.rs | 2 + src/common/meta/src/datanode.rs | 9 + src/common/meta/src/region_registry.rs | 53 ++- src/common/meta/src/wal_options_allocator.rs | 4 +- .../wal_options_allocator/topic_creator.rs | 21 +- src/common/wal/src/config.rs | 26 ++ src/common/wal/src/config/kafka/common.rs | 7 + src/common/wal/src/config/kafka/datanode.rs | 18 +- src/common/wal/src/config/kafka/metasrv.rs | 17 +- src/meta-srv/src/error.rs | 19 +- .../handler/collect_leader_region_handler.rs | 9 +- src/meta-srv/src/handler/failure_handler.rs | 2 + .../src/handler/region_lease_handler.rs | 2 + src/meta-srv/src/lib.rs | 1 + src/meta-srv/src/metasrv.rs | 5 + src/meta-srv/src/metasrv/builder.rs | 39 +- src/meta-srv/src/metrics.rs | 3 + src/meta-srv/src/procedure/test_util.rs | 19 +- src/meta-srv/src/procedure/wal_prune.rs | 261 +++++------ .../src/procedure/wal_prune/manager.rs | 438 ++++++++++++++++++ .../src/procedure/wal_prune/test_util.rs | 94 ++++ src/meta-srv/src/selector/weight_compute.rs | 6 + tests/conf/metasrv-test.toml.template | 1 + 26 files changed, 881 insertions(+), 198 deletions(-) create mode 100644 src/meta-srv/src/procedure/wal_prune/manager.rs create mode 100644 src/meta-srv/src/procedure/wal_prune/test_util.rs diff --git a/.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml b/.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml index a97f921f8c..58cc188985 100644 --- a/.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml +++ b/.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml @@ -2,13 +2,13 @@ meta: configData: |- [runtime] global_rt_size = 4 - + [wal] provider = "kafka" broker_endpoints = ["kafka.kafka-cluster.svc.cluster.local:9092"] num_topics = 3 + auto_prune_topic_records = true - [datanode] [datanode.client] timeout = "120s" diff --git a/config/config.md b/config/config.md index d0d7582db5..f34a41d861 100644 --- a/config/config.md +++ b/config/config.md @@ -343,6 +343,9 @@ | `wal.provider` | String | `raft_engine` | -- | | `wal.broker_endpoints` | Array | -- | The broker endpoints of the Kafka cluster. | | `wal.auto_create_topics` | Bool | `true` | Automatically create topics for WAL.
Set to `true` to automatically create topics for WAL.
Otherwise, use topics named `topic_name_prefix_[0..num_topics)` | +| `wal.auto_prune_interval` | String | `0s` | Interval of automatically WAL pruning.
Set to `0s` to disable automatically WAL pruning which delete unused remote WAL entries periodically. | +| `wal.trigger_flush_threshold` | Integer | `0` | The threshold to trigger a flush operation of a region in automatically WAL pruning.
Metasrv will send a flush request to flush the region when:
`trigger_flush_threshold` + `prunable_entry_id` < `max_prunable_entry_id`
where:
- `prunable_entry_id` is the maximum entry id that can be pruned of the region.
- `max_prunable_entry_id` is the maximum prunable entry id among all regions in the same topic.
Set to `0` to disable the flush operation. | +| `wal.auto_prune_parallelism` | Integer | `10` | Concurrent task limit for automatically WAL pruning. | | `wal.num_topics` | Integer | `64` | Number of topics. | | `wal.selector_type` | String | `round_robin` | Topic selector type.
Available selector types:
- `round_robin` (default) | | `wal.topic_name_prefix` | String | `greptimedb_wal_topic` | A Kafka topic is constructed by concatenating `topic_name_prefix` and `topic_id`.
Only accepts strings that match the following regular expression pattern:
[a-zA-Z_:-][a-zA-Z0-9_:\-\.@#]*
i.g., greptimedb_wal_topic_0, greptimedb_wal_topic_1. | diff --git a/config/metasrv.example.toml b/config/metasrv.example.toml index 0eb9900c2a..89c92352b2 100644 --- a/config/metasrv.example.toml +++ b/config/metasrv.example.toml @@ -130,6 +130,22 @@ broker_endpoints = ["127.0.0.1:9092"] ## Otherwise, use topics named `topic_name_prefix_[0..num_topics)` auto_create_topics = true +## Interval of automatically WAL pruning. +## Set to `0s` to disable automatically WAL pruning which delete unused remote WAL entries periodically. +auto_prune_interval = "0s" + +## The threshold to trigger a flush operation of a region in automatically WAL pruning. +## Metasrv will send a flush request to flush the region when: +## `trigger_flush_threshold` + `prunable_entry_id` < `max_prunable_entry_id` +## where: +## - `prunable_entry_id` is the maximum entry id that can be pruned of the region. +## - `max_prunable_entry_id` is the maximum prunable entry id among all regions in the same topic. +## Set to `0` to disable the flush operation. +trigger_flush_threshold = 0 + +## Concurrent task limit for automatically WAL pruning. +auto_prune_parallelism = 10 + ## Number of topics. num_topics = 64 diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index 3177a2446f..320a2849ed 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -779,6 +779,8 @@ impl InformationExtension for StandaloneInformationExtension { sst_size: region_stat.sst_size, index_size: region_stat.index_size, region_manifest: region_stat.manifest.into(), + data_topic_latest_entry_id: region_stat.data_topic_latest_entry_id, + metadata_topic_latest_entry_id: region_stat.metadata_topic_latest_entry_id, } }) .collect::>(); diff --git a/src/common/meta/src/datanode.rs b/src/common/meta/src/datanode.rs index 2f45e6bdb9..499ed865a2 100644 --- a/src/common/meta/src/datanode.rs +++ b/src/common/meta/src/datanode.rs @@ -94,6 +94,13 @@ pub struct RegionStat { pub index_size: u64, /// The manifest infoof the region. pub region_manifest: RegionManifestInfo, + /// The latest entry id of topic used by data. + /// **Only used by remote WAL prune.** + pub data_topic_latest_entry_id: u64, + /// The latest entry id of topic used by metadata. + /// **Only used by remote WAL prune.** + /// In mito engine, this is the same as `data_topic_latest_entry_id`. + pub metadata_topic_latest_entry_id: u64, } #[derive(Debug, Clone, Copy, Serialize, Deserialize)] @@ -264,6 +271,8 @@ impl From<&api::v1::meta::RegionStat> for RegionStat { sst_size: region_stat.sst_size, index_size: region_stat.index_size, region_manifest: region_stat.manifest.into(), + data_topic_latest_entry_id: region_stat.data_topic_latest_entry_id, + metadata_topic_latest_entry_id: region_stat.metadata_topic_latest_entry_id, } } } diff --git a/src/common/meta/src/region_registry.rs b/src/common/meta/src/region_registry.rs index 344e0fef5b..49d97f8a2f 100644 --- a/src/common/meta/src/region_registry.rs +++ b/src/common/meta/src/region_registry.rs @@ -19,7 +19,7 @@ use std::sync::{Arc, RwLock}; use common_telemetry::warn; use store_api::storage::RegionId; -use crate::datanode::RegionManifestInfo; +use crate::datanode::{RegionManifestInfo, RegionStat}; /// Represents information about a leader region in the cluster. /// Contains the datanode id where the leader is located, @@ -35,25 +35,22 @@ pub enum LeaderRegionManifestInfo { Mito { manifest_version: u64, flushed_entry_id: u64, + topic_latest_entry_id: u64, }, Metric { data_manifest_version: u64, data_flushed_entry_id: u64, + data_topic_latest_entry_id: u64, metadata_manifest_version: u64, metadata_flushed_entry_id: u64, + metadata_topic_latest_entry_id: u64, }, } -impl From for LeaderRegionManifestInfo { - fn from(value: RegionManifestInfo) -> Self { - match value { - RegionManifestInfo::Mito { - manifest_version, - flushed_entry_id, - } => LeaderRegionManifestInfo::Mito { - manifest_version, - flushed_entry_id, - }, +impl LeaderRegionManifestInfo { + /// Generate a [LeaderRegionManifestInfo] from [RegionStat]. + pub fn from_region_stat(region_stat: &RegionStat) -> LeaderRegionManifestInfo { + match region_stat.region_manifest { RegionManifestInfo::Metric { data_manifest_version, data_flushed_entry_id, @@ -62,14 +59,22 @@ impl From for LeaderRegionManifestInfo { } => LeaderRegionManifestInfo::Metric { data_manifest_version, data_flushed_entry_id, + data_topic_latest_entry_id: region_stat.data_topic_latest_entry_id, metadata_manifest_version, metadata_flushed_entry_id, + metadata_topic_latest_entry_id: region_stat.metadata_topic_latest_entry_id, + }, + RegionManifestInfo::Mito { + manifest_version, + flushed_entry_id, + } => LeaderRegionManifestInfo::Mito { + manifest_version, + flushed_entry_id, + topic_latest_entry_id: region_stat.data_topic_latest_entry_id, }, } } -} -impl LeaderRegionManifestInfo { /// Returns the manifest version of the leader region. pub fn manifest_version(&self) -> u64 { match self { @@ -96,17 +101,33 @@ impl LeaderRegionManifestInfo { } } - /// Returns the minimum flushed entry id of the leader region. - pub fn min_flushed_entry_id(&self) -> u64 { + /// Returns prunable entry id of the leader region. + /// It is used to determine the entry id that can be pruned in remote wal. + /// + /// For a mito region, the prunable entry id should max(flushed_entry_id, latest_entry_id_since_flush). + /// + /// For a metric region, the prunable entry id should min( + /// max(data_flushed_entry_id, data_latest_entry_id_since_flush), + /// max(metadata_flushed_entry_id, metadata_latest_entry_id_since_flush) + /// ). + pub fn prunable_entry_id(&self) -> u64 { match self { LeaderRegionManifestInfo::Mito { flushed_entry_id, .. } => *flushed_entry_id, LeaderRegionManifestInfo::Metric { data_flushed_entry_id, + data_topic_latest_entry_id, metadata_flushed_entry_id, + metadata_topic_latest_entry_id, .. - } => (*data_flushed_entry_id).min(*metadata_flushed_entry_id), + } => { + let data_prunable_entry_id = + (*data_flushed_entry_id).max(*data_topic_latest_entry_id); + let metadata_prunable_entry_id = + (*metadata_flushed_entry_id).max(*metadata_topic_latest_entry_id); + data_prunable_entry_id.min(metadata_prunable_entry_id) + } } } } diff --git a/src/common/meta/src/wal_options_allocator.rs b/src/common/meta/src/wal_options_allocator.rs index 2aba2a5ee3..a6e1482f04 100644 --- a/src/common/meta/src/wal_options_allocator.rs +++ b/src/common/meta/src/wal_options_allocator.rs @@ -30,7 +30,9 @@ use crate::error::{EncodeWalOptionsSnafu, InvalidTopicNamePrefixSnafu, Result}; use crate::key::NAME_PATTERN_REGEX; use crate::kv_backend::KvBackendRef; use crate::leadership_notifier::LeadershipChangeListener; -pub use crate::wal_options_allocator::topic_creator::build_kafka_topic_creator; +pub use crate::wal_options_allocator::topic_creator::{ + build_kafka_client, build_kafka_topic_creator, +}; use crate::wal_options_allocator::topic_pool::KafkaTopicPool; /// Allocates wal options in region granularity. diff --git a/src/common/meta/src/wal_options_allocator/topic_creator.rs b/src/common/meta/src/wal_options_allocator/topic_creator.rs index f49d1bf1ca..1a023546d3 100644 --- a/src/common/meta/src/wal_options_allocator/topic_creator.rs +++ b/src/common/meta/src/wal_options_allocator/topic_creator.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::Arc; - use common_telemetry::{error, info}; use common_wal::config::kafka::common::DEFAULT_BACKOFF_CONFIG; use common_wal::config::kafka::MetasrvKafkaConfig; @@ -34,11 +32,9 @@ use crate::error::{ // The `DEFAULT_PARTITION` refers to the index of the partition. const DEFAULT_PARTITION: i32 = 0; -type KafkaClientRef = Arc; - /// Creates topics in kafka. pub struct KafkaTopicCreator { - client: KafkaClientRef, + client: Client, /// The number of partitions per topic. num_partitions: i32, /// The replication factor of each topic. @@ -48,7 +44,7 @@ pub struct KafkaTopicCreator { } impl KafkaTopicCreator { - pub fn client(&self) -> &KafkaClientRef { + pub fn client(&self) -> &Client { &self.client } @@ -133,7 +129,8 @@ impl KafkaTopicCreator { } } -pub async fn build_kafka_topic_creator(config: &MetasrvKafkaConfig) -> Result { +/// Builds a kafka [Client](rskafka::client::Client). +pub async fn build_kafka_client(config: &MetasrvKafkaConfig) -> Result { // Builds an kafka controller client for creating topics. let broker_endpoints = common_wal::resolve_to_ipv4(&config.connection.broker_endpoints) .await @@ -145,15 +142,19 @@ pub async fn build_kafka_topic_creator(config: &MetasrvKafkaConfig) -> Result Result { + let client = build_kafka_client(config).await?; Ok(KafkaTopicCreator { - client: Arc::new(client), + client, num_partitions: config.kafka_topic.num_partitions, replication_factor: config.kafka_topic.replication_factor, create_topic_timeout: config.kafka_topic.create_topic_timeout.as_millis() as i32, diff --git a/src/common/wal/src/config.rs b/src/common/wal/src/config.rs index 831faef772..921c1faaa3 100644 --- a/src/common/wal/src/config.rs +++ b/src/common/wal/src/config.rs @@ -15,6 +15,8 @@ pub mod kafka; pub mod raft_engine; +use std::time::Duration; + use serde::{Deserialize, Serialize}; use crate::config::kafka::{DatanodeKafkaConfig, MetasrvKafkaConfig}; @@ -53,11 +55,32 @@ impl From for MetasrvWalConfig { connection: config.connection, kafka_topic: config.kafka_topic, auto_create_topics: config.auto_create_topics, + auto_prune_interval: config.auto_prune_interval, + trigger_flush_threshold: config.trigger_flush_threshold, + auto_prune_parallelism: config.auto_prune_parallelism, }), } } } +impl MetasrvWalConfig { + /// Returns if active wal pruning is enabled. + pub fn enable_active_wal_pruning(&self) -> bool { + match self { + MetasrvWalConfig::RaftEngine => false, + MetasrvWalConfig::Kafka(config) => config.auto_prune_interval > Duration::ZERO, + } + } + + /// Gets the kafka connection config. + pub fn remote_wal_options(&self) -> Option<&MetasrvKafkaConfig> { + match self { + MetasrvWalConfig::RaftEngine => None, + MetasrvWalConfig::Kafka(config) => Some(config), + } + } +} + impl From for DatanodeWalConfig { fn from(config: MetasrvWalConfig) -> Self { match config { @@ -181,6 +204,9 @@ mod tests { create_topic_timeout: Duration::from_secs(30), }, auto_create_topics: true, + auto_prune_interval: Duration::from_secs(0), + trigger_flush_threshold: 0, + auto_prune_parallelism: 10, }; assert_eq!(metasrv_wal_config, MetasrvWalConfig::Kafka(expected)); diff --git a/src/common/wal/src/config/kafka/common.rs b/src/common/wal/src/config/kafka/common.rs index ea58a3d49e..6b2c9992f4 100644 --- a/src/common/wal/src/config/kafka/common.rs +++ b/src/common/wal/src/config/kafka/common.rs @@ -30,6 +30,13 @@ pub const DEFAULT_BACKOFF_CONFIG: BackoffConfig = BackoffConfig { deadline: Some(Duration::from_secs(120)), }; +/// Default interval for active WAL pruning. +pub const DEFAULT_ACTIVE_PRUNE_INTERVAL: Duration = Duration::ZERO; +/// Default limit for concurrent active pruning tasks. +pub const DEFAULT_ACTIVE_PRUNE_TASK_LIMIT: usize = 10; +/// Default interval for sending flush request to regions when pruning remote WAL. +pub const DEFAULT_TRIGGER_FLUSH_THRESHOLD: u64 = 0; + use crate::error::{self, Result}; use crate::{TopicSelectorType, BROKER_ENDPOINT, TOPIC_NAME_PREFIX}; diff --git a/src/common/wal/src/config/kafka/datanode.rs b/src/common/wal/src/config/kafka/datanode.rs index 77cf05397d..dd659d636e 100644 --- a/src/common/wal/src/config/kafka/datanode.rs +++ b/src/common/wal/src/config/kafka/datanode.rs @@ -17,7 +17,10 @@ use std::time::Duration; use common_base::readable_size::ReadableSize; use serde::{Deserialize, Serialize}; -use crate::config::kafka::common::{KafkaConnectionConfig, KafkaTopicConfig}; +use crate::config::kafka::common::{ + KafkaConnectionConfig, KafkaTopicConfig, DEFAULT_ACTIVE_PRUNE_INTERVAL, + DEFAULT_ACTIVE_PRUNE_TASK_LIMIT, DEFAULT_TRIGGER_FLUSH_THRESHOLD, +}; /// Kafka wal configurations for datanode. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -44,6 +47,15 @@ pub struct DatanodeKafkaConfig { pub dump_index_interval: Duration, /// Ignore missing entries during read WAL. pub overwrite_entry_start_id: bool, + // Active WAL pruning. + pub auto_prune_topic_records: bool, + // Interval of WAL pruning. + pub auto_prune_interval: Duration, + // Threshold for sending flush request when pruning remote WAL. + // `None` stands for never sending flush request. + pub trigger_flush_threshold: u64, + // Limit of concurrent active pruning procedures. + pub auto_prune_parallelism: usize, } impl Default for DatanodeKafkaConfig { @@ -58,6 +70,10 @@ impl Default for DatanodeKafkaConfig { create_index: true, dump_index_interval: Duration::from_secs(60), overwrite_entry_start_id: false, + auto_prune_topic_records: false, + auto_prune_interval: DEFAULT_ACTIVE_PRUNE_INTERVAL, + trigger_flush_threshold: DEFAULT_TRIGGER_FLUSH_THRESHOLD, + auto_prune_parallelism: DEFAULT_ACTIVE_PRUNE_TASK_LIMIT, } } } diff --git a/src/common/wal/src/config/kafka/metasrv.rs b/src/common/wal/src/config/kafka/metasrv.rs index 27df3569b8..acbfbe05c6 100644 --- a/src/common/wal/src/config/kafka/metasrv.rs +++ b/src/common/wal/src/config/kafka/metasrv.rs @@ -12,9 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::time::Duration; + use serde::{Deserialize, Serialize}; -use crate::config::kafka::common::{KafkaConnectionConfig, KafkaTopicConfig}; +use crate::config::kafka::common::{ + KafkaConnectionConfig, KafkaTopicConfig, DEFAULT_ACTIVE_PRUNE_INTERVAL, + DEFAULT_ACTIVE_PRUNE_TASK_LIMIT, DEFAULT_TRIGGER_FLUSH_THRESHOLD, +}; /// Kafka wal configurations for metasrv. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -28,6 +33,13 @@ pub struct MetasrvKafkaConfig { pub kafka_topic: KafkaTopicConfig, // Automatically create topics for WAL. pub auto_create_topics: bool, + // Interval of WAL pruning. + pub auto_prune_interval: Duration, + // Threshold for sending flush request when pruning remote WAL. + // `None` stands for never sending flush request. + pub trigger_flush_threshold: u64, + // Limit of concurrent active pruning procedures. + pub auto_prune_parallelism: usize, } impl Default for MetasrvKafkaConfig { @@ -36,6 +48,9 @@ impl Default for MetasrvKafkaConfig { connection: Default::default(), kafka_topic: Default::default(), auto_create_topics: true, + auto_prune_interval: DEFAULT_ACTIVE_PRUNE_INTERVAL, + trigger_flush_threshold: DEFAULT_TRIGGER_FLUSH_THRESHOLD, + auto_prune_parallelism: DEFAULT_ACTIVE_PRUNE_TASK_LIMIT, } } } diff --git a/src/meta-srv/src/error.rs b/src/meta-srv/src/error.rs index 7c45fa408f..bd249c5178 100644 --- a/src/meta-srv/src/error.rs +++ b/src/meta-srv/src/error.rs @@ -518,6 +518,13 @@ pub enum Error { source: common_procedure::Error, }, + #[snafu(display("A prune task for topic {} is already running", topic))] + PruneTaskAlreadyRunning { + topic: String, + #[snafu(implicit)] + location: Location, + }, + #[snafu(display("Schema already exists, name: {schema_name}"))] SchemaAlreadyExists { schema_name: String, @@ -788,6 +795,14 @@ pub enum Error { source: common_meta::error::Error, }, + #[snafu(display("Failed to build kafka client."))] + BuildKafkaClient { + #[snafu(implicit)] + location: Location, + #[snafu(source)] + error: common_meta::error::Error, + }, + #[snafu(display( "Failed to build a Kafka partition client, topic: {}, partition: {}", topic, @@ -875,7 +890,9 @@ impl ErrorExt for Error { | Error::FlowStateHandler { .. } | Error::BuildWalOptionsAllocator { .. } | Error::BuildPartitionClient { .. } - | Error::DeleteRecords { .. } => StatusCode::Internal, + | Error::BuildKafkaClient { .. } + | Error::DeleteRecords { .. } + | Error::PruneTaskAlreadyRunning { .. } => StatusCode::Internal, Error::Unsupported { .. } => StatusCode::Unsupported, diff --git a/src/meta-srv/src/handler/collect_leader_region_handler.rs b/src/meta-srv/src/handler/collect_leader_region_handler.rs index fd5fab3639..13aee5d234 100644 --- a/src/meta-srv/src/handler/collect_leader_region_handler.rs +++ b/src/meta-srv/src/handler/collect_leader_region_handler.rs @@ -13,7 +13,7 @@ // limitations under the License. use api::v1::meta::{HeartbeatRequest, Role}; -use common_meta::region_registry::LeaderRegion; +use common_meta::region_registry::{LeaderRegion, LeaderRegionManifestInfo}; use store_api::region_engine::RegionRole; use crate::error::Result; @@ -44,7 +44,7 @@ impl HeartbeatHandler for CollectLeaderRegionHandler { continue; } - let manifest = stat.region_manifest.into(); + let manifest = LeaderRegionManifestInfo::from_region_stat(stat); let value = LeaderRegion { datanode_id: current_stat.id, manifest, @@ -122,6 +122,8 @@ mod tests { manifest_size: 0, sst_size: 0, index_size: 0, + data_topic_latest_entry_id: 0, + metadata_topic_latest_entry_id: 0, } } @@ -161,6 +163,7 @@ mod tests { manifest: LeaderRegionManifestInfo::Mito { manifest_version: 1, flushed_entry_id: 0, + topic_latest_entry_id: 0, }, }) ); @@ -192,6 +195,7 @@ mod tests { manifest: LeaderRegionManifestInfo::Mito { manifest_version: 2, flushed_entry_id: 0, + topic_latest_entry_id: 0, }, }) ); @@ -224,6 +228,7 @@ mod tests { manifest: LeaderRegionManifestInfo::Mito { manifest_version: 2, flushed_entry_id: 0, + topic_latest_entry_id: 0, }, }) ); diff --git a/src/meta-srv/src/handler/failure_handler.rs b/src/meta-srv/src/handler/failure_handler.rs index 81e9bfaeb9..cdcd9d3228 100644 --- a/src/meta-srv/src/handler/failure_handler.rs +++ b/src/meta-srv/src/handler/failure_handler.rs @@ -102,6 +102,8 @@ mod tests { manifest_version: 0, flushed_entry_id: 0, }, + data_topic_latest_entry_id: 0, + metadata_topic_latest_entry_id: 0, } } acc.stat = Some(Stat { diff --git a/src/meta-srv/src/handler/region_lease_handler.rs b/src/meta-srv/src/handler/region_lease_handler.rs index b89a570b80..d900078897 100644 --- a/src/meta-srv/src/handler/region_lease_handler.rs +++ b/src/meta-srv/src/handler/region_lease_handler.rs @@ -160,6 +160,8 @@ mod test { manifest_version: 0, flushed_entry_id: 0, }, + data_topic_latest_entry_id: 0, + metadata_topic_latest_entry_id: 0, } } diff --git a/src/meta-srv/src/lib.rs b/src/meta-srv/src/lib.rs index 9a9f0861a8..4b61eeeae3 100644 --- a/src/meta-srv/src/lib.rs +++ b/src/meta-srv/src/lib.rs @@ -15,6 +15,7 @@ #![feature(result_flattening)] #![feature(assert_matches)] #![feature(extract_if)] +#![feature(hash_set_entry)] pub mod bootstrap; pub mod cache_invalidator; diff --git a/src/meta-srv/src/metasrv.rs b/src/meta-srv/src/metasrv.rs index 00ac628b61..34b3cac25e 100644 --- a/src/meta-srv/src/metasrv.rs +++ b/src/meta-srv/src/metasrv.rs @@ -61,6 +61,7 @@ use crate::failure_detector::PhiAccrualFailureDetectorOptions; use crate::handler::{HeartbeatHandlerGroupBuilder, HeartbeatHandlerGroupRef}; use crate::lease::lookup_datanode_peer; use crate::procedure::region_migration::manager::RegionMigrationManagerRef; +use crate::procedure::wal_prune::manager::WalPruneTickerRef; use crate::procedure::ProcedureManagerListenerAdapter; use crate::pubsub::{PublisherRef, SubscriptionManagerRef}; use crate::region::supervisor::RegionSupervisorTickerRef; @@ -407,6 +408,7 @@ pub struct Metasrv { region_supervisor_ticker: Option, cache_invalidator: CacheInvalidatorRef, leader_region_registry: LeaderRegionRegistryRef, + wal_prune_ticker: Option, plugins: Plugins, } @@ -461,6 +463,9 @@ impl Metasrv { if let Some(region_supervisor_ticker) = &self.region_supervisor_ticker { leadership_change_notifier.add_listener(region_supervisor_ticker.clone() as _); } + if let Some(wal_prune_ticker) = &self.wal_prune_ticker { + leadership_change_notifier.add_listener(wal_prune_ticker.clone() as _); + } if let Some(customizer) = self.plugins.get::() { customizer.customize(&mut leadership_change_notifier); } diff --git a/src/meta-srv/src/metasrv/builder.rs b/src/meta-srv/src/metasrv/builder.rs index 0f1a04e47b..ec8f6ef253 100644 --- a/src/meta-srv/src/metasrv/builder.rs +++ b/src/meta-srv/src/metasrv/builder.rs @@ -37,7 +37,7 @@ use common_meta::region_keeper::MemoryRegionKeeper; use common_meta::region_registry::LeaderRegionRegistry; use common_meta::sequence::SequenceBuilder; use common_meta::state_store::KvStateStore; -use common_meta::wal_options_allocator::build_wal_options_allocator; +use common_meta::wal_options_allocator::{build_kafka_client, build_wal_options_allocator}; use common_procedure::local::{LocalManager, ManagerConfig}; use common_procedure::ProcedureManagerRef; use snafu::ResultExt; @@ -58,6 +58,8 @@ use crate::metasrv::{ }; use crate::procedure::region_migration::manager::RegionMigrationManager; use crate::procedure::region_migration::DefaultContextFactory; +use crate::procedure::wal_prune::manager::{WalPruneManager, WalPruneTicker}; +use crate::procedure::wal_prune::Context as WalPruneContext; use crate::region::supervisor::{ HeartbeatAcceptor, RegionFailureDetectorControl, RegionSupervisor, RegionSupervisorTicker, DEFAULT_TICK_INTERVAL, @@ -346,6 +348,40 @@ impl MetasrvBuilder { .context(error::InitDdlManagerSnafu)?, ); + // remote WAL prune ticker and manager + let wal_prune_ticker = if is_remote_wal && options.wal.enable_active_wal_pruning() { + let (tx, rx) = WalPruneManager::channel(); + // Safety: Must be remote WAL. + let remote_wal_options = options.wal.remote_wal_options().unwrap(); + let kafka_client = build_kafka_client(remote_wal_options) + .await + .context(error::BuildKafkaClientSnafu)?; + let wal_prune_context = WalPruneContext { + client: Arc::new(kafka_client), + table_metadata_manager: table_metadata_manager.clone(), + leader_region_registry: leader_region_registry.clone(), + server_addr: options.server_addr.clone(), + mailbox: mailbox.clone(), + }; + let wal_prune_manager = WalPruneManager::new( + table_metadata_manager.clone(), + remote_wal_options.auto_prune_parallelism, + rx, + procedure_manager.clone(), + wal_prune_context, + remote_wal_options.trigger_flush_threshold, + ); + // Start manager in background. Ticker will be started in the main thread to send ticks. + wal_prune_manager.try_start().await?; + let wal_prune_ticker = Arc::new(WalPruneTicker::new( + remote_wal_options.auto_prune_interval, + tx.clone(), + )); + Some(wal_prune_ticker) + } else { + None + }; + let customized_region_lease_renewer = plugins .as_ref() .and_then(|plugins| plugins.get::()); @@ -406,6 +442,7 @@ impl MetasrvBuilder { region_supervisor_ticker, cache_invalidator, leader_region_registry, + wal_prune_ticker, }) } } diff --git a/src/meta-srv/src/metrics.rs b/src/meta-srv/src/metrics.rs index c5e2d3df4d..a9750ae5f3 100644 --- a/src/meta-srv/src/metrics.rs +++ b/src/meta-srv/src/metrics.rs @@ -66,4 +66,7 @@ lazy_static! { // The heartbeat rate counter. pub static ref METRIC_META_HEARTBEAT_RATE: IntCounter = register_int_counter!("greptime_meta_heartbeat_rate", "meta heartbeat arrival rate").unwrap(); + /// The remote WAL prune execute counter. + pub static ref METRIC_META_REMOTE_WAL_PRUNE_EXECUTE: IntCounterVec = + register_int_counter_vec!("greptime_meta_remote_wal_prune_execute", "meta remote wal prune execute", &["topic_name"]).unwrap(); } diff --git a/src/meta-srv/src/procedure/test_util.rs b/src/meta-srv/src/procedure/test_util.rs index 34ce23abd4..ca6da59f2a 100644 --- a/src/meta-srv/src/procedure/test_util.rs +++ b/src/meta-srv/src/procedure/test_util.rs @@ -179,8 +179,8 @@ pub async fn new_wal_prune_metadata( ) -> (EntryId, Vec) { let datanode_id = 1; let from_peer = Peer::empty(datanode_id); - let mut min_flushed_entry_id = u64::MAX; - let mut max_flushed_entry_id = 0; + let mut min_prunable_entry_id = u64::MAX; + let mut max_prunable_entry_id = 0; let mut region_entry_ids = HashMap::with_capacity(n_table as usize * n_region as usize); for table_id in 0..n_table { let region_ids = (0..n_region) @@ -221,10 +221,10 @@ pub async fn new_wal_prune_metadata( .iter() .map(|region_id| { let rand_n = rand::random::() as usize; - let current_flushed_entry_id = offsets[rand_n % offsets.len()] as u64; - min_flushed_entry_id = min_flushed_entry_id.min(current_flushed_entry_id); - max_flushed_entry_id = max_flushed_entry_id.max(current_flushed_entry_id); - (*region_id, current_flushed_entry_id) + let current_prunable_entry_id = offsets[rand_n % offsets.len()] as u64; + min_prunable_entry_id = min_prunable_entry_id.min(current_prunable_entry_id); + max_prunable_entry_id = max_prunable_entry_id.max(current_prunable_entry_id); + (*region_id, current_prunable_entry_id) }) .collect::>(); region_entry_ids.extend(current_region_entry_ids.clone()); @@ -235,15 +235,15 @@ pub async fn new_wal_prune_metadata( let regions_to_flush = region_entry_ids .iter() - .filter_map(|(region_id, flushed_entry_id)| { - if max_flushed_entry_id - flushed_entry_id > threshold { + .filter_map(|(region_id, prunable_entry_id)| { + if max_prunable_entry_id - prunable_entry_id > threshold { Some(*region_id) } else { None } }) .collect::>(); - (min_flushed_entry_id, regions_to_flush) + (min_prunable_entry_id, regions_to_flush) } pub async fn update_in_memory_region_flushed_entry_id( @@ -257,6 +257,7 @@ pub async fn update_in_memory_region_flushed_entry_id( manifest: LeaderRegionManifestInfo::Mito { manifest_version: 0, flushed_entry_id, + topic_latest_entry_id: 0, }, }; key_values.push((region_id, value)); diff --git a/src/meta-srv/src/procedure/wal_prune.rs b/src/meta-srv/src/procedure/wal_prune.rs index b7f145fffc..0247c928ec 100644 --- a/src/meta-srv/src/procedure/wal_prune.rs +++ b/src/meta-srv/src/procedure/wal_prune.rs @@ -12,6 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub(crate) mod manager; +#[cfg(test)] +mod test_util; + use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::time::Duration; @@ -28,9 +32,10 @@ use common_procedure::{ Context as ProcedureContext, Error as ProcedureError, LockKey, Procedure, Result as ProcedureResult, Status, StringKey, }; -use common_telemetry::warn; +use common_telemetry::{info, warn}; use itertools::{Itertools, MinMaxResult}; use log_store::kafka::DEFAULT_PARTITION; +use manager::{WalPruneProcedureGuard, WalPruneProcedureTracker}; use rskafka::client::partition::UnknownTopicHandling; use rskafka::client::Client; use serde::{Deserialize, Serialize}; @@ -45,9 +50,8 @@ use crate::error::{ use crate::service::mailbox::{Channel, MailboxRef}; use crate::Result; -type KafkaClientRef = Arc; +pub type KafkaClientRef = Arc; -/// No timeout for flush request. const DELETE_RECORDS_TIMEOUT: Duration = Duration::from_secs(1); /// The state of WAL pruning. @@ -58,17 +62,18 @@ pub enum WalPruneState { Prune, } +#[derive(Clone)] pub struct Context { /// The Kafka client. - client: KafkaClientRef, + pub client: KafkaClientRef, /// The table metadata manager. - table_metadata_manager: TableMetadataManagerRef, + pub table_metadata_manager: TableMetadataManagerRef, /// The leader region registry. - leader_region_registry: LeaderRegionRegistryRef, + pub leader_region_registry: LeaderRegionRegistryRef, /// Server address of metasrv. - server_addr: String, + pub server_addr: String, /// The mailbox to send messages. - mailbox: MailboxRef, + pub mailbox: MailboxRef, } /// The data of WAL pruning. @@ -77,10 +82,11 @@ pub struct WalPruneData { /// The topic name to prune. pub topic: String, /// The minimum flush entry id for topic, which is used to prune the WAL. - pub min_flushed_entry_id: EntryId, + pub prunable_entry_id: EntryId, pub regions_to_flush: Vec, - /// If `flushed_entry_id` + `trigger_flush_threshold` < `max_flushed_entry_id`, send a flush request to the region. - pub trigger_flush_threshold: Option, + /// If `prunable_entry_id` + `trigger_flush_threshold` < `max_prunable_entry_id`, send a flush request to the region. + /// If `None`, never send flush requests. + pub trigger_flush_threshold: u64, /// The state. pub state: WalPruneState, } @@ -89,27 +95,43 @@ pub struct WalPruneData { pub struct WalPruneProcedure { pub data: WalPruneData, pub context: Context, + pub _guard: Option, } impl WalPruneProcedure { const TYPE_NAME: &'static str = "metasrv-procedure::WalPrune"; - pub fn new(topic: String, context: Context, trigger_flush_threshold: Option) -> Self { + pub fn new( + topic: String, + context: Context, + trigger_flush_threshold: u64, + guard: Option, + ) -> Self { Self { data: WalPruneData { topic, - min_flushed_entry_id: 0, + prunable_entry_id: 0, trigger_flush_threshold, regions_to_flush: vec![], state: WalPruneState::Prepare, }, context, + _guard: guard, } } - pub fn from_json(json: &str, context: Context) -> ProcedureResult { + pub fn from_json( + json: &str, + context: &Context, + tracker: WalPruneProcedureTracker, + ) -> ProcedureResult { let data: WalPruneData = serde_json::from_str(json).context(ToJsonSnafu)?; - Ok(Self { data, context }) + let guard = tracker.insert_running_procedure(data.topic.clone()); + Ok(Self { + data, + context: context.clone(), + _guard: guard, + }) } async fn build_peer_to_region_ids_map( @@ -182,20 +204,20 @@ impl WalPruneProcedure { .with_context(|_| error::RetryLaterWithSourceSnafu { reason: "Failed to get topic-region map", })?; - let flush_entry_ids_map: HashMap<_, _> = self + let prunable_entry_ids_map: HashMap<_, _> = self .context .leader_region_registry .batch_get(region_ids.iter().cloned()) .into_iter() .map(|(region_id, region)| { - let flushed_entry_id = region.manifest.min_flushed_entry_id(); - (region_id, flushed_entry_id) + let prunable_entry_id = region.manifest.prunable_entry_id(); + (region_id, prunable_entry_id) }) .collect(); - // Check if the `flush_entry_ids_map` contains all region ids. + // Check if the `prunable_entry_ids_map` contains all region ids. let non_collected_region_ids = - check_heartbeat_collected_region_ids(®ion_ids, &flush_entry_ids_map); + check_heartbeat_collected_region_ids(®ion_ids, &prunable_entry_ids_map); if !non_collected_region_ids.is_empty() { // The heartbeat collected region ids do not contain all region ids in the topic-region map. // In this case, we should not prune the WAL. @@ -204,23 +226,23 @@ impl WalPruneProcedure { return Ok(Status::done()); } - let min_max_result = flush_entry_ids_map.values().minmax(); - let max_flushed_entry_id = match min_max_result { + let min_max_result = prunable_entry_ids_map.values().minmax(); + let max_prunable_entry_id = match min_max_result { MinMaxResult::NoElements => { return Ok(Status::done()); } - MinMaxResult::OneElement(flushed_entry_id) => { - self.data.min_flushed_entry_id = *flushed_entry_id; - *flushed_entry_id + MinMaxResult::OneElement(prunable_entry_id) => { + self.data.prunable_entry_id = *prunable_entry_id; + *prunable_entry_id } - MinMaxResult::MinMax(min_flushed_entry_id, max_flushed_entry_id) => { - self.data.min_flushed_entry_id = *min_flushed_entry_id; - *max_flushed_entry_id + MinMaxResult::MinMax(min_prunable_entry_id, max_prunable_entry_id) => { + self.data.prunable_entry_id = *min_prunable_entry_id; + *max_prunable_entry_id } }; - if let Some(threshold) = self.data.trigger_flush_threshold { - for (region_id, flushed_entry_id) in flush_entry_ids_map { - if flushed_entry_id + threshold < max_flushed_entry_id { + if self.data.trigger_flush_threshold != 0 { + for (region_id, prunable_entry_id) in prunable_entry_ids_map { + if prunable_entry_id + self.data.trigger_flush_threshold < max_prunable_entry_id { self.data.regions_to_flush.push(region_id); } } @@ -232,10 +254,17 @@ impl WalPruneProcedure { } /// Send the flush request to regions with low flush entry id. + /// + /// Retry: + /// - Failed to build peer to region ids map. It means failure in retrieving metadata. pub async fn on_sending_flush_request(&mut self) -> Result { let peer_to_region_ids_map = self .build_peer_to_region_ids_map(&self.context, &self.data.regions_to_flush) - .await?; + .await + .map_err(BoxedError::new) + .with_context(|_| error::RetryLaterWithSourceSnafu { + reason: "Failed to build peer to region ids map", + })?; let flush_instructions = self.build_flush_region_instruction(peer_to_region_ids_map)?; for (peer, flush_instruction) in flush_instructions.into_iter() { let msg = MailboxMessage::json_message( @@ -255,13 +284,13 @@ impl WalPruneProcedure { Ok(Status::executing(true)) } - /// Prune the WAL and persist the minimum flushed entry id. + /// Prune the WAL and persist the minimum prunable entry id. /// /// Retry: - /// - Failed to update the minimum flushed entry id in kvbackend. + /// - Failed to update the minimum prunable entry id in kvbackend. /// - Failed to delete records. pub async fn on_prune(&mut self) -> Result { - // Safety: flushed_entry_ids are loaded in on_prepare. + // Safety: `prunable_entry_id`` are loaded in on_prepare. let partition_client = self .context .client @@ -276,7 +305,7 @@ impl WalPruneProcedure { partition: DEFAULT_PARTITION, })?; - // Should update the min flushed entry id in the kv backend before deleting records. + // Should update the min prunable entry id in the kv backend before deleting records. // Otherwise, when a datanode restarts, it will not be able to find the wal entries. let prev = self .context @@ -292,7 +321,7 @@ impl WalPruneProcedure { self.context .table_metadata_manager .topic_name_manager() - .update(&self.data.topic, self.data.min_flushed_entry_id, prev) + .update(&self.data.topic, self.data.prunable_entry_id, prev) .await .context(UpdateTopicNameValueSnafu { topic: &self.data.topic, @@ -306,14 +335,14 @@ impl WalPruneProcedure { })?; partition_client .delete_records( - (self.data.min_flushed_entry_id + 1) as i64, + (self.data.prunable_entry_id + 1) as i64, DELETE_RECORDS_TIMEOUT.as_millis() as i32, ) .await .context(DeleteRecordsSnafu { topic: &self.data.topic, partition: DEFAULT_PARTITION, - offset: (self.data.min_flushed_entry_id + 1), + offset: (self.data.prunable_entry_id + 1), }) .map_err(BoxedError::new) .with_context(|_| error::RetryLaterWithSourceSnafu { @@ -321,9 +350,13 @@ impl WalPruneProcedure { "Failed to delete records for topic: {}, partition: {}, offset: {}", self.data.topic, DEFAULT_PARTITION, - self.data.min_flushed_entry_id + 1 + self.data.prunable_entry_id + 1 ), })?; + info!( + "Successfully pruned WAL for topic: {}, entry id: {}", + self.data.topic, self.data.prunable_entry_id + ); Ok(Status::done()) } } @@ -388,123 +421,41 @@ mod tests { use std::assert_matches::assert_matches; use api::v1::meta::HeartbeatResponse; - use common_meta::key::TableMetadataManager; - use common_meta::kv_backend::memory::MemoryKvBackend; - use common_meta::region_registry::LeaderRegionRegistry; - use common_meta::sequence::SequenceBuilder; - use common_meta::wal_options_allocator::build_kafka_topic_creator; - use common_wal::config::kafka::common::{KafkaConnectionConfig, KafkaTopicConfig}; - use common_wal::config::kafka::MetasrvKafkaConfig; use common_wal::test_util::run_test_with_kafka_wal; use rskafka::record::Record; use tokio::sync::mpsc::Receiver; use super::*; use crate::handler::HeartbeatMailbox; - use crate::procedure::test_util::{new_wal_prune_metadata, MailboxContext}; - - struct TestEnv { - table_metadata_manager: TableMetadataManagerRef, - leader_region_registry: LeaderRegionRegistryRef, - mailbox: MailboxContext, - server_addr: String, - } - - impl TestEnv { - fn new() -> Self { - let kv_backend = Arc::new(MemoryKvBackend::new()); - let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend.clone())); - let leader_region_registry = Arc::new(LeaderRegionRegistry::new()); - let mailbox_sequence = - SequenceBuilder::new("test_heartbeat_mailbox", kv_backend.clone()).build(); - - let mailbox_ctx = MailboxContext::new(mailbox_sequence); - - Self { - table_metadata_manager, - leader_region_registry, - mailbox: mailbox_ctx, - server_addr: "localhost".to_string(), - } - } - - fn table_metadata_manager(&self) -> &TableMetadataManagerRef { - &self.table_metadata_manager - } - - fn leader_region_registry(&self) -> &LeaderRegionRegistryRef { - &self.leader_region_registry - } - - fn mailbox_context(&self) -> &MailboxContext { - &self.mailbox - } - - fn server_addr(&self) -> &str { - &self.server_addr - } - } + use crate::procedure::test_util::new_wal_prune_metadata; + // Fix this import to correctly point to the test_util module + use crate::procedure::wal_prune::test_util::TestEnv; /// Mock a test env for testing. /// Including: /// 1. Prepare some data in the table metadata manager and in-memory kv backend. - /// 2. Generate a `WalPruneProcedure` with the test env. - /// 3. Return the procedure, the minimum last entry id to prune and the regions to flush. - async fn mock_test_env( - topic: String, - broker_endpoints: Vec, - env: &TestEnv, - ) -> (WalPruneProcedure, u64, Vec) { - // Creates a topic manager. - let kafka_topic = KafkaTopicConfig { - replication_factor: broker_endpoints.len() as i16, - ..Default::default() - }; - let config = MetasrvKafkaConfig { - connection: KafkaConnectionConfig { - broker_endpoints, - ..Default::default() - }, - kafka_topic, - ..Default::default() - }; - let topic_creator = build_kafka_topic_creator(&config).await.unwrap(); - let table_metadata_manager = env.table_metadata_manager().clone(); - let leader_region_registry = env.leader_region_registry().clone(); - let mailbox = env.mailbox_context().mailbox().clone(); - + /// 2. Return the procedure, the minimum last entry id to prune and the regions to flush. + async fn mock_test_data(procedure: &WalPruneProcedure) -> (u64, Vec) { let n_region = 10; let n_table = 5; - let threshold = 10; // 5 entries per region. let offsets = mock_wal_entries( - topic_creator.client().clone(), - &topic, + procedure.context.client.clone(), + &procedure.data.topic, (n_region * n_table * 5) as usize, ) .await; - - let (min_flushed_entry_id, regions_to_flush) = new_wal_prune_metadata( - table_metadata_manager.clone(), - leader_region_registry.clone(), + let (prunable_entry_id, regions_to_flush) = new_wal_prune_metadata( + procedure.context.table_metadata_manager.clone(), + procedure.context.leader_region_registry.clone(), n_region, n_table, &offsets, - threshold, - topic.clone(), + procedure.data.trigger_flush_threshold, + procedure.data.topic.clone(), ) .await; - - let context = Context { - client: topic_creator.client().clone(), - table_metadata_manager, - leader_region_registry, - mailbox, - server_addr: env.server_addr().to_string(), - }; - - let wal_prune_procedure = WalPruneProcedure::new(topic, context, Some(threshold)); - (wal_prune_procedure, min_flushed_entry_id, regions_to_flush) + (prunable_entry_id, regions_to_flush) } fn record(i: usize) -> Record { @@ -603,10 +554,18 @@ mod tests { run_test_with_kafka_wal(|broker_endpoints| { Box::pin(async { common_telemetry::init_default_ut_logging(); - let topic_name = "greptime_test_topic".to_string(); + let mut topic_name = uuid::Uuid::new_v4().to_string(); + // Topic should start with a letter. + topic_name = format!("test_procedure_execution-{}", topic_name); let mut env = TestEnv::new(); - let (mut procedure, min_flushed_entry_id, regions_to_flush) = - mock_test_env(topic_name.clone(), broker_endpoints, &env).await; + let context = env.build_wal_prune_context(broker_endpoints).await; + let mut procedure = WalPruneProcedure::new(topic_name.clone(), context, 10, None); + + // Before any data in kvbackend is mocked, should return a retryable error. + let result = procedure.on_prune().await; + assert_matches!(result, Err(e) if e.is_retryable()); + + let (prunable_entry_id, regions_to_flush) = mock_test_data(&procedure).await; // Step 1: Test `on_prepare`. let status = procedure.on_prepare().await.unwrap(); @@ -618,7 +577,7 @@ mod tests { } ); assert_matches!(procedure.data.state, WalPruneState::FlushRegion); - assert_eq!(procedure.data.min_flushed_entry_id, min_flushed_entry_id); + assert_eq!(procedure.data.prunable_entry_id, prunable_entry_id); assert_eq!( procedure.data.regions_to_flush.len(), regions_to_flush.len() @@ -646,34 +605,31 @@ mod tests { // Step 3: Test `on_prune`. let status = procedure.on_prune().await.unwrap(); assert_matches!(status, Status::Done { output: None }); - // Check if the entry ids after `min_flushed_entry_id` still exist. + // Check if the entry ids after `prunable_entry_id` still exist. check_entry_id_existence( procedure.context.client.clone(), &topic_name, - procedure.data.min_flushed_entry_id as i64 + 1, + procedure.data.prunable_entry_id as i64 + 1, true, ) .await; - // Check if the entry s before `min_flushed_entry_id` are deleted. + // Check if the entry s before `prunable_entry_id` are deleted. check_entry_id_existence( procedure.context.client.clone(), &topic_name, - procedure.data.min_flushed_entry_id as i64, + procedure.data.prunable_entry_id as i64, false, ) .await; - let min_entry_id = env - .table_metadata_manager() + let value = env + .table_metadata_manager .topic_name_manager() .get(&topic_name) .await .unwrap() .unwrap(); - assert_eq!( - min_entry_id.pruned_entry_id, - procedure.data.min_flushed_entry_id - ); + assert_eq!(value.pruned_entry_id, procedure.data.prunable_entry_id); // Step 4: Test `on_prepare`, `check_heartbeat_collected_region_ids` fails. // Should log a warning and return `Status::Done`. @@ -682,13 +638,10 @@ mod tests { assert_matches!(status, Status::Done { output: None }); // Step 5: Test `on_prepare`, don't flush regions. - procedure.data.trigger_flush_threshold = None; + procedure.data.trigger_flush_threshold = 0; procedure.on_prepare().await.unwrap(); assert_matches!(procedure.data.state, WalPruneState::Prune); - assert_eq!( - min_entry_id.pruned_entry_id, - procedure.data.min_flushed_entry_id - ); + assert_eq!(value.pruned_entry_id, procedure.data.prunable_entry_id); // Clean up the topic. delete_topic(procedure.context.client, &topic_name).await; diff --git a/src/meta-srv/src/procedure/wal_prune/manager.rs b/src/meta-srv/src/procedure/wal_prune/manager.rs new file mode 100644 index 0000000000..8e5072ad11 --- /dev/null +++ b/src/meta-srv/src/procedure/wal_prune/manager.rs @@ -0,0 +1,438 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::hash_set::Entry; +use std::collections::HashSet; +use std::fmt::{Debug, Formatter}; +use std::sync::{Arc, Mutex, RwLock}; +use std::time::Duration; + +use common_meta::key::TableMetadataManagerRef; +use common_meta::leadership_notifier::LeadershipChangeListener; +use common_procedure::{watcher, ProcedureId, ProcedureManagerRef, ProcedureWithId}; +use common_runtime::JoinHandle; +use common_telemetry::{error, info, warn}; +use futures::future::join_all; +use snafu::{OptionExt, ResultExt}; +use tokio::sync::mpsc::{Receiver, Sender}; +use tokio::sync::Semaphore; +use tokio::time::{interval_at, Instant, MissedTickBehavior}; + +use crate::error::{self, Result}; +use crate::metrics::METRIC_META_REMOTE_WAL_PRUNE_EXECUTE; +use crate::procedure::wal_prune::{Context as WalPruneContext, WalPruneProcedure}; + +pub type WalPruneTickerRef = Arc; + +/// Tracks running [WalPruneProcedure]s and the resources they hold. +/// A [WalPruneProcedure] is holding a semaphore permit to limit the number of concurrent procedures. +/// +/// TODO(CookiePie): Similar to [RegionMigrationProcedureTracker], maybe can refactor to a unified framework. +#[derive(Clone)] +pub struct WalPruneProcedureTracker { + running_procedures: Arc>>, +} + +impl WalPruneProcedureTracker { + /// Insert a running [WalPruneProcedure] for the given topic name and + /// consume acquire a semaphore permit for the given topic name. + pub fn insert_running_procedure(&self, topic_name: String) -> Option { + let mut running_procedures = self.running_procedures.write().unwrap(); + match running_procedures.entry(topic_name.clone()) { + Entry::Occupied(_) => None, + Entry::Vacant(entry) => { + entry.insert(); + Some(WalPruneProcedureGuard { + topic_name, + running_procedures: self.running_procedures.clone(), + }) + } + } + } + + /// Number of running [WalPruneProcedure]s. + pub fn len(&self) -> usize { + self.running_procedures.read().unwrap().len() + } +} + +/// [WalPruneProcedureGuard] is a guard for [WalPruneProcedure]. +/// It is used to track the running [WalPruneProcedure]s. +/// When the guard is dropped, it will remove the topic name from the running procedures and release the semaphore. +pub struct WalPruneProcedureGuard { + topic_name: String, + running_procedures: Arc>>, +} + +impl Drop for WalPruneProcedureGuard { + fn drop(&mut self) { + let mut running_procedures = self.running_procedures.write().unwrap(); + running_procedures.remove(&self.topic_name); + } +} + +/// Event is used to notify the [WalPruneManager] to do some work. +/// +/// - `Tick`: Trigger a submission of [WalPruneProcedure] to prune remote WAL. +pub enum Event { + Tick, +} + +impl Debug for Event { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Event::Tick => write!(f, "Tick"), + } + } +} + +/// [WalPruneTicker] is a ticker that periodically sends [Event]s to the [WalPruneManager]. +/// It is used to trigger the [WalPruneManager] to submit [WalPruneProcedure]s. +pub(crate) struct WalPruneTicker { + /// Handle of ticker thread. + pub(crate) tick_handle: Mutex>>, + /// The interval of tick. + pub(crate) tick_interval: Duration, + /// Sends [Event]s. + pub(crate) sender: Sender, +} + +#[async_trait::async_trait] +impl LeadershipChangeListener for WalPruneTicker { + fn name(&self) -> &'static str { + "WalPruneTicker" + } + + async fn on_leader_start(&self) -> common_meta::error::Result<()> { + self.start(); + Ok(()) + } + + async fn on_leader_stop(&self) -> common_meta::error::Result<()> { + self.stop(); + Ok(()) + } +} + +/// TODO(CookiePie): Similar to [RegionSupervisorTicker], maybe can refactor to a unified framework. +impl WalPruneTicker { + pub(crate) fn new(tick_interval: Duration, sender: Sender) -> Self { + Self { + tick_handle: Mutex::new(None), + tick_interval, + sender, + } + } + + /// Starts the ticker. + pub fn start(&self) { + let mut handle = self.tick_handle.lock().unwrap(); + if handle.is_none() { + let sender = self.sender.clone(); + let tick_interval = self.tick_interval; + let ticker_loop = tokio::spawn(async move { + let mut interval = interval_at(Instant::now() + tick_interval, tick_interval); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + interval.tick().await; + if sender.send(Event::Tick).await.is_err() { + info!("EventReceiver is dropped, tick loop is stopped"); + break; + } + } + }); + *handle = Some(ticker_loop); + } + info!("WalPruneTicker started."); + } + + /// Stops the ticker. + pub fn stop(&self) { + let mut handle = self.tick_handle.lock().unwrap(); + if let Some(handle) = handle.take() { + handle.abort(); + } + info!("WalPruneTicker stopped."); + } +} + +impl Drop for WalPruneTicker { + fn drop(&mut self) { + self.stop(); + } +} + +/// [WalPruneManager] manages all remote WAL related tasks in metasrv. +/// +/// [WalPruneManager] is responsible for: +/// 1. Registering [WalPruneProcedure] loader in the procedure manager. +/// 2. Periodically receive [Event::Tick] to submit [WalPruneProcedure] to prune remote WAL. +/// 3. Use a semaphore to limit the number of concurrent [WalPruneProcedure]s. +pub(crate) struct WalPruneManager { + /// Table metadata manager to restore topics from kvbackend. + table_metadata_manager: TableMetadataManagerRef, + /// Receives [Event]s. + receiver: Receiver, + /// Procedure manager. + procedure_manager: ProcedureManagerRef, + /// Tracker for running [WalPruneProcedure]s. + tracker: WalPruneProcedureTracker, + /// Semaphore to limit the number of concurrent [WalPruneProcedure]s. + semaphore: Arc, + + /// Context for [WalPruneProcedure]. + wal_prune_context: WalPruneContext, + /// Trigger flush threshold for [WalPruneProcedure]. + /// If `None`, never send flush requests. + trigger_flush_threshold: u64, +} + +impl WalPruneManager { + /// Returns a new empty [WalPruneManager]. + pub fn new( + table_metadata_manager: TableMetadataManagerRef, + limit: usize, + receiver: Receiver, + procedure_manager: ProcedureManagerRef, + wal_prune_context: WalPruneContext, + trigger_flush_threshold: u64, + ) -> Self { + Self { + table_metadata_manager, + receiver, + procedure_manager, + wal_prune_context, + tracker: WalPruneProcedureTracker { + running_procedures: Arc::new(RwLock::new(HashSet::new())), + }, + semaphore: Arc::new(Semaphore::new(limit)), + trigger_flush_threshold, + } + } + + /// Start the [WalPruneManager]. It will register [WalPruneProcedure] loader in the procedure manager. + pub async fn try_start(mut self) -> Result<()> { + let context = self.wal_prune_context.clone(); + let tracker = self.tracker.clone(); + self.procedure_manager + .register_loader( + WalPruneProcedure::TYPE_NAME, + Box::new(move |json| { + let tracker = tracker.clone(); + WalPruneProcedure::from_json(json, &context, tracker).map(|p| Box::new(p) as _) + }), + ) + .context(error::RegisterProcedureLoaderSnafu { + type_name: WalPruneProcedure::TYPE_NAME, + })?; + common_runtime::spawn_global(async move { + self.run().await; + }); + info!("WalPruneProcedureManager Started."); + Ok(()) + } + + /// Returns a mpsc channel with a buffer capacity of 1024 for sending and receiving `Event` messages. + pub(crate) fn channel() -> (Sender, Receiver) { + tokio::sync::mpsc::channel(1024) + } + + /// Runs the main loop. Performs actions on received events. + /// + /// - `Tick`: Submit `limit` [WalPruneProcedure]s to prune remote WAL. + pub(crate) async fn run(&mut self) { + while let Some(event) = self.receiver.recv().await { + match event { + Event::Tick => self.handle_tick_request().await.unwrap_or_else(|e| { + error!(e; "Failed to handle tick request"); + }), + } + } + } + + /// Submits a [WalPruneProcedure] for the given topic name. + pub async fn submit_procedure(&self, topic_name: &str) -> Result { + let guard = self + .tracker + .insert_running_procedure(topic_name.to_string()) + .with_context(|| error::PruneTaskAlreadyRunningSnafu { topic: topic_name })?; + + let procedure = WalPruneProcedure::new( + topic_name.to_string(), + self.wal_prune_context.clone(), + self.trigger_flush_threshold, + Some(guard), + ); + let procedure_with_id = ProcedureWithId::with_random_id(Box::new(procedure)); + let procedure_id = procedure_with_id.id; + METRIC_META_REMOTE_WAL_PRUNE_EXECUTE + .with_label_values(&[topic_name]) + .inc(); + let procedure_manager = self.procedure_manager.clone(); + let mut watcher = procedure_manager + .submit(procedure_with_id) + .await + .context(error::SubmitProcedureSnafu)?; + watcher::wait(&mut watcher) + .await + .context(error::WaitProcedureSnafu)?; + + Ok(procedure_id) + } + + async fn handle_tick_request(&self) -> Result<()> { + let topics = self.retrieve_sorted_topics().await?; + let mut tasks = Vec::with_capacity(topics.len()); + for topic_name in topics.iter() { + tasks.push(async { + let _permit = self.semaphore.acquire().await.unwrap(); + match self.submit_procedure(topic_name).await { + Ok(_) => {} + Err(error::Error::PruneTaskAlreadyRunning { topic, .. }) => { + warn!("Prune task for topic {} is already running", topic); + } + Err(e) => { + error!( + "Failed to submit prune task for topic {}: {}", + topic_name.clone(), + e + ); + } + } + }); + } + + join_all(tasks).await; + Ok(()) + } + + /// Retrieve topics from the table metadata manager. + /// Since [WalPruneManager] submits procedures depending on the order of the topics, we should sort the topics. + /// TODO(CookiePie): Can register topics in memory instead of retrieving from the table metadata manager every time. + async fn retrieve_sorted_topics(&self) -> Result> { + self.table_metadata_manager + .topic_name_manager() + .range() + .await + .context(error::TableMetadataManagerSnafu) + } +} + +#[cfg(test)] +mod test { + use std::assert_matches::assert_matches; + + use common_meta::key::topic_name::TopicNameKey; + use common_wal::test_util::run_test_with_kafka_wal; + use tokio::time::{sleep, timeout}; + + use super::*; + use crate::procedure::wal_prune::test_util::TestEnv; + + #[tokio::test] + async fn test_wal_prune_ticker() { + let (tx, mut rx) = WalPruneManager::channel(); + let interval = Duration::from_millis(10); + let ticker = WalPruneTicker::new(interval, tx); + assert_eq!(ticker.name(), "WalPruneTicker"); + + for _ in 0..2 { + ticker.start(); + sleep(2 * interval).await; + assert!(!rx.is_empty()); + while let Ok(event) = rx.try_recv() { + assert_matches!(event, Event::Tick); + } + } + ticker.stop(); + } + + #[tokio::test] + async fn test_wal_prune_tracker_and_guard() { + let tracker = WalPruneProcedureTracker { + running_procedures: Arc::new(RwLock::new(HashSet::new())), + }; + let topic_name = uuid::Uuid::new_v4().to_string(); + { + let guard = tracker + .insert_running_procedure(topic_name.clone()) + .unwrap(); + assert_eq!(guard.topic_name, topic_name); + assert_eq!(guard.running_procedures.read().unwrap().len(), 1); + + let result = tracker.insert_running_procedure(topic_name.clone()); + assert!(result.is_none()); + } + assert_eq!(tracker.running_procedures.read().unwrap().len(), 0); + } + + async fn mock_wal_prune_manager( + broker_endpoints: Vec, + limit: usize, + ) -> (Sender, WalPruneManager) { + let test_env = TestEnv::new(); + let (tx, rx) = WalPruneManager::channel(); + let wal_prune_context = test_env.build_wal_prune_context(broker_endpoints).await; + ( + tx, + WalPruneManager::new( + test_env.table_metadata_manager.clone(), + limit, + rx, + test_env.procedure_manager.clone(), + wal_prune_context, + 0, + ), + ) + } + + async fn mock_topics(manager: &WalPruneManager, topics: &[String]) { + let topic_name_keys = topics + .iter() + .map(|topic| TopicNameKey::new(topic)) + .collect::>(); + manager + .table_metadata_manager + .topic_name_manager() + .batch_put(topic_name_keys) + .await + .unwrap(); + } + + #[tokio::test] + async fn test_wal_prune_manager() { + run_test_with_kafka_wal(|broker_endpoints| { + Box::pin(async { + let limit = 6; + let (tx, manager) = mock_wal_prune_manager(broker_endpoints, limit).await; + let topics = (0..limit * 2) + .map(|_| uuid::Uuid::new_v4().to_string()) + .collect::>(); + mock_topics(&manager, &topics).await; + + let tracker = manager.tracker.clone(); + let handler = + common_runtime::spawn_global(async move { manager.try_start().await.unwrap() }); + handler.await.unwrap(); + + tx.send(Event::Tick).await.unwrap(); + // Wait for at least one procedure to be submitted. + timeout(Duration::from_millis(100), async move { tracker.len() > 0 }) + .await + .unwrap(); + }) + }) + .await; + } +} diff --git a/src/meta-srv/src/procedure/wal_prune/test_util.rs b/src/meta-srv/src/procedure/wal_prune/test_util.rs new file mode 100644 index 0000000000..b7cdbad286 --- /dev/null +++ b/src/meta-srv/src/procedure/wal_prune/test_util.rs @@ -0,0 +1,94 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use common_meta::key::{TableMetadataManager, TableMetadataManagerRef}; +use common_meta::kv_backend::memory::MemoryKvBackend; +use common_meta::region_registry::{LeaderRegionRegistry, LeaderRegionRegistryRef}; +use common_meta::sequence::SequenceBuilder; +use common_meta::state_store::KvStateStore; +use common_meta::wal_options_allocator::build_kafka_client; +use common_procedure::local::{LocalManager, ManagerConfig}; +use common_procedure::test_util::InMemoryPoisonStore; +use common_procedure::ProcedureManagerRef; +use common_wal::config::kafka::common::{KafkaConnectionConfig, KafkaTopicConfig}; +use common_wal::config::kafka::MetasrvKafkaConfig; +use rskafka::client::Client; + +use crate::procedure::test_util::MailboxContext; +use crate::procedure::wal_prune::Context as WalPruneContext; + +pub struct TestEnv { + pub table_metadata_manager: TableMetadataManagerRef, + pub leader_region_registry: LeaderRegionRegistryRef, + pub procedure_manager: ProcedureManagerRef, + pub mailbox: MailboxContext, + pub server_addr: String, +} + +impl TestEnv { + pub fn new() -> Self { + let kv_backend = Arc::new(MemoryKvBackend::new()); + let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend.clone())); + let leader_region_registry = Arc::new(LeaderRegionRegistry::new()); + let mailbox_sequence = + SequenceBuilder::new("test_heartbeat_mailbox", kv_backend.clone()).build(); + + let state_store = Arc::new(KvStateStore::new(kv_backend.clone())); + let poison_manager = Arc::new(InMemoryPoisonStore::default()); + let procedure_manager = Arc::new(LocalManager::new( + ManagerConfig::default(), + state_store, + poison_manager, + )); + + let mailbox_ctx = MailboxContext::new(mailbox_sequence); + + Self { + table_metadata_manager, + leader_region_registry, + procedure_manager, + mailbox: mailbox_ctx, + server_addr: "localhost".to_string(), + } + } + + async fn build_kafka_client(broker_endpoints: Vec) -> Arc { + let kafka_topic = KafkaTopicConfig { + replication_factor: broker_endpoints.len() as i16, + ..Default::default() + }; + let config = MetasrvKafkaConfig { + connection: KafkaConnectionConfig { + broker_endpoints, + ..Default::default() + }, + kafka_topic, + ..Default::default() + }; + Arc::new(build_kafka_client(&config).await.unwrap()) + } + + pub async fn build_wal_prune_context(&self, broker_endpoints: Vec) -> WalPruneContext { + let client = Self::build_kafka_client(broker_endpoints).await; + WalPruneContext { + client, + table_metadata_manager: self.table_metadata_manager.clone(), + leader_region_registry: self.leader_region_registry.clone(), + server_addr: self.server_addr.to_string(), + mailbox: self.mailbox.mailbox().clone(), + } + } +} diff --git a/src/meta-srv/src/selector/weight_compute.rs b/src/meta-srv/src/selector/weight_compute.rs index eb35f43f19..904fc1f4e7 100644 --- a/src/meta-srv/src/selector/weight_compute.rs +++ b/src/meta-srv/src/selector/weight_compute.rs @@ -195,6 +195,8 @@ mod tests { manifest_version: 0, flushed_entry_id: 0, }, + data_topic_latest_entry_id: 0, + metadata_topic_latest_entry_id: 0, }], ..Default::default() } @@ -220,6 +222,8 @@ mod tests { manifest_version: 0, flushed_entry_id: 0, }, + data_topic_latest_entry_id: 0, + metadata_topic_latest_entry_id: 0, }], ..Default::default() } @@ -245,6 +249,8 @@ mod tests { manifest_version: 0, flushed_entry_id: 0, }, + data_topic_latest_entry_id: 0, + metadata_topic_latest_entry_id: 0, }], ..Default::default() } diff --git a/tests/conf/metasrv-test.toml.template b/tests/conf/metasrv-test.toml.template index 1196403a26..3daf6150f3 100644 --- a/tests/conf/metasrv-test.toml.template +++ b/tests/conf/metasrv-test.toml.template @@ -18,4 +18,5 @@ broker_endpoints = {kafka_wal_broker_endpoints | unescaped} num_topics = 3 selector_type = "round_robin" topic_name_prefix = "distributed_test_greptimedb_wal_topic" +auto_prune_topic_records = true {{ endif }} From ad1b77ab04934d718bace75d6e3938ffd9b958c4 Mon Sep 17 00:00:00 2001 From: dennis zhuang Date: Mon, 21 Apr 2025 10:44:44 +0800 Subject: [PATCH 44/82] feat: update readme (#5936) * fix: title * chore: format * chore: format * chore: format --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 912183f5bb..b7381a1e4d 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@

-

Unified & Cost-Effective Observability Database for Metrics, Logs, and Events

+

Real-Time & Cloud-Native Observability Database
for metrics, logs, and traces

From 9df493988b40961b398ea6156e15375cb2e3fd51 Mon Sep 17 00:00:00 2001 From: shuiyisong <113876041+shuiyisong@users.noreply.github.com> Date: Mon, 21 Apr 2025 12:05:46 +0800 Subject: [PATCH 45/82] fix: wrong error msg in pipeline (#5937) --- src/pipeline/src/etl/transform/transformer/greptime/coerce.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pipeline/src/etl/transform/transformer/greptime/coerce.rs b/src/pipeline/src/etl/transform/transformer/greptime/coerce.rs index 71c83dc477..41172a2876 100644 --- a/src/pipeline/src/etl/transform/transformer/greptime/coerce.rs +++ b/src/pipeline/src/etl/transform/transformer/greptime/coerce.rs @@ -436,7 +436,8 @@ fn coerce_string_value(s: &String, transform: &Transform) -> Result CoerceUnsupportedEpochTypeSnafu { ty: "String" }.fail(), }, - Value::Array(_) | Value::Map(_) => CoerceJsonTypeToSnafu { + Value::Array(_) | Value::Map(_) => CoerceStringToTypeSnafu { + s, ty: transform.type_.to_str_type(), } .fail(), From 56f319a7070cb6e8542eca66d800144b6fadf126 Mon Sep 17 00:00:00 2001 From: Yingwen Date: Mon, 21 Apr 2025 14:32:26 +0800 Subject: [PATCH 46/82] fix: filter doesn't consider default values after schema change (#5912) * test: sqlness test case * feat: use correct default while pruning row groups * fix: consider default in SimpleFilterContext * test: update sqlness test * test: add order by --- src/mito2/src/sst/parquet/file_range.rs | 29 ++++---- src/mito2/src/sst/parquet/reader.rs | 66 +++++++++++++++---- src/mito2/src/sst/parquet/stats.rs | 50 ++++++++++++-- .../common/alter/add_col_default.result | 39 ++++++++++- .../common/alter/add_col_default.sql | 16 ++++- 5 files changed, 171 insertions(+), 29 deletions(-) diff --git a/src/mito2/src/sst/parquet/file_range.rs b/src/mito2/src/sst/parquet/file_range.rs index e8241a453f..e8dbca04fb 100644 --- a/src/mito2/src/sst/parquet/file_range.rs +++ b/src/mito2/src/sst/parquet/file_range.rs @@ -36,7 +36,9 @@ use crate::read::Batch; use crate::row_converter::{CompositeValues, PrimaryKeyCodec}; use crate::sst::file::FileHandle; use crate::sst::parquet::format::ReadFormat; -use crate::sst::parquet::reader::{RowGroupReader, RowGroupReaderBuilder, SimpleFilterContext}; +use crate::sst::parquet::reader::{ + MaybeFilter, RowGroupReader, RowGroupReaderBuilder, SimpleFilterContext, +}; /// A range of a parquet SST. Now it is a row group. /// We can read different file ranges in parallel. @@ -255,8 +257,15 @@ impl RangeBase { // Run filter one by one and combine them result // TODO(ruihang): run primary key filter first. It may short circuit other filters - for filter in &self.filters { - let result = match filter.semantic_type() { + for filter_ctx in &self.filters { + let filter = match filter_ctx.filter() { + MaybeFilter::Filter(f) => f, + // Column matches. + MaybeFilter::Matched => continue, + // Column doesn't match, filter the entire batch. + MaybeFilter::Pruned => return Ok(None), + }; + let result = match filter_ctx.semantic_type() { SemanticType::Tag => { let pk_values = if let Some(pk_values) = input.pk_values() { pk_values @@ -270,21 +279,20 @@ impl RangeBase { let pk_index = self .read_format .metadata() - .primary_key_index(filter.column_id()) + .primary_key_index(filter_ctx.column_id()) .unwrap(); v[pk_index] .1 - .try_to_scalar_value(filter.data_type()) + .try_to_scalar_value(filter_ctx.data_type()) .context(FieldTypeMismatchSnafu)? } CompositeValues::Sparse(v) => { - let v = v.get_or_null(filter.column_id()); - v.try_to_scalar_value(filter.data_type()) + let v = v.get_or_null(filter_ctx.column_id()); + v.try_to_scalar_value(filter_ctx.data_type()) .context(FieldTypeMismatchSnafu)? } }; if filter - .filter() .evaluate_scalar(&pk_value) .context(FilterRecordBatchSnafu)? { @@ -295,18 +303,17 @@ impl RangeBase { } } SemanticType::Field => { - let Some(field_index) = self.read_format.field_index_by_id(filter.column_id()) + let Some(field_index) = + self.read_format.field_index_by_id(filter_ctx.column_id()) else { continue; }; let field_col = &input.fields()[field_index].data; filter - .filter() .evaluate_vector(field_col) .context(FilterRecordBatchSnafu)? } SemanticType::Timestamp => filter - .filter() .evaluate_vector(input.timestamps()) .context(FilterRecordBatchSnafu)?, }; diff --git a/src/mito2/src/sst/parquet/reader.rs b/src/mito2/src/sst/parquet/reader.rs index 5c2ab17591..ffa5c5d003 100644 --- a/src/mito2/src/sst/parquet/reader.rs +++ b/src/mito2/src/sst/parquet/reader.rs @@ -34,7 +34,7 @@ use parquet::arrow::{parquet_to_arrow_field_levels, FieldLevels, ProjectionMask} use parquet::file::metadata::ParquetMetaData; use parquet::format::KeyValue; use snafu::{OptionExt, ResultExt}; -use store_api::metadata::{RegionMetadata, RegionMetadataRef}; +use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataRef}; use store_api::storage::ColumnId; use table::predicate::Predicate; @@ -191,6 +191,7 @@ impl ParquetReaderBuilder { let file_path = self.file_handle.file_path(&self.file_dir); let file_size = self.file_handle.meta_ref().file_size; + // Loads parquet metadata of the file. let parquet_meta = self.read_parquet_metadata(&file_path, file_size).await?; // Decodes region metadata. @@ -550,11 +551,17 @@ impl ParquetReaderBuilder { let row_groups = parquet_meta.row_groups(); let stats = RowGroupPruningStats::new(row_groups, read_format, self.expected_metadata.clone()); + let prune_schema = self + .expected_metadata + .as_ref() + .map(|meta| meta.schema.arrow_schema()) + .unwrap_or_else(|| region_meta.schema.arrow_schema()); + // Here we use the schema of the SST to build the physical expression. If the column // in the SST doesn't have the same column id as the column in the expected metadata, // we will get a None statistics for that column. let res = predicate - .prune_with_stats(&stats, region_meta.schema.arrow_schema()) + .prune_with_stats(&stats, prune_schema) .iter() .zip(0..parquet_meta.num_row_groups()) .filter_map(|(mask, row_group)| { @@ -1009,10 +1016,20 @@ impl ReaderState { } } -/// Context to evaluate the column filter. +/// The filter to evaluate or the prune result of the default value. +pub(crate) enum MaybeFilter { + /// The filter to evaluate. + Filter(SimpleFilterEvaluator), + /// The filter matches the default value. + Matched, + /// The filter is pruned. + Pruned, +} + +/// Context to evaluate the column filter for a parquet file. pub(crate) struct SimpleFilterContext { /// Filter to evaluate. - filter: SimpleFilterEvaluator, + filter: MaybeFilter, /// Id of the column to evaluate. column_id: ColumnId, /// Semantic type of the column. @@ -1032,22 +1049,38 @@ impl SimpleFilterContext { expr: &Expr, ) -> Option { let filter = SimpleFilterEvaluator::try_new(expr)?; - let column_metadata = match expected_meta { + let (column_metadata, maybe_filter) = match expected_meta { Some(meta) => { // Gets the column metadata from the expected metadata. let column = meta.column_by_name(filter.column_name())?; // Checks if the column is present in the SST metadata. We still uses the // column from the expected metadata. - let sst_column = sst_meta.column_by_id(column.column_id)?; - debug_assert_eq!(column.semantic_type, sst_column.semantic_type); + match sst_meta.column_by_id(column.column_id) { + Some(sst_column) => { + debug_assert_eq!(column.semantic_type, sst_column.semantic_type); - column + (column, MaybeFilter::Filter(filter)) + } + None => { + // If the column is not present in the SST metadata, we evaluate the filter + // against the default value of the column. + // If we can't evaluate the filter, we return None. + if pruned_by_default(&filter, column)? { + (column, MaybeFilter::Pruned) + } else { + (column, MaybeFilter::Matched) + } + } + } + } + None => { + let column = sst_meta.column_by_name(filter.column_name())?; + (column, MaybeFilter::Filter(filter)) } - None => sst_meta.column_by_name(filter.column_name())?, }; Some(Self { - filter, + filter: maybe_filter, column_id: column_metadata.column_id, semantic_type: column_metadata.semantic_type, data_type: column_metadata.column_schema.data_type.clone(), @@ -1055,7 +1088,7 @@ impl SimpleFilterContext { } /// Returns the filter to evaluate. - pub(crate) fn filter(&self) -> &SimpleFilterEvaluator { + pub(crate) fn filter(&self) -> &MaybeFilter { &self.filter } @@ -1075,6 +1108,17 @@ impl SimpleFilterContext { } } +/// Prune a column by its default value. +/// Returns false if we can't create the default value or evaluate the filter. +fn pruned_by_default(filter: &SimpleFilterEvaluator, column: &ColumnMetadata) -> Option { + let value = column.column_schema.create_default().ok().flatten()?; + let scalar_value = value + .try_to_scalar_value(&column.column_schema.data_type) + .ok()?; + let matches = filter.evaluate_scalar(&scalar_value).ok()?; + Some(!matches) +} + type RowGroupMap = BTreeMap>; /// Parquet batch reader to read our SST format. diff --git a/src/mito2/src/sst/parquet/stats.rs b/src/mito2/src/sst/parquet/stats.rs index 09b837698c..ead3679397 100644 --- a/src/mito2/src/sst/parquet/stats.rs +++ b/src/mito2/src/sst/parquet/stats.rs @@ -16,10 +16,11 @@ use std::borrow::Borrow; use std::collections::HashSet; +use std::sync::Arc; use datafusion::physical_optimizer::pruning::PruningStatistics; use datafusion_common::{Column, ScalarValue}; -use datatypes::arrow::array::{ArrayRef, BooleanArray}; +use datatypes::arrow::array::{ArrayRef, BooleanArray, UInt64Array}; use parquet::file::metadata::RowGroupMetaData; use store_api::metadata::RegionMetadataRef; use store_api::storage::ColumnId; @@ -54,25 +55,62 @@ impl<'a, T> RowGroupPruningStats<'a, T> { } /// Returns the column id of specific column name if we need to read it. + /// Prefers the column id in the expected metadata if it exists. fn column_id_to_prune(&self, name: &str) -> Option { let metadata = self .expected_metadata .as_ref() .unwrap_or_else(|| self.read_format.metadata()); - // Only use stats when the column to read has the same id as the column in the SST. metadata.column_by_name(name).map(|col| col.column_id) } + + /// Returns the default value of all row groups for `column` according to the metadata. + fn compat_default_value(&self, column: &str) -> Option { + let metadata = self.expected_metadata.as_ref()?; + let col_metadata = metadata.column_by_name(column)?; + col_metadata + .column_schema + .create_default_vector(self.row_groups.len()) + .unwrap_or(None) + .map(|vector| vector.to_arrow_array()) + } +} + +impl> RowGroupPruningStats<'_, T> { + /// Returns the null count of all row groups for `column` according to the metadata. + fn compat_null_count(&self, column: &str) -> Option { + let metadata = self.expected_metadata.as_ref()?; + let col_metadata = metadata.column_by_name(column)?; + let value = col_metadata + .column_schema + .create_default() + .unwrap_or(None)?; + let values = self.row_groups.iter().map(|meta| { + if value.is_null() { + u64::try_from(meta.borrow().num_rows()).ok() + } else { + Some(0) + } + }); + Some(Arc::new(UInt64Array::from_iter(values))) + } } impl> PruningStatistics for RowGroupPruningStats<'_, T> { fn min_values(&self, column: &Column) -> Option { let column_id = self.column_id_to_prune(&column.name)?; - self.read_format.min_values(self.row_groups, column_id) + match self.read_format.min_values(self.row_groups, column_id) { + Some(values) => Some(values), + None => self.compat_default_value(&column.name), + } } fn max_values(&self, column: &Column) -> Option { let column_id = self.column_id_to_prune(&column.name)?; - self.read_format.max_values(self.row_groups, column_id) + match self.read_format.max_values(self.row_groups, column_id) { + Some(values) => Some(values), + None => self.compat_default_value(&column.name), + } } fn num_containers(&self) -> usize { @@ -80,7 +118,9 @@ impl> PruningStatistics for RowGroupPruningStats<'_, } fn null_counts(&self, column: &Column) -> Option { - let column_id = self.column_id_to_prune(&column.name)?; + let Some(column_id) = self.column_id_to_prune(&column.name) else { + return self.compat_null_count(&column.name); + }; self.read_format.null_counts(self.row_groups, column_id) } diff --git a/tests/cases/standalone/common/alter/add_col_default.result b/tests/cases/standalone/common/alter/add_col_default.result index 6d8c523ba3..5a9baf7186 100644 --- a/tests/cases/standalone/common/alter/add_col_default.result +++ b/tests/cases/standalone/common/alter/add_col_default.result @@ -6,19 +6,56 @@ INSERT INTO test VALUES (1, 1), (2, 2); Affected Rows: 2 +ADMIN FLUSH_TABLE('test'); + ++---------------------------+ +| ADMIN FLUSH_TABLE('test') | ++---------------------------+ +| 0 | ++---------------------------+ + +ALTER TABLE test MODIFY COLUMN i SET INVERTED INDEX; + +Affected Rows: 0 + +INSERT INTO test VALUES (3, 3), (4, 4); + +Affected Rows: 2 + ALTER TABLE test ADD COLUMN k INTEGER DEFAULT 3; Affected Rows: 0 -SELECT * FROM test; +SELECT * FROM test order by j; +---+-------------------------+---+ | i | j | k | +---+-------------------------+---+ | 1 | 1970-01-01T00:00:00.001 | 3 | | 2 | 1970-01-01T00:00:00.002 | 3 | +| 3 | 1970-01-01T00:00:00.003 | 3 | +| 4 | 1970-01-01T00:00:00.004 | 3 | +---+-------------------------+---+ +SELECT * FROM test where k != 3; + +++ +++ + +ALTER TABLE test ADD COLUMN host STRING DEFAULT '' PRIMARY KEY; + +Affected Rows: 0 + +SELECT * FROM test where host != ''; + +++ +++ + +SELECT * FROM test where host != '' AND i = 3; + +++ +++ + DROP TABLE test; Affected Rows: 0 diff --git a/tests/cases/standalone/common/alter/add_col_default.sql b/tests/cases/standalone/common/alter/add_col_default.sql index 187e96821e..258ec1683a 100644 --- a/tests/cases/standalone/common/alter/add_col_default.sql +++ b/tests/cases/standalone/common/alter/add_col_default.sql @@ -2,8 +2,22 @@ CREATE TABLE test(i INTEGER, j TIMESTAMP TIME INDEX); INSERT INTO test VALUES (1, 1), (2, 2); +ADMIN FLUSH_TABLE('test'); + +ALTER TABLE test MODIFY COLUMN i SET INVERTED INDEX; + +INSERT INTO test VALUES (3, 3), (4, 4); + ALTER TABLE test ADD COLUMN k INTEGER DEFAULT 3; -SELECT * FROM test; +SELECT * FROM test order by j; + +SELECT * FROM test where k != 3; + +ALTER TABLE test ADD COLUMN host STRING DEFAULT '' PRIMARY KEY; + +SELECT * FROM test where host != ''; + +SELECT * FROM test where host != '' AND i = 3; DROP TABLE test; From 90ffaa8a62290cc38171fffb512da6c6e5f5d79b Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" <6406592+v0y4g3r@users.noreply.github.com> Date: Mon, 21 Apr 2025 15:24:23 +0800 Subject: [PATCH 47/82] feat: implement otel-arrow protocol for GreptimeDB (#5840) * [wip]: implement arrow service * add service * feat/otel-arrow: ### Add OpenTelemetry Arrow Support - **`Cargo.toml`, `Cargo.lock`**: Updated `otel-arrow-rust` dependency to use a local path and added `arrow-ipc` as a dependency. - **`src/servers/src/grpc.rs`, `src/servers/src/grpc/builder.rs`**: Integrated `ArrowMetricsServiceServer` with gRPC server, including support for custom header interception and message compression. - **`src/servers/src/otel_arrow.rs`**: Implemented `OtelArrowServiceHandler` for handling OpenTelemetry Arrow metrics and added `HeaderInterceptor` for custom header handling. * feat/otel-arrow: Add error handling for OpenTelemetry Arrow requests - **`src/error.rs`**: Introduced a new error variant `HandleOtelArrowRequest` to handle failures in processing OpenTelemetry Arrow requests. - **`src/otel_arrow.rs`**: Implemented error handling for receiving and consuming batches from the OpenTelemetry Arrow client. Added logging for errors and updated the response status accordingly. * feat/otel-arrow: Remove `otel_arrow` Module from gRPC Server - Deleted the `otel_arrow` module from the gRPC server implementation. - Removed the `otel_arrow` module import from `grpc.rs`. - Deleted the `otel_arrow.rs` file, which contained the `OtelArrowServer` struct and its implementation. * feat/otel-arrow: ## Remove `Arc` Implementations for Protocol and Pipeline Handlers - **Removed `Arc` Implementations**: Deleted `Arc` implementations for `OpenTelemetryProtocolHandler` and `PipelineHandler` traits in `query_handler.rs`. This change simplifies the code by removing redundant async trait implementations for `Arc`. - **File Affected**: `src/servers/src/query_handler.rs` * feat/otel-arrow: Improve error handling and metadata processing in `otel_arrow.rs` - Updated error handling by ignoring the result of `sender.send` to prevent panic on failure. - Enhanced metadata processing in `HeaderInterceptor` by using `Ok` to safely handle `grpc-encoding` entry retrieval. * fix dependency * feat/otel-arrow: - **Update Dependencies**: - Moved `otel-arrow-rust` dependency in `Cargo.toml`. - Adjusted workspace dependencies in `src/frontend/Cargo.toml`. - **Error Handling**: - Removed `MissingQueryContext` error variant from `src/servers/src/error.rs`. * fix: toml format * remove useless code * chore: resolve conflicts --- Cargo.lock | 504 ++++++++++++++++++++++++-------- Cargo.toml | 3 + src/frontend/Cargo.toml | 2 + src/frontend/src/server.rs | 2 + src/servers/Cargo.toml | 1 + src/servers/src/error.rs | 17 +- src/servers/src/grpc.rs | 20 +- src/servers/src/grpc/builder.rs | 29 ++ src/servers/src/grpc/otlp.rs | 97 ------ src/servers/src/lib.rs | 1 + src/servers/src/otel_arrow.rs | 119 ++++++++ 11 files changed, 566 insertions(+), 229 deletions(-) delete mode 100644 src/servers/src/grpc/otlp.rs create mode 100644 src/servers/src/otel_arrow.rs diff --git a/Cargo.lock b/Cargo.lock index 520b33c929..a98826107b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -266,25 +266,61 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "arrow" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3a3ec4fe573f9d1f59d99c085197ef669b00b088ba1d7bb75224732d9357a74" +dependencies = [ + "arrow-arith 53.4.1", + "arrow-array 53.4.1", + "arrow-buffer 53.4.1", + "arrow-cast 53.4.1", + "arrow-csv 53.4.1", + "arrow-data 53.4.1", + "arrow-ipc 53.4.1", + "arrow-json 53.4.1", + "arrow-ord 53.4.1", + "arrow-row 53.4.1", + "arrow-schema 53.4.1", + "arrow-select 53.4.1", + "arrow-string 53.4.1", +] + [[package]] name = "arrow" version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc208515aa0151028e464cc94a692156e945ce5126abd3537bb7fd6ba2143ed1" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-csv", - "arrow-data", - "arrow-ipc", - "arrow-json", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-arith 54.2.1", + "arrow-array 54.2.1", + "arrow-buffer 54.3.1", + "arrow-cast 54.2.1", + "arrow-csv 54.2.1", + "arrow-data 54.3.1", + "arrow-ipc 54.2.1", + "arrow-json 54.2.1", + "arrow-ord 54.2.1", + "arrow-row 54.2.1", + "arrow-schema 54.3.1", + "arrow-select 54.2.1", + "arrow-string 54.2.1", +] + +[[package]] +name = "arrow-arith" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dcf19f07792d8c7f91086c67b574a79301e367029b17fcf63fb854332246a10" +dependencies = [ + "arrow-array 53.4.1", + "arrow-buffer 53.4.1", + "arrow-data 53.4.1", + "arrow-schema 53.4.1", + "chrono", + "half", + "num", ] [[package]] @@ -293,14 +329,30 @@ version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e07e726e2b3f7816a85c6a45b6ec118eeeabf0b2a8c208122ad949437181f49a" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 54.2.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", "chrono", "num", ] +[[package]] +name = "arrow-array" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7845c32b41f7053e37a075b3c2f29c6f5ea1b3ca6e5df7a2d325ee6e1b4a63cf" +dependencies = [ + "ahash 0.8.11", + "arrow-buffer 53.4.1", + "arrow-data 53.4.1", + "arrow-schema 53.4.1", + "chrono", + "half", + "hashbrown 0.15.2", + "num", +] + [[package]] name = "arrow-array" version = "54.2.1" @@ -308,9 +360,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2262eba4f16c78496adfd559a29fe4b24df6088efc9985a873d58e92be022d5" dependencies = [ "ahash 0.8.11", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", "chrono", "chrono-tz", "half", @@ -318,6 +370,17 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-buffer" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b5c681a99606f3316f2a99d9c8b6fa3aad0b1d34d8f6d7a1b471893940219d8" +dependencies = [ + "bytes", + "half", + "num", +] + [[package]] name = "arrow-buffer" version = "54.3.1" @@ -329,17 +392,37 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-cast" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6365f8527d4f87b133eeb862f9b8093c009d41a210b8f101f91aa2392f61daac" +dependencies = [ + "arrow-array 53.4.1", + "arrow-buffer 53.4.1", + "arrow-data 53.4.1", + "arrow-schema 53.4.1", + "arrow-select 53.4.1", + "atoi", + "base64 0.22.1", + "chrono", + "half", + "lexical-core", + "num", + "ryu", +] + [[package]] name = "arrow-cast" version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4103d88c5b441525ed4ac23153be7458494c2b0c9a11115848fdb9b81f6f886a" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 54.2.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "arrow-select 54.2.1", "atoi", "base64 0.22.1", "chrono", @@ -350,15 +433,34 @@ dependencies = [ "ryu", ] +[[package]] +name = "arrow-csv" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30dac4d23ac769300349197b845e0fd18c7f9f15d260d4659ae6b5a9ca06f586" +dependencies = [ + "arrow-array 53.4.1", + "arrow-buffer 53.4.1", + "arrow-cast 53.4.1", + "arrow-data 53.4.1", + "arrow-schema 53.4.1", + "chrono", + "csv", + "csv-core", + "lazy_static", + "lexical-core", + "regex", +] + [[package]] name = "arrow-csv" version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43d3cb0914486a3cae19a5cad2598e44e225d53157926d0ada03c20521191a65" dependencies = [ - "arrow-array", - "arrow-cast", - "arrow-schema", + "arrow-array 54.2.1", + "arrow-cast 54.2.1", + "arrow-schema 54.3.1", "chrono", "csv", "csv-core", @@ -366,14 +468,26 @@ dependencies = [ "regex", ] +[[package]] +name = "arrow-data" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd962fc3bf7f60705b25bcaa8eb3318b2545aa1d528656525ebdd6a17a6cd6fb" +dependencies = [ + "arrow-buffer 53.4.1", + "arrow-schema 53.4.1", + "half", + "num", +] + [[package]] name = "arrow-data" version = "54.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61cfdd7d99b4ff618f167e548b2411e5dd2c98c0ddebedd7df433d34c20a4429" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 54.3.1", + "arrow-schema 54.3.1", "half", "num", ] @@ -384,11 +498,11 @@ version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7408f2bf3b978eddda272c7699f439760ebc4ac70feca25fefa82c5b8ce808d" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-ipc", - "arrow-schema", + "arrow-array 54.2.1", + "arrow-buffer 54.3.1", + "arrow-cast 54.2.1", + "arrow-ipc 54.2.1", + "arrow-schema 54.3.1", "base64 0.22.1", "bytes", "futures", @@ -397,32 +511,67 @@ dependencies = [ "tonic 0.12.3", ] +[[package]] +name = "arrow-ipc" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3527365b24372f9c948f16e53738eb098720eea2093ae73c7af04ac5e30a39b" +dependencies = [ + "arrow-array 53.4.1", + "arrow-buffer 53.4.1", + "arrow-cast 53.4.1", + "arrow-data 53.4.1", + "arrow-schema 53.4.1", + "flatbuffers", + "zstd 0.13.2", +] + [[package]] name = "arrow-ipc" version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddecdeab02491b1ce88885986e25002a3da34dd349f682c7cfe67bab7cc17b86" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 54.2.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", "flatbuffers", "lz4_flex", "zstd 0.13.2", ] +[[package]] +name = "arrow-json" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdec0024749fc0d95e025c0b0266d78613727b3b3a5d4cf8ea47eb6d38afdd1" +dependencies = [ + "arrow-array 53.4.1", + "arrow-buffer 53.4.1", + "arrow-cast 53.4.1", + "arrow-data 53.4.1", + "arrow-schema 53.4.1", + "chrono", + "half", + "indexmap 2.9.0", + "lexical-core", + "num", + "serde", + "serde_json", +] + [[package]] name = "arrow-json" version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d03b9340013413eb84868682ace00a1098c81a5ebc96d279f7ebf9a4cac3c0fd" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", + "arrow-array 54.2.1", + "arrow-buffer 54.3.1", + "arrow-cast 54.2.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", "chrono", "half", "indexmap 2.9.0", @@ -432,17 +581,46 @@ dependencies = [ "serde_json", ] +[[package]] +name = "arrow-ord" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79af2db0e62a508d34ddf4f76bfd6109b6ecc845257c9cba6f939653668f89ac" +dependencies = [ + "arrow-array 53.4.1", + "arrow-buffer 53.4.1", + "arrow-data 53.4.1", + "arrow-schema 53.4.1", + "arrow-select 53.4.1", + "half", + "num", +] + [[package]] name = "arrow-ord" version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f841bfcc1997ef6ac48ee0305c4dfceb1f7c786fe31e67c1186edf775e1f1160" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 54.2.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "arrow-select 54.2.1", +] + +[[package]] +name = "arrow-row" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da30e9d10e9c52f09ea0cf15086d6d785c11ae8dcc3ea5f16d402221b6ac7735" +dependencies = [ + "ahash 0.8.11", + "arrow-array 53.4.1", + "arrow-buffer 53.4.1", + "arrow-data 53.4.1", + "arrow-schema 53.4.1", + "half", ] [[package]] @@ -451,13 +629,19 @@ version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1eeb55b0a0a83851aa01f2ca5ee5648f607e8506ba6802577afdda9d75cdedcd" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 54.2.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", "half", ] +[[package]] +name = "arrow-schema" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35b0f9c0c3582dd55db0f136d3b44bfa0189df07adcf7dc7f2f2e74db0f52eb8" + [[package]] name = "arrow-schema" version = "54.3.1" @@ -467,6 +651,20 @@ dependencies = [ "serde", ] +[[package]] +name = "arrow-select" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92fc337f01635218493c23da81a364daf38c694b05fc20569c3193c11c561984" +dependencies = [ + "ahash 0.8.11", + "arrow-array 53.4.1", + "arrow-buffer 53.4.1", + "arrow-data 53.4.1", + "arrow-schema 53.4.1", + "num", +] + [[package]] name = "arrow-select" version = "54.2.1" @@ -474,24 +672,41 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e2932aece2d0c869dd2125feb9bd1709ef5c445daa3838ac4112dcfa0fda52c" dependencies = [ "ahash 0.8.11", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 54.2.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", "num", ] +[[package]] +name = "arrow-string" +version = "53.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d596a9fc25dae556672d5069b090331aca8acb93cae426d8b7dcdf1c558fa0ce" +dependencies = [ + "arrow-array 53.4.1", + "arrow-buffer 53.4.1", + "arrow-data 53.4.1", + "arrow-schema 53.4.1", + "arrow-select 53.4.1", + "memchr", + "num", + "regex", + "regex-syntax 0.8.5", +] + [[package]] name = "arrow-string" version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "912e38bd6a7a7714c1d9b61df80315685553b7455e8a6045c27531d8ecd5b458" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 54.2.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "arrow-select 54.2.1", "memchr", "num", "regex", @@ -1349,8 +1564,8 @@ name = "catalog" version = "0.14.0" dependencies = [ "api", - "arrow", - "arrow-schema", + "arrow 54.2.1", + "arrow-schema 54.3.1", "async-stream", "async-trait", "bytes", @@ -1940,8 +2155,8 @@ dependencies = [ name = "common-datasource" version = "0.14.0" dependencies = [ - "arrow", - "arrow-schema", + "arrow 54.2.1", + "arrow-schema 54.3.1", "async-compression 0.3.15", "async-trait", "bytes", @@ -2403,7 +2618,7 @@ dependencies = [ name = "common-time" version = "0.14.0" dependencies = [ - "arrow", + "arrow 54.2.1", "chrono", "chrono-tz", "common-error", @@ -2904,10 +3119,10 @@ name = "datafusion" version = "45.0.0" source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ - "arrow", - "arrow-array", - "arrow-ipc", - "arrow-schema", + "arrow 54.2.1", + "arrow-array 54.2.1", + "arrow-ipc 54.2.1", + "arrow-schema 54.3.1", "async-compression 0.4.13", "async-trait", "bytes", @@ -2955,7 +3170,7 @@ name = "datafusion-catalog" version = "45.0.0" source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ - "arrow", + "arrow 54.2.1", "async-trait", "dashmap", "datafusion-common", @@ -2975,8 +3190,8 @@ name = "datafusion-catalog-listing" version = "45.0.0" source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ - "arrow", - "arrow-schema", + "arrow 54.2.1", + "arrow-schema 54.3.1", "chrono", "datafusion-catalog", "datafusion-common", @@ -2999,10 +3214,10 @@ version = "45.0.0" source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "ahash 0.8.11", - "arrow", - "arrow-array", - "arrow-ipc", - "arrow-schema", + "arrow 54.2.1", + "arrow-array 54.2.1", + "arrow-ipc 54.2.1", + "arrow-schema 54.3.1", "base64 0.22.1", "half", "hashbrown 0.14.5", @@ -3037,7 +3252,7 @@ name = "datafusion-execution" version = "45.0.0" source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ - "arrow", + "arrow 54.2.1", "dashmap", "datafusion-common", "datafusion-expr", @@ -3055,7 +3270,7 @@ name = "datafusion-expr" version = "45.0.0" source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ - "arrow", + "arrow 54.2.1", "chrono", "datafusion-common", "datafusion-doc", @@ -3075,7 +3290,7 @@ name = "datafusion-expr-common" version = "45.0.0" source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ - "arrow", + "arrow 54.2.1", "datafusion-common", "itertools 0.14.0", "paste", @@ -3086,8 +3301,8 @@ name = "datafusion-functions" version = "45.0.0" source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ - "arrow", - "arrow-buffer", + "arrow 54.2.1", + "arrow-buffer 54.3.1", "base64 0.22.1", "blake2", "blake3", @@ -3116,8 +3331,8 @@ version = "45.0.0" source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "ahash 0.8.11", - "arrow", - "arrow-schema", + "arrow 54.2.1", + "arrow-schema 54.3.1", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -3137,7 +3352,7 @@ version = "45.0.0" source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "ahash 0.8.11", - "arrow", + "arrow 54.2.1", "datafusion-common", "datafusion-expr-common", "datafusion-physical-expr-common", @@ -3148,10 +3363,10 @@ name = "datafusion-functions-nested" version = "45.0.0" source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ - "arrow", - "arrow-array", - "arrow-ord", - "arrow-schema", + "arrow 54.2.1", + "arrow-array 54.2.1", + "arrow-ord 54.2.1", + "arrow-schema 54.3.1", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -3170,7 +3385,7 @@ name = "datafusion-functions-table" version = "45.0.0" source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ - "arrow", + "arrow 54.2.1", "async-trait", "datafusion-catalog", "datafusion-common", @@ -3220,7 +3435,7 @@ name = "datafusion-optimizer" version = "45.0.0" source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ - "arrow", + "arrow 54.2.1", "chrono", "datafusion-common", "datafusion-expr", @@ -3239,9 +3454,9 @@ version = "45.0.0" source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "ahash 0.8.11", - "arrow", - "arrow-array", - "arrow-schema", + "arrow 54.2.1", + "arrow-array 54.2.1", + "arrow-schema 54.3.1", "datafusion-common", "datafusion-expr", "datafusion-expr-common", @@ -3262,7 +3477,7 @@ version = "45.0.0" source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "ahash 0.8.11", - "arrow", + "arrow 54.2.1", "datafusion-common", "datafusion-expr-common", "hashbrown 0.14.5", @@ -3274,8 +3489,8 @@ name = "datafusion-physical-optimizer" version = "45.0.0" source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ - "arrow", - "arrow-schema", + "arrow 54.2.1", + "arrow-schema 54.3.1", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -3296,10 +3511,10 @@ version = "45.0.0" source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ "ahash 0.8.11", - "arrow", - "arrow-array", - "arrow-ord", - "arrow-schema", + "arrow 54.2.1", + "arrow-array 54.2.1", + "arrow-ord 54.2.1", + "arrow-schema 54.3.1", "async-trait", "chrono", "datafusion-common", @@ -3325,9 +3540,9 @@ name = "datafusion-sql" version = "45.0.0" source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" dependencies = [ - "arrow", - "arrow-array", - "arrow-schema", + "arrow 54.2.1", + "arrow-array 54.2.1", + "arrow-schema 54.3.1", "bigdecimal 0.4.8", "datafusion-common", "datafusion-expr", @@ -3420,9 +3635,9 @@ dependencies = [ name = "datatypes" version = "0.14.0" dependencies = [ - "arrow", - "arrow-array", - "arrow-schema", + "arrow 54.2.1", + "arrow-array 54.2.1", + "arrow-schema 54.3.1", "base64 0.22.1", "common-base", "common-decimal", @@ -4170,8 +4385,8 @@ name = "flow" version = "0.14.0" dependencies = [ "api", - "arrow", - "arrow-schema", + "arrow 54.2.1", + "arrow-schema 54.3.1", "async-recursion", "async-trait", "bytes", @@ -4324,6 +4539,7 @@ dependencies = [ "num_cpus", "opentelemetry-proto 0.27.0", "operator", + "otel-arrow-rust", "partition", "pipeline", "prometheus", @@ -7546,6 +7762,27 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" +dependencies = [ + "proc-macro-crate 1.3.1", + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "num_threads" version = "0.1.7" @@ -7940,7 +8177,7 @@ name = "orc-rust" version = "0.6.0" source = "git+https://github.com/datafusion-contrib/orc-rust?rev=3134cab581a8e91b942d6a23aca2916ea965f6bb#3134cab581a8e91b942d6a23aca2916ea965f6bb" dependencies = [ - "arrow", + "arrow 54.2.1", "async-trait", "bytemuck", "bytes", @@ -8026,6 +8263,24 @@ version = "6.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" +[[package]] +name = "otel-arrow-rust" +version = "0.1.0" +source = "git+https://github.com/open-telemetry/otel-arrow?rev=5d551412d2a12e689cde4d84c14ef29e36784e51#5d551412d2a12e689cde4d84c14ef29e36784e51" +dependencies = [ + "arrow 53.4.1", + "arrow-ipc 53.4.1", + "lazy_static", + "num_enum", + "opentelemetry-proto 0.27.0", + "paste", + "prost 0.13.5", + "serde", + "snafu 0.8.5", + "tonic 0.12.3", + "tonic-build 0.12.3", +] + [[package]] name = "overload" version = "0.1.1" @@ -8124,13 +8379,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f88838dca3b84d41444a0341b19f347e8098a3898b0f21536654b8b799e11abd" dependencies = [ "ahash 0.8.11", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ipc", - "arrow-schema", - "arrow-select", + "arrow-array 54.2.1", + "arrow-buffer 54.3.1", + "arrow-cast 54.2.1", + "arrow-data 54.3.1", + "arrow-ipc 54.2.1", + "arrow-schema 54.3.1", + "arrow-select 54.2.1", "base64 0.22.1", "brotli", "bytes", @@ -8451,7 +8706,7 @@ version = "0.14.0" dependencies = [ "ahash 0.8.11", "api", - "arrow", + "arrow 54.2.1", "async-trait", "catalog", "chrono", @@ -9161,8 +9416,8 @@ dependencies = [ "ahash 0.8.11", "api", "arc-swap", - "arrow", - "arrow-schema", + "arrow 54.2.1", + "arrow-schema 54.3.1", "async-recursion", "async-stream", "async-trait", @@ -10577,10 +10832,10 @@ version = "0.14.0" dependencies = [ "ahash 0.8.11", "api", - "arrow", + "arrow 54.2.1", "arrow-flight", - "arrow-ipc", - "arrow-schema", + "arrow-ipc 54.2.1", + "arrow-schema 54.3.1", "async-trait", "auth", "axum 0.8.1", @@ -10643,6 +10898,7 @@ dependencies = [ "openmetrics-parser", "opensrv-mysql", "opentelemetry-proto 0.27.0", + "otel-arrow-rust", "parking_lot 0.12.3", "permutation", "pgwire", diff --git a/Cargo.toml b/Cargo.toml index 3753c457f0..8aef7b6d7b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -269,6 +269,9 @@ metric-engine = { path = "src/metric-engine" } mito2 = { path = "src/mito2" } object-store = { path = "src/object-store" } operator = { path = "src/operator" } +otel-arrow-rust = { git = "https://github.com/open-telemetry/otel-arrow", rev = "5d551412d2a12e689cde4d84c14ef29e36784e51", features = [ + "server", +] } partition = { path = "src/partition" } pipeline = { path = "src/pipeline" } plugins = { path = "src/plugins" } diff --git a/src/frontend/Cargo.toml b/src/frontend/Cargo.toml index e6c6cf940d..54017bc8d6 100644 --- a/src/frontend/Cargo.toml +++ b/src/frontend/Cargo.toml @@ -39,6 +39,7 @@ datafusion.workspace = true datafusion-expr.workspace = true datanode.workspace = true datatypes.workspace = true +futures.workspace = true humantime-serde.workspace = true lazy_static.workspace = true log-query.workspace = true @@ -47,6 +48,7 @@ meta-client.workspace = true num_cpus.workspace = true opentelemetry-proto.workspace = true operator.workspace = true +otel-arrow-rust.workspace = true partition.workspace = true pipeline.workspace = true prometheus.workspace = true diff --git a/src/frontend/src/server.rs b/src/frontend/src/server.rs index fe1e15e7d9..d64fa4200f 100644 --- a/src/frontend/src/server.rs +++ b/src/frontend/src/server.rs @@ -27,6 +27,7 @@ use servers::http::{HttpServer, HttpServerBuilder}; use servers::interceptor::LogIngestInterceptorRef; use servers::metrics_handler::MetricsHandler; use servers::mysql::server::{MysqlServer, MysqlSpawnConfig, MysqlSpawnRef}; +use servers::otel_arrow::OtelArrowServiceHandler; use servers::postgres::PostgresServer; use servers::query_handler::grpc::ServerGrpcQueryHandlerAdapter; use servers::query_handler::sql::ServerSqlQueryHandlerAdapter; @@ -162,6 +163,7 @@ where let grpc_server = builder .database_handler(greptime_request_handler.clone()) .prometheus_handler(self.instance.clone(), user_provider.clone()) + .otel_arrow_handler(OtelArrowServiceHandler(self.instance.clone())) .flight_handler(Arc::new(greptime_request_handler)) .build(); Ok(grpc_server) diff --git a/src/servers/Cargo.toml b/src/servers/Cargo.toml index 2ad288c1f2..b29ff0bd40 100644 --- a/src/servers/Cargo.toml +++ b/src/servers/Cargo.toml @@ -85,6 +85,7 @@ socket2 = "0.5" # 2. Use ring, instead of aws-lc-rs in https://github.com/databendlabs/opensrv/pull/72 opensrv-mysql = { git = "https://github.com/datafuselabs/opensrv", rev = "a1fb4da215c8693c7e4f62be249a01b7fec52997" } opentelemetry-proto.workspace = true +otel-arrow-rust.workspace = true parking_lot.workspace = true pgwire = { version = "0.28.0", default-features = false, features = ["server-api-ring"] } pin-project = "1.0" diff --git a/src/servers/src/error.rs b/src/servers/src/error.rs index bfb36c32ca..e9ecca366a 100644 --- a/src/servers/src/error.rs +++ b/src/servers/src/error.rs @@ -540,12 +540,6 @@ pub enum Error { location: Location, }, - #[snafu(display("Missing query context"))] - MissingQueryContext { - #[snafu(implicit)] - location: Location, - }, - #[snafu(display("Invalid table name"))] InvalidTableName { #[snafu(source)] @@ -619,6 +613,13 @@ pub enum Error { #[snafu(display("Overflow while casting `{:?}` to Interval", val))] DurationOverflow { val: Duration }, + + #[snafu(display("Failed to handle otel-arrow request, error message: {}", err_msg))] + HandleOtelArrowRequest { + err_msg: String, + #[snafu(implicit)] + location: Location, + }, } pub type Result = std::result::Result; @@ -677,7 +678,6 @@ impl ErrorExt for Error { | TimePrecision { .. } | UrlDecode { .. } | IncompatibleSchema { .. } - | MissingQueryContext { .. } | MysqlValueConversion { .. } | ParseJson { .. } | InvalidLokiLabels { .. } @@ -738,7 +738,10 @@ impl ErrorExt for Error { ConvertSqlValue { source, .. } => source.status_code(), InFlightWriteBytesExceeded { .. } => StatusCode::RateLimited, + DurationOverflow { .. } => StatusCode::InvalidArguments, + + HandleOtelArrowRequest { .. } => StatusCode::Internal, } } diff --git a/src/servers/src/grpc.rs b/src/servers/src/grpc.rs index 0d7d185d76..dd591d7805 100644 --- a/src/servers/src/grpc.rs +++ b/src/servers/src/grpc.rs @@ -32,11 +32,13 @@ use common_grpc::channel_manager::{ }; use common_telemetry::{error, info, warn}; use futures::FutureExt; +use otel_arrow_rust::opentelemetry::ArrowMetricsServiceServer; use serde::{Deserialize, Serialize}; use snafu::{ensure, OptionExt, ResultExt}; use tokio::net::TcpListener; use tokio::sync::oneshot::{self, Receiver, Sender}; use tokio::sync::Mutex; +use tonic::service::interceptor::InterceptedService; use tonic::service::Routes; use tonic::transport::server::TcpIncoming; use tonic::transport::ServerTlsConfig; @@ -47,6 +49,8 @@ use crate::error::{ AlreadyStartedSnafu, InternalSnafu, Result, StartGrpcSnafu, TcpBindSnafu, TcpIncomingSnafu, }; use crate::metrics::MetricsMiddlewareLayer; +use crate::otel_arrow::{HeaderInterceptor, OtelArrowServiceHandler}; +use crate::query_handler::OpenTelemetryProtocolHandlerRef; use crate::server::Server; use crate::tls::TlsOption; @@ -138,6 +142,15 @@ pub struct GrpcServer { routes: Mutex>, // tls config tls_config: Option, + // Otel arrow service + otel_arrow_service: Mutex< + Option< + InterceptedService< + ArrowMetricsServiceServer>, + HeaderInterceptor, + >, + >, + >, } /// Grpc Server configuration @@ -264,11 +277,16 @@ impl Server for GrpcServer { if let Some(tls_config) = self.tls_config.clone() { builder = builder.tls_config(tls_config).context(StartGrpcSnafu)?; } - let builder = builder + + let mut builder = builder .add_routes(routes) .add_service(self.create_healthcheck_service()) .add_service(self.create_reflection_service()); + if let Some(otel_arrow_service) = self.otel_arrow_service.lock().await.take() { + builder = builder.add_service(otel_arrow_service); + } + let (serve_state_tx, serve_state_rx) = oneshot::channel(); let mut serve_state = self.serve_state.lock().await; *serve_state = Some(serve_state_rx); diff --git a/src/servers/src/grpc/builder.rs b/src/servers/src/grpc/builder.rs index fc6bbba7ec..65d439fada 100644 --- a/src/servers/src/grpc/builder.rs +++ b/src/servers/src/grpc/builder.rs @@ -19,8 +19,11 @@ use arrow_flight::flight_service_server::FlightServiceServer; use auth::UserProviderRef; use common_grpc::error::{Error, InvalidConfigFilePathSnafu, Result}; use common_runtime::Runtime; +use otel_arrow_rust::opentelemetry::ArrowMetricsServiceServer; use snafu::ResultExt; use tokio::sync::Mutex; +use tonic::codec::CompressionEncoding; +use tonic::service::interceptor::InterceptedService; use tonic::service::RoutesBuilder; use tonic::transport::{Identity, ServerTlsConfig}; @@ -30,7 +33,9 @@ use crate::grpc::greptime_handler::GreptimeRequestHandler; use crate::grpc::prom_query_gateway::PrometheusGatewayService; use crate::grpc::region_server::{RegionServerHandlerRef, RegionServerRequestHandler}; use crate::grpc::{GrpcServer, GrpcServerConfig}; +use crate::otel_arrow::{HeaderInterceptor, OtelArrowServiceHandler}; use crate::prometheus_handler::PrometheusHandlerRef; +use crate::query_handler::OpenTelemetryProtocolHandlerRef; use crate::tls::TlsOption; /// Add a gRPC service (`service`) to a `builder`([RoutesBuilder]). @@ -59,6 +64,12 @@ pub struct GrpcServerBuilder { runtime: Runtime, routes_builder: RoutesBuilder, tls_config: Option, + otel_arrow_service: Option< + InterceptedService< + ArrowMetricsServiceServer>, + HeaderInterceptor, + >, + >, } impl GrpcServerBuilder { @@ -68,6 +79,7 @@ impl GrpcServerBuilder { runtime, routes_builder: RoutesBuilder::default(), tls_config: None, + otel_arrow_service: None, } } @@ -113,6 +125,22 @@ impl GrpcServerBuilder { self } + /// Add handler for [OtelArrowService]. + pub fn otel_arrow_handler( + mut self, + handler: OtelArrowServiceHandler, + ) -> Self { + let mut server = ArrowMetricsServiceServer::new(handler); + server = server + .max_decoding_message_size(self.config.max_recv_message_size) + .max_encoding_message_size(self.config.max_send_message_size) + .accept_compressed(CompressionEncoding::Zstd) + .send_compressed(CompressionEncoding::Zstd); + let svc = InterceptedService::new(server, HeaderInterceptor {}); + self.otel_arrow_service = Some(svc); + self + } + /// Add handler for [RegionServer]. pub fn region_server_handler(mut self, region_server_handler: RegionServerHandlerRef) -> Self { let handler = RegionServerRequestHandler::new(region_server_handler, self.runtime.clone()); @@ -152,6 +180,7 @@ impl GrpcServerBuilder { shutdown_tx: Mutex::new(None), serve_state: Mutex::new(None), tls_config: self.tls_config, + otel_arrow_service: Mutex::new(self.otel_arrow_service), } } } diff --git a/src/servers/src/grpc/otlp.rs b/src/servers/src/grpc/otlp.rs deleted file mode 100644 index f3f71900eb..0000000000 --- a/src/servers/src/grpc/otlp.rs +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::result::Result as StdResult; -use std::sync::Arc; - -use opentelemetry_proto::tonic::collector::metrics::v1::metrics_service_server::MetricsService; -use opentelemetry_proto::tonic::collector::metrics::v1::{ - ExportMetricsServiceRequest, ExportMetricsServiceResponse, -}; -use opentelemetry_proto::tonic::collector::trace::v1::trace_service_server::TraceService; -use opentelemetry_proto::tonic::collector::trace::v1::{ - ExportTraceServiceRequest, ExportTraceServiceResponse, -}; -use session::context::{Channel, QueryContext}; -use snafu::{OptionExt, ResultExt}; -use tonic::{Request, Response, Status}; - -use crate::error; -use crate::http::header::constants::GREPTIME_TRACE_TABLE_NAME_HEADER_NAME; -use crate::otlp::trace::TRACE_TABLE_NAME; -use crate::query_handler::OpenTelemetryProtocolHandlerRef; - -pub struct OtlpService { - handler: OpenTelemetryProtocolHandlerRef, -} - -impl OtlpService { - pub fn new(handler: OpenTelemetryProtocolHandlerRef) -> Self { - Self { handler } - } -} - -#[async_trait::async_trait] -impl TraceService for OtlpService { - async fn export( - &self, - request: Request, - ) -> StdResult, Status> { - let (headers, extensions, req) = request.into_parts(); - - let table_name = match headers.get(GREPTIME_TRACE_TABLE_NAME_HEADER_NAME) { - Some(table_name) => table_name - .to_str() - .context(error::InvalidTableNameSnafu)? - .to_string(), - None => TRACE_TABLE_NAME.to_string(), - }; - - let mut ctx = extensions - .get::() - .cloned() - .context(error::MissingQueryContextSnafu)?; - ctx.set_channel(Channel::Otlp); - let ctx = Arc::new(ctx); - - let _ = self.handler.traces(req, table_name, ctx).await?; - - Ok(Response::new(ExportTraceServiceResponse { - partial_success: None, - })) - } -} - -#[async_trait::async_trait] -impl MetricsService for OtlpService { - async fn export( - &self, - request: Request, - ) -> StdResult, Status> { - let (_headers, extensions, req) = request.into_parts(); - - let mut ctx = extensions - .get::() - .cloned() - .context(error::MissingQueryContextSnafu)?; - ctx.set_channel(Channel::Otlp); - let ctx = Arc::new(ctx); - - let _ = self.handler.metrics(req, ctx).await?; - - Ok(Response::new(ExportMetricsServiceResponse { - partial_success: None, - })) - } -} diff --git a/src/servers/src/lib.rs b/src/servers/src/lib.rs index 61bf041f52..a13cd0ce1f 100644 --- a/src/servers/src/lib.rs +++ b/src/servers/src/lib.rs @@ -37,6 +37,7 @@ mod metrics; pub mod metrics_handler; pub mod mysql; pub mod opentsdb; +pub mod otel_arrow; pub mod otlp; mod pipeline; pub mod postgres; diff --git a/src/servers/src/otel_arrow.rs b/src/servers/src/otel_arrow.rs new file mode 100644 index 0000000000..f279c7f7b8 --- /dev/null +++ b/src/servers/src/otel_arrow.rs @@ -0,0 +1,119 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use common_error::ext::ErrorExt; +use common_error::status_code::status_to_tonic_code; +use common_telemetry::error; +use futures::SinkExt; +use otel_arrow_rust::opentelemetry::{ArrowMetricsService, BatchArrowRecords, BatchStatus}; +use otel_arrow_rust::Consumer; +use session::context::QueryContext; +use tonic::metadata::{Entry, MetadataValue}; +use tonic::service::Interceptor; +use tonic::{Request, Response, Status, Streaming}; + +use crate::error; +use crate::query_handler::OpenTelemetryProtocolHandlerRef; + +pub struct OtelArrowServiceHandler(pub T); + +impl OtelArrowServiceHandler { + pub fn new(handler: T) -> Self { + Self(handler) + } +} + +#[async_trait::async_trait] +impl ArrowMetricsService for OtelArrowServiceHandler { + type ArrowMetricsStream = futures::channel::mpsc::Receiver>; + async fn arrow_metrics( + &self, + request: Request>, + ) -> Result, Status> { + let (mut sender, receiver) = futures::channel::mpsc::channel(100); + let mut incoming_requests = request.into_inner(); + let handler = self.0.clone(); + let query_context = QueryContext::arc(); + // handles incoming requests + common_runtime::spawn_global(async move { + let mut consumer = Consumer::default(); + while let Some(batch_res) = incoming_requests.message().await.transpose() { + let mut batch = match batch_res { + Ok(batch) => batch, + Err(e) => { + error!( + "Failed to receive batch from otel-arrow client, error: {}", + e + ); + let _ = sender.send(Err(e)).await; + return; + } + }; + let batch_status = BatchStatus { + batch_id: batch.batch_id, + status_code: 0, + status_message: Default::default(), + }; + let request = match consumer.consume_batches(&mut batch).map_err(|e| { + error::HandleOtelArrowRequestSnafu { + err_msg: e.to_string(), + } + .build() + }) { + Ok(request) => request, + Err(e) => { + let _ = sender + .send(Err(Status::new( + status_to_tonic_code(e.status_code()), + e.to_string(), + ))) + .await; + error!(e; + "Failed to consume batch from otel-arrow client" + ); + return; + } + }; + if let Err(e) = handler.metrics(request, query_context.clone()).await { + let _ = sender + .send(Err(Status::new( + status_to_tonic_code(e.status_code()), + e.to_string(), + ))) + .await; + error!(e; "Failed to ingest metrics from otel-arrow"); + return; + } + let _ = sender.send(Ok(batch_status)).await; + } + }); + Ok(Response::new(receiver)) + } +} + +/// This serves as a workaround for otel-arrow collector's custom header. +#[derive(Clone)] +pub struct HeaderInterceptor; + +impl Interceptor for HeaderInterceptor { + fn call(&mut self, mut request: Request<()>) -> Result, Status> { + if let Ok(Entry::Occupied(mut e)) = request.metadata_mut().entry("grpc-encoding") { + // This works as a workaround to handle customized compression type (zstdarrow*) in otel-arrow. + if e.get().as_bytes().starts_with(b"zstdarrow") { + e.insert(MetadataValue::from_static("zstd")); + } + } + Ok(request) + } +} From ee07b9bfa81029dfaf50144b97c802e9d8d48e64 Mon Sep 17 00:00:00 2001 From: Yuhan Wang Date: Mon, 21 Apr 2025 15:57:43 +0800 Subject: [PATCH 48/82] test: update configs to enable auto wal prune (#5938) * test: update configs to enable auto wal prune * fix: add humantime_serde * fix: enable overwrite_entry_start_id * fix: not in metasrv * chore: update default value name * Apply suggestions from code review Co-authored-by: jeremyhi * fix: kafka use overwrite_entry_start_id --------- Co-authored-by: jeremyhi --- .../setup-greptimedb-cluster/with-remote-wal.yaml | 4 +++- src/common/wal/src/config/kafka/common.rs | 8 ++++---- src/common/wal/src/config/kafka/datanode.rs | 12 +++++------- src/common/wal/src/config/kafka/metasrv.rs | 9 +++++---- tests/conf/datanode-test.toml.template | 1 + tests/conf/metasrv-test.toml.template | 3 ++- 6 files changed, 20 insertions(+), 17 deletions(-) diff --git a/.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml b/.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml index 58cc188985..26e354f3d0 100644 --- a/.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml +++ b/.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml @@ -7,7 +7,8 @@ meta: provider = "kafka" broker_endpoints = ["kafka.kafka-cluster.svc.cluster.local:9092"] num_topics = 3 - auto_prune_topic_records = true + auto_prune_interval = "30s" + trigger_flush_threshold = 100 [datanode] [datanode.client] @@ -22,6 +23,7 @@ datanode: provider = "kafka" broker_endpoints = ["kafka.kafka-cluster.svc.cluster.local:9092"] linger = "2ms" + overwrite_entry_start_id = true frontend: configData: |- [runtime] diff --git a/src/common/wal/src/config/kafka/common.rs b/src/common/wal/src/config/kafka/common.rs index 6b2c9992f4..e4763687cd 100644 --- a/src/common/wal/src/config/kafka/common.rs +++ b/src/common/wal/src/config/kafka/common.rs @@ -30,10 +30,10 @@ pub const DEFAULT_BACKOFF_CONFIG: BackoffConfig = BackoffConfig { deadline: Some(Duration::from_secs(120)), }; -/// Default interval for active WAL pruning. -pub const DEFAULT_ACTIVE_PRUNE_INTERVAL: Duration = Duration::ZERO; -/// Default limit for concurrent active pruning tasks. -pub const DEFAULT_ACTIVE_PRUNE_TASK_LIMIT: usize = 10; +/// Default interval for auto WAL pruning. +pub const DEFAULT_AUTO_PRUNE_INTERVAL: Duration = Duration::ZERO; +/// Default limit for concurrent auto pruning tasks. +pub const DEFAULT_AUTO_PRUNE_PARALLELISM: usize = 10; /// Default interval for sending flush request to regions when pruning remote WAL. pub const DEFAULT_TRIGGER_FLUSH_THRESHOLD: u64 = 0; diff --git a/src/common/wal/src/config/kafka/datanode.rs b/src/common/wal/src/config/kafka/datanode.rs index dd659d636e..278e3dd1a5 100644 --- a/src/common/wal/src/config/kafka/datanode.rs +++ b/src/common/wal/src/config/kafka/datanode.rs @@ -18,8 +18,8 @@ use common_base::readable_size::ReadableSize; use serde::{Deserialize, Serialize}; use crate::config::kafka::common::{ - KafkaConnectionConfig, KafkaTopicConfig, DEFAULT_ACTIVE_PRUNE_INTERVAL, - DEFAULT_ACTIVE_PRUNE_TASK_LIMIT, DEFAULT_TRIGGER_FLUSH_THRESHOLD, + KafkaConnectionConfig, KafkaTopicConfig, DEFAULT_AUTO_PRUNE_INTERVAL, + DEFAULT_AUTO_PRUNE_PARALLELISM, DEFAULT_TRIGGER_FLUSH_THRESHOLD, }; /// Kafka wal configurations for datanode. @@ -47,9 +47,8 @@ pub struct DatanodeKafkaConfig { pub dump_index_interval: Duration, /// Ignore missing entries during read WAL. pub overwrite_entry_start_id: bool, - // Active WAL pruning. - pub auto_prune_topic_records: bool, // Interval of WAL pruning. + #[serde(with = "humantime_serde")] pub auto_prune_interval: Duration, // Threshold for sending flush request when pruning remote WAL. // `None` stands for never sending flush request. @@ -70,10 +69,9 @@ impl Default for DatanodeKafkaConfig { create_index: true, dump_index_interval: Duration::from_secs(60), overwrite_entry_start_id: false, - auto_prune_topic_records: false, - auto_prune_interval: DEFAULT_ACTIVE_PRUNE_INTERVAL, + auto_prune_interval: DEFAULT_AUTO_PRUNE_INTERVAL, trigger_flush_threshold: DEFAULT_TRIGGER_FLUSH_THRESHOLD, - auto_prune_parallelism: DEFAULT_ACTIVE_PRUNE_TASK_LIMIT, + auto_prune_parallelism: DEFAULT_AUTO_PRUNE_PARALLELISM, } } } diff --git a/src/common/wal/src/config/kafka/metasrv.rs b/src/common/wal/src/config/kafka/metasrv.rs index acbfbe05c6..d20100af89 100644 --- a/src/common/wal/src/config/kafka/metasrv.rs +++ b/src/common/wal/src/config/kafka/metasrv.rs @@ -17,8 +17,8 @@ use std::time::Duration; use serde::{Deserialize, Serialize}; use crate::config::kafka::common::{ - KafkaConnectionConfig, KafkaTopicConfig, DEFAULT_ACTIVE_PRUNE_INTERVAL, - DEFAULT_ACTIVE_PRUNE_TASK_LIMIT, DEFAULT_TRIGGER_FLUSH_THRESHOLD, + KafkaConnectionConfig, KafkaTopicConfig, DEFAULT_AUTO_PRUNE_INTERVAL, + DEFAULT_AUTO_PRUNE_PARALLELISM, DEFAULT_TRIGGER_FLUSH_THRESHOLD, }; /// Kafka wal configurations for metasrv. @@ -34,6 +34,7 @@ pub struct MetasrvKafkaConfig { // Automatically create topics for WAL. pub auto_create_topics: bool, // Interval of WAL pruning. + #[serde(with = "humantime_serde")] pub auto_prune_interval: Duration, // Threshold for sending flush request when pruning remote WAL. // `None` stands for never sending flush request. @@ -48,9 +49,9 @@ impl Default for MetasrvKafkaConfig { connection: Default::default(), kafka_topic: Default::default(), auto_create_topics: true, - auto_prune_interval: DEFAULT_ACTIVE_PRUNE_INTERVAL, + auto_prune_interval: DEFAULT_AUTO_PRUNE_INTERVAL, trigger_flush_threshold: DEFAULT_TRIGGER_FLUSH_THRESHOLD, - auto_prune_parallelism: DEFAULT_ACTIVE_PRUNE_TASK_LIMIT, + auto_prune_parallelism: DEFAULT_AUTO_PRUNE_PARALLELISM, } } } diff --git a/tests/conf/datanode-test.toml.template b/tests/conf/datanode-test.toml.template index 20987eed9a..a58d802262 100644 --- a/tests/conf/datanode-test.toml.template +++ b/tests/conf/datanode-test.toml.template @@ -15,6 +15,7 @@ sync_write = false provider = "kafka" broker_endpoints = {kafka_wal_broker_endpoints | unescaped} linger = "5ms" +overwrite_entry_start_id = true {{ endif }} [storage] diff --git a/tests/conf/metasrv-test.toml.template b/tests/conf/metasrv-test.toml.template index 3daf6150f3..f4bbbf3ae9 100644 --- a/tests/conf/metasrv-test.toml.template +++ b/tests/conf/metasrv-test.toml.template @@ -18,5 +18,6 @@ broker_endpoints = {kafka_wal_broker_endpoints | unescaped} num_topics = 3 selector_type = "round_robin" topic_name_prefix = "distributed_test_greptimedb_wal_topic" -auto_prune_topic_records = true +auto_prune_interval = "30s" +trigger_flush_threshold = 100 {{ endif }} From 7a8e1bc3f993a109fcd47d1a3cb74df26846e291 Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Mon, 21 Apr 2025 18:59:24 +0800 Subject: [PATCH 49/82] feat: support building `metasrv` with selector from plugins (#5942) * chore: expose selector * feat: use f64 * chore: expose selector::common * feat: build metasrv with selector from plugins --- src/meta-srv/src/bootstrap.rs | 18 ++++++++++++++---- src/meta-srv/src/selector.rs | 7 ++++--- src/meta-srv/src/selector/common.rs | 10 +++++----- src/meta-srv/src/selector/lease_based.rs | 2 +- src/meta-srv/src/selector/weight_compute.rs | 4 ++-- src/meta-srv/src/selector/weighted_choose.rs | 17 ++++++++++------- 6 files changed, 36 insertions(+), 22 deletions(-) diff --git a/src/meta-srv/src/bootstrap.rs b/src/meta-srv/src/bootstrap.rs index 33c32443e2..3b27295301 100644 --- a/src/meta-srv/src/bootstrap.rs +++ b/src/meta-srv/src/bootstrap.rs @@ -294,10 +294,20 @@ pub async fn metasrv_builder( let in_memory = Arc::new(MemoryKvBackend::new()) as ResettableKvBackendRef; - let selector = match opts.selector { - SelectorType::LoadBased => Arc::new(LoadBasedSelector::default()) as SelectorRef, - SelectorType::LeaseBased => Arc::new(LeaseBasedSelector) as SelectorRef, - SelectorType::RoundRobin => Arc::new(RoundRobinSelector::default()) as SelectorRef, + let selector = if let Some(selector) = plugins.get::() { + info!("Using selector from plugins"); + selector + } else { + let selector = match opts.selector { + SelectorType::LoadBased => Arc::new(LoadBasedSelector::default()) as SelectorRef, + SelectorType::LeaseBased => Arc::new(LeaseBasedSelector) as SelectorRef, + SelectorType::RoundRobin => Arc::new(RoundRobinSelector::default()) as SelectorRef, + }; + info!( + "Using selector from options, selector type: {}", + opts.selector.as_ref() + ); + selector }; Ok(MetasrvBuilder::new() diff --git a/src/meta-srv/src/selector.rs b/src/meta-srv/src/selector.rs index c197f04e59..9a3d2ceadd 100644 --- a/src/meta-srv/src/selector.rs +++ b/src/meta-srv/src/selector.rs @@ -12,15 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod common; +pub mod common; pub mod lease_based; pub mod load_based; pub mod round_robin; #[cfg(test)] pub(crate) mod test_utils; mod weight_compute; -mod weighted_choose; +pub mod weighted_choose; use serde::{Deserialize, Serialize}; +use strum::AsRefStr; use crate::error; use crate::error::Result; @@ -51,7 +52,7 @@ impl Default for SelectorOptions { } /// [`SelectorType`] refers to the load balancer used when creating tables. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default, AsRefStr)] #[serde(try_from = "String")] pub enum SelectorType { /// The current load balancing is based on the number of regions on each datanode node; diff --git a/src/meta-srv/src/selector/common.rs b/src/meta-srv/src/selector/common.rs index 24352c14c2..b44043f60e 100644 --- a/src/meta-srv/src/selector/common.rs +++ b/src/meta-srv/src/selector/common.rs @@ -92,35 +92,35 @@ mod tests { id: 1, addr: "127.0.0.1:3001".to_string(), }, - weight: 1, + weight: 1.0, }, WeightedItem { item: Peer { id: 2, addr: "127.0.0.1:3001".to_string(), }, - weight: 1, + weight: 1.0, }, WeightedItem { item: Peer { id: 3, addr: "127.0.0.1:3001".to_string(), }, - weight: 1, + weight: 1.0, }, WeightedItem { item: Peer { id: 4, addr: "127.0.0.1:3001".to_string(), }, - weight: 1, + weight: 1.0, }, WeightedItem { item: Peer { id: 5, addr: "127.0.0.1:3001".to_string(), }, - weight: 1, + weight: 1.0, }, ]; diff --git a/src/meta-srv/src/selector/lease_based.rs b/src/meta-srv/src/selector/lease_based.rs index a7ce7c7321..770c8aac8a 100644 --- a/src/meta-srv/src/selector/lease_based.rs +++ b/src/meta-srv/src/selector/lease_based.rs @@ -42,7 +42,7 @@ impl Selector for LeaseBasedSelector { id: k.node_id, addr: v.node_addr.clone(), }, - weight: 1, + weight: 1.0, }) .collect(); diff --git a/src/meta-srv/src/selector/weight_compute.rs b/src/meta-srv/src/selector/weight_compute.rs index 904fc1f4e7..d69e2f56d3 100644 --- a/src/meta-srv/src/selector/weight_compute.rs +++ b/src/meta-srv/src/selector/weight_compute.rs @@ -84,7 +84,7 @@ impl WeightCompute for RegionNumsBasedWeightCompute { .zip(region_nums) .map(|(peer, region_num)| WeightedItem { item: peer, - weight: (max_weight - region_num + base_weight) as usize, + weight: (max_weight - region_num + base_weight) as f64, }) .collect() } @@ -148,7 +148,7 @@ mod tests { 2, ); for weight in weight_array.iter() { - assert_eq!(*expected.get(&weight.item).unwrap(), weight.weight,); + assert_eq!(*expected.get(&weight.item).unwrap(), weight.weight as usize); } let mut expected = HashMap::new(); diff --git a/src/meta-srv/src/selector/weighted_choose.rs b/src/meta-srv/src/selector/weighted_choose.rs index 749df57edf..0890be9137 100644 --- a/src/meta-srv/src/selector/weighted_choose.rs +++ b/src/meta-srv/src/selector/weighted_choose.rs @@ -42,10 +42,10 @@ pub trait WeightedChoose: Send + Sync { } /// The struct represents a weighted item. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq)] pub struct WeightedItem { pub item: Item, - pub weight: usize, + pub weight: f64, } /// A implementation of weighted balance: random weighted choose. @@ -87,7 +87,7 @@ where // unwrap safety: whether weighted_index is none has been checked before. let item = self .items - .choose_weighted(&mut rng(), |item| item.weight as f64) + .choose_weighted(&mut rng(), |item| item.weight) .context(error::ChooseItemsSnafu)? .item .clone(); @@ -95,11 +95,11 @@ where } fn choose_multiple(&mut self, amount: usize) -> Result> { - let amount = amount.min(self.items.iter().filter(|item| item.weight > 0).count()); + let amount = amount.min(self.items.iter().filter(|item| item.weight > 0.0).count()); Ok(self .items - .choose_multiple_weighted(&mut rng(), amount, |item| item.weight as f64) + .choose_multiple_weighted(&mut rng(), amount, |item| item.weight) .context(error::ChooseItemsSnafu)? .cloned() .map(|item| item.item) @@ -120,9 +120,12 @@ mod tests { let mut choose = RandomWeightedChoose::new(vec![ WeightedItem { item: 1, - weight: 100, + weight: 100.0, + }, + WeightedItem { + item: 2, + weight: 0.0, }, - WeightedItem { item: 2, weight: 0 }, ]); for _ in 0..100 { From 3b8c6d5ce3e50e8311be505cec837a553679f357 Mon Sep 17 00:00:00 2001 From: shuiyisong <113876041+shuiyisong@users.noreply.github.com> Date: Mon, 21 Apr 2025 20:55:48 +0800 Subject: [PATCH 50/82] chore: use `once_cell` to avoid parse everytime in pipeline exec (#5943) * chore: use once_cell to avoid parse everytime * chore: remove pub on options --- .../src/etl/transform/transformer/greptime.rs | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/pipeline/src/etl/transform/transformer/greptime.rs b/src/pipeline/src/etl/transform/transformer/greptime.rs index fa0f9c3b49..033feda0c5 100644 --- a/src/pipeline/src/etl/transform/transformer/greptime.rs +++ b/src/pipeline/src/etl/transform/transformer/greptime.rs @@ -25,6 +25,7 @@ use api::v1::{ColumnDataType, ColumnDataTypeExtension, JsonTypeExtension, Semant use coerce::{coerce_columns, coerce_value}; use greptime_proto::v1::{ColumnSchema, Row, Rows, Value as GreptimeValue}; use itertools::Itertools; +use once_cell::sync::OnceCell; use serde_json::Number; use crate::error::{ @@ -54,8 +55,12 @@ pub struct GreptimeTransformer { /// Parameters that can be used to configure the greptime pipelines. #[derive(Debug, Clone, Default)] pub struct GreptimePipelineParams { - /// The options for configuring the greptime pipelines. - pub options: HashMap, + /// The original options for configuring the greptime pipelines. + /// This should not be used directly, instead, use the parsed shortcut option values. + options: HashMap, + + /// Parsed shortcut option values + pub flatten_json_object: OnceCell, } impl GreptimePipelineParams { @@ -70,15 +75,20 @@ impl GreptimePipelineParams { .map(|(k, v)| (k.to_string(), v.to_string())) .collect::>(); - Self { options } + Self { + options, + flatten_json_object: OnceCell::new(), + } } /// Whether to flatten the JSON object. pub fn flatten_json_object(&self) -> bool { - self.options - .get("flatten_json_object") - .map(|v| v == "true") - .unwrap_or(false) + *self.flatten_json_object.get_or_init(|| { + self.options + .get("flatten_json_object") + .map(|v| v == "true") + .unwrap_or(false) + }) } } From 60e4607b642c0292ea4bab817b81f2830baf2570 Mon Sep 17 00:00:00 2001 From: jeremyhi Date: Tue, 22 Apr 2025 00:12:27 +0800 Subject: [PATCH 51/82] chore: better buckets for heartbeat stat size histogram (#5945) chore: better buckets for METRIC_META_HEARTBEAT_STAT_MEMORY_SIZE --- src/meta-srv/src/metrics.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/meta-srv/src/metrics.rs b/src/meta-srv/src/metrics.rs index a9750ae5f3..ffbe986d72 100644 --- a/src/meta-srv/src/metrics.rs +++ b/src/meta-srv/src/metrics.rs @@ -62,7 +62,9 @@ lazy_static! { register_int_counter!("greptime_meta_region_migration_fail", "meta region migration fail").unwrap(); // The heartbeat stat memory size histogram. pub static ref METRIC_META_HEARTBEAT_STAT_MEMORY_SIZE: Histogram = - register_histogram!("greptime_meta_heartbeat_stat_memory_size", "meta heartbeat stat memory size").unwrap(); + register_histogram!("greptime_meta_heartbeat_stat_memory_size", "meta heartbeat stat memory size", vec![ + 100.0, 500.0, 1000.0, 1500.0, 2000.0, 3000.0, 5000.0, 10000.0, 20000.0 + ]).unwrap(); // The heartbeat rate counter. pub static ref METRIC_META_HEARTBEAT_RATE: IntCounter = register_int_counter!("greptime_meta_heartbeat_rate", "meta heartbeat arrival rate").unwrap(); From 35f4fa3c3e8596cd0435f0de04f10fafdf639e86 Mon Sep 17 00:00:00 2001 From: zyy17 Date: Tue, 22 Apr 2025 14:03:01 +0800 Subject: [PATCH 52/82] refactor: unify all dashboards and use `dac` tool to generate intermediate dashboards (#5933) * refactor: split cluster metrics into multiple dashboards * chore: merge multiple dashboards into one dashboard * refactor: add 'dac' tool to generate a intermediate dashboards * refactor: generate markdown docs for dashboards --- .github/workflows/grafana.yml | 30 +- Makefile | 10 + grafana/README.md | 108 +- grafana/check.sh | 19 - grafana/dashboards/cluster/dashboard.json | 7082 +++++++++++++++++ grafana/dashboards/cluster/dashboard.md | 96 + grafana/dashboards/cluster/dashboard.yaml | 761 ++ grafana/dashboards/standalone/dashboard.json | 7082 +++++++++++++++++ grafana/dashboards/standalone/dashboard.md | 96 + grafana/dashboards/standalone/dashboard.yaml | 761 ++ grafana/greptimedb-cluster.json | 7518 ------------------ grafana/greptimedb.json | 4159 ---------- grafana/scripts/check.sh | 54 + grafana/scripts/gen-dashboards.sh | 18 + grafana/summary.sh | 11 - 15 files changed, 16027 insertions(+), 11778 deletions(-) delete mode 100755 grafana/check.sh create mode 100644 grafana/dashboards/cluster/dashboard.json create mode 100644 grafana/dashboards/cluster/dashboard.md create mode 100644 grafana/dashboards/cluster/dashboard.yaml create mode 100644 grafana/dashboards/standalone/dashboard.json create mode 100644 grafana/dashboards/standalone/dashboard.md create mode 100644 grafana/dashboards/standalone/dashboard.yaml delete mode 100644 grafana/greptimedb-cluster.json delete mode 100644 grafana/greptimedb.json create mode 100755 grafana/scripts/check.sh create mode 100755 grafana/scripts/gen-dashboards.sh delete mode 100755 grafana/summary.sh diff --git a/.github/workflows/grafana.yml b/.github/workflows/grafana.yml index 139ea85b05..29fa182998 100644 --- a/.github/workflows/grafana.yml +++ b/.github/workflows/grafana.yml @@ -21,32 +21,6 @@ jobs: run: sudo apt-get install -y jq # Make the check.sh script executable - - name: Make check.sh executable - run: chmod +x grafana/check.sh - - # Run the check.sh script - - name: Run check.sh - run: ./grafana/check.sh - - # Only run summary.sh for pull_request events (not for merge queues or final pushes) - - name: Check if this is a pull request - id: check-pr + - name: Check grafana dashboards run: | - if [[ "${{ github.event_name }}" == "pull_request" ]]; then - echo "is_pull_request=true" >> $GITHUB_OUTPUT - else - echo "is_pull_request=false" >> $GITHUB_OUTPUT - fi - - # Make the summary.sh script executable - - name: Make summary.sh executable - if: steps.check-pr.outputs.is_pull_request == 'true' - run: chmod +x grafana/summary.sh - - # Run the summary.sh script and add its output to the GitHub Job Summary - - name: Run summary.sh and add to Job Summary - if: steps.check-pr.outputs.is_pull_request == 'true' - run: | - SUMMARY=$(./grafana/summary.sh) - echo "### Summary of Grafana Panels" >> $GITHUB_STEP_SUMMARY - echo "$SUMMARY" >> $GITHUB_STEP_SUMMARY + make check-dashboards diff --git a/Makefile b/Makefile index c40b7f67cb..7c1cff9821 100644 --- a/Makefile +++ b/Makefile @@ -222,6 +222,16 @@ start-cluster: ## Start the greptimedb cluster with etcd by using docker compose stop-cluster: ## Stop the greptimedb cluster that created by docker compose. docker compose -f ./docker/docker-compose/cluster-with-etcd.yaml stop +##@ Grafana + +.PHONY: check-dashboards +check-dashboards: ## Check the Grafana dashboards. + @./grafana/scripts/check.sh + +.PHONY: dashboards +dashboards: ## Generate the Grafana dashboards for standalone mode and intermediate dashboards. + @./grafana/scripts/gen-dashboards.sh + ##@ Docs config-docs: ## Generate configuration documentation from toml files. docker run --rm \ diff --git a/grafana/README.md b/grafana/README.md index 233dcdd4d6..9c13b0e322 100644 --- a/grafana/README.md +++ b/grafana/README.md @@ -1,61 +1,83 @@ -Grafana dashboard for GreptimeDB --------------------------------- +# Grafana dashboards for GreptimeDB -GreptimeDB's official Grafana dashboard. +## Overview -Status notify: we are still working on this config. It's expected to change frequently in the recent days. Please feel free to submit your feedback and/or contribution to this dashboard 🤗 +This repository maintains the Grafana dashboards for GreptimeDB. It has two types of dashboards: -If you use Helm [chart](https://github.com/GreptimeTeam/helm-charts) to deploy GreptimeDB cluster, you can enable self-monitoring by setting the following values in your Helm chart: +- `cluster/`: The dashboard for the GreptimeDB cluster. Read the [dashboard.md](./dashboards/cluster/dashboard.md) for more details. +- `standalone/`: The dashboard for the standalone GreptimeDB instance. Read the [dashboard.md](./dashboards/standalone/dashboard.md) for more details. + +As the rapid development of GreptimeDB, the metrics may be changed, and please feel free to submit your feedback and/or contribution to this dashboard 🤗 + +To maintain the dashboards, we use the [`dac`](https://github.com/zyy17/dac) tool to generate the intermediate dashboards and markdown documents: + +- `cluster/dashboard.yaml`: The intermediate dashboard for the GreptimeDB cluster. +- `standalone/dashboard.yaml`: The intermediatedashboard for the standalone GreptimeDB instance. + +## Data Sources + +There are two data sources for the dashboards to fetch the metrics: + +- **Prometheus**: Expose the metrics of GreptimeDB. +- **Information Schema**: It is the MySQL port of the current monitored instance. The `overview` dashboard will use this datasource to show the information schema of the current instance. + +## Instance Filters + +To deploy the dashboards for multiple scenarios (K8s, bare metal, etc.), we prefer to use the `instance` label when filtering instances. + +Additionally, we recommend including the `pod` label in the legend to make it easier to identify each instance, even though this field will be empty in bare metal scenarios. + +For example, the following query is recommended: + +```promql +sum(process_resident_memory_bytes{instance=~"$datanode"}) by (instance, pod) +``` + +And the legend will be like: `[{{instance}}]-[{{ pod }}]`. + +## Deployment + +### Helm + +If you use the Helm [chart](https://github.com/GreptimeTeam/helm-charts) to deploy a GreptimeDB cluster, you can enable self-monitoring by setting the following values in your Helm chart: - `monitoring.enabled=true`: Deploys a standalone GreptimeDB instance dedicated to monitoring the cluster; - `grafana.enabled=true`: Deploys Grafana and automatically imports the monitoring dashboard; -The standalone GreptimeDB instance will collect metrics from your cluster and the dashboard will be available in the Grafana UI. For detailed deployment instructions, please refer to our [Kubernetes deployment guide](https://docs.greptime.com/nightly/user-guide/deployments/deploy-on-kubernetes/getting-started). +The standalone GreptimeDB instance will collect metrics from your cluster, and the dashboard will be available in the Grafana UI. For detailed deployment instructions, please refer to our [Kubernetes deployment guide](https://docs.greptime.com/nightly/user-guide/deployments/deploy-on-kubernetes/getting-started). -# How to use +### Self-host Prometheus and import dashboards manually -## `greptimedb.json` +1. **Configure Prometheus to scrape the cluster** -Open Grafana Dashboard page, choose `New` -> `Import`. And upload `greptimedb.json` file. + The following is an example configuration(**Please modify it according to your actual situation**): -## `greptimedb-cluster.json` + ```yml + # example config + # only to indicate how to assign labels to each target + # modify yours accordingly + scrape_configs: + - job_name: metasrv + static_configs: + - targets: [':'] -This cluster dashboard provides a comprehensive view of incoming requests, response statuses, and internal activities such as flush and compaction, with a layered structure from frontend to datanode. Designed with a focus on alert functionality, its primary aim is to highlight any anomalies in metrics, allowing users to quickly pinpoint the cause of errors. + - job_name: datanode + static_configs: + - targets: [':', ':', ':'] -We use Prometheus to scrape off metrics from nodes in GreptimeDB cluster, Grafana to visualize the diagram. Any compatible stack should work too. + - job_name: frontend + static_configs: + - targets: [':'] + ``` -__Note__: This dashboard is still in an early stage of development. Any issue or advice on improvement is welcomed. +2. **Configure the data sources in Grafana** -### Configuration + You need to add two data sources in Grafana: -Please ensure the following configuration before importing the dashboard into Grafana. + - Prometheus: It is the Prometheus instance that scrapes the GreptimeDB metrics. + - Information Schema: It is the MySQL port of the current monitored instance. The dashboard will use this datasource to show the information schema of the current instance. -__1. Prometheus scrape config__ +3. **Import the dashboards based on your deployment scenario** -Configure Prometheus to scrape the cluster. - -```yml -# example config -# only to indicate how to assign labels to each target -# modify yours accordingly -scrape_configs: - - job_name: metasrv - static_configs: - - targets: [':'] - - - job_name: datanode - static_configs: - - targets: [':', ':', ':'] - - - job_name: frontend - static_configs: - - targets: [':'] -``` - -__2. Grafana config__ - -Create a Prometheus data source in Grafana before using this dashboard. We use `datasource` as a variable in Grafana dashboard so that multiple environments are supported. - -### Usage - -Use `datasource` or `instance` on the upper-left corner to filter data from certain node. + - **Cluster**: Import the `cluster/dashboard.json` dashboard. + - **Standalone**: Import the `standalone/dashboard.json` dashboard. diff --git a/grafana/check.sh b/grafana/check.sh deleted file mode 100755 index 9cab07391c..0000000000 --- a/grafana/check.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -BASEDIR=$(dirname "$0") - -# Use jq to check for panels with empty or missing descriptions -invalid_panels=$(cat $BASEDIR/greptimedb-cluster.json | jq -r ' - .panels[] - | select((.type == "stats" or .type == "timeseries") and (.description == "" or .description == null)) -') - -# Check if any invalid panels were found -if [[ -n "$invalid_panels" ]]; then - echo "Error: The following panels have empty or missing descriptions:" - echo "$invalid_panels" - exit 1 -else - echo "All panels with type 'stats' or 'timeseries' have valid descriptions." - exit 0 -fi diff --git a/grafana/dashboards/cluster/dashboard.json b/grafana/dashboards/cluster/dashboard.json new file mode 100644 index 0000000000..ef5490c888 --- /dev/null +++ b/grafana/dashboards/cluster/dashboard.json @@ -0,0 +1,7082 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "The Grafana dashboards for GreptimeDB.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 48, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 279, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "The start time of GreptimeDB.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "max": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 265, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "limit": 1, + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "time() - process_start_time_seconds", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "GreptimeDB version.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 3, + "y": 1 + }, + "id": 239, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^pkg_version$/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT pkg_version FROM information_schema.build_info", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Version", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total ingestion rate.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "max": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "rowsps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 5, + "y": 1 + }, + "id": 249, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total Ingestion Rate", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "Total number of data file size.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 9, + "y": 1 + }, + "id": 248, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select SUM(disk_size) from information_schema.region_statistics;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Total Storage Size", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "Total number of data rows in the cluster. Calculated by sum of rows from each region.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "sishort" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 13, + "y": 1 + }, + "id": 254, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select SUM(region_rows) from information_schema.region_statistics;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Total Rows", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "The deployment topology of GreptimeDB.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 0, + "y": 5 + }, + "id": 243, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT count(*) as datanode FROM information_schema.cluster_info WHERE peer_type = 'DATANODE';", + "refId": "datanode", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT count(*) as frontend FROM information_schema.cluster_info WHERE peer_type = 'FRONTEND';", + "refId": "frontend", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT count(*) as metasrv FROM information_schema.cluster_info WHERE peer_type = 'METASRV';", + "refId": "metasrv", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT count(*) as flownode FROM information_schema.cluster_info WHERE peer_type = 'FLOWNODE';", + "refId": "flownode", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Deployment", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "The number of the key resources in GreptimeDB.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 5, + "y": 5 + }, + "id": 247, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(*) as databases FROM information_schema.schemata WHERE schema_name NOT IN ('greptime_private', 'information_schema')", + "refId": "databases", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) as tables FROM information_schema.tables WHERE table_schema != 'information_schema'", + "refId": "tables", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(region_id) as regions FROM information_schema.region_peers", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(*) as flows FROM information_schema.flows", + "refId": "B", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Database Resources", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "The data size of wal/index/manifest in the GreptimeDB.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 10, + "y": 5 + }, + "id": 278, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(memtable_size) * 0.42825 as WAL FROM information_schema.region_statistics;\n", + "refId": "WAL", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(index_size) as index FROM information_schema.region_statistics;\n", + "refId": "Index", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(manifest_size) as manifest FROM information_schema.region_statistics;\n", + "refId": "manifest", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Data Size", + "type": "stat" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 275, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rowsps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 386 + }, + "id": 193, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_table_operator_ingest_rows{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "ingestion", + "range": true, + "refId": "C" + } + ], + "title": "Total Ingestion Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rowsps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 392 + }, + "id": 277, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http-logs", + "range": true, + "refId": "http_logs" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "prometheus-remote-write", + "range": true, + "refId": "prometheus-remote-write" + } + ], + "title": "Ingestion Rate by Type", + "type": "timeseries" + } + ], + "title": "Ingestion", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 276, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total rate of query API calls by protocol. This metric is collected from frontends.\n\nHere we listed 3 main protocols:\n- MySQL\n- Postgres\n- Prometheus API\n\nNote that there are some other minor query APIs like /sql are not included", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 407 + }, + "id": 255, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum (rate(greptime_servers_mysql_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "mysql", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum (rate(greptime_servers_postgres_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "pg", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum (rate(greptime_servers_http_promql_elapsed_counte{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "promql", + "range": true, + "refId": "C" + } + ], + "title": "Total Query Rate", + "type": "timeseries" + } + ], + "title": "Queries", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 274, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 256, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{instance=~\"$datanode\"}) by (instance, pod)", + "instant": false, + "legendFormat": "[{{instance}}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Datanode Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 262, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$datanode\"}[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Datanode CPU Usage per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 266, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{instance=~\"$frontend\"}) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Frontend Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 268, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$frontend\"}[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]-cpu", + "range": true, + "refId": "A" + } + ], + "title": "Frontend CPU Usage per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 269, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{instance=~\"$metasrv\"}) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]-resident", + "range": true, + "refId": "A" + } + ], + "title": "Metasrv Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 271, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$metasrv\"}[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Metasrv CPU Usage per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 272, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{instance=~\"$flownode\"}) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Flownode Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 273, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$flownode\"}[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Flownode CPU Usage per Instance", + "type": "timeseries" + } + ], + "title": "Resources", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 280, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "HTTP QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "[10.244.1.81:4000]-[mycluster-frontend-5bdf57f86-kshxt]-[/v1/prometheus/write]-[POST]-[500]" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 325 + }, + "id": 281, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~\"$frontend\",path!~\"/health|/metrics\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]", + "range": true, + "refId": "A" + } + ], + "title": "HTTP QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "HTTP P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 325 + }, + "id": 282, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~\"$frontend\",path!~\"/health|/metrics\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "HTTP P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "gRPC QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 333 + }, + "id": 283, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]", + "range": true, + "refId": "A" + } + ], + "title": "gRPC QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "gRPC P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 333 + }, + "id": 284, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "gRPC P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "MySQL QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 341 + }, + "id": 285, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "MySQL QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "MySQL P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 341 + }, + "id": 286, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "MySQL P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "PostgreSQL QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 349 + }, + "id": 287, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "PostgreSQL QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "PostgreSQL P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 349 + }, + "id": 288, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "PostgreSQL P99 per Instance", + "type": "timeseries" + } + ], + "title": "Frontend Requests", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 289, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Ingestion rate by row as in each frontend", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rowsps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 292, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{instance=~\"$frontend\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Ingest Rows per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Region Call QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 290, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{instance=~\"$frontend\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{request_type}}]", + "range": true, + "refId": "A" + } + ], + "title": "Region Call QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Region Call P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 291, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{instance=~\"$frontend\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{request_type}}]", + "range": true, + "refId": "A" + } + ], + "title": "Region Call P99 per Instance", + "type": "timeseries" + } + ], + "title": "Frontend to Datanode", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 293, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Request QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 294, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", + "range": true, + "refId": "A" + } + ], + "title": "Request OPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Request P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 295, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", + "range": true, + "refId": "A" + } + ], + "title": "Request P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write Buffer per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 296, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_mito_write_buffer_bytes{instance=~\"$datanode\"}", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write Buffer per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Ingestion size by row counts.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rowsps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 297, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_mito_write_rows_total{instance=~\"$datanode\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write Rows per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Flush QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 298, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~\"$datanode\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]", + "range": true, + "refId": "A" + } + ], + "title": "Flush OPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write Stall per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 299, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (greptime_mito_write_stall_total{instance=~\"$datanode\"})", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write Stall per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Read Stage OPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 300, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{instance=~\"$datanode\", stage=\"total\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Read Stage OPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Read Stage P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 301, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]", + "range": true, + "refId": "A" + } + ], + "title": "Read Stage P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write Stage P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 302, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write Stage P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction OPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 303, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Compaction OPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction latency by stage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 304, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "Compaction P99 per Instance by Stage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 305, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction", + "range": true, + "refId": "A" + } + ], + "title": "Compaction P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 55 + }, + "id": 306, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-throughput", + "range": true, + "refId": "C" + } + ], + "title": "WAL write size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Cached Bytes per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 55 + }, + "id": 307, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_mito_cache_bytes{instance=~\"$datanode\"}", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", + "range": true, + "refId": "A" + } + ], + "title": "Cached Bytes per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Ongoing compaction task count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 63 + }, + "id": 308, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_mito_inflight_compaction_count", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Inflight Compaction", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Raft engine (local disk) log store sync latency, p99", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 63 + }, + "id": 310, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "WAL sync duration seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write-ahead log operations latency at p99", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 71 + }, + "id": 311, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "Log Store op duration seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Ongoing flush task count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 71 + }, + "id": 312, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_mito_inflight_flush_count", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Inflight Flush", + "type": "timeseries" + } + ], + "title": "Mito Engine", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 313, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "[10.244.2.103:4000]-[mycluster-datanode-0]-[fs]-[delete]" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 314, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + } + ], + "title": "QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Read QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 315, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=\"read\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", + "range": true, + "refId": "A" + } + ], + "title": "Read QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Read P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 316, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\",operation=\"read\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-{{scheme}}", + "range": true, + "refId": "A" + } + ], + "title": "Read P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 317, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=\"write\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-{{scheme}}", + "range": true, + "refId": "A" + } + ], + "title": "Write QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 318, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation=\"write\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "List QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 319, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=\"list\"}[$__rate_interval]))", + "instant": false, + "interval": "", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", + "range": true, + "refId": "A" + } + ], + "title": "List QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "List P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 320, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation=\"list\"}[$__rate_interval])))", + "instant": false, + "interval": "", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", + "range": true, + "refId": "A" + } + ], + "title": "List P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Other Requests per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 321, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\",operation!~\"read|write|list|stat\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + } + ], + "title": "Other Requests per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Other Request P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 322, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation!~\"read|write|list\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + } + ], + "title": "Other Request P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total traffic as in bytes by instance and operation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "[mycluster-datanode-0]-[fs]-[Writer::write]" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 323, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~\"$datanode\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Opendal traffic", + "type": "timeseries" + } + ], + "title": "OpenDAL", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 324, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Counter of region migration by source and destination", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 325, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "auto", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_meta_region_migration_stat{datanode_type=\"src\"}", + "instant": false, + "legendFormat": "from-datanode-{{datanode_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_meta_region_migration_stat{datanode_type=\"desc\"}", + "hide": false, + "instant": false, + "legendFormat": "to-datanode-{{datanode_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Region migration datanode", + "type": "state-timeline" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Counter of region migration error", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 326, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_meta_region_migration_error", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Region migration error", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 327, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_datanode_load", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Datanode load", + "type": "timeseries" + } + ], + "title": "Metasrv", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 328, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Flow Ingest / Output Rate.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 329, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{pod}}]-[{{instance}}]-[{{direction}}]", + "range": true, + "refId": "A" + } + ], + "title": "Flow Ingest / Output Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Flow Ingest Latency.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 330, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-p95", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-p99", + "range": true, + "refId": "B" + } + ], + "title": "Flow Ingest Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Flow Operation Latency.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 0, + "y": 26 + }, + "id": 331, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-p95", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-p99", + "range": true, + "refId": "B" + } + ], + "title": "Flow Operation Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Flow Buffer Size per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 9, + "y": 26 + }, + "id": 332, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_flow_input_buf_size", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}]", + "range": true, + "refId": "A" + } + ], + "title": "Flow Buffer Size per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Flow Processing Error per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 26 + }, + "id": 333, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{code}}]", + "range": true, + "refId": "A" + } + ], + "title": "Flow Processing Error per Instance", + "type": "timeseries" + } + ], + "title": "Flownode", + "type": "row" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "metrics", + "value": "P177A7EA3611FE6B1" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "metrics", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "information_schema", + "value": "P0CE5E4D2C4819379" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "information_schema", + "options": [], + "query": "mysql", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": "", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "definition": "label_values(greptime_app_version{app=\"greptime-datanode\"},instance)", + "hide": 2, + "includeAll": true, + "multi": true, + "name": "datanode", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(greptime_app_version{app=\"greptime-datanode\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "definition": "label_values(greptime_app_version{app=\"greptime-frontend\"},instance)", + "hide": 2, + "includeAll": true, + "multi": true, + "name": "frontend", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(greptime_app_version{app=\"greptime-frontend\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": false, + "text": "10.244.1.79:4000", + "value": "10.244.1.79:4000" + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "definition": "label_values(greptime_app_version{app=\"greptime-metasrv\"},instance)", + "hide": 2, + "includeAll": true, + "multi": true, + "name": "metasrv", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(greptime_app_version{app=\"greptime-metasrv\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "definition": "label_values(greptime_app_version{app=\"greptime-flownode\"},instance)", + "hide": 2, + "includeAll": true, + "multi": true, + "name": "flownode", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(greptime_app_version{app=\"greptime-flownode\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GreptimeDB", + "uid": "dejf3k5e7g2kgb", + "version": 1, + "weekStart": "" +} diff --git a/grafana/dashboards/cluster/dashboard.md b/grafana/dashboards/cluster/dashboard.md new file mode 100644 index 0000000000..2bc100f860 --- /dev/null +++ b/grafana/dashboards/cluster/dashboard.md @@ -0,0 +1,96 @@ +# Overview +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Uptime | `time() - process_start_time_seconds` | `stat` | The start time of GreptimeDB. | `s` | `prometheus` | `__auto` | +| Version | `SELECT pkg_version FROM information_schema.build_info` | `stat` | GreptimeDB version. | -- | `mysql` | -- | +| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))` | `stat` | Total ingestion rate. | `rowsps` | `prometheus` | `__auto` | +| Total Storage Size | `select SUM(disk_size) from information_schema.region_statistics;` | `stat` | Total number of data file size. | `decbytes` | `mysql` | -- | +| Total Rows | `select SUM(region_rows) from information_schema.region_statistics;` | `stat` | Total number of data rows in the cluster. Calculated by sum of rows from each region. | `sishort` | `mysql` | -- | +| Deployment | `SELECT count(*) as datanode FROM information_schema.cluster_info WHERE peer_type = 'DATANODE';`
`SELECT count(*) as frontend FROM information_schema.cluster_info WHERE peer_type = 'FRONTEND';`
`SELECT count(*) as metasrv FROM information_schema.cluster_info WHERE peer_type = 'METASRV';`
`SELECT count(*) as flownode FROM information_schema.cluster_info WHERE peer_type = 'FLOWNODE';` | `stat` | The deployment topology of GreptimeDB. | -- | `mysql` | -- | +| Database Resources | `SELECT COUNT(*) as databases FROM information_schema.schemata WHERE schema_name NOT IN ('greptime_private', 'information_schema')`
`SELECT COUNT(*) as tables FROM information_schema.tables WHERE table_schema != 'information_schema'`
`SELECT COUNT(region_id) as regions FROM information_schema.region_peers`
`SELECT COUNT(*) as flows FROM information_schema.flows` | `stat` | The number of the key resources in GreptimeDB. | -- | `mysql` | -- | +| Data Size | `SELECT SUM(memtable_size) * 0.42825 as WAL FROM information_schema.region_statistics;`
`SELECT SUM(index_size) as index FROM information_schema.region_statistics;`
`SELECT SUM(manifest_size) as manifest FROM information_schema.region_statistics;` | `stat` | The data size of wal/index/manifest in the GreptimeDB. | `decbytes` | `mysql` | -- | +# Ingestion +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Total ingestion rate.

Here we listed 3 primary protocols:

- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `rowsps` | `prometheus` | `ingestion` | +| Ingestion Rate by Type | `sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))`
`sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))` | `timeseries` | Total ingestion rate.

Here we listed 3 primary protocols:

- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `rowsps` | `prometheus` | `http-logs` | +# Queries +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Total Query Rate | `sum (rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum (rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum (rate(greptime_servers_http_promql_elapsed_counte{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Total rate of query API calls by protocol. This metric is collected from frontends.

Here we listed 3 main protocols:
- MySQL
- Postgres
- Prometheus API

Note that there are some other minor query APIs like /sql are not included | `reqps` | `prometheus` | `mysql` | +# Resources +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Datanode Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$datanode"}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `decbytes` | `prometheus` | `[{{instance}}]-[{{ pod }}]` | +| Datanode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$datanode"}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `none` | `prometheus` | `[{{ instance }}]-[{{ pod }}]` | +| Frontend Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$frontend"}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `decbytes` | `prometheus` | `[{{ instance }}]-[{{ pod }}]` | +| Frontend CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$frontend"}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `none` | `prometheus` | `[{{ instance }}]-[{{ pod }}]-cpu` | +| Metasrv Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$metasrv"}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `decbytes` | `prometheus` | `[{{ instance }}]-[{{ pod }}]-resident` | +| Metasrv CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$metasrv"}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `none` | `prometheus` | `[{{ instance }}]-[{{ pod }}]` | +| Flownode Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$flownode"}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `decbytes` | `prometheus` | `[{{ instance }}]-[{{ pod }}]` | +| Flownode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$flownode"}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `none` | `prometheus` | `[{{ instance }}]-[{{ pod }}]` | +# Frontend Requests +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| HTTP QPS per Instance | `sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | HTTP QPS per Instance. | `reqps` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]` | +| HTTP P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval])))` | `timeseries` | HTTP P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` | +| gRPC QPS per Instance | `sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | gRPC QPS per Instance. | `reqps` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]` | +| gRPC P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | gRPC P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` | +| MySQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | MySQL QPS per Instance. | `reqps` | `prometheus` | `[{{instance}}]-[{{pod}}]` | +| MySQL P99 per Instance | `histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | MySQL P99 per Instance. | `s` | `prometheus` | `[{{ instance }}]-[{{ pod }}]-p99` | +| PostgreSQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | PostgreSQL QPS per Instance. | `reqps` | `prometheus` | `[{{instance}}]-[{{pod}}]` | +| PostgreSQL P99 per Instance | `histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | PostgreSQL P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-p99` | +# Frontend to Datanode +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Ingest Rows per Instance | `sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Ingestion rate by row as in each frontend | `rowsps` | `prometheus` | `[{{instance}}]-[{{pod}}]` | +| Region Call QPS per Instance | `sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Region Call QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` | +| Region Call P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | Region Call P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` | +# Mito Engine +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Request OPS per Instance | `sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Request QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | +| Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Request P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | +| Write Buffer per Instance | `greptime_mito_write_buffer_bytes{instance=~"$datanode"}` | `timeseries` | Write Buffer per Instance. | `decbytes` | `prometheus` | `[{{instance}}]-[{{pod}}]` | +| Write Rows per Instance | `sum by (instance, pod) (rate(greptime_mito_write_rows_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Ingestion size by row counts. | `rowsps` | `prometheus` | `[{{instance}}]-[{{pod}}]` | +| Flush OPS per Instance | `sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Flush QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{reason}}]` | +| Write Stall per Instance | `sum by(instance, pod) (greptime_mito_write_stall_total{instance=~"$datanode"})` | `timeseries` | Write Stall per Instance. | `decbytes` | `prometheus` | `[{{instance}}]-[{{pod}}]` | +| Read Stage OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{instance=~"$datanode", stage="total"}[$__rate_interval]))` | `timeseries` | Read Stage OPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]` | +| Read Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Read Stage P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | +| Write Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Write Stage P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | +| Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `ops` | `prometheus` | `[{{ instance }}]-[{{pod}}]` | +| Compaction P99 per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Compaction latency by stage | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` | +| Compaction P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Compaction P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction` | +| WAL write size | `histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))` | `timeseries` | Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. | `bytes` | `prometheus` | `[{{instance}}]-[{{pod}}]-req-size-p95` | +| Cached Bytes per Instance | `greptime_mito_cache_bytes{instance=~"$datanode"}` | `timeseries` | Cached Bytes per Instance. | `decbytes` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | +| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction task count | `none` | `prometheus` | `[{{instance}}]-[{{pod}}]` | +| WAL sync duration seconds | `histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))` | `timeseries` | Raft engine (local disk) log store sync latency, p99 | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-p99` | +| Log Store op duration seconds | `histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))` | `timeseries` | Write-ahead log operations latency at p99 | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99` | +| Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `none` | `prometheus` | `[{{instance}}]-[{{pod}}]` | +# OpenDAL +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Read QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | +| Read P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode",operation="read"}[$__rate_interval])))` | `timeseries` | Read P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-{{scheme}}` | +| Write QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="write"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-{{scheme}}` | +| Write P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="write"}[$__rate_interval])))` | `timeseries` | Write P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | +| List QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="list"}[$__rate_interval]))` | `timeseries` | List QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | +| List P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="list"}[$__rate_interval])))` | `timeseries` | List P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | +| Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode",operation!~"read\|write\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Other Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation!~"read\|write\|list"}[$__rate_interval])))` | `timeseries` | Other Request P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +# Metasrv +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Region migration datanode | `greptime_meta_region_migration_stat{datanode_type="src"}`
`greptime_meta_region_migration_stat{datanode_type="desc"}` | `state-timeline` | Counter of region migration by source and destination | `none` | `prometheus` | `from-datanode-{{datanode_id}}` | +| Region migration error | `greptime_meta_region_migration_error` | `timeseries` | Counter of region migration error | `none` | `prometheus` | `__auto` | +| Datanode load | `greptime_datanode_load` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `none` | `prometheus` | `__auto` | +# Flownode +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Flow Ingest / Output Rate | `sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))` | `timeseries` | Flow Ingest / Output Rate. | -- | `prometheus` | `[{{pod}}]-[{{instance}}]-[{{direction}}]` | +| Flow Ingest Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`
`histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))` | `timeseries` | Flow Ingest Latency. | -- | `prometheus` | `[{{instance}}]-[{{pod}}]-p95` | +| Flow Operation Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))`
`histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))` | `timeseries` | Flow Operation Latency. | -- | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` | +| Flow Buffer Size per Instance | `greptime_flow_input_buf_size` | `timeseries` | Flow Buffer Size per Instance. | -- | `prometheus` | `[{{instance}}]-[{{pod}]` | +| Flow Processing Error per Instance | `sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))` | `timeseries` | Flow Processing Error per Instance. | -- | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{code}}]` | diff --git a/grafana/dashboards/cluster/dashboard.yaml b/grafana/dashboards/cluster/dashboard.yaml new file mode 100644 index 0000000000..622cb2d548 --- /dev/null +++ b/grafana/dashboards/cluster/dashboard.yaml @@ -0,0 +1,761 @@ +groups: + - title: Overview + panels: + - title: Uptime + type: stat + description: The start time of GreptimeDB. + unit: s + queries: + - expr: time() - process_start_time_seconds + datasource: + type: prometheus + uid: ${metrics} + legendFormat: __auto + - title: Version + type: stat + description: GreptimeDB version. + queries: + - expr: SELECT pkg_version FROM information_schema.build_info + datasource: + type: mysql + uid: ${information_schema} + - title: Total Ingestion Rate + type: stat + description: Total ingestion rate. + unit: rowsps + queries: + - expr: sum(rate(greptime_table_operator_ingest_rows[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: __auto + - title: Total Storage Size + type: stat + description: Total number of data file size. + unit: decbytes + queries: + - expr: select SUM(disk_size) from information_schema.region_statistics; + datasource: + type: mysql + uid: ${information_schema} + - title: Total Rows + type: stat + description: Total number of data rows in the cluster. Calculated by sum of rows from each region. + unit: sishort + queries: + - expr: select SUM(region_rows) from information_schema.region_statistics; + datasource: + type: mysql + uid: ${information_schema} + - title: Deployment + type: stat + description: The deployment topology of GreptimeDB. + queries: + - expr: SELECT count(*) as datanode FROM information_schema.cluster_info WHERE peer_type = 'DATANODE'; + datasource: + type: mysql + uid: ${information_schema} + - expr: SELECT count(*) as frontend FROM information_schema.cluster_info WHERE peer_type = 'FRONTEND'; + datasource: + type: mysql + uid: ${information_schema} + - expr: SELECT count(*) as metasrv FROM information_schema.cluster_info WHERE peer_type = 'METASRV'; + datasource: + type: mysql + uid: ${information_schema} + - expr: SELECT count(*) as flownode FROM information_schema.cluster_info WHERE peer_type = 'FLOWNODE'; + datasource: + type: mysql + uid: ${information_schema} + - title: Database Resources + type: stat + description: The number of the key resources in GreptimeDB. + queries: + - expr: SELECT COUNT(*) as databases FROM information_schema.schemata WHERE schema_name NOT IN ('greptime_private', 'information_schema') + datasource: + type: mysql + uid: ${information_schema} + - expr: SELECT COUNT(*) as tables FROM information_schema.tables WHERE table_schema != 'information_schema' + datasource: + type: mysql + uid: ${information_schema} + - expr: SELECT COUNT(region_id) as regions FROM information_schema.region_peers + datasource: + type: mysql + uid: ${information_schema} + - expr: SELECT COUNT(*) as flows FROM information_schema.flows + datasource: + type: mysql + uid: ${information_schema} + - title: Data Size + type: stat + description: The data size of wal/index/manifest in the GreptimeDB. + unit: decbytes + queries: + - expr: SELECT SUM(memtable_size) * 0.42825 as WAL FROM information_schema.region_statistics; + datasource: + type: mysql + uid: ${information_schema} + - expr: SELECT SUM(index_size) as index FROM information_schema.region_statistics; + datasource: + type: mysql + uid: ${information_schema} + - expr: SELECT SUM(manifest_size) as manifest FROM information_schema.region_statistics; + datasource: + type: mysql + uid: ${information_schema} + - title: Ingestion + panels: + - title: Total Ingestion Rate + type: timeseries + description: | + Total ingestion rate. + + Here we listed 3 primary protocols: + + - Prometheus remote write + - Greptime's gRPC API (when using our ingest SDK) + - Log ingestion http API + unit: rowsps + queries: + - expr: sum(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: ingestion + - title: Ingestion Rate by Type + type: timeseries + description: | + Total ingestion rate. + + Here we listed 3 primary protocols: + + - Prometheus remote write + - Greptime's gRPC API (when using our ingest SDK) + - Log ingestion http API + unit: rowsps + queries: + - expr: sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: http-logs + - expr: sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: prometheus-remote-write + - title: Queries + panels: + - title: Total Query Rate + type: timeseries + description: |- + Total rate of query API calls by protocol. This metric is collected from frontends. + + Here we listed 3 main protocols: + - MySQL + - Postgres + - Prometheus API + + Note that there are some other minor query APIs like /sql are not included + unit: reqps + queries: + - expr: sum (rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: mysql + - expr: sum (rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: pg + - expr: sum (rate(greptime_servers_http_promql_elapsed_counte{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: promql + - title: Resources + panels: + - title: Datanode Memory per Instance + type: timeseries + description: Current memory usage by instance + unit: decbytes + queries: + - expr: sum(process_resident_memory_bytes{instance=~"$datanode"}) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{ pod }}]' + - title: Datanode CPU Usage per Instance + type: timeseries + description: Current cpu usage by instance + unit: none + queries: + - expr: sum(rate(process_cpu_seconds_total{instance=~"$datanode"}[$__rate_interval]) * 1000) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{ pod }}]' + - title: Frontend Memory per Instance + type: timeseries + description: Current memory usage by instance + unit: decbytes + queries: + - expr: sum(process_resident_memory_bytes{instance=~"$frontend"}) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{ pod }}]' + - title: Frontend CPU Usage per Instance + type: timeseries + description: Current cpu usage by instance + unit: none + queries: + - expr: sum(rate(process_cpu_seconds_total{instance=~"$frontend"}[$__rate_interval]) * 1000) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{ pod }}]-cpu' + - title: Metasrv Memory per Instance + type: timeseries + description: Current memory usage by instance + unit: decbytes + queries: + - expr: sum(process_resident_memory_bytes{instance=~"$metasrv"}) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{ pod }}]-resident' + - title: Metasrv CPU Usage per Instance + type: timeseries + description: Current cpu usage by instance + unit: none + queries: + - expr: sum(rate(process_cpu_seconds_total{instance=~"$metasrv"}[$__rate_interval]) * 1000) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{ pod }}]' + - title: Flownode Memory per Instance + type: timeseries + description: Current memory usage by instance + unit: decbytes + queries: + - expr: sum(process_resident_memory_bytes{instance=~"$flownode"}) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{ pod }}]' + - title: Flownode CPU Usage per Instance + type: timeseries + description: Current cpu usage by instance + unit: none + queries: + - expr: sum(rate(process_cpu_seconds_total{instance=~"$flownode"}[$__rate_interval]) * 1000) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{ pod }}]' + - title: Frontend Requests + panels: + - title: HTTP QPS per Instance + type: timeseries + description: HTTP QPS per Instance. + unit: reqps + queries: + - expr: sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health|/metrics"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]' + - title: HTTP P99 per Instance + type: timeseries + description: HTTP P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~"$frontend",path!~"/health|/metrics"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99' + - title: gRPC QPS per Instance + type: timeseries + description: gRPC QPS per Instance. + unit: reqps + queries: + - expr: sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]' + - title: gRPC P99 per Instance + type: timeseries + description: gRPC P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~"$frontend"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99' + - title: MySQL QPS per Instance + type: timeseries + description: MySQL QPS per Instance. + unit: reqps + queries: + - expr: sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: MySQL P99 per Instance + type: timeseries + description: MySQL P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{ pod }}]-p99' + - title: PostgreSQL QPS per Instance + type: timeseries + description: PostgreSQL QPS per Instance. + unit: reqps + queries: + - expr: sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: PostgreSQL P99 per Instance + type: timeseries + description: PostgreSQL P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-p99' + - title: Frontend to Datanode + panels: + - title: Ingest Rows per Instance + type: timeseries + description: Ingestion rate by row as in each frontend + unit: rowsps + queries: + - expr: sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Region Call QPS per Instance + type: timeseries + description: Region Call QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{request_type}}]' + - title: Region Call P99 per Instance + type: timeseries + description: Region Call P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{instance=~"$frontend"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{request_type}}]' + - title: Mito Engine + panels: + - title: Request OPS per Instance + type: timeseries + description: Request QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]' + - title: Request P99 per Instance + type: timeseries + description: Request P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{instance=~"$datanode"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]' + - title: Write Buffer per Instance + type: timeseries + description: Write Buffer per Instance. + unit: decbytes + queries: + - expr: greptime_mito_write_buffer_bytes{instance=~"$datanode"} + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Write Rows per Instance + type: timeseries + description: Ingestion size by row counts. + unit: rowsps + queries: + - expr: sum by (instance, pod) (rate(greptime_mito_write_rows_total{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Flush OPS per Instance + type: timeseries + description: Flush QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{reason}}]' + - title: Write Stall per Instance + type: timeseries + description: Write Stall per Instance. + unit: decbytes + queries: + - expr: sum by(instance, pod) (greptime_mito_write_stall_total{instance=~"$datanode"}) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Read Stage OPS per Instance + type: timeseries + description: Read Stage OPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{instance=~"$datanode", stage="total"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Read Stage P99 per Instance + type: timeseries + description: Read Stage P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]' + - title: Write Stage P99 per Instance + type: timeseries + description: Write Stage P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]' + - title: Compaction OPS per Instance + type: timeseries + description: Compaction OPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{pod}}]' + - title: Compaction P99 per Instance by Stage + type: timeseries + description: Compaction latency by stage + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-p99' + - title: Compaction P99 per Instance + type: timeseries + description: Compaction P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~"$datanode"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction' + - title: WAL write size + type: timeseries + description: Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. + unit: bytes + queries: + - expr: histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-req-size-p95' + - expr: histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-req-size-p99' + - expr: sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-throughput' + - title: Cached Bytes per Instance + type: timeseries + description: Cached Bytes per Instance. + unit: decbytes + queries: + - expr: greptime_mito_cache_bytes{instance=~"$datanode"} + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]' + - title: Inflight Compaction + type: timeseries + description: Ongoing compaction task count + unit: none + queries: + - expr: greptime_mito_inflight_compaction_count + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: WAL sync duration seconds + type: timeseries + description: Raft engine (local disk) log store sync latency, p99 + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-p99' + - title: Log Store op duration seconds + type: timeseries + description: Write-ahead log operations latency at p99 + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99' + - title: Inflight Flush + type: timeseries + description: Ongoing flush task count + unit: none + queries: + - expr: greptime_mito_inflight_flush_count + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: OpenDAL + panels: + - title: QPS per Instance + type: timeseries + description: QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - title: Read QPS per Instance + type: timeseries + description: Read QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="read"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]' + - title: Read P99 per Instance + type: timeseries + description: Read P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode",operation="read"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-{{scheme}}' + - title: Write QPS per Instance + type: timeseries + description: Write QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="write"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-{{scheme}}' + - title: Write P99 per Instance + type: timeseries + description: Write P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="write"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]' + - title: List QPS per Instance + type: timeseries + description: List QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="list"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]' + - title: List P99 per Instance + type: timeseries + description: List P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="list"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]' + - title: Other Requests per Instance + type: timeseries + description: Other Requests per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode",operation!~"read|write|list|stat"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - title: Other Request P99 per Instance + type: timeseries + description: Other Request P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation!~"read|write|list"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - title: Opendal traffic + type: timeseries + description: Total traffic as in bytes by instance and operation + unit: ops + queries: + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - title: Metasrv + panels: + - title: Region migration datanode + type: state-timeline + description: Counter of region migration by source and destination + unit: none + queries: + - expr: greptime_meta_region_migration_stat{datanode_type="src"} + datasource: + type: prometheus + uid: ${metrics} + legendFormat: from-datanode-{{datanode_id}} + - expr: greptime_meta_region_migration_stat{datanode_type="desc"} + datasource: + type: prometheus + uid: ${metrics} + legendFormat: to-datanode-{{datanode_id}} + - title: Region migration error + type: timeseries + description: Counter of region migration error + unit: none + queries: + - expr: greptime_meta_region_migration_error + datasource: + type: prometheus + uid: ${metrics} + legendFormat: __auto + - title: Datanode load + type: timeseries + description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. + unit: none + queries: + - expr: greptime_datanode_load + datasource: + type: prometheus + uid: ${metrics} + legendFormat: __auto + - title: Flownode + panels: + - title: Flow Ingest / Output Rate + type: timeseries + description: Flow Ingest / Output Rate. + queries: + - expr: sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{pod}}]-[{{instance}}]-[{{direction}}]' + - title: Flow Ingest Latency + type: timeseries + description: Flow Ingest Latency. + queries: + - expr: histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod)) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-p95' + - expr: histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod)) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-p99' + - title: Flow Operation Latency + type: timeseries + description: Flow Operation Latency. + queries: + - expr: histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type)) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p95' + - expr: histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type)) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p99' + - title: Flow Buffer Size per Instance + type: timeseries + description: Flow Buffer Size per Instance. + queries: + - expr: greptime_flow_input_buf_size + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}]' + - title: Flow Processing Error per Instance + type: timeseries + description: Flow Processing Error per Instance. + queries: + - expr: sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{code}}]' diff --git a/grafana/dashboards/standalone/dashboard.json b/grafana/dashboards/standalone/dashboard.json new file mode 100644 index 0000000000..fcf7f0d04e --- /dev/null +++ b/grafana/dashboards/standalone/dashboard.json @@ -0,0 +1,7082 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "The Grafana dashboards for GreptimeDB.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 48, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 279, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "The start time of GreptimeDB.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "max": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 265, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "limit": 1, + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "time() - process_start_time_seconds", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "GreptimeDB version.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 3, + "y": 1 + }, + "id": 239, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^pkg_version$/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT pkg_version FROM information_schema.build_info", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Version", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total ingestion rate.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "max": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "rowsps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 5, + "y": 1 + }, + "id": 249, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total Ingestion Rate", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "Total number of data file size.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 9, + "y": 1 + }, + "id": 248, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select SUM(disk_size) from information_schema.region_statistics;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Total Storage Size", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "Total number of data rows in the cluster. Calculated by sum of rows from each region.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "sishort" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 13, + "y": 1 + }, + "id": 254, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select SUM(region_rows) from information_schema.region_statistics;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Total Rows", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "The deployment topology of GreptimeDB.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 0, + "y": 5 + }, + "id": 243, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT count(*) as datanode FROM information_schema.cluster_info WHERE peer_type = 'DATANODE';", + "refId": "datanode", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT count(*) as frontend FROM information_schema.cluster_info WHERE peer_type = 'FRONTEND';", + "refId": "frontend", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT count(*) as metasrv FROM information_schema.cluster_info WHERE peer_type = 'METASRV';", + "refId": "metasrv", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT count(*) as flownode FROM information_schema.cluster_info WHERE peer_type = 'FLOWNODE';", + "refId": "flownode", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Deployment", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "The number of the key resources in GreptimeDB.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 5, + "y": 5 + }, + "id": 247, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(*) as databases FROM information_schema.schemata WHERE schema_name NOT IN ('greptime_private', 'information_schema')", + "refId": "databases", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) as tables FROM information_schema.tables WHERE table_schema != 'information_schema'", + "refId": "tables", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(region_id) as regions FROM information_schema.region_peers", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(*) as flows FROM information_schema.flows", + "refId": "B", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Database Resources", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "The data size of wal/index/manifest in the GreptimeDB.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 10, + "y": 5 + }, + "id": 278, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(memtable_size) * 0.42825 as WAL FROM information_schema.region_statistics;\n", + "refId": "WAL", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(index_size) as index FROM information_schema.region_statistics;\n", + "refId": "Index", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(manifest_size) as manifest FROM information_schema.region_statistics;\n", + "refId": "manifest", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Data Size", + "type": "stat" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 275, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rowsps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 386 + }, + "id": 193, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "ingestion", + "range": true, + "refId": "C" + } + ], + "title": "Total Ingestion Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rowsps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 392 + }, + "id": 277, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http-logs", + "range": true, + "refId": "http_logs" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "prometheus-remote-write", + "range": true, + "refId": "prometheus-remote-write" + } + ], + "title": "Ingestion Rate by Type", + "type": "timeseries" + } + ], + "title": "Ingestion", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 276, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total rate of query API calls by protocol. This metric is collected from frontends.\n\nHere we listed 3 main protocols:\n- MySQL\n- Postgres\n- Prometheus API\n\nNote that there are some other minor query APIs like /sql are not included", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 407 + }, + "id": 255, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum (rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval]))", + "instant": false, + "legendFormat": "mysql", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum (rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "pg", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum (rate(greptime_servers_http_promql_elapsed_counte{}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "promql", + "range": true, + "refId": "C" + } + ], + "title": "Total Query Rate", + "type": "timeseries" + } + ], + "title": "Queries", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 274, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 256, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{}) by (instance, pod)", + "instant": false, + "legendFormat": "[{{instance}}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Datanode Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 262, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Datanode CPU Usage per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 266, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{}) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Frontend Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 268, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]-cpu", + "range": true, + "refId": "A" + } + ], + "title": "Frontend CPU Usage per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 269, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{}) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]-resident", + "range": true, + "refId": "A" + } + ], + "title": "Metasrv Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 271, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Metasrv CPU Usage per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 272, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{}) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Flownode Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 273, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Flownode CPU Usage per Instance", + "type": "timeseries" + } + ], + "title": "Resources", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 280, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "HTTP QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "[10.244.1.81:4000]-[mycluster-frontend-5bdf57f86-kshxt]-[/v1/prometheus/write]-[POST]-[500]" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 325 + }, + "id": 281, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~\"/health|/metrics\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]", + "range": true, + "refId": "A" + } + ], + "title": "HTTP QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "HTTP P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 325 + }, + "id": 282, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{path!~\"/health|/metrics\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "HTTP P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "gRPC QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 333 + }, + "id": 283, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]", + "range": true, + "refId": "A" + } + ], + "title": "gRPC QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "gRPC P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 333 + }, + "id": 284, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "gRPC P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "MySQL QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 341 + }, + "id": 285, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "MySQL QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "MySQL P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 341 + }, + "id": 286, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "MySQL P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "PostgreSQL QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 349 + }, + "id": 287, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "PostgreSQL QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "PostgreSQL P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 349 + }, + "id": 288, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "PostgreSQL P99 per Instance", + "type": "timeseries" + } + ], + "title": "Frontend Requests", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 289, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Ingestion rate by row as in each frontend", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rowsps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 292, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Ingest Rows per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Region Call QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 290, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{request_type}}]", + "range": true, + "refId": "A" + } + ], + "title": "Region Call QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Region Call P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 291, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{request_type}}]", + "range": true, + "refId": "A" + } + ], + "title": "Region Call P99 per Instance", + "type": "timeseries" + } + ], + "title": "Frontend to Datanode", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 293, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Request QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 294, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", + "range": true, + "refId": "A" + } + ], + "title": "Request OPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Request P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 295, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", + "range": true, + "refId": "A" + } + ], + "title": "Request P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write Buffer per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 296, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_mito_write_buffer_bytes{}", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write Buffer per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Ingestion size by row counts.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rowsps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 297, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_mito_write_rows_total{}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write Rows per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Flush QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 298, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]", + "range": true, + "refId": "A" + } + ], + "title": "Flush OPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write Stall per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 299, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (greptime_mito_write_stall_total{})", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write Stall per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Read Stage OPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 300, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{ stage=\"total\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Read Stage OPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Read Stage P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 301, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]", + "range": true, + "refId": "A" + } + ], + "title": "Read Stage P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write Stage P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 302, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write Stage P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction OPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 303, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Compaction OPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction latency by stage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 304, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "Compaction P99 per Instance by Stage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 305, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction", + "range": true, + "refId": "A" + } + ], + "title": "Compaction P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 55 + }, + "id": 306, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-throughput", + "range": true, + "refId": "C" + } + ], + "title": "WAL write size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Cached Bytes per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 55 + }, + "id": 307, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_mito_cache_bytes{}", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", + "range": true, + "refId": "A" + } + ], + "title": "Cached Bytes per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Ongoing compaction task count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 63 + }, + "id": 308, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_mito_inflight_compaction_count", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Inflight Compaction", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Raft engine (local disk) log store sync latency, p99", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 63 + }, + "id": 310, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "WAL sync duration seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write-ahead log operations latency at p99", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 71 + }, + "id": 311, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "Log Store op duration seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Ongoing flush task count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 71 + }, + "id": 312, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_mito_inflight_flush_count", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Inflight Flush", + "type": "timeseries" + } + ], + "title": "Mito Engine", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 313, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "[10.244.2.103:4000]-[mycluster-datanode-0]-[fs]-[delete]" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 314, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + } + ], + "title": "QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Read QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 315, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation=\"read\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", + "range": true, + "refId": "A" + } + ], + "title": "Read QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Read P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 316, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{operation=\"read\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-{{scheme}}", + "range": true, + "refId": "A" + } + ], + "title": "Read P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 317, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation=\"write\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-{{scheme}}", + "range": true, + "refId": "A" + } + ], + "title": "Write QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 318, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation=\"write\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "List QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 319, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation=\"list\"}[$__rate_interval]))", + "instant": false, + "interval": "", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", + "range": true, + "refId": "A" + } + ], + "title": "List QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "List P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 320, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation=\"list\"}[$__rate_interval])))", + "instant": false, + "interval": "", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", + "range": true, + "refId": "A" + } + ], + "title": "List P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Other Requests per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 321, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~\"read|write|list|stat\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + } + ], + "title": "Other Requests per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Other Request P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 322, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation!~\"read|write|list\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + } + ], + "title": "Other Request P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total traffic as in bytes by instance and operation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "[mycluster-datanode-0]-[fs]-[Writer::write]" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 323, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Opendal traffic", + "type": "timeseries" + } + ], + "title": "OpenDAL", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 324, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Counter of region migration by source and destination", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 325, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "auto", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_meta_region_migration_stat{datanode_type=\"src\"}", + "instant": false, + "legendFormat": "from-datanode-{{datanode_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_meta_region_migration_stat{datanode_type=\"desc\"}", + "hide": false, + "instant": false, + "legendFormat": "to-datanode-{{datanode_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Region migration datanode", + "type": "state-timeline" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Counter of region migration error", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 326, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_meta_region_migration_error", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Region migration error", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 327, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_datanode_load", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Datanode load", + "type": "timeseries" + } + ], + "title": "Metasrv", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 328, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Flow Ingest / Output Rate.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 329, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{pod}}]-[{{instance}}]-[{{direction}}]", + "range": true, + "refId": "A" + } + ], + "title": "Flow Ingest / Output Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Flow Ingest Latency.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 330, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-p95", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-p99", + "range": true, + "refId": "B" + } + ], + "title": "Flow Ingest Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Flow Operation Latency.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 0, + "y": 26 + }, + "id": 331, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-p95", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-p99", + "range": true, + "refId": "B" + } + ], + "title": "Flow Operation Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Flow Buffer Size per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 9, + "y": 26 + }, + "id": 332, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_flow_input_buf_size", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}]", + "range": true, + "refId": "A" + } + ], + "title": "Flow Buffer Size per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Flow Processing Error per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 26 + }, + "id": 333, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{code}}]", + "range": true, + "refId": "A" + } + ], + "title": "Flow Processing Error per Instance", + "type": "timeseries" + } + ], + "title": "Flownode", + "type": "row" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "metrics", + "value": "P177A7EA3611FE6B1" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "metrics", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "information_schema", + "value": "P0CE5E4D2C4819379" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "information_schema", + "options": [], + "query": "mysql", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": "", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "definition": "label_values(greptime_app_version{app=\"greptime-datanode\"},instance)", + "hide": 2, + "includeAll": true, + "multi": true, + "name": "datanode", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(greptime_app_version{app=\"greptime-datanode\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "definition": "label_values(greptime_app_version{app=\"greptime-frontend\"},instance)", + "hide": 2, + "includeAll": true, + "multi": true, + "name": "frontend", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(greptime_app_version{app=\"greptime-frontend\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": false, + "text": "10.244.1.79:4000", + "value": "10.244.1.79:4000" + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "definition": "label_values(greptime_app_version{app=\"greptime-metasrv\"},instance)", + "hide": 2, + "includeAll": true, + "multi": true, + "name": "metasrv", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(greptime_app_version{app=\"greptime-metasrv\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "definition": "label_values(greptime_app_version{app=\"greptime-flownode\"},instance)", + "hide": 2, + "includeAll": true, + "multi": true, + "name": "flownode", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(greptime_app_version{app=\"greptime-flownode\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GreptimeDB", + "uid": "dejf3k5e7g2kgb", + "version": 1, + "weekStart": "" +} diff --git a/grafana/dashboards/standalone/dashboard.md b/grafana/dashboards/standalone/dashboard.md new file mode 100644 index 0000000000..a0ff6c03ca --- /dev/null +++ b/grafana/dashboards/standalone/dashboard.md @@ -0,0 +1,96 @@ +# Overview +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Uptime | `time() - process_start_time_seconds` | `stat` | The start time of GreptimeDB. | `s` | `prometheus` | `__auto` | +| Version | `SELECT pkg_version FROM information_schema.build_info` | `stat` | GreptimeDB version. | -- | `mysql` | -- | +| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))` | `stat` | Total ingestion rate. | `rowsps` | `prometheus` | `__auto` | +| Total Storage Size | `select SUM(disk_size) from information_schema.region_statistics;` | `stat` | Total number of data file size. | `decbytes` | `mysql` | -- | +| Total Rows | `select SUM(region_rows) from information_schema.region_statistics;` | `stat` | Total number of data rows in the cluster. Calculated by sum of rows from each region. | `sishort` | `mysql` | -- | +| Deployment | `SELECT count(*) as datanode FROM information_schema.cluster_info WHERE peer_type = 'DATANODE';`
`SELECT count(*) as frontend FROM information_schema.cluster_info WHERE peer_type = 'FRONTEND';`
`SELECT count(*) as metasrv FROM information_schema.cluster_info WHERE peer_type = 'METASRV';`
`SELECT count(*) as flownode FROM information_schema.cluster_info WHERE peer_type = 'FLOWNODE';` | `stat` | The deployment topology of GreptimeDB. | -- | `mysql` | -- | +| Database Resources | `SELECT COUNT(*) as databases FROM information_schema.schemata WHERE schema_name NOT IN ('greptime_private', 'information_schema')`
`SELECT COUNT(*) as tables FROM information_schema.tables WHERE table_schema != 'information_schema'`
`SELECT COUNT(region_id) as regions FROM information_schema.region_peers`
`SELECT COUNT(*) as flows FROM information_schema.flows` | `stat` | The number of the key resources in GreptimeDB. | -- | `mysql` | -- | +| Data Size | `SELECT SUM(memtable_size) * 0.42825 as WAL FROM information_schema.region_statistics;`
`SELECT SUM(index_size) as index FROM information_schema.region_statistics;`
`SELECT SUM(manifest_size) as manifest FROM information_schema.region_statistics;` | `stat` | The data size of wal/index/manifest in the GreptimeDB. | `decbytes` | `mysql` | -- | +# Ingestion +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))` | `timeseries` | Total ingestion rate.

Here we listed 3 primary protocols:

- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `rowsps` | `prometheus` | `ingestion` | +| Ingestion Rate by Type | `sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))`
`sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))` | `timeseries` | Total ingestion rate.

Here we listed 3 primary protocols:

- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `rowsps` | `prometheus` | `http-logs` | +# Queries +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Total Query Rate | `sum (rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval]))`
`sum (rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval]))`
`sum (rate(greptime_servers_http_promql_elapsed_counte{}[$__rate_interval]))` | `timeseries` | Total rate of query API calls by protocol. This metric is collected from frontends.

Here we listed 3 main protocols:
- MySQL
- Postgres
- Prometheus API

Note that there are some other minor query APIs like /sql are not included | `reqps` | `prometheus` | `mysql` | +# Resources +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Datanode Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `decbytes` | `prometheus` | `[{{instance}}]-[{{ pod }}]` | +| Datanode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `none` | `prometheus` | `[{{ instance }}]-[{{ pod }}]` | +| Frontend Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `decbytes` | `prometheus` | `[{{ instance }}]-[{{ pod }}]` | +| Frontend CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `none` | `prometheus` | `[{{ instance }}]-[{{ pod }}]-cpu` | +| Metasrv Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `decbytes` | `prometheus` | `[{{ instance }}]-[{{ pod }}]-resident` | +| Metasrv CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `none` | `prometheus` | `[{{ instance }}]-[{{ pod }}]` | +| Flownode Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `decbytes` | `prometheus` | `[{{ instance }}]-[{{ pod }}]` | +| Flownode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `none` | `prometheus` | `[{{ instance }}]-[{{ pod }}]` | +# Frontend Requests +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| HTTP QPS per Instance | `sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | HTTP QPS per Instance. | `reqps` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]` | +| HTTP P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{path!~"/health\|/metrics"}[$__rate_interval])))` | `timeseries` | HTTP P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` | +| gRPC QPS per Instance | `sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{}[$__rate_interval]))` | `timeseries` | gRPC QPS per Instance. | `reqps` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]` | +| gRPC P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | gRPC P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` | +| MySQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval]))` | `timeseries` | MySQL QPS per Instance. | `reqps` | `prometheus` | `[{{instance}}]-[{{pod}}]` | +| MySQL P99 per Instance | `histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | MySQL P99 per Instance. | `s` | `prometheus` | `[{{ instance }}]-[{{ pod }}]-p99` | +| PostgreSQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval]))` | `timeseries` | PostgreSQL QPS per Instance. | `reqps` | `prometheus` | `[{{instance}}]-[{{pod}}]` | +| PostgreSQL P99 per Instance | `histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | PostgreSQL P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-p99` | +# Frontend to Datanode +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Ingest Rows per Instance | `sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))` | `timeseries` | Ingestion rate by row as in each frontend | `rowsps` | `prometheus` | `[{{instance}}]-[{{pod}}]` | +| Region Call QPS per Instance | `sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{}[$__rate_interval]))` | `timeseries` | Region Call QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` | +| Region Call P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{}[$__rate_interval])))` | `timeseries` | Region Call P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` | +# Mito Engine +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Request OPS per Instance | `sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{}[$__rate_interval]))` | `timeseries` | Request QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | +| Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Request P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | +| Write Buffer per Instance | `greptime_mito_write_buffer_bytes{}` | `timeseries` | Write Buffer per Instance. | `decbytes` | `prometheus` | `[{{instance}}]-[{{pod}}]` | +| Write Rows per Instance | `sum by (instance, pod) (rate(greptime_mito_write_rows_total{}[$__rate_interval]))` | `timeseries` | Ingestion size by row counts. | `rowsps` | `prometheus` | `[{{instance}}]-[{{pod}}]` | +| Flush OPS per Instance | `sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{}[$__rate_interval]))` | `timeseries` | Flush QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{reason}}]` | +| Write Stall per Instance | `sum by(instance, pod) (greptime_mito_write_stall_total{})` | `timeseries` | Write Stall per Instance. | `decbytes` | `prometheus` | `[{{instance}}]-[{{pod}}]` | +| Read Stage OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{ stage="total"}[$__rate_interval]))` | `timeseries` | Read Stage OPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]` | +| Read Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Read Stage P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | +| Write Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Write Stage P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | +| Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{}[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `ops` | `prometheus` | `[{{ instance }}]-[{{pod}}]` | +| Compaction P99 per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Compaction latency by stage | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` | +| Compaction P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Compaction P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction` | +| WAL write size | `histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))` | `timeseries` | Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. | `bytes` | `prometheus` | `[{{instance}}]-[{{pod}}]-req-size-p95` | +| Cached Bytes per Instance | `greptime_mito_cache_bytes{}` | `timeseries` | Cached Bytes per Instance. | `decbytes` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | +| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction task count | `none` | `prometheus` | `[{{instance}}]-[{{pod}}]` | +| WAL sync duration seconds | `histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))` | `timeseries` | Raft engine (local disk) log store sync latency, p99 | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-p99` | +| Log Store op duration seconds | `histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))` | `timeseries` | Write-ahead log operations latency at p99 | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99` | +| Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `none` | `prometheus` | `[{{instance}}]-[{{pod}}]` | +# OpenDAL +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{}[$__rate_interval]))` | `timeseries` | QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Read QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | +| Read P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{operation="read"}[$__rate_interval])))` | `timeseries` | Read P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-{{scheme}}` | +| Write QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="write"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-{{scheme}}` | +| Write P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation="write"}[$__rate_interval])))` | `timeseries` | Write P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | +| List QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="list"}[$__rate_interval]))` | `timeseries` | List QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | +| List P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation="list"}[$__rate_interval])))` | `timeseries` | List P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | +| Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~"read\|write\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Other Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation!~"read\|write\|list"}[$__rate_interval])))` | `timeseries` | Other Request P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{}[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +# Metasrv +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Region migration datanode | `greptime_meta_region_migration_stat{datanode_type="src"}`
`greptime_meta_region_migration_stat{datanode_type="desc"}` | `state-timeline` | Counter of region migration by source and destination | `none` | `prometheus` | `from-datanode-{{datanode_id}}` | +| Region migration error | `greptime_meta_region_migration_error` | `timeseries` | Counter of region migration error | `none` | `prometheus` | `__auto` | +| Datanode load | `greptime_datanode_load` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `none` | `prometheus` | `__auto` | +# Flownode +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Flow Ingest / Output Rate | `sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))` | `timeseries` | Flow Ingest / Output Rate. | -- | `prometheus` | `[{{pod}}]-[{{instance}}]-[{{direction}}]` | +| Flow Ingest Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`
`histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))` | `timeseries` | Flow Ingest Latency. | -- | `prometheus` | `[{{instance}}]-[{{pod}}]-p95` | +| Flow Operation Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))`
`histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))` | `timeseries` | Flow Operation Latency. | -- | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` | +| Flow Buffer Size per Instance | `greptime_flow_input_buf_size` | `timeseries` | Flow Buffer Size per Instance. | -- | `prometheus` | `[{{instance}}]-[{{pod}]` | +| Flow Processing Error per Instance | `sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))` | `timeseries` | Flow Processing Error per Instance. | -- | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{code}}]` | diff --git a/grafana/dashboards/standalone/dashboard.yaml b/grafana/dashboards/standalone/dashboard.yaml new file mode 100644 index 0000000000..197478b5b1 --- /dev/null +++ b/grafana/dashboards/standalone/dashboard.yaml @@ -0,0 +1,761 @@ +groups: + - title: Overview + panels: + - title: Uptime + type: stat + description: The start time of GreptimeDB. + unit: s + queries: + - expr: time() - process_start_time_seconds + datasource: + type: prometheus + uid: ${metrics} + legendFormat: __auto + - title: Version + type: stat + description: GreptimeDB version. + queries: + - expr: SELECT pkg_version FROM information_schema.build_info + datasource: + type: mysql + uid: ${information_schema} + - title: Total Ingestion Rate + type: stat + description: Total ingestion rate. + unit: rowsps + queries: + - expr: sum(rate(greptime_table_operator_ingest_rows[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: __auto + - title: Total Storage Size + type: stat + description: Total number of data file size. + unit: decbytes + queries: + - expr: select SUM(disk_size) from information_schema.region_statistics; + datasource: + type: mysql + uid: ${information_schema} + - title: Total Rows + type: stat + description: Total number of data rows in the cluster. Calculated by sum of rows from each region. + unit: sishort + queries: + - expr: select SUM(region_rows) from information_schema.region_statistics; + datasource: + type: mysql + uid: ${information_schema} + - title: Deployment + type: stat + description: The deployment topology of GreptimeDB. + queries: + - expr: SELECT count(*) as datanode FROM information_schema.cluster_info WHERE peer_type = 'DATANODE'; + datasource: + type: mysql + uid: ${information_schema} + - expr: SELECT count(*) as frontend FROM information_schema.cluster_info WHERE peer_type = 'FRONTEND'; + datasource: + type: mysql + uid: ${information_schema} + - expr: SELECT count(*) as metasrv FROM information_schema.cluster_info WHERE peer_type = 'METASRV'; + datasource: + type: mysql + uid: ${information_schema} + - expr: SELECT count(*) as flownode FROM information_schema.cluster_info WHERE peer_type = 'FLOWNODE'; + datasource: + type: mysql + uid: ${information_schema} + - title: Database Resources + type: stat + description: The number of the key resources in GreptimeDB. + queries: + - expr: SELECT COUNT(*) as databases FROM information_schema.schemata WHERE schema_name NOT IN ('greptime_private', 'information_schema') + datasource: + type: mysql + uid: ${information_schema} + - expr: SELECT COUNT(*) as tables FROM information_schema.tables WHERE table_schema != 'information_schema' + datasource: + type: mysql + uid: ${information_schema} + - expr: SELECT COUNT(region_id) as regions FROM information_schema.region_peers + datasource: + type: mysql + uid: ${information_schema} + - expr: SELECT COUNT(*) as flows FROM information_schema.flows + datasource: + type: mysql + uid: ${information_schema} + - title: Data Size + type: stat + description: The data size of wal/index/manifest in the GreptimeDB. + unit: decbytes + queries: + - expr: SELECT SUM(memtable_size) * 0.42825 as WAL FROM information_schema.region_statistics; + datasource: + type: mysql + uid: ${information_schema} + - expr: SELECT SUM(index_size) as index FROM information_schema.region_statistics; + datasource: + type: mysql + uid: ${information_schema} + - expr: SELECT SUM(manifest_size) as manifest FROM information_schema.region_statistics; + datasource: + type: mysql + uid: ${information_schema} + - title: Ingestion + panels: + - title: Total Ingestion Rate + type: timeseries + description: | + Total ingestion rate. + + Here we listed 3 primary protocols: + + - Prometheus remote write + - Greptime's gRPC API (when using our ingest SDK) + - Log ingestion http API + unit: rowsps + queries: + - expr: sum(rate(greptime_table_operator_ingest_rows{}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: ingestion + - title: Ingestion Rate by Type + type: timeseries + description: | + Total ingestion rate. + + Here we listed 3 primary protocols: + + - Prometheus remote write + - Greptime's gRPC API (when using our ingest SDK) + - Log ingestion http API + unit: rowsps + queries: + - expr: sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: http-logs + - expr: sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: prometheus-remote-write + - title: Queries + panels: + - title: Total Query Rate + type: timeseries + description: |- + Total rate of query API calls by protocol. This metric is collected from frontends. + + Here we listed 3 main protocols: + - MySQL + - Postgres + - Prometheus API + + Note that there are some other minor query APIs like /sql are not included + unit: reqps + queries: + - expr: sum (rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: mysql + - expr: sum (rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: pg + - expr: sum (rate(greptime_servers_http_promql_elapsed_counte{}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: promql + - title: Resources + panels: + - title: Datanode Memory per Instance + type: timeseries + description: Current memory usage by instance + unit: decbytes + queries: + - expr: sum(process_resident_memory_bytes{}) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{ pod }}]' + - title: Datanode CPU Usage per Instance + type: timeseries + description: Current cpu usage by instance + unit: none + queries: + - expr: sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{ pod }}]' + - title: Frontend Memory per Instance + type: timeseries + description: Current memory usage by instance + unit: decbytes + queries: + - expr: sum(process_resident_memory_bytes{}) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{ pod }}]' + - title: Frontend CPU Usage per Instance + type: timeseries + description: Current cpu usage by instance + unit: none + queries: + - expr: sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{ pod }}]-cpu' + - title: Metasrv Memory per Instance + type: timeseries + description: Current memory usage by instance + unit: decbytes + queries: + - expr: sum(process_resident_memory_bytes{}) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{ pod }}]-resident' + - title: Metasrv CPU Usage per Instance + type: timeseries + description: Current cpu usage by instance + unit: none + queries: + - expr: sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{ pod }}]' + - title: Flownode Memory per Instance + type: timeseries + description: Current memory usage by instance + unit: decbytes + queries: + - expr: sum(process_resident_memory_bytes{}) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{ pod }}]' + - title: Flownode CPU Usage per Instance + type: timeseries + description: Current cpu usage by instance + unit: none + queries: + - expr: sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{ pod }}]' + - title: Frontend Requests + panels: + - title: HTTP QPS per Instance + type: timeseries + description: HTTP QPS per Instance. + unit: reqps + queries: + - expr: sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~"/health|/metrics"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]' + - title: HTTP P99 per Instance + type: timeseries + description: HTTP P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{path!~"/health|/metrics"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99' + - title: gRPC QPS per Instance + type: timeseries + description: gRPC QPS per Instance. + unit: reqps + queries: + - expr: sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]' + - title: gRPC P99 per Instance + type: timeseries + description: gRPC P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99' + - title: MySQL QPS per Instance + type: timeseries + description: MySQL QPS per Instance. + unit: reqps + queries: + - expr: sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: MySQL P99 per Instance + type: timeseries + description: MySQL P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{ pod }}]-p99' + - title: PostgreSQL QPS per Instance + type: timeseries + description: PostgreSQL QPS per Instance. + unit: reqps + queries: + - expr: sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: PostgreSQL P99 per Instance + type: timeseries + description: PostgreSQL P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-p99' + - title: Frontend to Datanode + panels: + - title: Ingest Rows per Instance + type: timeseries + description: Ingestion rate by row as in each frontend + unit: rowsps + queries: + - expr: sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Region Call QPS per Instance + type: timeseries + description: Region Call QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{request_type}}]' + - title: Region Call P99 per Instance + type: timeseries + description: Region Call P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{request_type}}]' + - title: Mito Engine + panels: + - title: Request OPS per Instance + type: timeseries + description: Request QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]' + - title: Request P99 per Instance + type: timeseries + description: Request P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]' + - title: Write Buffer per Instance + type: timeseries + description: Write Buffer per Instance. + unit: decbytes + queries: + - expr: greptime_mito_write_buffer_bytes{} + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Write Rows per Instance + type: timeseries + description: Ingestion size by row counts. + unit: rowsps + queries: + - expr: sum by (instance, pod) (rate(greptime_mito_write_rows_total{}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Flush OPS per Instance + type: timeseries + description: Flush QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{reason}}]' + - title: Write Stall per Instance + type: timeseries + description: Write Stall per Instance. + unit: decbytes + queries: + - expr: sum by(instance, pod) (greptime_mito_write_stall_total{}) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Read Stage OPS per Instance + type: timeseries + description: Read Stage OPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{ stage="total"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Read Stage P99 per Instance + type: timeseries + description: Read Stage P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]' + - title: Write Stage P99 per Instance + type: timeseries + description: Write Stage P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]' + - title: Compaction OPS per Instance + type: timeseries + description: Compaction OPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{pod}}]' + - title: Compaction P99 per Instance by Stage + type: timeseries + description: Compaction latency by stage + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-p99' + - title: Compaction P99 per Instance + type: timeseries + description: Compaction P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction' + - title: WAL write size + type: timeseries + description: Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. + unit: bytes + queries: + - expr: histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-req-size-p95' + - expr: histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-req-size-p99' + - expr: sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-throughput' + - title: Cached Bytes per Instance + type: timeseries + description: Cached Bytes per Instance. + unit: decbytes + queries: + - expr: greptime_mito_cache_bytes{} + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]' + - title: Inflight Compaction + type: timeseries + description: Ongoing compaction task count + unit: none + queries: + - expr: greptime_mito_inflight_compaction_count + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: WAL sync duration seconds + type: timeseries + description: Raft engine (local disk) log store sync latency, p99 + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-p99' + - title: Log Store op duration seconds + type: timeseries + description: Write-ahead log operations latency at p99 + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99' + - title: Inflight Flush + type: timeseries + description: Ongoing flush task count + unit: none + queries: + - expr: greptime_mito_inflight_flush_count + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: OpenDAL + panels: + - title: QPS per Instance + type: timeseries + description: QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - title: Read QPS per Instance + type: timeseries + description: Read QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="read"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]' + - title: Read P99 per Instance + type: timeseries + description: Read P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{operation="read"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-{{scheme}}' + - title: Write QPS per Instance + type: timeseries + description: Write QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="write"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-{{scheme}}' + - title: Write P99 per Instance + type: timeseries + description: Write P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation="write"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]' + - title: List QPS per Instance + type: timeseries + description: List QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="list"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]' + - title: List P99 per Instance + type: timeseries + description: List P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation="list"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]' + - title: Other Requests per Instance + type: timeseries + description: Other Requests per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~"read|write|list|stat"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - title: Other Request P99 per Instance + type: timeseries + description: Other Request P99 per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation!~"read|write|list"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - title: Opendal traffic + type: timeseries + description: Total traffic as in bytes by instance and operation + unit: ops + queries: + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - title: Metasrv + panels: + - title: Region migration datanode + type: state-timeline + description: Counter of region migration by source and destination + unit: none + queries: + - expr: greptime_meta_region_migration_stat{datanode_type="src"} + datasource: + type: prometheus + uid: ${metrics} + legendFormat: from-datanode-{{datanode_id}} + - expr: greptime_meta_region_migration_stat{datanode_type="desc"} + datasource: + type: prometheus + uid: ${metrics} + legendFormat: to-datanode-{{datanode_id}} + - title: Region migration error + type: timeseries + description: Counter of region migration error + unit: none + queries: + - expr: greptime_meta_region_migration_error + datasource: + type: prometheus + uid: ${metrics} + legendFormat: __auto + - title: Datanode load + type: timeseries + description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. + unit: none + queries: + - expr: greptime_datanode_load + datasource: + type: prometheus + uid: ${metrics} + legendFormat: __auto + - title: Flownode + panels: + - title: Flow Ingest / Output Rate + type: timeseries + description: Flow Ingest / Output Rate. + queries: + - expr: sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{pod}}]-[{{instance}}]-[{{direction}}]' + - title: Flow Ingest Latency + type: timeseries + description: Flow Ingest Latency. + queries: + - expr: histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod)) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-p95' + - expr: histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod)) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-p99' + - title: Flow Operation Latency + type: timeseries + description: Flow Operation Latency. + queries: + - expr: histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type)) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p95' + - expr: histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type)) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p99' + - title: Flow Buffer Size per Instance + type: timeseries + description: Flow Buffer Size per Instance. + queries: + - expr: greptime_flow_input_buf_size + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}]' + - title: Flow Processing Error per Instance + type: timeseries + description: Flow Processing Error per Instance. + queries: + - expr: sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{code}}]' diff --git a/grafana/greptimedb-cluster.json b/grafana/greptimedb-cluster.json deleted file mode 100644 index 512139db07..0000000000 --- a/grafana/greptimedb-cluster.json +++ /dev/null @@ -1,7518 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_MYSQL", - "label": "mysql", - "description": "", - "type": "datasource", - "pluginId": "mysql", - "pluginName": "MySQL" - }, - { - "name": "DS_PROMETHEUS", - "label": "prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__elements": {}, - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "10.2.3" - }, - { - "type": "datasource", - "id": "mysql", - "name": "MySQL", - "version": "1.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "state-timeline", - "name": "State timeline", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "The Grafana dashboard of GreptimeDB cluster.", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 189, - "panels": [], - "title": "Overview", - "type": "row" - }, - { - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "description": "Greptime DB version.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 0, - "y": 1 - }, - "id": 239, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^pkg_version$/", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "dataset": "information_schema", - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT pkg_version FROM information_schema.build_info", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Version", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total memory allocated by frontend node. Calculated from jemalloc metrics and may vary from system metrics.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 2, - "y": 1 - }, - "id": 240, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(sys_jemalloc_resident{pod=~\"$frontend\"})", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Frontend Memory", - "type": "stat" - }, - { - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "description": "Total number of active frontend nodes in the cluster.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 5, - "y": 1 - }, - "id": 241, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "dataset": "information_schema", - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT count(*) FROM information_schema.cluster_info WHERE peer_type = 'FRONTEND';", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Frontend Num", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total memory allocated by datanodes. Calculated from jemalloc metrics and may vary from system metrics.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 7, - "y": 1 - }, - "id": 242, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(sys_jemalloc_resident{pod=~\"$datanode\"})", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Datanode Memory", - "type": "stat" - }, - { - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "description": "Total number of active datanodes in the cluster.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 10, - "y": 1 - }, - "id": 243, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "dataset": "information_schema", - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT count(*) FROM information_schema.cluster_info WHERE peer_type = 'DATANODE';", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Datanode Num", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total memory allocated by metasrv. Calculated from jemalloc metrics and may vary from system metrics.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 12, - "y": 1 - }, - "id": 244, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(sys_jemalloc_resident{pod=~\"$metasrv\"})", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Metasrv Memory", - "type": "stat" - }, - { - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "description": "Total number of active metasrv instances", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 15, - "y": 1 - }, - "id": 245, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "dataset": "information_schema", - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT count(*) FROM information_schema.cluster_info WHERE peer_type = 'METASRV';", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Metasrv Num", - "type": "stat" - }, - { - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "description": "User-created database count.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 4 - }, - "id": 246, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "dataset": "information_schema", - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name NOT IN ('greptime_private', 'information_schema')", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Databases", - "type": "stat" - }, - { - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "description": "Total number of tables.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 4 - }, - "id": 247, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "dataset": "information_schema", - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema != 'information_schema'", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Tables", - "type": "stat" - }, - { - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "description": "Total number of data file size.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 6, - "y": 4 - }, - "id": 248, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "dataset": "information_schema", - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "select SUM(disk_size) from information_schema.region_statistics;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Storage Size", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total number of rows ingested into the cluster, per second.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "fieldMinMax": false, - "mappings": [], - "max": 2, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "rowsps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 10, - "y": 4 - }, - "id": 249, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Ingest Rows Rate", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total number of rows ingested via /events/logs endpoint, per second.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "fieldMinMax": false, - "mappings": [], - "max": 2, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "rowsps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 14, - "y": 4 - }, - "id": 265, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Log Ingest Rows Rate", - "type": "stat" - }, - { - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "description": "The approximate size of write-ahead logs", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 0, - "y": 8 - }, - "id": 250, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "dataset": "information_schema", - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "select SUM(memtable_size) * 0.42825 from information_schema.region_statistics;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "WAL Size", - "type": "stat" - }, - { - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "description": "Total size of index files.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 2, - "y": 8 - }, - "id": 251, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "dataset": "information_schema", - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "select SUM(index_size) from information_schema.region_statistics;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Index Size", - "type": "stat" - }, - { - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "description": "Total size of manifest file size. Manifest is a our table format metadata stored on object storage. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 4, - "y": 8 - }, - "id": 252, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "dataset": "information_schema", - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "select SUM(manifest_size) from information_schema.region_statistics;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Manifest Size", - "type": "stat" - }, - { - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "description": "Total number of partitions in the cluster.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 6, - "y": 8 - }, - "id": 253, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "dataset": "information_schema", - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT COUNT(region_id) FROM information_schema.region_peers", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Regions Count", - "type": "stat" - }, - { - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "description": "Total number of data rows in the cluster. Calculated by sum of rows from each region.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "sishort" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 8, - "y": 8 - }, - "id": 254, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "dataset": "information_schema", - "datasource": { - "type": "mysql", - "uid": "${DS_MYSQL}" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "select SUM(region_rows) from information_schema.region_statistics;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Region Rows", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total rate data ingestion API calls by protocol.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 193, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum (rate(greptime_servers_http_prometheus_write_elapsed_count{pod=~\"$frontend\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "prometheus-remote-write", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum (rate(greptime_servers_grpc_requests_elapsed_count{pod=~\"$frontend\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "gRPC", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum (rate(greptime_servers_http_logs_ingestion_elapsed_count{pod=~\"$frontend\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "http_logs", - "range": true, - "refId": "C" - } - ], - "title": "Ingestion", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total rate of query API calls by protocol. This metric is collected from frontends.\n\nHere we listed 3 main protocols:\n- MySQL\n- Postgres\n- Prometheus API\n\nNote that there are some other minor query APIs like /sql are not included", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 255, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum (rate(greptime_servers_mysql_query_elapsed_count{pod=~\"$frontend\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "mysql", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum (rate(greptime_servers_postgres_query_elapsed_count{pod=~\"$frontend\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "pg", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum (rate(greptime_servers_http_promql_elapsed_counte{pod=~\"$frontend\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "promql", - "range": true, - "refId": "C" - } - ], - "title": "Queries", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 237, - "panels": [], - "title": "Resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Memory usage information of datanodes.\n\nThere are three types of the metrics:\n\n- allocated from jemalloc\n- resident memory as stat from jemalloc\n- process virtual memory", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 8, - "x": 0, - "y": 24 - }, - "id": 234, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(sys_jemalloc_allocated{pod=~\"$datanode\"})", - "instant": false, - "legendFormat": "allocated", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(sys_jemalloc_resident{pod=~\"$datanode\"})", - "hide": false, - "instant": false, - "legendFormat": "resident", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(process_virtual_memory_bytes{pod=~\"$datanode\"})", - "hide": false, - "instant": false, - "legendFormat": "virtual-memory", - "range": true, - "refId": "C" - } - ], - "title": "Datanode Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Memory usage information of frontend.\n\nThere are three types of the metrics:\n\n- allocated from jemalloc\n- resident memory as stat from jemalloc\n- process virtual memory", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 8, - "x": 8, - "y": 24 - }, - "id": 233, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(sys_jemalloc_allocated{pod=~\"$frontend\"})", - "instant": false, - "legendFormat": "allocated", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(sys_jemalloc_resident{pod=~\"$frontend\"})", - "hide": false, - "instant": false, - "legendFormat": "resident", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(process_virtual_memory_bytes{pod=~\"$frontend\"})", - "hide": false, - "instant": false, - "legendFormat": "virtual-memory", - "range": true, - "refId": "C" - } - ], - "title": "Frontend Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Memory usage information of metasrv.\n\nThere are three types of the metrics:\n\n- allocated from jemalloc\n- resident memory as stat from jemalloc\n- process virtual memory", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 8, - "x": 16, - "y": 24 - }, - "id": 235, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(sys_jemalloc_allocated{pod=~\"$metasrv\"})", - "instant": false, - "legendFormat": "allocated", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(sys_jemalloc_resident{pod=~\"$metasrv\"})", - "hide": false, - "instant": false, - "legendFormat": "resident", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(process_virtual_memory_bytes{pod=~\"$metasrv\"})", - "hide": false, - "instant": false, - "legendFormat": "virtual-memory", - "range": true, - "refId": "C" - } - ], - "title": "Metasrv Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Current memory usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 8, - "x": 0, - "y": 34 - }, - "id": 256, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(sys_jemalloc_resident{pod=~\"$datanode\"}) by (pod)", - "instant": false, - "legendFormat": "[{{ pod }}]-resident", - "range": true, - "refId": "A" - } - ], - "title": "Datanode Memory per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Current memory usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 8, - "x": 8, - "y": 34 - }, - "id": 257, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(sys_jemalloc_resident{pod=~\"$frontend\"}) by (pod)", - "instant": false, - "legendFormat": "[{{ pod }}]-resident", - "range": true, - "refId": "A" - } - ], - "title": "Frontend Memory per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Current memory usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 8, - "x": 16, - "y": 34 - }, - "id": 258, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(sys_jemalloc_resident{pod=~\"$metasrv\"}) by (pod)", - "instant": false, - "legendFormat": "[{{ pod }}]-resident", - "range": true, - "refId": "A" - } - ], - "title": "Metasrv Memory per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Current cpu usage of all instances accumulated", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 8, - "x": 0, - "y": 44 - }, - "id": 259, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(rate(process_cpu_seconds_total{pod=~\"$datanode\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "cpu", - "range": true, - "refId": "A" - } - ], - "title": "Datanode CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Current cpu usage of all instances accumulated", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 8, - "x": 8, - "y": 44 - }, - "id": 260, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(rate(process_cpu_seconds_total{pod=~\"$frontend\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "cpu", - "range": true, - "refId": "A" - } - ], - "title": "Frontend CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Current cpu usage of all instances accumulated", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 8, - "x": 16, - "y": 44 - }, - "id": 261, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(rate(process_cpu_seconds_total{pod=~\"$metasrv\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "cpu", - "range": true, - "refId": "A" - } - ], - "title": "Metasrv CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Current cpu usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 8, - "x": 0, - "y": 54 - }, - "id": 262, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(rate(process_cpu_seconds_total{pod=~\"$datanode\"}[$__rate_interval])) by (pod)", - "instant": false, - "legendFormat": "[{{ pod }}]-cpu", - "range": true, - "refId": "A" - } - ], - "title": "Datanode CPU Usage per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Current cpu usage of all instances accumulated", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 8, - "x": 8, - "y": 54 - }, - "id": 263, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(rate(process_cpu_seconds_total{pod=~\"$frontend\"}[$__rate_interval])) by (pod)", - "instant": false, - "legendFormat": "[{{ pod }}]-cpu", - "range": true, - "refId": "A" - } - ], - "title": "Frontend CPU Usage per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Current cpu usage of all instances accumulated", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 8, - "x": 16, - "y": 54 - }, - "id": 264, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(rate(process_cpu_seconds_total{pod=~\"$metasrv\"}[$__rate_interval])) by (pod)", - "instant": false, - "legendFormat": "[{{ pod }}]-cpu", - "range": true, - "refId": "A" - } - ], - "title": "Metasrv CPU Usage per Instance", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 64 - }, - "id": 192, - "panels": [], - "title": "Frontend APIs", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "HTTP APIs QPS by instance, request url, http method and response status code", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 65 - }, - "id": 202, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{pod=~\"$frontend\",path!~\"/health|/metrics\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-qps", - "range": true, - "refId": "A" - } - ], - "title": "HTTP QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "P99 latency of HTTP requests by instance, request url, http method and response code", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 65 - }, - "id": 203, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{pod=~\"$frontend\",path!~\"/health|/metrics\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99", - "range": true, - "refId": "A" - } - ], - "title": "HTTP P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "gRPC requests QPS on frontends by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[mycluster-frontend-5f94445cf8-mcmhf]-[/v1/prometheus/write]-[POST]-[204]-qps" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 73 - }, - "id": 211, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{pod=~\"$frontend\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{pod}}]-[{{path}}]-[{{code}}]-qps", - "range": true, - "refId": "A" - } - ], - "title": "gRPC QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "gRPC latency p99 by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 73 - }, - "id": 212, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{pod=~\"$frontend\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99", - "range": true, - "refId": "A" - } - ], - "title": "gRPC P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "MySQL query rate by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[mycluster-frontend-5c59b4cc9b-kpb6q]-qps" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 81 - }, - "id": 213, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(pod)(rate(greptime_servers_mysql_query_elapsed_count{pod=~\"$frontend\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{pod}}]-qps", - "range": true, - "refId": "A" - } - ], - "title": "MySQL QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "MySQL query latency p99 by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 81 - }, - "id": 214, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(pod, le) (rate(greptime_servers_mysql_query_elapsed_bucket{pod=~\"$frontend\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{ pod }}]-p99", - "range": true, - "refId": "A" - } - ], - "title": "MySQL P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Postgres query rate by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[mycluster-frontend-5f94445cf8-mcmhf]-[/v1/prometheus/write]-[POST]-[204]-qps" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 89 - }, - "id": 215, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(pod) (rate(greptime_servers_postgres_query_elapsed_count{pod=~\"$frontend\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{pod}}]-qps", - "range": true, - "refId": "A" - } - ], - "title": "PostgreSQL QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Postgres query latency p99 by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 89 - }, - "id": 216, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(pod, le) (rate(greptime_servers_postgres_query_elapsed_count{pod=~\"$frontend\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{pod}}]-p99", - "range": true, - "refId": "A" - } - ], - "title": "PostgreSQL P99 per Instance", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 97 - }, - "id": 217, - "panels": [], - "title": "Frontend <-> Datanode", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Ingestion rate by row as in each frontend", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "rowsps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 98 - }, - "id": 218, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(pod)(rate(greptime_table_operator_ingest_rows{pod=~\"$frontend\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{pod}}]-rps", - "range": true, - "refId": "A" - } - ], - "title": "Ingest Rows per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Datanode query rate issued by each frontend", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 104 - }, - "id": 219, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(pod, request_type) (rate(greptime_grpc_region_request_count{pod=~\"$frontend\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{pod}}]-[{{request_type}}]-qps", - "range": true, - "refId": "A" - } - ], - "title": "Region Call QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Datanode query latency at p99 as seen by each frontend", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 104 - }, - "id": 220, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(pod, le, request_type) (rate(greptime_grpc_region_request_bucket{pod=~\"$frontend\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{pod}}]-[{{request_type}}]-p99", - "range": true, - "refId": "A" - } - ], - "title": "Region Call P99 per Instance", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 112 - }, - "id": 273, - "panels": [], - "title": "Metasrv", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Counter of region migration by source and destination", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "fillOpacity": 70, - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineWidth": 0, - "spanNulls": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 113 - }, - "id": 274, - "options": { - "alignValue": "left", - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "mergeValues": true, - "rowHeight": 0.9, - "showValue": "auto", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "greptime_meta_region_migration_stat{datanode_type=\"src\"}", - "instant": false, - "legendFormat": "from-datanode-{{datanode_id}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "greptime_meta_region_migration_stat{datanode_type=\"desc\"}", - "hide": false, - "instant": false, - "legendFormat": "to-datanode-{{datanode_id}}", - "range": true, - "refId": "B" - } - ], - "title": "Region migration datanode", - "type": "state-timeline" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Counter of region migration error", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 113 - }, - "id": 275, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "greptime_meta_region_migration_error", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Region migration error", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 121 - }, - "id": 276, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "greptime_datanode_load", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Datanode load", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 129 - }, - "id": 194, - "panels": [], - "title": "Mito Engine", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Datanode storage engine QPS by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 130 - }, - "id": 201, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(pod, type) (rate(greptime_mito_handle_request_elapsed_count{pod=~\"$datanode\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{pod}}]-[{{type}}]-qps", - "range": true, - "refId": "A" - } - ], - "title": "Request QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Storage query latency at p99 by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 130 - }, - "id": 222, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{pod=~\"$datanode\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{pod}}]-[{{type}}]-p99", - "range": true, - "refId": "A" - } - ], - "title": "Request P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Memtable size on each instance.\n\nThe memtable holds unflushed data in memory and will flush it to object storage periodically or when size exceed configured limit.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 138 - }, - "id": 200, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "greptime_mito_write_buffer_bytes{pod=~\"$datanode\"}", - "instant": false, - "legendFormat": "{{pod}}", - "range": true, - "refId": "A" - } - ], - "title": "Write Buffer per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Ingestion size by row counts.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 138 - }, - "id": 277, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(greptime_mito_write_rows_total{pod=~\"$datanode\"}[$__rate_interval])", - "instant": false, - "legendFormat": "{{pod}}", - "range": true, - "refId": "A" - } - ], - "title": "Write Rows per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Memtable flush rate by reason and instance.\n\nThere are several reasons when memtable get flushed. For example, it's full as in size, or reaching the time-to-flush, or by an artificial request.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 146 - }, - "id": 224, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(pod, reason) (rate(greptime_mito_flush_requests_total{pod=~\"$datanode\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{pod}}]-[{{reason}}]-success", - "range": true, - "refId": "A" - } - ], - "title": "Flush QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Current counts for stalled write requests by instance\n\nWrite stalls when memtable is full and pending for flush\n\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 146 - }, - "id": 221, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(pod) (greptime_mito_write_stall_total{pod=~\"$datanode\"})", - "instant": false, - "legendFormat": "{{pod}}", - "range": true, - "refId": "A" - } - ], - "title": "Write Stall per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Read QPS from the storage engine by instance.\n\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 154 - }, - "id": 227, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(pod) (rate(greptime_mito_read_stage_elapsed_count{pod=~\"$datanode\", stage=\"total\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "{{pod}}-p99", - "range": true, - "refId": "A" - } - ], - "title": "Read Stage QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Cache size by instance.\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 154 - }, - "id": 229, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "greptime_mito_cache_bytes{pod=~\"$datanode\"}", - "instant": false, - "legendFormat": "{{pod}}-{{type}}", - "range": true, - "refId": "A" - } - ], - "title": "Cached Bytes per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Compaction operation rate.\n\nCompaction happens when storage to merge and optimise data files.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 162 - }, - "id": 231, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(pod) (rate(greptime_mito_compaction_total_elapsed_count{pod=~\"$datanode\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "{{pod}}", - "range": true, - "refId": "A" - } - ], - "title": "Compaction OPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "P99 latency of each type of reads by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 162 - }, - "id": 228, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{pod=~\"$datanode\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "{{pod}}-{{stage}}-p99", - "range": true, - "refId": "A" - } - ], - "title": "Read Stage P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Write latency by instance and stage type", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 170 - }, - "id": 225, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{pod=~\"$datanode\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "{{pod}}-{{stage}}-p99", - "range": true, - "refId": "A" - } - ], - "title": "Write Stage P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Latency of compaction task, at p99", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 170 - }, - "id": 230, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket{pod=~\"$datanode\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{pod}}]-compaction-p99", - "range": true, - "refId": "A" - } - ], - "title": "Compaction P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 178 - }, - "id": 268, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", - "instant": false, - "legendFormat": "{{pod}}-req-size-p95", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "{{pod}}-req-size-p99", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(raft_engine_write_size_sum[$__rate_interval])", - "hide": false, - "instant": false, - "legendFormat": "{{pod}}-throughput", - "range": true, - "refId": "C" - } - ], - "title": "WAL write size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Compaction latency by stage", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 178 - }, - "id": 232, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{pod=~\"$datanode\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "{{pod}}-{{stage}}-p99", - "range": true, - "refId": "A" - } - ], - "title": "Compaction P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Raft engine (local disk) log store sync latency, p99", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 186 - }, - "id": 270, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, type, node, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))", - "instant": false, - "legendFormat": "{{pod}}-p99", - "range": true, - "refId": "A" - } - ], - "title": "WAL sync duration seconds", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Write-ahead log operations latency at p99", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 186 - }, - "id": 269, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))", - "instant": false, - "legendFormat": "{{pod}}-{{logstore}}-{{optype}}-p99", - "range": true, - "refId": "A" - } - ], - "title": "Log Store op duration seconds", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Ongoing flush task count", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 194 - }, - "id": 272, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "greptime_mito_inflight_flush_count", - "instant": false, - "legendFormat": "{{pod}}", - "range": true, - "refId": "A" - } - ], - "title": "Inflight Flush", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Ongoing compaction task count", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 194 - }, - "id": 271, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "greptime_mito_inflight_compaction_count", - "instant": false, - "legendFormat": "{{pod}}", - "range": true, - "refId": "A" - } - ], - "title": "Inflight Compaction", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 202 - }, - "id": 195, - "panels": [], - "title": "OpenDAL", - "type": "row" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "object storage query rate by datanode and operation type", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 203 - }, - "id": 209, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "sum by(pod, scheme, operation) (rate(opendal_operation_bytes_count{pod=~\"$datanode\"}[$__rate_interval]))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "[{{pod}}]-[{{scheme}}]-[{{operation}}]-qps", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total traffic as in bytes by instance and operation", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[mycluster-datanode-0]-[fs]-[Writer::write]" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 203 - }, - "id": 267, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "sum by(pod, scheme, operation) (rate(opendal_operation_bytes_sum{pod=~\"$datanode\"}[$__rate_interval]))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "[{{pod}}]-[{{scheme}}]-[{{operation}}]", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Opendal traffic", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Object storage read traffic rate as in bytes per second by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 210 - }, - "id": 196, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum by(pod, scheme) (rate(opendal_operation_bytes_count{pod=~\"$datanode\", operation=\"Reader::read\"}[$__rate_interval]))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "[{{pod}}]-[{{scheme}}]-qps", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Read QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Read operation latency at p99", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 210 - }, - "id": 198, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.99, sum by(pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{pod=~\"$datanode\", operation=\"Reader::read\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "[{{pod}}]-{{scheme}}-p99", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Read P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Object storage write traffic rate as in bytes per second by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 217 - }, - "id": 199, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum by(pod, scheme) (rate(opendal_operation_duration_seconds_count{pod=~\"$datanode\", operation=\"Writer::write\"}[$__rate_interval]))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "[{{pod}}]-[{{scheme}}]-qps", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Write QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Write operation latency at p99", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 217 - }, - "id": 204, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{pod=~\"$datanode\", operation=\"Writer::write\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{pod}}]-[{{scheme}}]-p99", - "range": true, - "refId": "A" - } - ], - "title": "Write P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Object storage list traffic rate as in bytes per second by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 224 - }, - "id": 205, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "sum by(pod, scheme) (rate(opendal_operation_duration_seconds_count{pod=~\"$datanode\", operation=\"list\"}[$__rate_interval]))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "interval": "", - "legendFormat": "[{{pod}}]-[{{scheme}}]-qps", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "List QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "List operation latency at p99", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 224 - }, - "id": 206, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{pod=~\"$datanode\", operation=\"list\"}[$__rate_interval])))", - "instant": false, - "interval": "", - "legendFormat": "[{{pod}}]-[{{scheme}}]-p99", - "range": true, - "refId": "A" - } - ], - "title": "List P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Object storage traffic rate other than read/write/list/stat as in bytes per second by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 231 - }, - "id": 207, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{pod=~\"$datanode\",operation!~\"read|write|list|stat\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{pod}}]-[{{scheme}}]-[{{operation}}]-qps", - "range": true, - "refId": "A" - } - ], - "title": "Other Requests per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "All other operation latency at p99", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 231 - }, - "id": 210, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{pod=~\"$datanode\", operation!~\"read|write|list\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{pod}}]-[{{scheme}}]-[{{operation}}]-p99", - "range": true, - "refId": "A" - } - ], - "title": "Other Request P99 per Instance", - "type": "timeseries" - } - ], - "refresh": "10s", - "schemaVersion": 39, - "tags": [], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "No data sources found", - "value": "" - }, - "hide": 0, - "includeAll": false, - "multi": false, - "name": "metrics", - "options": [], - "query": "prometheus", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "selected": false, - "text": "No data sources found", - "value": "" - }, - "hide": 0, - "includeAll": false, - "multi": false, - "name": "information_schema", - "options": [], - "query": "mysql", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(greptime_app_version{app=~\"greptime-datanode|greptime-frontend|greptime-metasrv|greptime-flownode\"},app)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "roles", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(greptime_app_version{app=~\"greptime-datanode|greptime-frontend|greptime-metasrv|greptime-flownode\"},app)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(greptime_app_version{app=~\"$roles\"},pod)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "pods", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(greptime_app_version{app=~\"$roles\"},pod)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(greptime_app_version{app=\"greptime-datanode\"},pod)", - "hide": 2, - "includeAll": true, - "multi": true, - "name": "datanode", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(greptime_app_version{app=\"greptime-datanode\"},pod)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(greptime_app_version{app=\"greptime-frontend\"},pod)", - "hide": 2, - "includeAll": true, - "multi": true, - "name": "frontend", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(greptime_app_version{app=\"greptime-frontend\"},pod)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(greptime_app_version{app=\"greptime-metasrv\"},pod)", - "hide": 2, - "includeAll": true, - "multi": true, - "name": "metasrv", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(greptime_app_version{app=\"greptime-metasrv\"},pod)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "label_values(greptime_app_version{app=\"greptime-flownode\"},pod)", - "hide": 2, - "includeAll": true, - "multi": true, - "name": "flownode", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(greptime_app_version{app=\"greptime-flownode\"},pod)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "GreptimeDB Cluster Metrics", - "uid": "ce3q6xwn3xa0qs", - "version": 10, - "weekStart": "" -} diff --git a/grafana/greptimedb.json b/grafana/greptimedb.json deleted file mode 100644 index a5913ee8e8..0000000000 --- a/grafana/greptimedb.json +++ /dev/null @@ -1,4159 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__elements": {}, - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "10.2.3" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "state-timeline", - "name": "State timeline", - "version": "" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 23, - "panels": [], - "title": "Resource", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "center", - "cellOptions": { - "type": "auto" - }, - "filterable": false, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "short_version" - }, - "properties": [ - { - "id": "custom.width", - "value": 147 - } - ] - } - ] - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 0, - "y": 1 - }, - "id": 29, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": false, - "fields": [], - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "greptime_app_version", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "{{short_version}}", - "range": false, - "refId": "A" - } - ], - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Value": true, - "__name__": true, - "instance": true, - "job": true - }, - "includeByName": {}, - "indexByName": {}, - "renameByName": {} - } - } - ], - "transparent": true, - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 1 - }, - "id": 30, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "greptime_catalog_schema_count", - "instant": false, - "legendFormat": "database", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "greptime_table_operator_create_table_count", - "hide": false, - "instant": false, - "legendFormat": "table", - "range": true, - "refId": "B" - } - ], - "transparent": true, - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 12, - "y": 1 - }, - "id": 31, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "process_threads", - "instant": false, - "legendFormat": "threads", - "range": true, - "refId": "A" - } - ], - "title": "Threads", - "transformations": [], - "transparent": true, - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "fillOpacity": 70, - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineWidth": 0, - "spanNulls": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 18, - "y": 1 - }, - "id": 32, - "options": { - "alignValue": "center", - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "mergeValues": true, - "rowHeight": 0.9, - "showValue": "auto", - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "up{}", - "instant": false, - "legendFormat": "_", - "range": true, - "refId": "A" - } - ], - "title": "Up", - "transparent": true, - "type": "state-timeline" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 700 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 4 - }, - "id": 27, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "irate(process_cpu_seconds_total[1m])", - "instant": false, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "CPU", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 4 - }, - "id": 28, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "process_resident_memory_bytes", - "instant": false, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Memory", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 24, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 34, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le, db) (rate(greptime_servers_http_promql_elapsed_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "promql-{{db}}-p95", - "range": true, - "refId": "PromQL P95", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, db) (rate(greptime_servers_http_promql_elapsed_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "promql-{{db}}-p99", - "range": true, - "refId": "PromQL P99", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le, db) (rate(greptime_servers_http_sql_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "sql-{{db}}-p95", - "range": true, - "refId": "SQL P95" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, db) (rate(greptime_servers_http_sql_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "sql-{{db}}-p99", - "range": true, - "refId": "SQL P99" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le, db) (rate(greptime_servers_http_prometheus_read_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "promstore-read-{{db}}-p95", - "range": true, - "refId": "PromStore Read P95" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, db) (rate(greptime_servers_http_prometheus_read_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "promstore-read-{{db}}-p99", - "range": true, - "refId": "PromStore Read P99" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le, db, method) (rate(greptime_servers_http_prometheus_promql_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "prom-promql-{{db}}-{{method}}-p95", - "range": true, - "refId": "Prometheus PromQL P95" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, db, method) (rate(greptime_servers_http_prometheus_promql_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "prom-promql-{{db}}-{{method}}-p99", - "range": true, - "refId": "Prometheus PromQL P99" - } - ], - "title": "HTTP query elapsed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 35, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le, db) (rate(greptime_servers_http_influxdb_write_elapsed_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "influx-{{db}}-p95", - "range": true, - "refId": "InfluxDB Line Protocol P95", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, db) (rate(greptime_servers_http_influxdb_write_elapsed_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "influx-{{db}}-p99", - "range": true, - "refId": "InfluxDB Line Protocol P99", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le, db) (rate(greptime_servers_http_prometheus_write_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "promstore-{{db}}-p95", - "range": true, - "refId": "PromStore Write P95" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, db) (rate(greptime_servers_http_prometheus_write_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "promstore-{{db}}-p99", - "range": true, - "refId": "PromStore Write P99" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le, db) (rate(greptime_servers_http_otlp_metrics_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "otlp-metric-{{db}}-p95", - "range": true, - "refId": "OTLP Metric P95" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, db) (rate(greptime_servers_http_otlp_metrics_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "otlp-metric-{{db}}-p99", - "range": true, - "refId": "OTLP Metric P99" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le, db) (rate(greptime_servers_http_otlp_traces_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "otlp-trace-{{db}}-p95", - "range": true, - "refId": "OTLP Trace P95" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, db) (rate(greptime_servers_http_otlp_traces_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "otlp-trace-{{db}}-p99", - "range": true, - "refId": "OTLP Trace P99" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le, db) (rate(greptime_servers_http_logs_transform_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "log-transform-{{db}}-p95", - "range": true, - "refId": "Log Transform P95" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, db) (rate(greptime_servers_http_logs_transform_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "log-transform-{{db}}-p99", - "range": true, - "refId": "Log Transform P99" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le, db) (rate(greptime_servers_http_logs_ingestion_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "log-ingest-{{db}}-p99", - "range": true, - "refId": "Log Ingest P95" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, db) (rate(greptime_servers_http_logs_ingestion_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "log-ingest-{{db}}-p99", - "range": true, - "refId": "Log Ingest P99" - } - ], - "title": "HTTP write elapsed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 38, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "sum by(path) (rate(greptime_servers_http_requests_total[$__rate_interval]))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "HTTP request rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 36, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "sum by(db) (rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{db}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Logs ingest rate (number of lines)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 25 - }, - "id": 13, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le, db) (rate(greptime_servers_grpc_requests_elapsed_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{db}}-p95", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, db) (rate(greptime_servers_grpc_requests_elapsed_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{db}}-p99", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "gRPC insert elapsed", - "type": "timeseries" - } - ], - "title": "Protocol", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 25, - "panels": [], - "title": "Mito Engine", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 12 - }, - "id": 1, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le, type) (rate(greptime_mito_handle_request_elapsed_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{type}}-p95", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, type) (rate(greptime_mito_handle_request_elapsed_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{type}}-p99", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Handle request elapsed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 12 - }, - "id": 7, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "rate(greptime_mito_write_rows_total[$__rate_interval])", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Write rows total", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 19 - }, - "id": 3, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le, stage) (rate(greptime_mito_read_stage_elapsed_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Mito engine read stage duration", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 19 - }, - "id": 11, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le, stage) (rate(greptime_mito_write_stage_elapsed_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{stage}}-p95", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, stage) (rate(greptime_mito_write_stage_elapsed_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{stage}}-p99", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Write stage duration", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 26 - }, - "id": 15, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "idelta(greptime_mito_compaction_stage_elapsed_count{stage=\"merge\"}[5m])", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "compaction-{{stage}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le, type) (rate(greptime_mito_flush_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "flush-{{type}}", - "range": true, - "refId": "B" - } - ], - "title": "Flush / compaction duration", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 26 - }, - "id": 39, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "greptime_mito_inflight_compaction_count", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "compaction-{{instance}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "greptime_mito_inflight_flush_count", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "flush-{{instance}}", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Flush / compaction count", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 9, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "greptime_mito_write_buffer_bytes", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "greptime_mito_memtable_dict_bytes", - "hide": false, - "instant": false, - "legendFormat": "{{instance}}", - "range": true, - "refId": "B" - } - ], - "title": "Write buffer size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 33 - }, - "id": 40, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(greptime_mito_write_stall_total[$__rate_interval])", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{instance}}-worker-{{worker}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Write stall count", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 41, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "greptime_mito_cache_bytes", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{instance}}-{{type}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Cache size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 42, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "sum(increase(greptime_mito_cache_hit[$__rate_interval])) by (instance, type) / (sum(increase(greptime_mito_cache_miss[$__rate_interval])) by (instance, type) + sum(increase(greptime_mito_cache_hit[$__rate_interval])) by (instance, type))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{instance}}-{{type}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Cache hit", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 47 - }, - "id": 26, - "panels": [], - "title": "Metric Engine", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 48 - }, - "id": 22, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le, operation) (rate(greptime_metric_engine_mito_op_elapsed_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "p95-{{operation}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, operation) (rate(greptime_metric_engine_mito_op_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "p99-{{operation}}", - "range": true, - "refId": "B" - } - ], - "title": "Metric engine to mito R/W duration", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 48 - }, - "id": 33, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le, operation) (rate(greptime_metric_engine_mito_ddl_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "p95-{{operation}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, operation) (rate(greptime_metric_engine_mito_ddl_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "p99-{{label_name}}", - "range": true, - "refId": "B" - } - ], - "title": "Metric engine to mito DDL duration", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 55 - }, - "id": 21, - "panels": [], - "title": "Storage Components", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 56 - }, - "id": 18, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "rate(opendal_operation_bytes_sum[$__rate_interval])", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{scheme}}-{{operation}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "OpenDAL traffic", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 56 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le, operation, schema) (rate(opendal_operation_duration_seconds_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "OpenDAL operation duration", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 63 - }, - "id": 43, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "greptime_object_store_lru_cache_bytes", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{instance}}-{{type}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Object store read cache size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 63 - }, - "id": 44, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "sum(increase(greptime_object_store_lru_cache_hit[$__rate_interval])) by (instance) / (sum(increase(greptime_object_store_lru_cache_miss[$__rate_interval])) by (instance) + sum(increase(greptime_object_store_lru_cache_hit[$__rate_interval])) by (instance))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Object store read cache hit", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 70 - }, - "id": 10, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le,logstore,optype) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "{{logstore}}-{{optype}}-p95", - "range": true, - "refId": "Log Store P95" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le,logstore,optype) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "{{logstore}}-{{optype}}-p99", - "range": true, - "refId": "Log Store P99" - } - ], - "title": "Log Store op duration seconds", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 70 - }, - "id": 12, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le) (rate(raft_engine_write_size_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "req-size-p95", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(raft_engine_write_size_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "req-size-p99", - "range": true, - "refId": "C", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(raft_engine_write_size_sum[$__rate_interval])", - "hide": false, - "instant": false, - "legendFormat": "throughput", - "range": true, - "refId": "B" - } - ], - "title": "WAL write size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 77 - }, - "id": 37, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, type, node) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "{{node}}-{{type}}-p99", - "range": true, - "refId": "Log Store P95" - } - ], - "title": "WAL sync duration seconds", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 84 - }, - "id": 46, - "panels": [], - "title": "Index", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 85 - }, - "id": 45, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "greptime_index_create_memory_usage", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{instance}}-{{type}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "greptime_index_apply_memory_usage", - "hide": false, - "instant": false, - "legendFormat": "{{instance}}", - "range": true, - "refId": "B" - } - ], - "title": "Index memory usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 85 - }, - "id": 19, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "apply-{{type}}-p95", - "range": true, - "refId": "Apply P95", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "apply-{{type}}-p95", - "range": true, - "refId": "Apply P99", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "create-{{type}}-p95", - "range": true, - "refId": "Create P95", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "create-{{type}}-p95", - "range": true, - "refId": "Create P99", - "useBackend": false - } - ], - "title": "Index elapsed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 92 - }, - "id": 47, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "rate(greptime_index_create_rows_total[$__rate_interval])", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Index create rows total", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 92 - }, - "id": 48, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum by(instance, type) (rate(greptime_index_create_bytes_total[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{instance}}-{{type}}", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Index create bytes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 99 - }, - "id": 49, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum by(instance, type, file_type) (rate(greptime_index_io_bytes_total[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{instance}}-{{type}}-{{file_type}}", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Index IO bytes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 99 - }, - "id": 50, - "interval": "1s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum by(instance, type, file_type) (rate(greptime_index_io_op_total[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{instance}}-{{type}}-{{file_type}}", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Index IO op", - "type": "timeseries" - } - ], - "refresh": "10s", - "schemaVersion": 39, - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "GreptimeDB", - "uid": "e7097237-669b-4f8d-b751-13067afbfb68", - "version": 19, - "weekStart": "" -} diff --git a/grafana/scripts/check.sh b/grafana/scripts/check.sh new file mode 100755 index 0000000000..78d133e105 --- /dev/null +++ b/grafana/scripts/check.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +DASHBOARD_DIR=${1:-grafana/dashboards} + +check_dashboard_description() { + for dashboard in $(find $DASHBOARD_DIR -name "*.json"); do + echo "Checking $dashboard description" + + # Use jq to check for panels with empty or missing descriptions + invalid_panels=$(cat $dashboard | jq -r ' + .panels[] + | select((.type == "stats" or .type == "timeseries") and (.description == "" or .description == null))') + + # Check if any invalid panels were found + if [[ -n "$invalid_panels" ]]; then + echo "Error: The following panels have empty or missing descriptions:" + echo "$invalid_panels" + exit 1 + else + echo "All panels with type 'stats' or 'timeseries' have valid descriptions." + fi + done +} + +check_dashboards_generation() { + ./grafana/scripts/gen-dashboards.sh + + if [[ -n "$(git diff --name-only grafana/dashboards)" ]]; then + echo "Error: The dashboards are not generated correctly. You should execute the `make dashboards` command." + exit 1 + fi +} + +check_datasource() { + for dashboard in $(find $DASHBOARD_DIR -name "*.json"); do + echo "Checking $dashboard datasource" + jq -r '.panels[] | select(.type != "row") | .targets[] | [.datasource.type, .datasource.uid] | @tsv' $dashboard | while read -r type uid; do + # if the datasource is prometheus, check if the uid is ${metrics} + if [[ "$type" == "prometheus" && "$uid" != "\${metrics}" ]]; then + echo "Error: The datasource uid of $dashboard is not valid. It should be \${metrics}, got $uid" + exit 1 + fi + # if the datasource is mysql, check if the uid is ${information_schema} + if [[ "$type" == "mysql" && "$uid" != "\${information_schema}" ]]; then + echo "Error: The datasource uid of $dashboard is not valid. It should be \${information_schema}, got $uid" + exit 1 + fi + done + done +} + +check_dashboards_generation +check_dashboard_description +check_datasource diff --git a/grafana/scripts/gen-dashboards.sh b/grafana/scripts/gen-dashboards.sh new file mode 100755 index 0000000000..bfd710d0d5 --- /dev/null +++ b/grafana/scripts/gen-dashboards.sh @@ -0,0 +1,18 @@ +#! /usr/bin/env bash + +CLUSTER_DASHBOARD_DIR=${1:-grafana/dashboards/cluster} +STANDALONE_DASHBOARD_DIR=${2:-grafana/dashboards/standalone} +DAC_IMAGE=ghcr.io/zyy17/dac:20250422-c9435ce + +remove_instance_filters() { + # Remove the instance filters for the standalone dashboards. + sed 's/instance=~\\"$datanode\\",//; s/instance=~\\"$datanode\\"//; s/instance=~\\"$frontend\\",//; s/instance=~\\"$frontend\\"//; s/instance=~\\"$metasrv\\",//; s/instance=~\\"$metasrv\\"//; s/instance=~\\"$flownode\\",//; s/instance=~\\"$flownode\\"//;' $CLUSTER_DASHBOARD_DIR/dashboard.json > $STANDALONE_DASHBOARD_DIR/dashboard.json +} + +generate_intermediate_dashboards_and_docs() { + docker run -v ${PWD}:/greptimedb --rm ${DAC_IMAGE} -i /greptimedb/$CLUSTER_DASHBOARD_DIR/dashboard.json -o /greptimedb/$CLUSTER_DASHBOARD_DIR/dashboard.yaml -m > $CLUSTER_DASHBOARD_DIR/dashboard.md + docker run -v ${PWD}:/greptimedb --rm ${DAC_IMAGE} -i /greptimedb/$STANDALONE_DASHBOARD_DIR/dashboard.json -o /greptimedb/$STANDALONE_DASHBOARD_DIR/dashboard.yaml -m > $STANDALONE_DASHBOARD_DIR/dashboard.md +} + +remove_instance_filters +generate_intermediate_dashboards_and_docs diff --git a/grafana/summary.sh b/grafana/summary.sh deleted file mode 100755 index 4e63fd3bd7..0000000000 --- a/grafana/summary.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -BASEDIR=$(dirname "$0") -echo '| Title | Description | Expressions | -|---|---|---|' - -cat $BASEDIR/greptimedb-cluster.json | jq -r ' - .panels | - map(select(.type == "stat" or .type == "timeseries")) | - .[] | "| \(.title) | \(.description | gsub("\n"; "
")) | \(.targets | map(.expr // .rawSql | "`\(.|gsub("\n"; "
"))`") | join("
")) |" -' From d9437c6da735290f998d238d716d02afffb4b85f Mon Sep 17 00:00:00 2001 From: LFC <990479+MichaelScofield@users.noreply.github.com> Date: Tue, 22 Apr 2025 14:04:06 +0800 Subject: [PATCH 53/82] chore: assert plugin uniqueness (#5947) --- src/common/base/src/plugins.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/common/base/src/plugins.rs b/src/common/base/src/plugins.rs index c392422b64..5d27ad1ac1 100644 --- a/src/common/base/src/plugins.rs +++ b/src/common/base/src/plugins.rs @@ -31,7 +31,8 @@ impl Plugins { } pub fn insert(&self, value: T) { - let _ = self.write().insert(value); + let last = self.write().insert(value); + assert!(last.is_none(), "each type of plugins must be one and only"); } pub fn get(&self) -> Option { @@ -137,4 +138,12 @@ mod tests { assert_eq!(plugins.len(), 2); assert!(!plugins.is_empty()); } + + #[test] + #[should_panic(expected = "each type of plugins must be one and only")] + fn test_plugin_uniqueness() { + let plugins = Plugins::new(); + plugins.insert(1i32); + plugins.insert(2i32); + } } From 0a4594c9e20aed8cdf0438422d075994c3a657dd Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Tue, 22 Apr 2025 14:15:47 +0800 Subject: [PATCH 54/82] fix: remove obsolete failover detectors after region leader change (#5944) * fix: remove obsolete failover detectors after region leader change * chore: apply suggestions from CR * fix: fix unit tests * fix: fix unit test * fix: failover logic --- src/meta-srv/src/error.rs | 10 +++++- .../src/procedure/region_migration/manager.rs | 8 ++--- src/meta-srv/src/region/supervisor.rs | 36 +++++++++++++++---- tests-integration/tests/region_migration.rs | 2 +- 4 files changed, 43 insertions(+), 13 deletions(-) diff --git a/src/meta-srv/src/error.rs b/src/meta-srv/src/error.rs index bd249c5178..d6c4ef4208 100644 --- a/src/meta-srv/src/error.rs +++ b/src/meta-srv/src/error.rs @@ -336,6 +336,13 @@ pub enum Error { location: Location, }, + #[snafu(display("Region's leader peer changed: {}", msg))] + LeaderPeerChanged { + msg: String, + #[snafu(implicit)] + location: Location, + }, + #[snafu(display("Invalid arguments: {}", err_msg))] InvalidArguments { err_msg: String, @@ -914,7 +921,8 @@ impl ErrorExt for Error { | Error::ProcedureNotFound { .. } | Error::TooManyPartitions { .. } | Error::TomlFormat { .. } - | Error::HandlerNotFound { .. } => StatusCode::InvalidArguments, + | Error::HandlerNotFound { .. } + | Error::LeaderPeerChanged { .. } => StatusCode::InvalidArguments, Error::LeaseKeyFromUtf8 { .. } | Error::LeaseValueFromUtf8 { .. } | Error::InvalidRegionKeyFromUtf8 { .. } diff --git a/src/meta-srv/src/procedure/region_migration/manager.rs b/src/meta-srv/src/procedure/region_migration/manager.rs index 417881f549..9286bb8778 100644 --- a/src/meta-srv/src/procedure/region_migration/manager.rs +++ b/src/meta-srv/src/procedure/region_migration/manager.rs @@ -267,8 +267,8 @@ impl RegionMigrationManager { ensure!( leader_peer.id == task.from_peer.id, - error::InvalidArgumentsSnafu { - err_msg: format!( + error::LeaderPeerChangedSnafu { + msg: format!( "Region's leader peer({}) is not the `from_peer`({}), region: {}", leader_peer.id, task.from_peer.id, task.region_id ), @@ -507,8 +507,8 @@ mod test { .await; let err = manager.submit_procedure(task).await.unwrap_err(); - assert_matches!(err, error::Error::InvalidArguments { .. }); - assert_eq!(err.to_string(), "Invalid arguments: Region's leader peer(3) is not the `from_peer`(1), region: 4398046511105(1024, 1)"); + assert_matches!(err, error::Error::LeaderPeerChanged { .. }); + assert_eq!(err.to_string(), "Region's leader peer changed: Region's leader peer(3) is not the `from_peer`(1), region: 4398046511105(1024, 1)"); } #[tokio::test] diff --git a/src/meta-srv/src/region/supervisor.rs b/src/meta-srv/src/region/supervisor.rs index 9393474f16..f37face68f 100644 --- a/src/meta-srv/src/region/supervisor.rs +++ b/src/meta-srv/src/region/supervisor.rs @@ -26,7 +26,7 @@ use common_meta::DatanodeId; use common_runtime::JoinHandle; use common_telemetry::{error, info, warn}; use common_time::util::current_time_millis; -use error::Error::{MigrationRunning, TableRouteNotFound}; +use error::Error::{LeaderPeerChanged, MigrationRunning, TableRouteNotFound}; use snafu::{OptionExt, ResultExt}; use store_api::storage::RegionId; use tokio::sync::mpsc::{Receiver, Sender}; @@ -363,15 +363,15 @@ impl RegionSupervisor { for (datanode_id, region_id) in migrating_regions { self.failure_detector.remove(&(datanode_id, region_id)); + warn!( + "Removed region failover for region: {region_id}, datanode: {datanode_id} because it's migrating" + ); } warn!("Detects region failures: {:?}", regions); for (datanode_id, region_id) in regions { - match self.do_failover(datanode_id, region_id).await { - Ok(_) => self.failure_detector.remove(&(datanode_id, region_id)), - Err(err) => { - error!(err; "Failed to execute region failover for region: {region_id}, datanode: {datanode_id}"); - } + if let Err(err) = self.do_failover(datanode_id, region_id).await { + error!(err; "Failed to execute region failover for region: {region_id}, datanode: {datanode_id}"); } } } @@ -421,7 +421,29 @@ impl RegionSupervisor { if let Err(err) = self.region_migration_manager.submit_procedure(task).await { return match err { // Returns Ok if it's running or table is dropped. - MigrationRunning { .. } | TableRouteNotFound { .. } => Ok(()), + MigrationRunning { .. } => { + info!( + "Another region migration is running, skip failover for region: {}, datanode: {}", + region_id, datanode_id + ); + Ok(()) + } + TableRouteNotFound { .. } => { + self.failure_detector.remove(&(datanode_id, region_id)); + info!( + "Table route is not found, the table is dropped, removed failover detector for region: {}, datanode: {}", + region_id, datanode_id + ); + Ok(()) + } + LeaderPeerChanged { .. } => { + self.failure_detector.remove(&(datanode_id, region_id)); + info!( + "Region's leader peer changed, removed failover detector for region: {}, datanode: {}", + region_id, datanode_id + ); + Ok(()) + } err => Err(err), }; }; diff --git a/tests-integration/tests/region_migration.rs b/tests-integration/tests/region_migration.rs index b24850420e..28d6371c8b 100644 --- a/tests-integration/tests/region_migration.rs +++ b/tests-integration/tests/region_migration.rs @@ -847,7 +847,7 @@ pub async fn test_region_migration_incorrect_from_peer( assert!(matches!( err, - meta_srv::error::Error::InvalidArguments { .. } + meta_srv::error::Error::LeaderPeerChanged { .. } )); } From 0f77135ef9a33f8d9dd5d70ac99d84a0a04ccc31 Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Tue, 22 Apr 2025 15:49:22 +0800 Subject: [PATCH 55/82] feat: add `exclude_peer_ids` to `SelectorOptions` (#5949) * feat: add `exclude_peer_ids` to `SelectorOptions` * chore: apply suggestions from CR * fix: clippy --- src/meta-srv/src/flow_meta_alloc.rs | 3 ++ .../src/procedure/region_migration.rs | 3 ++ src/meta-srv/src/region/supervisor.rs | 8 +++- src/meta-srv/src/selector.rs | 5 +++ src/meta-srv/src/selector/common.rs | 43 ++++++++++++++++++- src/meta-srv/src/selector/lease_based.rs | 5 ++- src/meta-srv/src/selector/load_based.rs | 5 ++- src/meta-srv/src/selector/round_robin.rs | 4 ++ src/meta-srv/src/table_meta_alloc.rs | 3 ++ 9 files changed, 71 insertions(+), 8 deletions(-) diff --git a/src/meta-srv/src/flow_meta_alloc.rs b/src/meta-srv/src/flow_meta_alloc.rs index bdfac158aa..5c9dd82301 100644 --- a/src/meta-srv/src/flow_meta_alloc.rs +++ b/src/meta-srv/src/flow_meta_alloc.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashSet; + use common_error::ext::BoxedError; use common_meta::ddl::flow_meta::PartitionPeerAllocator; use common_meta::peer::Peer; @@ -40,6 +42,7 @@ impl PartitionPeerAllocator for FlowPeerAllocator { SelectorOptions { min_required_items: partitions, allow_duplication: true, + exclude_peer_ids: HashSet::new(), }, ) .await diff --git a/src/meta-srv/src/procedure/region_migration.rs b/src/meta-srv/src/procedure/region_migration.rs index a98320f9e7..b2f1eed711 100644 --- a/src/meta-srv/src/procedure/region_migration.rs +++ b/src/meta-srv/src/procedure/region_migration.rs @@ -58,6 +58,9 @@ use crate::error::{self, Result}; use crate::metrics::{METRIC_META_REGION_MIGRATION_ERROR, METRIC_META_REGION_MIGRATION_EXECUTE}; use crate::service::mailbox::MailboxRef; +/// The default timeout for region migration. +pub const DEFAULT_REGION_MIGRATION_TIMEOUT: Duration = Duration::from_secs(120); + /// It's shared in each step and available even after recovering. /// /// It will only be updated/stored after the Red node has succeeded. diff --git a/src/meta-srv/src/region/supervisor.rs b/src/meta-srv/src/region/supervisor.rs index f37face68f..6318c0d128 100644 --- a/src/meta-srv/src/region/supervisor.rs +++ b/src/meta-srv/src/region/supervisor.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashSet; use std::fmt::Debug; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -36,7 +37,9 @@ use crate::error::{self, Result}; use crate::failure_detector::PhiAccrualFailureDetectorOptions; use crate::metasrv::{SelectorContext, SelectorRef}; use crate::procedure::region_migration::manager::RegionMigrationManagerRef; -use crate::procedure::region_migration::RegionMigrationProcedureTask; +use crate::procedure::region_migration::{ + RegionMigrationProcedureTask, DEFAULT_REGION_MIGRATION_TIMEOUT, +}; use crate::region::failure_detector::RegionFailureDetector; use crate::selector::SelectorOptions; @@ -401,6 +404,7 @@ impl RegionSupervisor { SelectorOptions { min_required_items: 1, allow_duplication: false, + exclude_peer_ids: HashSet::from([from_peer.id]), }, ) .await?; @@ -415,7 +419,7 @@ impl RegionSupervisor { region_id, from_peer, to_peer, - timeout: Duration::from_secs(60), + timeout: DEFAULT_REGION_MIGRATION_TIMEOUT, }; if let Err(err) = self.region_migration_manager.submit_procedure(task).await { diff --git a/src/meta-srv/src/selector.rs b/src/meta-srv/src/selector.rs index 9a3d2ceadd..ce166ae05c 100644 --- a/src/meta-srv/src/selector.rs +++ b/src/meta-srv/src/selector.rs @@ -20,6 +20,8 @@ pub mod round_robin; pub(crate) mod test_utils; mod weight_compute; pub mod weighted_choose; +use std::collections::HashSet; + use serde::{Deserialize, Serialize}; use strum::AsRefStr; @@ -40,6 +42,8 @@ pub struct SelectorOptions { pub min_required_items: usize, /// Whether duplicates are allowed in the selected result, default false. pub allow_duplication: bool, + /// The peers to exclude from the selection. + pub exclude_peer_ids: HashSet, } impl Default for SelectorOptions { @@ -47,6 +51,7 @@ impl Default for SelectorOptions { Self { min_required_items: 1, allow_duplication: false, + exclude_peer_ids: HashSet::new(), } } } diff --git a/src/meta-srv/src/selector/common.rs b/src/meta-srv/src/selector/common.rs index b44043f60e..24c3597c26 100644 --- a/src/meta-srv/src/selector/common.rs +++ b/src/meta-srv/src/selector/common.rs @@ -12,15 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashSet; + use common_meta::peer::Peer; use snafu::ensure; use crate::error; use crate::error::Result; use crate::metasrv::SelectTarget; -use crate::selector::weighted_choose::WeightedChoose; +use crate::selector::weighted_choose::{WeightedChoose, WeightedItem}; use crate::selector::SelectorOptions; +/// Filter out the excluded peers from the `weight_array`. +pub fn filter_out_excluded_peers( + weight_array: &mut Vec>, + exclude_peer_ids: &HashSet, +) { + weight_array.retain(|peer| !exclude_peer_ids.contains(&peer.item.id)); +} + /// According to the `opts`, choose peers from the `weight_array` through `weighted_choose`. pub fn choose_items(opts: &SelectorOptions, weighted_choose: &mut W) -> Result> where @@ -80,7 +90,7 @@ mod tests { use common_meta::peer::Peer; - use crate::selector::common::choose_items; + use crate::selector::common::{choose_items, filter_out_excluded_peers}; use crate::selector::weighted_choose::{RandomWeightedChoose, WeightedItem}; use crate::selector::SelectorOptions; @@ -128,6 +138,7 @@ mod tests { let opts = SelectorOptions { min_required_items: i, allow_duplication: false, + exclude_peer_ids: HashSet::new(), }; let selected_peers: HashSet<_> = @@ -142,6 +153,7 @@ mod tests { let opts = SelectorOptions { min_required_items: 6, allow_duplication: false, + exclude_peer_ids: HashSet::new(), }; let selected_result = @@ -152,6 +164,7 @@ mod tests { let opts = SelectorOptions { min_required_items: i, allow_duplication: true, + exclude_peer_ids: HashSet::new(), }; let selected_peers = @@ -160,4 +173,30 @@ mod tests { assert_eq!(i, selected_peers.len()); } } + + #[test] + fn test_filter_out_excluded_peers() { + let mut weight_array = vec![ + WeightedItem { + item: Peer { + id: 1, + addr: "127.0.0.1:3001".to_string(), + }, + weight: 1.0, + }, + WeightedItem { + item: Peer { + id: 2, + addr: "127.0.0.1:3002".to_string(), + }, + weight: 1.0, + }, + ]; + + let exclude_peer_ids = HashSet::from([1]); + filter_out_excluded_peers(&mut weight_array, &exclude_peer_ids); + + assert_eq!(weight_array.len(), 1); + assert_eq!(weight_array[0].item.id, 2); + } } diff --git a/src/meta-srv/src/selector/lease_based.rs b/src/meta-srv/src/selector/lease_based.rs index 770c8aac8a..e60157989d 100644 --- a/src/meta-srv/src/selector/lease_based.rs +++ b/src/meta-srv/src/selector/lease_based.rs @@ -17,7 +17,7 @@ use common_meta::peer::Peer; use crate::error::Result; use crate::lease; use crate::metasrv::SelectorContext; -use crate::selector::common::choose_items; +use crate::selector::common::{choose_items, filter_out_excluded_peers}; use crate::selector::weighted_choose::{RandomWeightedChoose, WeightedItem}; use crate::selector::{Selector, SelectorOptions}; @@ -35,7 +35,7 @@ impl Selector for LeaseBasedSelector { lease::alive_datanodes(&ctx.meta_peer_client, ctx.datanode_lease_secs).await?; // 2. compute weight array, but the weight of each item is the same. - let weight_array = lease_kvs + let mut weight_array = lease_kvs .into_iter() .map(|(k, v)| WeightedItem { item: Peer { @@ -47,6 +47,7 @@ impl Selector for LeaseBasedSelector { .collect(); // 3. choose peers by weight_array. + filter_out_excluded_peers(&mut weight_array, &opts.exclude_peer_ids); let mut weighted_choose = RandomWeightedChoose::new(weight_array); let selected = choose_items(&opts, &mut weighted_choose)?; diff --git a/src/meta-srv/src/selector/load_based.rs b/src/meta-srv/src/selector/load_based.rs index 2628990bf4..d98a4ace5d 100644 --- a/src/meta-srv/src/selector/load_based.rs +++ b/src/meta-srv/src/selector/load_based.rs @@ -26,7 +26,7 @@ use crate::error::{self, Result}; use crate::key::{DatanodeLeaseKey, LeaseValue}; use crate::lease; use crate::metasrv::SelectorContext; -use crate::selector::common::choose_items; +use crate::selector::common::{choose_items, filter_out_excluded_peers}; use crate::selector::weight_compute::{RegionNumsBasedWeightCompute, WeightCompute}; use crate::selector::weighted_choose::RandomWeightedChoose; use crate::selector::{Selector, SelectorOptions}; @@ -85,9 +85,10 @@ where }; // 4. compute weight array. - let weight_array = self.weight_compute.compute(&stat_kvs); + let mut weight_array = self.weight_compute.compute(&stat_kvs); // 5. choose peers by weight_array. + filter_out_excluded_peers(&mut weight_array, &opts.exclude_peer_ids); let mut weighted_choose = RandomWeightedChoose::new(weight_array); let selected = choose_items(&opts, &mut weighted_choose)?; diff --git a/src/meta-srv/src/selector/round_robin.rs b/src/meta-srv/src/selector/round_robin.rs index f11a36555f..2c849cb194 100644 --- a/src/meta-srv/src/selector/round_robin.rs +++ b/src/meta-srv/src/selector/round_robin.rs @@ -120,6 +120,8 @@ impl Selector for RoundRobinSelector { #[cfg(test)] mod test { + use std::collections::HashSet; + use super::*; use crate::test_util::{create_selector_context, put_datanodes}; @@ -149,6 +151,7 @@ mod test { SelectorOptions { min_required_items: 4, allow_duplication: true, + exclude_peer_ids: HashSet::new(), }, ) .await @@ -165,6 +168,7 @@ mod test { SelectorOptions { min_required_items: 2, allow_duplication: true, + exclude_peer_ids: HashSet::new(), }, ) .await diff --git a/src/meta-srv/src/table_meta_alloc.rs b/src/meta-srv/src/table_meta_alloc.rs index 8578e6cd19..54dd34ea86 100644 --- a/src/meta-srv/src/table_meta_alloc.rs +++ b/src/meta-srv/src/table_meta_alloc.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashSet; + use async_trait::async_trait; use common_error::ext::BoxedError; use common_meta::ddl::table_meta::PeerAllocator; @@ -51,6 +53,7 @@ impl MetasrvPeerAllocator { SelectorOptions { min_required_items: regions, allow_duplication: true, + exclude_peer_ids: HashSet::new(), }, ) .await?; From bcefc6b83ff31c819ccbeb193c17adbe172a3386 Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Tue, 22 Apr 2025 16:10:35 +0800 Subject: [PATCH 56/82] feat: add format support for promql http api (not prometheus) (#5939) * feat: add format support for promql http api (not prometheus) * test: add csv format test --- src/servers/src/http/handler.rs | 40 ++++++++++++++++++++++++++++++++- tests-integration/tests/http.rs | 15 +++++++++++-- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/src/servers/src/http/handler.rs b/src/servers/src/http/handler.rs index fe4d1aa034..b21af27f10 100644 --- a/src/servers/src/http/handler.rs +++ b/src/servers/src/http/handler.rs @@ -251,6 +251,23 @@ pub struct PromqlQuery { pub step: String, pub lookback: Option, pub db: Option, + // (Optional) result format: [`greptimedb_v1`, `influxdb_v1`, `csv`, + // `arrow`], + // the default value is `greptimedb_v1` + pub format: Option, + // For arrow output + pub compression: Option, + // Returns epoch timestamps with the specified precision. + // Both u and µ indicate microseconds. + // epoch = [ns,u,µ,ms,s], + // + // For influx output only + // + // TODO(jeremy): currently, only InfluxDB result format is supported, + // and all columns of the `Timestamp` type will be converted to their + // specified time precision. Maybe greptimedb format can support this + // param too. + pub epoch: Option, } impl From for PromQuery { @@ -292,9 +309,30 @@ pub async fn promql( let resp = ErrorResponse::from_error_message(status, msg); HttpResponse::Error(resp) } else { + let format = params + .format + .as_ref() + .map(|s| s.to_lowercase()) + .map(|s| ResponseFormat::parse(s.as_str()).unwrap_or(ResponseFormat::GreptimedbV1)) + .unwrap_or(ResponseFormat::GreptimedbV1); + let epoch = params + .epoch + .as_ref() + .map(|s| s.to_lowercase()) + .map(|s| Epoch::parse(s.as_str()).unwrap_or(Epoch::Millisecond)); + let compression = params.compression.clone(); + let prom_query = params.into(); let outputs = sql_handler.do_promql_query(&prom_query, query_ctx).await; - GreptimedbV1Response::from_output(outputs).await + + match format { + ResponseFormat::Arrow => ArrowResponse::from_output(outputs, compression).await, + ResponseFormat::Csv => CsvResponse::from_output(outputs).await, + ResponseFormat::Table => TableResponse::from_output(outputs).await, + ResponseFormat::GreptimedbV1 => GreptimedbV1Response::from_output(outputs).await, + ResponseFormat::InfluxdbV1 => InfluxdbV1Response::from_output(outputs, epoch).await, + ResponseFormat::Json => JsonResponse::from_output(outputs).await, + } }; resp.with_execution_time(exec_start.elapsed().as_millis() as u64) diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index 9e369e654d..2c663e99b9 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -483,7 +483,7 @@ pub async fn test_sql_api(store_type: StorageType) { } pub async fn test_prometheus_promql_api(store_type: StorageType) { - let (app, mut guard) = setup_test_http_app_with_frontend(store_type, "sql_api").await; + let (app, mut guard) = setup_test_http_app_with_frontend(store_type, "promql_api").await; let client = TestClient::new(app).await; let res = client @@ -492,7 +492,18 @@ pub async fn test_prometheus_promql_api(store_type: StorageType) { .await; assert_eq!(res.status(), StatusCode::OK); - let _body = serde_json::from_str::(&res.text().await).unwrap(); + let json_text = res.text().await; + assert!(serde_json::from_str::(&json_text).is_ok()); + + let res = client + .get("/v1/promql?query=1&start=0&end=100&step=5s&format=csv") + .send() + .await; + assert_eq!(res.status(), StatusCode::OK); + + let csv_body = &res.text().await; + assert_eq!("0,1.0\n5000,1.0\n10000,1.0\n15000,1.0\n20000,1.0\n25000,1.0\n30000,1.0\n35000,1.0\n40000,1.0\n45000,1.0\n50000,1.0\n55000,1.0\n60000,1.0\n65000,1.0\n70000,1.0\n75000,1.0\n80000,1.0\n85000,1.0\n90000,1.0\n95000,1.0\n100000,1.0\n", csv_body); + guard.remove_all().await; } From 6e407ae4b916caee42bf5c623f636f0c59eaf35f Mon Sep 17 00:00:00 2001 From: discord9 <55937128+discord9@users.noreply.github.com> Date: Tue, 22 Apr 2025 16:14:27 +0800 Subject: [PATCH 57/82] test: use random seed for window sort fuzz test (#5950) tests: use random seed for window sort fuzz test --- src/query/src/window_sort.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/query/src/window_sort.rs b/src/query/src/window_sort.rs index 57309f3c2a..2e367c5ea0 100644 --- a/src/query/src/window_sort.rs +++ b/src/query/src/window_sort.rs @@ -3156,7 +3156,8 @@ mod test { let fetch_bound = 100; let mut rng = fastrand::Rng::new(); - rng.seed(1337); + let rng_seed = rng.u64(..); + rng.seed(rng_seed); let mut bound_val = None; // construct testcases type CmpFn = Box std::cmp::Ordering>; @@ -3299,8 +3300,8 @@ mod test { } assert_eq!( res_concat, expected_concat, - "case failed, case id: {}", - case_id + "case failed, case id: {}, rng seed: {}", + case_id, rng_seed ); } } From 9fb0487e67d1b4383de3400863f980f9d2b04de1 Mon Sep 17 00:00:00 2001 From: discord9 <55937128+discord9@users.noreply.github.com> Date: Tue, 22 Apr 2025 16:44:04 +0800 Subject: [PATCH 58/82] fix: parse flow expire after interval (#5953) * fix: parse flow expire after interval * fix: correct 30.44&comments --- src/sql/src/parsers/create_parser.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/sql/src/parsers/create_parser.rs b/src/sql/src/parsers/create_parser.rs index 35797c380b..561216e876 100644 --- a/src/sql/src/parsers/create_parser.rs +++ b/src/sql/src/parsers/create_parser.rs @@ -288,8 +288,14 @@ impl<'a> ParserContext<'a> { .with_context(|| InvalidIntervalSnafu { reason: format!("cannot cast {} to interval type", expire_after_expr), })?; - if let ScalarValue::IntervalMonthDayNano(Some(nanoseconds)) = expire_after_lit { - Some(nanoseconds.nanoseconds / 1_000_000_000) + if let ScalarValue::IntervalMonthDayNano(Some(interval)) = expire_after_lit { + Some( + interval.nanoseconds / 1_000_000_000 + + interval.days as i64 * 60 * 60 * 24 + + interval.months as i64 * 60 * 60 * 24 * 3044 / 1000, // 1 month=365.25/12=30.44 days + // this is to keep the same as https://docs.rs/humantime/latest/humantime/fn.parse_duration.html + // which we use in database to parse i.e. ttl interval and many other intervals + ) } else { unreachable!() } @@ -1325,6 +1331,7 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;"; let sql = r" CREATE FLOW `task_2` SINK TO schema_1.table_1 +EXPIRE AFTER '1 month 2 days 1h 2 min' AS SELECT max(c1), min(c2) FROM schema_2.table_2;"; let stmts = @@ -1337,7 +1344,10 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;"; }; assert!(!create_task.or_replace); assert!(!create_task.if_not_exists); - assert!(create_task.expire_after.is_none()); + assert_eq!( + create_task.expire_after, + Some(86400 * 3044 / 1000 + 2 * 86400 + 3600 + 2 * 60) + ); assert!(create_task.comment.is_none()); assert_eq!(create_task.flow_name.to_string(), "`task_2`"); } From 5c07f0dec73103dfb57e3f44af38a4d9f0439f5b Mon Sep 17 00:00:00 2001 From: shuiyisong <113876041+shuiyisong@users.noreply.github.com> Date: Tue, 22 Apr 2025 19:34:19 +0800 Subject: [PATCH 59/82] refactor: `run_pipeline` parameters (#5954) * refactor: simplify run_pipeline params * refactor: remove unnecessory function wrap --- src/pipeline/src/lib.rs | 4 +- src/pipeline/src/manager.rs | 18 ++++++++- src/servers/src/elasticsearch.rs | 32 ++++++++-------- src/servers/src/http/event.rs | 38 +++++++++---------- src/servers/src/http/prom_store.rs | 32 +++++----------- src/servers/src/otlp/logs.rs | 13 ++++--- src/servers/src/pipeline.rs | 59 ++++++++++++++---------------- 7 files changed, 98 insertions(+), 98 deletions(-) diff --git a/src/pipeline/src/lib.rs b/src/pipeline/src/lib.rs index 857c321d33..d656094c2e 100644 --- a/src/pipeline/src/lib.rs +++ b/src/pipeline/src/lib.rs @@ -29,7 +29,7 @@ pub use etl::{ DispatchedTo, Pipeline, PipelineExecOutput, PipelineMap, }; pub use manager::{ - pipeline_operator, table, util, IdentityTimeIndex, PipelineDefinition, PipelineInfo, - PipelineRef, PipelineTableRef, PipelineVersion, PipelineWay, SelectInfo, + pipeline_operator, table, util, IdentityTimeIndex, PipelineContext, PipelineDefinition, + PipelineInfo, PipelineRef, PipelineTableRef, PipelineVersion, PipelineWay, SelectInfo, GREPTIME_INTERNAL_IDENTITY_PIPELINE_NAME, GREPTIME_INTERNAL_TRACE_PIPELINE_V1_NAME, }; diff --git a/src/pipeline/src/manager.rs b/src/pipeline/src/manager.rs index b929cf3c23..3928e00b35 100644 --- a/src/pipeline/src/manager.rs +++ b/src/pipeline/src/manager.rs @@ -26,7 +26,7 @@ use util::to_pipeline_version; use crate::error::{CastTypeSnafu, InvalidCustomTimeIndexSnafu, PipelineMissingSnafu, Result}; use crate::etl::value::time::{MS_RESOLUTION, NS_RESOLUTION, S_RESOLUTION, US_RESOLUTION}; use crate::table::PipelineTable; -use crate::{Pipeline, Value}; +use crate::{GreptimePipelineParams, Pipeline, Value}; pub mod pipeline_operator; pub mod table; @@ -104,6 +104,22 @@ impl PipelineDefinition { } } +pub struct PipelineContext<'a> { + pub pipeline_definition: &'a PipelineDefinition, + pub pipeline_param: &'a GreptimePipelineParams, +} + +impl<'a> PipelineContext<'a> { + pub fn new( + pipeline_definition: &'a PipelineDefinition, + pipeline_param: &'a GreptimePipelineParams, + ) -> Self { + Self { + pipeline_definition, + pipeline_param, + } + } +} pub enum PipelineWay { OtlpLogDirect(Box), Pipeline(PipelineDefinition), diff --git a/src/servers/src/elasticsearch.rs b/src/servers/src/elasticsearch.rs index ba0e97acbf..23d7f1b2dc 100644 --- a/src/servers/src/elasticsearch.rs +++ b/src/servers/src/elasticsearch.rs @@ -33,7 +33,9 @@ use crate::error::{ status_code_to_http_status, InvalidElasticsearchInputSnafu, ParseJsonSnafu, PipelineSnafu, Result as ServersResult, }; -use crate::http::event::{ingest_logs_inner, LogIngestRequest, LogIngesterQueryParams, LogState}; +use crate::http::event::{ + ingest_logs_inner, LogIngesterQueryParams, LogState, PipelineIngestRequest, +}; use crate::metrics::{ METRIC_ELASTICSEARCH_LOGS_DOCS_COUNT, METRIC_ELASTICSEARCH_LOGS_INGESTION_ELAPSED, }; @@ -276,7 +278,7 @@ fn parse_bulk_request( input: &str, index_from_url: &Option, msg_field: &Option, -) -> ServersResult> { +) -> ServersResult> { // Read the ndjson payload and convert it to `Vec`. Return error if the input is not a valid JSON. let values: Vec = Deserializer::from_str(input) .into_iter::() @@ -291,7 +293,7 @@ fn parse_bulk_request( } ); - let mut requests: Vec = Vec::with_capacity(values.len() / 2); + let mut requests: Vec = Vec::with_capacity(values.len() / 2); let mut values = values.into_iter(); // Read the ndjson payload and convert it to a (index, value) vector. @@ -331,7 +333,7 @@ fn parse_bulk_request( ); let log_value = pipeline::json_to_map(log_value).context(PipelineSnafu)?; - requests.push(LogIngestRequest { + requests.push(PipelineIngestRequest { table: index.unwrap_or_else(|| index_from_url.as_ref().unwrap().clone()), values: vec![log_value], }); @@ -402,13 +404,13 @@ mod tests { None, None, Ok(vec![ - LogIngestRequest { + PipelineIngestRequest { table: "test".to_string(), values: vec![ pipeline::json_to_map(json!({"foo1": "foo1_value", "bar1": "bar1_value"})).unwrap(), ], }, - LogIngestRequest { + PipelineIngestRequest { table: "test".to_string(), values: vec![pipeline::json_to_map(json!({"foo2": "foo2_value", "bar2": "bar2_value"})).unwrap()], }, @@ -425,11 +427,11 @@ mod tests { Some("logs".to_string()), None, Ok(vec![ - LogIngestRequest { + PipelineIngestRequest { table: "test".to_string(), values: vec![pipeline::json_to_map(json!({"foo1": "foo1_value", "bar1": "bar1_value"})).unwrap()], }, - LogIngestRequest { + PipelineIngestRequest { table: "logs".to_string(), values: vec![pipeline::json_to_map(json!({"foo2": "foo2_value", "bar2": "bar2_value"})).unwrap()], }, @@ -446,11 +448,11 @@ mod tests { Some("logs".to_string()), None, Ok(vec![ - LogIngestRequest { + PipelineIngestRequest { table: "test".to_string(), values: vec![pipeline::json_to_map(json!({"foo1": "foo1_value", "bar1": "bar1_value"})).unwrap()], }, - LogIngestRequest { + PipelineIngestRequest { table: "logs".to_string(), values: vec![pipeline::json_to_map(json!({"foo2": "foo2_value", "bar2": "bar2_value"})).unwrap()], }, @@ -466,7 +468,7 @@ mod tests { Some("logs".to_string()), None, Ok(vec![ - LogIngestRequest { + PipelineIngestRequest { table: "test".to_string(), values: vec![pipeline::json_to_map(json!({"foo1": "foo1_value", "bar1": "bar1_value"})).unwrap()], }, @@ -483,11 +485,11 @@ mod tests { None, Some("data".to_string()), Ok(vec![ - LogIngestRequest { + PipelineIngestRequest { table: "test".to_string(), values: vec![pipeline::json_to_map(json!({"foo1": "foo1_value", "bar1": "bar1_value"})).unwrap()], }, - LogIngestRequest { + PipelineIngestRequest { table: "test".to_string(), values: vec![pipeline::json_to_map(json!({"foo2": "foo2_value", "bar2": "bar2_value"})).unwrap()], }, @@ -504,13 +506,13 @@ mod tests { None, Some("message".to_string()), Ok(vec![ - LogIngestRequest { + PipelineIngestRequest { table: "logs-generic-default".to_string(), values: vec![ pipeline::json_to_map(json!({"message": "172.16.0.1 - - [25/May/2024:20:19:37 +0000] \"GET /contact HTTP/1.1\" 404 162 \"-\" \"Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1\""})).unwrap(), ], }, - LogIngestRequest { + PipelineIngestRequest { table: "logs-generic-default".to_string(), values: vec![ pipeline::json_to_map(json!({"message": "10.0.0.1 - - [25/May/2024:20:18:37 +0000] \"GET /images/logo.png HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0\""})).unwrap(), diff --git a/src/servers/src/http/event.rs b/src/servers/src/http/event.rs index cfc0694ae0..fba3e33582 100644 --- a/src/servers/src/http/event.rs +++ b/src/servers/src/http/event.rs @@ -33,7 +33,7 @@ use datatypes::value::column_data_to_json; use headers::ContentType; use lazy_static::lazy_static; use pipeline::util::to_pipeline_version; -use pipeline::{GreptimePipelineParams, PipelineDefinition, PipelineMap}; +use pipeline::{GreptimePipelineParams, PipelineContext, PipelineDefinition, PipelineMap}; use serde::{Deserialize, Serialize}; use serde_json::{json, Deserializer, Map, Value}; use session::context::{Channel, QueryContext, QueryContextRef}; @@ -100,7 +100,7 @@ pub struct LogIngesterQueryParams { /// LogIngestRequest is the internal request for log ingestion. The raw log input can be transformed into multiple LogIngestRequests. /// Multiple LogIngestRequests will be ingested into the same database with the same pipeline. #[derive(Debug, PartialEq)] -pub(crate) struct LogIngestRequest { +pub(crate) struct PipelineIngestRequest { /// The table where the log data will be written to. pub table: String, /// The log data to be ingested. @@ -325,12 +325,15 @@ async fn dryrun_pipeline_inner( ) -> Result { let params = GreptimePipelineParams::default(); + let pipeline_def = PipelineDefinition::Resolved(pipeline); + let pipeline_ctx = PipelineContext::new(&pipeline_def, ¶ms); let results = run_pipeline( &pipeline_handler, - &PipelineDefinition::Resolved(pipeline), - ¶ms, - value, - "dry_run".to_owned(), + &pipeline_ctx, + PipelineIngestRequest { + table: "dry_run".to_owned(), + values: value, + }, query_ctx, true, ) @@ -603,7 +606,7 @@ pub async fn log_ingester( ingest_logs_inner( handler, pipeline, - vec![LogIngestRequest { + vec![PipelineIngestRequest { table: table_name, values: value, }], @@ -673,9 +676,9 @@ fn extract_pipeline_value_by_content_type( } pub(crate) async fn ingest_logs_inner( - state: PipelineHandlerRef, + handler: PipelineHandlerRef, pipeline: PipelineDefinition, - log_ingest_requests: Vec, + log_ingest_requests: Vec, query_ctx: QueryContextRef, headers: HeaderMap, ) -> Result { @@ -690,22 +693,15 @@ pub(crate) async fn ingest_logs_inner( .and_then(|v| v.to_str().ok()), ); - for request in log_ingest_requests { - let requests = run_pipeline( - &state, - &pipeline, - &pipeline_params, - request.values, - request.table, - &query_ctx, - true, - ) - .await?; + let pipeline_ctx = PipelineContext::new(&pipeline, &pipeline_params); + for pipeline_req in log_ingest_requests { + let requests = + run_pipeline(&handler, &pipeline_ctx, pipeline_req, &query_ctx, true).await?; insert_requests.extend(requests); } - let output = state + let output = handler .insert( RowInsertRequests { inserts: insert_requests, diff --git a/src/servers/src/http/prom_store.rs b/src/servers/src/http/prom_store.rs index caafd0f1f3..8c849bf735 100644 --- a/src/servers/src/http/prom_store.rs +++ b/src/servers/src/http/prom_store.rs @@ -83,33 +83,17 @@ impl Default for RemoteWriteQuery { )] pub async fn remote_write( State(state): State, - query: Query, - extension: Extension, - content_encoding: TypedHeader, - raw_body: Bytes, -) -> Result { - remote_write_impl( - state.prom_store_handler, - query, - extension, - content_encoding, - raw_body, - state.is_strict_mode, - state.prom_store_with_metric_engine, - ) - .await -} - -async fn remote_write_impl( - handler: PromStoreProtocolHandlerRef, Query(params): Query, Extension(mut query_ctx): Extension, content_encoding: TypedHeader, body: Bytes, - is_strict_mode: bool, - is_metric_engine: bool, ) -> Result { - // VictoriaMetrics handshake + let PromStoreState { + prom_store_handler, + prom_store_with_metric_engine, + is_strict_mode, + } = state; + if let Some(_vm_handshake) = params.get_vm_proto_version { return Ok(VM_PROTO_VERSION.into_response()); } @@ -128,7 +112,9 @@ async fn remote_write_impl( } let query_ctx = Arc::new(query_ctx); - let output = handler.write(request, query_ctx, is_metric_engine).await?; + let output = prom_store_handler + .write(request, query_ctx, prom_store_with_metric_engine) + .await?; crate::metrics::PROM_STORE_REMOTE_WRITE_SAMPLES.inc_by(samples as u64); Ok(( StatusCode::NO_CONTENT, diff --git a/src/servers/src/otlp/logs.rs b/src/servers/src/otlp/logs.rs index 95b3b21e99..cb6a3d89cf 100644 --- a/src/servers/src/otlp/logs.rs +++ b/src/servers/src/otlp/logs.rs @@ -24,7 +24,7 @@ use jsonb::{Number as JsonbNumber, Value as JsonbValue}; use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; use opentelemetry_proto::tonic::common::v1::{any_value, AnyValue, InstrumentationScope, KeyValue}; use opentelemetry_proto::tonic::logs::v1::{LogRecord, ResourceLogs, ScopeLogs}; -use pipeline::{GreptimePipelineParams, PipelineWay, SchemaInfo, SelectInfo}; +use pipeline::{GreptimePipelineParams, PipelineContext, PipelineWay, SchemaInfo, SelectInfo}; use serde_json::{Map, Value}; use session::context::QueryContextRef; use snafu::{ensure, ResultExt}; @@ -33,6 +33,7 @@ use crate::error::{ IncompatibleSchemaSnafu, NotSupportedSnafu, PipelineSnafu, Result, UnsupportedJsonDataTypeForTagSnafu, }; +use crate::http::event::PipelineIngestRequest; use crate::otlp::trace::attributes::OtlpAnyValue; use crate::otlp::utils::{bytes_to_hex_string, key_value_to_jsonb}; use crate::pipeline::run_pipeline; @@ -74,12 +75,14 @@ pub async fn to_grpc_insert_requests( let data = parse_export_logs_service_request(request); let array = pipeline::json_array_to_map(data).context(PipelineSnafu)?; + let pipeline_ctx = PipelineContext::new(&pipeline_def, &pipeline_params); let inserts = run_pipeline( &pipeline_handler, - &pipeline_def, - &pipeline_params, - array, - table_name, + &pipeline_ctx, + PipelineIngestRequest { + table: table_name, + values: array, + }, query_ctx, true, ) diff --git a/src/servers/src/pipeline.rs b/src/servers/src/pipeline.rs index 8d8538a319..2a7970e2f7 100644 --- a/src/servers/src/pipeline.rs +++ b/src/servers/src/pipeline.rs @@ -18,13 +18,14 @@ use std::sync::Arc; use api::v1::{RowInsertRequest, Rows}; use hashbrown::HashMap; use pipeline::{ - DispatchedTo, GreptimePipelineParams, IdentityTimeIndex, Pipeline, PipelineDefinition, - PipelineExecOutput, PipelineMap, GREPTIME_INTERNAL_IDENTITY_PIPELINE_NAME, + DispatchedTo, GreptimePipelineParams, IdentityTimeIndex, Pipeline, PipelineContext, + PipelineDefinition, PipelineExecOutput, PipelineMap, GREPTIME_INTERNAL_IDENTITY_PIPELINE_NAME, }; use session::context::QueryContextRef; use snafu::ResultExt; use crate::error::{CatalogSnafu, PipelineSnafu, Result}; +use crate::http::event::PipelineIngestRequest; use crate::metrics::{ METRIC_FAILURE_VALUE, METRIC_HTTP_LOGS_TRANSFORM_ELAPSED, METRIC_SUCCESS_VALUE, }; @@ -51,36 +52,24 @@ pub async fn get_pipeline( pub(crate) async fn run_pipeline( handler: &PipelineHandlerRef, - pipeline_definition: &PipelineDefinition, - pipeline_parameters: &GreptimePipelineParams, - data_array: Vec, - table_name: String, + pipeline_ctx: &PipelineContext<'_>, + pipeline_req: PipelineIngestRequest, query_ctx: &QueryContextRef, is_top_level: bool, ) -> Result> { - match pipeline_definition { + match &pipeline_ctx.pipeline_definition { PipelineDefinition::GreptimeIdentityPipeline(custom_ts) => { run_identity_pipeline( handler, custom_ts.as_ref(), - pipeline_parameters, - data_array, - table_name, + pipeline_ctx.pipeline_param, + pipeline_req, query_ctx, ) .await } _ => { - run_custom_pipeline( - handler, - pipeline_definition, - pipeline_parameters, - data_array, - table_name, - query_ctx, - is_top_level, - ) - .await + run_custom_pipeline(handler, pipeline_ctx, pipeline_req, query_ctx, is_top_level).await } } } @@ -89,10 +78,13 @@ async fn run_identity_pipeline( handler: &PipelineHandlerRef, custom_ts: Option<&IdentityTimeIndex>, pipeline_parameters: &GreptimePipelineParams, - data_array: Vec, - table_name: String, + pipeline_req: PipelineIngestRequest, query_ctx: &QueryContextRef, ) -> Result> { + let PipelineIngestRequest { + table: table_name, + values: data_array, + } = pipeline_req; let table = handler .get_table(&table_name, query_ctx) .await @@ -109,18 +101,20 @@ async fn run_identity_pipeline( async fn run_custom_pipeline( handler: &PipelineHandlerRef, - pipeline_definition: &PipelineDefinition, - pipeline_parameters: &GreptimePipelineParams, - data_array: Vec, - table_name: String, + pipeline_ctx: &PipelineContext<'_>, + pipeline_req: PipelineIngestRequest, query_ctx: &QueryContextRef, is_top_level: bool, ) -> Result> { let db = query_ctx.get_db_string(); - let pipeline = get_pipeline(pipeline_definition, handler, query_ctx).await?; + let pipeline = get_pipeline(pipeline_ctx.pipeline_definition, handler, query_ctx).await?; let transform_timer = std::time::Instant::now(); + let PipelineIngestRequest { + table: table_name, + values: data_array, + } = pipeline_req; let arr_len = data_array.len(); let mut req_map = HashMap::new(); let mut dispatched: BTreeMap> = BTreeMap::new(); @@ -185,12 +179,15 @@ async fn run_custom_pipeline( // run pipeline recursively. let next_pipeline_def = PipelineDefinition::from_name(next_pipeline_name, None, None).context(PipelineSnafu)?; + let next_pipeline_ctx = + PipelineContext::new(&next_pipeline_def, pipeline_ctx.pipeline_param); let requests = Box::pin(run_pipeline( handler, - &next_pipeline_def, - pipeline_parameters, - coll, - table_name, + &next_pipeline_ctx, + PipelineIngestRequest { + table: table_name, + values: coll, + }, query_ctx, false, )) From 7e5f6cbeae1e0ad4b8966f191d09a97a2e6499f4 Mon Sep 17 00:00:00 2001 From: ZonaHe Date: Tue, 22 Apr 2025 19:35:33 +0800 Subject: [PATCH 60/82] feat: update dashboard to v0.9.0 (#5948) Co-authored-by: ZonaHex --- src/servers/dashboard/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/servers/dashboard/VERSION b/src/servers/dashboard/VERSION index b19b521185..f979adec64 100644 --- a/src/servers/dashboard/VERSION +++ b/src/servers/dashboard/VERSION @@ -1 +1 @@ -v0.8.0 +v0.9.0 From 919956999b964322bf7a6e14ba745dc7ccdfed1f Mon Sep 17 00:00:00 2001 From: Yuhan Wang Date: Wed, 23 Apr 2025 07:48:32 +0800 Subject: [PATCH 61/82] fix: use max in flushed entry id and topic latest entry id (#5946) --- src/common/meta/src/region_registry.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/common/meta/src/region_registry.rs b/src/common/meta/src/region_registry.rs index 49d97f8a2f..4beb24c008 100644 --- a/src/common/meta/src/region_registry.rs +++ b/src/common/meta/src/region_registry.rs @@ -113,8 +113,10 @@ impl LeaderRegionManifestInfo { pub fn prunable_entry_id(&self) -> u64 { match self { LeaderRegionManifestInfo::Mito { - flushed_entry_id, .. - } => *flushed_entry_id, + flushed_entry_id, + topic_latest_entry_id, + .. + } => (*flushed_entry_id).max(*topic_latest_entry_id), LeaderRegionManifestInfo::Metric { data_flushed_entry_id, data_topic_latest_entry_id, From 8c4796734ac78140368e77fabadc3f467559daed Mon Sep 17 00:00:00 2001 From: fys <40801205+fengys1996@users.noreply.github.com> Date: Wed, 23 Apr 2025 11:17:13 +0800 Subject: [PATCH 62/82] chore: remove unused attribute (#5960) --- src/flow/src/adapter.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/flow/src/adapter.rs b/src/flow/src/adapter.rs index 516254ae55..d0d88d978a 100644 --- a/src/flow/src/adapter.rs +++ b/src/flow/src/adapter.rs @@ -746,7 +746,6 @@ impl FlowWorkerManager { /// steps to create task: /// 1. parse query into typed plan(and optional parse expire_after expr) /// 2. render source/sink with output table id and used input table id - #[allow(clippy::too_many_arguments)] pub async fn create_flow_inner(&self, args: CreateFlowArgs) -> Result, Error> { let CreateFlowArgs { flow_id, From 55cadcd2c04b58e6ddbdbeacb9eb1d4d1c1337f5 Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Wed, 23 Apr 2025 12:51:22 +0800 Subject: [PATCH 63/82] feat: introduce flush metadata region task for metric engine (#5951) * feat: introduce flush metadata region task for metric engine * docs: generate config.md * chore: add header * test: fix unit test * fix: fix unit tests * chore: apply suggestions from CR * chore: remove docs * fix: fix unit tests --- Cargo.lock | 2 + src/cmd/tests/load_config_test.rs | 2 + src/datanode/src/datanode.rs | 11 +- src/datanode/src/error.rs | 8 ++ src/metric-engine/Cargo.toml | 2 + src/metric-engine/src/config.rs | 42 ++++++- src/metric-engine/src/engine.rs | 49 +++++--- src/metric-engine/src/engine/create.rs | 2 +- src/metric-engine/src/error.rs | 10 ++ src/metric-engine/src/lib.rs | 1 + src/metric-engine/src/repeated_task.rs | 167 +++++++++++++++++++++++++ src/metric-engine/src/test_util.rs | 9 +- src/mito2/src/worker/handle_flush.rs | 17 ++- 13 files changed, 289 insertions(+), 33 deletions(-) create mode 100644 src/metric-engine/src/repeated_task.rs diff --git a/Cargo.lock b/Cargo.lock index a98826107b..3b74af8971 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7040,12 +7040,14 @@ dependencies = [ "common-macro", "common-query", "common-recordbatch", + "common-runtime", "common-telemetry", "common-test-util", "common-time", "datafusion", "datatypes", "futures-util", + "humantime-serde", "itertools 0.14.0", "lazy_static", "mito2", diff --git a/src/cmd/tests/load_config_test.rs b/src/cmd/tests/load_config_test.rs index 73d1295417..07be39dddb 100644 --- a/src/cmd/tests/load_config_test.rs +++ b/src/cmd/tests/load_config_test.rs @@ -74,6 +74,7 @@ fn test_load_datanode_example_config() { RegionEngineConfig::File(FileEngineConfig {}), RegionEngineConfig::Metric(MetricEngineConfig { experimental_sparse_primary_key_encoding: false, + flush_metadata_region_interval: Duration::from_secs(30), }), ], logging: LoggingOptions { @@ -216,6 +217,7 @@ fn test_load_standalone_example_config() { RegionEngineConfig::File(FileEngineConfig {}), RegionEngineConfig::Metric(MetricEngineConfig { experimental_sparse_primary_key_encoding: false, + flush_metadata_region_interval: Duration::from_secs(30), }), ], storage: StorageConfig { diff --git a/src/datanode/src/datanode.rs b/src/datanode/src/datanode.rs index 4b1e720032..25cb5a9d0c 100644 --- a/src/datanode/src/datanode.rs +++ b/src/datanode/src/datanode.rs @@ -57,9 +57,9 @@ use tokio::sync::Notify; use crate::config::{DatanodeOptions, RegionEngineConfig, StorageConfig}; use crate::error::{ - self, BuildMitoEngineSnafu, CreateDirSnafu, GetMetadataSnafu, MissingCacheSnafu, - MissingKvBackendSnafu, MissingNodeIdSnafu, OpenLogStoreSnafu, Result, ShutdownInstanceSnafu, - ShutdownServerSnafu, StartServerSnafu, + self, BuildMetricEngineSnafu, BuildMitoEngineSnafu, CreateDirSnafu, GetMetadataSnafu, + MissingCacheSnafu, MissingKvBackendSnafu, MissingNodeIdSnafu, OpenLogStoreSnafu, Result, + ShutdownInstanceSnafu, ShutdownServerSnafu, StartServerSnafu, }; use crate::event_listener::{ new_region_server_event_channel, NoopRegionServerEventListener, RegionServerEventListenerRef, @@ -416,10 +416,11 @@ impl DatanodeBuilder { ) .await?; - let metric_engine = MetricEngine::new( + let metric_engine = MetricEngine::try_new( mito_engine.clone(), metric_engine_config.take().unwrap_or_default(), - ); + ) + .context(BuildMetricEngineSnafu)?; engines.push(Arc::new(mito_engine) as _); engines.push(Arc::new(metric_engine) as _); } diff --git a/src/datanode/src/error.rs b/src/datanode/src/error.rs index 6c63e9115f..84e8168d91 100644 --- a/src/datanode/src/error.rs +++ b/src/datanode/src/error.rs @@ -336,6 +336,13 @@ pub enum Error { location: Location, }, + #[snafu(display("Failed to build metric engine"))] + BuildMetricEngine { + source: metric_engine::error::Error, + #[snafu(implicit)] + location: Location, + }, + #[snafu(display("Failed to serialize options to TOML"))] TomlFormat { #[snafu(implicit)] @@ -452,6 +459,7 @@ impl ErrorExt for Error { FindLogicalRegions { source, .. } => source.status_code(), BuildMitoEngine { source, .. } => source.status_code(), + BuildMetricEngine { source, .. } => source.status_code(), ConcurrentQueryLimiterClosed { .. } | ConcurrentQueryLimiterTimeout { .. } => { StatusCode::RegionBusy } diff --git a/src/metric-engine/Cargo.toml b/src/metric-engine/Cargo.toml index 5fe5ed3cb5..9e7a5b8545 100644 --- a/src/metric-engine/Cargo.toml +++ b/src/metric-engine/Cargo.toml @@ -18,11 +18,13 @@ common-error.workspace = true common-macro.workspace = true common-query.workspace = true common-recordbatch.workspace = true +common-runtime.workspace = true common-telemetry.workspace = true common-time.workspace = true datafusion.workspace = true datatypes.workspace = true futures-util.workspace = true +humantime-serde.workspace = true itertools.workspace = true lazy_static = "1.4" mito2.workspace = true diff --git a/src/metric-engine/src/config.rs b/src/metric-engine/src/config.rs index b35b412188..20df8fa739 100644 --- a/src/metric-engine/src/config.rs +++ b/src/metric-engine/src/config.rs @@ -12,9 +12,49 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::time::Duration; + +use common_telemetry::warn; use serde::{Deserialize, Serialize}; -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +/// The default flush interval of the metadata region. +pub(crate) const DEFAULT_FLUSH_METADATA_REGION_INTERVAL: Duration = Duration::from_secs(30); + +/// Configuration for the metric engine. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct EngineConfig { + /// Experimental feature to use sparse primary key encoding. pub experimental_sparse_primary_key_encoding: bool, + /// The flush interval of the metadata region. + #[serde( + with = "humantime_serde", + default = "EngineConfig::default_flush_metadata_region_interval" + )] + pub flush_metadata_region_interval: Duration, +} + +impl Default for EngineConfig { + fn default() -> Self { + Self { + flush_metadata_region_interval: DEFAULT_FLUSH_METADATA_REGION_INTERVAL, + experimental_sparse_primary_key_encoding: false, + } + } +} + +impl EngineConfig { + fn default_flush_metadata_region_interval() -> Duration { + DEFAULT_FLUSH_METADATA_REGION_INTERVAL + } + + /// Sanitizes the configuration. + pub fn sanitize(&mut self) { + if self.flush_metadata_region_interval.is_zero() { + warn!( + "Flush metadata region interval is zero, override with default value: {:?}. Disable metadata region flush is forbidden.", + DEFAULT_FLUSH_METADATA_REGION_INTERVAL + ); + self.flush_metadata_region_interval = DEFAULT_FLUSH_METADATA_REGION_INTERVAL; + } + } } diff --git a/src/metric-engine/src/engine.rs b/src/metric-engine/src/engine.rs index 28709264b3..15caa2c456 100644 --- a/src/metric-engine/src/engine.rs +++ b/src/metric-engine/src/engine.rs @@ -34,9 +34,11 @@ use api::region::RegionResponse; use async_trait::async_trait; use common_error::ext::{BoxedError, ErrorExt}; use common_error::status_code::StatusCode; +use common_runtime::RepeatedTask; use mito2::engine::MitoEngine; pub(crate) use options::IndexOptions; use snafu::ResultExt; +pub(crate) use state::MetricEngineState; use store_api::metadata::RegionMetadataRef; use store_api::metric_engine_consts::METRIC_ENGINE_NAME; use store_api::region_engine::{ @@ -47,11 +49,11 @@ use store_api::region_engine::{ use store_api::region_request::{BatchRegionDdlRequest, RegionRequest}; use store_api::storage::{RegionId, ScanRequest, SequenceNumber}; -use self::state::MetricEngineState; use crate::config::EngineConfig; use crate::data_region::DataRegion; -use crate::error::{self, Result, UnsupportedRegionRequestSnafu}; +use crate::error::{self, Error, Result, StartRepeatedTaskSnafu, UnsupportedRegionRequestSnafu}; use crate::metadata_region::MetadataRegion; +use crate::repeated_task::FlushMetadataRegionTask; use crate::row_modifier::RowModifier; use crate::utils::{self, get_region_statistic}; @@ -359,19 +361,32 @@ impl RegionEngine for MetricEngine { } impl MetricEngine { - pub fn new(mito: MitoEngine, config: EngineConfig) -> Self { + pub fn try_new(mito: MitoEngine, mut config: EngineConfig) -> Result { let metadata_region = MetadataRegion::new(mito.clone()); let data_region = DataRegion::new(mito.clone()); - Self { - inner: Arc::new(MetricEngineInner { - mito, - metadata_region, - data_region, - state: RwLock::default(), - config, - row_modifier: RowModifier::new(), - }), - } + let state = Arc::new(RwLock::default()); + config.sanitize(); + let flush_interval = config.flush_metadata_region_interval; + let inner = Arc::new(MetricEngineInner { + mito: mito.clone(), + metadata_region, + data_region, + state: state.clone(), + config, + row_modifier: RowModifier::new(), + flush_task: RepeatedTask::new( + flush_interval, + Box::new(FlushMetadataRegionTask { + state: state.clone(), + mito: mito.clone(), + }), + ), + }); + inner + .flush_task + .start(common_runtime::global_runtime()) + .context(StartRepeatedTaskSnafu { name: "flush_task" })?; + Ok(Self { inner }) } pub fn mito(&self) -> MitoEngine { @@ -426,15 +441,21 @@ impl MetricEngine { ) -> Result { self.inner.scan_to_stream(region_id, request).await } + + /// Returns the configuration of the engine. + pub fn config(&self) -> &EngineConfig { + &self.inner.config + } } struct MetricEngineInner { mito: MitoEngine, metadata_region: MetadataRegion, data_region: DataRegion, - state: RwLock, + state: Arc>, config: EngineConfig, row_modifier: RowModifier, + flush_task: RepeatedTask, } #[cfg(test)] diff --git a/src/metric-engine/src/engine/create.rs b/src/metric-engine/src/engine/create.rs index 78d2293302..596054623d 100644 --- a/src/metric-engine/src/engine/create.rs +++ b/src/metric-engine/src/engine/create.rs @@ -737,7 +737,7 @@ mod test { // set up let env = TestEnv::new().await; - let engine = MetricEngine::new(env.mito(), EngineConfig::default()); + let engine = MetricEngine::try_new(env.mito(), EngineConfig::default()).unwrap(); let engine_inner = engine.inner; // check create data region request diff --git a/src/metric-engine/src/error.rs b/src/metric-engine/src/error.rs index 015e9d9f98..c0ae55e402 100644 --- a/src/metric-engine/src/error.rs +++ b/src/metric-engine/src/error.rs @@ -282,6 +282,14 @@ pub enum Error { #[snafu(implicit)] location: Location, }, + + #[snafu(display("Failed to start repeated task: {}", name))] + StartRepeatedTask { + name: String, + source: common_runtime::error::Error, + #[snafu(implicit)] + location: Location, + }, } pub type Result = std::result::Result; @@ -335,6 +343,8 @@ impl ErrorExt for Error { CollectRecordBatchStream { source, .. } => source.status_code(), + StartRepeatedTask { source, .. } => source.status_code(), + MetricManifestInfo { .. } => StatusCode::Internal, } } diff --git a/src/metric-engine/src/lib.rs b/src/metric-engine/src/lib.rs index 597e8f5897..7722f3acd1 100644 --- a/src/metric-engine/src/lib.rs +++ b/src/metric-engine/src/lib.rs @@ -59,6 +59,7 @@ pub mod engine; pub mod error; mod metadata_region; mod metrics; +mod repeated_task; pub mod row_modifier; #[cfg(test)] mod test_util; diff --git a/src/metric-engine/src/repeated_task.rs b/src/metric-engine/src/repeated_task.rs new file mode 100644 index 0000000000..e5e7f7025f --- /dev/null +++ b/src/metric-engine/src/repeated_task.rs @@ -0,0 +1,167 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::{Arc, RwLock}; +use std::time::Instant; + +use common_runtime::TaskFunction; +use common_telemetry::{debug, error}; +use mito2::engine::MitoEngine; +use store_api::region_engine::{RegionEngine, RegionRole}; +use store_api::region_request::{RegionFlushRequest, RegionRequest}; + +use crate::engine::MetricEngineState; +use crate::error::{Error, Result}; +use crate::utils; + +/// Task to flush metadata regions. +/// +/// This task is used to send flush requests to the metadata regions +/// periodically. +pub(crate) struct FlushMetadataRegionTask { + pub(crate) state: Arc>, + pub(crate) mito: MitoEngine, +} + +#[async_trait::async_trait] +impl TaskFunction for FlushMetadataRegionTask { + fn name(&self) -> &str { + "FlushMetadataRegionTask" + } + + async fn call(&mut self) -> Result<()> { + let region_ids = { + let state = self.state.read().unwrap(); + state + .physical_region_states() + .keys() + .cloned() + .collect::>() + }; + + let num_region = region_ids.len(); + let now = Instant::now(); + for region_id in region_ids { + let Some(role) = self.mito.role(region_id) else { + continue; + }; + if role == RegionRole::Follower { + continue; + } + let metadata_region_id = utils::to_metadata_region_id(region_id); + if let Err(e) = self + .mito + .handle_request( + metadata_region_id, + RegionRequest::Flush(RegionFlushRequest { + row_group_size: None, + }), + ) + .await + { + error!(e; "Failed to flush metadata region {}", metadata_region_id); + } + } + debug!( + "Flushed {} metadata regions, elapsed: {:?}", + num_region, + now.elapsed() + ); + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::assert_matches::assert_matches; + use std::time::Duration; + + use store_api::region_engine::{RegionEngine, RegionManifestInfo}; + + use crate::config::{EngineConfig, DEFAULT_FLUSH_METADATA_REGION_INTERVAL}; + use crate::test_util::TestEnv; + + #[tokio::test] + async fn test_flush_metadata_region_task() { + let env = TestEnv::with_prefix_and_config( + "test_flush_metadata_region_task", + EngineConfig { + flush_metadata_region_interval: Duration::from_millis(100), + ..Default::default() + }, + ) + .await; + env.init_metric_region().await; + let engine = env.metric(); + // Wait for flush task run + tokio::time::sleep(Duration::from_millis(200)).await; + let physical_region_id = env.default_physical_region_id(); + let stat = engine.region_statistic(physical_region_id).unwrap(); + + assert_matches!( + stat.manifest, + RegionManifestInfo::Metric { + metadata_manifest_version: 1, + metadata_flushed_entry_id: 1, + .. + } + ) + } + + #[tokio::test] + async fn test_flush_metadata_region_task_with_long_interval() { + let env = TestEnv::with_prefix_and_config( + "test_flush_metadata_region_task_with_long_interval", + EngineConfig { + flush_metadata_region_interval: Duration::from_secs(60), + ..Default::default() + }, + ) + .await; + env.init_metric_region().await; + let engine = env.metric(); + // Wait for flush task run, should not flush metadata region + tokio::time::sleep(Duration::from_millis(200)).await; + let physical_region_id = env.default_physical_region_id(); + let stat = engine.region_statistic(physical_region_id).unwrap(); + + assert_matches!( + stat.manifest, + RegionManifestInfo::Metric { + metadata_manifest_version: 0, + metadata_flushed_entry_id: 0, + .. + } + ) + } + + #[tokio::test] + async fn test_flush_metadata_region_sanitize() { + let env = TestEnv::with_prefix_and_config( + "test_flush_metadata_region_sanitize", + EngineConfig { + flush_metadata_region_interval: Duration::from_secs(0), + ..Default::default() + }, + ) + .await; + let metric = env.metric(); + let config = metric.config(); + assert_eq!( + config.flush_metadata_region_interval, + DEFAULT_FLUSH_METADATA_REGION_INTERVAL + ); + } +} diff --git a/src/metric-engine/src/test_util.rs b/src/metric-engine/src/test_util.rs index 6bcc002908..e750b516f1 100644 --- a/src/metric-engine/src/test_util.rs +++ b/src/metric-engine/src/test_util.rs @@ -54,9 +54,14 @@ impl TestEnv { /// Returns a new env with specific `prefix` for test. pub async fn with_prefix(prefix: &str) -> Self { + Self::with_prefix_and_config(prefix, EngineConfig::default()).await + } + + /// Returns a new env with specific `prefix` and `config` for test. + pub async fn with_prefix_and_config(prefix: &str, config: EngineConfig) -> Self { let mut mito_env = MitoTestEnv::with_prefix(prefix); let mito = mito_env.create_engine(MitoConfig::default()).await; - let metric = MetricEngine::new(mito.clone(), EngineConfig::default()); + let metric = MetricEngine::try_new(mito.clone(), config).unwrap(); Self { mito_env, mito, @@ -84,7 +89,7 @@ impl TestEnv { .mito_env .create_follower_engine(MitoConfig::default()) .await; - let metric = MetricEngine::new(mito.clone(), EngineConfig::default()); + let metric = MetricEngine::try_new(mito.clone(), EngineConfig::default()).unwrap(); let region_id = self.default_physical_region_id(); debug!("opening default physical region: {region_id}"); diff --git a/src/mito2/src/worker/handle_flush.rs b/src/mito2/src/worker/handle_flush.rs index dc5a58d794..ca7499f326 100644 --- a/src/mito2/src/worker/handle_flush.rs +++ b/src/mito2/src/worker/handle_flush.rs @@ -141,11 +141,6 @@ impl RegionWorkerLoop { // But the flush is skipped if memtables are empty. Thus should update the `topic_latest_entry_id` // when handling flush request instead of in `schedule_flush` or `flush_finished`. self.update_topic_latest_entry_id(®ion); - info!( - "Region {} flush request, high watermark: {}", - region_id, - region.topic_latest_entry_id.load(Ordering::Relaxed) - ); let reason = if region.is_downgrading() { FlushReason::Downgrading @@ -268,15 +263,17 @@ impl RegionWorkerLoop { .store() .high_watermark(®ion.provider) .unwrap_or(0); - if high_watermark != 0 { + let topic_last_entry_id = region.topic_latest_entry_id.load(Ordering::Relaxed); + + if high_watermark != 0 && high_watermark > topic_last_entry_id { region .topic_latest_entry_id .store(high_watermark, Ordering::Relaxed); + info!( + "Region {} high watermark updated to {}", + region.region_id, high_watermark + ); } - info!( - "Region {} high watermark updated to {}", - region.region_id, high_watermark - ); } } } From 02e9a66d7a35b8f6bf912b122631b2b864f50115 Mon Sep 17 00:00:00 2001 From: zyy17 Date: Wed, 23 Apr 2025 13:00:37 +0800 Subject: [PATCH 64/82] chore: update dac tools image and docs (#5961) --- grafana/README.md | 6 +- grafana/dashboards/cluster/dashboard.md | 132 ++++++++++----------- grafana/dashboards/standalone/dashboard.md | 132 ++++++++++----------- grafana/scripts/gen-dashboards.sh | 13 +- 4 files changed, 146 insertions(+), 137 deletions(-) diff --git a/grafana/README.md b/grafana/README.md index 9c13b0e322..31cfcccd06 100644 --- a/grafana/README.md +++ b/grafana/README.md @@ -4,11 +4,13 @@ This repository maintains the Grafana dashboards for GreptimeDB. It has two types of dashboards: -- `cluster/`: The dashboard for the GreptimeDB cluster. Read the [dashboard.md](./dashboards/cluster/dashboard.md) for more details. -- `standalone/`: The dashboard for the standalone GreptimeDB instance. Read the [dashboard.md](./dashboards/standalone/dashboard.md) for more details. +- `cluster/dashboard.json`: The Grafana dashboard for the GreptimeDB cluster. Read the [dashboard.md](./dashboards/cluster/dashboard.md) for more details. +- `standalone/dashboard.json`: The Grafana dashboard for the standalone GreptimeDB instance. **It's generated from the `cluster/dashboard.json` by removing the instance filter through the `make dashboards` command**. Read the [dashboard.md](./dashboards/standalone/dashboard.md) for more details. As the rapid development of GreptimeDB, the metrics may be changed, and please feel free to submit your feedback and/or contribution to this dashboard 🤗 +**NOTE**: If you want to modify the dashboards, you only need to modify the `cluster/dashboard.json` and run the `make dashboards` command to generate the `standalone/dashboard.json` and other related files. + To maintain the dashboards, we use the [`dac`](https://github.com/zyy17/dac) tool to generate the intermediate dashboards and markdown documents: - `cluster/dashboard.yaml`: The intermediate dashboard for the GreptimeDB cluster. diff --git a/grafana/dashboards/cluster/dashboard.md b/grafana/dashboards/cluster/dashboard.md index 2bc100f860..feef106365 100644 --- a/grafana/dashboards/cluster/dashboard.md +++ b/grafana/dashboards/cluster/dashboard.md @@ -1,96 +1,96 @@ # Overview | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Uptime | `time() - process_start_time_seconds` | `stat` | The start time of GreptimeDB. | `s` | `prometheus` | `__auto` | -| Version | `SELECT pkg_version FROM information_schema.build_info` | `stat` | GreptimeDB version. | -- | `mysql` | -- | -| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))` | `stat` | Total ingestion rate. | `rowsps` | `prometheus` | `__auto` | -| Total Storage Size | `select SUM(disk_size) from information_schema.region_statistics;` | `stat` | Total number of data file size. | `decbytes` | `mysql` | -- | -| Total Rows | `select SUM(region_rows) from information_schema.region_statistics;` | `stat` | Total number of data rows in the cluster. Calculated by sum of rows from each region. | `sishort` | `mysql` | -- | -| Deployment | `SELECT count(*) as datanode FROM information_schema.cluster_info WHERE peer_type = 'DATANODE';`
`SELECT count(*) as frontend FROM information_schema.cluster_info WHERE peer_type = 'FRONTEND';`
`SELECT count(*) as metasrv FROM information_schema.cluster_info WHERE peer_type = 'METASRV';`
`SELECT count(*) as flownode FROM information_schema.cluster_info WHERE peer_type = 'FLOWNODE';` | `stat` | The deployment topology of GreptimeDB. | -- | `mysql` | -- | -| Database Resources | `SELECT COUNT(*) as databases FROM information_schema.schemata WHERE schema_name NOT IN ('greptime_private', 'information_schema')`
`SELECT COUNT(*) as tables FROM information_schema.tables WHERE table_schema != 'information_schema'`
`SELECT COUNT(region_id) as regions FROM information_schema.region_peers`
`SELECT COUNT(*) as flows FROM information_schema.flows` | `stat` | The number of the key resources in GreptimeDB. | -- | `mysql` | -- | -| Data Size | `SELECT SUM(memtable_size) * 0.42825 as WAL FROM information_schema.region_statistics;`
`SELECT SUM(index_size) as index FROM information_schema.region_statistics;`
`SELECT SUM(manifest_size) as manifest FROM information_schema.region_statistics;` | `stat` | The data size of wal/index/manifest in the GreptimeDB. | `decbytes` | `mysql` | -- | +| Uptime | `time() - process_start_time_seconds` | `stat` | The start time of GreptimeDB. | `prometheus` | `s` | `__auto` | +| Version | `SELECT pkg_version FROM information_schema.build_info` | `stat` | GreptimeDB version. | `mysql` | -- | -- | +| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))` | `stat` | Total ingestion rate. | `prometheus` | `rowsps` | `__auto` | +| Total Storage Size | `select SUM(disk_size) from information_schema.region_statistics;` | `stat` | Total number of data file size. | `mysql` | `decbytes` | -- | +| Total Rows | `select SUM(region_rows) from information_schema.region_statistics;` | `stat` | Total number of data rows in the cluster. Calculated by sum of rows from each region. | `mysql` | `sishort` | -- | +| Deployment | `SELECT count(*) as datanode FROM information_schema.cluster_info WHERE peer_type = 'DATANODE';`
`SELECT count(*) as frontend FROM information_schema.cluster_info WHERE peer_type = 'FRONTEND';`
`SELECT count(*) as metasrv FROM information_schema.cluster_info WHERE peer_type = 'METASRV';`
`SELECT count(*) as flownode FROM information_schema.cluster_info WHERE peer_type = 'FLOWNODE';` | `stat` | The deployment topology of GreptimeDB. | `mysql` | -- | -- | +| Database Resources | `SELECT COUNT(*) as databases FROM information_schema.schemata WHERE schema_name NOT IN ('greptime_private', 'information_schema')`
`SELECT COUNT(*) as tables FROM information_schema.tables WHERE table_schema != 'information_schema'`
`SELECT COUNT(region_id) as regions FROM information_schema.region_peers`
`SELECT COUNT(*) as flows FROM information_schema.flows` | `stat` | The number of the key resources in GreptimeDB. | `mysql` | -- | -- | +| Data Size | `SELECT SUM(memtable_size) * 0.42825 as WAL FROM information_schema.region_statistics;`
`SELECT SUM(index_size) as index FROM information_schema.region_statistics;`
`SELECT SUM(manifest_size) as manifest FROM information_schema.region_statistics;` | `stat` | The data size of wal/index/manifest in the GreptimeDB. | `mysql` | `decbytes` | -- | # Ingestion | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Total ingestion rate.

Here we listed 3 primary protocols:

- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `rowsps` | `prometheus` | `ingestion` | -| Ingestion Rate by Type | `sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))`
`sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))` | `timeseries` | Total ingestion rate.

Here we listed 3 primary protocols:

- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `rowsps` | `prometheus` | `http-logs` | +| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Total ingestion rate.

Here we listed 3 primary protocols:

- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `prometheus` | `rowsps` | `ingestion` | +| Ingestion Rate by Type | `sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))`
`sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))` | `timeseries` | Total ingestion rate.

Here we listed 3 primary protocols:

- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `prometheus` | `rowsps` | `http-logs` | # Queries | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Total Query Rate | `sum (rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum (rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum (rate(greptime_servers_http_promql_elapsed_counte{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Total rate of query API calls by protocol. This metric is collected from frontends.

Here we listed 3 main protocols:
- MySQL
- Postgres
- Prometheus API

Note that there are some other minor query APIs like /sql are not included | `reqps` | `prometheus` | `mysql` | +| Total Query Rate | `sum (rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum (rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum (rate(greptime_servers_http_promql_elapsed_counte{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Total rate of query API calls by protocol. This metric is collected from frontends.

Here we listed 3 main protocols:
- MySQL
- Postgres
- Prometheus API

Note that there are some other minor query APIs like /sql are not included | `prometheus` | `reqps` | `mysql` | # Resources | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Datanode Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$datanode"}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `decbytes` | `prometheus` | `[{{instance}}]-[{{ pod }}]` | -| Datanode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$datanode"}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `none` | `prometheus` | `[{{ instance }}]-[{{ pod }}]` | -| Frontend Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$frontend"}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `decbytes` | `prometheus` | `[{{ instance }}]-[{{ pod }}]` | -| Frontend CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$frontend"}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `none` | `prometheus` | `[{{ instance }}]-[{{ pod }}]-cpu` | -| Metasrv Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$metasrv"}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `decbytes` | `prometheus` | `[{{ instance }}]-[{{ pod }}]-resident` | -| Metasrv CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$metasrv"}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `none` | `prometheus` | `[{{ instance }}]-[{{ pod }}]` | -| Flownode Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$flownode"}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `decbytes` | `prometheus` | `[{{ instance }}]-[{{ pod }}]` | -| Flownode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$flownode"}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `none` | `prometheus` | `[{{ instance }}]-[{{ pod }}]` | +| Datanode Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$datanode"}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `prometheus` | `decbytes` | `[{{instance}}]-[{{ pod }}]` | +| Datanode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$datanode"}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` | +| Frontend Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$frontend"}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `prometheus` | `decbytes` | `[{{ instance }}]-[{{ pod }}]` | +| Frontend CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$frontend"}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]-cpu` | +| Metasrv Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$metasrv"}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `prometheus` | `decbytes` | `[{{ instance }}]-[{{ pod }}]-resident` | +| Metasrv CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$metasrv"}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` | +| Flownode Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$flownode"}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `prometheus` | `decbytes` | `[{{ instance }}]-[{{ pod }}]` | +| Flownode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$flownode"}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` | # Frontend Requests | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| HTTP QPS per Instance | `sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | HTTP QPS per Instance. | `reqps` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]` | -| HTTP P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval])))` | `timeseries` | HTTP P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` | -| gRPC QPS per Instance | `sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | gRPC QPS per Instance. | `reqps` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]` | -| gRPC P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | gRPC P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` | -| MySQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | MySQL QPS per Instance. | `reqps` | `prometheus` | `[{{instance}}]-[{{pod}}]` | -| MySQL P99 per Instance | `histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | MySQL P99 per Instance. | `s` | `prometheus` | `[{{ instance }}]-[{{ pod }}]-p99` | -| PostgreSQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | PostgreSQL QPS per Instance. | `reqps` | `prometheus` | `[{{instance}}]-[{{pod}}]` | -| PostgreSQL P99 per Instance | `histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | PostgreSQL P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-p99` | +| HTTP QPS per Instance | `sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | HTTP QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]` | +| HTTP P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval])))` | `timeseries` | HTTP P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` | +| gRPC QPS per Instance | `sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | gRPC QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]` | +| gRPC P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | gRPC P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` | +| MySQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | MySQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` | +| MySQL P99 per Instance | `histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | MySQL P99 per Instance. | `prometheus` | `s` | `[{{ instance }}]-[{{ pod }}]-p99` | +| PostgreSQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | PostgreSQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` | +| PostgreSQL P99 per Instance | `histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | PostgreSQL P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` | # Frontend to Datanode | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Ingest Rows per Instance | `sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Ingestion rate by row as in each frontend | `rowsps` | `prometheus` | `[{{instance}}]-[{{pod}}]` | -| Region Call QPS per Instance | `sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Region Call QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` | -| Region Call P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | Region Call P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` | +| Ingest Rows per Instance | `sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Ingestion rate by row as in each frontend | `prometheus` | `rowsps` | `[{{instance}}]-[{{pod}}]` | +| Region Call QPS per Instance | `sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Region Call QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` | +| Region Call P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | Region Call P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` | # Mito Engine | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Request OPS per Instance | `sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Request QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | -| Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Request P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | -| Write Buffer per Instance | `greptime_mito_write_buffer_bytes{instance=~"$datanode"}` | `timeseries` | Write Buffer per Instance. | `decbytes` | `prometheus` | `[{{instance}}]-[{{pod}}]` | -| Write Rows per Instance | `sum by (instance, pod) (rate(greptime_mito_write_rows_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Ingestion size by row counts. | `rowsps` | `prometheus` | `[{{instance}}]-[{{pod}}]` | -| Flush OPS per Instance | `sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Flush QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{reason}}]` | -| Write Stall per Instance | `sum by(instance, pod) (greptime_mito_write_stall_total{instance=~"$datanode"})` | `timeseries` | Write Stall per Instance. | `decbytes` | `prometheus` | `[{{instance}}]-[{{pod}}]` | -| Read Stage OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{instance=~"$datanode", stage="total"}[$__rate_interval]))` | `timeseries` | Read Stage OPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]` | -| Read Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Read Stage P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | -| Write Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Write Stage P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | -| Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `ops` | `prometheus` | `[{{ instance }}]-[{{pod}}]` | -| Compaction P99 per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Compaction latency by stage | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` | -| Compaction P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Compaction P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction` | -| WAL write size | `histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))` | `timeseries` | Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. | `bytes` | `prometheus` | `[{{instance}}]-[{{pod}}]-req-size-p95` | -| Cached Bytes per Instance | `greptime_mito_cache_bytes{instance=~"$datanode"}` | `timeseries` | Cached Bytes per Instance. | `decbytes` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | -| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction task count | `none` | `prometheus` | `[{{instance}}]-[{{pod}}]` | -| WAL sync duration seconds | `histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))` | `timeseries` | Raft engine (local disk) log store sync latency, p99 | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-p99` | -| Log Store op duration seconds | `histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))` | `timeseries` | Write-ahead log operations latency at p99 | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99` | -| Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `none` | `prometheus` | `[{{instance}}]-[{{pod}}]` | +| Request OPS per Instance | `sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Request QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | +| Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | +| Write Buffer per Instance | `greptime_mito_write_buffer_bytes{instance=~"$datanode"}` | `timeseries` | Write Buffer per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]` | +| Write Rows per Instance | `sum by (instance, pod) (rate(greptime_mito_write_rows_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Ingestion size by row counts. | `prometheus` | `rowsps` | `[{{instance}}]-[{{pod}}]` | +| Flush OPS per Instance | `sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Flush QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{reason}}]` | +| Write Stall per Instance | `sum by(instance, pod) (greptime_mito_write_stall_total{instance=~"$datanode"})` | `timeseries` | Write Stall per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]` | +| Read Stage OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{instance=~"$datanode", stage="total"}[$__rate_interval]))` | `timeseries` | Read Stage OPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]` | +| Read Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Read Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | +| Write Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Write Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | +| Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `prometheus` | `ops` | `[{{ instance }}]-[{{pod}}]` | +| Compaction P99 per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Compaction latency by stage | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` | +| Compaction P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Compaction P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction` | +| WAL write size | `histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))` | `timeseries` | Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-req-size-p95` | +| Cached Bytes per Instance | `greptime_mito_cache_bytes{instance=~"$datanode"}` | `timeseries` | Cached Bytes per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | +| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` | +| WAL sync duration seconds | `histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))` | `timeseries` | Raft engine (local disk) log store sync latency, p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` | +| Log Store op duration seconds | `histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))` | `timeseries` | Write-ahead log operations latency at p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99` | +| Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` | # OpenDAL | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| Read QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | -| Read P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode",operation="read"}[$__rate_interval])))` | `timeseries` | Read P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-{{scheme}}` | -| Write QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="write"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-{{scheme}}` | -| Write P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="write"}[$__rate_interval])))` | `timeseries` | Write P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | -| List QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="list"}[$__rate_interval]))` | `timeseries` | List QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | -| List P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="list"}[$__rate_interval])))` | `timeseries` | List P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | -| Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode",operation!~"read\|write\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| Other Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation!~"read\|write\|list"}[$__rate_interval])))` | `timeseries` | Other Request P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Read QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | +| Read P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode",operation="read"}[$__rate_interval])))` | `timeseries` | Read P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-{{scheme}}` | +| Write QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="write"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-{{scheme}}` | +| Write P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="write"}[$__rate_interval])))` | `timeseries` | Write P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | +| List QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="list"}[$__rate_interval]))` | `timeseries` | List QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | +| List P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="list"}[$__rate_interval])))` | `timeseries` | List P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | +| Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode",operation!~"read\|write\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Other Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation!~"read\|write\|list"}[$__rate_interval])))` | `timeseries` | Other Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | # Metasrv | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Region migration datanode | `greptime_meta_region_migration_stat{datanode_type="src"}`
`greptime_meta_region_migration_stat{datanode_type="desc"}` | `state-timeline` | Counter of region migration by source and destination | `none` | `prometheus` | `from-datanode-{{datanode_id}}` | -| Region migration error | `greptime_meta_region_migration_error` | `timeseries` | Counter of region migration error | `none` | `prometheus` | `__auto` | -| Datanode load | `greptime_datanode_load` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `none` | `prometheus` | `__auto` | +| Region migration datanode | `greptime_meta_region_migration_stat{datanode_type="src"}`
`greptime_meta_region_migration_stat{datanode_type="desc"}` | `state-timeline` | Counter of region migration by source and destination | `prometheus` | `none` | `from-datanode-{{datanode_id}}` | +| Region migration error | `greptime_meta_region_migration_error` | `timeseries` | Counter of region migration error | `prometheus` | `none` | `__auto` | +| Datanode load | `greptime_datanode_load` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `none` | `__auto` | # Flownode | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Flow Ingest / Output Rate | `sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))` | `timeseries` | Flow Ingest / Output Rate. | -- | `prometheus` | `[{{pod}}]-[{{instance}}]-[{{direction}}]` | -| Flow Ingest Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`
`histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))` | `timeseries` | Flow Ingest Latency. | -- | `prometheus` | `[{{instance}}]-[{{pod}}]-p95` | -| Flow Operation Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))`
`histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))` | `timeseries` | Flow Operation Latency. | -- | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` | -| Flow Buffer Size per Instance | `greptime_flow_input_buf_size` | `timeseries` | Flow Buffer Size per Instance. | -- | `prometheus` | `[{{instance}}]-[{{pod}]` | -| Flow Processing Error per Instance | `sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))` | `timeseries` | Flow Processing Error per Instance. | -- | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{code}}]` | +| Flow Ingest / Output Rate | `sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))` | `timeseries` | Flow Ingest / Output Rate. | `prometheus` | -- | `[{{pod}}]-[{{instance}}]-[{{direction}}]` | +| Flow Ingest Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`
`histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))` | `timeseries` | Flow Ingest Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-p95` | +| Flow Operation Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))`
`histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))` | `timeseries` | Flow Operation Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` | +| Flow Buffer Size per Instance | `greptime_flow_input_buf_size` | `timeseries` | Flow Buffer Size per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}]` | +| Flow Processing Error per Instance | `sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))` | `timeseries` | Flow Processing Error per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{code}}]` | diff --git a/grafana/dashboards/standalone/dashboard.md b/grafana/dashboards/standalone/dashboard.md index a0ff6c03ca..fbd7158c9e 100644 --- a/grafana/dashboards/standalone/dashboard.md +++ b/grafana/dashboards/standalone/dashboard.md @@ -1,96 +1,96 @@ # Overview | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Uptime | `time() - process_start_time_seconds` | `stat` | The start time of GreptimeDB. | `s` | `prometheus` | `__auto` | -| Version | `SELECT pkg_version FROM information_schema.build_info` | `stat` | GreptimeDB version. | -- | `mysql` | -- | -| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))` | `stat` | Total ingestion rate. | `rowsps` | `prometheus` | `__auto` | -| Total Storage Size | `select SUM(disk_size) from information_schema.region_statistics;` | `stat` | Total number of data file size. | `decbytes` | `mysql` | -- | -| Total Rows | `select SUM(region_rows) from information_schema.region_statistics;` | `stat` | Total number of data rows in the cluster. Calculated by sum of rows from each region. | `sishort` | `mysql` | -- | -| Deployment | `SELECT count(*) as datanode FROM information_schema.cluster_info WHERE peer_type = 'DATANODE';`
`SELECT count(*) as frontend FROM information_schema.cluster_info WHERE peer_type = 'FRONTEND';`
`SELECT count(*) as metasrv FROM information_schema.cluster_info WHERE peer_type = 'METASRV';`
`SELECT count(*) as flownode FROM information_schema.cluster_info WHERE peer_type = 'FLOWNODE';` | `stat` | The deployment topology of GreptimeDB. | -- | `mysql` | -- | -| Database Resources | `SELECT COUNT(*) as databases FROM information_schema.schemata WHERE schema_name NOT IN ('greptime_private', 'information_schema')`
`SELECT COUNT(*) as tables FROM information_schema.tables WHERE table_schema != 'information_schema'`
`SELECT COUNT(region_id) as regions FROM information_schema.region_peers`
`SELECT COUNT(*) as flows FROM information_schema.flows` | `stat` | The number of the key resources in GreptimeDB. | -- | `mysql` | -- | -| Data Size | `SELECT SUM(memtable_size) * 0.42825 as WAL FROM information_schema.region_statistics;`
`SELECT SUM(index_size) as index FROM information_schema.region_statistics;`
`SELECT SUM(manifest_size) as manifest FROM information_schema.region_statistics;` | `stat` | The data size of wal/index/manifest in the GreptimeDB. | `decbytes` | `mysql` | -- | +| Uptime | `time() - process_start_time_seconds` | `stat` | The start time of GreptimeDB. | `prometheus` | `s` | `__auto` | +| Version | `SELECT pkg_version FROM information_schema.build_info` | `stat` | GreptimeDB version. | `mysql` | -- | -- | +| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))` | `stat` | Total ingestion rate. | `prometheus` | `rowsps` | `__auto` | +| Total Storage Size | `select SUM(disk_size) from information_schema.region_statistics;` | `stat` | Total number of data file size. | `mysql` | `decbytes` | -- | +| Total Rows | `select SUM(region_rows) from information_schema.region_statistics;` | `stat` | Total number of data rows in the cluster. Calculated by sum of rows from each region. | `mysql` | `sishort` | -- | +| Deployment | `SELECT count(*) as datanode FROM information_schema.cluster_info WHERE peer_type = 'DATANODE';`
`SELECT count(*) as frontend FROM information_schema.cluster_info WHERE peer_type = 'FRONTEND';`
`SELECT count(*) as metasrv FROM information_schema.cluster_info WHERE peer_type = 'METASRV';`
`SELECT count(*) as flownode FROM information_schema.cluster_info WHERE peer_type = 'FLOWNODE';` | `stat` | The deployment topology of GreptimeDB. | `mysql` | -- | -- | +| Database Resources | `SELECT COUNT(*) as databases FROM information_schema.schemata WHERE schema_name NOT IN ('greptime_private', 'information_schema')`
`SELECT COUNT(*) as tables FROM information_schema.tables WHERE table_schema != 'information_schema'`
`SELECT COUNT(region_id) as regions FROM information_schema.region_peers`
`SELECT COUNT(*) as flows FROM information_schema.flows` | `stat` | The number of the key resources in GreptimeDB. | `mysql` | -- | -- | +| Data Size | `SELECT SUM(memtable_size) * 0.42825 as WAL FROM information_schema.region_statistics;`
`SELECT SUM(index_size) as index FROM information_schema.region_statistics;`
`SELECT SUM(manifest_size) as manifest FROM information_schema.region_statistics;` | `stat` | The data size of wal/index/manifest in the GreptimeDB. | `mysql` | `decbytes` | -- | # Ingestion | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))` | `timeseries` | Total ingestion rate.

Here we listed 3 primary protocols:

- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `rowsps` | `prometheus` | `ingestion` | -| Ingestion Rate by Type | `sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))`
`sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))` | `timeseries` | Total ingestion rate.

Here we listed 3 primary protocols:

- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `rowsps` | `prometheus` | `http-logs` | +| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))` | `timeseries` | Total ingestion rate.

Here we listed 3 primary protocols:

- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `prometheus` | `rowsps` | `ingestion` | +| Ingestion Rate by Type | `sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))`
`sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))` | `timeseries` | Total ingestion rate.

Here we listed 3 primary protocols:

- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `prometheus` | `rowsps` | `http-logs` | # Queries | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Total Query Rate | `sum (rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval]))`
`sum (rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval]))`
`sum (rate(greptime_servers_http_promql_elapsed_counte{}[$__rate_interval]))` | `timeseries` | Total rate of query API calls by protocol. This metric is collected from frontends.

Here we listed 3 main protocols:
- MySQL
- Postgres
- Prometheus API

Note that there are some other minor query APIs like /sql are not included | `reqps` | `prometheus` | `mysql` | +| Total Query Rate | `sum (rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval]))`
`sum (rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval]))`
`sum (rate(greptime_servers_http_promql_elapsed_counte{}[$__rate_interval]))` | `timeseries` | Total rate of query API calls by protocol. This metric is collected from frontends.

Here we listed 3 main protocols:
- MySQL
- Postgres
- Prometheus API

Note that there are some other minor query APIs like /sql are not included | `prometheus` | `reqps` | `mysql` | # Resources | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Datanode Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `decbytes` | `prometheus` | `[{{instance}}]-[{{ pod }}]` | -| Datanode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `none` | `prometheus` | `[{{ instance }}]-[{{ pod }}]` | -| Frontend Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `decbytes` | `prometheus` | `[{{ instance }}]-[{{ pod }}]` | -| Frontend CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `none` | `prometheus` | `[{{ instance }}]-[{{ pod }}]-cpu` | -| Metasrv Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `decbytes` | `prometheus` | `[{{ instance }}]-[{{ pod }}]-resident` | -| Metasrv CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `none` | `prometheus` | `[{{ instance }}]-[{{ pod }}]` | -| Flownode Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `decbytes` | `prometheus` | `[{{ instance }}]-[{{ pod }}]` | -| Flownode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `none` | `prometheus` | `[{{ instance }}]-[{{ pod }}]` | +| Datanode Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `prometheus` | `decbytes` | `[{{instance}}]-[{{ pod }}]` | +| Datanode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` | +| Frontend Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `prometheus` | `decbytes` | `[{{ instance }}]-[{{ pod }}]` | +| Frontend CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]-cpu` | +| Metasrv Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `prometheus` | `decbytes` | `[{{ instance }}]-[{{ pod }}]-resident` | +| Metasrv CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` | +| Flownode Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)` | `timeseries` | Current memory usage by instance | `prometheus` | `decbytes` | `[{{ instance }}]-[{{ pod }}]` | +| Flownode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` | # Frontend Requests | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| HTTP QPS per Instance | `sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | HTTP QPS per Instance. | `reqps` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]` | -| HTTP P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{path!~"/health\|/metrics"}[$__rate_interval])))` | `timeseries` | HTTP P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` | -| gRPC QPS per Instance | `sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{}[$__rate_interval]))` | `timeseries` | gRPC QPS per Instance. | `reqps` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]` | -| gRPC P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | gRPC P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` | -| MySQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval]))` | `timeseries` | MySQL QPS per Instance. | `reqps` | `prometheus` | `[{{instance}}]-[{{pod}}]` | -| MySQL P99 per Instance | `histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | MySQL P99 per Instance. | `s` | `prometheus` | `[{{ instance }}]-[{{ pod }}]-p99` | -| PostgreSQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval]))` | `timeseries` | PostgreSQL QPS per Instance. | `reqps` | `prometheus` | `[{{instance}}]-[{{pod}}]` | -| PostgreSQL P99 per Instance | `histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | PostgreSQL P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-p99` | +| HTTP QPS per Instance | `sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | HTTP QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]` | +| HTTP P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{path!~"/health\|/metrics"}[$__rate_interval])))` | `timeseries` | HTTP P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` | +| gRPC QPS per Instance | `sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{}[$__rate_interval]))` | `timeseries` | gRPC QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]` | +| gRPC P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | gRPC P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` | +| MySQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval]))` | `timeseries` | MySQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` | +| MySQL P99 per Instance | `histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | MySQL P99 per Instance. | `prometheus` | `s` | `[{{ instance }}]-[{{ pod }}]-p99` | +| PostgreSQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval]))` | `timeseries` | PostgreSQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` | +| PostgreSQL P99 per Instance | `histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | PostgreSQL P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` | # Frontend to Datanode | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Ingest Rows per Instance | `sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))` | `timeseries` | Ingestion rate by row as in each frontend | `rowsps` | `prometheus` | `[{{instance}}]-[{{pod}}]` | -| Region Call QPS per Instance | `sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{}[$__rate_interval]))` | `timeseries` | Region Call QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` | -| Region Call P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{}[$__rate_interval])))` | `timeseries` | Region Call P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` | +| Ingest Rows per Instance | `sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))` | `timeseries` | Ingestion rate by row as in each frontend | `prometheus` | `rowsps` | `[{{instance}}]-[{{pod}}]` | +| Region Call QPS per Instance | `sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{}[$__rate_interval]))` | `timeseries` | Region Call QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` | +| Region Call P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{}[$__rate_interval])))` | `timeseries` | Region Call P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` | # Mito Engine | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Request OPS per Instance | `sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{}[$__rate_interval]))` | `timeseries` | Request QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | -| Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Request P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | -| Write Buffer per Instance | `greptime_mito_write_buffer_bytes{}` | `timeseries` | Write Buffer per Instance. | `decbytes` | `prometheus` | `[{{instance}}]-[{{pod}}]` | -| Write Rows per Instance | `sum by (instance, pod) (rate(greptime_mito_write_rows_total{}[$__rate_interval]))` | `timeseries` | Ingestion size by row counts. | `rowsps` | `prometheus` | `[{{instance}}]-[{{pod}}]` | -| Flush OPS per Instance | `sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{}[$__rate_interval]))` | `timeseries` | Flush QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{reason}}]` | -| Write Stall per Instance | `sum by(instance, pod) (greptime_mito_write_stall_total{})` | `timeseries` | Write Stall per Instance. | `decbytes` | `prometheus` | `[{{instance}}]-[{{pod}}]` | -| Read Stage OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{ stage="total"}[$__rate_interval]))` | `timeseries` | Read Stage OPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]` | -| Read Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Read Stage P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | -| Write Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Write Stage P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | -| Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{}[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `ops` | `prometheus` | `[{{ instance }}]-[{{pod}}]` | -| Compaction P99 per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Compaction latency by stage | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` | -| Compaction P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Compaction P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction` | -| WAL write size | `histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))` | `timeseries` | Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. | `bytes` | `prometheus` | `[{{instance}}]-[{{pod}}]-req-size-p95` | -| Cached Bytes per Instance | `greptime_mito_cache_bytes{}` | `timeseries` | Cached Bytes per Instance. | `decbytes` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | -| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction task count | `none` | `prometheus` | `[{{instance}}]-[{{pod}}]` | -| WAL sync duration seconds | `histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))` | `timeseries` | Raft engine (local disk) log store sync latency, p99 | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-p99` | -| Log Store op duration seconds | `histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))` | `timeseries` | Write-ahead log operations latency at p99 | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99` | -| Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `none` | `prometheus` | `[{{instance}}]-[{{pod}}]` | +| Request OPS per Instance | `sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{}[$__rate_interval]))` | `timeseries` | Request QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | +| Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | +| Write Buffer per Instance | `greptime_mito_write_buffer_bytes{}` | `timeseries` | Write Buffer per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]` | +| Write Rows per Instance | `sum by (instance, pod) (rate(greptime_mito_write_rows_total{}[$__rate_interval]))` | `timeseries` | Ingestion size by row counts. | `prometheus` | `rowsps` | `[{{instance}}]-[{{pod}}]` | +| Flush OPS per Instance | `sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{}[$__rate_interval]))` | `timeseries` | Flush QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{reason}}]` | +| Write Stall per Instance | `sum by(instance, pod) (greptime_mito_write_stall_total{})` | `timeseries` | Write Stall per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]` | +| Read Stage OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{ stage="total"}[$__rate_interval]))` | `timeseries` | Read Stage OPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]` | +| Read Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Read Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | +| Write Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Write Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | +| Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{}[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `prometheus` | `ops` | `[{{ instance }}]-[{{pod}}]` | +| Compaction P99 per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Compaction latency by stage | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` | +| Compaction P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Compaction P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction` | +| WAL write size | `histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))` | `timeseries` | Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-req-size-p95` | +| Cached Bytes per Instance | `greptime_mito_cache_bytes{}` | `timeseries` | Cached Bytes per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | +| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` | +| WAL sync duration seconds | `histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))` | `timeseries` | Raft engine (local disk) log store sync latency, p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` | +| Log Store op duration seconds | `histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))` | `timeseries` | Write-ahead log operations latency at p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99` | +| Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` | # OpenDAL | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{}[$__rate_interval]))` | `timeseries` | QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| Read QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | -| Read P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{operation="read"}[$__rate_interval])))` | `timeseries` | Read P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-{{scheme}}` | -| Write QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="write"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-{{scheme}}` | -| Write P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation="write"}[$__rate_interval])))` | `timeseries` | Write P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | -| List QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="list"}[$__rate_interval]))` | `timeseries` | List QPS per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | -| List P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation="list"}[$__rate_interval])))` | `timeseries` | List P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | -| Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~"read\|write\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| Other Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation!~"read\|write\|list"}[$__rate_interval])))` | `timeseries` | Other Request P99 per Instance. | `s` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{}[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `ops` | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{}[$__rate_interval]))` | `timeseries` | QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Read QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | +| Read P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{operation="read"}[$__rate_interval])))` | `timeseries` | Read P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-{{scheme}}` | +| Write QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="write"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-{{scheme}}` | +| Write P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation="write"}[$__rate_interval])))` | `timeseries` | Write P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | +| List QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="list"}[$__rate_interval]))` | `timeseries` | List QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | +| List P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation="list"}[$__rate_interval])))` | `timeseries` | List P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | +| Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~"read\|write\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Other Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation!~"read\|write\|list"}[$__rate_interval])))` | `timeseries` | Other Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{}[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | # Metasrv | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Region migration datanode | `greptime_meta_region_migration_stat{datanode_type="src"}`
`greptime_meta_region_migration_stat{datanode_type="desc"}` | `state-timeline` | Counter of region migration by source and destination | `none` | `prometheus` | `from-datanode-{{datanode_id}}` | -| Region migration error | `greptime_meta_region_migration_error` | `timeseries` | Counter of region migration error | `none` | `prometheus` | `__auto` | -| Datanode load | `greptime_datanode_load` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `none` | `prometheus` | `__auto` | +| Region migration datanode | `greptime_meta_region_migration_stat{datanode_type="src"}`
`greptime_meta_region_migration_stat{datanode_type="desc"}` | `state-timeline` | Counter of region migration by source and destination | `prometheus` | `none` | `from-datanode-{{datanode_id}}` | +| Region migration error | `greptime_meta_region_migration_error` | `timeseries` | Counter of region migration error | `prometheus` | `none` | `__auto` | +| Datanode load | `greptime_datanode_load` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `none` | `__auto` | # Flownode | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Flow Ingest / Output Rate | `sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))` | `timeseries` | Flow Ingest / Output Rate. | -- | `prometheus` | `[{{pod}}]-[{{instance}}]-[{{direction}}]` | -| Flow Ingest Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`
`histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))` | `timeseries` | Flow Ingest Latency. | -- | `prometheus` | `[{{instance}}]-[{{pod}}]-p95` | -| Flow Operation Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))`
`histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))` | `timeseries` | Flow Operation Latency. | -- | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` | -| Flow Buffer Size per Instance | `greptime_flow_input_buf_size` | `timeseries` | Flow Buffer Size per Instance. | -- | `prometheus` | `[{{instance}}]-[{{pod}]` | -| Flow Processing Error per Instance | `sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))` | `timeseries` | Flow Processing Error per Instance. | -- | `prometheus` | `[{{instance}}]-[{{pod}}]-[{{code}}]` | +| Flow Ingest / Output Rate | `sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))` | `timeseries` | Flow Ingest / Output Rate. | `prometheus` | -- | `[{{pod}}]-[{{instance}}]-[{{direction}}]` | +| Flow Ingest Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`
`histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))` | `timeseries` | Flow Ingest Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-p95` | +| Flow Operation Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))`
`histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))` | `timeseries` | Flow Operation Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` | +| Flow Buffer Size per Instance | `greptime_flow_input_buf_size` | `timeseries` | Flow Buffer Size per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}]` | +| Flow Processing Error per Instance | `sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))` | `timeseries` | Flow Processing Error per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{code}}]` | diff --git a/grafana/scripts/gen-dashboards.sh b/grafana/scripts/gen-dashboards.sh index bfd710d0d5..9488986bf9 100755 --- a/grafana/scripts/gen-dashboards.sh +++ b/grafana/scripts/gen-dashboards.sh @@ -2,7 +2,7 @@ CLUSTER_DASHBOARD_DIR=${1:-grafana/dashboards/cluster} STANDALONE_DASHBOARD_DIR=${2:-grafana/dashboards/standalone} -DAC_IMAGE=ghcr.io/zyy17/dac:20250422-c9435ce +DAC_IMAGE=ghcr.io/zyy17/dac:20250423-522bd35 remove_instance_filters() { # Remove the instance filters for the standalone dashboards. @@ -10,8 +10,15 @@ remove_instance_filters() { } generate_intermediate_dashboards_and_docs() { - docker run -v ${PWD}:/greptimedb --rm ${DAC_IMAGE} -i /greptimedb/$CLUSTER_DASHBOARD_DIR/dashboard.json -o /greptimedb/$CLUSTER_DASHBOARD_DIR/dashboard.yaml -m > $CLUSTER_DASHBOARD_DIR/dashboard.md - docker run -v ${PWD}:/greptimedb --rm ${DAC_IMAGE} -i /greptimedb/$STANDALONE_DASHBOARD_DIR/dashboard.json -o /greptimedb/$STANDALONE_DASHBOARD_DIR/dashboard.yaml -m > $STANDALONE_DASHBOARD_DIR/dashboard.md + docker run -v ${PWD}:/greptimedb --rm ${DAC_IMAGE} \ + -i /greptimedb/$CLUSTER_DASHBOARD_DIR/dashboard.json \ + -o /greptimedb/$CLUSTER_DASHBOARD_DIR/dashboard.yaml \ + -m /greptimedb/$CLUSTER_DASHBOARD_DIR/dashboard.md + + docker run -v ${PWD}:/greptimedb --rm ${DAC_IMAGE} \ + -i /greptimedb/$STANDALONE_DASHBOARD_DIR/dashboard.json \ + -o /greptimedb/$STANDALONE_DASHBOARD_DIR/dashboard.yaml \ + -m /greptimedb/$STANDALONE_DASHBOARD_DIR/dashboard.md } remove_instance_filters From 79ed7bbc44d0b349373d2008773d4c2421b4f051 Mon Sep 17 00:00:00 2001 From: discord9 <55937128+discord9@users.noreply.github.com> Date: Wed, 23 Apr 2025 17:59:09 +0800 Subject: [PATCH 65/82] fix: store flow query ctx on creation (#5963) * fix: store flow schema on creation * chore: update sqlness * refactor: save the entire query context to flow info * chore: sqlness update * chore: rm pub * fix: keep old version compatibility --- .../meta/src/cache/flow/table_flownode.rs | 1 + src/common/meta/src/ddl/create_flow.rs | 1 + src/common/meta/src/error.rs | 11 +- src/common/meta/src/key/flow.rs | 3 + src/common/meta/src/key/flow/flow_info.rs | 11 ++ src/common/meta/src/rpc/ddl.rs | 24 +++- src/flow/src/server.rs | 25 +++- .../common/flow/flow_rebuild.result | 114 ++++++++++++++++++ .../standalone/common/flow/flow_rebuild.sql | 63 ++++++++++ 9 files changed, 243 insertions(+), 10 deletions(-) diff --git a/src/common/meta/src/cache/flow/table_flownode.rs b/src/common/meta/src/cache/flow/table_flownode.rs index c9eb883b76..b285088822 100644 --- a/src/common/meta/src/cache/flow/table_flownode.rs +++ b/src/common/meta/src/cache/flow/table_flownode.rs @@ -187,6 +187,7 @@ mod tests { }, flownode_ids: BTreeMap::from([(0, 1), (1, 2), (2, 3)]), catalog_name: DEFAULT_CATALOG_NAME.to_string(), + query_context: None, flow_name: "my_flow".to_string(), raw_sql: "sql".to_string(), expire_after: Some(300), diff --git a/src/common/meta/src/ddl/create_flow.rs b/src/common/meta/src/ddl/create_flow.rs index 8b1c0354d4..1d751746d3 100644 --- a/src/common/meta/src/ddl/create_flow.rs +++ b/src/common/meta/src/ddl/create_flow.rs @@ -437,6 +437,7 @@ impl From<&CreateFlowData> for (FlowInfoValue, Vec<(FlowPartitionId, FlowRouteVa sink_table_name, flownode_ids, catalog_name, + query_context: Some(value.query_context.clone()), flow_name, raw_sql: sql, expire_after, diff --git a/src/common/meta/src/error.rs b/src/common/meta/src/error.rs index 1bdb3d0857..5242698458 100644 --- a/src/common/meta/src/error.rs +++ b/src/common/meta/src/error.rs @@ -783,6 +783,14 @@ pub enum Error { #[snafu(source)] source: common_procedure::error::Error, }, + + #[snafu(display("Failed to parse timezone"))] + InvalidTimeZone { + #[snafu(implicit)] + location: Location, + #[snafu(source)] + error: common_time::error::Error, + }, } pub type Result = std::result::Result; @@ -853,7 +861,8 @@ impl ErrorExt for Error { | TlsConfig { .. } | InvalidSetDatabaseOption { .. } | InvalidUnsetDatabaseOption { .. } - | InvalidTopicNamePrefix { .. } => StatusCode::InvalidArguments, + | InvalidTopicNamePrefix { .. } + | InvalidTimeZone { .. } => StatusCode::InvalidArguments, FlowNotFound { .. } => StatusCode::FlowNotFound, FlowRouteNotFound { .. } => StatusCode::Unexpected, diff --git a/src/common/meta/src/key/flow.rs b/src/common/meta/src/key/flow.rs index 6af910b2fc..bc66c08a9d 100644 --- a/src/common/meta/src/key/flow.rs +++ b/src/common/meta/src/key/flow.rs @@ -452,6 +452,7 @@ mod tests { }; FlowInfoValue { catalog_name: catalog_name.to_string(), + query_context: None, flow_name: flow_name.to_string(), source_table_ids, sink_table_name, @@ -625,6 +626,7 @@ mod tests { }; let flow_value = FlowInfoValue { catalog_name: "greptime".to_string(), + query_context: None, flow_name: "flow".to_string(), source_table_ids: vec![1024, 1025, 1026], sink_table_name: another_sink_table_name, @@ -864,6 +866,7 @@ mod tests { }; let flow_value = FlowInfoValue { catalog_name: "greptime".to_string(), + query_context: None, flow_name: "flow".to_string(), source_table_ids: vec![1024, 1025, 1026], sink_table_name: another_sink_table_name, diff --git a/src/common/meta/src/key/flow/flow_info.rs b/src/common/meta/src/key/flow/flow_info.rs index eeb37da81d..1ed3f1e6f4 100644 --- a/src/common/meta/src/key/flow/flow_info.rs +++ b/src/common/meta/src/key/flow/flow_info.rs @@ -121,6 +121,13 @@ pub struct FlowInfoValue { pub(crate) flownode_ids: BTreeMap, /// The catalog name. pub(crate) catalog_name: String, + /// The query context used when create flow. + /// Although flow doesn't belong to any schema, this query_context is needed to remember + /// the query context when `create_flow` is executed + /// for recovering flow using the same sql&query_context after db restart. + /// if none, should use default query context + #[serde(default)] + pub(crate) query_context: Option, /// The flow name. pub(crate) flow_name: String, /// The raw sql. @@ -155,6 +162,10 @@ impl FlowInfoValue { &self.catalog_name } + pub fn query_context(&self) -> &Option { + &self.query_context + } + pub fn flow_name(&self) -> &String { &self.flow_name } diff --git a/src/common/meta/src/rpc/ddl.rs b/src/common/meta/src/rpc/ddl.rs index ae7794d9bd..2797c6bee0 100644 --- a/src/common/meta/src/rpc/ddl.rs +++ b/src/common/meta/src/rpc/ddl.rs @@ -35,17 +35,20 @@ use api::v1::{ }; use base64::engine::general_purpose; use base64::Engine as _; -use common_time::DatabaseTimeToLive; +use common_time::{DatabaseTimeToLive, Timezone}; use prost::Message; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DefaultOnNull}; -use session::context::QueryContextRef; +use session::context::{QueryContextBuilder, QueryContextRef}; use snafu::{OptionExt, ResultExt}; use table::metadata::{RawTableInfo, TableId}; use table::table_name::TableName; use table::table_reference::TableReference; -use crate::error::{self, InvalidSetDatabaseOptionSnafu, InvalidUnsetDatabaseOptionSnafu, Result}; +use crate::error::{ + self, InvalidSetDatabaseOptionSnafu, InvalidTimeZoneSnafu, InvalidUnsetDatabaseOptionSnafu, + Result, +}; use crate::key::FlowId; /// DDL tasks @@ -1202,7 +1205,7 @@ impl From for PbDropFlowTask { } } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct QueryContext { current_catalog: String, current_schema: String, @@ -1223,6 +1226,19 @@ impl From for QueryContext { } } +impl TryFrom for session::context::QueryContext { + type Error = error::Error; + fn try_from(value: QueryContext) -> std::result::Result { + Ok(QueryContextBuilder::default() + .current_catalog(value.current_catalog) + .current_schema(value.current_schema) + .timezone(Timezone::from_tz_string(&value.timezone).context(InvalidTimeZoneSnafu)?) + .extensions(value.extensions) + .channel((value.channel as u32).into()) + .build()) + } +} + impl From for PbQueryContext { fn from( QueryContext { diff --git a/src/flow/src/server.rs b/src/flow/src/server.rs index 53712ffb67..99f7a7c089 100644 --- a/src/flow/src/server.rs +++ b/src/flow/src/server.rs @@ -401,6 +401,7 @@ impl FlownodeBuilder { let cnt = to_be_recovered.len(); // TODO(discord9): recover in parallel + info!("Recovering {} flows: {:?}", cnt, to_be_recovered); for flow_id in to_be_recovered { let info = self .flow_metadata_manager @@ -416,6 +417,7 @@ impl FlownodeBuilder { info.sink_table_name().schema_name.clone(), info.sink_table_name().table_name.clone(), ]; + let args = CreateFlowArgs { flow_id: flow_id as _, sink_table_name, @@ -429,11 +431,24 @@ impl FlownodeBuilder { comment: Some(info.comment().clone()), sql: info.raw_sql().clone(), flow_options: info.options().clone(), - query_ctx: Some( - QueryContextBuilder::default() - .current_catalog(info.catalog_name().clone()) - .build(), - ), + query_ctx: info + .query_context() + .clone() + .map(|ctx| { + ctx.try_into() + .map_err(BoxedError::new) + .context(ExternalSnafu) + }) + .transpose()? + // or use default QueryContext with catalog_name from info + // to keep compatibility with old version + .or_else(|| { + Some( + QueryContextBuilder::default() + .current_catalog(info.catalog_name().to_string()) + .build(), + ) + }), }; manager .create_flow_inner(args) diff --git a/tests/cases/standalone/common/flow/flow_rebuild.result b/tests/cases/standalone/common/flow/flow_rebuild.result index 67fd43a032..0cc5f1ce9c 100644 --- a/tests/cases/standalone/common/flow/flow_rebuild.result +++ b/tests/cases/standalone/common/flow/flow_rebuild.result @@ -576,3 +576,117 @@ DROP TABLE out_basic; Affected Rows: 0 +-- check if different schema is working as expected +CREATE DATABASE jsdp_log; + +Affected Rows: 1 + +USE jsdp_log; + +Affected Rows: 0 + +CREATE TABLE IF NOT EXISTS `api_log` ( + `time` TIMESTAMP(9) NOT NULL, + `key` STRING NULL SKIPPING INDEX WITH(granularity = '1024', type = 'BLOOM'), + `status_code` TINYINT NULL, + `method` STRING NULL, + `path` STRING NULL, + `raw_query` STRING NULL, + `user_agent` STRING NULL, + `client_ip` STRING NULL, + `duration` INT NULL, + `count` INT NULL, + TIME INDEX (`time`) +) ENGINE=mito WITH( + append_mode = 'true' +); + +Affected Rows: 0 + +CREATE TABLE IF NOT EXISTS `api_stats` ( + `time` TIMESTAMP(0) NOT NULL, + `key` STRING NULL, + `qpm` BIGINT NULL, + `rpm` BIGINT NULL, + `update_at` TIMESTAMP(3) NULL, + TIME INDEX (`time`), + PRIMARY KEY (`key`) +) ENGINE=mito WITH( + append_mode = 'false', + merge_mode = 'last_row' +); + +Affected Rows: 0 + +CREATE FLOW IF NOT EXISTS api_stats_flow +SINK TO api_stats EXPIRE AFTER '10 minute'::INTERVAL AS +SELECT date_trunc('minute', `time`::TimestampSecond) AS `time1`, `key`, count(*), sum(`count`) +FROM api_log +GROUP BY `time1`, `key`; + +Affected Rows: 0 + +INSERT INTO `api_log` (`time`, `key`, `status_code`, `method`, `path`, `raw_query`, `user_agent`, `client_ip`, `duration`, `count`) VALUES (now(), '1', 0, 'GET', '/lightning/v1/query', 'key=1&since=600', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', '1', 21, 1); + +Affected Rows: 1 + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('api_stats_flow'); + ++------------------------------------+ +| ADMIN FLUSH_FLOW('api_stats_flow') | ++------------------------------------+ +| FLOW_FLUSHED | ++------------------------------------+ + +SELECT key FROM api_stats; + ++-----+ +| key | ++-----+ +| 1 | ++-----+ + +-- SQLNESS ARG restart=true +INSERT INTO `api_log` (`time`, `key`, `status_code`, `method`, `path`, `raw_query`, `user_agent`, `client_ip`, `duration`, `count`) VALUES (now(), '2', 0, 'GET', '/lightning/v1/query', 'key=1&since=600', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', '1', 21, 1); + +Affected Rows: 1 + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('api_stats_flow'); + ++------------------------------------+ +| ADMIN FLUSH_FLOW('api_stats_flow') | ++------------------------------------+ +| FLOW_FLUSHED | ++------------------------------------+ + +SELECT key FROM api_stats; + ++-----+ +| key | ++-----+ +| 1 | +| 2 | ++-----+ + +DROP FLOW api_stats_flow; + +Affected Rows: 0 + +DROP TABLE api_log; + +Affected Rows: 0 + +DROP TABLE api_stats; + +Affected Rows: 0 + +USE public; + +Affected Rows: 0 + +DROP DATABASE jsdp_log; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/flow/flow_rebuild.sql b/tests/cases/standalone/common/flow/flow_rebuild.sql index 288d6f1f03..ad07ea4a40 100644 --- a/tests/cases/standalone/common/flow/flow_rebuild.sql +++ b/tests/cases/standalone/common/flow/flow_rebuild.sql @@ -317,3 +317,66 @@ DROP FLOW test_wildcard_basic; DROP TABLE input_basic; DROP TABLE out_basic; + +-- check if different schema is working as expected + +CREATE DATABASE jsdp_log; +USE jsdp_log; + +CREATE TABLE IF NOT EXISTS `api_log` ( + `time` TIMESTAMP(9) NOT NULL, + `key` STRING NULL SKIPPING INDEX WITH(granularity = '1024', type = 'BLOOM'), + `status_code` TINYINT NULL, + `method` STRING NULL, + `path` STRING NULL, + `raw_query` STRING NULL, + `user_agent` STRING NULL, + `client_ip` STRING NULL, + `duration` INT NULL, + `count` INT NULL, + TIME INDEX (`time`) +) ENGINE=mito WITH( + append_mode = 'true' +); + +CREATE TABLE IF NOT EXISTS `api_stats` ( + `time` TIMESTAMP(0) NOT NULL, + `key` STRING NULL, + `qpm` BIGINT NULL, + `rpm` BIGINT NULL, + `update_at` TIMESTAMP(3) NULL, + TIME INDEX (`time`), + PRIMARY KEY (`key`) +) ENGINE=mito WITH( + append_mode = 'false', + merge_mode = 'last_row' +); + +CREATE FLOW IF NOT EXISTS api_stats_flow +SINK TO api_stats EXPIRE AFTER '10 minute'::INTERVAL AS +SELECT date_trunc('minute', `time`::TimestampSecond) AS `time1`, `key`, count(*), sum(`count`) +FROM api_log +GROUP BY `time1`, `key`; + +INSERT INTO `api_log` (`time`, `key`, `status_code`, `method`, `path`, `raw_query`, `user_agent`, `client_ip`, `duration`, `count`) VALUES (now(), '1', 0, 'GET', '/lightning/v1/query', 'key=1&since=600', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', '1', 21, 1); + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('api_stats_flow'); + +SELECT key FROM api_stats; + +-- SQLNESS ARG restart=true +INSERT INTO `api_log` (`time`, `key`, `status_code`, `method`, `path`, `raw_query`, `user_agent`, `client_ip`, `duration`, `count`) VALUES (now(), '2', 0, 'GET', '/lightning/v1/query', 'key=1&since=600', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', '1', 21, 1); + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('api_stats_flow'); + +SELECT key FROM api_stats; + +DROP FLOW api_stats_flow; + +DROP TABLE api_log; +DROP TABLE api_stats; + +USE public; +DROP DATABASE jsdp_log; From 71db79c8d64bd4a3be1d163c83baaba567f09753 Mon Sep 17 00:00:00 2001 From: LFC <990479+MichaelScofield@users.noreply.github.com> Date: Wed, 23 Apr 2025 18:48:46 +0800 Subject: [PATCH 66/82] feat: node excluder (#5964) * feat: node excluder * fix ci * fix ci --- src/cmd/src/metasrv.rs | 6 +++--- src/meta-srv/src/lib.rs | 1 + src/meta-srv/src/node_excluder.rs | 26 ++++++++++++++++++++++++++ typos.toml | 1 + 4 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 src/meta-srv/src/node_excluder.rs diff --git a/src/cmd/src/metasrv.rs b/src/cmd/src/metasrv.rs index dc455dcf65..da017e71cd 100644 --- a/src/cmd/src/metasrv.rs +++ b/src/cmd/src/metasrv.rs @@ -132,7 +132,7 @@ impl SubCommand { } #[derive(Debug, Default, Parser)] -struct StartCommand { +pub struct StartCommand { /// The address to bind the gRPC server. #[clap(long, alias = "bind-addr")] rpc_bind_addr: Option, @@ -172,7 +172,7 @@ struct StartCommand { } impl StartCommand { - fn load_options(&self, global_options: &GlobalOptions) -> Result { + pub fn load_options(&self, global_options: &GlobalOptions) -> Result { let mut opts = MetasrvOptions::load_layered_options( self.config_file.as_deref(), self.env_prefix.as_ref(), @@ -261,7 +261,7 @@ impl StartCommand { Ok(()) } - async fn build(&self, opts: MetasrvOptions) -> Result { + pub async fn build(&self, opts: MetasrvOptions) -> Result { common_runtime::init_global_runtimes(&opts.runtime); let guard = common_telemetry::init_global_logging( diff --git a/src/meta-srv/src/lib.rs b/src/meta-srv/src/lib.rs index 4b61eeeae3..ebd3b7b54f 100644 --- a/src/meta-srv/src/lib.rs +++ b/src/meta-srv/src/lib.rs @@ -31,6 +31,7 @@ pub mod metasrv; pub mod metrics; #[cfg(feature = "mock")] pub mod mocks; +pub mod node_excluder; pub mod procedure; pub mod pubsub; pub mod region; diff --git a/src/meta-srv/src/node_excluder.rs b/src/meta-srv/src/node_excluder.rs new file mode 100644 index 0000000000..f9e892f092 --- /dev/null +++ b/src/meta-srv/src/node_excluder.rs @@ -0,0 +1,26 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use common_meta::DatanodeId; + +pub type NodeExcluderRef = Arc; + +/// [NodeExcluder] is used to help decide whether some nodes should be excluded (out of consideration) +/// in certain situations. For example, in some node selectors. +pub trait NodeExcluder: Send + Sync { + /// Returns the excluded datanode ids. + fn excluded_datanode_ids(&self) -> &Vec; +} diff --git a/typos.toml b/typos.toml index da6570d224..57b7279d0f 100644 --- a/typos.toml +++ b/typos.toml @@ -6,6 +6,7 @@ ot = "ot" typ = "typ" typs = "typs" unqualifed = "unqualifed" +excluder = "excluder" [files] extend-exclude = [ From 45a05fb08cf83307816c9fb5419ea706162b8e03 Mon Sep 17 00:00:00 2001 From: Yingwen Date: Wed, 23 Apr 2025 21:31:29 +0800 Subject: [PATCH 67/82] docs: fix some units and adds the opendal errors panel (#5962) * docs: fixes units in the dashboard * docs: add opendal errors panel * docs: opendal traffic use decbytes * docs: update readme --------- Co-authored-by: zyy17 --- grafana/README.md | 10 +- grafana/dashboards/cluster/dashboard.json | 4009 +++++++++--------- grafana/dashboards/cluster/dashboard.md | 5 +- grafana/dashboards/cluster/dashboard.yaml | 12 +- grafana/dashboards/standalone/dashboard.json | 4009 +++++++++--------- grafana/dashboards/standalone/dashboard.md | 5 +- grafana/dashboards/standalone/dashboard.yaml | 12 +- 7 files changed, 4153 insertions(+), 3909 deletions(-) diff --git a/grafana/README.md b/grafana/README.md index 31cfcccd06..db86581e0b 100644 --- a/grafana/README.md +++ b/grafana/README.md @@ -9,12 +9,16 @@ This repository maintains the Grafana dashboards for GreptimeDB. It has two type As the rapid development of GreptimeDB, the metrics may be changed, and please feel free to submit your feedback and/or contribution to this dashboard 🤗 -**NOTE**: If you want to modify the dashboards, you only need to modify the `cluster/dashboard.json` and run the `make dashboards` command to generate the `standalone/dashboard.json` and other related files. +**NOTE**: -To maintain the dashboards, we use the [`dac`](https://github.com/zyy17/dac) tool to generate the intermediate dashboards and markdown documents: +- The Grafana version should be greater than 9.0. + +- If you want to modify the dashboards, you only need to modify the `cluster/dashboard.json` and run the `make dashboards` command to generate the `standalone/dashboard.json` and other related files. + +To maintain the dashboards easily, we use the [`dac`](https://github.com/zyy17/dac) tool to generate the intermediate dashboards and markdown documents: - `cluster/dashboard.yaml`: The intermediate dashboard for the GreptimeDB cluster. -- `standalone/dashboard.yaml`: The intermediatedashboard for the standalone GreptimeDB instance. +- `standalone/dashboard.yaml`: The intermediate dashboard for the standalone GreptimeDB instance. ## Data Sources diff --git a/grafana/dashboards/cluster/dashboard.json b/grafana/dashboards/cluster/dashboard.json index ef5490c888..91633ae6ab 100644 --- a/grafana/dashboards/cluster/dashboard.json +++ b/grafana/dashboards/cluster/dashboard.json @@ -25,9 +25,8 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": 48, + "id": null, "links": [], - "liveNow": false, "panels": [ { "collapsed": false, @@ -963,6 +962,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1009,7 +1009,7 @@ "h": 6, "w": 24, "x": 0, - "y": 386 + "y": 10 }, "id": 193, "options": { @@ -1022,10 +1022,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -1062,6 +1064,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1108,7 +1111,7 @@ "h": 6, "w": 24, "x": 0, - "y": 392 + "y": 784 }, "id": 277, "options": { @@ -1121,10 +1124,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -1234,7 +1239,7 @@ "h": 6, "w": 24, "x": 0, - "y": 407 + "y": 1589 }, "id": 255, "options": { @@ -1299,7 +1304,7 @@ "type": "row" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, @@ -1307,816 +1312,843 @@ "y": 11 }, "id": 274, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current memory usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 256, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum(process_resident_memory_bytes{instance=~\"$datanode\"}) by (instance, pod)", - "instant": false, - "legendFormat": "[{{instance}}]-[{{ pod }}]", - "range": true, - "refId": "A" - } - ], - "title": "Datanode Memory per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current cpu usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 262, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$datanode\"}[$__rate_interval]) * 1000) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]", - "range": true, - "refId": "A" - } - ], - "title": "Datanode CPU Usage per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current memory usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 266, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum(process_resident_memory_bytes{instance=~\"$frontend\"}) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]", - "range": true, - "refId": "A" - } - ], - "title": "Frontend Memory per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current cpu usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 268, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$frontend\"}[$__rate_interval]) * 1000) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]-cpu", - "range": true, - "refId": "A" - } - ], - "title": "Frontend CPU Usage per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current memory usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 21 - }, - "id": 269, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum(process_resident_memory_bytes{instance=~\"$metasrv\"}) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]-resident", - "range": true, - "refId": "A" - } - ], - "title": "Metasrv Memory per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current cpu usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 21 - }, - "id": 271, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$metasrv\"}[$__rate_interval]) * 1000) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]", - "range": true, - "refId": "A" - } - ], - "title": "Metasrv CPU Usage per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current memory usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 31 - }, - "id": 272, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum(process_resident_memory_bytes{instance=~\"$flownode\"}) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]", - "range": true, - "refId": "A" - } - ], - "title": "Flownode Memory per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current cpu usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 31 - }, - "id": 273, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$flownode\"}[$__rate_interval]) * 1000) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]", - "range": true, - "refId": "A" - } - ], - "title": "Flownode CPU Usage per Instance", - "type": "timeseries" - } - ], + "panels": [], "title": "Resources", "type": "row" }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 256, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{instance=~\"$datanode\"}) by (instance, pod)", + "instant": false, + "legendFormat": "[{{instance}}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Datanode Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 262, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$datanode\"}[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Datanode CPU Usage per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 266, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{instance=~\"$frontend\"}) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Frontend Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 268, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$frontend\"}[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]-cpu", + "range": true, + "refId": "A" + } + ], + "title": "Frontend CPU Usage per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 269, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{instance=~\"$metasrv\"}) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]-resident", + "range": true, + "refId": "A" + } + ], + "title": "Metasrv Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 271, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$metasrv\"}[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Metasrv CPU Usage per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 272, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{instance=~\"$flownode\"}) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Flownode Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 273, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$flownode\"}[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Flownode CPU Usage per Instance", + "type": "timeseries" + }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 52 }, "id": 280, "panels": [ @@ -2209,7 +2241,7 @@ "h": 8, "w": 12, "x": 0, - "y": 325 + "y": 1507 }, "id": 281, "options": { @@ -2307,7 +2339,7 @@ "h": 8, "w": 12, "x": 12, - "y": 325 + "y": 1507 }, "id": 282, "options": { @@ -2405,7 +2437,7 @@ "h": 8, "w": 12, "x": 0, - "y": 333 + "y": 1515 }, "id": 283, "options": { @@ -2503,7 +2535,7 @@ "h": 8, "w": 12, "x": 12, - "y": 333 + "y": 1515 }, "id": 284, "options": { @@ -2599,7 +2631,7 @@ "h": 8, "w": 12, "x": 0, - "y": 341 + "y": 1523 }, "id": 285, "options": { @@ -2697,7 +2729,7 @@ "h": 8, "w": 12, "x": 12, - "y": 341 + "y": 1523 }, "id": 286, "options": { @@ -2796,7 +2828,7 @@ "h": 8, "w": 12, "x": 0, - "y": 349 + "y": 1531 }, "id": 287, "options": { @@ -2894,7 +2926,7 @@ "h": 8, "w": 12, "x": 12, - "y": 349 + "y": 1531 }, "id": 288, "options": { @@ -2936,7 +2968,7 @@ "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 53 }, "id": 289, "panels": [ @@ -3102,7 +3134,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 792 }, "id": 290, "options": { @@ -3200,7 +3232,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 792 }, "id": 291, "options": { @@ -3244,7 +3276,7 @@ "h": 1, "w": 24, "x": 0, - "y": 14 + "y": 54 }, "id": 293, "panels": [ @@ -3266,6 +3298,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3312,7 +3345,7 @@ "h": 8, "w": 12, "x": 0, - "y": 7 + "y": 212 }, "id": 294, "options": { @@ -3325,10 +3358,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -3364,6 +3399,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", @@ -3410,7 +3446,7 @@ "h": 8, "w": 12, "x": 12, - "y": 7 + "y": 212 }, "id": 295, "options": { @@ -3423,10 +3459,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -3462,6 +3500,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3508,7 +3547,7 @@ "h": 8, "w": 12, "x": 0, - "y": 15 + "y": 265 }, "id": 296, "options": { @@ -3521,10 +3560,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -3560,6 +3601,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3606,7 +3648,7 @@ "h": 8, "w": 12, "x": 12, - "y": 15 + "y": 265 }, "id": 297, "options": { @@ -3619,10 +3661,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -3658,6 +3702,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3704,7 +3749,7 @@ "h": 8, "w": 12, "x": 0, - "y": 23 + "y": 273 }, "id": 298, "options": { @@ -3717,10 +3762,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -3756,6 +3803,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3793,8 +3841,7 @@ "value": 80 } ] - }, - "unit": "decbytes" + } }, "overrides": [] }, @@ -3802,7 +3849,7 @@ "h": 8, "w": 12, "x": 12, - "y": 23 + "y": 273 }, "id": 299, "options": { @@ -3815,10 +3862,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -3854,6 +3903,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3900,7 +3950,7 @@ "h": 8, "w": 12, "x": 0, - "y": 31 + "y": 281 }, "id": 300, "options": { @@ -3913,10 +3963,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -3952,6 +4004,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", @@ -3998,7 +4051,7 @@ "h": 8, "w": 12, "x": 12, - "y": 31 + "y": 281 }, "id": 301, "options": { @@ -4011,10 +4064,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -4050,6 +4105,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", @@ -4096,7 +4152,7 @@ "h": 8, "w": 12, "x": 0, - "y": 39 + "y": 289 }, "id": 302, "options": { @@ -4111,10 +4167,12 @@ "sortDesc": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -4150,6 +4208,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -4196,7 +4255,7 @@ "h": 8, "w": 12, "x": 12, - "y": 39 + "y": 289 }, "id": 303, "options": { @@ -4209,10 +4268,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -4248,6 +4309,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -4294,7 +4356,7 @@ "h": 8, "w": 12, "x": 0, - "y": 47 + "y": 297 }, "id": 304, "options": { @@ -4307,10 +4369,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -4346,6 +4410,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", @@ -4392,7 +4457,7 @@ "h": 8, "w": 12, "x": 12, - "y": 47 + "y": 297 }, "id": 305, "options": { @@ -4405,10 +4470,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -4444,6 +4511,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -4490,7 +4558,7 @@ "h": 8, "w": 12, "x": 0, - "y": 55 + "y": 305 }, "id": 306, "options": { @@ -4503,10 +4571,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -4568,6 +4638,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", @@ -4614,7 +4685,7 @@ "h": 8, "w": 12, "x": 12, - "y": 55 + "y": 305 }, "id": 307, "options": { @@ -4625,10 +4696,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -4664,6 +4737,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", @@ -4710,7 +4784,7 @@ "h": 8, "w": 12, "x": 0, - "y": 63 + "y": 313 }, "id": 308, "options": { @@ -4723,6 +4797,7 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } @@ -4763,6 +4838,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", @@ -4809,7 +4885,7 @@ "h": 8, "w": 12, "x": 12, - "y": 63 + "y": 313 }, "id": 310, "options": { @@ -4822,6 +4898,7 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } @@ -4862,6 +4939,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", @@ -4908,7 +4986,7 @@ "h": 8, "w": 12, "x": 0, - "y": 71 + "y": 321 }, "id": 311, "options": { @@ -4921,6 +4999,7 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } @@ -4961,6 +5040,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", @@ -5007,7 +5087,7 @@ "h": 8, "w": 12, "x": 12, - "y": 71 + "y": 321 }, "id": 312, "options": { @@ -5020,6 +5100,7 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } @@ -5047,1062 +5128,1141 @@ "type": "row" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 15 + "y": 55 }, "id": 313, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "QPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[10.244.2.103:4000]-[mycluster-datanode-0]-[fs]-[delete]" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 314, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", - "range": true, - "refId": "A" - } - ], - "title": "QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Read QPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 315, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=\"read\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", - "range": true, - "refId": "A" - } - ], - "title": "Read QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Read P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 316, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\",operation=\"read\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-{{scheme}}", - "range": true, - "refId": "A" - } - ], - "title": "Read P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write QPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 25 - }, - "id": 317, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=\"write\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-{{scheme}}", - "range": true, - "refId": "A" - } - ], - "title": "Write QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 25 - }, - "id": 318, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation=\"write\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", - "range": true, - "refId": "A" - } - ], - "title": "Write P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "List QPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 319, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=\"list\"}[$__rate_interval]))", - "instant": false, - "interval": "", - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", - "range": true, - "refId": "A" - } - ], - "title": "List QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "List P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 320, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation=\"list\"}[$__rate_interval])))", - "instant": false, - "interval": "", - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", - "range": true, - "refId": "A" - } - ], - "title": "List P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Other Requests per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 39 - }, - "id": 321, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\",operation!~\"read|write|list|stat\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", - "range": true, - "refId": "A" - } - ], - "title": "Other Requests per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Other Request P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 39 - }, - "id": 322, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation!~\"read|write|list\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", - "range": true, - "refId": "A" - } - ], - "title": "Other Request P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Total traffic as in bytes by instance and operation", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[mycluster-datanode-0]-[fs]-[Writer::write]" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 46 - }, - "id": 323, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~\"$datanode\"}[$__rate_interval]))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Opendal traffic", - "type": "timeseries" - } - ], + "panels": [], "title": "OpenDAL", "type": "row" }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 314, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + } + ], + "title": "QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Read QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 66 + }, + "id": 315, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=\"read\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", + "range": true, + "refId": "A" + } + ], + "title": "Read QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Read P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 66 + }, + "id": 316, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\",operation=\"read\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-{{scheme}}", + "range": true, + "refId": "A" + } + ], + "title": "Read P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 73 + }, + "id": 317, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=\"write\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-{{scheme}}", + "range": true, + "refId": "A" + } + ], + "title": "Write QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 73 + }, + "id": 318, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation=\"write\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "List QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 80 + }, + "id": 319, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=\"list\"}[$__rate_interval]))", + "instant": false, + "interval": "", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", + "range": true, + "refId": "A" + } + ], + "title": "List QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "List P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 80 + }, + "id": 320, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation=\"list\"}[$__rate_interval])))", + "instant": false, + "interval": "", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", + "range": true, + "refId": "A" + } + ], + "title": "List P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Other Requests per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 87 + }, + "id": 321, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\",operation!~\"read|write|list|stat\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + } + ], + "title": "Other Requests per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Other Request P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 87 + }, + "id": 322, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation!~\"read|write|list\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + } + ], + "title": "Other Request P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total traffic as in bytes by instance and operation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 94 + }, + "id": 323, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~\"$datanode\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Opendal traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "OpenDAL error counts per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 94 + }, + "id": 334, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{instance=~\"$datanode\", error!=\"NotFound\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]", + "range": true, + "refId": "A" + } + ], + "title": "OpenDAL errors per Instance", + "type": "timeseries" + }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 16 + "y": 101 }, "id": 324, "panels": [ @@ -6357,7 +6517,7 @@ "h": 8, "w": 12, "x": 0, - "y": 17 + "y": 1199 }, "id": 327, "options": { @@ -6399,7 +6559,7 @@ "h": 1, "w": 24, "x": 0, - "y": 17 + "y": 102 }, "id": 328, "panels": [ @@ -6451,8 +6611,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6467,7 +6626,7 @@ "h": 8, "w": 12, "x": 0, - "y": 18 + "y": 1200 }, "id": 329, "options": { @@ -6547,8 +6706,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6563,7 +6721,7 @@ "h": 8, "w": 12, "x": 12, - "y": 18 + "y": 1200 }, "id": 330, "options": { @@ -6656,8 +6814,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6672,7 +6829,7 @@ "h": 8, "w": 9, "x": 0, - "y": 26 + "y": 1208 }, "id": 331, "options": { @@ -6765,8 +6922,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6781,7 +6937,7 @@ "h": 8, "w": 9, "x": 9, - "y": 26 + "y": 1208 }, "id": 332, "options": { @@ -6861,8 +7017,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6877,7 +7032,7 @@ "h": 8, "w": 6, "x": 18, - "y": 26 + "y": 1208 }, "id": 333, "options": { @@ -6915,53 +7070,32 @@ } ], "refresh": "10s", - "schemaVersion": 39, + "schemaVersion": 40, "tags": [], "templating": { "list": [ { - "current": { - "selected": false, - "text": "metrics", - "value": "P177A7EA3611FE6B1" - }, - "hide": 0, + "current": {}, "includeAll": false, - "multi": false, "name": "metrics", "options": [], "query": "prometheus", - "queryValue": "", "refresh": 1, "regex": "", - "skipUrlSync": false, "type": "datasource" }, { - "current": { - "selected": false, - "text": "information_schema", - "value": "P0CE5E4D2C4819379" - }, - "hide": 0, + "current": {}, "includeAll": false, - "multi": false, "name": "information_schema", "options": [], "query": "mysql", - "queryValue": "", "refresh": 1, "regex": "", - "skipUrlSync": false, "type": "datasource" }, { - "allValue": "", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, + "current": {}, "datasource": { "type": "prometheus", "uid": "${metrics}" @@ -6979,17 +7113,10 @@ }, "refresh": 1, "regex": "", - "skipUrlSync": false, - "sort": 0, "type": "query" }, { - "allValue": "", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, + "current": {}, "datasource": { "type": "prometheus", "uid": "${metrics}" @@ -7007,17 +7134,10 @@ }, "refresh": 1, "regex": "", - "skipUrlSync": false, - "sort": 0, "type": "query" }, { - "allValue": "", - "current": { - "selected": false, - "text": "10.244.1.79:4000", - "value": "10.244.1.79:4000" - }, + "current": {}, "datasource": { "type": "prometheus", "uid": "${metrics}" @@ -7035,17 +7155,10 @@ }, "refresh": 1, "regex": "", - "skipUrlSync": false, - "sort": 0, "type": "query" }, { - "allValue": "", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, + "current": {}, "datasource": { "type": "prometheus", "uid": "${metrics}" @@ -7063,8 +7176,6 @@ }, "refresh": 1, "regex": "", - "skipUrlSync": false, - "sort": 0, "type": "query" } ] @@ -7077,6 +7188,6 @@ "timezone": "", "title": "GreptimeDB", "uid": "dejf3k5e7g2kgb", - "version": 1, + "version": 2, "weekStart": "" } diff --git a/grafana/dashboards/cluster/dashboard.md b/grafana/dashboards/cluster/dashboard.md index feef106365..2de3016bbf 100644 --- a/grafana/dashboards/cluster/dashboard.md +++ b/grafana/dashboards/cluster/dashboard.md @@ -54,7 +54,7 @@ | Write Buffer per Instance | `greptime_mito_write_buffer_bytes{instance=~"$datanode"}` | `timeseries` | Write Buffer per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]` | | Write Rows per Instance | `sum by (instance, pod) (rate(greptime_mito_write_rows_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Ingestion size by row counts. | `prometheus` | `rowsps` | `[{{instance}}]-[{{pod}}]` | | Flush OPS per Instance | `sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Flush QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{reason}}]` | -| Write Stall per Instance | `sum by(instance, pod) (greptime_mito_write_stall_total{instance=~"$datanode"})` | `timeseries` | Write Stall per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]` | +| Write Stall per Instance | `sum by(instance, pod) (greptime_mito_write_stall_total{instance=~"$datanode"})` | `timeseries` | Write Stall per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` | | Read Stage OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{instance=~"$datanode", stage="total"}[$__rate_interval]))` | `timeseries` | Read Stage OPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]` | | Read Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Read Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | | Write Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Write Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | @@ -79,7 +79,8 @@ | List P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="list"}[$__rate_interval])))` | `timeseries` | List P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | | Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode",operation!~"read\|write\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | | Other Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation!~"read\|write\|list"}[$__rate_interval])))` | `timeseries` | Other Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| OpenDAL errors per Instance | `sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{instance=~"$datanode", error!="NotFound"}[$__rate_interval]))` | `timeseries` | OpenDAL error counts per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]` | # Metasrv | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | diff --git a/grafana/dashboards/cluster/dashboard.yaml b/grafana/dashboards/cluster/dashboard.yaml index 622cb2d548..e67ed3e960 100644 --- a/grafana/dashboards/cluster/dashboard.yaml +++ b/grafana/dashboards/cluster/dashboard.yaml @@ -426,7 +426,6 @@ groups: - title: Write Stall per Instance type: timeseries description: Write Stall per Instance. - unit: decbytes queries: - expr: sum by(instance, pod) (greptime_mito_write_stall_total{instance=~"$datanode"}) datasource: @@ -658,13 +657,22 @@ groups: - title: Opendal traffic type: timeseries description: Total traffic as in bytes by instance and operation - unit: ops + unit: decbytes queries: - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~"$datanode"}[$__rate_interval])) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - title: OpenDAL errors per Instance + type: timeseries + description: OpenDAL error counts per Instance. + queries: + - expr: sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{instance=~"$datanode", error!="NotFound"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]' - title: Metasrv panels: - title: Region migration datanode diff --git a/grafana/dashboards/standalone/dashboard.json b/grafana/dashboards/standalone/dashboard.json index fcf7f0d04e..d799cd7e0f 100644 --- a/grafana/dashboards/standalone/dashboard.json +++ b/grafana/dashboards/standalone/dashboard.json @@ -25,9 +25,8 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": 48, + "id": null, "links": [], - "liveNow": false, "panels": [ { "collapsed": false, @@ -963,6 +962,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1009,7 +1009,7 @@ "h": 6, "w": 24, "x": 0, - "y": 386 + "y": 10 }, "id": 193, "options": { @@ -1022,10 +1022,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -1062,6 +1064,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1108,7 +1111,7 @@ "h": 6, "w": 24, "x": 0, - "y": 392 + "y": 784 }, "id": 277, "options": { @@ -1121,10 +1124,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -1234,7 +1239,7 @@ "h": 6, "w": 24, "x": 0, - "y": 407 + "y": 1589 }, "id": 255, "options": { @@ -1299,7 +1304,7 @@ "type": "row" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, @@ -1307,816 +1312,843 @@ "y": 11 }, "id": 274, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current memory usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 256, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum(process_resident_memory_bytes{}) by (instance, pod)", - "instant": false, - "legendFormat": "[{{instance}}]-[{{ pod }}]", - "range": true, - "refId": "A" - } - ], - "title": "Datanode Memory per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current cpu usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 262, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]", - "range": true, - "refId": "A" - } - ], - "title": "Datanode CPU Usage per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current memory usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 266, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum(process_resident_memory_bytes{}) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]", - "range": true, - "refId": "A" - } - ], - "title": "Frontend Memory per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current cpu usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 268, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]-cpu", - "range": true, - "refId": "A" - } - ], - "title": "Frontend CPU Usage per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current memory usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 21 - }, - "id": 269, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum(process_resident_memory_bytes{}) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]-resident", - "range": true, - "refId": "A" - } - ], - "title": "Metasrv Memory per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current cpu usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 21 - }, - "id": 271, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]", - "range": true, - "refId": "A" - } - ], - "title": "Metasrv CPU Usage per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current memory usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 31 - }, - "id": 272, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum(process_resident_memory_bytes{}) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]", - "range": true, - "refId": "A" - } - ], - "title": "Flownode Memory per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current cpu usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 31 - }, - "id": 273, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]", - "range": true, - "refId": "A" - } - ], - "title": "Flownode CPU Usage per Instance", - "type": "timeseries" - } - ], + "panels": [], "title": "Resources", "type": "row" }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 256, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{}) by (instance, pod)", + "instant": false, + "legendFormat": "[{{instance}}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Datanode Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 262, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Datanode CPU Usage per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 266, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{}) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Frontend Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 268, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]-cpu", + "range": true, + "refId": "A" + } + ], + "title": "Frontend CPU Usage per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 269, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{}) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]-resident", + "range": true, + "refId": "A" + } + ], + "title": "Metasrv Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 271, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Metasrv CPU Usage per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 272, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{}) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Flownode Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 273, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + } + ], + "title": "Flownode CPU Usage per Instance", + "type": "timeseries" + }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 52 }, "id": 280, "panels": [ @@ -2209,7 +2241,7 @@ "h": 8, "w": 12, "x": 0, - "y": 325 + "y": 1507 }, "id": 281, "options": { @@ -2307,7 +2339,7 @@ "h": 8, "w": 12, "x": 12, - "y": 325 + "y": 1507 }, "id": 282, "options": { @@ -2405,7 +2437,7 @@ "h": 8, "w": 12, "x": 0, - "y": 333 + "y": 1515 }, "id": 283, "options": { @@ -2503,7 +2535,7 @@ "h": 8, "w": 12, "x": 12, - "y": 333 + "y": 1515 }, "id": 284, "options": { @@ -2599,7 +2631,7 @@ "h": 8, "w": 12, "x": 0, - "y": 341 + "y": 1523 }, "id": 285, "options": { @@ -2697,7 +2729,7 @@ "h": 8, "w": 12, "x": 12, - "y": 341 + "y": 1523 }, "id": 286, "options": { @@ -2796,7 +2828,7 @@ "h": 8, "w": 12, "x": 0, - "y": 349 + "y": 1531 }, "id": 287, "options": { @@ -2894,7 +2926,7 @@ "h": 8, "w": 12, "x": 12, - "y": 349 + "y": 1531 }, "id": 288, "options": { @@ -2936,7 +2968,7 @@ "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 53 }, "id": 289, "panels": [ @@ -3102,7 +3134,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 792 }, "id": 290, "options": { @@ -3200,7 +3232,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 792 }, "id": 291, "options": { @@ -3244,7 +3276,7 @@ "h": 1, "w": 24, "x": 0, - "y": 14 + "y": 54 }, "id": 293, "panels": [ @@ -3266,6 +3298,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3312,7 +3345,7 @@ "h": 8, "w": 12, "x": 0, - "y": 7 + "y": 212 }, "id": 294, "options": { @@ -3325,10 +3358,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -3364,6 +3399,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", @@ -3410,7 +3446,7 @@ "h": 8, "w": 12, "x": 12, - "y": 7 + "y": 212 }, "id": 295, "options": { @@ -3423,10 +3459,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -3462,6 +3500,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3508,7 +3547,7 @@ "h": 8, "w": 12, "x": 0, - "y": 15 + "y": 265 }, "id": 296, "options": { @@ -3521,10 +3560,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -3560,6 +3601,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3606,7 +3648,7 @@ "h": 8, "w": 12, "x": 12, - "y": 15 + "y": 265 }, "id": 297, "options": { @@ -3619,10 +3661,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -3658,6 +3702,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3704,7 +3749,7 @@ "h": 8, "w": 12, "x": 0, - "y": 23 + "y": 273 }, "id": 298, "options": { @@ -3717,10 +3762,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -3756,6 +3803,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3793,8 +3841,7 @@ "value": 80 } ] - }, - "unit": "decbytes" + } }, "overrides": [] }, @@ -3802,7 +3849,7 @@ "h": 8, "w": 12, "x": 12, - "y": 23 + "y": 273 }, "id": 299, "options": { @@ -3815,10 +3862,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -3854,6 +3903,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3900,7 +3950,7 @@ "h": 8, "w": 12, "x": 0, - "y": 31 + "y": 281 }, "id": 300, "options": { @@ -3913,10 +3963,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -3952,6 +4004,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", @@ -3998,7 +4051,7 @@ "h": 8, "w": 12, "x": 12, - "y": 31 + "y": 281 }, "id": 301, "options": { @@ -4011,10 +4064,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -4050,6 +4105,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", @@ -4096,7 +4152,7 @@ "h": 8, "w": 12, "x": 0, - "y": 39 + "y": 289 }, "id": 302, "options": { @@ -4111,10 +4167,12 @@ "sortDesc": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -4150,6 +4208,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -4196,7 +4255,7 @@ "h": 8, "w": 12, "x": 12, - "y": 39 + "y": 289 }, "id": 303, "options": { @@ -4209,10 +4268,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -4248,6 +4309,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -4294,7 +4356,7 @@ "h": 8, "w": 12, "x": 0, - "y": 47 + "y": 297 }, "id": 304, "options": { @@ -4307,10 +4369,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -4346,6 +4410,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", @@ -4392,7 +4457,7 @@ "h": 8, "w": 12, "x": 12, - "y": 47 + "y": 297 }, "id": 305, "options": { @@ -4405,10 +4470,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -4444,6 +4511,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -4490,7 +4558,7 @@ "h": 8, "w": 12, "x": 0, - "y": 55 + "y": 305 }, "id": 306, "options": { @@ -4503,10 +4571,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -4568,6 +4638,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", @@ -4614,7 +4685,7 @@ "h": 8, "w": 12, "x": 12, - "y": 55 + "y": 305 }, "id": 307, "options": { @@ -4625,10 +4696,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "desc" } }, + "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -4664,6 +4737,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", @@ -4710,7 +4784,7 @@ "h": 8, "w": 12, "x": 0, - "y": 63 + "y": 313 }, "id": 308, "options": { @@ -4723,6 +4797,7 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } @@ -4763,6 +4838,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", @@ -4809,7 +4885,7 @@ "h": 8, "w": 12, "x": 12, - "y": 63 + "y": 313 }, "id": 310, "options": { @@ -4822,6 +4898,7 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } @@ -4862,6 +4939,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", @@ -4908,7 +4986,7 @@ "h": 8, "w": 12, "x": 0, - "y": 71 + "y": 321 }, "id": 311, "options": { @@ -4921,6 +4999,7 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } @@ -4961,6 +5040,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", @@ -5007,7 +5087,7 @@ "h": 8, "w": 12, "x": 12, - "y": 71 + "y": 321 }, "id": 312, "options": { @@ -5020,6 +5100,7 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } @@ -5047,1062 +5128,1141 @@ "type": "row" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 15 + "y": 55 }, "id": 313, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "QPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[10.244.2.103:4000]-[mycluster-datanode-0]-[fs]-[delete]" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 314, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", - "range": true, - "refId": "A" - } - ], - "title": "QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Read QPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 315, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation=\"read\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", - "range": true, - "refId": "A" - } - ], - "title": "Read QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Read P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 316, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{operation=\"read\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-{{scheme}}", - "range": true, - "refId": "A" - } - ], - "title": "Read P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write QPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 25 - }, - "id": 317, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation=\"write\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-{{scheme}}", - "range": true, - "refId": "A" - } - ], - "title": "Write QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 25 - }, - "id": 318, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation=\"write\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", - "range": true, - "refId": "A" - } - ], - "title": "Write P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "List QPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 319, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation=\"list\"}[$__rate_interval]))", - "instant": false, - "interval": "", - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", - "range": true, - "refId": "A" - } - ], - "title": "List QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "List P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 320, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation=\"list\"}[$__rate_interval])))", - "instant": false, - "interval": "", - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", - "range": true, - "refId": "A" - } - ], - "title": "List P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Other Requests per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 39 - }, - "id": 321, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~\"read|write|list|stat\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", - "range": true, - "refId": "A" - } - ], - "title": "Other Requests per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Other Request P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 39 - }, - "id": 322, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation!~\"read|write|list\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", - "range": true, - "refId": "A" - } - ], - "title": "Other Request P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Total traffic as in bytes by instance and operation", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[mycluster-datanode-0]-[fs]-[Writer::write]" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 46 - }, - "id": 323, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{}[$__rate_interval]))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Opendal traffic", - "type": "timeseries" - } - ], + "panels": [], "title": "OpenDAL", "type": "row" }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 314, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + } + ], + "title": "QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Read QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 66 + }, + "id": 315, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation=\"read\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", + "range": true, + "refId": "A" + } + ], + "title": "Read QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Read P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 66 + }, + "id": 316, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{operation=\"read\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-{{scheme}}", + "range": true, + "refId": "A" + } + ], + "title": "Read P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 73 + }, + "id": 317, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation=\"write\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-{{scheme}}", + "range": true, + "refId": "A" + } + ], + "title": "Write QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 73 + }, + "id": 318, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation=\"write\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "List QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 80 + }, + "id": 319, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation=\"list\"}[$__rate_interval]))", + "instant": false, + "interval": "", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", + "range": true, + "refId": "A" + } + ], + "title": "List QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "List P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 80 + }, + "id": 320, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation=\"list\"}[$__rate_interval])))", + "instant": false, + "interval": "", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", + "range": true, + "refId": "A" + } + ], + "title": "List P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Other Requests per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 87 + }, + "id": 321, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~\"read|write|list|stat\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + } + ], + "title": "Other Requests per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Other Request P99 per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 87 + }, + "id": 322, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation!~\"read|write|list\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + } + ], + "title": "Other Request P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total traffic as in bytes by instance and operation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 94 + }, + "id": 323, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Opendal traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "OpenDAL error counts per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 94 + }, + "id": 334, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{ error!=\"NotFound\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]", + "range": true, + "refId": "A" + } + ], + "title": "OpenDAL errors per Instance", + "type": "timeseries" + }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 16 + "y": 101 }, "id": 324, "panels": [ @@ -6357,7 +6517,7 @@ "h": 8, "w": 12, "x": 0, - "y": 17 + "y": 1199 }, "id": 327, "options": { @@ -6399,7 +6559,7 @@ "h": 1, "w": 24, "x": 0, - "y": 17 + "y": 102 }, "id": 328, "panels": [ @@ -6451,8 +6611,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6467,7 +6626,7 @@ "h": 8, "w": 12, "x": 0, - "y": 18 + "y": 1200 }, "id": 329, "options": { @@ -6547,8 +6706,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6563,7 +6721,7 @@ "h": 8, "w": 12, "x": 12, - "y": 18 + "y": 1200 }, "id": 330, "options": { @@ -6656,8 +6814,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6672,7 +6829,7 @@ "h": 8, "w": 9, "x": 0, - "y": 26 + "y": 1208 }, "id": 331, "options": { @@ -6765,8 +6922,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6781,7 +6937,7 @@ "h": 8, "w": 9, "x": 9, - "y": 26 + "y": 1208 }, "id": 332, "options": { @@ -6861,8 +7017,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6877,7 +7032,7 @@ "h": 8, "w": 6, "x": 18, - "y": 26 + "y": 1208 }, "id": 333, "options": { @@ -6915,53 +7070,32 @@ } ], "refresh": "10s", - "schemaVersion": 39, + "schemaVersion": 40, "tags": [], "templating": { "list": [ { - "current": { - "selected": false, - "text": "metrics", - "value": "P177A7EA3611FE6B1" - }, - "hide": 0, + "current": {}, "includeAll": false, - "multi": false, "name": "metrics", "options": [], "query": "prometheus", - "queryValue": "", "refresh": 1, "regex": "", - "skipUrlSync": false, "type": "datasource" }, { - "current": { - "selected": false, - "text": "information_schema", - "value": "P0CE5E4D2C4819379" - }, - "hide": 0, + "current": {}, "includeAll": false, - "multi": false, "name": "information_schema", "options": [], "query": "mysql", - "queryValue": "", "refresh": 1, "regex": "", - "skipUrlSync": false, "type": "datasource" }, { - "allValue": "", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, + "current": {}, "datasource": { "type": "prometheus", "uid": "${metrics}" @@ -6979,17 +7113,10 @@ }, "refresh": 1, "regex": "", - "skipUrlSync": false, - "sort": 0, "type": "query" }, { - "allValue": "", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, + "current": {}, "datasource": { "type": "prometheus", "uid": "${metrics}" @@ -7007,17 +7134,10 @@ }, "refresh": 1, "regex": "", - "skipUrlSync": false, - "sort": 0, "type": "query" }, { - "allValue": "", - "current": { - "selected": false, - "text": "10.244.1.79:4000", - "value": "10.244.1.79:4000" - }, + "current": {}, "datasource": { "type": "prometheus", "uid": "${metrics}" @@ -7035,17 +7155,10 @@ }, "refresh": 1, "regex": "", - "skipUrlSync": false, - "sort": 0, "type": "query" }, { - "allValue": "", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, + "current": {}, "datasource": { "type": "prometheus", "uid": "${metrics}" @@ -7063,8 +7176,6 @@ }, "refresh": 1, "regex": "", - "skipUrlSync": false, - "sort": 0, "type": "query" } ] @@ -7077,6 +7188,6 @@ "timezone": "", "title": "GreptimeDB", "uid": "dejf3k5e7g2kgb", - "version": 1, + "version": 2, "weekStart": "" } diff --git a/grafana/dashboards/standalone/dashboard.md b/grafana/dashboards/standalone/dashboard.md index fbd7158c9e..673faf3357 100644 --- a/grafana/dashboards/standalone/dashboard.md +++ b/grafana/dashboards/standalone/dashboard.md @@ -54,7 +54,7 @@ | Write Buffer per Instance | `greptime_mito_write_buffer_bytes{}` | `timeseries` | Write Buffer per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]` | | Write Rows per Instance | `sum by (instance, pod) (rate(greptime_mito_write_rows_total{}[$__rate_interval]))` | `timeseries` | Ingestion size by row counts. | `prometheus` | `rowsps` | `[{{instance}}]-[{{pod}}]` | | Flush OPS per Instance | `sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{}[$__rate_interval]))` | `timeseries` | Flush QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{reason}}]` | -| Write Stall per Instance | `sum by(instance, pod) (greptime_mito_write_stall_total{})` | `timeseries` | Write Stall per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]` | +| Write Stall per Instance | `sum by(instance, pod) (greptime_mito_write_stall_total{})` | `timeseries` | Write Stall per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` | | Read Stage OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{ stage="total"}[$__rate_interval]))` | `timeseries` | Read Stage OPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]` | | Read Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Read Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | | Write Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Write Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | @@ -79,7 +79,8 @@ | List P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation="list"}[$__rate_interval])))` | `timeseries` | List P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | | Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~"read\|write\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | | Other Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation!~"read\|write\|list"}[$__rate_interval])))` | `timeseries` | Other Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{}[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{}[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| OpenDAL errors per Instance | `sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{ error!="NotFound"}[$__rate_interval]))` | `timeseries` | OpenDAL error counts per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]` | # Metasrv | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | diff --git a/grafana/dashboards/standalone/dashboard.yaml b/grafana/dashboards/standalone/dashboard.yaml index 197478b5b1..828ac1dffc 100644 --- a/grafana/dashboards/standalone/dashboard.yaml +++ b/grafana/dashboards/standalone/dashboard.yaml @@ -426,7 +426,6 @@ groups: - title: Write Stall per Instance type: timeseries description: Write Stall per Instance. - unit: decbytes queries: - expr: sum by(instance, pod) (greptime_mito_write_stall_total{}) datasource: @@ -658,13 +657,22 @@ groups: - title: Opendal traffic type: timeseries description: Total traffic as in bytes by instance and operation - unit: ops + unit: decbytes queries: - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{}[$__rate_interval])) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - title: OpenDAL errors per Instance + type: timeseries + description: OpenDAL error counts per Instance. + queries: + - expr: sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{ error!="NotFound"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]' - title: Metasrv panels: - title: Region migration datanode From a0900f5b90165536edb7fa141ed20da6f58125eb Mon Sep 17 00:00:00 2001 From: discord9 <55937128+discord9@users.noreply.github.com> Date: Wed, 23 Apr 2025 23:12:16 +0800 Subject: [PATCH 68/82] feat(flow): use batching mode&fix sqlness (#5903) * feat: use flow batching engine broken: try using logical plan fix: use dummy catalog for logical plan fix: insert plan exec&sqlness grpc addr feat: use frontend instance in flownode in standalone feat: flow type in metasrv&fix: flush flow out of sync& column name alias tests: sqlness update tests: sqlness flow rebuild udpate chore: per review refactor: keep chnl mgr refactor: use catalog mgr for get table tests: use valid sql fix: add more check refactor: put flow type determine to frontend * chore: update proto * chore: update proto to main branch * fix: add locks for create/drop flow&docs: update docs * feat: flush_flow flush all ranges now * test: add align time window test * docs: explain `nodeid` use in check task * refactor: AddAutoColumnRewriter check for Projection * refactor: per review * fix: query without time window also clean dirty time window * chore: better logging * chore: add comments per review * refactor: per review * chore: per review * chore: per review rename args * refactor: per review partially * chore: update docs * chore: use better error variant * chore: better error variant * refactor: rename FlowWorkerManager to FlowStreamingEngine * rename again * refactor: per review * chore: rebase after #5963 merged * refactor: rename all flow_worker_manager occurs * docs: rm resolved TODO --- Cargo.lock | 3 +- Cargo.toml | 2 +- src/api/src/helper.rs | 1 + src/catalog/src/table_source.rs | 2 +- src/cmd/src/flownode.rs | 6 +- src/cmd/src/standalone.rs | 47 +- src/common/meta/src/ddl/create_flow.rs | 24 +- src/common/meta/src/ddl/tests/create_flow.rs | 2 +- src/common/meta/src/error.rs | 8 + src/common/query/src/logical_plan.rs | 77 ++- src/flow/src/adapter.rs | 22 +- src/flow/src/adapter/flownode_impl.rs | 453 ++++++++++++++++-- src/flow/src/adapter/refill.rs | 12 +- src/flow/src/adapter/stat.rs | 4 +- src/flow/src/adapter/util.rs | 4 +- src/flow/src/batching_mode/engine.rs | 66 ++- src/flow/src/batching_mode/frontend_client.rs | 239 +++++++-- src/flow/src/batching_mode/state.rs | 124 ++++- src/flow/src/batching_mode/task.rs | 331 +++++++------ src/flow/src/batching_mode/time_window.rs | 13 +- src/flow/src/batching_mode/utils.rs | 181 +++++-- src/flow/src/engine.rs | 2 + src/flow/src/error.rs | 36 +- src/flow/src/lib.rs | 4 +- src/flow/src/server.rs | 88 +++- src/frontend/Cargo.toml | 1 + src/frontend/src/error.rs | 14 +- src/frontend/src/instance.rs | 2 +- src/frontend/src/instance/grpc.rs | 124 ++++- src/operator/src/statement/ddl.rs | 81 +++- src/servers/tests/mod.rs | 2 +- tests-integration/src/standalone.rs | 22 +- .../common/flow/flow_advance_ttl.result | 107 ++++- .../common/flow/flow_advance_ttl.sql | 32 ++ .../common/flow/flow_auto_sink_table.result | 31 +- .../common/flow/flow_auto_sink_table.sql | 5 +- .../standalone/common/flow/flow_basic.result | 90 ++-- .../standalone/common/flow/flow_basic.sql | 31 +- .../standalone/common/flow/flow_blog.result | 6 +- .../standalone/common/flow/flow_blog.sql | 6 +- .../common/flow/flow_call_df_func.result | 56 +-- .../common/flow/flow_call_df_func.sql | 12 +- .../standalone/common/flow/flow_flush.result | 62 +++ .../standalone/common/flow/flow_flush.sql | 37 ++ .../standalone/common/flow/flow_null.result | 4 + .../standalone/common/flow/flow_null.sql | 4 + .../common/flow/flow_rebuild.result | 88 +++- .../standalone/common/flow/flow_rebuild.sql | 39 +- .../common/flow/flow_user_guide.result | 21 +- .../common/flow/flow_user_guide.sql | 9 +- .../standalone/common/flow/flow_view.result | 3 +- .../standalone/common/flow/flow_view.sql | 1 + tests/conf/frontend-test.toml.template | 4 +- tests/runner/src/server_mode.rs | 3 + 54 files changed, 2123 insertions(+), 525 deletions(-) create mode 100644 tests/cases/standalone/common/flow/flow_flush.result create mode 100644 tests/cases/standalone/common/flow/flow_flush.sql diff --git a/Cargo.lock b/Cargo.lock index 3b74af8971..839ceafddc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4505,6 +4505,7 @@ dependencies = [ "arc-swap", "async-trait", "auth", + "bytes", "cache", "catalog", "client", @@ -4943,7 +4944,7 @@ dependencies = [ [[package]] name = "greptime-proto" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=b6d9cffd43c4e6358805a798f17e03e232994b82#b6d9cffd43c4e6358805a798f17e03e232994b82" +source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=e82b0158cd38d4021edb4e4c0ae77f999051e62f#e82b0158cd38d4021edb4e4c0ae77f999051e62f" dependencies = [ "prost 0.13.5", "serde", diff --git a/Cargo.toml b/Cargo.toml index 8aef7b6d7b..92dba96d00 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -129,7 +129,7 @@ etcd-client = "0.14" fst = "0.4.7" futures = "0.3" futures-util = "0.3" -greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "b6d9cffd43c4e6358805a798f17e03e232994b82" } +greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "e82b0158cd38d4021edb4e4c0ae77f999051e62f" } hex = "0.4" http = "1" humantime = "2.1" diff --git a/src/api/src/helper.rs b/src/api/src/helper.rs index 6a398b05d3..5c4a02f335 100644 --- a/src/api/src/helper.rs +++ b/src/api/src/helper.rs @@ -514,6 +514,7 @@ fn query_request_type(request: &QueryRequest) -> &'static str { Some(Query::Sql(_)) => "query.sql", Some(Query::LogicalPlan(_)) => "query.logical_plan", Some(Query::PromRangeQuery(_)) => "query.prom_range", + Some(Query::InsertIntoPlan(_)) => "query.insert_into_plan", None => "query.empty", } } diff --git a/src/catalog/src/table_source.rs b/src/catalog/src/table_source.rs index 3cb3b5087d..caf6778214 100644 --- a/src/catalog/src/table_source.rs +++ b/src/catalog/src/table_source.rs @@ -27,7 +27,7 @@ use session::context::QueryContextRef; use snafu::{ensure, OptionExt, ResultExt}; use table::metadata::TableType; use table::table::adapter::DfTableProviderAdapter; -mod dummy_catalog; +pub mod dummy_catalog; use dummy_catalog::DummyCatalogList; use table::TableRef; diff --git a/src/cmd/src/flownode.rs b/src/cmd/src/flownode.rs index a7b530e558..3fc8249349 100644 --- a/src/cmd/src/flownode.rs +++ b/src/cmd/src/flownode.rs @@ -345,7 +345,7 @@ impl StartCommand { let client = Arc::new(NodeClients::new(channel_config)); let invoker = FrontendInvoker::build_from( - flownode.flow_worker_manager().clone(), + flownode.flow_engine().streaming_engine(), catalog_manager.clone(), cached_meta_backend.clone(), layered_cache_registry.clone(), @@ -355,7 +355,9 @@ impl StartCommand { .await .context(StartFlownodeSnafu)?; flownode - .flow_worker_manager() + .flow_engine() + .streaming_engine() + // TODO(discord9): refactor and avoid circular reference .set_frontend_invoker(invoker) .await; diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index 320a2849ed..5877329698 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -56,8 +56,8 @@ use datanode::datanode::{Datanode, DatanodeBuilder}; use datanode::region_server::RegionServer; use file_engine::config::EngineConfig as FileEngineConfig; use flow::{ - FlowConfig, FlowWorkerManager, FlownodeBuilder, FlownodeInstance, FlownodeOptions, - FrontendClient, FrontendInvoker, + FlowConfig, FlownodeBuilder, FlownodeInstance, FlownodeOptions, FrontendClient, + FrontendInvoker, GrpcQueryHandlerWithBoxedError, StreamingEngine, }; use frontend::frontend::{Frontend, FrontendOptions}; use frontend::instance::builder::FrontendBuilder; @@ -524,17 +524,17 @@ impl StartCommand { ..Default::default() }; - // TODO(discord9): for standalone not use grpc, but just somehow get a handler to frontend grpc client without + // for standalone not use grpc, but get a handler to frontend grpc client without // actually make a connection - let fe_server_addr = fe_opts.grpc.bind_addr.clone(); - let frontend_client = FrontendClient::from_static_grpc_addr(fe_server_addr); + let (frontend_client, frontend_instance_handler) = + FrontendClient::from_empty_grpc_handler(); let flow_builder = FlownodeBuilder::new( flownode_options, plugins.clone(), table_metadata_manager.clone(), catalog_manager.clone(), flow_metadata_manager.clone(), - Arc::new(frontend_client), + Arc::new(frontend_client.clone()), ); let flownode = flow_builder .build() @@ -544,15 +544,15 @@ impl StartCommand { // set the ref to query for the local flow state { - let flow_worker_manager = flownode.flow_worker_manager(); + let flow_streaming_engine = flownode.flow_engine().streaming_engine(); information_extension - .set_flow_worker_manager(flow_worker_manager.clone()) + .set_flow_streaming_engine(flow_streaming_engine) .await; } let node_manager = Arc::new(StandaloneDatanodeManager { region_server: datanode.region_server(), - flow_server: flownode.flow_worker_manager(), + flow_server: flownode.flow_engine(), }); let table_id_sequence = Arc::new( @@ -606,10 +606,19 @@ impl StartCommand { .context(error::StartFrontendSnafu)?; let fe_instance = Arc::new(fe_instance); - let flow_worker_manager = flownode.flow_worker_manager(); + // set the frontend client for flownode + let grpc_handler = fe_instance.clone() as Arc; + let weak_grpc_handler = Arc::downgrade(&grpc_handler); + frontend_instance_handler + .lock() + .unwrap() + .replace(weak_grpc_handler); + + // set the frontend invoker for flownode + let flow_streaming_engine = flownode.flow_engine().streaming_engine(); // flow server need to be able to use frontend to write insert requests back let invoker = FrontendInvoker::build_from( - flow_worker_manager.clone(), + flow_streaming_engine.clone(), catalog_manager.clone(), kv_backend.clone(), layered_cache_registry.clone(), @@ -618,7 +627,7 @@ impl StartCommand { ) .await .context(error::StartFlownodeSnafu)?; - flow_worker_manager.set_frontend_invoker(invoker).await; + flow_streaming_engine.set_frontend_invoker(invoker).await; let export_metrics_task = ExportMetricsTask::try_new(&opts.export_metrics, Some(&plugins)) .context(error::ServersSnafu)?; @@ -694,7 +703,7 @@ pub struct StandaloneInformationExtension { region_server: RegionServer, procedure_manager: ProcedureManagerRef, start_time_ms: u64, - flow_worker_manager: RwLock>>, + flow_streaming_engine: RwLock>>, } impl StandaloneInformationExtension { @@ -703,14 +712,14 @@ impl StandaloneInformationExtension { region_server, procedure_manager, start_time_ms: common_time::util::current_time_millis() as u64, - flow_worker_manager: RwLock::new(None), + flow_streaming_engine: RwLock::new(None), } } - /// Set the flow worker manager for the standalone instance. - pub async fn set_flow_worker_manager(&self, flow_worker_manager: Arc) { - let mut guard = self.flow_worker_manager.write().await; - *guard = Some(flow_worker_manager); + /// Set the flow streaming engine for the standalone instance. + pub async fn set_flow_streaming_engine(&self, flow_streaming_engine: Arc) { + let mut guard = self.flow_streaming_engine.write().await; + *guard = Some(flow_streaming_engine); } } @@ -789,7 +798,7 @@ impl InformationExtension for StandaloneInformationExtension { async fn flow_stats(&self) -> std::result::Result, Self::Error> { Ok(Some( - self.flow_worker_manager + self.flow_streaming_engine .read() .await .as_ref() diff --git a/src/common/meta/src/ddl/create_flow.rs b/src/common/meta/src/ddl/create_flow.rs index 1d751746d3..278a3a6c9e 100644 --- a/src/common/meta/src/ddl/create_flow.rs +++ b/src/common/meta/src/ddl/create_flow.rs @@ -38,7 +38,7 @@ use table::metadata::TableId; use crate::cache_invalidator::Context; use crate::ddl::utils::{add_peer_context_if_needed, handle_retry_error}; use crate::ddl::DdlContext; -use crate::error::{self, Result}; +use crate::error::{self, Result, UnexpectedSnafu}; use crate::instruction::{CacheIdent, CreateFlow}; use crate::key::flow::flow_info::FlowInfoValue; use crate::key::flow::flow_route::FlowRouteValue; @@ -171,7 +171,7 @@ impl CreateFlowProcedure { } self.data.state = CreateFlowState::CreateFlows; // determine flow type - self.data.flow_type = Some(determine_flow_type(&self.data.task)); + self.data.flow_type = Some(get_flow_type_from_options(&self.data.task)?); Ok(Status::executing(true)) } @@ -196,8 +196,8 @@ impl CreateFlowProcedure { }); } info!( - "Creating flow({:?}) on flownodes with peers={:?}", - self.data.flow_id, self.data.peers + "Creating flow({:?}, type={:?}) on flownodes with peers={:?}", + self.data.flow_id, self.data.flow_type, self.data.peers ); join_all(create_flow) .await @@ -306,8 +306,20 @@ impl Procedure for CreateFlowProcedure { } } -pub fn determine_flow_type(_flow_task: &CreateFlowTask) -> FlowType { - FlowType::Batching +pub fn get_flow_type_from_options(flow_task: &CreateFlowTask) -> Result { + let flow_type = flow_task + .flow_options + .get(FlowType::FLOW_TYPE_KEY) + .map(|s| s.as_str()); + match flow_type { + Some(FlowType::BATCHING) => Ok(FlowType::Batching), + Some(FlowType::STREAMING) => Ok(FlowType::Streaming), + Some(unknown) => UnexpectedSnafu { + err_msg: format!("Unknown flow type: {}", unknown), + } + .fail(), + None => Ok(FlowType::Batching), + } } /// The state of [CreateFlowProcedure]. diff --git a/src/common/meta/src/ddl/tests/create_flow.rs b/src/common/meta/src/ddl/tests/create_flow.rs index 4c9f86fe09..3b24e86400 100644 --- a/src/common/meta/src/ddl/tests/create_flow.rs +++ b/src/common/meta/src/ddl/tests/create_flow.rs @@ -46,7 +46,7 @@ pub(crate) fn test_create_flow_task( create_if_not_exists, expire_after: Some(300), comment: "".to_string(), - sql: "raw_sql".to_string(), + sql: "select 1".to_string(), flow_options: Default::default(), } } diff --git a/src/common/meta/src/error.rs b/src/common/meta/src/error.rs index 5242698458..1bc85a898d 100644 --- a/src/common/meta/src/error.rs +++ b/src/common/meta/src/error.rs @@ -401,6 +401,13 @@ pub enum Error { location: Location, }, + #[snafu(display("Invalid flow request body: {:?}", body))] + InvalidFlowRequestBody { + body: Box>, + #[snafu(implicit)] + location: Location, + }, + #[snafu(display("Failed to get kv cache, err: {}", err_msg))] GetKvCache { err_msg: String }, @@ -863,6 +870,7 @@ impl ErrorExt for Error { | InvalidUnsetDatabaseOption { .. } | InvalidTopicNamePrefix { .. } | InvalidTimeZone { .. } => StatusCode::InvalidArguments, + InvalidFlowRequestBody { .. } => StatusCode::InvalidArguments, FlowNotFound { .. } => StatusCode::FlowNotFound, FlowRouteNotFound { .. } => StatusCode::Unexpected, diff --git a/src/common/query/src/logical_plan.rs b/src/common/query/src/logical_plan.rs index 974a30a15a..418e7a52ae 100644 --- a/src/common/query/src/logical_plan.rs +++ b/src/common/query/src/logical_plan.rs @@ -18,16 +18,19 @@ mod udaf; use std::sync::Arc; +use api::v1::TableName; use datafusion::catalog::CatalogProviderList; use datafusion::error::Result as DatafusionResult; use datafusion::logical_expr::{LogicalPlan, LogicalPlanBuilder}; -use datafusion_common::Column; -use datafusion_expr::col; +use datafusion_common::{Column, TableReference}; +use datafusion_expr::dml::InsertOp; +use datafusion_expr::{col, DmlStatement, WriteOp}; pub use expr::{build_filter_from_timestamp, build_same_type_ts_filter}; +use snafu::ResultExt; pub use self::accumulator::{Accumulator, AggregateFunctionCreator, AggregateFunctionCreatorRef}; pub use self::udaf::AggregateFunction; -use crate::error::Result; +use crate::error::{GeneralDataFusionSnafu, Result}; use crate::logical_plan::accumulator::*; use crate::signature::{Signature, Volatility}; @@ -79,6 +82,74 @@ pub fn rename_logical_plan_columns( LogicalPlanBuilder::from(plan).project(projection)?.build() } +/// Convert a insert into logical plan to an (table_name, logical_plan) +/// where table_name is the name of the table to insert into. +/// logical_plan is the plan to be executed. +/// +/// if input logical plan is not `insert into table_name `, return None +/// +/// Returned TableName will use provided catalog and schema if not specified in the logical plan, +/// if table scan in logical plan have full table name, will **NOT** override it. +pub fn breakup_insert_plan( + plan: &LogicalPlan, + default_catalog: &str, + default_schema: &str, +) -> Option<(TableName, Arc)> { + if let LogicalPlan::Dml(dml) = plan { + if dml.op != WriteOp::Insert(InsertOp::Append) { + return None; + } + let table_name = &dml.table_name; + let table_name = match table_name { + TableReference::Bare { table } => TableName { + catalog_name: default_catalog.to_string(), + schema_name: default_schema.to_string(), + table_name: table.to_string(), + }, + TableReference::Partial { schema, table } => TableName { + catalog_name: default_catalog.to_string(), + schema_name: schema.to_string(), + table_name: table.to_string(), + }, + TableReference::Full { + catalog, + schema, + table, + } => TableName { + catalog_name: catalog.to_string(), + schema_name: schema.to_string(), + table_name: table.to_string(), + }, + }; + let logical_plan = dml.input.clone(); + Some((table_name, logical_plan)) + } else { + None + } +} + +/// create a `insert into table_name ` logical plan +pub fn add_insert_to_logical_plan( + table_name: TableName, + table_schema: datafusion_common::DFSchemaRef, + input: LogicalPlan, +) -> Result { + let table_name = TableReference::Full { + catalog: table_name.catalog_name.into(), + schema: table_name.schema_name.into(), + table: table_name.table_name.into(), + }; + + let plan = LogicalPlan::Dml(DmlStatement::new( + table_name, + table_schema, + WriteOp::Insert(InsertOp::Append), + Arc::new(input), + )); + let plan = plan.recompute_schema().context(GeneralDataFusionSnafu)?; + Ok(plan) +} + /// The datafusion `[LogicalPlan]` decoder. #[async_trait::async_trait] pub trait SubstraitPlanDecoder { diff --git a/src/flow/src/adapter.rs b/src/flow/src/adapter.rs index d0d88d978a..a613e3f83a 100644 --- a/src/flow/src/adapter.rs +++ b/src/flow/src/adapter.rs @@ -58,7 +58,7 @@ use crate::metrics::{METRIC_FLOW_INSERT_ELAPSED, METRIC_FLOW_ROWS, METRIC_FLOW_R use crate::repr::{self, DiffRow, RelationDesc, Row, BATCH_SIZE}; use crate::{CreateFlowArgs, FlowId, TableName}; -mod flownode_impl; +pub(crate) mod flownode_impl; mod parse_expr; pub(crate) mod refill; mod stat; @@ -135,12 +135,13 @@ impl Configurable for FlownodeOptions { } /// Arc-ed FlowNodeManager, cheaper to clone -pub type FlowWorkerManagerRef = Arc; +pub type FlowStreamingEngineRef = Arc; /// FlowNodeManager manages the state of all tasks in the flow node, which should be run on the same thread /// /// The choice of timestamp is just using current system timestamp for now -pub struct FlowWorkerManager { +/// +pub struct StreamingEngine { /// The handler to the worker that will run the dataflow /// which is `!Send` so a handle is used pub worker_handles: Vec, @@ -158,7 +159,8 @@ pub struct FlowWorkerManager { flow_err_collectors: RwLock>, src_send_buf_lens: RwLock>>, tick_manager: FlowTickManager, - node_id: Option, + /// This node id is only available in distributed mode, on standalone mode this is guaranteed to be `None` + pub node_id: Option, /// Lock for flushing, will be `read` by `handle_inserts` and `write` by `flush_flow` /// /// So that a series of event like `inserts -> flush` can be handled correctly @@ -168,7 +170,7 @@ pub struct FlowWorkerManager { } /// Building FlownodeManager -impl FlowWorkerManager { +impl StreamingEngine { /// set frontend invoker pub async fn set_frontend_invoker(&self, frontend: FrontendInvoker) { *self.frontend_invoker.write().await = Some(frontend); @@ -187,7 +189,7 @@ impl FlowWorkerManager { let node_context = FlownodeContext::new(Box::new(srv_map.clone()) as _); let tick_manager = FlowTickManager::new(); let worker_handles = Vec::new(); - FlowWorkerManager { + StreamingEngine { worker_handles, worker_selector: Mutex::new(0), query_engine, @@ -263,7 +265,7 @@ pub fn batches_to_rows_req(batches: Vec) -> Result, Erro } /// This impl block contains methods to send writeback requests to frontend -impl FlowWorkerManager { +impl StreamingEngine { /// Return the number of requests it made pub async fn send_writeback_requests(&self) -> Result { let all_reqs = self.generate_writeback_request().await?; @@ -534,7 +536,7 @@ impl FlowWorkerManager { } /// Flow Runtime related methods -impl FlowWorkerManager { +impl StreamingEngine { /// Start state report handler, which will receive a sender from HeartbeatTask to send state size report back /// /// if heartbeat task is shutdown, this future will exit too @@ -659,7 +661,7 @@ impl FlowWorkerManager { } // flow is now shutdown, drop frontend_invoker early so a ref cycle(in standalone mode) can be prevent: // FlowWorkerManager.frontend_invoker -> FrontendInvoker.inserter - // -> Inserter.node_manager -> NodeManager.flownode -> Flownode.flow_worker_manager.frontend_invoker + // -> Inserter.node_manager -> NodeManager.flownode -> Flownode.flow_streaming_engine.frontend_invoker self.frontend_invoker.write().await.take(); } @@ -728,7 +730,7 @@ impl FlowWorkerManager { } /// Create&Remove flow -impl FlowWorkerManager { +impl StreamingEngine { /// remove a flow by it's id pub async fn remove_flow_inner(&self, flow_id: FlowId) -> Result<(), Error> { for handle in self.worker_handles.iter() { diff --git a/src/flow/src/adapter/flownode_impl.rs b/src/flow/src/adapter/flownode_impl.rs index b7d218ef21..c49cbb97ef 100644 --- a/src/flow/src/adapter/flownode_impl.rs +++ b/src/flow/src/adapter/flownode_impl.rs @@ -20,35 +20,394 @@ use api::v1::flow::{ flow_request, CreateRequest, DropRequest, FlowRequest, FlowResponse, FlushFlow, }; use api::v1::region::InsertRequests; +use catalog::CatalogManager; use common_error::ext::BoxedError; use common_meta::ddl::create_flow::FlowType; -use common_meta::error::{Result as MetaResult, UnexpectedSnafu}; +use common_meta::error::Result as MetaResult; +use common_meta::key::flow::FlowMetadataManager; use common_runtime::JoinHandle; -use common_telemetry::{trace, warn}; +use common_telemetry::{error, info, trace, warn}; use datatypes::value::Value; +use futures::TryStreamExt; use itertools::Itertools; -use snafu::{IntoError, OptionExt, ResultExt}; +use session::context::QueryContextBuilder; +use snafu::{ensure, IntoError, OptionExt, ResultExt}; use store_api::storage::{RegionId, TableId}; +use tokio::sync::{Mutex, RwLock}; -use crate::adapter::{CreateFlowArgs, FlowWorkerManager}; +use crate::adapter::{CreateFlowArgs, StreamingEngine}; use crate::batching_mode::engine::BatchingEngine; use crate::engine::FlowEngine; -use crate::error::{CreateFlowSnafu, FlowNotFoundSnafu, InsertIntoFlowSnafu, InternalSnafu}; +use crate::error::{ + CreateFlowSnafu, ExternalSnafu, FlowNotFoundSnafu, IllegalCheckTaskStateSnafu, + InsertIntoFlowSnafu, InternalSnafu, JoinTaskSnafu, ListFlowsSnafu, SyncCheckTaskSnafu, + UnexpectedSnafu, +}; use crate::metrics::METRIC_FLOW_TASK_COUNT; use crate::repr::{self, DiffRow}; use crate::{Error, FlowId}; +/// Ref to [`FlowDualEngine`] +pub type FlowDualEngineRef = Arc; + /// Manage both streaming and batching mode engine /// /// including create/drop/flush flow /// and redirect insert requests to the appropriate engine pub struct FlowDualEngine { - streaming_engine: Arc, + streaming_engine: Arc, batching_engine: Arc, /// helper struct for faster query flow by table id or vice versa - src_table2flow: std::sync::RwLock, + src_table2flow: RwLock, + flow_metadata_manager: Arc, + catalog_manager: Arc, + check_task: tokio::sync::Mutex>, } +impl FlowDualEngine { + pub fn new( + streaming_engine: Arc, + batching_engine: Arc, + flow_metadata_manager: Arc, + catalog_manager: Arc, + ) -> Self { + Self { + streaming_engine, + batching_engine, + src_table2flow: RwLock::new(SrcTableToFlow::default()), + flow_metadata_manager, + catalog_manager, + check_task: Mutex::new(None), + } + } + + pub fn streaming_engine(&self) -> Arc { + self.streaming_engine.clone() + } + + pub fn batching_engine(&self) -> Arc { + self.batching_engine.clone() + } + + /// Try to sync with check task, this is only used in drop flow&flush flow, so a flow id is required + /// + /// the need to sync is to make sure flush flow actually get called + async fn try_sync_with_check_task( + &self, + flow_id: FlowId, + allow_drop: bool, + ) -> Result<(), Error> { + // this function rarely get called so adding some log is helpful + info!("Try to sync with check task for flow {}", flow_id); + let mut retry = 0; + let max_retry = 10; + // keep trying to trigger consistent check + while retry < max_retry { + if let Some(task) = self.check_task.lock().await.as_ref() { + task.trigger(false, allow_drop).await?; + break; + } + retry += 1; + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + } + + if retry == max_retry { + error!( + "Can't sync with check task for flow {} with allow_drop={}", + flow_id, allow_drop + ); + return SyncCheckTaskSnafu { + flow_id, + allow_drop, + } + .fail(); + } + info!("Successfully sync with check task for flow {}", flow_id); + + Ok(()) + } + + /// Spawn a task to consistently check if all flow tasks in metasrv is created on flownode, + /// so on startup, this will create all missing flow tasks, and constantly check at a interval + async fn check_flow_consistent( + &self, + allow_create: bool, + allow_drop: bool, + ) -> Result<(), Error> { + // use nodeid to determine if this is standalone/distributed mode, and retrieve all flows in this node(in distributed mode)/or all flows(in standalone mode) + let nodeid = self.streaming_engine.node_id; + let should_exists: Vec<_> = if let Some(nodeid) = nodeid { + // nodeid is available, so we only need to check flows on this node + // which also means we are in distributed mode + let to_be_recover = self + .flow_metadata_manager + .flownode_flow_manager() + .flows(nodeid.into()) + .try_collect::>() + .await + .context(ListFlowsSnafu { + id: Some(nodeid.into()), + })?; + to_be_recover.into_iter().map(|(id, _)| id).collect() + } else { + // nodeid is not available, so we need to check all flows + // which also means we are in standalone mode + let all_catalogs = self + .catalog_manager + .catalog_names() + .await + .map_err(BoxedError::new) + .context(ExternalSnafu)?; + let mut all_flow_ids = vec![]; + for catalog in all_catalogs { + let flows = self + .flow_metadata_manager + .flow_name_manager() + .flow_names(&catalog) + .await + .try_collect::>() + .await + .map_err(BoxedError::new) + .context(ExternalSnafu)?; + + all_flow_ids.extend(flows.into_iter().map(|(_, id)| id.flow_id())); + } + all_flow_ids + }; + let should_exists = should_exists + .into_iter() + .map(|i| i as FlowId) + .collect::>(); + let actual_exists = self.list_flows().await?.into_iter().collect::>(); + let to_be_created = should_exists + .iter() + .filter(|id| !actual_exists.contains(id)) + .collect::>(); + let to_be_dropped = actual_exists + .iter() + .filter(|id| !should_exists.contains(id)) + .collect::>(); + + if !to_be_created.is_empty() { + if allow_create { + info!( + "Recovering {} flows: {:?}", + to_be_created.len(), + to_be_created + ); + let mut errors = vec![]; + for flow_id in to_be_created { + let flow_id = *flow_id; + let info = self + .flow_metadata_manager + .flow_info_manager() + .get(flow_id as u32) + .await + .map_err(BoxedError::new) + .context(ExternalSnafu)? + .context(FlowNotFoundSnafu { id: flow_id })?; + + let sink_table_name = [ + info.sink_table_name().catalog_name.clone(), + info.sink_table_name().schema_name.clone(), + info.sink_table_name().table_name.clone(), + ]; + let args = CreateFlowArgs { + flow_id, + sink_table_name, + source_table_ids: info.source_table_ids().to_vec(), + // because recover should only happen on restart the `create_if_not_exists` and `or_replace` can be arbitrary value(since flow doesn't exist) + // but for the sake of consistency and to make sure recover of flow actually happen, we set both to true + // (which is also fine since checks for not allow both to be true is on metasrv and we already pass that) + create_if_not_exists: true, + or_replace: true, + expire_after: info.expire_after(), + comment: Some(info.comment().clone()), + sql: info.raw_sql().clone(), + flow_options: info.options().clone(), + query_ctx: info + .query_context() + .clone() + .map(|ctx| { + ctx.try_into() + .map_err(BoxedError::new) + .context(ExternalSnafu) + }) + .transpose()? + // or use default QueryContext with catalog_name from info + // to keep compatibility with old version + .or_else(|| { + Some( + QueryContextBuilder::default() + .current_catalog(info.catalog_name().to_string()) + .build(), + ) + }), + }; + if let Err(err) = self + .create_flow(args) + .await + .map_err(BoxedError::new) + .with_context(|_| CreateFlowSnafu { + sql: info.raw_sql().clone(), + }) + { + errors.push((flow_id, err)); + } + } + for (flow_id, err) in errors { + warn!("Failed to recreate flow {}, err={:#?}", flow_id, err); + } + } else { + warn!( + "Flownode {:?} found flows not exist in flownode, flow_ids={:?}", + nodeid, to_be_created + ); + } + } + if !to_be_dropped.is_empty() { + if allow_drop { + info!("Dropping flows: {:?}", to_be_dropped); + let mut errors = vec![]; + for flow_id in to_be_dropped { + let flow_id = *flow_id; + if let Err(err) = self.remove_flow(flow_id).await { + errors.push((flow_id, err)); + } + } + for (flow_id, err) in errors { + warn!("Failed to drop flow {}, err={:#?}", flow_id, err); + } + } else { + warn!( + "Flownode {:?} found flows not exist in flownode, flow_ids={:?}", + nodeid, to_be_dropped + ); + } + } + Ok(()) + } + + // TODO(discord9): consider sync this with heartbeat(might become necessary in the future) + pub async fn start_flow_consistent_check_task(self: &Arc) -> Result<(), Error> { + let mut check_task = self.check_task.lock().await; + ensure!( + check_task.is_none(), + IllegalCheckTaskStateSnafu { + reason: "Flow consistent check task already exists", + } + ); + let task = ConsistentCheckTask::start_check_task(self).await?; + *check_task = Some(task); + Ok(()) + } + + pub async fn stop_flow_consistent_check_task(&self) -> Result<(), Error> { + info!("Stopping flow consistent check task"); + let mut check_task = self.check_task.lock().await; + + ensure!( + check_task.is_some(), + IllegalCheckTaskStateSnafu { + reason: "Flow consistent check task does not exist", + } + ); + + check_task.take().unwrap().stop().await?; + info!("Stopped flow consistent check task"); + Ok(()) + } + + /// TODO(discord9): also add a `exists` api using flow metadata manager's `exists` method + async fn flow_exist_in_metadata(&self, flow_id: FlowId) -> Result { + self.flow_metadata_manager + .flow_info_manager() + .get(flow_id as u32) + .await + .map_err(BoxedError::new) + .context(ExternalSnafu) + .map(|info| info.is_some()) + } +} + +struct ConsistentCheckTask { + handle: JoinHandle<()>, + shutdown_tx: tokio::sync::mpsc::Sender<()>, + trigger_tx: tokio::sync::mpsc::Sender<(bool, bool, tokio::sync::oneshot::Sender<()>)>, +} + +impl ConsistentCheckTask { + async fn start_check_task(engine: &Arc) -> Result { + // first do recover flows + engine.check_flow_consistent(true, false).await?; + + let inner = engine.clone(); + let (tx, mut rx) = tokio::sync::mpsc::channel(1); + let (trigger_tx, mut trigger_rx) = + tokio::sync::mpsc::channel::<(bool, bool, tokio::sync::oneshot::Sender<()>)>(10); + let handle = common_runtime::spawn_global(async move { + let (mut allow_create, mut allow_drop) = (false, false); + let mut ret_signal: Option> = None; + loop { + if let Err(err) = inner.check_flow_consistent(allow_create, allow_drop).await { + error!(err; "Failed to check flow consistent"); + } + if let Some(done) = ret_signal.take() { + let _ = done.send(()); + } + tokio::select! { + _ = rx.recv() => break, + incoming = trigger_rx.recv() => if let Some(incoming) = incoming { + (allow_create, allow_drop) = (incoming.0, incoming.1); + ret_signal = Some(incoming.2); + }, + _ = tokio::time::sleep(std::time::Duration::from_secs(10)) => { + (allow_create, allow_drop) = (false, false); + }, + } + } + }); + Ok(ConsistentCheckTask { + handle, + shutdown_tx: tx, + trigger_tx, + }) + } + + async fn trigger(&self, allow_create: bool, allow_drop: bool) -> Result<(), Error> { + let (tx, rx) = tokio::sync::oneshot::channel(); + self.trigger_tx + .send((allow_create, allow_drop, tx)) + .await + .map_err(|_| { + IllegalCheckTaskStateSnafu { + reason: "Failed to send trigger signal", + } + .build() + })?; + rx.await.map_err(|_| { + IllegalCheckTaskStateSnafu { + reason: "Failed to receive trigger signal", + } + .build() + })?; + Ok(()) + } + + async fn stop(self) -> Result<(), Error> { + self.shutdown_tx.send(()).await.map_err(|_| { + IllegalCheckTaskStateSnafu { + reason: "Failed to send shutdown signal", + } + .build() + })?; + // abort so no need to wait + self.handle.abort(); + Ok(()) + } +} + +#[derive(Default)] struct SrcTableToFlow { /// mapping of table ids to flow ids for streaming mode stream: HashMap>, @@ -138,35 +497,49 @@ impl FlowEngine for FlowDualEngine { self.src_table2flow .write() - .unwrap() + .await .add_flow(flow_id, flow_type, src_table_ids); Ok(res) } async fn remove_flow(&self, flow_id: FlowId) -> Result<(), Error> { - let flow_type = self.src_table2flow.read().unwrap().get_flow_type(flow_id); + let flow_type = self.src_table2flow.read().await.get_flow_type(flow_id); + match flow_type { Some(FlowType::Batching) => self.batching_engine.remove_flow(flow_id).await, Some(FlowType::Streaming) => self.streaming_engine.remove_flow(flow_id).await, - None => FlowNotFoundSnafu { id: flow_id }.fail(), + None => { + // this can happen if flownode just restart, and is stilling creating the flow + // since now that this flow should dropped, we need to trigger the consistent check and allow drop + // this rely on drop flow ddl delete metadata first, see src/common/meta/src/ddl/drop_flow.rs + warn!( + "Flow {} is not exist in the underlying engine, but exist in metadata", + flow_id + ); + self.try_sync_with_check_task(flow_id, true).await?; + + Ok(()) + } }?; // remove mapping - self.src_table2flow.write().unwrap().remove_flow(flow_id); + self.src_table2flow.write().await.remove_flow(flow_id); Ok(()) } async fn flush_flow(&self, flow_id: FlowId) -> Result { - let flow_type = self.src_table2flow.read().unwrap().get_flow_type(flow_id); + // sync with check task + self.try_sync_with_check_task(flow_id, false).await?; + let flow_type = self.src_table2flow.read().await.get_flow_type(flow_id); match flow_type { Some(FlowType::Batching) => self.batching_engine.flush_flow(flow_id).await, Some(FlowType::Streaming) => self.streaming_engine.flush_flow(flow_id).await, - None => FlowNotFoundSnafu { id: flow_id }.fail(), + None => Ok(0), } } async fn flow_exist(&self, flow_id: FlowId) -> Result { - let flow_type = self.src_table2flow.read().unwrap().get_flow_type(flow_id); + let flow_type = self.src_table2flow.read().await.get_flow_type(flow_id); // not using `flow_type.is_some()` to make sure the flow is actually exist in the underlying engine match flow_type { Some(FlowType::Batching) => self.batching_engine.flow_exist(flow_id).await, @@ -175,6 +548,13 @@ impl FlowEngine for FlowDualEngine { } } + async fn list_flows(&self) -> Result, Error> { + let stream_flows = self.streaming_engine.list_flows().await?; + let batch_flows = self.batching_engine.list_flows().await?; + + Ok(stream_flows.into_iter().chain(batch_flows)) + } + async fn handle_flow_inserts( &self, request: api::v1::region::InsertRequests, @@ -184,7 +564,7 @@ impl FlowEngine for FlowDualEngine { let mut to_batch_engine = request.requests; { - let src_table2flow = self.src_table2flow.read().unwrap(); + let src_table2flow = self.src_table2flow.read().await; to_batch_engine.retain(|req| { let region_id = RegionId::from(req.region_id); let table_id = region_id.table_id(); @@ -221,12 +601,7 @@ impl FlowEngine for FlowDualEngine { requests: to_batch_engine, }) .await?; - stream_handler.await.map_err(|e| { - crate::error::UnexpectedSnafu { - reason: format!("JoinError when handle inserts for flow stream engine: {e:?}"), - } - .build() - })??; + stream_handler.await.context(JoinTaskSnafu)??; Ok(()) } @@ -307,14 +682,7 @@ impl common_meta::node_manager::Flownode for FlowDualEngine { ..Default::default() }) } - None => UnexpectedSnafu { - err_msg: "Missing request body", - } - .fail(), - _ => UnexpectedSnafu { - err_msg: "Invalid request body.", - } - .fail(), + other => common_meta::error::InvalidFlowRequestBodySnafu { body: other }.fail(), } } @@ -339,7 +707,7 @@ fn to_meta_err( } #[async_trait::async_trait] -impl common_meta::node_manager::Flownode for FlowWorkerManager { +impl common_meta::node_manager::Flownode for StreamingEngine { async fn handle(&self, request: FlowRequest) -> MetaResult { let query_ctx = request .header @@ -413,14 +781,7 @@ impl common_meta::node_manager::Flownode for FlowWorkerManager { ..Default::default() }) } - None => UnexpectedSnafu { - err_msg: "Missing request body", - } - .fail(), - _ => UnexpectedSnafu { - err_msg: "Invalid request body.", - } - .fail(), + other => common_meta::error::InvalidFlowRequestBodySnafu { body: other }.fail(), } } @@ -432,7 +793,7 @@ impl common_meta::node_manager::Flownode for FlowWorkerManager { } } -impl FlowEngine for FlowWorkerManager { +impl FlowEngine for StreamingEngine { async fn create_flow(&self, args: CreateFlowArgs) -> Result, Error> { self.create_flow_inner(args).await } @@ -449,6 +810,16 @@ impl FlowEngine for FlowWorkerManager { self.flow_exist_inner(flow_id).await } + async fn list_flows(&self) -> Result, Error> { + Ok(self + .flow_err_collectors + .read() + .await + .keys() + .cloned() + .collect::>()) + } + async fn handle_flow_inserts( &self, request: api::v1::region::InsertRequests, @@ -474,7 +845,7 @@ impl FetchFromRow { } } -impl FlowWorkerManager { +impl StreamingEngine { async fn handle_inserts_inner( &self, request: InsertRequests, @@ -552,7 +923,7 @@ impl FlowWorkerManager { .copied() .map(FetchFromRow::Idx) .or_else(|| col_default_val.clone().map(FetchFromRow::Default)) - .with_context(|| crate::error::UnexpectedSnafu { + .with_context(|| UnexpectedSnafu { reason: format!( "Column not found: {}, default_value: {:?}", col_name, col_default_val diff --git a/src/flow/src/adapter/refill.rs b/src/flow/src/adapter/refill.rs index 25287df62b..a29e120f0a 100644 --- a/src/flow/src/adapter/refill.rs +++ b/src/flow/src/adapter/refill.rs @@ -31,7 +31,7 @@ use snafu::{ensure, OptionExt, ResultExt}; use table::metadata::TableId; use crate::adapter::table_source::ManagedTableSource; -use crate::adapter::{FlowId, FlowWorkerManager, FlowWorkerManagerRef}; +use crate::adapter::{FlowId, FlowStreamingEngineRef, StreamingEngine}; use crate::error::{FlowNotFoundSnafu, JoinTaskSnafu, UnexpectedSnafu}; use crate::expr::error::ExternalSnafu; use crate::expr::utils::find_plan_time_window_expr_lower_bound; @@ -39,10 +39,10 @@ use crate::repr::RelationDesc; use crate::server::get_all_flow_ids; use crate::{Error, FrontendInvoker}; -impl FlowWorkerManager { +impl StreamingEngine { /// Create and start refill flow tasks in background pub async fn create_and_start_refill_flow_tasks( - self: &FlowWorkerManagerRef, + self: &FlowStreamingEngineRef, flow_metadata_manager: &FlowMetadataManagerRef, catalog_manager: &CatalogManagerRef, ) -> Result<(), Error> { @@ -130,7 +130,7 @@ impl FlowWorkerManager { /// Starting to refill flows, if any error occurs, will rebuild the flow and retry pub(crate) async fn starting_refill_flows( - self: &FlowWorkerManagerRef, + self: &FlowStreamingEngineRef, tasks: Vec, ) -> Result<(), Error> { // TODO(discord9): add a back pressure mechanism @@ -266,7 +266,7 @@ impl TaskState<()> { fn start_running( &mut self, task_data: &TaskData, - manager: FlowWorkerManagerRef, + manager: FlowStreamingEngineRef, mut output_stream: SendableRecordBatchStream, ) -> Result<(), Error> { let data = (*task_data).clone(); @@ -383,7 +383,7 @@ impl RefillTask { /// Start running the task in background, non-blocking pub async fn start_running( &mut self, - manager: FlowWorkerManagerRef, + manager: FlowStreamingEngineRef, invoker: &FrontendInvoker, ) -> Result<(), Error> { let TaskState::Prepared { sql } = &mut self.state else { diff --git a/src/flow/src/adapter/stat.rs b/src/flow/src/adapter/stat.rs index bf272cd9d0..fe1727a30e 100644 --- a/src/flow/src/adapter/stat.rs +++ b/src/flow/src/adapter/stat.rs @@ -16,9 +16,9 @@ use std::collections::BTreeMap; use common_meta::key::flow::flow_state::FlowStat; -use crate::FlowWorkerManager; +use crate::StreamingEngine; -impl FlowWorkerManager { +impl StreamingEngine { pub async fn gen_state_report(&self) -> FlowStat { let mut full_report = BTreeMap::new(); let mut last_exec_time_map = BTreeMap::new(); diff --git a/src/flow/src/adapter/util.rs b/src/flow/src/adapter/util.rs index 6811c29c96..3bb0031eee 100644 --- a/src/flow/src/adapter/util.rs +++ b/src/flow/src/adapter/util.rs @@ -33,8 +33,8 @@ use crate::adapter::table_source::TableDesc; use crate::adapter::{TableName, WorkerHandle, AUTO_CREATED_PLACEHOLDER_TS_COL}; use crate::error::{Error, ExternalSnafu, UnexpectedSnafu}; use crate::repr::{ColumnType, RelationDesc, RelationType}; -use crate::FlowWorkerManager; -impl FlowWorkerManager { +use crate::StreamingEngine; +impl StreamingEngine { /// Get a worker handle for creating flow, using round robin to select a worker pub(crate) async fn get_worker_handle_for_create_flow(&self) -> &WorkerHandle { let use_idx = { diff --git a/src/flow/src/batching_mode/engine.rs b/src/flow/src/batching_mode/engine.rs index c53107f695..6c667d56d5 100644 --- a/src/flow/src/batching_mode/engine.rs +++ b/src/flow/src/batching_mode/engine.rs @@ -17,14 +17,16 @@ use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; +use catalog::CatalogManagerRef; use common_error::ext::BoxedError; use common_meta::ddl::create_flow::FlowType; use common_meta::key::flow::FlowMetadataManagerRef; -use common_meta::key::table_info::TableInfoManager; +use common_meta::key::table_info::{TableInfoManager, TableInfoValue}; use common_meta::key::TableMetadataManagerRef; use common_runtime::JoinHandle; -use common_telemetry::info; use common_telemetry::tracing::warn; +use common_telemetry::{debug, info}; +use common_time::TimeToLive; use query::QueryEngineRef; use snafu::{ensure, OptionExt, ResultExt}; use store_api::storage::RegionId; @@ -36,7 +38,9 @@ use crate::batching_mode::task::BatchingTask; use crate::batching_mode::time_window::{find_time_window_expr, TimeWindowExpr}; use crate::batching_mode::utils::sql_to_df_plan; use crate::engine::FlowEngine; -use crate::error::{ExternalSnafu, FlowAlreadyExistSnafu, TableNotFoundMetaSnafu, UnexpectedSnafu}; +use crate::error::{ + ExternalSnafu, FlowAlreadyExistSnafu, TableNotFoundMetaSnafu, UnexpectedSnafu, UnsupportedSnafu, +}; use crate::{CreateFlowArgs, Error, FlowId, TableName}; /// Batching mode Engine, responsible for driving all the batching mode tasks @@ -48,6 +52,7 @@ pub struct BatchingEngine { frontend_client: Arc, flow_metadata_manager: FlowMetadataManagerRef, table_meta: TableMetadataManagerRef, + catalog_manager: CatalogManagerRef, query_engine: QueryEngineRef, } @@ -57,6 +62,7 @@ impl BatchingEngine { query_engine: QueryEngineRef, flow_metadata_manager: FlowMetadataManagerRef, table_meta: TableMetadataManagerRef, + catalog_manager: CatalogManagerRef, ) -> Self { Self { tasks: Default::default(), @@ -64,6 +70,7 @@ impl BatchingEngine { frontend_client, flow_metadata_manager, table_meta, + catalog_manager, query_engine, } } @@ -179,6 +186,16 @@ async fn get_table_name( table_info: &TableInfoManager, table_id: &TableId, ) -> Result { + get_table_info(table_info, table_id).await.map(|info| { + let name = info.table_name(); + [name.catalog_name, name.schema_name, name.table_name] + }) +} + +async fn get_table_info( + table_info: &TableInfoManager, + table_id: &TableId, +) -> Result { table_info .get(*table_id) .await @@ -187,8 +204,7 @@ async fn get_table_name( .with_context(|| UnexpectedSnafu { reason: format!("Table id = {:?}, couldn't found table name", table_id), }) - .map(|name| name.table_name()) - .map(|name| [name.catalog_name, name.schema_name, name.table_name]) + .map(|info| info.into_inner()) } impl BatchingEngine { @@ -248,7 +264,20 @@ impl BatchingEngine { let query_ctx = Arc::new(query_ctx); let mut source_table_names = Vec::with_capacity(2); for src_id in source_table_ids { + // also check table option to see if ttl!=instant let table_name = get_table_name(self.table_meta.table_info_manager(), &src_id).await?; + let table_info = get_table_info(self.table_meta.table_info_manager(), &src_id).await?; + ensure!( + table_info.table_info.meta.options.ttl != Some(TimeToLive::Instant), + UnsupportedSnafu { + reason: format!( + "Source table `{}`(id={}) has instant TTL, Instant TTL is not supported under batching mode. Consider using a TTL longer than flush interval", + table_name.join("."), + src_id + ), + } + ); + source_table_names.push(table_name); } @@ -273,7 +302,14 @@ impl BatchingEngine { }) .transpose()?; - info!("Flow id={}, found time window expr={:?}", flow_id, phy_expr); + info!( + "Flow id={}, found time window expr={}", + flow_id, + phy_expr + .as_ref() + .map(|phy_expr| phy_expr.to_string()) + .unwrap_or("None".to_string()) + ); let task = BatchingTask::new( flow_id, @@ -284,7 +320,7 @@ impl BatchingEngine { sink_table_name, source_table_names, query_ctx, - self.table_meta.clone(), + self.catalog_manager.clone(), rx, ); @@ -295,10 +331,11 @@ impl BatchingEngine { // check execute once first to detect any error early task.check_execute(&engine, &frontend).await?; - // TODO(discord9): also save handle & use time wheel or what for better - let _handle = common_runtime::spawn_global(async move { + // TODO(discord9): use time wheel or what for better + let handle = common_runtime::spawn_global(async move { task_inner.start_executing_loop(engine, frontend).await; }); + task.state.write().unwrap().task_handle = Some(handle); // only replace here not earlier because we want the old one intact if something went wrong before this line let replaced_old_task_opt = self.tasks.write().await.insert(flow_id, task); @@ -326,15 +363,23 @@ impl BatchingEngine { } pub async fn flush_flow_inner(&self, flow_id: FlowId) -> Result { + debug!("Try flush flow {flow_id}"); let task = self.tasks.read().await.get(&flow_id).cloned(); let task = task.with_context(|| UnexpectedSnafu { reason: format!("Can't found task for flow {flow_id}"), })?; + task.mark_all_windows_as_dirty()?; + let res = task .gen_exec_once(&self.query_engine, &self.frontend_client) .await?; + let affected_rows = res.map(|(r, _)| r).unwrap_or_default() as usize; + debug!( + "Successfully flush flow {flow_id}, affected rows={}", + affected_rows + ); Ok(affected_rows) } @@ -357,6 +402,9 @@ impl FlowEngine for BatchingEngine { async fn flow_exist(&self, flow_id: FlowId) -> Result { Ok(self.flow_exist_inner(flow_id).await) } + async fn list_flows(&self) -> Result, Error> { + Ok(self.tasks.read().await.keys().cloned().collect::>()) + } async fn handle_flow_inserts( &self, request: api::v1::region::InsertRequests, diff --git a/src/flow/src/batching_mode/frontend_client.rs b/src/flow/src/batching_mode/frontend_client.rs index 3b62986422..2454e86251 100644 --- a/src/flow/src/batching_mode/frontend_client.rs +++ b/src/flow/src/batching_mode/frontend_client.rs @@ -14,44 +14,104 @@ //! Frontend client to run flow as batching task which is time-window-aware normal query triggered every tick set by user -use std::sync::Arc; +use std::sync::{Arc, Weak}; -use client::{Client, Database, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; -use common_error::ext::BoxedError; +use api::v1::greptime_request::Request; +use api::v1::CreateTableExpr; +use client::{Client, Database}; +use common_error::ext::{BoxedError, ErrorExt}; use common_grpc::channel_manager::{ChannelConfig, ChannelManager}; use common_meta::cluster::{NodeInfo, NodeInfoKey, Role}; use common_meta::peer::Peer; use common_meta::rpc::store::RangeRequest; +use common_query::Output; use meta_client::client::MetaClient; -use snafu::ResultExt; +use servers::query_handler::grpc::GrpcQueryHandler; +use session::context::{QueryContextBuilder, QueryContextRef}; +use snafu::{OptionExt, ResultExt}; use crate::batching_mode::DEFAULT_BATCHING_ENGINE_QUERY_TIMEOUT; -use crate::error::{ExternalSnafu, UnexpectedSnafu}; +use crate::error::{ExternalSnafu, InvalidRequestSnafu, UnexpectedSnafu}; use crate::Error; -fn default_channel_mgr() -> ChannelManager { - let cfg = ChannelConfig::new().timeout(DEFAULT_BATCHING_ENGINE_QUERY_TIMEOUT); - ChannelManager::with_config(cfg) +/// Just like [`GrpcQueryHandler`] but use BoxedError +/// +/// basically just a specialized `GrpcQueryHandler` +/// +/// this is only useful for flownode to +/// invoke frontend Instance in standalone mode +#[async_trait::async_trait] +pub trait GrpcQueryHandlerWithBoxedError: Send + Sync + 'static { + async fn do_query( + &self, + query: Request, + ctx: QueryContextRef, + ) -> std::result::Result; } -fn client_from_urls(addrs: Vec) -> Client { - Client::with_manager_and_urls(default_channel_mgr(), addrs) +/// auto impl +#[async_trait::async_trait] +impl< + E: ErrorExt + Send + Sync + 'static, + T: GrpcQueryHandler + Send + Sync + 'static, + > GrpcQueryHandlerWithBoxedError for T +{ + async fn do_query( + &self, + query: Request, + ctx: QueryContextRef, + ) -> std::result::Result { + self.do_query(query, ctx).await.map_err(BoxedError::new) + } } +type HandlerMutable = Arc>>>; + /// A simple frontend client able to execute sql using grpc protocol -#[derive(Debug)] +/// +/// This is for computation-heavy query which need to offload computation to frontend, lifting the load from flownode +#[derive(Debug, Clone)] pub enum FrontendClient { Distributed { meta_client: Arc, + chnl_mgr: ChannelManager, }, Standalone { /// for the sake of simplicity still use grpc even in standalone mode /// notice the client here should all be lazy, so that can wait after frontend is booted then make conn - /// TODO(discord9): not use grpc under standalone mode - database_client: DatabaseWithPeer, + database_client: HandlerMutable, }, } +impl FrontendClient { + /// Create a new empty frontend client, with a `HandlerMutable` to set the grpc handler later + pub fn from_empty_grpc_handler() -> (Self, HandlerMutable) { + let handler = Arc::new(std::sync::Mutex::new(None)); + ( + Self::Standalone { + database_client: handler.clone(), + }, + handler, + ) + } + + pub fn from_meta_client(meta_client: Arc) -> Self { + Self::Distributed { + meta_client, + chnl_mgr: { + let cfg = ChannelConfig::new().timeout(DEFAULT_BATCHING_ENGINE_QUERY_TIMEOUT); + ChannelManager::with_config(cfg) + }, + } + } + + pub fn from_grpc_handler(grpc_handler: Weak) -> Self { + Self::Standalone { + database_client: Arc::new(std::sync::Mutex::new(Some(grpc_handler))), + } + } +} + #[derive(Debug, Clone)] pub struct DatabaseWithPeer { pub database: Database, @@ -64,25 +124,6 @@ impl DatabaseWithPeer { } } -impl FrontendClient { - pub fn from_meta_client(meta_client: Arc) -> Self { - Self::Distributed { meta_client } - } - - pub fn from_static_grpc_addr(addr: String) -> Self { - let peer = Peer { - id: 0, - addr: addr.clone(), - }; - - let client = client_from_urls(vec![addr]); - let database = Database::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, client); - Self::Standalone { - database_client: DatabaseWithPeer::new(database, peer), - } - } -} - impl FrontendClient { async fn scan_for_frontend(&self) -> Result, Error> { let Self::Distributed { meta_client, .. } = self else { @@ -115,10 +156,21 @@ impl FrontendClient { } /// Get the database with max `last_activity_ts` - async fn get_last_active_frontend(&self) -> Result { - if let Self::Standalone { database_client } = self { - return Ok(database_client.clone()); - } + async fn get_last_active_frontend( + &self, + catalog: &str, + schema: &str, + ) -> Result { + let Self::Distributed { + meta_client: _, + chnl_mgr, + } = self + else { + return UnexpectedSnafu { + reason: "Expect distributed mode", + } + .fail(); + }; let frontends = self.scan_for_frontend().await?; let mut peer = None; @@ -133,16 +185,119 @@ impl FrontendClient { } .fail()? }; - let client = client_from_urls(vec![peer.addr.clone()]); - let database = Database::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, client); + let client = Client::with_manager_and_urls(chnl_mgr.clone(), vec![peer.addr.clone()]); + let database = Database::new(catalog, schema, client); Ok(DatabaseWithPeer::new(database, peer)) } - /// Get a database client, and possibly update it before returning. - pub async fn get_database_client(&self) -> Result { + pub async fn create( + &self, + create: CreateTableExpr, + catalog: &str, + schema: &str, + ) -> Result { + self.handle( + Request::Ddl(api::v1::DdlRequest { + expr: Some(api::v1::ddl_request::Expr::CreateTable(create)), + }), + catalog, + schema, + &mut None, + ) + .await + } + + /// Handle a request to frontend + pub(crate) async fn handle( + &self, + req: api::v1::greptime_request::Request, + catalog: &str, + schema: &str, + peer_desc: &mut Option, + ) -> Result { match self { - Self::Standalone { database_client } => Ok(database_client.clone()), - Self::Distributed { meta_client: _ } => self.get_last_active_frontend().await, + FrontendClient::Distributed { .. } => { + let db = self.get_last_active_frontend(catalog, schema).await?; + + *peer_desc = Some(PeerDesc::Dist { + peer: db.peer.clone(), + }); + + db.database + .handle(req.clone()) + .await + .with_context(|_| InvalidRequestSnafu { + context: format!("Failed to handle request: {:?}", req), + }) + } + FrontendClient::Standalone { database_client } => { + let ctx = QueryContextBuilder::default() + .current_catalog(catalog.to_string()) + .current_schema(schema.to_string()) + .build(); + let ctx = Arc::new(ctx); + { + let database_client = { + database_client + .lock() + .map_err(|e| { + UnexpectedSnafu { + reason: format!("Failed to lock database client: {e}"), + } + .build() + })? + .as_ref() + .context(UnexpectedSnafu { + reason: "Standalone's frontend instance is not set", + })? + .upgrade() + .context(UnexpectedSnafu { + reason: "Failed to upgrade database client", + })? + }; + let resp: common_query::Output = database_client + .do_query(req.clone(), ctx) + .await + .map_err(BoxedError::new) + .context(ExternalSnafu)?; + match resp.data { + common_query::OutputData::AffectedRows(rows) => { + Ok(rows.try_into().map_err(|_| { + UnexpectedSnafu { + reason: format!("Failed to convert rows to u32: {}", rows), + } + .build() + })?) + } + _ => UnexpectedSnafu { + reason: "Unexpected output data", + } + .fail(), + } + } + } + } + } +} + +/// Describe a peer of frontend +#[derive(Debug, Default)] +pub(crate) enum PeerDesc { + /// Distributed mode's frontend peer address + Dist { + /// frontend peer address + peer: Peer, + }, + /// Standalone mode + #[default] + Standalone, +} + +impl std::fmt::Display for PeerDesc { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + PeerDesc::Dist { peer } => write!(f, "{}", peer.addr), + PeerDesc::Standalone => write!(f, "standalone"), } } } diff --git a/src/flow/src/batching_mode/state.rs b/src/flow/src/batching_mode/state.rs index 3a9802713c..4c6f608af9 100644 --- a/src/flow/src/batching_mode/state.rs +++ b/src/flow/src/batching_mode/state.rs @@ -22,13 +22,14 @@ use common_telemetry::tracing::warn; use common_time::Timestamp; use datatypes::value::Value; use session::context::QueryContextRef; -use snafu::ResultExt; +use snafu::{OptionExt, ResultExt}; use tokio::sync::oneshot; use tokio::time::Instant; use crate::batching_mode::task::BatchingTask; +use crate::batching_mode::time_window::TimeWindowExpr; use crate::batching_mode::MIN_REFRESH_DURATION; -use crate::error::{DatatypesSnafu, InternalSnafu, TimeSnafu}; +use crate::error::{DatatypesSnafu, InternalSnafu, TimeSnafu, UnexpectedSnafu}; use crate::{Error, FlowId}; /// The state of the [`BatchingTask`]. @@ -46,6 +47,8 @@ pub struct TaskState { exec_state: ExecState, /// Shutdown receiver pub(crate) shutdown_rx: oneshot::Receiver<()>, + /// Task handle + pub(crate) task_handle: Option>, } impl TaskState { pub fn new(query_ctx: QueryContextRef, shutdown_rx: oneshot::Receiver<()>) -> Self { @@ -56,6 +59,7 @@ impl TaskState { dirty_time_windows: Default::default(), exec_state: ExecState::Idle, shutdown_rx, + task_handle: None, } } @@ -70,7 +74,11 @@ impl TaskState { /// wait for at least `last_query_duration`, at most `max_timeout` to start next query /// /// if have more dirty time window, exec next query immediately - pub fn get_next_start_query_time(&self, max_timeout: Option) -> Instant { + pub fn get_next_start_query_time( + &self, + flow_id: FlowId, + max_timeout: Option, + ) -> Instant { let next_duration = max_timeout .unwrap_or(self.last_query_duration) .min(self.last_query_duration); @@ -80,6 +88,12 @@ impl TaskState { if self.dirty_time_windows.windows.is_empty() { self.last_update_time + next_duration } else { + debug!( + "Flow id = {}, still have {} dirty time window({:?}), execute immediately", + flow_id, + self.dirty_time_windows.windows.len(), + self.dirty_time_windows.windows + ); Instant::now() } } @@ -115,6 +129,15 @@ impl DirtyTimeWindows { } } + pub fn add_window(&mut self, start: Timestamp, end: Option) { + self.windows.insert(start, end); + } + + /// Clean all dirty time windows, useful when can't found time window expr + pub fn clean(&mut self) { + self.windows.clear(); + } + /// Generate all filter expressions consuming all time windows pub fn gen_filter_exprs( &mut self, @@ -177,6 +200,18 @@ impl DirtyTimeWindows { let mut expr_lst = vec![]; for (start, end) in first_nth.into_iter() { + // align using time window exprs + let (start, end) = if let Some(ctx) = task_ctx { + let Some(time_window_expr) = &ctx.config.time_window_expr else { + UnexpectedSnafu { + reason: "time_window_expr is not set", + } + .fail()? + }; + self.align_time_window(start, end, time_window_expr)? + } else { + (start, end) + }; debug!( "Time window start: {:?}, end: {:?}", start.to_iso8601_string(), @@ -199,6 +234,30 @@ impl DirtyTimeWindows { Ok(expr) } + fn align_time_window( + &self, + start: Timestamp, + end: Option, + time_window_expr: &TimeWindowExpr, + ) -> Result<(Timestamp, Option), Error> { + let align_start = time_window_expr.eval(start)?.0.context(UnexpectedSnafu { + reason: format!( + "Failed to align start time {:?} with time window expr {:?}", + start, time_window_expr + ), + })?; + let align_end = end + .and_then(|end| { + time_window_expr + .eval(end) + // if after aligned, end is the same, then use end(because it's already aligned) else use aligned end + .map(|r| if r.0 == Some(end) { r.0 } else { r.1 }) + .transpose() + }) + .transpose()?; + Ok((align_start, align_end)) + } + /// Merge time windows that overlaps or get too close pub fn merge_dirty_time_windows( &mut self, @@ -287,8 +346,12 @@ enum ExecState { #[cfg(test)] mod test { use pretty_assertions::assert_eq; + use session::context::QueryContext; use super::*; + use crate::batching_mode::time_window::find_time_window_expr; + use crate::batching_mode::utils::sql_to_df_plan; + use crate::test_utils::create_test_query_engine; #[test] fn test_merge_dirty_time_windows() { @@ -404,4 +467,59 @@ mod test { assert_eq!(expected_filter_expr, to_sql.as_deref()); } } + + #[tokio::test] + async fn test_align_time_window() { + type TimeWindow = (Timestamp, Option); + struct TestCase { + sql: String, + aligns: Vec<(TimeWindow, TimeWindow)>, + } + let testcases: Vec = vec![TestCase{ + sql: "SELECT date_bin(INTERVAL '5 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window;".to_string(), + aligns: vec![ + ((Timestamp::new_second(3), None), (Timestamp::new_second(0), None)), + ((Timestamp::new_second(8), None), (Timestamp::new_second(5), None)), + ((Timestamp::new_second(8), Some(Timestamp::new_second(10))), (Timestamp::new_second(5), Some(Timestamp::new_second(10)))), + ((Timestamp::new_second(8), Some(Timestamp::new_second(9))), (Timestamp::new_second(5), Some(Timestamp::new_second(10)))), + ], + }]; + + let query_engine = create_test_query_engine(); + let ctx = QueryContext::arc(); + for TestCase { sql, aligns } in testcases { + let plan = sql_to_df_plan(ctx.clone(), query_engine.clone(), &sql, true) + .await + .unwrap(); + + let (column_name, time_window_expr, _, df_schema) = find_time_window_expr( + &plan, + query_engine.engine_state().catalog_manager().clone(), + ctx.clone(), + ) + .await + .unwrap(); + + let time_window_expr = time_window_expr + .map(|expr| { + TimeWindowExpr::from_expr( + &expr, + &column_name, + &df_schema, + &query_engine.engine_state().session_state(), + ) + }) + .transpose() + .unwrap() + .unwrap(); + + let dirty = DirtyTimeWindows::default(); + for (before_align, expected_after_align) in aligns { + let after_align = dirty + .align_time_window(before_align.0, before_align.1, &time_window_expr) + .unwrap(); + assert_eq!(expected_after_align, after_align); + } + } + } } diff --git a/src/flow/src/batching_mode/task.rs b/src/flow/src/batching_mode/task.rs index f4280f54bd..1547faae11 100644 --- a/src/flow/src/batching_mode/task.rs +++ b/src/flow/src/batching_mode/task.rs @@ -12,33 +12,32 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashSet; +use std::collections::{BTreeSet, HashSet}; use std::ops::Deref; use std::sync::{Arc, RwLock}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use api::v1::CreateTableExpr; use arrow_schema::Fields; +use catalog::CatalogManagerRef; use common_error::ext::BoxedError; -use common_meta::key::table_name::TableNameKey; -use common_meta::key::TableMetadataManagerRef; +use common_query::logical_plan::breakup_insert_plan; use common_telemetry::tracing::warn; use common_telemetry::{debug, info}; use common_time::Timestamp; +use datafusion::optimizer::analyzer::count_wildcard_rule::CountWildcardRule; +use datafusion::optimizer::AnalyzerRule; use datafusion::sql::unparser::expr_to_sql; -use datafusion_common::tree_node::TreeNode; +use datafusion_common::tree_node::{Transformed, TreeNode}; use datafusion_expr::{DmlStatement, LogicalPlan, WriteOp}; use datatypes::prelude::ConcreteDataType; -use datatypes::schema::constraint::NOW_FN; -use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema}; -use datatypes::value::Value; +use datatypes::schema::{ColumnSchema, Schema}; use operator::expr_helper::column_schemas_to_defs; use query::query_engine::DefaultSerializer; use query::QueryEngineRef; use session::context::QueryContextRef; -use snafu::{OptionExt, ResultExt}; +use snafu::{ensure, OptionExt, ResultExt}; use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan}; -use table::metadata::RawTableMeta; use tokio::sync::oneshot; use tokio::sync::oneshot::error::TryRecvError; use tokio::time::Instant; @@ -48,14 +47,15 @@ use crate::batching_mode::frontend_client::FrontendClient; use crate::batching_mode::state::TaskState; use crate::batching_mode::time_window::TimeWindowExpr; use crate::batching_mode::utils::{ - sql_to_df_plan, AddAutoColumnRewriter, AddFilterRewriter, FindGroupByFinalName, + get_table_info_df_schema, sql_to_df_plan, AddAutoColumnRewriter, AddFilterRewriter, + FindGroupByFinalName, }; use crate::batching_mode::{ DEFAULT_BATCHING_ENGINE_QUERY_TIMEOUT, MIN_REFRESH_DURATION, SLOW_QUERY_THRESHOLD, }; use crate::error::{ - ConvertColumnSchemaSnafu, DatafusionSnafu, DatatypesSnafu, ExternalSnafu, InvalidRequestSnafu, - SubstraitEncodeLogicalPlanSnafu, TableNotFoundMetaSnafu, TableNotFoundSnafu, UnexpectedSnafu, + ConvertColumnSchemaSnafu, DatafusionSnafu, ExternalSnafu, InvalidQuerySnafu, + SubstraitEncodeLogicalPlanSnafu, UnexpectedSnafu, }; use crate::metrics::{ METRIC_FLOW_BATCHING_ENGINE_QUERY_TIME, METRIC_FLOW_BATCHING_ENGINE_SLOW_QUERY, @@ -73,7 +73,7 @@ pub struct TaskConfig { pub expire_after: Option, sink_table_name: [String; 3], pub source_table_names: HashSet<[String; 3]>, - table_meta: TableMetadataManagerRef, + catalog_manager: CatalogManagerRef, } #[derive(Clone)] @@ -93,7 +93,7 @@ impl BatchingTask { sink_table_name: [String; 3], source_table_names: Vec<[String; 3]>, query_ctx: QueryContextRef, - table_meta: TableMetadataManagerRef, + catalog_manager: CatalogManagerRef, shutdown_rx: oneshot::Receiver<()>, ) -> Self { Self { @@ -105,12 +105,42 @@ impl BatchingTask { expire_after, sink_table_name, source_table_names: source_table_names.into_iter().collect(), - table_meta, + catalog_manager, }), state: Arc::new(RwLock::new(TaskState::new(query_ctx, shutdown_rx))), } } + /// mark time window range (now - expire_after, now) as dirty (or (0, now) if expire_after not set) + /// + /// useful for flush_flow to flush dirty time windows range + pub fn mark_all_windows_as_dirty(&self) -> Result<(), Error> { + let now = SystemTime::now(); + let now = Timestamp::new_second( + now.duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs() as _, + ); + let lower_bound = self + .config + .expire_after + .map(|e| now.sub_duration(Duration::from_secs(e as _))) + .transpose() + .map_err(BoxedError::new) + .context(ExternalSnafu)? + .unwrap_or(Timestamp::new_second(0)); + debug!( + "Flow {} mark range ({:?}, {:?}) as dirty", + self.config.flow_id, lower_bound, now + ); + self.state + .write() + .unwrap() + .dirty_time_windows + .add_window(lower_bound, Some(now)); + Ok(()) + } + /// Test execute, for check syntax or such pub async fn check_execute( &self, @@ -148,13 +178,8 @@ impl BatchingTask { async fn is_table_exist(&self, table_name: &[String; 3]) -> Result { self.config - .table_meta - .table_name_manager() - .exists(TableNameKey { - catalog: &table_name[0], - schema: &table_name[1], - table: &table_name[2], - }) + .catalog_manager + .table_exists(&table_name[0], &table_name[1], &table_name[2], None) .await .map_err(BoxedError::new) .context(ExternalSnafu) @@ -166,8 +191,10 @@ impl BatchingTask { frontend_client: &Arc, ) -> Result, Error> { if let Some(new_query) = self.gen_insert_plan(engine).await? { + debug!("Generate new query: {:#?}", new_query); self.execute_logical_plan(frontend_client, &new_query).await } else { + debug!("Generate no query"); Ok(None) } } @@ -176,67 +203,35 @@ impl BatchingTask { &self, engine: &QueryEngineRef, ) -> Result, Error> { - let full_table_name = self.config.sink_table_name.clone().join("."); - - let table_id = self - .config - .table_meta - .table_name_manager() - .get(common_meta::key::table_name::TableNameKey::new( - &self.config.sink_table_name[0], - &self.config.sink_table_name[1], - &self.config.sink_table_name[2], - )) - .await - .with_context(|_| TableNotFoundMetaSnafu { - msg: full_table_name.clone(), - })? - .map(|t| t.table_id()) - .with_context(|| TableNotFoundSnafu { - name: full_table_name.clone(), - })?; - - let table = self - .config - .table_meta - .table_info_manager() - .get(table_id) - .await - .with_context(|_| TableNotFoundMetaSnafu { - msg: full_table_name.clone(), - })? - .with_context(|| TableNotFoundSnafu { - name: full_table_name.clone(), - })? - .into_inner(); - - let schema: datatypes::schema::Schema = table - .table_info - .meta - .schema - .clone() - .try_into() - .with_context(|_| DatatypesSnafu { - extra: format!( - "Failed to convert schema from raw schema, raw_schema={:?}", - table.table_info.meta.schema - ), - })?; - - let df_schema = Arc::new(schema.arrow_schema().clone().try_into().with_context(|_| { - DatafusionSnafu { - context: format!( - "Failed to convert arrow schema to datafusion schema, arrow_schema={:?}", - schema.arrow_schema() - ), - } - })?); + let (table, df_schema) = get_table_info_df_schema( + self.config.catalog_manager.clone(), + self.config.sink_table_name.clone(), + ) + .await?; let new_query = self - .gen_query_with_time_window(engine.clone(), &table.table_info.meta) + .gen_query_with_time_window(engine.clone(), &table.meta.schema) .await?; let insert_into = if let Some((new_query, _column_cnt)) = new_query { + // first check if all columns in input query exists in sink table + // since insert into ref to names in record batch generate by given query + let table_columns = df_schema + .columns() + .into_iter() + .map(|c| c.name) + .collect::>(); + for column in new_query.schema().columns() { + ensure!( + table_columns.contains(column.name()), + InvalidQuerySnafu { + reason: format!( + "Column {} not found in sink table with columns {:?}", + column, table_columns + ), + } + ); + } // update_at& time index placeholder (if exists) should have default value LogicalPlan::Dml(DmlStatement::new( datafusion_common::TableReference::Full { @@ -251,6 +246,9 @@ impl BatchingTask { } else { return Ok(None); }; + let insert_into = insert_into.recompute_schema().context(DatafusionSnafu { + context: "Failed to recompute schema", + })?; Ok(Some(insert_into)) } @@ -259,14 +257,11 @@ impl BatchingTask { frontend_client: &Arc, expr: CreateTableExpr, ) -> Result<(), Error> { - let db_client = frontend_client.get_database_client().await?; - db_client - .database - .create(expr.clone()) - .await - .with_context(|_| InvalidRequestSnafu { - context: format!("Failed to create table with expr: {:?}", expr), - })?; + let catalog = &self.config.sink_table_name[0]; + let schema = &self.config.sink_table_name[1]; + frontend_client + .create(expr.clone(), catalog, schema) + .await?; Ok(()) } @@ -277,27 +272,78 @@ impl BatchingTask { ) -> Result, Error> { let instant = Instant::now(); let flow_id = self.config.flow_id; - let db_client = frontend_client.get_database_client().await?; - let peer_addr = db_client.peer.addr; + debug!( - "Executing flow {flow_id}(expire_after={:?} secs) on {:?} with query {}", - self.config.expire_after, peer_addr, &plan + "Executing flow {flow_id}(expire_after={:?} secs) with query {}", + self.config.expire_after, &plan ); - let timer = METRIC_FLOW_BATCHING_ENGINE_QUERY_TIME - .with_label_values(&[flow_id.to_string().as_str()]) - .start_timer(); + let catalog = &self.config.sink_table_name[0]; + let schema = &self.config.sink_table_name[1]; - let message = DFLogicalSubstraitConvertor {} - .encode(plan, DefaultSerializer) - .context(SubstraitEncodeLogicalPlanSnafu)?; + // fix all table ref by make it fully qualified, i.e. "table_name" => "catalog_name.schema_name.table_name" + let fixed_plan = plan + .clone() + .transform_down_with_subqueries(|p| { + if let LogicalPlan::TableScan(mut table_scan) = p { + let resolved = table_scan.table_name.resolve(catalog, schema); + table_scan.table_name = resolved.into(); + Ok(Transformed::yes(LogicalPlan::TableScan(table_scan))) + } else { + Ok(Transformed::no(p)) + } + }) + .with_context(|_| DatafusionSnafu { + context: format!("Failed to fix table ref in logical plan, plan={:?}", plan), + })? + .data; - let req = api::v1::greptime_request::Request::Query(api::v1::QueryRequest { - query: Some(api::v1::query_request::Query::LogicalPlan(message.to_vec())), - }); + let expanded_plan = CountWildcardRule::new() + .analyze(fixed_plan.clone(), &Default::default()) + .with_context(|_| DatafusionSnafu { + context: format!( + "Failed to expand wildcard in logical plan, plan={:?}", + fixed_plan + ), + })?; - let res = db_client.database.handle(req).await; - drop(timer); + let plan = expanded_plan; + let mut peer_desc = None; + + let res = { + let _timer = METRIC_FLOW_BATCHING_ENGINE_QUERY_TIME + .with_label_values(&[flow_id.to_string().as_str()]) + .start_timer(); + + // hack and special handling the insert logical plan + let req = if let Some((insert_to, insert_plan)) = + breakup_insert_plan(&plan, catalog, schema) + { + let message = DFLogicalSubstraitConvertor {} + .encode(&insert_plan, DefaultSerializer) + .context(SubstraitEncodeLogicalPlanSnafu)?; + api::v1::greptime_request::Request::Query(api::v1::QueryRequest { + query: Some(api::v1::query_request::Query::InsertIntoPlan( + api::v1::InsertIntoPlan { + table_name: Some(insert_to), + logical_plan: message.to_vec(), + }, + )), + }) + } else { + let message = DFLogicalSubstraitConvertor {} + .encode(&plan, DefaultSerializer) + .context(SubstraitEncodeLogicalPlanSnafu)?; + + api::v1::greptime_request::Request::Query(api::v1::QueryRequest { + query: Some(api::v1::query_request::Query::LogicalPlan(message.to_vec())), + }) + }; + + frontend_client + .handle(req, catalog, schema, &mut peer_desc) + .await + }; let elapsed = instant.elapsed(); if let Ok(affected_rows) = &res { @@ -307,19 +353,23 @@ impl BatchingTask { ); } else if let Err(err) = &res { warn!( - "Failed to execute Flow {flow_id} on frontend {}, result: {err:?}, elapsed: {:?} with query: {}", - peer_addr, elapsed, &plan + "Failed to execute Flow {flow_id} on frontend {:?}, result: {err:?}, elapsed: {:?} with query: {}", + peer_desc, elapsed, &plan ); } // record slow query if elapsed >= SLOW_QUERY_THRESHOLD { warn!( - "Flow {flow_id} on frontend {} executed for {:?} before complete, query: {}", - peer_addr, elapsed, &plan + "Flow {flow_id} on frontend {:?} executed for {:?} before complete, query: {}", + peer_desc, elapsed, &plan ); METRIC_FLOW_BATCHING_ENGINE_SLOW_QUERY - .with_label_values(&[flow_id.to_string().as_str(), &plan.to_string(), &peer_addr]) + .with_label_values(&[ + flow_id.to_string().as_str(), + &plan.to_string(), + &peer_desc.unwrap_or_default().to_string(), + ]) .observe(elapsed.as_secs_f64()); } @@ -328,12 +378,7 @@ impl BatchingTask { .unwrap() .after_query_exec(elapsed, res.is_ok()); - let res = res.context(InvalidRequestSnafu { - context: format!( - "Failed to execute query for flow={}: \'{}\'", - self.config.flow_id, &plan - ), - })?; + let res = res?; Ok(Some((res, elapsed))) } @@ -372,7 +417,10 @@ impl BatchingTask { } Err(TryRecvError::Empty) => (), } - state.get_next_start_query_time(Some(DEFAULT_BATCHING_ENGINE_QUERY_TIMEOUT)) + state.get_next_start_query_time( + self.config.flow_id, + Some(DEFAULT_BATCHING_ENGINE_QUERY_TIMEOUT), + ) }; tokio::time::sleep_until(sleep_until).await; } @@ -386,14 +434,18 @@ impl BatchingTask { continue; } // TODO(discord9): this error should have better place to go, but for now just print error, also more context is needed - Err(err) => match new_query { - Some(query) => { - common_telemetry::error!(err; "Failed to execute query for flow={} with query: {query}", self.config.flow_id) + Err(err) => { + match new_query { + Some(query) => { + common_telemetry::error!(err; "Failed to execute query for flow={} with query: {query}", self.config.flow_id) + } + None => { + common_telemetry::error!(err; "Failed to generate query for flow={}", self.config.flow_id) + } } - None => { - common_telemetry::error!(err; "Failed to generate query for flow={}", self.config.flow_id) - } - }, + // also sleep for a little while before try again to prevent flooding logs + tokio::time::sleep(MIN_REFRESH_DURATION).await; + } } } } @@ -418,7 +470,7 @@ impl BatchingTask { async fn gen_query_with_time_window( &self, engine: QueryEngineRef, - sink_table_meta: &RawTableMeta, + sink_table_schema: &Arc, ) -> Result, Error> { let query_ctx = self.state.read().unwrap().query_ctx.clone(); let start = SystemTime::now(); @@ -477,9 +529,11 @@ impl BatchingTask { debug!( "Flow id = {:?}, can't get window size: precise_lower_bound={expire_time_window_bound:?}, using the same query", self.config.flow_id ); + // clean dirty time window too, this could be from create flow's check_execute + self.state.write().unwrap().dirty_time_windows.clean(); let mut add_auto_column = - AddAutoColumnRewriter::new(sink_table_meta.schema.clone()); + AddAutoColumnRewriter::new(sink_table_schema.clone()); let plan = self .config .plan @@ -515,8 +569,10 @@ impl BatchingTask { return Ok(None); }; + // TODO(discord9): add auto column or not? This might break compatibility for auto created sink table before this, but that's ok right? + let mut add_filter = AddFilterRewriter::new(expr); - let mut add_auto_column = AddAutoColumnRewriter::new(sink_table_meta.schema.clone()); + let mut add_auto_column = AddAutoColumnRewriter::new(sink_table_schema.clone()); // make a not optimized plan for clearer unparse let plan = sql_to_df_plan(query_ctx.clone(), engine.clone(), &self.config.query, false) .await?; @@ -534,7 +590,7 @@ impl BatchingTask { } // auto created table have a auto added column `update_at`, and optional have a `AUTO_CREATED_PLACEHOLDER_TS_COL` column for time index placeholder if no timestamp column is specified -// TODO(discord9): unit test +// TODO(discord9): for now no default value is set for auto added column for compatibility reason with streaming mode, but this might change in favor of simpler code? fn create_table_with_expr( plan: &LogicalPlan, sink_table_name: &[String; 3], @@ -558,11 +614,7 @@ fn create_table_with_expr( AUTO_CREATED_UPDATE_AT_TS_COL, ConcreteDataType::timestamp_millisecond_datatype(), true, - ) - .with_default_constraint(Some(ColumnDefaultConstraint::Function(NOW_FN.to_string()))) - .context(DatatypesSnafu { - extra: "Failed to build column `update_at TimestampMillisecond default now()`", - })?; + ); column_schemas.push(update_at_schema); let time_index = if let Some(time_index) = first_time_stamp { @@ -574,16 +626,7 @@ fn create_table_with_expr( ConcreteDataType::timestamp_millisecond_datatype(), false, ) - .with_time_index(true) - .with_default_constraint(Some(ColumnDefaultConstraint::Value(Value::Timestamp( - Timestamp::new_millisecond(0), - )))) - .context(DatatypesSnafu { - extra: format!( - "Failed to build column `{} TimestampMillisecond TIME INDEX default 0`", - AUTO_CREATED_PLACEHOLDER_TS_COL - ), - })?, + .with_time_index(true), ); AUTO_CREATED_PLACEHOLDER_TS_COL.to_string() }; @@ -675,20 +718,14 @@ mod test { AUTO_CREATED_UPDATE_AT_TS_COL, ConcreteDataType::timestamp_millisecond_datatype(), true, - ) - .with_default_constraint(Some(ColumnDefaultConstraint::Function(NOW_FN.to_string()))) - .unwrap(); + ); let ts_placeholder_schema = ColumnSchema::new( AUTO_CREATED_PLACEHOLDER_TS_COL, ConcreteDataType::timestamp_millisecond_datatype(), false, ) - .with_time_index(true) - .with_default_constraint(Some(ColumnDefaultConstraint::Value(Value::Timestamp( - Timestamp::new_millisecond(0), - )))) - .unwrap(); + .with_time_index(true); let testcases = vec![ TestCase { diff --git a/src/flow/src/batching_mode/time_window.rs b/src/flow/src/batching_mode/time_window.rs index f154847499..e6a0d6ad8c 100644 --- a/src/flow/src/batching_mode/time_window.rs +++ b/src/flow/src/batching_mode/time_window.rs @@ -72,6 +72,17 @@ pub struct TimeWindowExpr { df_schema: DFSchema, } +impl std::fmt::Display for TimeWindowExpr { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TimeWindowExpr") + .field("phy_expr", &self.phy_expr.to_string()) + .field("column_name", &self.column_name) + .field("logical_expr", &self.logical_expr.to_string()) + .field("df_schema", &self.df_schema) + .finish() + } +} + impl TimeWindowExpr { pub fn from_expr( expr: &Expr, @@ -256,7 +267,7 @@ fn columnar_to_ts_vector(columnar: &ColumnarValue) -> Result Result<(Arc, Arc), Error> { + let full_table_name = table_name.clone().join("."); + let table = catalog_mr + .table(&table_name[0], &table_name[1], &table_name[2], None) + .await + .map_err(BoxedError::new) + .context(ExternalSnafu)? + .context(TableNotFoundSnafu { + name: &full_table_name, + })?; + let table_info = table.table_info().clone(); + + let schema = table_info.meta.schema.clone(); + + let df_schema: Arc = Arc::new( + schema + .arrow_schema() + .clone() + .try_into() + .with_context(|_| DatafusionSnafu { + context: format!( + "Failed to convert arrow schema to datafusion schema, arrow_schema={:?}", + schema.arrow_schema() + ), + })?, + ); + Ok((table_info, df_schema)) +} /// Convert sql to datafusion logical plan pub async fn sql_to_df_plan( @@ -164,14 +198,16 @@ impl TreeNodeVisitor<'_> for FindGroupByFinalName { /// (which doesn't necessary need to have exact name just need to be a extra timestamp column) /// and `__ts_placeholder`(this column need to have exact this name and be a timestamp) /// with values like `now()` and `0` +/// +/// it also give existing columns alias to column in sink table if needed #[derive(Debug)] pub struct AddAutoColumnRewriter { - pub schema: RawSchema, + pub schema: SchemaRef, pub is_rewritten: bool, } impl AddAutoColumnRewriter { - pub fn new(schema: RawSchema) -> Self { + pub fn new(schema: SchemaRef) -> Self { Self { schema, is_rewritten: false, @@ -181,37 +217,97 @@ impl AddAutoColumnRewriter { impl TreeNodeRewriter for AddAutoColumnRewriter { type Node = LogicalPlan; - fn f_down(&mut self, node: Self::Node) -> DfResult> { + fn f_down(&mut self, mut node: Self::Node) -> DfResult> { if self.is_rewritten { return Ok(Transformed::no(node)); } - // if is distinct all, go one level down - if let LogicalPlan::Distinct(Distinct::All(_)) = node { - return Ok(Transformed::no(node)); + // if is distinct all, wrap it in a projection + if let LogicalPlan::Distinct(Distinct::All(_)) = &node { + let mut exprs = vec![]; + + for field in node.schema().fields().iter() { + exprs.push(Expr::Column(datafusion::common::Column::new_unqualified( + field.name(), + ))); + } + + let projection = + LogicalPlan::Projection(Projection::try_new(exprs, Arc::new(node.clone()))?); + + node = projection; + } + // handle table_scan by wrap it in a projection + else if let LogicalPlan::TableScan(table_scan) = node { + let mut exprs = vec![]; + + for field in table_scan.projected_schema.fields().iter() { + exprs.push(Expr::Column(datafusion::common::Column::new( + Some(table_scan.table_name.clone()), + field.name(), + ))); + } + + let projection = LogicalPlan::Projection(Projection::try_new( + exprs, + Arc::new(LogicalPlan::TableScan(table_scan)), + )?); + + node = projection; } - // FIXME(discord9): just read plan.expr and do stuffs - let mut exprs = node.expressions(); + // only do rewrite if found the outermost projection + let mut exprs = if let LogicalPlan::Projection(project) = &node { + project.expr.clone() + } else { + return Ok(Transformed::no(node)); + }; + + let all_names = self + .schema + .column_schemas() + .iter() + .map(|c| c.name.clone()) + .collect::>(); + // first match by position + for (idx, expr) in exprs.iter_mut().enumerate() { + if !all_names.contains(&expr.qualified_name().1) { + if let Some(col_name) = self + .schema + .column_schemas() + .get(idx) + .map(|c| c.name.clone()) + { + // if the data type mismatched, later check_execute will error out + // hence no need to check it here, beside, optimize pass might be able to cast it + // so checking here is not necessary + *expr = expr.clone().alias(col_name); + } + } + } // add columns if have different column count let query_col_cnt = exprs.len(); - let table_col_cnt = self.schema.column_schemas.len(); - info!("query_col_cnt={query_col_cnt}, table_col_cnt={table_col_cnt}"); + let table_col_cnt = self.schema.column_schemas().len(); + debug!("query_col_cnt={query_col_cnt}, table_col_cnt={table_col_cnt}"); + + let placeholder_ts_expr = + datafusion::logical_expr::lit(ScalarValue::TimestampMillisecond(Some(0), None)) + .alias(AUTO_CREATED_PLACEHOLDER_TS_COL); + if query_col_cnt == table_col_cnt { - self.is_rewritten = true; - return Ok(Transformed::no(node)); + // still need to add alias, see below } else if query_col_cnt + 1 == table_col_cnt { - let last_col_schema = self.schema.column_schemas.last().unwrap(); + let last_col_schema = self.schema.column_schemas().last().unwrap(); // if time index column is auto created add it if last_col_schema.name == AUTO_CREATED_PLACEHOLDER_TS_COL - && self.schema.timestamp_index == Some(table_col_cnt - 1) + && self.schema.timestamp_index() == Some(table_col_cnt - 1) { - exprs.push(datafusion::logical_expr::lit(0)); + exprs.push(placeholder_ts_expr); } else if last_col_schema.data_type.is_timestamp() { // is the update at column - exprs.push(datafusion::prelude::now()); + exprs.push(datafusion::prelude::now().alias(&last_col_schema.name)); } else { // helpful error message return Err(DataFusionError::Plan(format!( @@ -221,11 +317,11 @@ impl TreeNodeRewriter for AddAutoColumnRewriter { ))); } } else if query_col_cnt + 2 == table_col_cnt { - let mut col_iter = self.schema.column_schemas.iter().rev(); + let mut col_iter = self.schema.column_schemas().iter().rev(); let last_col_schema = col_iter.next().unwrap(); let second_last_col_schema = col_iter.next().unwrap(); if second_last_col_schema.data_type.is_timestamp() { - exprs.push(datafusion::prelude::now()); + exprs.push(datafusion::prelude::now().alias(&second_last_col_schema.name)); } else { return Err(DataFusionError::Plan(format!( "Expect the second last column in the table to be timestamp column, found column {} with type {:?}", @@ -235,9 +331,9 @@ impl TreeNodeRewriter for AddAutoColumnRewriter { } if last_col_schema.name == AUTO_CREATED_PLACEHOLDER_TS_COL - && self.schema.timestamp_index == Some(table_col_cnt - 1) + && self.schema.timestamp_index() == Some(table_col_cnt - 1) { - exprs.push(datafusion::logical_expr::lit(0)); + exprs.push(placeholder_ts_expr); } else { return Err(DataFusionError::Plan(format!( "Expect timestamp column {}, found {:?}", @@ -246,8 +342,8 @@ impl TreeNodeRewriter for AddAutoColumnRewriter { } } else { return Err(DataFusionError::Plan(format!( - "Expect table have 0,1 or 2 columns more than query columns, found {} query columns {:?}, {} table columns {:?}", - query_col_cnt, node.expressions(), table_col_cnt, self.schema.column_schemas + "Expect table have 0,1 or 2 columns more than query columns, found {} query columns {:?}, {} table columns {:?} at node {:?}", + query_col_cnt, exprs, table_col_cnt, self.schema.column_schemas(), node ))); } @@ -255,9 +351,12 @@ impl TreeNodeRewriter for AddAutoColumnRewriter { let new_plan = node.with_new_exprs(exprs, node.inputs().into_iter().cloned().collect())?; Ok(Transformed::yes(new_plan)) } -} -// TODO(discord9): a method to found out the precise time window + /// We might add new columns, so we need to recompute the schema + fn f_up(&mut self, node: Self::Node) -> DfResult> { + node.recompute_schema().map(Transformed::yes) + } +} /// Find out the `Filter` Node corresponding to innermost(deepest) `WHERE` and add a new filter expr to it #[derive(Debug)] @@ -301,9 +400,11 @@ impl TreeNodeRewriter for AddFilterRewriter { #[cfg(test)] mod test { + use std::sync::Arc; + use datafusion_common::tree_node::TreeNode as _; use datatypes::prelude::ConcreteDataType; - use datatypes::schema::ColumnSchema; + use datatypes::schema::{ColumnSchema, Schema}; use pretty_assertions::assert_eq; use session::context::QueryContext; @@ -386,7 +487,7 @@ mod test { // add update_at ( "SELECT number FROM numbers_with_ts", - Ok("SELECT numbers_with_ts.number, now() FROM numbers_with_ts"), + Ok("SELECT numbers_with_ts.number, now() AS ts FROM numbers_with_ts"), vec![ ColumnSchema::new("number", ConcreteDataType::int32_datatype(), true), ColumnSchema::new( @@ -400,7 +501,7 @@ mod test { // add ts placeholder ( "SELECT number FROM numbers_with_ts", - Ok("SELECT numbers_with_ts.number, 0 FROM numbers_with_ts"), + Ok("SELECT numbers_with_ts.number, CAST('1970-01-01 00:00:00' AS TIMESTAMP) AS __ts_placeholder FROM numbers_with_ts"), vec![ ColumnSchema::new("number", ConcreteDataType::int32_datatype(), true), ColumnSchema::new( @@ -428,7 +529,7 @@ mod test { // add update_at and ts placeholder ( "SELECT number FROM numbers_with_ts", - Ok("SELECT numbers_with_ts.number, now(), 0 FROM numbers_with_ts"), + Ok("SELECT numbers_with_ts.number, now() AS update_at, CAST('1970-01-01 00:00:00' AS TIMESTAMP) AS __ts_placeholder FROM numbers_with_ts"), vec![ ColumnSchema::new("number", ConcreteDataType::int32_datatype(), true), ColumnSchema::new( @@ -447,7 +548,7 @@ mod test { // add ts placeholder ( "SELECT number, ts FROM numbers_with_ts", - Ok("SELECT numbers_with_ts.number, numbers_with_ts.ts, 0 FROM numbers_with_ts"), + Ok("SELECT numbers_with_ts.number, numbers_with_ts.ts AS update_at, CAST('1970-01-01 00:00:00' AS TIMESTAMP) AS __ts_placeholder FROM numbers_with_ts"), vec![ ColumnSchema::new("number", ConcreteDataType::int32_datatype(), true), ColumnSchema::new( @@ -466,7 +567,7 @@ mod test { // add update_at after time index column ( "SELECT number, ts FROM numbers_with_ts", - Ok("SELECT numbers_with_ts.number, numbers_with_ts.ts, now() FROM numbers_with_ts"), + Ok("SELECT numbers_with_ts.number, numbers_with_ts.ts, now() AS update_atat FROM numbers_with_ts"), vec![ ColumnSchema::new("number", ConcreteDataType::int32_datatype(), true), ColumnSchema::new( @@ -528,8 +629,8 @@ mod test { let query_engine = create_test_query_engine(); let ctx = QueryContext::arc(); for (before, after, column_schemas) in testcases { - let raw_schema = RawSchema::new(column_schemas); - let mut add_auto_column_rewriter = AddAutoColumnRewriter::new(raw_schema); + let schema = Arc::new(Schema::new(column_schemas)); + let mut add_auto_column_rewriter = AddAutoColumnRewriter::new(schema); let plan = sql_to_df_plan(ctx.clone(), query_engine.clone(), before, false) .await diff --git a/src/flow/src/engine.rs b/src/flow/src/engine.rs index 33da5252d7..1338e893f3 100644 --- a/src/flow/src/engine.rs +++ b/src/flow/src/engine.rs @@ -49,6 +49,8 @@ pub trait FlowEngine { async fn flush_flow(&self, flow_id: FlowId) -> Result; /// Check if the flow exists async fn flow_exist(&self, flow_id: FlowId) -> Result; + /// List all flows + async fn list_flows(&self) -> Result, Error>; /// Handle the insert requests for the flow async fn handle_flow_inserts( &self, diff --git a/src/flow/src/error.rs b/src/flow/src/error.rs index 1741f8cb1b..904a0a8fa7 100644 --- a/src/flow/src/error.rs +++ b/src/flow/src/error.rs @@ -149,6 +149,13 @@ pub enum Error { location: Location, }, + #[snafu(display("Unsupported: {reason}"))] + Unsupported { + reason: String, + #[snafu(implicit)] + location: Location, + }, + #[snafu(display("Unsupported temporal filter: {reason}"))] UnsupportedTemporalFilter { reason: String, @@ -189,6 +196,25 @@ pub enum Error { location: Location, }, + #[snafu(display("Illegal check task state: {reason}"))] + IllegalCheckTaskState { + reason: String, + #[snafu(implicit)] + location: Location, + }, + + #[snafu(display( + "Failed to sync with check task for flow {} with allow_drop={}", + flow_id, + allow_drop + ))] + SyncCheckTask { + flow_id: FlowId, + allow_drop: bool, + #[snafu(implicit)] + location: Location, + }, + #[snafu(display("Failed to start server"))] StartServer { #[snafu(implicit)] @@ -280,10 +306,12 @@ impl ErrorExt for Error { Self::CreateFlow { .. } | Self::Arrow { .. } | Self::Time { .. } => { StatusCode::EngineExecuteQuery } - Self::Unexpected { .. } => StatusCode::Unexpected, - Self::NotImplemented { .. } | Self::UnsupportedTemporalFilter { .. } => { - StatusCode::Unsupported - } + Self::Unexpected { .. } + | Self::SyncCheckTask { .. } + | Self::IllegalCheckTaskState { .. } => StatusCode::Unexpected, + Self::NotImplemented { .. } + | Self::UnsupportedTemporalFilter { .. } + | Self::Unsupported { .. } => StatusCode::Unsupported, Self::External { source, .. } => source.status_code(), Self::Internal { .. } | Self::CacheRequired { .. } => StatusCode::Internal, Self::StartServer { source, .. } | Self::ShutdownServer { source, .. } => { diff --git a/src/flow/src/lib.rs b/src/flow/src/lib.rs index 5dc3c67491..8caa659f07 100644 --- a/src/flow/src/lib.rs +++ b/src/flow/src/lib.rs @@ -43,8 +43,8 @@ mod utils; #[cfg(test)] mod test_utils; -pub use adapter::{FlowConfig, FlowWorkerManager, FlowWorkerManagerRef, FlownodeOptions}; -pub use batching_mode::frontend_client::FrontendClient; +pub use adapter::{FlowConfig, FlowStreamingEngineRef, FlownodeOptions, StreamingEngine}; +pub use batching_mode::frontend_client::{FrontendClient, GrpcQueryHandlerWithBoxedError}; pub(crate) use engine::{CreateFlowArgs, FlowId, TableName}; pub use error::{Error, Result}; pub use server::{ diff --git a/src/flow/src/server.rs b/src/flow/src/server.rs index 99f7a7c089..46c0d93301 100644 --- a/src/flow/src/server.rs +++ b/src/flow/src/server.rs @@ -29,6 +29,7 @@ use common_meta::key::TableMetadataManagerRef; use common_meta::kv_backend::KvBackendRef; use common_meta::node_manager::{Flownode, NodeManagerRef}; use common_query::Output; +use common_runtime::JoinHandle; use common_telemetry::tracing::info; use futures::{FutureExt, TryStreamExt}; use greptime_proto::v1::flow::{flow_server, FlowRequest, FlowResponse, InsertRequests}; @@ -50,7 +51,10 @@ use tonic::codec::CompressionEncoding; use tonic::transport::server::TcpIncoming; use tonic::{Request, Response, Status}; -use crate::adapter::{create_worker, FlowWorkerManagerRef}; +use crate::adapter::flownode_impl::{FlowDualEngine, FlowDualEngineRef}; +use crate::adapter::{create_worker, FlowStreamingEngineRef}; +use crate::batching_mode::engine::BatchingEngine; +use crate::engine::FlowEngine; use crate::error::{ to_status_with_last_err, CacheRequiredSnafu, CreateFlowSnafu, ExternalSnafu, FlowNotFoundSnafu, ListFlowsSnafu, ParseAddrSnafu, ShutdownServerSnafu, StartServerSnafu, UnexpectedSnafu, @@ -59,19 +63,20 @@ use crate::heartbeat::HeartbeatTask; use crate::metrics::{METRIC_FLOW_PROCESSING_TIME, METRIC_FLOW_ROWS}; use crate::transform::register_function_to_query_engine; use crate::utils::{SizeReportSender, StateReportHandler}; -use crate::{CreateFlowArgs, Error, FlowWorkerManager, FlownodeOptions, FrontendClient}; +use crate::{CreateFlowArgs, Error, FlownodeOptions, FrontendClient, StreamingEngine}; pub const FLOW_NODE_SERVER_NAME: &str = "FLOW_NODE_SERVER"; /// wrapping flow node manager to avoid orphan rule with Arc<...> #[derive(Clone)] pub struct FlowService { - /// TODO(discord9): replace with dual engine - pub manager: FlowWorkerManagerRef, + pub dual_engine: FlowDualEngineRef, } impl FlowService { - pub fn new(manager: FlowWorkerManagerRef) -> Self { - Self { manager } + pub fn new(manager: FlowDualEngineRef) -> Self { + Self { + dual_engine: manager, + } } } @@ -86,7 +91,7 @@ impl flow_server::Flow for FlowService { .start_timer(); let request = request.into_inner(); - self.manager + self.dual_engine .handle(request) .await .map_err(|err| { @@ -126,7 +131,7 @@ impl flow_server::Flow for FlowService { .with_label_values(&["in"]) .inc_by(row_count as u64); - self.manager + self.dual_engine .handle_inserts(request) .await .map(Response::new) @@ -139,11 +144,16 @@ pub struct FlownodeServer { inner: Arc, } +/// FlownodeServerInner is the inner state of FlownodeServer, +/// this struct mostly useful for construct/start and stop the +/// flow node server struct FlownodeServerInner { /// worker shutdown signal, not to be confused with server_shutdown_tx worker_shutdown_tx: Mutex>, /// server shutdown signal for shutdown grpc server server_shutdown_tx: Mutex>, + /// streaming task handler + streaming_task_handler: Mutex>>, flow_service: FlowService, } @@ -156,16 +166,28 @@ impl FlownodeServer { flow_service, worker_shutdown_tx: Mutex::new(tx), server_shutdown_tx: Mutex::new(server_tx), + streaming_task_handler: Mutex::new(None), }), } } /// Start the background task for streaming computation. async fn start_workers(&self) -> Result<(), Error> { - let manager_ref = self.inner.flow_service.manager.clone(); - let _handle = manager_ref - .clone() + let manager_ref = self.inner.flow_service.dual_engine.clone(); + let handle = manager_ref + .streaming_engine() .run_background(Some(self.inner.worker_shutdown_tx.lock().await.subscribe())); + self.inner + .streaming_task_handler + .lock() + .await + .replace(handle); + + self.inner + .flow_service + .dual_engine + .start_flow_consistent_check_task() + .await?; Ok(()) } @@ -176,6 +198,11 @@ impl FlownodeServer { if tx.send(()).is_err() { info!("Receiver dropped, the flow node server has already shutdown"); } + self.inner + .flow_service + .dual_engine + .stop_flow_consistent_check_task() + .await?; Ok(()) } } @@ -272,8 +299,8 @@ impl FlownodeInstance { &self.flownode_server } - pub fn flow_worker_manager(&self) -> FlowWorkerManagerRef { - self.flownode_server.inner.flow_service.manager.clone() + pub fn flow_engine(&self) -> FlowDualEngineRef { + self.flownode_server.inner.flow_service.dual_engine.clone() } pub fn setup_services(&mut self, services: ServerHandlers) { @@ -342,12 +369,21 @@ impl FlownodeBuilder { self.build_manager(query_engine_factory.query_engine()) .await?, ); + let batching = Arc::new(BatchingEngine::new( + self.frontend_client.clone(), + query_engine_factory.query_engine(), + self.flow_metadata_manager.clone(), + self.table_meta.clone(), + self.catalog_manager.clone(), + )); + let dual = FlowDualEngine::new( + manager.clone(), + batching, + self.flow_metadata_manager.clone(), + self.catalog_manager.clone(), + ); - if let Err(err) = self.recover_flows(&manager).await { - common_telemetry::error!(err; "Failed to recover flows"); - } - - let server = FlownodeServer::new(FlowService::new(manager.clone())); + let server = FlownodeServer::new(FlowService::new(Arc::new(dual))); let heartbeat_task = self.heartbeat_task; @@ -364,7 +400,7 @@ impl FlownodeBuilder { /// or recover all existing flow tasks if in standalone mode(nodeid is None) /// /// TODO(discord9): persistent flow tasks with internal state - async fn recover_flows(&self, manager: &FlowWorkerManagerRef) -> Result { + async fn recover_flows(&self, manager: &FlowDualEngine) -> Result { let nodeid = self.opts.node_id; let to_be_recovered: Vec<_> = if let Some(nodeid) = nodeid { let to_be_recover = self @@ -451,7 +487,7 @@ impl FlownodeBuilder { }), }; manager - .create_flow_inner(args) + .create_flow(args) .await .map_err(BoxedError::new) .with_context(|_| CreateFlowSnafu { @@ -467,7 +503,7 @@ impl FlownodeBuilder { async fn build_manager( &mut self, query_engine: Arc, - ) -> Result { + ) -> Result { let table_meta = self.table_meta.clone(); register_function_to_query_engine(&query_engine); @@ -476,7 +512,7 @@ impl FlownodeBuilder { let node_id = self.opts.node_id.map(|id| id as u32); - let mut man = FlowWorkerManager::new(node_id, query_engine, table_meta); + let mut man = StreamingEngine::new(node_id, query_engine, table_meta); for worker_id in 0..num_workers { let (tx, rx) = oneshot::channel(); @@ -558,6 +594,10 @@ impl<'a> FlownodeServiceBuilder<'a> { } } +/// Basically a tiny frontend that communicates with datanode, different from [`FrontendClient`] which +/// connect to a real frontend instead, this is used for flow's streaming engine. And is for simple query. +/// +/// For heavy query use [`FrontendClient`] which offload computation to frontend, lifting the load from flownode #[derive(Clone)] pub struct FrontendInvoker { inserter: Arc, @@ -579,7 +619,7 @@ impl FrontendInvoker { } pub async fn build_from( - flow_worker_manager: FlowWorkerManagerRef, + flow_streaming_engine: FlowStreamingEngineRef, catalog_manager: CatalogManagerRef, kv_backend: KvBackendRef, layered_cache_registry: LayeredCacheRegistryRef, @@ -614,7 +654,7 @@ impl FrontendInvoker { node_manager.clone(), )); - let query_engine = flow_worker_manager.query_engine.clone(); + let query_engine = flow_streaming_engine.query_engine.clone(); let statement_executor = Arc::new(StatementExecutor::new( catalog_manager.clone(), diff --git a/src/frontend/Cargo.toml b/src/frontend/Cargo.toml index 54017bc8d6..153fbcdd83 100644 --- a/src/frontend/Cargo.toml +++ b/src/frontend/Cargo.toml @@ -15,6 +15,7 @@ api.workspace = true arc-swap = "1.0" async-trait.workspace = true auth.workspace = true +bytes.workspace = true cache.workspace = true catalog.workspace = true client.workspace = true diff --git a/src/frontend/src/error.rs b/src/frontend/src/error.rs index 99edbdbc62..7880656f51 100644 --- a/src/frontend/src/error.rs +++ b/src/frontend/src/error.rs @@ -19,6 +19,8 @@ use common_error::define_into_tonic_status; use common_error::ext::{BoxedError, ErrorExt}; use common_error::status_code::StatusCode; use common_macro::stack_trace_debug; +use common_query::error::datafusion_status_code; +use datafusion::error::DataFusionError; use session::ReadPreference; use snafu::{Location, Snafu}; use store_api::storage::RegionId; @@ -345,7 +347,15 @@ pub enum Error { SubstraitDecodeLogicalPlan { #[snafu(implicit)] location: Location, - source: substrait::error::Error, + source: common_query::error::Error, + }, + + #[snafu(display("DataFusionError"))] + DataFusion { + #[snafu(source)] + error: DataFusionError, + #[snafu(implicit)] + location: Location, }, } @@ -423,6 +433,8 @@ impl ErrorExt for Error { Error::TableOperation { source, .. } => source.status_code(), Error::InFlightWriteBytesExceeded { .. } => StatusCode::RateLimited, + + Error::DataFusion { error, .. } => datafusion_status_code::(error, None), } } diff --git a/src/frontend/src/instance.rs b/src/frontend/src/instance.rs index 9c94ec326a..1477a2b133 100644 --- a/src/frontend/src/instance.rs +++ b/src/frontend/src/instance.rs @@ -278,7 +278,7 @@ impl SqlQueryHandler for Instance { // plan should be prepared before exec // we'll do check there self.query_engine - .execute(plan, query_ctx) + .execute(plan.clone(), query_ctx) .await .context(ExecLogicalPlanSnafu) } diff --git a/src/frontend/src/instance/grpc.rs b/src/frontend/src/instance/grpc.rs index 915d884d7e..4a83a99d12 100644 --- a/src/frontend/src/instance/grpc.rs +++ b/src/frontend/src/instance/grpc.rs @@ -12,29 +12,33 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; + use api::v1::ddl_request::{Expr as DdlExpr, Expr}; use api::v1::greptime_request::Request; use api::v1::query_request::Query; -use api::v1::{DeleteRequests, DropFlowExpr, InsertRequests, RowDeleteRequests, RowInsertRequests}; +use api::v1::{ + DeleteRequests, DropFlowExpr, InsertIntoPlan, InsertRequests, RowDeleteRequests, + RowInsertRequests, +}; use async_trait::async_trait; use auth::{PermissionChecker, PermissionCheckerRef, PermissionReq}; use common_base::AffectedRows; +use common_query::logical_plan::add_insert_to_logical_plan; use common_query::Output; use common_telemetry::tracing::{self}; -use datafusion::execution::SessionStateBuilder; use query::parser::PromQuery; use servers::interceptor::{GrpcQueryInterceptor, GrpcQueryInterceptorRef}; use servers::query_handler::grpc::{GrpcQueryHandler, RawRecordBatch}; use servers::query_handler::sql::SqlQueryHandler; use session::context::QueryContextRef; use snafu::{ensure, OptionExt, ResultExt}; -use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan}; use table::table_name::TableName; use crate::error::{ - CatalogSnafu, Error, InFlightWriteBytesExceededSnafu, IncompleteGrpcRequestSnafu, - NotSupportedSnafu, PermissionSnafu, Result, SubstraitDecodeLogicalPlanSnafu, - TableNotFoundSnafu, TableOperationSnafu, + CatalogSnafu, DataFusionSnafu, Error, InFlightWriteBytesExceededSnafu, + IncompleteGrpcRequestSnafu, NotSupportedSnafu, PermissionSnafu, PlanStatementSnafu, Result, + SubstraitDecodeLogicalPlanSnafu, TableNotFoundSnafu, TableOperationSnafu, }; use crate::instance::{attach_timer, Instance}; use crate::metrics::{ @@ -91,14 +95,31 @@ impl GrpcQueryHandler for Instance { Query::LogicalPlan(plan) => { // this path is useful internally when flownode needs to execute a logical plan through gRPC interface let timer = GRPC_HANDLE_PLAN_ELAPSED.start_timer(); - let plan = DFLogicalSubstraitConvertor {} - .decode(&*plan, SessionStateBuilder::default().build()) + + // use dummy catalog to provide table + let plan_decoder = self + .query_engine() + .engine_context(ctx.clone()) + .new_plan_decoder() + .context(PlanStatementSnafu)?; + + let dummy_catalog_list = + Arc::new(catalog::table_source::dummy_catalog::DummyCatalogList::new( + self.catalog_manager().clone(), + )); + + let logical_plan = plan_decoder + .decode(bytes::Bytes::from(plan), dummy_catalog_list, true) .await .context(SubstraitDecodeLogicalPlanSnafu)?; - let output = SqlQueryHandler::do_exec_plan(self, plan, ctx.clone()).await?; + let output = + SqlQueryHandler::do_exec_plan(self, logical_plan, ctx.clone()).await?; attach_timer(output, timer) } + Query::InsertIntoPlan(insert) => { + self.handle_insert_plan(insert, ctx.clone()).await? + } Query::PromRangeQuery(promql) => { let timer = GRPC_HANDLE_PROMQL_ELAPSED.start_timer(); let prom_query = PromQuery { @@ -284,6 +305,91 @@ fn fill_catalog_and_schema_from_context(ddl_expr: &mut DdlExpr, ctx: &QueryConte } impl Instance { + async fn handle_insert_plan( + &self, + insert: InsertIntoPlan, + ctx: QueryContextRef, + ) -> Result { + let timer = GRPC_HANDLE_PLAN_ELAPSED.start_timer(); + let table_name = insert.table_name.context(IncompleteGrpcRequestSnafu { + err_msg: "'table_name' is absent in InsertIntoPlan", + })?; + + // use dummy catalog to provide table + let plan_decoder = self + .query_engine() + .engine_context(ctx.clone()) + .new_plan_decoder() + .context(PlanStatementSnafu)?; + + let dummy_catalog_list = + Arc::new(catalog::table_source::dummy_catalog::DummyCatalogList::new( + self.catalog_manager().clone(), + )); + + // no optimize yet since we still need to add stuff + let logical_plan = plan_decoder + .decode( + bytes::Bytes::from(insert.logical_plan), + dummy_catalog_list, + false, + ) + .await + .context(SubstraitDecodeLogicalPlanSnafu)?; + + let table = self + .catalog_manager() + .table( + &table_name.catalog_name, + &table_name.schema_name, + &table_name.table_name, + None, + ) + .await + .context(CatalogSnafu)? + .with_context(|| TableNotFoundSnafu { + table_name: [ + table_name.catalog_name.clone(), + table_name.schema_name.clone(), + table_name.table_name.clone(), + ] + .join("."), + })?; + + let table_info = table.table_info(); + + let df_schema = Arc::new( + table_info + .meta + .schema + .arrow_schema() + .clone() + .try_into() + .context(DataFusionSnafu)?, + ); + + let insert_into = add_insert_to_logical_plan(table_name, df_schema, logical_plan) + .context(SubstraitDecodeLogicalPlanSnafu)?; + + let engine_ctx = self.query_engine().engine_context(ctx.clone()); + let state = engine_ctx.state(); + // Analyze the plan + let analyzed_plan = state + .analyzer() + .execute_and_check(insert_into, state.config_options(), |_, _| {}) + .context(common_query::error::GeneralDataFusionSnafu) + .context(SubstraitDecodeLogicalPlanSnafu)?; + + // Optimize the plan + let optimized_plan = state + .optimize(&analyzed_plan) + .context(common_query::error::GeneralDataFusionSnafu) + .context(SubstraitDecodeLogicalPlanSnafu)?; + + let output = SqlQueryHandler::do_exec_plan(self, optimized_plan, ctx.clone()).await?; + + Ok(attach_timer(output, timer)) + } #[tracing::instrument(skip_all)] pub async fn handle_inserts( &self, diff --git a/src/operator/src/statement/ddl.rs b/src/operator/src/statement/ddl.rs index 8450c73c4d..afae466dc0 100644 --- a/src/operator/src/statement/ddl.rs +++ b/src/operator/src/statement/ddl.rs @@ -26,6 +26,7 @@ use common_catalog::consts::{is_readonly_schema, DEFAULT_CATALOG_NAME, DEFAULT_S use common_catalog::{format_full_flow_name, format_full_table_name}; use common_error::ext::BoxedError; use common_meta::cache_invalidator::Context; +use common_meta::ddl::create_flow::FlowType; use common_meta::ddl::ExecutorContext; use common_meta::instruction::CacheIdent; use common_meta::key::schema_name::{SchemaName, SchemaNameKey}; @@ -38,6 +39,8 @@ use common_meta::rpc::router::{Partition, Partition as MetaPartition}; use common_query::Output; use common_telemetry::{debug, info, tracing}; use common_time::Timezone; +use datafusion_common::tree_node::TreeNodeVisitor; +use datafusion_expr::LogicalPlan; use datatypes::prelude::ConcreteDataType; use datatypes::schema::{RawSchema, Schema}; use datatypes::value::Value; @@ -45,7 +48,7 @@ use lazy_static::lazy_static; use partition::expr::{Operand, PartitionExpr, RestrictedOp}; use partition::multi_dim::MultiDimPartitionRule; use partition::partition::{PartitionBound, PartitionDef}; -use query::parser::QueryStatement; +use query::parser::{QueryLanguageParser, QueryStatement}; use query::plan::extract_and_rewrite_full_table_names; use query::query_engine::DefaultSerializer; use query::sql::create_table_stmt; @@ -69,13 +72,14 @@ use table::table_name::TableName; use table::TableRef; use crate::error::{ - self, AlterExprToRequestSnafu, CatalogSnafu, ColumnDataTypeSnafu, ColumnNotFoundSnafu, - ConvertSchemaSnafu, CreateLogicalTablesSnafu, CreateTableInfoSnafu, DeserializePartitionSnafu, - EmptyDdlExprSnafu, ExtractTableNamesSnafu, FlowNotFoundSnafu, InvalidPartitionRuleSnafu, - InvalidPartitionSnafu, InvalidSqlSnafu, InvalidTableNameSnafu, InvalidViewNameSnafu, - InvalidViewStmtSnafu, ParseSqlValueSnafu, Result, SchemaInUseSnafu, SchemaNotFoundSnafu, - SchemaReadOnlySnafu, SubstraitCodecSnafu, TableAlreadyExistsSnafu, TableMetadataManagerSnafu, - TableNotFoundSnafu, UnrecognizedTableOptionSnafu, ViewAlreadyExistsSnafu, + self, AlterExprToRequestSnafu, BuildDfLogicalPlanSnafu, CatalogSnafu, ColumnDataTypeSnafu, + ColumnNotFoundSnafu, ConvertSchemaSnafu, CreateLogicalTablesSnafu, CreateTableInfoSnafu, + DeserializePartitionSnafu, EmptyDdlExprSnafu, ExternalSnafu, ExtractTableNamesSnafu, + FlowNotFoundSnafu, InvalidPartitionRuleSnafu, InvalidPartitionSnafu, InvalidSqlSnafu, + InvalidTableNameSnafu, InvalidViewNameSnafu, InvalidViewStmtSnafu, ParseSqlValueSnafu, Result, + SchemaInUseSnafu, SchemaNotFoundSnafu, SchemaReadOnlySnafu, SubstraitCodecSnafu, + TableAlreadyExistsSnafu, TableMetadataManagerSnafu, TableNotFoundSnafu, + UnrecognizedTableOptionSnafu, ViewAlreadyExistsSnafu, }; use crate::expr_helper; use crate::statement::show::create_partitions_stmt; @@ -364,6 +368,18 @@ impl StatementExecutor { expr: CreateFlowExpr, query_context: QueryContextRef, ) -> Result { + let flow_type = self + .determine_flow_type(&expr.sql, query_context.clone()) + .await?; + info!("determined flow={} type: {:#?}", expr.flow_name, flow_type); + + let expr = { + let mut expr = expr; + expr.flow_options + .insert(FlowType::FLOW_TYPE_KEY.to_string(), flow_type.to_string()); + expr + }; + let task = CreateFlowTask::try_from(PbCreateFlowTask { create_flow: Some(expr), }) @@ -379,6 +395,55 @@ impl StatementExecutor { .context(error::ExecuteDdlSnafu) } + /// Determine the flow type based on the SQL query + /// + /// If it contains aggregation or distinct, then it is a batch flow, otherwise it is a streaming flow + async fn determine_flow_type(&self, sql: &str, query_ctx: QueryContextRef) -> Result { + let engine = &self.query_engine; + let stmt = QueryLanguageParser::parse_sql(sql, &query_ctx) + .map_err(BoxedError::new) + .context(ExternalSnafu)?; + let plan = engine + .planner() + .plan(&stmt, query_ctx) + .await + .map_err(BoxedError::new) + .context(ExternalSnafu)?; + + /// Visitor to find aggregation or distinct + struct FindAggr { + is_aggr: bool, + } + + impl TreeNodeVisitor<'_> for FindAggr { + type Node = LogicalPlan; + fn f_down( + &mut self, + node: &Self::Node, + ) -> datafusion_common::Result + { + match node { + LogicalPlan::Aggregate(_) | LogicalPlan::Distinct(_) => { + self.is_aggr = true; + return Ok(datafusion_common::tree_node::TreeNodeRecursion::Stop); + } + _ => (), + } + Ok(datafusion_common::tree_node::TreeNodeRecursion::Continue) + } + } + + let mut find_aggr = FindAggr { is_aggr: false }; + + plan.visit_with_subqueries(&mut find_aggr) + .context(BuildDfLogicalPlanSnafu)?; + if find_aggr.is_aggr { + Ok(FlowType::Batching) + } else { + Ok(FlowType::Streaming) + } + } + #[tracing::instrument(skip_all)] pub async fn create_view( &self, diff --git a/src/servers/tests/mod.rs b/src/servers/tests/mod.rs index 43aeb362fa..7bd8a696be 100644 --- a/src/servers/tests/mod.rs +++ b/src/servers/tests/mod.rs @@ -132,7 +132,7 @@ impl GrpcQueryHandler for DummyInstance { ); result.remove(0)? } - Query::LogicalPlan(_) => unimplemented!(), + Query::LogicalPlan(_) | Query::InsertIntoPlan(_) => unimplemented!(), Query::PromRangeQuery(promql) => { let prom_query = PromQuery { query: promql.query, diff --git a/tests-integration/src/standalone.rs b/tests-integration/src/standalone.rs index b85c848c88..e25e2a9682 100644 --- a/tests-integration/src/standalone.rs +++ b/tests-integration/src/standalone.rs @@ -41,7 +41,7 @@ use common_procedure::options::ProcedureConfig; use common_procedure::ProcedureManagerRef; use common_wal::config::{DatanodeWalConfig, MetasrvWalConfig}; use datanode::datanode::DatanodeBuilder; -use flow::{FlownodeBuilder, FrontendClient}; +use flow::{FlownodeBuilder, FrontendClient, GrpcQueryHandlerWithBoxedError}; use frontend::frontend::Frontend; use frontend::instance::builder::FrontendBuilder; use frontend::instance::{Instance, StandaloneDatanodeManager}; @@ -174,8 +174,8 @@ impl GreptimeDbStandaloneBuilder { Some(procedure_manager.clone()), ); - let fe_server_addr = opts.frontend_options().grpc.bind_addr.clone(); - let frontend_client = FrontendClient::from_static_grpc_addr(fe_server_addr); + let (frontend_client, frontend_instance_handler) = + FrontendClient::from_empty_grpc_handler(); let flow_builder = FlownodeBuilder::new( Default::default(), plugins.clone(), @@ -188,7 +188,7 @@ impl GreptimeDbStandaloneBuilder { let node_manager = Arc::new(StandaloneDatanodeManager { region_server: datanode.region_server(), - flow_server: flownode.flow_worker_manager(), + flow_server: flownode.flow_engine(), }); let table_id_sequence = Arc::new( @@ -250,9 +250,17 @@ impl GreptimeDbStandaloneBuilder { .unwrap(); let instance = Arc::new(instance); - let flow_worker_manager = flownode.flow_worker_manager(); + // set the frontend client for flownode + let grpc_handler = instance.clone() as Arc; + let weak_grpc_handler = Arc::downgrade(&grpc_handler); + frontend_instance_handler + .lock() + .unwrap() + .replace(weak_grpc_handler); + + let flow_streaming_engine = flownode.flow_engine().streaming_engine(); let invoker = flow::FrontendInvoker::build_from( - flow_worker_manager.clone(), + flow_streaming_engine.clone(), catalog_manager.clone(), kv_backend.clone(), cache_registry.clone(), @@ -263,7 +271,7 @@ impl GreptimeDbStandaloneBuilder { .context(StartFlownodeSnafu) .unwrap(); - flow_worker_manager.set_frontend_invoker(invoker).await; + flow_streaming_engine.set_frontend_invoker(invoker).await; procedure_manager.start().await.unwrap(); wal_options_allocator.start().await.unwrap(); diff --git a/tests/cases/standalone/common/flow/flow_advance_ttl.result b/tests/cases/standalone/common/flow/flow_advance_ttl.result index 38d14d6b31..3f5a940977 100644 --- a/tests/cases/standalone/common/flow/flow_advance_ttl.result +++ b/tests/cases/standalone/common/flow/flow_advance_ttl.result @@ -8,6 +8,20 @@ CREATE TABLE distinct_basic ( Affected Rows: 0 +-- should fail +-- SQLNESS REPLACE id=\d+ id=REDACTED +CREATE FLOW test_distinct_basic SINK TO out_distinct_basic AS +SELECT + DISTINCT number as dis +FROM + distinct_basic; + +Error: 3001(EngineExecuteQuery), Unsupported: Source table `greptime.public.distinct_basic`(id=REDACTED) has instant TTL, Instant TTL is not supported under batching mode. Consider using a TTL longer than flush interval + +ALTER TABLE distinct_basic SET 'ttl' = '5s'; + +Affected Rows: 0 + CREATE FLOW test_distinct_basic SINK TO out_distinct_basic AS SELECT DISTINCT number as dis @@ -24,7 +38,7 @@ VALUES (20, "2021-07-01 00:00:00.200"), (22, "2021-07-01 00:00:00.600"); -Affected Rows: 0 +Affected Rows: 3 -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_distinct_basic'); @@ -49,7 +63,7 @@ SHOW CREATE TABLE distinct_basic; | | | | | ENGINE=mito | | | WITH( | -| | ttl = 'instant' | +| | ttl = '5s' | | | ) | +----------------+-----------------------------------------------------------+ @@ -84,8 +98,93 @@ FROM SELECT number FROM distinct_basic; -++ -++ ++--------+ +| number | ++--------+ +| 20 | +| 22 | ++--------+ + +-- SQLNESS SLEEP 6s +ADMIN FLUSH_TABLE('distinct_basic'); + ++-------------------------------------+ +| ADMIN FLUSH_TABLE('distinct_basic') | ++-------------------------------------+ +| 0 | ++-------------------------------------+ + +INSERT INTO + distinct_basic +VALUES + (23, "2021-07-01 00:00:01.600"); + +Affected Rows: 1 + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('test_distinct_basic'); + ++-----------------------------------------+ +| ADMIN FLUSH_FLOW('test_distinct_basic') | ++-----------------------------------------+ +| FLOW_FLUSHED | ++-----------------------------------------+ + +SHOW CREATE TABLE distinct_basic; + ++----------------+-----------------------------------------------------------+ +| Table | Create Table | ++----------------+-----------------------------------------------------------+ +| distinct_basic | CREATE TABLE IF NOT EXISTS "distinct_basic" ( | +| | "number" INT NULL, | +| | "ts" TIMESTAMP(3) NOT NULL DEFAULT current_timestamp(), | +| | TIME INDEX ("ts"), | +| | PRIMARY KEY ("number") | +| | ) | +| | | +| | ENGINE=mito | +| | WITH( | +| | ttl = '5s' | +| | ) | ++----------------+-----------------------------------------------------------+ + +SHOW CREATE TABLE out_distinct_basic; + ++--------------------+---------------------------------------------------+ +| Table | Create Table | ++--------------------+---------------------------------------------------+ +| out_distinct_basic | CREATE TABLE IF NOT EXISTS "out_distinct_basic" ( | +| | "dis" INT NULL, | +| | "update_at" TIMESTAMP(3) NULL, | +| | "__ts_placeholder" TIMESTAMP(3) NOT NULL, | +| | TIME INDEX ("__ts_placeholder"), | +| | PRIMARY KEY ("dis") | +| | ) | +| | | +| | ENGINE=mito | +| | | ++--------------------+---------------------------------------------------+ + +SELECT + dis +FROM + out_distinct_basic; + ++-----+ +| dis | ++-----+ +| 20 | +| 22 | +| 23 | ++-----+ + +SELECT number FROM distinct_basic; + ++--------+ +| number | ++--------+ +| 23 | ++--------+ DROP FLOW test_distinct_basic; diff --git a/tests/cases/standalone/common/flow/flow_advance_ttl.sql b/tests/cases/standalone/common/flow/flow_advance_ttl.sql index 18dfea25db..2691af2b0c 100644 --- a/tests/cases/standalone/common/flow/flow_advance_ttl.sql +++ b/tests/cases/standalone/common/flow/flow_advance_ttl.sql @@ -6,6 +6,16 @@ CREATE TABLE distinct_basic ( TIME INDEX(ts) )WITH ('ttl' = 'instant'); +-- should fail +-- SQLNESS REPLACE id=\d+ id=REDACTED +CREATE FLOW test_distinct_basic SINK TO out_distinct_basic AS +SELECT + DISTINCT number as dis +FROM + distinct_basic; + +ALTER TABLE distinct_basic SET 'ttl' = '5s'; + CREATE FLOW test_distinct_basic SINK TO out_distinct_basic AS SELECT DISTINCT number as dis @@ -34,6 +44,28 @@ FROM SELECT number FROM distinct_basic; +-- SQLNESS SLEEP 6s +ADMIN FLUSH_TABLE('distinct_basic'); + +INSERT INTO + distinct_basic +VALUES + (23, "2021-07-01 00:00:01.600"); + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('test_distinct_basic'); + +SHOW CREATE TABLE distinct_basic; + +SHOW CREATE TABLE out_distinct_basic; + +SELECT + dis +FROM + out_distinct_basic; + +SELECT number FROM distinct_basic; + DROP FLOW test_distinct_basic; DROP TABLE distinct_basic; DROP TABLE out_distinct_basic; \ No newline at end of file diff --git a/tests/cases/standalone/common/flow/flow_auto_sink_table.result b/tests/cases/standalone/common/flow/flow_auto_sink_table.result index ebd19a828d..de8a44fad7 100644 --- a/tests/cases/standalone/common/flow/flow_auto_sink_table.result +++ b/tests/cases/standalone/common/flow/flow_auto_sink_table.result @@ -9,11 +9,12 @@ Affected Rows: 0 CREATE FLOW test_numbers_basic SINK TO out_num_cnt_basic AS SELECT - sum(number) + sum(number), + date_bin(INTERVAL '1 second', ts, '2021-07-01 00:00:00') as time_window FROM numbers_input_basic GROUP BY - tumble(ts, '1 second', '2021-07-01 00:00:00'); + time_window; Affected Rows: 0 @@ -24,11 +25,9 @@ SHOW CREATE TABLE out_num_cnt_basic; +-------------------+--------------------------------------------------+ | out_num_cnt_basic | CREATE TABLE IF NOT EXISTS "out_num_cnt_basic" ( | | | "sum(numbers_input_basic.number)" BIGINT NULL, | -| | "window_start" TIMESTAMP(3) NOT NULL, | -| | "window_end" TIMESTAMP(3) NULL, | +| | "time_window" TIMESTAMP(9) NOT NULL, | | | "update_at" TIMESTAMP(3) NULL, | -| | TIME INDEX ("window_start"), | -| | PRIMARY KEY ("window_end") | +| | TIME INDEX ("time_window") | | | ) | | | | | | ENGINE=mito | @@ -52,11 +51,9 @@ SHOW CREATE TABLE out_num_cnt_basic; +-------------------+--------------------------------------------------+ | out_num_cnt_basic | CREATE TABLE IF NOT EXISTS "out_num_cnt_basic" ( | | | "sum(numbers_input_basic.number)" BIGINT NULL, | -| | "window_start" TIMESTAMP(3) NOT NULL, | -| | "window_end" TIMESTAMP(3) NULL, | +| | "time_window" TIMESTAMP(9) NOT NULL, | | | "update_at" TIMESTAMP(3) NULL, | -| | TIME INDEX ("window_start"), | -| | PRIMARY KEY ("window_end") | +| | TIME INDEX ("time_window") | | | ) | | | | | | ENGINE=mito | @@ -65,13 +62,13 @@ SHOW CREATE TABLE out_num_cnt_basic; SHOW CREATE FLOW test_numbers_basic; -+--------------------+-------------------------------------------------------------------------------------------------------+ -| Flow | Create Flow | -+--------------------+-------------------------------------------------------------------------------------------------------+ -| test_numbers_basic | CREATE FLOW IF NOT EXISTS test_numbers_basic | -| | SINK TO out_num_cnt_basic | -| | AS SELECT sum(number) FROM numbers_input_basic GROUP BY tumble(ts, '1 second', '2021-07-01 00:00:00') | -+--------------------+-------------------------------------------------------------------------------------------------------+ ++--------------------+----------------------------------------------------------------------------------------------------------------------------------------------+ +| Flow | Create Flow | ++--------------------+----------------------------------------------------------------------------------------------------------------------------------------------+ +| test_numbers_basic | CREATE FLOW IF NOT EXISTS test_numbers_basic | +| | SINK TO out_num_cnt_basic | +| | AS SELECT sum(number), date_bin(INTERVAL '1 second', ts, '2021-07-01 00:00:00') AS time_window FROM numbers_input_basic GROUP BY time_window | ++--------------------+----------------------------------------------------------------------------------------------------------------------------------------------+ DROP FLOW test_numbers_basic; diff --git a/tests/cases/standalone/common/flow/flow_auto_sink_table.sql b/tests/cases/standalone/common/flow/flow_auto_sink_table.sql index 0af723770c..ca76ba767e 100644 --- a/tests/cases/standalone/common/flow/flow_auto_sink_table.sql +++ b/tests/cases/standalone/common/flow/flow_auto_sink_table.sql @@ -7,11 +7,12 @@ CREATE TABLE numbers_input_basic ( CREATE FLOW test_numbers_basic SINK TO out_num_cnt_basic AS SELECT - sum(number) + sum(number), + date_bin(INTERVAL '1 second', ts, '2021-07-01 00:00:00') as time_window FROM numbers_input_basic GROUP BY - tumble(ts, '1 second', '2021-07-01 00:00:00'); + time_window; SHOW CREATE TABLE out_num_cnt_basic; diff --git a/tests/cases/standalone/common/flow/flow_basic.result b/tests/cases/standalone/common/flow/flow_basic.result index 511468f5a5..5b4e6b32ab 100644 --- a/tests/cases/standalone/common/flow/flow_basic.result +++ b/tests/cases/standalone/common/flow/flow_basic.result @@ -9,11 +9,12 @@ Affected Rows: 0 CREATE FLOW test_numbers_basic SINK TO out_num_cnt_basic AS SELECT - sum(number) + sum(number), + date_bin(INTERVAL '1 second', ts, '2021-07-01 00:00:00') as time_window FROM numbers_input_basic GROUP BY - tumble(ts, '1 second', '2021-07-01 00:00:00'); + time_window; Affected Rows: 0 @@ -24,11 +25,9 @@ SHOW CREATE TABLE out_num_cnt_basic; +-------------------+--------------------------------------------------+ | out_num_cnt_basic | CREATE TABLE IF NOT EXISTS "out_num_cnt_basic" ( | | | "sum(numbers_input_basic.number)" BIGINT NULL, | -| | "window_start" TIMESTAMP(3) NOT NULL, | -| | "window_end" TIMESTAMP(3) NULL, | +| | "time_window" TIMESTAMP(9) NOT NULL, | | | "update_at" TIMESTAMP(3) NULL, | -| | TIME INDEX ("window_start"), | -| | PRIMARY KEY ("window_end") | +| | TIME INDEX ("time_window") | | | ) | | | | | | ENGINE=mito | @@ -53,11 +52,9 @@ SHOW CREATE TABLE out_num_cnt_basic; +-------------------+--------------------------------------------------+ | out_num_cnt_basic | CREATE TABLE IF NOT EXISTS "out_num_cnt_basic" ( | | | "sum(numbers_input_basic.number)" BIGINT NULL, | -| | "window_start" TIMESTAMP(3) NOT NULL, | -| | "window_end" TIMESTAMP(3) NULL, | +| | "time_window" TIMESTAMP(9) NOT NULL, | | | "update_at" TIMESTAMP(3) NULL, | -| | TIME INDEX ("window_start"), | -| | PRIMARY KEY ("window_end") | +| | TIME INDEX ("time_window") | | | ) | | | | | | ENGINE=mito | @@ -84,16 +81,15 @@ ADMIN FLUSH_FLOW('test_numbers_basic'); SELECT "sum(numbers_input_basic.number)", - window_start, - window_end + time_window FROM out_num_cnt_basic; -+---------------------------------+---------------------+---------------------+ -| sum(numbers_input_basic.number) | window_start | window_end | -+---------------------------------+---------------------+---------------------+ -| 42 | 2021-07-01T00:00:00 | 2021-07-01T00:00:01 | -+---------------------------------+---------------------+---------------------+ ++---------------------------------+---------------------+ +| sum(numbers_input_basic.number) | time_window | ++---------------------------------+---------------------+ +| 42 | 2021-07-01T00:00:00 | ++---------------------------------+---------------------+ -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_numbers_basic'); @@ -124,17 +120,16 @@ ADMIN FLUSH_FLOW('test_numbers_basic'); -- note that this quote-unquote column is a column-name, **not** a aggregation expr, generated by datafusion SELECT "sum(numbers_input_basic.number)", - window_start, - window_end + time_window FROM out_num_cnt_basic; -+---------------------------------+---------------------+---------------------+ -| sum(numbers_input_basic.number) | window_start | window_end | -+---------------------------------+---------------------+---------------------+ -| 42 | 2021-07-01T00:00:00 | 2021-07-01T00:00:01 | -| 47 | 2021-07-01T00:00:01 | 2021-07-01T00:00:02 | -+---------------------------------+---------------------+---------------------+ ++---------------------------------+---------------------+ +| sum(numbers_input_basic.number) | time_window | ++---------------------------------+---------------------+ +| 42 | 2021-07-01T00:00:00 | +| 47 | 2021-07-01T00:00:01 | ++---------------------------------+---------------------+ DROP FLOW test_numbers_basic; @@ -896,6 +891,8 @@ CREATE TABLE temp_sensor_data ( loc STRING, temperature DOUBLE, ts TIMESTAMP TIME INDEX +)WITH( + append_mode = 'true' ); Affected Rows: 0 @@ -904,7 +901,8 @@ CREATE TABLE temp_alerts ( sensor_id INT, loc STRING, max_temp DOUBLE, - ts TIMESTAMP TIME INDEX + event_ts TIMESTAMP TIME INDEX, + update_at TIMESTAMP ); Affected Rows: 0 @@ -914,6 +912,7 @@ SELECT sensor_id, loc, max(temperature) as max_temp, + max(ts) as event_ts FROM temp_sensor_data GROUP BY @@ -933,8 +932,9 @@ SHOW CREATE TABLE temp_alerts; | | "sensor_id" INT NULL, | | | "loc" STRING NULL, | | | "max_temp" DOUBLE NULL, | -| | "ts" TIMESTAMP(3) NOT NULL, | -| | TIME INDEX ("ts") | +| | "event_ts" TIMESTAMP(3) NOT NULL, | +| | "update_at" TIMESTAMP(3) NULL, | +| | TIME INDEX ("event_ts") | | | ) | | | | | | ENGINE=mito | @@ -993,15 +993,16 @@ SHOW TABLES LIKE 'temp_alerts'; SELECT sensor_id, loc, - max_temp + max_temp, + event_ts FROM temp_alerts; -+-----------+-------+----------+ -| sensor_id | loc | max_temp | -+-----------+-------+----------+ -| 1 | room1 | 150.0 | -+-----------+-------+----------+ ++-----------+-------+----------+-------------------------+ +| sensor_id | loc | max_temp | event_ts | ++-----------+-------+----------+-------------------------+ +| 1 | room1 | 150.0 | 1970-01-01T00:00:00.001 | ++-----------+-------+----------+-------------------------+ INSERT INTO temp_sensor_data @@ -1022,15 +1023,16 @@ ADMIN FLUSH_FLOW('temp_monitoring'); SELECT sensor_id, loc, - max_temp + max_temp, + event_ts FROM temp_alerts; -+-----------+-------+----------+ -| sensor_id | loc | max_temp | -+-----------+-------+----------+ -| 1 | room1 | 150.0 | -+-----------+-------+----------+ ++-----------+-------+----------+-------------------------+ +| sensor_id | loc | max_temp | event_ts | ++-----------+-------+----------+-------------------------+ +| 1 | room1 | 150.0 | 1970-01-01T00:00:00.001 | ++-----------+-------+----------+-------------------------+ DROP FLOW temp_monitoring; @@ -1049,6 +1051,8 @@ CREATE TABLE ngx_access_log ( stat INT, size INT, access_time TIMESTAMP TIME INDEX +)WITH( + append_mode = 'true' ); Affected Rows: 0 @@ -1183,6 +1187,8 @@ CREATE TABLE requests ( service_ip STRING, val INT, ts TIMESTAMP TIME INDEX +)WITH( + append_mode = 'true' ); Affected Rows: 0 @@ -1392,6 +1398,8 @@ CREATE TABLE android_log ( `log` STRING, ts TIMESTAMP(9), TIME INDEX(ts) +)WITH( + append_mode = 'true' ); Affected Rows: 0 @@ -1503,6 +1511,8 @@ CREATE TABLE android_log ( `log` STRING, ts TIMESTAMP(9), TIME INDEX(ts) +)WITH( + append_mode = 'true' ); Affected Rows: 0 diff --git a/tests/cases/standalone/common/flow/flow_basic.sql b/tests/cases/standalone/common/flow/flow_basic.sql index bafc4c266e..32598927ab 100644 --- a/tests/cases/standalone/common/flow/flow_basic.sql +++ b/tests/cases/standalone/common/flow/flow_basic.sql @@ -7,11 +7,12 @@ CREATE TABLE numbers_input_basic ( CREATE FLOW test_numbers_basic SINK TO out_num_cnt_basic AS SELECT - sum(number) + sum(number), + date_bin(INTERVAL '1 second', ts, '2021-07-01 00:00:00') as time_window FROM numbers_input_basic GROUP BY - tumble(ts, '1 second', '2021-07-01 00:00:00'); + time_window; SHOW CREATE TABLE out_num_cnt_basic; @@ -34,8 +35,7 @@ ADMIN FLUSH_FLOW('test_numbers_basic'); SELECT "sum(numbers_input_basic.number)", - window_start, - window_end + time_window FROM out_num_cnt_basic; @@ -54,8 +54,7 @@ ADMIN FLUSH_FLOW('test_numbers_basic'); -- note that this quote-unquote column is a column-name, **not** a aggregation expr, generated by datafusion SELECT "sum(numbers_input_basic.number)", - window_start, - window_end + time_window FROM out_num_cnt_basic; @@ -403,13 +402,16 @@ CREATE TABLE temp_sensor_data ( loc STRING, temperature DOUBLE, ts TIMESTAMP TIME INDEX +)WITH( + append_mode = 'true' ); CREATE TABLE temp_alerts ( sensor_id INT, loc STRING, max_temp DOUBLE, - ts TIMESTAMP TIME INDEX + event_ts TIMESTAMP TIME INDEX, + update_at TIMESTAMP ); CREATE FLOW temp_monitoring SINK TO temp_alerts AS @@ -417,6 +419,7 @@ SELECT sensor_id, loc, max(temperature) as max_temp, + max(ts) as event_ts FROM temp_sensor_data GROUP BY @@ -451,7 +454,8 @@ SHOW TABLES LIKE 'temp_alerts'; SELECT sensor_id, loc, - max_temp + max_temp, + event_ts FROM temp_alerts; @@ -466,7 +470,8 @@ ADMIN FLUSH_FLOW('temp_monitoring'); SELECT sensor_id, loc, - max_temp + max_temp, + event_ts FROM temp_alerts; @@ -481,6 +486,8 @@ CREATE TABLE ngx_access_log ( stat INT, size INT, access_time TIMESTAMP TIME INDEX +)WITH( + append_mode = 'true' ); CREATE TABLE ngx_distribution ( @@ -555,6 +562,8 @@ CREATE TABLE requests ( service_ip STRING, val INT, ts TIMESTAMP TIME INDEX +)WITH( + append_mode = 'true' ); CREATE TABLE requests_without_ip ( @@ -650,6 +659,8 @@ CREATE TABLE android_log ( `log` STRING, ts TIMESTAMP(9), TIME INDEX(ts) +)WITH( + append_mode = 'true' ); CREATE TABLE android_log_abnormal ( @@ -704,6 +715,8 @@ CREATE TABLE android_log ( `log` STRING, ts TIMESTAMP(9), TIME INDEX(ts) +)WITH( + append_mode = 'true' ); CREATE TABLE android_log_abnormal ( diff --git a/tests/cases/standalone/common/flow/flow_blog.result b/tests/cases/standalone/common/flow/flow_blog.result index 3046e147c0..9b90d1b068 100644 --- a/tests/cases/standalone/common/flow/flow_blog.result +++ b/tests/cases/standalone/common/flow/flow_blog.result @@ -19,7 +19,9 @@ Affected Rows: 0 CREATE FLOW calc_avg_speed SINK TO avg_speed AS SELECT - avg((left_wheel + right_wheel) / 2) + avg((left_wheel + right_wheel) / 2) as avg_speed, + date_bin(INTERVAL '5 second', ts) as start_window, + date_bin(INTERVAL '5 second', ts) + INTERVAL '5 second' as end_window, FROM velocity WHERE @@ -28,7 +30,7 @@ WHERE AND left_wheel < 60 AND right_wheel < 60 GROUP BY - tumble(ts, '5 second'); + start_window; Affected Rows: 0 diff --git a/tests/cases/standalone/common/flow/flow_blog.sql b/tests/cases/standalone/common/flow/flow_blog.sql index f40614bd9a..4255aa1875 100644 --- a/tests/cases/standalone/common/flow/flow_blog.sql +++ b/tests/cases/standalone/common/flow/flow_blog.sql @@ -15,7 +15,9 @@ CREATE TABLE avg_speed ( CREATE FLOW calc_avg_speed SINK TO avg_speed AS SELECT - avg((left_wheel + right_wheel) / 2) + avg((left_wheel + right_wheel) / 2) as avg_speed, + date_bin(INTERVAL '5 second', ts) as start_window, + date_bin(INTERVAL '5 second', ts) + INTERVAL '5 second' as end_window, FROM velocity WHERE @@ -24,7 +26,7 @@ WHERE AND left_wheel < 60 AND right_wheel < 60 GROUP BY - tumble(ts, '5 second'); + start_window; INSERT INTO velocity diff --git a/tests/cases/standalone/common/flow/flow_call_df_func.result b/tests/cases/standalone/common/flow/flow_call_df_func.result index d6423c7c7f..17f172c738 100644 --- a/tests/cases/standalone/common/flow/flow_call_df_func.result +++ b/tests/cases/standalone/common/flow/flow_call_df_func.result @@ -11,7 +11,7 @@ Affected Rows: 0 CREATE FLOW test_numbers_df_func SINK TO out_num_cnt_df_func AS -SELECT sum(abs(number)) FROM numbers_input_df_func GROUP BY tumble(ts, '1 second', '2021-07-01 00:00:00'); +SELECT sum(abs(number)), date_bin(INTERVAL '1 second', ts, '2021-07-01 00:00:00') as time_window FROM numbers_input_df_func GROUP BY time_window; Affected Rows: 0 @@ -42,13 +42,13 @@ ADMIN FLUSH_FLOW('test_numbers_df_func'); +------------------------------------------+ -- note that this quote-unquote column is a column-name, **not** a aggregation expr, generated by datafusion -SELECT "sum(abs(numbers_input_df_func.number))", window_start, window_end FROM out_num_cnt_df_func; +SELECT "sum(abs(numbers_input_df_func.number))", time_window FROM out_num_cnt_df_func; -+----------------------------------------+---------------------+---------------------+ -| sum(abs(numbers_input_df_func.number)) | window_start | window_end | -+----------------------------------------+---------------------+---------------------+ -| 42 | 2021-07-01T00:00:00 | 2021-07-01T00:00:01 | -+----------------------------------------+---------------------+---------------------+ ++----------------------------------------+---------------------+ +| sum(abs(numbers_input_df_func.number)) | time_window | ++----------------------------------------+---------------------+ +| 42 | 2021-07-01T00:00:00 | ++----------------------------------------+---------------------+ -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_numbers_df_func'); @@ -76,14 +76,14 @@ ADMIN FLUSH_FLOW('test_numbers_df_func'); +------------------------------------------+ -- note that this quote-unquote column is a column-name, **not** a aggregation expr, generated by datafusion -SELECT "sum(abs(numbers_input_df_func.number))", window_start, window_end FROM out_num_cnt_df_func; +SELECT "sum(abs(numbers_input_df_func.number))", time_window FROM out_num_cnt_df_func; -+----------------------------------------+---------------------+---------------------+ -| sum(abs(numbers_input_df_func.number)) | window_start | window_end | -+----------------------------------------+---------------------+---------------------+ -| 42 | 2021-07-01T00:00:00 | 2021-07-01T00:00:01 | -| 47 | 2021-07-01T00:00:01 | 2021-07-01T00:00:02 | -+----------------------------------------+---------------------+---------------------+ ++----------------------------------------+---------------------+ +| sum(abs(numbers_input_df_func.number)) | time_window | ++----------------------------------------+---------------------+ +| 42 | 2021-07-01T00:00:00 | +| 47 | 2021-07-01T00:00:01 | ++----------------------------------------+---------------------+ DROP FLOW test_numbers_df_func; @@ -110,7 +110,7 @@ Affected Rows: 0 CREATE FLOW test_numbers_df_func SINK TO out_num_cnt_df_func AS -SELECT abs(sum(number)) FROM numbers_input_df_func GROUP BY tumble(ts, '1 second', '2021-07-01 00:00:00'); +SELECT abs(sum(number)), date_bin(INTERVAL '1 second', ts, '2021-07-01 00:00:00') as time_window FROM numbers_input_df_func GROUP BY time_window; Affected Rows: 0 @@ -140,13 +140,13 @@ ADMIN FLUSH_FLOW('test_numbers_df_func'); | FLOW_FLUSHED | +------------------------------------------+ -SELECT "abs(sum(numbers_input_df_func.number))", window_start, window_end FROM out_num_cnt_df_func; +SELECT "abs(sum(numbers_input_df_func.number))", time_window FROM out_num_cnt_df_func; -+----------------------------------------+---------------------+---------------------+ -| abs(sum(numbers_input_df_func.number)) | window_start | window_end | -+----------------------------------------+---------------------+---------------------+ -| 2 | 2021-07-01T00:00:00 | 2021-07-01T00:00:01 | -+----------------------------------------+---------------------+---------------------+ ++----------------------------------------+---------------------+ +| abs(sum(numbers_input_df_func.number)) | time_window | ++----------------------------------------+---------------------+ +| 2 | 2021-07-01T00:00:00 | ++----------------------------------------+---------------------+ -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_numbers_df_func'); @@ -173,14 +173,14 @@ ADMIN FLUSH_FLOW('test_numbers_df_func'); | FLOW_FLUSHED | +------------------------------------------+ -SELECT "abs(sum(numbers_input_df_func.number))", window_start, window_end FROM out_num_cnt_df_func; +SELECT "abs(sum(numbers_input_df_func.number))", time_window FROM out_num_cnt_df_func; -+----------------------------------------+---------------------+---------------------+ -| abs(sum(numbers_input_df_func.number)) | window_start | window_end | -+----------------------------------------+---------------------+---------------------+ -| 2 | 2021-07-01T00:00:00 | 2021-07-01T00:00:01 | -| 1 | 2021-07-01T00:00:01 | 2021-07-01T00:00:02 | -+----------------------------------------+---------------------+---------------------+ ++----------------------------------------+---------------------+ +| abs(sum(numbers_input_df_func.number)) | time_window | ++----------------------------------------+---------------------+ +| 2 | 2021-07-01T00:00:00 | +| 1 | 2021-07-01T00:00:01 | ++----------------------------------------+---------------------+ DROP FLOW test_numbers_df_func; diff --git a/tests/cases/standalone/common/flow/flow_call_df_func.sql b/tests/cases/standalone/common/flow/flow_call_df_func.sql index 6143f493f4..83c4090094 100644 --- a/tests/cases/standalone/common/flow/flow_call_df_func.sql +++ b/tests/cases/standalone/common/flow/flow_call_df_func.sql @@ -9,7 +9,7 @@ CREATE TABLE numbers_input_df_func ( CREATE FLOW test_numbers_df_func SINK TO out_num_cnt_df_func AS -SELECT sum(abs(number)) FROM numbers_input_df_func GROUP BY tumble(ts, '1 second', '2021-07-01 00:00:00'); +SELECT sum(abs(number)), date_bin(INTERVAL '1 second', ts, '2021-07-01 00:00:00') as time_window FROM numbers_input_df_func GROUP BY time_window; -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_numbers_df_func'); @@ -24,7 +24,7 @@ VALUES ADMIN FLUSH_FLOW('test_numbers_df_func'); -- note that this quote-unquote column is a column-name, **not** a aggregation expr, generated by datafusion -SELECT "sum(abs(numbers_input_df_func.number))", window_start, window_end FROM out_num_cnt_df_func; +SELECT "sum(abs(numbers_input_df_func.number))", time_window FROM out_num_cnt_df_func; -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_numbers_df_func'); @@ -38,7 +38,7 @@ VALUES ADMIN FLUSH_FLOW('test_numbers_df_func'); -- note that this quote-unquote column is a column-name, **not** a aggregation expr, generated by datafusion -SELECT "sum(abs(numbers_input_df_func.number))", window_start, window_end FROM out_num_cnt_df_func; +SELECT "sum(abs(numbers_input_df_func.number))", time_window FROM out_num_cnt_df_func; DROP FLOW test_numbers_df_func; DROP TABLE numbers_input_df_func; @@ -55,7 +55,7 @@ CREATE TABLE numbers_input_df_func ( CREATE FLOW test_numbers_df_func SINK TO out_num_cnt_df_func AS -SELECT abs(sum(number)) FROM numbers_input_df_func GROUP BY tumble(ts, '1 second', '2021-07-01 00:00:00'); +SELECT abs(sum(number)), date_bin(INTERVAL '1 second', ts, '2021-07-01 00:00:00') as time_window FROM numbers_input_df_func GROUP BY time_window; -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_numbers_df_func'); @@ -69,7 +69,7 @@ VALUES -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_numbers_df_func'); -SELECT "abs(sum(numbers_input_df_func.number))", window_start, window_end FROM out_num_cnt_df_func; +SELECT "abs(sum(numbers_input_df_func.number))", time_window FROM out_num_cnt_df_func; -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_numbers_df_func'); @@ -82,7 +82,7 @@ VALUES -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_numbers_df_func'); -SELECT "abs(sum(numbers_input_df_func.number))", window_start, window_end FROM out_num_cnt_df_func; +SELECT "abs(sum(numbers_input_df_func.number))", time_window FROM out_num_cnt_df_func; DROP FLOW test_numbers_df_func; DROP TABLE numbers_input_df_func; diff --git a/tests/cases/standalone/common/flow/flow_flush.result b/tests/cases/standalone/common/flow/flow_flush.result new file mode 100644 index 0000000000..f9a8a43af8 --- /dev/null +++ b/tests/cases/standalone/common/flow/flow_flush.result @@ -0,0 +1,62 @@ +-- test if flush_flow works and flush old data to flow for compute +CREATE TABLE numbers_input_basic ( + number INT, + ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(number), + TIME INDEX(ts) +); + +Affected Rows: 0 + +INSERT INTO + numbers_input_basic +VALUES + (20, "2021-07-01 00:00:00.200"), + (22, "2021-07-01 00:00:00.600"); + +Affected Rows: 2 + +CREATE FLOW test_numbers_basic SINK TO out_num_cnt_basic AS +SELECT + sum(number), + date_bin(INTERVAL '1 second', ts, '2021-07-01 00:00:00.1') as time_window +FROM + numbers_input_basic +GROUP BY + time_window; + +Affected Rows: 0 + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('test_numbers_basic'); + ++----------------------------------------+ +| ADMIN FLUSH_FLOW('test_numbers_basic') | ++----------------------------------------+ +| FLOW_FLUSHED | ++----------------------------------------+ + +SELECT + "sum(numbers_input_basic.number)", + time_window +FROM + out_num_cnt_basic; + ++---------------------------------+-------------------------+ +| sum(numbers_input_basic.number) | time_window | ++---------------------------------+-------------------------+ +| 42 | 2021-07-01T00:00:00.100 | ++---------------------------------+-------------------------+ + +DROP FLOW test_numbers_basic; + +Affected Rows: 0 + +DROP TABLE numbers_input_basic; + +Affected Rows: 0 + +DROP TABLE out_num_cnt_basic; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/flow/flow_flush.sql b/tests/cases/standalone/common/flow/flow_flush.sql new file mode 100644 index 0000000000..9dca98baf7 --- /dev/null +++ b/tests/cases/standalone/common/flow/flow_flush.sql @@ -0,0 +1,37 @@ +-- test if flush_flow works and flush old data to flow for compute +CREATE TABLE numbers_input_basic ( + number INT, + ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(number), + TIME INDEX(ts) +); + +INSERT INTO + numbers_input_basic +VALUES + (20, "2021-07-01 00:00:00.200"), + (22, "2021-07-01 00:00:00.600"); + +CREATE FLOW test_numbers_basic SINK TO out_num_cnt_basic AS +SELECT + sum(number), + date_bin(INTERVAL '1 second', ts, '2021-07-01 00:00:00.1') as time_window +FROM + numbers_input_basic +GROUP BY + time_window; + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('test_numbers_basic'); + +SELECT + "sum(numbers_input_basic.number)", + time_window +FROM + out_num_cnt_basic; + +DROP FLOW test_numbers_basic; + +DROP TABLE numbers_input_basic; + +DROP TABLE out_num_cnt_basic; diff --git a/tests/cases/standalone/common/flow/flow_null.result b/tests/cases/standalone/common/flow/flow_null.result index aaa151c51e..4ee94cfc09 100644 --- a/tests/cases/standalone/common/flow/flow_null.result +++ b/tests/cases/standalone/common/flow/flow_null.result @@ -5,6 +5,8 @@ CREATE TABLE requests ( service_ip STRING, val INT, ts TIMESTAMP TIME INDEX +)WITH( + append_mode = 'true' ); Affected Rows: 0 @@ -93,6 +95,8 @@ CREATE TABLE ngx_access_log ( client STRING, country STRING, access_time TIMESTAMP TIME INDEX +)WITH( + append_mode = 'true' ); Affected Rows: 0 diff --git a/tests/cases/standalone/common/flow/flow_null.sql b/tests/cases/standalone/common/flow/flow_null.sql index b2bdfd74df..680d404b78 100644 --- a/tests/cases/standalone/common/flow/flow_null.sql +++ b/tests/cases/standalone/common/flow/flow_null.sql @@ -6,6 +6,8 @@ CREATE TABLE requests ( service_ip STRING, val INT, ts TIMESTAMP TIME INDEX +)WITH( + append_mode = 'true' ); CREATE TABLE sum_val_in_reqs ( @@ -59,6 +61,8 @@ CREATE TABLE ngx_access_log ( client STRING, country STRING, access_time TIMESTAMP TIME INDEX +)WITH( + append_mode = 'true' ); CREATE FLOW calc_ngx_country SINK TO ngx_country AS diff --git a/tests/cases/standalone/common/flow/flow_rebuild.result b/tests/cases/standalone/common/flow/flow_rebuild.result index 0cc5f1ce9c..521bcdb5ae 100644 --- a/tests/cases/standalone/common/flow/flow_rebuild.result +++ b/tests/cases/standalone/common/flow/flow_rebuild.result @@ -3,6 +3,8 @@ CREATE TABLE input_basic ( ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(number), TIME INDEX(ts) +)WITH( + append_mode = 'true' ); Affected Rows: 0 @@ -166,7 +168,7 @@ ADMIN FLUSH_FLOW('test_wildcard_basic'); | FLOW_FLUSHED | +-----------------------------------------+ --- 3 is also expected, since flow don't have persisent state +-- flow batching mode SELECT wildcard FROM out_basic; +----------+ @@ -175,6 +177,14 @@ SELECT wildcard FROM out_basic; | 3 | +----------+ +SELECT count(*) FROM input_basic; + ++----------+ +| count(*) | ++----------+ +| 3 | ++----------+ + DROP TABLE input_basic; Affected Rows: 0 @@ -302,6 +312,15 @@ FROM Affected Rows: 0 -- SQLNESS ARG restart=true +SELECT 1; + ++----------+ +| Int64(1) | ++----------+ +| 1 | ++----------+ + +-- SQLNESS SLEEP 3s INSERT INTO input_basic VALUES @@ -310,6 +329,8 @@ VALUES Affected Rows: 2 +-- give flownode a second to rebuild flow +-- SQLNESS SLEEP 3s -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_wildcard_basic'); @@ -358,6 +379,15 @@ FROM Affected Rows: 0 -- SQLNESS ARG restart=true +SELECT 1; + ++----------+ +| Int64(1) | ++----------+ +| 1 | ++----------+ + +-- SQLNESS SLEEP 3s INSERT INTO input_basic VALUES @@ -366,6 +396,8 @@ VALUES Affected Rows: 2 +-- give flownode a second to rebuild flow +-- SQLNESS SLEEP 3s -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_wildcard_basic'); @@ -397,6 +429,15 @@ CREATE TABLE input_basic ( Affected Rows: 0 -- SQLNESS ARG restart=true +SELECT 1; + ++----------+ +| Int64(1) | ++----------+ +| 1 | ++----------+ + +-- SQLNESS SLEEP 3s INSERT INTO input_basic VALUES @@ -406,6 +447,8 @@ VALUES Affected Rows: 3 +-- give flownode a second to rebuild flow +-- SQLNESS SLEEP 3s -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_wildcard_basic'); @@ -438,7 +481,17 @@ FROM Affected Rows: 0 +-- give flownode a second to rebuild flow -- SQLNESS ARG restart=true +SELECT 1; + ++----------+ +| Int64(1) | ++----------+ +| 1 | ++----------+ + +-- SQLNESS SLEEP 3s INSERT INTO input_basic VALUES @@ -457,13 +510,21 @@ ADMIN FLUSH_FLOW('test_wildcard_basic'); | FLOW_FLUSHED | +-----------------------------------------+ --- 3 is also expected, since flow don't have persisent state +-- 4 is also expected, since flow batching mode SELECT wildcard FROM out_basic; +----------+ | wildcard | +----------+ -| 3 | +| 4 | ++----------+ + +SELECT count(*) FROM input_basic; + ++----------+ +| count(*) | ++----------+ +| 4 | +----------+ DROP TABLE input_basic; @@ -496,6 +557,15 @@ FROM Affected Rows: 0 -- SQLNESS ARG restart=true +SELECT 1; + ++----------+ +| Int64(1) | ++----------+ +| 1 | ++----------+ + +-- SQLNESS SLEEP 3s INSERT INTO input_basic VALUES @@ -504,6 +574,8 @@ VALUES Affected Rows: 2 +-- give flownode a second to rebuild flow +-- SQLNESS SLEEP 3s -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_wildcard_basic'); @@ -538,6 +610,15 @@ FROM Affected Rows: 0 -- SQLNESS ARG restart=true +SELECT 1; + ++----------+ +| Int64(1) | ++----------+ +| 1 | ++----------+ + +-- SQLNESS SLEEP 3s INSERT INTO input_basic VALUES @@ -547,6 +628,7 @@ VALUES Affected Rows: 3 +-- give flownode a second to rebuild flow -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_wildcard_basic'); diff --git a/tests/cases/standalone/common/flow/flow_rebuild.sql b/tests/cases/standalone/common/flow/flow_rebuild.sql index ad07ea4a40..b2b6149761 100644 --- a/tests/cases/standalone/common/flow/flow_rebuild.sql +++ b/tests/cases/standalone/common/flow/flow_rebuild.sql @@ -3,6 +3,8 @@ CREATE TABLE input_basic ( ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(number), TIME INDEX(ts) +)WITH( + append_mode = 'true' ); CREATE FLOW test_wildcard_basic sink TO out_basic AS @@ -95,9 +97,11 @@ VALUES -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_wildcard_basic'); --- 3 is also expected, since flow don't have persisent state +-- flow batching mode SELECT wildcard FROM out_basic; +SELECT count(*) FROM input_basic; + DROP TABLE input_basic; DROP FLOW test_wildcard_basic; DROP TABLE out_basic; @@ -168,12 +172,17 @@ FROM input_basic; -- SQLNESS ARG restart=true +SELECT 1; + +-- SQLNESS SLEEP 3s INSERT INTO input_basic VALUES (23, "2021-07-01 00:00:01.000"), (24, "2021-07-01 00:00:01.500"); +-- give flownode a second to rebuild flow +-- SQLNESS SLEEP 3s -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_wildcard_basic'); @@ -201,12 +210,17 @@ FROM input_basic; -- SQLNESS ARG restart=true +SELECT 1; + +-- SQLNESS SLEEP 3s INSERT INTO input_basic VALUES (23, "2021-07-01 00:00:01.000"), (24, "2021-07-01 00:00:01.500"); +-- give flownode a second to rebuild flow +-- SQLNESS SLEEP 3s -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_wildcard_basic'); @@ -222,6 +236,9 @@ CREATE TABLE input_basic ( ); -- SQLNESS ARG restart=true +SELECT 1; + +-- SQLNESS SLEEP 3s INSERT INTO input_basic VALUES @@ -229,6 +246,8 @@ VALUES (24, "2021-07-01 00:00:01.500"), (26, "2021-07-01 00:00:02.000"); +-- give flownode a second to rebuild flow +-- SQLNESS SLEEP 3s -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_wildcard_basic'); @@ -245,7 +264,11 @@ SELECT FROM input_basic; +-- give flownode a second to rebuild flow -- SQLNESS ARG restart=true +SELECT 1; + +-- SQLNESS SLEEP 3s INSERT INTO input_basic VALUES @@ -256,9 +279,11 @@ VALUES -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_wildcard_basic'); --- 3 is also expected, since flow don't have persisent state +-- 4 is also expected, since flow batching mode SELECT wildcard FROM out_basic; +SELECT count(*) FROM input_basic; + DROP TABLE input_basic; DROP FLOW test_wildcard_basic; DROP TABLE out_basic; @@ -277,13 +302,17 @@ FROM input_basic; -- SQLNESS ARG restart=true +SELECT 1; + +-- SQLNESS SLEEP 3s INSERT INTO input_basic VALUES (23, "2021-07-01 00:00:01.000"), (24, "2021-07-01 00:00:01.500"); - +-- give flownode a second to rebuild flow +-- SQLNESS SLEEP 3s -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_wildcard_basic'); @@ -300,6 +329,9 @@ FROM input_basic; -- SQLNESS ARG restart=true +SELECT 1; + +-- SQLNESS SLEEP 3s INSERT INTO input_basic VALUES @@ -307,6 +339,7 @@ VALUES (24, "2021-07-01 00:00:01.500"), (25, "2021-07-01 00:00:01.700"); +-- give flownode a second to rebuild flow -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_wildcard_basic'); diff --git a/tests/cases/standalone/common/flow/flow_user_guide.result b/tests/cases/standalone/common/flow/flow_user_guide.result index c044fef367..7c7a16b4d0 100644 --- a/tests/cases/standalone/common/flow/flow_user_guide.result +++ b/tests/cases/standalone/common/flow/flow_user_guide.result @@ -397,7 +397,7 @@ CREATE TABLE temp_alerts ( sensor_id INT, loc STRING, max_temp DOUBLE, - update_at TIMESTAMP TIME INDEX, + event_ts TIMESTAMP TIME INDEX, PRIMARY KEY(sensor_id, loc) ); @@ -408,6 +408,7 @@ SELECT sensor_id, loc, max(temperature) as max_temp, + max(ts) as event_ts, FROM temp_sensor_data GROUP BY @@ -438,7 +439,8 @@ ADMIN FLUSH_FLOW('temp_monitoring'); SELECT sensor_id, loc, - max_temp + max_temp, + event_ts FROM temp_alerts; @@ -466,16 +468,17 @@ ADMIN FLUSH_FLOW('temp_monitoring'); SELECT sensor_id, loc, - max_temp + max_temp, + event_ts FROM temp_alerts; -+-----------+-------+----------+ -| sensor_id | loc | max_temp | -+-----------+-------+----------+ -| 1 | room1 | 101.5 | -| 2 | room2 | 102.5 | -+-----------+-------+----------+ ++-----------+-------+----------+---------------------+ +| sensor_id | loc | max_temp | event_ts | ++-----------+-------+----------+---------------------+ +| 1 | room1 | 101.5 | 2022-01-01T00:00:02 | +| 2 | room2 | 102.5 | 2022-01-01T00:00:03 | ++-----------+-------+----------+---------------------+ DROP FLOW temp_monitoring; diff --git a/tests/cases/standalone/common/flow/flow_user_guide.sql b/tests/cases/standalone/common/flow/flow_user_guide.sql index d882972393..deaf3a61cc 100644 --- a/tests/cases/standalone/common/flow/flow_user_guide.sql +++ b/tests/cases/standalone/common/flow/flow_user_guide.sql @@ -291,7 +291,7 @@ CREATE TABLE temp_alerts ( sensor_id INT, loc STRING, max_temp DOUBLE, - update_at TIMESTAMP TIME INDEX, + event_ts TIMESTAMP TIME INDEX, PRIMARY KEY(sensor_id, loc) ); @@ -300,6 +300,7 @@ SELECT sensor_id, loc, max(temperature) as max_temp, + max(ts) as event_ts, FROM temp_sensor_data GROUP BY @@ -320,7 +321,8 @@ ADMIN FLUSH_FLOW('temp_monitoring'); SELECT sensor_id, loc, - max_temp + max_temp, + event_ts FROM temp_alerts; @@ -337,7 +339,8 @@ ADMIN FLUSH_FLOW('temp_monitoring'); SELECT sensor_id, loc, - max_temp + max_temp, + event_ts FROM temp_alerts; diff --git a/tests/cases/standalone/common/flow/flow_view.result b/tests/cases/standalone/common/flow/flow_view.result index ec54a12aa8..086f823136 100644 --- a/tests/cases/standalone/common/flow/flow_view.result +++ b/tests/cases/standalone/common/flow/flow_view.result @@ -72,12 +72,13 @@ INSERT INTO ngx_access_log VALUES ('192.168.1.1', 'GET', '/index.html', 200, 512 Affected Rows: 4 +-- TODO(discord9): fix flow stat update for batching mode flow SELECT created_time < last_execution_time, created_time IS NOT NULL, last_execution_time IS NOT NULL, source_table_names FROM information_schema.flows WHERE flow_name = 'user_agent_flow'; +--------------------------------------------------------------------------------------+---------------------------------------------------+----------------------------------------------------------+--------------------------------+ | information_schema.flows.created_time < information_schema.flows.last_execution_time | information_schema.flows.created_time IS NOT NULL | information_schema.flows.last_execution_time IS NOT NULL | source_table_names | +--------------------------------------------------------------------------------------+---------------------------------------------------+----------------------------------------------------------+--------------------------------+ -| true | true | true | greptime.public.ngx_access_log | +| | true | false | greptime.public.ngx_access_log | +--------------------------------------------------------------------------------------+---------------------------------------------------+----------------------------------------------------------+--------------------------------+ DROP TABLE ngx_access_log; diff --git a/tests/cases/standalone/common/flow/flow_view.sql b/tests/cases/standalone/common/flow/flow_view.sql index 28e5e2608e..61aff064a9 100644 --- a/tests/cases/standalone/common/flow/flow_view.sql +++ b/tests/cases/standalone/common/flow/flow_view.sql @@ -32,6 +32,7 @@ INSERT INTO ngx_access_log VALUES ('192.168.1.1', 'GET', '/index.html', 200, 512 -- SQLNESS SLEEP 10s INSERT INTO ngx_access_log VALUES ('192.168.1.1', 'GET', '/index.html', 200, 512, 'Mozilla/5.0', 1024, '2023-10-01T10:00:00Z'), ('192.168.1.2', 'POST', '/submit', 201, 256, 'curl/7.68.0', 512, '2023-10-01T10:01:00Z'), ('192.168.1.1', 'GET', '/about.html', 200, 128, 'Mozilla/5.0', 256, '2023-10-01T10:02:00Z'), ('192.168.1.3', 'GET', '/contact', 404, 64, 'curl/7.68.0', 128, '2023-10-01T10:03:00Z'); +-- TODO(discord9): fix flow stat update for batching mode flow SELECT created_time < last_execution_time, created_time IS NOT NULL, last_execution_time IS NOT NULL, source_table_names FROM information_schema.flows WHERE flow_name = 'user_agent_flow'; DROP TABLE ngx_access_log; diff --git a/tests/conf/frontend-test.toml.template b/tests/conf/frontend-test.toml.template index 2f8c4f58b8..de4ce86adc 100644 --- a/tests/conf/frontend-test.toml.template +++ b/tests/conf/frontend-test.toml.template @@ -1,3 +1,3 @@ [grpc] -bind_addr = "127.0.0.1:29401" -server_addr = "127.0.0.1:29401" \ No newline at end of file +bind_addr = "{grpc_addr}" +server_addr = "{grpc_addr}" diff --git a/tests/runner/src/server_mode.rs b/tests/runner/src/server_mode.rs index e7971dc73a..b3d471da46 100644 --- a/tests/runner/src/server_mode.rs +++ b/tests/runner/src/server_mode.rs @@ -389,6 +389,9 @@ impl ServerMode { format!("--metasrv-addrs={metasrv_addr}"), format!("--http-addr={http_addr}"), format!("--rpc-addr={rpc_bind_addr}"), + // since sqlness run on local, bind addr is the same as server addr + // this is needed so that `cluster_info`'s server addr column can be correct + format!("--rpc-server-addr={rpc_bind_addr}"), format!("--mysql-addr={mysql_addr}"), format!("--postgres-addr={postgres_addr}"), format!( From 9557b76224d17ee1fc1cfed1c529a71ba1bf2ea2 Mon Sep 17 00:00:00 2001 From: discord9 <55937128+discord9@users.noreply.github.com> Date: Thu, 24 Apr 2025 00:57:54 +0800 Subject: [PATCH 69/82] fix: try prune one less (#5965) * try prune one less * test: also not add one * ci: use longer fuzz time * revert fuzz time&per review * chore: no ( * docs: add explain to offset used in delete records * test: fix test_procedure_execution --- src/meta-srv/src/procedure/wal_prune.rs | 27 ++++++++++++------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/meta-srv/src/procedure/wal_prune.rs b/src/meta-srv/src/procedure/wal_prune.rs index 0247c928ec..e9b6403942 100644 --- a/src/meta-srv/src/procedure/wal_prune.rs +++ b/src/meta-srv/src/procedure/wal_prune.rs @@ -335,22 +335,21 @@ impl WalPruneProcedure { })?; partition_client .delete_records( - (self.data.prunable_entry_id + 1) as i64, + // notice here no "+1" is needed because the offset arg is exclusive, and it's defensive programming just in case somewhere else have a off by one error, see https://kafka.apache.org/36/javadoc/org/apache/kafka/clients/consumer/KafkaConsumer.html#endOffsets(java.util.Collection) which we use to get the end offset from high watermark + self.data.prunable_entry_id as i64, DELETE_RECORDS_TIMEOUT.as_millis() as i32, ) .await .context(DeleteRecordsSnafu { topic: &self.data.topic, partition: DEFAULT_PARTITION, - offset: (self.data.prunable_entry_id + 1), + offset: self.data.prunable_entry_id, }) .map_err(BoxedError::new) .with_context(|_| error::RetryLaterWithSourceSnafu { reason: format!( "Failed to delete records for topic: {}, partition: {}, offset: {}", - self.data.topic, - DEFAULT_PARTITION, - self.data.prunable_entry_id + 1 + self.data.topic, DEFAULT_PARTITION, self.data.prunable_entry_id ), })?; info!( @@ -605,19 +604,19 @@ mod tests { // Step 3: Test `on_prune`. let status = procedure.on_prune().await.unwrap(); assert_matches!(status, Status::Done { output: None }); - // Check if the entry ids after `prunable_entry_id` still exist. - check_entry_id_existence( - procedure.context.client.clone(), - &topic_name, - procedure.data.prunable_entry_id as i64 + 1, - true, - ) - .await; - // Check if the entry s before `prunable_entry_id` are deleted. + // Check if the entry ids after(include) `prunable_entry_id` still exist. check_entry_id_existence( procedure.context.client.clone(), &topic_name, procedure.data.prunable_entry_id as i64, + true, + ) + .await; + // Check if the entry ids before `prunable_entry_id` are deleted. + check_entry_id_existence( + procedure.context.client.clone(), + &topic_name, + procedure.data.prunable_entry_id as i64 - 1, false, ) .await; From f3d000f6ecffcd7161b2b3bfd2715da93c601b81 Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Thu, 24 Apr 2025 02:19:18 +0800 Subject: [PATCH 70/82] feat: track region failover attempts and adjust timeout (#5952) --- src/meta-srv/src/region/supervisor.rs | 35 ++++++++++++++++++--------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/src/meta-srv/src/region/supervisor.rs b/src/meta-srv/src/region/supervisor.rs index 6318c0d128..42963fb1fd 100644 --- a/src/meta-srv/src/region/supervisor.rs +++ b/src/meta-srv/src/region/supervisor.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::fmt::Debug; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -25,7 +25,7 @@ use common_meta::leadership_notifier::LeadershipChangeListener; use common_meta::peer::PeerLookupServiceRef; use common_meta::DatanodeId; use common_runtime::JoinHandle; -use common_telemetry::{error, info, warn}; +use common_telemetry::{debug, error, info, warn}; use common_time::util::current_time_millis; use error::Error::{LeaderPeerChanged, MigrationRunning, TableRouteNotFound}; use snafu::{OptionExt, ResultExt}; @@ -208,6 +208,8 @@ pub const DEFAULT_TICK_INTERVAL: Duration = Duration::from_secs(1); pub struct RegionSupervisor { /// Used to detect the failure of regions. failure_detector: RegionFailureDetector, + /// Tracks the number of failovers for each region. + failover_counts: HashMap, /// Receives [Event]s. receiver: Receiver, /// The context of [`SelectorRef`] @@ -293,6 +295,7 @@ impl RegionSupervisor { ) -> Self { Self { failure_detector: RegionFailureDetector::new(options), + failover_counts: HashMap::new(), receiver: event_receiver, selector_context, selector, @@ -336,13 +339,14 @@ impl RegionSupervisor { } } - async fn deregister_failure_detectors(&self, detecting_regions: Vec) { + async fn deregister_failure_detectors(&mut self, detecting_regions: Vec) { for region in detecting_regions { - self.failure_detector.remove(®ion) + self.failure_detector.remove(®ion); + self.failover_counts.remove(®ion); } } - async fn handle_region_failures(&self, mut regions: Vec<(DatanodeId, RegionId)>) { + async fn handle_region_failures(&mut self, mut regions: Vec<(DatanodeId, RegionId)>) { if regions.is_empty() { return; } @@ -365,8 +369,7 @@ impl RegionSupervisor { .collect::>(); for (datanode_id, region_id) in migrating_regions { - self.failure_detector.remove(&(datanode_id, region_id)); - warn!( + debug!( "Removed region failover for region: {region_id}, datanode: {datanode_id} because it's migrating" ); } @@ -386,7 +389,12 @@ impl RegionSupervisor { .context(error::MaintenanceModeManagerSnafu) } - async fn do_failover(&self, datanode_id: DatanodeId, region_id: RegionId) -> Result<()> { + async fn do_failover(&mut self, datanode_id: DatanodeId, region_id: RegionId) -> Result<()> { + let count = *self + .failover_counts + .entry((datanode_id, region_id)) + .and_modify(|count| *count += 1) + .or_insert(1); let from_peer = self .peer_lookup .datanode(datanode_id) @@ -415,11 +423,14 @@ impl RegionSupervisor { ); return Ok(()); } + info!( + "Failover for region: {region_id}, from_peer: {from_peer}, to_peer: {to_peer}, tries: {count}" + ); let task = RegionMigrationProcedureTask { region_id, from_peer, to_peer, - timeout: DEFAULT_REGION_MIGRATION_TIMEOUT, + timeout: DEFAULT_REGION_MIGRATION_TIMEOUT * count, }; if let Err(err) = self.region_migration_manager.submit_procedure(task).await { @@ -433,7 +444,8 @@ impl RegionSupervisor { Ok(()) } TableRouteNotFound { .. } => { - self.failure_detector.remove(&(datanode_id, region_id)); + self.deregister_failure_detectors(vec![(datanode_id, region_id)]) + .await; info!( "Table route is not found, the table is dropped, removed failover detector for region: {}, datanode: {}", region_id, datanode_id @@ -441,7 +453,8 @@ impl RegionSupervisor { Ok(()) } LeaderPeerChanged { .. } => { - self.failure_detector.remove(&(datanode_id, region_id)); + self.deregister_failure_detectors(vec![(datanode_id, region_id)]) + .await; info!( "Region's leader peer changed, removed failover detector for region: {}, datanode: {}", region_id, datanode_id From cc5629b4a18afe03397d7c2e1a3ea099f5bec04a Mon Sep 17 00:00:00 2001 From: dennis zhuang Date: Wed, 23 Apr 2025 19:15:44 -0700 Subject: [PATCH 71/82] chore: remove coderabbit (#5969) --- .coderabbit.yaml | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 .coderabbit.yaml diff --git a/.coderabbit.yaml b/.coderabbit.yaml deleted file mode 100644 index 01bc346444..0000000000 --- a/.coderabbit.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json -language: "en-US" -early_access: false -reviews: - profile: "chill" - request_changes_workflow: false - high_level_summary: true - poem: true - review_status: true - collapse_walkthrough: false - auto_review: - enabled: false - drafts: false -chat: - auto_reply: true From a533ac2555eaf2b41a27e48e6355138e3230a607 Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Thu, 24 Apr 2025 10:27:27 +0800 Subject: [PATCH 72/82] feat: enhance selector with node exclusion support (#5966) --- src/meta-srv/src/bootstrap.rs | 21 ++++++-- src/meta-srv/src/metasrv/builder.rs | 12 +++-- src/meta-srv/src/node_excluder.rs | 6 +++ src/meta-srv/src/selector.rs | 2 +- src/meta-srv/src/selector/lease_based.rs | 31 ++++++++++- src/meta-srv/src/selector/load_based.rs | 22 ++++++-- src/meta-srv/src/selector/round_robin.rs | 66 ++++++++++++++++++++---- 7 files changed, 136 insertions(+), 24 deletions(-) diff --git a/src/meta-srv/src/bootstrap.rs b/src/meta-srv/src/bootstrap.rs index 3b27295301..40e41bb815 100644 --- a/src/meta-srv/src/bootstrap.rs +++ b/src/meta-srv/src/bootstrap.rs @@ -66,10 +66,12 @@ use crate::election::postgres::PgElection; #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))] use crate::election::CANDIDATE_LEASE_SECS; use crate::metasrv::builder::MetasrvBuilder; -use crate::metasrv::{BackendImpl, Metasrv, MetasrvOptions, SelectorRef}; +use crate::metasrv::{BackendImpl, Metasrv, MetasrvOptions, SelectTarget, SelectorRef}; +use crate::node_excluder::NodeExcluderRef; use crate::selector::lease_based::LeaseBasedSelector; use crate::selector::load_based::LoadBasedSelector; use crate::selector::round_robin::RoundRobinSelector; +use crate::selector::weight_compute::RegionNumsBasedWeightCompute; use crate::selector::SelectorType; use crate::service::admin; use crate::{error, Result}; @@ -294,14 +296,25 @@ pub async fn metasrv_builder( let in_memory = Arc::new(MemoryKvBackend::new()) as ResettableKvBackendRef; + let node_excluder = plugins + .get::() + .unwrap_or_else(|| Arc::new(Vec::new()) as NodeExcluderRef); let selector = if let Some(selector) = plugins.get::() { info!("Using selector from plugins"); selector } else { let selector = match opts.selector { - SelectorType::LoadBased => Arc::new(LoadBasedSelector::default()) as SelectorRef, - SelectorType::LeaseBased => Arc::new(LeaseBasedSelector) as SelectorRef, - SelectorType::RoundRobin => Arc::new(RoundRobinSelector::default()) as SelectorRef, + SelectorType::LoadBased => Arc::new(LoadBasedSelector::new( + RegionNumsBasedWeightCompute, + node_excluder, + )) as SelectorRef, + SelectorType::LeaseBased => { + Arc::new(LeaseBasedSelector::new(node_excluder)) as SelectorRef + } + SelectorType::RoundRobin => Arc::new(RoundRobinSelector::new( + SelectTarget::Datanode, + node_excluder, + )) as SelectorRef, }; info!( "Using selector from options, selector type: {}", diff --git a/src/meta-srv/src/metasrv/builder.rs b/src/meta-srv/src/metasrv/builder.rs index ec8f6ef253..02f835e226 100644 --- a/src/meta-srv/src/metasrv/builder.rs +++ b/src/meta-srv/src/metasrv/builder.rs @@ -190,7 +190,7 @@ impl MetasrvBuilder { let meta_peer_client = meta_peer_client .unwrap_or_else(|| build_default_meta_peer_client(&election, &in_memory)); - let selector = selector.unwrap_or_else(|| Arc::new(LeaseBasedSelector)); + let selector = selector.unwrap_or_else(|| Arc::new(LeaseBasedSelector::default())); let pushers = Pushers::default(); let mailbox = build_mailbox(&kv_backend, &pushers); let procedure_manager = build_procedure_manager(&options, &kv_backend); @@ -234,13 +234,17 @@ impl MetasrvBuilder { )) }); + let flow_selector = Arc::new(RoundRobinSelector::new( + SelectTarget::Flownode, + Arc::new(Vec::new()), + )) as SelectorRef; + let flow_metadata_allocator = { // for now flownode just use round-robin selector - let flow_selector = RoundRobinSelector::new(SelectTarget::Flownode); let flow_selector_ctx = selector_ctx.clone(); let peer_allocator = Arc::new(FlowPeerAllocator::new( flow_selector_ctx, - Arc::new(flow_selector), + flow_selector.clone(), )); let seq = Arc::new( SequenceBuilder::new(FLOW_ID_SEQ, kv_backend.clone()) @@ -420,7 +424,7 @@ impl MetasrvBuilder { meta_peer_client: meta_peer_client.clone(), selector, // TODO(jeremy): We do not allow configuring the flow selector. - flow_selector: Arc::new(RoundRobinSelector::new(SelectTarget::Flownode)), + flow_selector, handler_group: RwLock::new(None), handler_group_builder: Mutex::new(Some(handler_group_builder)), election, diff --git a/src/meta-srv/src/node_excluder.rs b/src/meta-srv/src/node_excluder.rs index f9e892f092..a7bc6e0f69 100644 --- a/src/meta-srv/src/node_excluder.rs +++ b/src/meta-srv/src/node_excluder.rs @@ -24,3 +24,9 @@ pub trait NodeExcluder: Send + Sync { /// Returns the excluded datanode ids. fn excluded_datanode_ids(&self) -> &Vec; } + +impl NodeExcluder for Vec { + fn excluded_datanode_ids(&self) -> &Vec { + self + } +} diff --git a/src/meta-srv/src/selector.rs b/src/meta-srv/src/selector.rs index ce166ae05c..96fbda241d 100644 --- a/src/meta-srv/src/selector.rs +++ b/src/meta-srv/src/selector.rs @@ -18,7 +18,7 @@ pub mod load_based; pub mod round_robin; #[cfg(test)] pub(crate) mod test_utils; -mod weight_compute; +pub mod weight_compute; pub mod weighted_choose; use std::collections::HashSet; diff --git a/src/meta-srv/src/selector/lease_based.rs b/src/meta-srv/src/selector/lease_based.rs index e60157989d..448c26b08e 100644 --- a/src/meta-srv/src/selector/lease_based.rs +++ b/src/meta-srv/src/selector/lease_based.rs @@ -12,17 +12,37 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashSet; +use std::sync::Arc; + use common_meta::peer::Peer; use crate::error::Result; use crate::lease; use crate::metasrv::SelectorContext; +use crate::node_excluder::NodeExcluderRef; use crate::selector::common::{choose_items, filter_out_excluded_peers}; use crate::selector::weighted_choose::{RandomWeightedChoose, WeightedItem}; use crate::selector::{Selector, SelectorOptions}; /// Select all alive datanodes based using a random weighted choose. -pub struct LeaseBasedSelector; +pub struct LeaseBasedSelector { + node_excluder: NodeExcluderRef, +} + +impl LeaseBasedSelector { + pub fn new(node_excluder: NodeExcluderRef) -> Self { + Self { node_excluder } + } +} + +impl Default for LeaseBasedSelector { + fn default() -> Self { + Self { + node_excluder: Arc::new(Vec::new()), + } + } +} #[async_trait::async_trait] impl Selector for LeaseBasedSelector { @@ -47,7 +67,14 @@ impl Selector for LeaseBasedSelector { .collect(); // 3. choose peers by weight_array. - filter_out_excluded_peers(&mut weight_array, &opts.exclude_peer_ids); + let mut exclude_peer_ids = self + .node_excluder + .excluded_datanode_ids() + .iter() + .cloned() + .collect::>(); + exclude_peer_ids.extend(opts.exclude_peer_ids.iter()); + filter_out_excluded_peers(&mut weight_array, &exclude_peer_ids); let mut weighted_choose = RandomWeightedChoose::new(weight_array); let selected = choose_items(&opts, &mut weighted_choose)?; diff --git a/src/meta-srv/src/selector/load_based.rs b/src/meta-srv/src/selector/load_based.rs index d98a4ace5d..4f33245a28 100644 --- a/src/meta-srv/src/selector/load_based.rs +++ b/src/meta-srv/src/selector/load_based.rs @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; use common_meta::datanode::{DatanodeStatKey, DatanodeStatValue}; use common_meta::key::TableMetadataManager; @@ -26,6 +27,7 @@ use crate::error::{self, Result}; use crate::key::{DatanodeLeaseKey, LeaseValue}; use crate::lease; use crate::metasrv::SelectorContext; +use crate::node_excluder::NodeExcluderRef; use crate::selector::common::{choose_items, filter_out_excluded_peers}; use crate::selector::weight_compute::{RegionNumsBasedWeightCompute, WeightCompute}; use crate::selector::weighted_choose::RandomWeightedChoose; @@ -33,11 +35,15 @@ use crate::selector::{Selector, SelectorOptions}; pub struct LoadBasedSelector { weight_compute: C, + node_excluder: NodeExcluderRef, } impl LoadBasedSelector { - pub fn new(weight_compute: C) -> Self { - Self { weight_compute } + pub fn new(weight_compute: C, node_excluder: NodeExcluderRef) -> Self { + Self { + weight_compute, + node_excluder, + } } } @@ -45,6 +51,7 @@ impl Default for LoadBasedSelector { fn default() -> Self { Self { weight_compute: RegionNumsBasedWeightCompute, + node_excluder: Arc::new(Vec::new()), } } } @@ -88,7 +95,14 @@ where let mut weight_array = self.weight_compute.compute(&stat_kvs); // 5. choose peers by weight_array. - filter_out_excluded_peers(&mut weight_array, &opts.exclude_peer_ids); + let mut exclude_peer_ids = self + .node_excluder + .excluded_datanode_ids() + .iter() + .cloned() + .collect::>(); + exclude_peer_ids.extend(opts.exclude_peer_ids.iter()); + filter_out_excluded_peers(&mut weight_array, &exclude_peer_ids); let mut weighted_choose = RandomWeightedChoose::new(weight_array); let selected = choose_items(&opts, &mut weighted_choose)?; diff --git a/src/meta-srv/src/selector/round_robin.rs b/src/meta-srv/src/selector/round_robin.rs index 2c849cb194..d930ca06cb 100644 --- a/src/meta-srv/src/selector/round_robin.rs +++ b/src/meta-srv/src/selector/round_robin.rs @@ -12,7 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashSet; use std::sync::atomic::AtomicUsize; +use std::sync::Arc; use common_meta::peer::Peer; use snafu::ensure; @@ -20,6 +22,7 @@ use snafu::ensure; use crate::error::{NoEnoughAvailableNodeSnafu, Result}; use crate::lease; use crate::metasrv::{SelectTarget, SelectorContext}; +use crate::node_excluder::NodeExcluderRef; use crate::selector::{Selector, SelectorOptions}; /// Round-robin selector that returns the next peer in the list in sequence. @@ -32,6 +35,7 @@ use crate::selector::{Selector, SelectorOptions}; pub struct RoundRobinSelector { select_target: SelectTarget, counter: AtomicUsize, + node_excluder: NodeExcluderRef, } impl Default for RoundRobinSelector { @@ -39,32 +43,38 @@ impl Default for RoundRobinSelector { Self { select_target: SelectTarget::Datanode, counter: AtomicUsize::new(0), + node_excluder: Arc::new(Vec::new()), } } } impl RoundRobinSelector { - pub fn new(select_target: SelectTarget) -> Self { + pub fn new(select_target: SelectTarget, node_excluder: NodeExcluderRef) -> Self { Self { select_target, + node_excluder, ..Default::default() } } - async fn get_peers( - &self, - min_required_items: usize, - ctx: &SelectorContext, - ) -> Result> { + async fn get_peers(&self, opts: &SelectorOptions, ctx: &SelectorContext) -> Result> { let mut peers = match self.select_target { SelectTarget::Datanode => { // 1. get alive datanodes. let lease_kvs = lease::alive_datanodes(&ctx.meta_peer_client, ctx.datanode_lease_secs).await?; + let mut exclude_peer_ids = self + .node_excluder + .excluded_datanode_ids() + .iter() + .cloned() + .collect::>(); + exclude_peer_ids.extend(opts.exclude_peer_ids.iter()); // 2. map into peers lease_kvs .into_iter() + .filter(|(k, _)| !exclude_peer_ids.contains(&k.node_id)) .map(|(k, v)| Peer::new(k.node_id, v.node_addr)) .collect::>() } @@ -84,8 +94,8 @@ impl RoundRobinSelector { ensure!( !peers.is_empty(), NoEnoughAvailableNodeSnafu { - required: min_required_items, - available: 0usize, + required: opts.min_required_items, + available: peers.len(), select_target: self.select_target } ); @@ -103,7 +113,7 @@ impl Selector for RoundRobinSelector { type Output = Vec; async fn select(&self, ctx: &Self::Context, opts: SelectorOptions) -> Result> { - let peers = self.get_peers(opts.min_required_items, ctx).await?; + let peers = self.get_peers(&opts, ctx).await?; // choose peers let mut selected = Vec::with_capacity(opts.min_required_items); for _ in 0..opts.min_required_items { @@ -176,4 +186,42 @@ mod test { assert_eq!(peers.len(), 2); assert_eq!(peers, vec![peer2.clone(), peer3.clone()]); } + + #[tokio::test] + async fn test_round_robin_selector_with_exclude_peer_ids() { + let selector = RoundRobinSelector::new(SelectTarget::Datanode, Arc::new(vec![5])); + let ctx = create_selector_context(); + // add three nodes + let peer1 = Peer { + id: 2, + addr: "node1".to_string(), + }; + let peer2 = Peer { + id: 5, + addr: "node2".to_string(), + }; + let peer3 = Peer { + id: 8, + addr: "node3".to_string(), + }; + put_datanodes( + &ctx.meta_peer_client, + vec![peer1.clone(), peer2.clone(), peer3.clone()], + ) + .await; + + let peers = selector + .select( + &ctx, + SelectorOptions { + min_required_items: 1, + allow_duplication: true, + exclude_peer_ids: HashSet::from([2]), + }, + ) + .await + .unwrap(); + assert_eq!(peers.len(), 1); + assert_eq!(peers, vec![peer3.clone()]); + } } From ff3a46b1d0276d924663b6404f0ee4fdd32c25b3 Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Thu, 24 Apr 2025 12:00:14 +0800 Subject: [PATCH 73/82] feat: improve observability of region migration procedure (#5967) * feat: improve observability of region migration procedure * chore: apply suggestions from CR * chore: observe non-zero value --- src/meta-srv/src/metrics.rs | 9 ++ .../src/procedure/region_migration.rs | 124 +++++++++++++++++- .../close_downgraded_region.rs | 8 +- .../downgrade_leader_region.rs | 8 +- .../region_migration/migration_abort.rs | 11 +- .../region_migration/open_candidate_region.rs | 5 +- .../upgrade_candidate_region.rs | 9 +- 7 files changed, 160 insertions(+), 14 deletions(-) diff --git a/src/meta-srv/src/metrics.rs b/src/meta-srv/src/metrics.rs index ffbe986d72..2984a91a1c 100644 --- a/src/meta-srv/src/metrics.rs +++ b/src/meta-srv/src/metrics.rs @@ -71,4 +71,13 @@ lazy_static! { /// The remote WAL prune execute counter. pub static ref METRIC_META_REMOTE_WAL_PRUNE_EXECUTE: IntCounterVec = register_int_counter_vec!("greptime_meta_remote_wal_prune_execute", "meta remote wal prune execute", &["topic_name"]).unwrap(); + /// The migration stage elapsed histogram. + pub static ref METRIC_META_REGION_MIGRATION_STAGE_ELAPSED: HistogramVec = register_histogram_vec!( + "greptime_meta_region_migration_stage_elapsed", + "meta region migration stage elapsed", + &["stage"], + // 0.01 ~ 1000 + exponential_buckets(0.01, 10.0, 7).unwrap(), + ) + .unwrap(); } diff --git a/src/meta-srv/src/procedure/region_migration.rs b/src/meta-srv/src/procedure/region_migration.rs index b2f1eed711..43b444d3b1 100644 --- a/src/meta-srv/src/procedure/region_migration.rs +++ b/src/meta-srv/src/procedure/region_migration.rs @@ -25,7 +25,7 @@ pub(crate) mod update_metadata; pub(crate) mod upgrade_candidate_region; use std::any::Any; -use std::fmt::Debug; +use std::fmt::{Debug, Display}; use std::time::Duration; use common_error::ext::BoxedError; @@ -43,7 +43,7 @@ use common_procedure::error::{ Error as ProcedureError, FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu, }; use common_procedure::{Context as ProcedureContext, LockKey, Procedure, Status, StringKey}; -use common_telemetry::info; +use common_telemetry::{error, info}; use manager::RegionMigrationProcedureGuard; pub use manager::{ RegionMigrationManagerRef, RegionMigrationProcedureTask, RegionMigrationProcedureTracker, @@ -55,7 +55,10 @@ use tokio::time::Instant; use self::migration_start::RegionMigrationStart; use crate::error::{self, Result}; -use crate::metrics::{METRIC_META_REGION_MIGRATION_ERROR, METRIC_META_REGION_MIGRATION_EXECUTE}; +use crate::metrics::{ + METRIC_META_REGION_MIGRATION_ERROR, METRIC_META_REGION_MIGRATION_EXECUTE, + METRIC_META_REGION_MIGRATION_STAGE_ELAPSED, +}; use crate::service::mailbox::MailboxRef; /// The default timeout for region migration. @@ -103,6 +106,82 @@ impl PersistentContext { } } +/// Metrics of region migration. +#[derive(Debug, Clone, Default)] +pub struct Metrics { + /// Elapsed time of downgrading region and upgrading region. + operations_elapsed: Duration, + /// Elapsed time of downgrading leader region. + downgrade_leader_region_elapsed: Duration, + /// Elapsed time of open candidate region. + open_candidate_region_elapsed: Duration, + /// Elapsed time of upgrade candidate region. + upgrade_candidate_region_elapsed: Duration, +} + +impl Display for Metrics { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "operations_elapsed: {:?}, downgrade_leader_region_elapsed: {:?}, open_candidate_region_elapsed: {:?}, upgrade_candidate_region_elapsed: {:?}", + self.operations_elapsed, + self.downgrade_leader_region_elapsed, + self.open_candidate_region_elapsed, + self.upgrade_candidate_region_elapsed + ) + } +} + +impl Metrics { + /// Updates the elapsed time of downgrading region and upgrading region. + pub fn update_operations_elapsed(&mut self, elapsed: Duration) { + self.operations_elapsed += elapsed; + } + + /// Updates the elapsed time of downgrading leader region. + pub fn update_downgrade_leader_region_elapsed(&mut self, elapsed: Duration) { + self.downgrade_leader_region_elapsed += elapsed; + } + + /// Updates the elapsed time of open candidate region. + pub fn update_open_candidate_region_elapsed(&mut self, elapsed: Duration) { + self.open_candidate_region_elapsed += elapsed; + } + + /// Updates the elapsed time of upgrade candidate region. + pub fn update_upgrade_candidate_region_elapsed(&mut self, elapsed: Duration) { + self.upgrade_candidate_region_elapsed += elapsed; + } +} + +impl Drop for Metrics { + fn drop(&mut self) { + if !self.operations_elapsed.is_zero() { + METRIC_META_REGION_MIGRATION_STAGE_ELAPSED + .with_label_values(&["operations"]) + .observe(self.operations_elapsed.as_secs_f64()); + } + + if !self.downgrade_leader_region_elapsed.is_zero() { + METRIC_META_REGION_MIGRATION_STAGE_ELAPSED + .with_label_values(&["downgrade_leader_region"]) + .observe(self.downgrade_leader_region_elapsed.as_secs_f64()); + } + + if !self.open_candidate_region_elapsed.is_zero() { + METRIC_META_REGION_MIGRATION_STAGE_ELAPSED + .with_label_values(&["open_candidate_region"]) + .observe(self.open_candidate_region_elapsed.as_secs_f64()); + } + + if !self.upgrade_candidate_region_elapsed.is_zero() { + METRIC_META_REGION_MIGRATION_STAGE_ELAPSED + .with_label_values(&["upgrade_candidate_region"]) + .observe(self.upgrade_candidate_region_elapsed.as_secs_f64()); + } + } +} + /// It's shared in each step and available in executing (including retrying). /// /// It will be dropped if the procedure runner crashes. @@ -132,8 +211,8 @@ pub struct VolatileContext { leader_region_last_entry_id: Option, /// The last_entry_id of leader metadata region (Only used for metric engine). leader_region_metadata_last_entry_id: Option, - /// Elapsed time of downgrading region and upgrading region. - operations_elapsed: Duration, + /// Metrics of region migration. + metrics: Metrics, } impl VolatileContext { @@ -231,12 +310,35 @@ impl Context { pub fn next_operation_timeout(&self) -> Option { self.persistent_ctx .timeout - .checked_sub(self.volatile_ctx.operations_elapsed) + .checked_sub(self.volatile_ctx.metrics.operations_elapsed) } /// Updates operations elapsed. pub fn update_operations_elapsed(&mut self, instant: Instant) { - self.volatile_ctx.operations_elapsed += instant.elapsed(); + self.volatile_ctx + .metrics + .update_operations_elapsed(instant.elapsed()); + } + + /// Updates the elapsed time of downgrading leader region. + pub fn update_downgrade_leader_region_elapsed(&mut self, instant: Instant) { + self.volatile_ctx + .metrics + .update_downgrade_leader_region_elapsed(instant.elapsed()); + } + + /// Updates the elapsed time of open candidate region. + pub fn update_open_candidate_region_elapsed(&mut self, instant: Instant) { + self.volatile_ctx + .metrics + .update_open_candidate_region_elapsed(instant.elapsed()); + } + + /// Updates the elapsed time of upgrade candidate region. + pub fn update_upgrade_candidate_region_elapsed(&mut self, instant: Instant) { + self.volatile_ctx + .metrics + .update_upgrade_candidate_region_elapsed(instant.elapsed()); } /// Returns address of meta server. @@ -550,6 +652,14 @@ impl Procedure for RegionMigrationProcedure { .inc(); ProcedureError::retry_later(e) } else { + error!( + e; + "Region migration procedure failed, region_id: {}, from_peer: {}, to_peer: {}, {}", + self.context.region_id(), + self.context.persistent_ctx.from_peer, + self.context.persistent_ctx.to_peer, + self.context.volatile_ctx.metrics, + ); METRIC_META_REGION_MIGRATION_ERROR .with_label_values(&[name, "external"]) .inc(); diff --git a/src/meta-srv/src/procedure/region_migration/close_downgraded_region.rs b/src/meta-srv/src/procedure/region_migration/close_downgraded_region.rs index 94256ba5ec..ba13f7cdea 100644 --- a/src/meta-srv/src/procedure/region_migration/close_downgraded_region.rs +++ b/src/meta-srv/src/procedure/region_migration/close_downgraded_region.rs @@ -46,7 +46,13 @@ impl State for CloseDowngradedRegion { let region_id = ctx.region_id(); warn!(err; "Failed to close downgraded leader region: {region_id} on datanode {:?}", downgrade_leader_datanode); } - + info!( + "Region migration is finished: region_id: {}, from_peer: {}, to_peer: {}, {}", + ctx.region_id(), + ctx.persistent_ctx.from_peer, + ctx.persistent_ctx.to_peer, + ctx.volatile_ctx.metrics, + ); Ok((Box::new(RegionMigrationEnd), Status::done())) } diff --git a/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs b/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs index 02b7216fe7..93481adc54 100644 --- a/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs +++ b/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs @@ -54,6 +54,7 @@ impl Default for DowngradeLeaderRegion { #[typetag::serde] impl State for DowngradeLeaderRegion { async fn next(&mut self, ctx: &mut Context) -> Result<(Box, Status)> { + let now = Instant::now(); // Ensures the `leader_region_lease_deadline` must exist after recovering. ctx.volatile_ctx .set_leader_region_lease_deadline(Duration::from_secs(REGION_LEASE_SECS)); @@ -77,6 +78,7 @@ impl State for DowngradeLeaderRegion { } } } + ctx.update_downgrade_leader_region_elapsed(now); Ok(( Box::new(UpgradeCandidateRegion::default()), @@ -348,7 +350,8 @@ mod tests { let env = TestingEnv::new(); let mut ctx = env.context_factory().new_context(persistent_context); prepare_table_metadata(&ctx, HashMap::default()).await; - ctx.volatile_ctx.operations_elapsed = ctx.persistent_ctx.timeout + Duration::from_secs(1); + ctx.volatile_ctx.metrics.operations_elapsed = + ctx.persistent_ctx.timeout + Duration::from_secs(1); let err = state.downgrade_region(&mut ctx).await.unwrap_err(); @@ -591,7 +594,8 @@ mod tests { let mut ctx = env.context_factory().new_context(persistent_context); let mailbox_ctx = env.mailbox_context(); let mailbox = mailbox_ctx.mailbox().clone(); - ctx.volatile_ctx.operations_elapsed = ctx.persistent_ctx.timeout + Duration::from_secs(1); + ctx.volatile_ctx.metrics.operations_elapsed = + ctx.persistent_ctx.timeout + Duration::from_secs(1); let (tx, rx) = tokio::sync::mpsc::channel(1); mailbox_ctx diff --git a/src/meta-srv/src/procedure/region_migration/migration_abort.rs b/src/meta-srv/src/procedure/region_migration/migration_abort.rs index af56843045..d364f0c8b9 100644 --- a/src/meta-srv/src/procedure/region_migration/migration_abort.rs +++ b/src/meta-srv/src/procedure/region_migration/migration_abort.rs @@ -15,6 +15,7 @@ use std::any::Any; use common_procedure::Status; +use common_telemetry::warn; use serde::{Deserialize, Serialize}; use crate::error::{self, Result}; @@ -37,7 +38,15 @@ impl RegionMigrationAbort { #[async_trait::async_trait] #[typetag::serde] impl State for RegionMigrationAbort { - async fn next(&mut self, _: &mut Context) -> Result<(Box, Status)> { + async fn next(&mut self, ctx: &mut Context) -> Result<(Box, Status)> { + warn!( + "Region migration is aborted: {}, region_id: {}, from_peer: {}, to_peer: {}, {}", + self.reason, + ctx.region_id(), + ctx.persistent_ctx.from_peer, + ctx.persistent_ctx.to_peer, + ctx.volatile_ctx.metrics, + ); error::MigrationAbortSnafu { reason: &self.reason, } diff --git a/src/meta-srv/src/procedure/region_migration/open_candidate_region.rs b/src/meta-srv/src/procedure/region_migration/open_candidate_region.rs index 6cacf75063..6d1c81d3ed 100644 --- a/src/meta-srv/src/procedure/region_migration/open_candidate_region.rs +++ b/src/meta-srv/src/procedure/region_migration/open_candidate_region.rs @@ -13,7 +13,7 @@ // limitations under the License. use std::any::Any; -use std::time::{Duration, Instant}; +use std::time::Duration; use api::v1::meta::MailboxMessage; use common_meta::distributed_time_constants::REGION_LEASE_SECS; @@ -24,6 +24,7 @@ use common_procedure::Status; use common_telemetry::info; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt}; +use tokio::time::Instant; use crate::error::{self, Result}; use crate::handler::HeartbeatMailbox; @@ -42,7 +43,9 @@ pub struct OpenCandidateRegion; impl State for OpenCandidateRegion { async fn next(&mut self, ctx: &mut Context) -> Result<(Box, Status)> { let instruction = self.build_open_region_instruction(ctx).await?; + let now = Instant::now(); self.open_candidate_region(ctx, instruction).await?; + ctx.update_open_candidate_region_elapsed(now); Ok(( Box::new(UpdateMetadata::Downgrade), diff --git a/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs b/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs index 552b9d3863..8f3741dbac 100644 --- a/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs +++ b/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs @@ -54,9 +54,12 @@ impl Default for UpgradeCandidateRegion { #[typetag::serde] impl State for UpgradeCandidateRegion { async fn next(&mut self, ctx: &mut Context) -> Result<(Box, Status)> { + let now = Instant::now(); if self.upgrade_region_with_retry(ctx).await { + ctx.update_upgrade_candidate_region_elapsed(now); Ok((Box::new(UpdateMetadata::Upgrade), Status::executing(false))) } else { + ctx.update_upgrade_candidate_region_elapsed(now); Ok((Box::new(UpdateMetadata::Rollback), Status::executing(false))) } } @@ -288,7 +291,8 @@ mod tests { let persistent_context = new_persistent_context(); let env = TestingEnv::new(); let mut ctx = env.context_factory().new_context(persistent_context); - ctx.volatile_ctx.operations_elapsed = ctx.persistent_ctx.timeout + Duration::from_secs(1); + ctx.volatile_ctx.metrics.operations_elapsed = + ctx.persistent_ctx.timeout + Duration::from_secs(1); let err = state.upgrade_region(&ctx).await.unwrap_err(); @@ -558,7 +562,8 @@ mod tests { let mut ctx = env.context_factory().new_context(persistent_context); let mailbox_ctx = env.mailbox_context(); let mailbox = mailbox_ctx.mailbox().clone(); - ctx.volatile_ctx.operations_elapsed = ctx.persistent_ctx.timeout + Duration::from_secs(1); + ctx.volatile_ctx.metrics.operations_elapsed = + ctx.persistent_ctx.timeout + Duration::from_secs(1); let (tx, rx) = tokio::sync::mpsc::channel(1); mailbox_ctx From b476584f56df1a3d18ace00835451755a2b3f6d7 Mon Sep 17 00:00:00 2001 From: Ruihang Xia Date: Thu, 24 Apr 2025 15:17:10 +0800 Subject: [PATCH 74/82] feat: remove hyper parameter from promql functions (#5955) * quantile udaf Signed-off-by: Ruihang Xia * extrapolate rate Signed-off-by: Ruihang Xia * predict_linear, round, holt_winters, quantile_overtime Signed-off-by: Ruihang Xia * fix clippy Signed-off-by: Ruihang Xia * fix quantile function Signed-off-by: Ruihang Xia --------- Signed-off-by: Ruihang Xia --- src/promql/src/functions.rs | 12 +- src/promql/src/functions/aggr_over_time.rs | 12 ++ src/promql/src/functions/changes.rs | 3 + src/promql/src/functions/deriv.rs | 2 + src/promql/src/functions/extrapolate_rate.rs | 51 ++++++-- src/promql/src/functions/holt_winters.rs | 48 +++++++- src/promql/src/functions/idelta.rs | 2 + src/promql/src/functions/predict_linear.rs | 41 +++++-- src/promql/src/functions/quantile.rs | 21 +++- src/promql/src/functions/quantile_aggr.rs | 80 +++++++++---- src/promql/src/functions/resets.rs | 3 + src/promql/src/functions/round.rs | 30 ++++- src/promql/src/functions/test_util.rs | 9 +- src/query/src/promql/planner.rs | 112 ++++++------------ .../standalone/common/promql/quantile.result | 56 ++++----- .../standalone/common/promql/round_fn.result | 80 ++++++------- .../common/promql/simple_histogram.result | 42 +++---- .../standalone/common/promql/subquery.result | 20 ++-- .../common/tql-explain-analyze/analyze.result | 4 +- 19 files changed, 383 insertions(+), 245 deletions(-) diff --git a/src/promql/src/functions.rs b/src/promql/src/functions.rs index dade00ea7b..81a9c9cedb 100644 --- a/src/promql/src/functions.rs +++ b/src/promql/src/functions.rs @@ -44,13 +44,13 @@ pub use quantile_aggr::quantile_udaf; pub use resets::Resets; pub use round::Round; +/// Extracts an array from a `ColumnarValue`. +/// +/// If the `ColumnarValue` is a scalar, it converts it to an array of size 1. pub(crate) fn extract_array(columnar_value: &ColumnarValue) -> Result { - if let ColumnarValue::Array(array) = columnar_value { - Ok(array.clone()) - } else { - Err(DataFusionError::Execution( - "expect array as input, found scalar value".to_string(), - )) + match columnar_value { + ColumnarValue::Array(array) => Ok(array.clone()), + ColumnarValue::Scalar(scalar) => Ok(scalar.to_array_of_size(1)?), } } diff --git a/src/promql/src/functions/aggr_over_time.rs b/src/promql/src/functions/aggr_over_time.rs index 298959ef35..841f28e0df 100644 --- a/src/promql/src/functions/aggr_over_time.rs +++ b/src/promql/src/functions/aggr_over_time.rs @@ -231,6 +231,7 @@ mod test { AvgOverTime::scalar_udf(), ts_array, value_array, + vec![], vec![ Some(49.9999995), Some(45.8618844), @@ -253,6 +254,7 @@ mod test { MinOverTime::scalar_udf(), ts_array, value_array, + vec![], vec![ Some(12.345678), Some(12.345678), @@ -275,6 +277,7 @@ mod test { MaxOverTime::scalar_udf(), ts_array, value_array, + vec![], vec![ Some(87.654321), Some(87.654321), @@ -297,6 +300,7 @@ mod test { SumOverTime::scalar_udf(), ts_array, value_array, + vec![], vec![ Some(99.999999), Some(229.309422), @@ -319,6 +323,7 @@ mod test { CountOverTime::scalar_udf(), ts_array, value_array, + vec![], vec![ Some(2.0), Some(5.0), @@ -341,6 +346,7 @@ mod test { LastOverTime::scalar_udf(), ts_array, value_array, + vec![], vec![ Some(87.654321), Some(70.710678), @@ -363,6 +369,7 @@ mod test { AbsentOverTime::scalar_udf(), ts_array, value_array, + vec![], vec![ None, None, @@ -385,6 +392,7 @@ mod test { PresentOverTime::scalar_udf(), ts_array, value_array, + vec![], vec![ Some(1.0), Some(1.0), @@ -407,6 +415,7 @@ mod test { StdvarOverTime::scalar_udf(), ts_array, value_array, + vec![], vec![ Some(1417.8479276253622), Some(808.999919713209), @@ -442,6 +451,7 @@ mod test { StdvarOverTime::scalar_udf(), RangeArray::from_ranges(ts_array, ranges).unwrap(), RangeArray::from_ranges(values_array, ranges).unwrap(), + vec![], vec![Some(0.0), Some(10.559999999999999)], ); } @@ -453,6 +463,7 @@ mod test { StddevOverTime::scalar_udf(), ts_array, value_array, + vec![], vec![ Some(37.6543215), Some(28.442923895289123), @@ -488,6 +499,7 @@ mod test { StddevOverTime::scalar_udf(), RangeArray::from_ranges(ts_array, ranges).unwrap(), RangeArray::from_ranges(values_array, ranges).unwrap(), + vec![], vec![Some(0.0), Some(3.249615361854384)], ); } diff --git a/src/promql/src/functions/changes.rs b/src/promql/src/functions/changes.rs index 743f941652..21819436e6 100644 --- a/src/promql/src/functions/changes.rs +++ b/src/promql/src/functions/changes.rs @@ -90,6 +90,7 @@ mod test { Changes::scalar_udf(), ts_array_1, value_array_1, + vec![], vec![Some(0.0), Some(3.0), Some(5.0), Some(8.0), None], ); @@ -101,6 +102,7 @@ mod test { Changes::scalar_udf(), ts_array_2, value_array_2, + vec![], vec![Some(0.0), Some(3.0), Some(5.0), Some(9.0), None], ); @@ -111,6 +113,7 @@ mod test { Changes::scalar_udf(), ts_array_3, value_array_3, + vec![], vec![Some(0.0), Some(0.0), Some(1.0), Some(1.0), None], ); } diff --git a/src/promql/src/functions/deriv.rs b/src/promql/src/functions/deriv.rs index 90b09f0d40..49e6718911 100644 --- a/src/promql/src/functions/deriv.rs +++ b/src/promql/src/functions/deriv.rs @@ -74,6 +74,7 @@ mod test { Deriv::scalar_udf(), ts_array, value_array, + vec![], vec![Some(10.606060606060607), None], ); } @@ -99,6 +100,7 @@ mod test { Deriv::scalar_udf(), ts_range_array, value_range_array, + vec![], vec![Some(0.0)], ); } diff --git a/src/promql/src/functions/extrapolate_rate.rs b/src/promql/src/functions/extrapolate_rate.rs index 8977eaf083..bdcd6a2b1b 100644 --- a/src/promql/src/functions/extrapolate_rate.rs +++ b/src/promql/src/functions/extrapolate_rate.rs @@ -34,11 +34,11 @@ use std::sync::Arc; use datafusion::arrow::array::{Float64Array, TimestampMillisecondArray}; use datafusion::arrow::datatypes::TimeUnit; -use datafusion::common::DataFusionError; +use datafusion::common::{DataFusionError, Result as DfResult}; use datafusion::logical_expr::{ScalarUDF, Volatility}; use datafusion::physical_plan::ColumnarValue; use datafusion_expr::create_udf; -use datatypes::arrow::array::Array; +use datatypes::arrow::array::{Array, Int64Array}; use datatypes::arrow::datatypes::DataType; use crate::extension_plan::Millisecond; @@ -53,7 +53,7 @@ pub type Increase = ExtrapolatedRate; /// from #[derive(Debug)] pub struct ExtrapolatedRate { - /// Range duration in millisecond + /// Range length in milliseconds. range_length: i64, } @@ -63,7 +63,7 @@ impl ExtrapolatedRate ScalarUDF { + fn scalar_udf_with_name(name: &str) -> ScalarUDF { let input_types = vec![ // timestamp range vector RangeArray::convert_data_type(DataType::Timestamp(TimeUnit::Millisecond, None)), @@ -71,6 +71,8 @@ impl ExtrapolatedRate ExtrapolatedRate Result { - assert_eq!(input.len(), 3); + fn create_function(inputs: &[ColumnarValue]) -> DfResult { + if inputs.len() != 4 { + return Err(DataFusionError::Plan( + "ExtrapolatedRate function should have 4 inputs".to_string(), + )); + } + + let range_length_array = extract_array(&inputs[3])?; + let range_length = range_length_array + .as_any() + .downcast_ref::() + .unwrap() + .value(0) as i64; + + Ok(Self::new(range_length)) + } + + /// Input parameters: + /// * 0: timestamp range vector + /// * 1: value range vector + /// * 2: timestamp vector + /// * 3: range length. Range duration in millisecond. Not used here + fn calc(&self, input: &[ColumnarValue]) -> DfResult { + assert_eq!(input.len(), 4); // construct matrix from input let ts_array = extract_array(&input[0])?; @@ -208,8 +232,8 @@ impl ExtrapolatedRate { "prom_delta" } - pub fn scalar_udf(range_length: i64) -> ScalarUDF { - Self::scalar_udf_with_name(Self::name(), range_length) + pub fn scalar_udf() -> ScalarUDF { + Self::scalar_udf_with_name(Self::name()) } } @@ -219,8 +243,8 @@ impl ExtrapolatedRate { "prom_rate" } - pub fn scalar_udf(range_length: i64) -> ScalarUDF { - Self::scalar_udf_with_name(Self::name(), range_length) + pub fn scalar_udf() -> ScalarUDF { + Self::scalar_udf_with_name(Self::name()) } } @@ -230,8 +254,8 @@ impl ExtrapolatedRate { "prom_increase" } - pub fn scalar_udf(range_length: i64) -> ScalarUDF { - Self::scalar_udf_with_name(Self::name(), range_length) + pub fn scalar_udf() -> ScalarUDF { + Self::scalar_udf_with_name(Self::name()) } } @@ -271,6 +295,7 @@ mod test { ColumnarValue::Array(Arc::new(ts_range.into_dict())), ColumnarValue::Array(Arc::new(value_range.into_dict())), ColumnarValue::Array(timestamps), + ColumnarValue::Array(Arc::new(Int64Array::from(vec![5]))), ]; let output = extract_array( &ExtrapolatedRate::::new(5) diff --git a/src/promql/src/functions/holt_winters.rs b/src/promql/src/functions/holt_winters.rs index 3f26abffb1..8e722c8651 100644 --- a/src/promql/src/functions/holt_winters.rs +++ b/src/promql/src/functions/holt_winters.rs @@ -22,6 +22,7 @@ use datafusion::arrow::datatypes::TimeUnit; use datafusion::common::DataFusionError; use datafusion::logical_expr::{ScalarUDF, Volatility}; use datafusion::physical_plan::ColumnarValue; +use datafusion_common::ScalarValue; use datafusion_expr::create_udf; use datatypes::arrow::array::Array; use datatypes::arrow::datatypes::DataType; @@ -62,6 +63,10 @@ impl HoltWinters { vec![ RangeArray::convert_data_type(DataType::Timestamp(TimeUnit::Millisecond, None)), RangeArray::convert_data_type(DataType::Float64), + // sf + DataType::Float64, + // tf + DataType::Float64, ] } @@ -69,20 +74,39 @@ impl HoltWinters { DataType::Float64 } - pub fn scalar_udf(level: f64, trend: f64) -> ScalarUDF { + pub fn scalar_udf() -> ScalarUDF { create_udf( Self::name(), Self::input_type(), Self::return_type(), Volatility::Volatile, - Arc::new(move |input: &_| Self::new(level, trend).calc(input)) as _, + Arc::new(move |input: &_| Self::create_function(input)?.calc(input)) as _, ) } + fn create_function(inputs: &[ColumnarValue]) -> Result { + if inputs.len() != 4 { + return Err(DataFusionError::Plan( + "HoltWinters function should have 4 inputs".to_string(), + )); + } + let ColumnarValue::Scalar(ScalarValue::Float64(Some(sf))) = inputs[2] else { + return Err(DataFusionError::Plan( + "HoltWinters function's third input should be a scalar float64".to_string(), + )); + }; + let ColumnarValue::Scalar(ScalarValue::Float64(Some(tf))) = inputs[3] else { + return Err(DataFusionError::Plan( + "HoltWinters function's fourth input should be a scalar float64".to_string(), + )); + }; + Ok(Self::new(sf, tf)) + } + fn calc(&self, input: &[ColumnarValue]) -> Result { // construct matrix from input. // The third one is level param, the fourth - trend param which are included in fields. - assert_eq!(input.len(), 2); + assert_eq!(input.len(), 4); let ts_array = extract_array(&input[0])?; let value_array = extract_array(&input[1])?; @@ -264,9 +288,13 @@ mod tests { let ts_range_array = RangeArray::from_ranges(ts_array, ranges).unwrap(); let value_range_array = RangeArray::from_ranges(values_array, ranges).unwrap(); simple_range_udf_runner( - HoltWinters::scalar_udf(0.5, 0.1), + HoltWinters::scalar_udf(), ts_range_array, value_range_array, + vec![ + ScalarValue::Float64(Some(0.5)), + ScalarValue::Float64(Some(0.1)), + ], vec![Some(5.0)], ); } @@ -287,9 +315,13 @@ mod tests { let ts_range_array = RangeArray::from_ranges(ts_array, ranges).unwrap(); let value_range_array = RangeArray::from_ranges(values_array, ranges).unwrap(); simple_range_udf_runner( - HoltWinters::scalar_udf(0.5, 0.1), + HoltWinters::scalar_udf(), ts_range_array, value_range_array, + vec![ + ScalarValue::Float64(Some(0.5)), + ScalarValue::Float64(Some(0.1)), + ], vec![Some(38.18119566835938)], ); } @@ -315,9 +347,13 @@ mod tests { let (ts_range_array, value_range_array) = create_ts_and_value_range_arrays(query, ranges.clone()); simple_range_udf_runner( - HoltWinters::scalar_udf(0.01, 0.1), + HoltWinters::scalar_udf(), ts_range_array, value_range_array, + vec![ + ScalarValue::Float64(Some(0.01)), + ScalarValue::Float64(Some(0.1)), + ], vec![Some(expected)], ); } diff --git a/src/promql/src/functions/idelta.rs b/src/promql/src/functions/idelta.rs index c5d1897a3e..a70a1dee3c 100644 --- a/src/promql/src/functions/idelta.rs +++ b/src/promql/src/functions/idelta.rs @@ -190,6 +190,7 @@ mod test { IDelta::::scalar_udf(), ts_range_array, value_range_array, + vec![], vec![Some(1.0), Some(-5.0), None, Some(6.0), None, None], ); @@ -200,6 +201,7 @@ mod test { IDelta::::scalar_udf(), ts_range_array, value_range_array, + vec![], // the second point represent counter reset vec![Some(0.5), Some(0.0), None, Some(3.0), None, None], ); diff --git a/src/promql/src/functions/predict_linear.rs b/src/promql/src/functions/predict_linear.rs index 4b945cabbb..d3c1e8214c 100644 --- a/src/promql/src/functions/predict_linear.rs +++ b/src/promql/src/functions/predict_linear.rs @@ -22,6 +22,7 @@ use datafusion::arrow::datatypes::TimeUnit; use datafusion::common::DataFusionError; use datafusion::logical_expr::{ScalarUDF, Volatility}; use datafusion::physical_plan::ColumnarValue; +use datafusion_common::ScalarValue; use datafusion_expr::create_udf; use datatypes::arrow::array::Array; use datatypes::arrow::datatypes::DataType; @@ -44,25 +45,41 @@ impl PredictLinear { "prom_predict_linear" } - pub fn scalar_udf(t: i64) -> ScalarUDF { + pub fn scalar_udf() -> ScalarUDF { let input_types = vec![ // time index column RangeArray::convert_data_type(DataType::Timestamp(TimeUnit::Millisecond, None)), // value column RangeArray::convert_data_type(DataType::Float64), + // t + DataType::Int64, ]; create_udf( Self::name(), input_types, DataType::Float64, Volatility::Volatile, - Arc::new(move |input: &_| Self::new(t).predict_linear(input)) as _, + Arc::new(move |input: &_| Self::create_function(input)?.predict_linear(input)) as _, ) } + fn create_function(inputs: &[ColumnarValue]) -> Result { + if inputs.len() != 3 { + return Err(DataFusionError::Plan( + "PredictLinear function should have 3 inputs".to_string(), + )); + } + let ColumnarValue::Scalar(ScalarValue::Int64(Some(t))) = inputs[2] else { + return Err(DataFusionError::Plan( + "PredictLinear function's third input should be a scalar int64".to_string(), + )); + }; + Ok(Self::new(t)) + } + fn predict_linear(&self, input: &[ColumnarValue]) -> Result { // construct matrix from input. - assert_eq!(input.len(), 2); + assert_eq!(input.len(), 3); let ts_array = extract_array(&input[0])?; let value_array = extract_array(&input[1])?; @@ -190,9 +207,10 @@ mod test { let ts_array = RangeArray::from_ranges(ts_array, ranges).unwrap(); let value_array = RangeArray::from_ranges(values_array, ranges).unwrap(); simple_range_udf_runner( - PredictLinear::scalar_udf(0), + PredictLinear::scalar_udf(), ts_array, value_array, + vec![ScalarValue::Int64(Some(0))], vec![None, None], ); } @@ -201,9 +219,10 @@ mod test { fn calculate_predict_linear_test1() { let (ts_array, value_array) = build_test_range_arrays(); simple_range_udf_runner( - PredictLinear::scalar_udf(0), + PredictLinear::scalar_udf(), ts_array, value_array, + vec![ScalarValue::Int64(Some(0))], // value at t = 0 vec![Some(38.63636363636364)], ); @@ -213,9 +232,10 @@ mod test { fn calculate_predict_linear_test2() { let (ts_array, value_array) = build_test_range_arrays(); simple_range_udf_runner( - PredictLinear::scalar_udf(3000), + PredictLinear::scalar_udf(), ts_array, value_array, + vec![ScalarValue::Int64(Some(3000))], // value at t = 3000 vec![Some(31856.818181818187)], ); @@ -225,9 +245,10 @@ mod test { fn calculate_predict_linear_test3() { let (ts_array, value_array) = build_test_range_arrays(); simple_range_udf_runner( - PredictLinear::scalar_udf(4200), + PredictLinear::scalar_udf(), ts_array, value_array, + vec![ScalarValue::Int64(Some(4200))], // value at t = 4200 vec![Some(44584.09090909091)], ); @@ -237,9 +258,10 @@ mod test { fn calculate_predict_linear_test4() { let (ts_array, value_array) = build_test_range_arrays(); simple_range_udf_runner( - PredictLinear::scalar_udf(6600), + PredictLinear::scalar_udf(), ts_array, value_array, + vec![ScalarValue::Int64(Some(6600))], // value at t = 6600 vec![Some(70038.63636363638)], ); @@ -249,9 +271,10 @@ mod test { fn calculate_predict_linear_test5() { let (ts_array, value_array) = build_test_range_arrays(); simple_range_udf_runner( - PredictLinear::scalar_udf(7800), + PredictLinear::scalar_udf(), ts_array, value_array, + vec![ScalarValue::Int64(Some(7800))], // value at t = 7800 vec![Some(82765.9090909091)], ); diff --git a/src/promql/src/functions/quantile.rs b/src/promql/src/functions/quantile.rs index f975a76cf4..7fd553287d 100644 --- a/src/promql/src/functions/quantile.rs +++ b/src/promql/src/functions/quantile.rs @@ -19,6 +19,7 @@ use datafusion::arrow::datatypes::TimeUnit; use datafusion::common::DataFusionError; use datafusion::logical_expr::{ScalarUDF, Volatility}; use datafusion::physical_plan::ColumnarValue; +use datafusion_common::ScalarValue; use datafusion_expr::create_udf; use datatypes::arrow::array::Array; use datatypes::arrow::datatypes::DataType; @@ -40,22 +41,38 @@ impl QuantileOverTime { "prom_quantile_over_time" } - pub fn scalar_udf(quantile: f64) -> ScalarUDF { + pub fn scalar_udf() -> ScalarUDF { let input_types = vec![ // time index column RangeArray::convert_data_type(DataType::Timestamp(TimeUnit::Millisecond, None)), // value column RangeArray::convert_data_type(DataType::Float64), + // quantile + DataType::Float64, ]; create_udf( Self::name(), input_types, DataType::Float64, Volatility::Volatile, - Arc::new(move |input: &_| Self::new(quantile).quantile_over_time(input)) as _, + Arc::new(move |input: &_| Self::create_function(input)?.quantile_over_time(input)) as _, ) } + fn create_function(inputs: &[ColumnarValue]) -> Result { + if inputs.len() != 3 { + return Err(DataFusionError::Plan( + "QuantileOverTime function should have 3 inputs".to_string(), + )); + } + let ColumnarValue::Scalar(ScalarValue::Float64(Some(quantile))) = inputs[2] else { + return Err(DataFusionError::Plan( + "QuantileOverTime function's third input should be a scalar float64".to_string(), + )); + }; + Ok(Self::new(quantile)) + } + fn quantile_over_time( &self, input: &[ColumnarValue], diff --git a/src/promql/src/functions/quantile_aggr.rs b/src/promql/src/functions/quantile_aggr.rs index 2f8d9edd9d..08d18c8c4f 100644 --- a/src/promql/src/functions/quantile_aggr.rs +++ b/src/promql/src/functions/quantile_aggr.rs @@ -16,10 +16,12 @@ use std::sync::Arc; use datafusion::arrow::array::{ArrayRef, AsArray}; use datafusion::common::cast::{as_list_array, as_primitive_array, as_struct_array}; -use datafusion::error::Result as DfResult; +use datafusion::error::{DataFusionError, Result as DfResult}; use datafusion::logical_expr::{Accumulator as DfAccumulator, AggregateUDF, Volatility}; +use datafusion::physical_plan::expressions::Literal; use datafusion::prelude::create_udaf; use datafusion_common::ScalarValue; +use datafusion_expr::function::AccumulatorArgs; use datatypes::arrow::array::{ListArray, StructArray}; use datatypes::arrow::datatypes::{DataType, Field, Float64Type}; @@ -38,16 +40,16 @@ pub struct QuantileAccumulator { /// Create a quantile `AggregateUDF` for PromQL quantile operator, /// which calculates φ-quantile (0 ≤ φ ≤ 1) over dimensions -pub fn quantile_udaf(q: f64) -> Arc { +pub fn quantile_udaf() -> Arc { Arc::new(create_udaf( QUANTILE_NAME, - // Input type: (values) - vec![DataType::Float64], + // Input type: (φ, values) + vec![DataType::Float64, DataType::Float64], // Output type: the φ-quantile Arc::new(DataType::Float64), Volatility::Volatile, // Create the accumulator - Arc::new(move |_| Ok(Box::new(QuantileAccumulator::new(q)))), + Arc::new(QuantileAccumulator::from_args), // Intermediate state types Arc::new(vec![DataType::Struct( vec![Field::new( @@ -65,17 +67,40 @@ pub fn quantile_udaf(q: f64) -> Arc { } impl QuantileAccumulator { - pub fn new(q: f64) -> Self { + fn new(q: f64) -> Self { Self { q, ..Default::default() } } + + pub fn from_args(args: AccumulatorArgs) -> DfResult> { + if args.exprs.len() != 2 { + return Err(DataFusionError::Plan( + "Quantile function should have 2 inputs".to_string(), + )); + } + + let q = match &args.exprs[0] + .as_any() + .downcast_ref::() + .map(|lit| lit.value()) + { + Some(ScalarValue::Float64(Some(q))) => *q, + _ => { + return Err(DataFusionError::Internal( + "Invalid quantile value".to_string(), + )) + } + }; + + Ok(Box::new(Self::new(q))) + } } impl DfAccumulator for QuantileAccumulator { fn update_batch(&mut self, values: &[ArrayRef]) -> DfResult<()> { - let f64_array = values[0].as_primitive::(); + let f64_array = values[1].as_primitive::(); self.values.extend(f64_array); @@ -162,9 +187,10 @@ mod tests { #[test] fn test_quantile_accumulator_single_value() { let mut accumulator = QuantileAccumulator::new(0.5); + let q = create_f64_array(vec![Some(0.5)]); let input = create_f64_array(vec![Some(10.0)]); - accumulator.update_batch(&[input]).unwrap(); + accumulator.update_batch(&[q, input]).unwrap(); let result = accumulator.evaluate().unwrap(); assert_eq!(result, ScalarValue::Float64(Some(10.0))); @@ -173,9 +199,10 @@ mod tests { #[test] fn test_quantile_accumulator_multiple_values() { let mut accumulator = QuantileAccumulator::new(0.5); + let q = create_f64_array(vec![Some(0.5)]); let input = create_f64_array(vec![Some(1.0), Some(2.0), Some(3.0), Some(4.0), Some(5.0)]); - accumulator.update_batch(&[input]).unwrap(); + accumulator.update_batch(&[q, input]).unwrap(); let result = accumulator.evaluate().unwrap(); assert_eq!(result, ScalarValue::Float64(Some(3.0))); @@ -184,9 +211,10 @@ mod tests { #[test] fn test_quantile_accumulator_with_nulls() { let mut accumulator = QuantileAccumulator::new(0.5); + let q = create_f64_array(vec![Some(0.5)]); let input = create_f64_array(vec![Some(1.0), None, Some(3.0), Some(4.0), Some(5.0)]); - accumulator.update_batch(&[input]).unwrap(); + accumulator.update_batch(&[q, input]).unwrap(); let result = accumulator.evaluate().unwrap(); assert_eq!(result, ScalarValue::Float64(Some(3.0))); @@ -195,11 +223,12 @@ mod tests { #[test] fn test_quantile_accumulator_multiple_batches() { let mut accumulator = QuantileAccumulator::new(0.5); + let q = create_f64_array(vec![Some(0.5)]); let input1 = create_f64_array(vec![Some(1.0), Some(2.0)]); let input2 = create_f64_array(vec![Some(3.0), Some(4.0), Some(5.0)]); - accumulator.update_batch(&[input1]).unwrap(); - accumulator.update_batch(&[input2]).unwrap(); + accumulator.update_batch(&[q.clone(), input1]).unwrap(); + accumulator.update_batch(&[q, input2]).unwrap(); let result = accumulator.evaluate().unwrap(); assert_eq!(result, ScalarValue::Float64(Some(3.0))); @@ -208,29 +237,33 @@ mod tests { #[test] fn test_quantile_accumulator_different_quantiles() { let mut min_accumulator = QuantileAccumulator::new(0.0); + let q = create_f64_array(vec![Some(0.0)]); let input = create_f64_array(vec![Some(1.0), Some(2.0), Some(3.0), Some(4.0), Some(5.0)]); - min_accumulator.update_batch(&[input.clone()]).unwrap(); + min_accumulator.update_batch(&[q, input.clone()]).unwrap(); assert_eq!( min_accumulator.evaluate().unwrap(), ScalarValue::Float64(Some(1.0)) ); let mut q1_accumulator = QuantileAccumulator::new(0.25); - q1_accumulator.update_batch(&[input.clone()]).unwrap(); + let q = create_f64_array(vec![Some(0.25)]); + q1_accumulator.update_batch(&[q, input.clone()]).unwrap(); assert_eq!( q1_accumulator.evaluate().unwrap(), ScalarValue::Float64(Some(2.0)) ); let mut q3_accumulator = QuantileAccumulator::new(0.75); - q3_accumulator.update_batch(&[input.clone()]).unwrap(); + let q = create_f64_array(vec![Some(0.75)]); + q3_accumulator.update_batch(&[q, input.clone()]).unwrap(); assert_eq!( q3_accumulator.evaluate().unwrap(), ScalarValue::Float64(Some(4.0)) ); let mut max_accumulator = QuantileAccumulator::new(1.0); - max_accumulator.update_batch(&[input]).unwrap(); + let q = create_f64_array(vec![Some(1.0)]); + max_accumulator.update_batch(&[q, input]).unwrap(); assert_eq!( max_accumulator.evaluate().unwrap(), ScalarValue::Float64(Some(5.0)) @@ -240,10 +273,11 @@ mod tests { #[test] fn test_quantile_accumulator_size() { let mut accumulator = QuantileAccumulator::new(0.5); + let q = create_f64_array(vec![Some(0.5)]); let input = create_f64_array(vec![Some(1.0), Some(2.0), Some(3.0)]); let initial_size = accumulator.size(); - accumulator.update_batch(&[input]).unwrap(); + accumulator.update_batch(&[q, input]).unwrap(); let after_update_size = accumulator.size(); assert!(after_update_size >= initial_size); @@ -252,14 +286,16 @@ mod tests { #[test] fn test_quantile_accumulator_state_and_merge() -> DfResult<()> { let mut acc1 = QuantileAccumulator::new(0.5); + let q = create_f64_array(vec![Some(0.5)]); let input1 = create_f64_array(vec![Some(1.0), Some(2.0)]); - acc1.update_batch(&[input1])?; + acc1.update_batch(&[q, input1])?; let state1 = acc1.state()?; let mut acc2 = QuantileAccumulator::new(0.5); + let q = create_f64_array(vec![Some(0.5)]); let input2 = create_f64_array(vec![Some(3.0), Some(4.0), Some(5.0)]); - acc2.update_batch(&[input2])?; + acc2.update_batch(&[q, input2])?; let mut struct_builders = vec![]; for scalar in &state1 { @@ -280,16 +316,16 @@ mod tests { #[test] fn test_quantile_accumulator_with_extreme_values() { let mut accumulator = QuantileAccumulator::new(0.5); + let q = create_f64_array(vec![Some(0.5)]); let input = create_f64_array(vec![Some(f64::MAX), Some(f64::MIN), Some(0.0)]); - accumulator.update_batch(&[input]).unwrap(); + accumulator.update_batch(&[q, input]).unwrap(); let _result = accumulator.evaluate().unwrap(); } #[test] fn test_quantile_udaf_creation() { - let q = 0.5; - let udaf = quantile_udaf(q); + let udaf = quantile_udaf(); assert_eq!(udaf.name(), QUANTILE_NAME); assert_eq!(udaf.return_type(&[]).unwrap(), DataType::Float64); diff --git a/src/promql/src/functions/resets.rs b/src/promql/src/functions/resets.rs index 7df44b5e76..05d091db0d 100644 --- a/src/promql/src/functions/resets.rs +++ b/src/promql/src/functions/resets.rs @@ -90,6 +90,7 @@ mod test { Resets::scalar_udf(), ts_array_1, value_array_1, + vec![], vec![Some(0.0), Some(1.0), Some(2.0), Some(3.0), None], ); @@ -101,6 +102,7 @@ mod test { Resets::scalar_udf(), ts_array_2, value_array_2, + vec![], vec![Some(0.0), Some(0.0), Some(1.0), Some(1.0), None], ); @@ -111,6 +113,7 @@ mod test { Resets::scalar_udf(), ts_array_3, value_array_3, + vec![], vec![Some(0.0), Some(0.0), Some(0.0), Some(0.0), None], ); } diff --git a/src/promql/src/functions/round.rs b/src/promql/src/functions/round.rs index d1c9d318d8..c3931c5424 100644 --- a/src/promql/src/functions/round.rs +++ b/src/promql/src/functions/round.rs @@ -15,6 +15,7 @@ use std::sync::Arc; use datafusion::error::DataFusionError; +use datafusion_common::ScalarValue; use datafusion_expr::{create_udf, ColumnarValue, ScalarUDF, Volatility}; use datatypes::arrow::array::AsArray; use datatypes::arrow::datatypes::{DataType, Float64Type}; @@ -36,25 +37,39 @@ impl Round { } fn input_type() -> Vec { - vec![DataType::Float64] + vec![DataType::Float64, DataType::Float64] } pub fn return_type() -> DataType { DataType::Float64 } - pub fn scalar_udf(nearest: f64) -> ScalarUDF { + pub fn scalar_udf() -> ScalarUDF { create_udf( Self::name(), Self::input_type(), Self::return_type(), Volatility::Volatile, - Arc::new(move |input: &_| Self::new(nearest).calc(input)) as _, + Arc::new(move |input: &_| Self::create_function(input)?.calc(input)) as _, ) } + fn create_function(inputs: &[ColumnarValue]) -> Result { + if inputs.len() != 2 { + return Err(DataFusionError::Plan( + "Round function should have 2 inputs".to_string(), + )); + } + let ColumnarValue::Scalar(ScalarValue::Float64(Some(nearest))) = inputs[1] else { + return Err(DataFusionError::Plan( + "Round function's second input should be a scalar float64".to_string(), + )); + }; + Ok(Self::new(nearest)) + } + fn calc(&self, input: &[ColumnarValue]) -> Result { - assert_eq!(input.len(), 1); + assert_eq!(input.len(), 2); let value_array = extract_array(&input[0])?; @@ -80,8 +95,11 @@ mod tests { use super::*; fn test_round_f64(value: Vec, nearest: f64, expected: Vec) { - let round_udf = Round::scalar_udf(nearest); - let input = vec![ColumnarValue::Array(Arc::new(Float64Array::from(value)))]; + let round_udf = Round::scalar_udf(); + let input = vec![ + ColumnarValue::Array(Arc::new(Float64Array::from(value))), + ColumnarValue::Scalar(ScalarValue::Float64(Some(nearest))), + ]; let args = ScalarFunctionArgs { args: input, number_rows: 1, diff --git a/src/promql/src/functions/test_util.rs b/src/promql/src/functions/test_util.rs index 46ad6ec1a8..fb76ca52b5 100644 --- a/src/promql/src/functions/test_util.rs +++ b/src/promql/src/functions/test_util.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use datafusion::arrow::array::Float64Array; use datafusion::logical_expr::ScalarUDF; use datafusion::physical_plan::ColumnarValue; +use datafusion_common::ScalarValue; use datafusion_expr::ScalarFunctionArgs; use datatypes::arrow::datatypes::DataType; @@ -28,13 +29,17 @@ pub fn simple_range_udf_runner( range_fn: ScalarUDF, input_ts: RangeArray, input_value: RangeArray, + other_args: Vec, expected: Vec>, ) { let num_rows = input_ts.len(); - let input = vec![ + let input = [ ColumnarValue::Array(Arc::new(input_ts.into_dict())), ColumnarValue::Array(Arc::new(input_value.into_dict())), - ]; + ] + .into_iter() + .chain(other_args.into_iter().map(ColumnarValue::Scalar)) + .collect::>(); let args = ScalarFunctionArgs { args: input, number_rows: num_rows, diff --git a/src/query/src/promql/planner.rs b/src/query/src/promql/planner.rs index f81d2052dc..9f5d67a578 100644 --- a/src/query/src/promql/planner.rs +++ b/src/query/src/promql/planner.rs @@ -31,7 +31,7 @@ use datafusion::functions_aggregate::stddev::stddev_pop_udaf; use datafusion::functions_aggregate::sum::sum_udaf; use datafusion::functions_aggregate::variance::var_pop_udaf; use datafusion::functions_window::row_number::RowNumber; -use datafusion::logical_expr::expr::{AggregateFunction, Alias, ScalarFunction, WindowFunction}; +use datafusion::logical_expr::expr::{Alias, ScalarFunction, WindowFunction}; use datafusion::logical_expr::expr_rewriter::normalize_cols; use datafusion::logical_expr::{ BinaryExpr, Cast, Extension, LogicalPlan, LogicalPlanBuilder, Operator, @@ -1425,15 +1425,18 @@ impl PromPlanner { let field_column_pos = 0; let mut exprs = Vec::with_capacity(self.ctx.field_columns.len()); let scalar_func = match func.name { - "increase" => ScalarFunc::ExtrapolateUdf(Arc::new(Increase::scalar_udf( + "increase" => ScalarFunc::ExtrapolateUdf( + Arc::new(Increase::scalar_udf()), self.ctx.range.context(ExpectRangeSelectorSnafu)?, - ))), - "rate" => ScalarFunc::ExtrapolateUdf(Arc::new(Rate::scalar_udf( + ), + "rate" => ScalarFunc::ExtrapolateUdf( + Arc::new(Rate::scalar_udf()), self.ctx.range.context(ExpectRangeSelectorSnafu)?, - ))), - "delta" => ScalarFunc::ExtrapolateUdf(Arc::new(Delta::scalar_udf( + ), + "delta" => ScalarFunc::ExtrapolateUdf( + Arc::new(Delta::scalar_udf()), self.ctx.range.context(ExpectRangeSelectorSnafu)?, - ))), + ), "idelta" => ScalarFunc::Udf(Arc::new(IDelta::::scalar_udf())), "irate" => ScalarFunc::Udf(Arc::new(IDelta::::scalar_udf())), "resets" => ScalarFunc::Udf(Arc::new(Resets::scalar_udf())), @@ -1449,50 +1452,9 @@ impl PromPlanner { "present_over_time" => ScalarFunc::Udf(Arc::new(PresentOverTime::scalar_udf())), "stddev_over_time" => ScalarFunc::Udf(Arc::new(StddevOverTime::scalar_udf())), "stdvar_over_time" => ScalarFunc::Udf(Arc::new(StdvarOverTime::scalar_udf())), - "quantile_over_time" => { - let quantile_expr = match other_input_exprs.pop_front() { - Some(DfExpr::Literal(ScalarValue::Float64(Some(quantile)))) => quantile, - other => UnexpectedPlanExprSnafu { - desc: format!("expected f64 literal as quantile, but found {:?}", other), - } - .fail()?, - }; - ScalarFunc::Udf(Arc::new(QuantileOverTime::scalar_udf(quantile_expr))) - } - "predict_linear" => { - let t_expr = match other_input_exprs.pop_front() { - Some(DfExpr::Literal(ScalarValue::Float64(Some(t)))) => t as i64, - Some(DfExpr::Literal(ScalarValue::Int64(Some(t)))) => t, - other => UnexpectedPlanExprSnafu { - desc: format!("expected i64 literal as t, but found {:?}", other), - } - .fail()?, - }; - ScalarFunc::Udf(Arc::new(PredictLinear::scalar_udf(t_expr))) - } - "holt_winters" => { - let sf_exp = match other_input_exprs.pop_front() { - Some(DfExpr::Literal(ScalarValue::Float64(Some(sf)))) => sf, - other => UnexpectedPlanExprSnafu { - desc: format!( - "expected f64 literal as smoothing factor, but found {:?}", - other - ), - } - .fail()?, - }; - let tf_exp = match other_input_exprs.pop_front() { - Some(DfExpr::Literal(ScalarValue::Float64(Some(tf)))) => tf, - other => UnexpectedPlanExprSnafu { - desc: format!( - "expected f64 literal as trend factor, but found {:?}", - other - ), - } - .fail()?, - }; - ScalarFunc::Udf(Arc::new(HoltWinters::scalar_udf(sf_exp, tf_exp))) - } + "quantile_over_time" => ScalarFunc::Udf(Arc::new(QuantileOverTime::scalar_udf())), + "predict_linear" => ScalarFunc::Udf(Arc::new(PredictLinear::scalar_udf())), + "holt_winters" => ScalarFunc::Udf(Arc::new(HoltWinters::scalar_udf())), "time" => { exprs.push(build_special_time_expr( self.ctx.time_index_column.as_ref().unwrap(), @@ -1627,17 +1589,10 @@ impl PromPlanner { ScalarFunc::GeneratedExpr } "round" => { - let nearest = match other_input_exprs.pop_front() { - Some(DfExpr::Literal(ScalarValue::Float64(Some(t)))) => t, - Some(DfExpr::Literal(ScalarValue::Int64(Some(t)))) => t as f64, - None => 0.0, - other => UnexpectedPlanExprSnafu { - desc: format!("expected f64 literal as t, but found {:?}", other), - } - .fail()?, - }; - - ScalarFunc::DataFusionUdf(Arc::new(Round::scalar_udf(nearest))) + if other_input_exprs.is_empty() { + other_input_exprs.push_front(DfExpr::Literal(ScalarValue::Float64(Some(0.0)))); + } + ScalarFunc::DataFusionUdf(Arc::new(Round::scalar_udf())) } _ => { @@ -1695,7 +1650,7 @@ impl PromPlanner { let _ = other_input_exprs.remove(field_column_pos + 1); let _ = other_input_exprs.remove(field_column_pos); } - ScalarFunc::ExtrapolateUdf(func) => { + ScalarFunc::ExtrapolateUdf(func, range_length) => { let ts_range_expr = DfExpr::Column(Column::from_name( RangeManipulate::build_timestamp_range_name( self.ctx.time_index_column.as_ref().unwrap(), @@ -1705,11 +1660,13 @@ impl PromPlanner { other_input_exprs.insert(field_column_pos + 1, col_expr); other_input_exprs .insert(field_column_pos + 2, self.create_time_index_column_expr()?); + other_input_exprs.push_back(lit(range_length)); let fn_expr = DfExpr::ScalarFunction(ScalarFunction { func, args: other_input_exprs.clone().into(), }); exprs.push(fn_expr); + let _ = other_input_exprs.pop_back(); let _ = other_input_exprs.remove(field_column_pos + 2); let _ = other_input_exprs.remove(field_column_pos + 1); let _ = other_input_exprs.remove(field_column_pos); @@ -1972,11 +1929,13 @@ impl PromPlanner { param: &Option>, input_plan: &LogicalPlan, ) -> Result<(Vec, Vec)> { + let mut non_col_args = Vec::new(); let aggr = match op.id() { token::T_SUM => sum_udaf(), token::T_QUANTILE => { let q = Self::get_param_value_as_f64(op, param)?; - quantile_udaf(q) + non_col_args.push(lit(q)); + quantile_udaf() } token::T_AVG => avg_udaf(), token::T_COUNT_VALUES | token::T_COUNT => count_udaf(), @@ -1998,16 +1957,12 @@ impl PromPlanner { .field_columns .iter() .map(|col| { - Ok(DfExpr::AggregateFunction(AggregateFunction { - func: aggr.clone(), - args: vec![DfExpr::Column(Column::from_name(col))], - distinct: false, - filter: None, - order_by: None, - null_treatment: None, - })) + non_col_args.push(DfExpr::Column(Column::from_name(col))); + let expr = aggr.call(non_col_args.clone()); + non_col_args.pop(); + expr }) - .collect::>>()?; + .collect::>(); // if the aggregator is `count_values`, it must be grouped by current fields. let prev_field_exprs = if op.id() == token::T_COUNT_VALUES { @@ -2941,7 +2896,8 @@ enum ScalarFunc { Udf(Arc), // todo(ruihang): maybe merge with Udf later /// UDF that require extra information like range length to be evaluated. - ExtrapolateUdf(Arc), + /// The second argument is range length. + ExtrapolateUdf(Arc, i64), /// Func that doesn't require input, like `time()`. GeneratedExpr, } @@ -3595,8 +3551,8 @@ mod test { async fn increase_aggr() { let query = "increase(some_metric[5m])"; let expected = String::from( - "Filter: prom_increase(timestamp_range,field_0,timestamp) IS NOT NULL [timestamp:Timestamp(Millisecond, None), prom_increase(timestamp_range,field_0,timestamp):Float64;N, tag_0:Utf8]\ - \n Projection: some_metric.timestamp, prom_increase(timestamp_range, field_0, some_metric.timestamp) AS prom_increase(timestamp_range,field_0,timestamp), some_metric.tag_0 [timestamp:Timestamp(Millisecond, None), prom_increase(timestamp_range,field_0,timestamp):Float64;N, tag_0:Utf8]\ + "Filter: prom_increase(timestamp_range,field_0,timestamp,Int64(300000)) IS NOT NULL [timestamp:Timestamp(Millisecond, None), prom_increase(timestamp_range,field_0,timestamp,Int64(300000)):Float64;N, tag_0:Utf8]\ + \n Projection: some_metric.timestamp, prom_increase(timestamp_range, field_0, some_metric.timestamp, Int64(300000)) AS prom_increase(timestamp_range,field_0,timestamp,Int64(300000)), some_metric.tag_0 [timestamp:Timestamp(Millisecond, None), prom_increase(timestamp_range,field_0,timestamp,Int64(300000)):Float64;N, tag_0:Utf8]\ \n PromRangeManipulate: req range=[0..100000000], interval=[5000], eval range=[300000], time index=[timestamp], values=[\"field_0\"] [tag_0:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Dictionary(Int64, Float64);N, timestamp_range:Dictionary(Int64, Timestamp(Millisecond, None))]\ \n PromSeriesNormalize: offset=[0], time index=[timestamp], filter NaN: [true] [tag_0:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N]\ \n PromSeriesDivide: tags=[\"tag_0\"] [tag_0:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N]\ @@ -4395,8 +4351,8 @@ mod test { let plan = PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &build_session_state()) .await .unwrap(); - let expected = "Sort: prometheus_tsdb_head_series.greptime_timestamp ASC NULLS LAST [greptime_timestamp:Timestamp(Millisecond, None), quantile(sum(prometheus_tsdb_head_series.greptime_value)):Float64;N]\ - \n Aggregate: groupBy=[[prometheus_tsdb_head_series.greptime_timestamp]], aggr=[[quantile(sum(prometheus_tsdb_head_series.greptime_value))]] [greptime_timestamp:Timestamp(Millisecond, None), quantile(sum(prometheus_tsdb_head_series.greptime_value)):Float64;N]\ + let expected = "Sort: prometheus_tsdb_head_series.greptime_timestamp ASC NULLS LAST [greptime_timestamp:Timestamp(Millisecond, None), quantile(Float64(0.3),sum(prometheus_tsdb_head_series.greptime_value)):Float64;N]\ + \n Aggregate: groupBy=[[prometheus_tsdb_head_series.greptime_timestamp]], aggr=[[quantile(Float64(0.3), sum(prometheus_tsdb_head_series.greptime_value))]] [greptime_timestamp:Timestamp(Millisecond, None), quantile(Float64(0.3),sum(prometheus_tsdb_head_series.greptime_value)):Float64;N]\ \n Sort: prometheus_tsdb_head_series.ip ASC NULLS LAST, prometheus_tsdb_head_series.greptime_timestamp ASC NULLS LAST [ip:Utf8, greptime_timestamp:Timestamp(Millisecond, None), sum(prometheus_tsdb_head_series.greptime_value):Float64;N]\ \n Aggregate: groupBy=[[prometheus_tsdb_head_series.ip, prometheus_tsdb_head_series.greptime_timestamp]], aggr=[[sum(prometheus_tsdb_head_series.greptime_value)]] [ip:Utf8, greptime_timestamp:Timestamp(Millisecond, None), sum(prometheus_tsdb_head_series.greptime_value):Float64;N]\ \n PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[greptime_timestamp] [ip:Utf8, greptime_timestamp:Timestamp(Millisecond, None), greptime_value:Float64;N]\ diff --git a/tests/cases/standalone/common/promql/quantile.result b/tests/cases/standalone/common/promql/quantile.result index 8676bbbb77..c3aa1ef1ec 100644 --- a/tests/cases/standalone/common/promql/quantile.result +++ b/tests/cases/standalone/common/promql/quantile.result @@ -30,40 +30,40 @@ Affected Rows: 16 TQL EVAL (0, 15, '5s') quantile(0.5, test); -+---------------------+--------------------+ -| ts | quantile(test.val) | -+---------------------+--------------------+ -| 1970-01-01T00:00:00 | 2.5 | -| 1970-01-01T00:00:05 | 6.5 | -| 1970-01-01T00:00:10 | 10.5 | -| 1970-01-01T00:00:15 | 14.5 | -+---------------------+--------------------+ ++---------------------+---------------------------------+ +| ts | quantile(Float64(0.5),test.val) | ++---------------------+---------------------------------+ +| 1970-01-01T00:00:00 | 2.5 | +| 1970-01-01T00:00:05 | 6.5 | +| 1970-01-01T00:00:10 | 10.5 | +| 1970-01-01T00:00:15 | 14.5 | ++---------------------+---------------------------------+ TQL EVAL (0, 15, '5s') quantile(0.5, test) by (idc); -+------+---------------------+--------------------+ -| idc | ts | quantile(test.val) | -+------+---------------------+--------------------+ -| idc1 | 1970-01-01T00:00:00 | 1.5 | -| idc1 | 1970-01-01T00:00:05 | 5.5 | -| idc1 | 1970-01-01T00:00:10 | 9.5 | -| idc1 | 1970-01-01T00:00:15 | 13.5 | -| idc2 | 1970-01-01T00:00:00 | 3.5 | -| idc2 | 1970-01-01T00:00:05 | 7.5 | -| idc2 | 1970-01-01T00:00:10 | 11.5 | -| idc2 | 1970-01-01T00:00:15 | 15.5 | -+------+---------------------+--------------------+ ++------+---------------------+---------------------------------+ +| idc | ts | quantile(Float64(0.5),test.val) | ++------+---------------------+---------------------------------+ +| idc1 | 1970-01-01T00:00:00 | 1.5 | +| idc1 | 1970-01-01T00:00:05 | 5.5 | +| idc1 | 1970-01-01T00:00:10 | 9.5 | +| idc1 | 1970-01-01T00:00:15 | 13.5 | +| idc2 | 1970-01-01T00:00:00 | 3.5 | +| idc2 | 1970-01-01T00:00:05 | 7.5 | +| idc2 | 1970-01-01T00:00:10 | 11.5 | +| idc2 | 1970-01-01T00:00:15 | 15.5 | ++------+---------------------+---------------------------------+ TQL EVAL (0, 15, '5s') quantile(0.5, sum(test) by (idc)); -+---------------------+-------------------------+ -| ts | quantile(sum(test.val)) | -+---------------------+-------------------------+ -| 1970-01-01T00:00:00 | 5.0 | -| 1970-01-01T00:00:05 | 13.0 | -| 1970-01-01T00:00:10 | 21.0 | -| 1970-01-01T00:00:15 | 29.0 | -+---------------------+-------------------------+ ++---------------------+--------------------------------------+ +| ts | quantile(Float64(0.5),sum(test.val)) | ++---------------------+--------------------------------------+ +| 1970-01-01T00:00:00 | 5.0 | +| 1970-01-01T00:00:05 | 13.0 | +| 1970-01-01T00:00:10 | 21.0 | +| 1970-01-01T00:00:15 | 29.0 | ++---------------------+--------------------------------------+ DROP TABLE test; diff --git a/tests/cases/standalone/common/promql/round_fn.result b/tests/cases/standalone/common/promql/round_fn.result index fe12ca6f67..5fe7e2beb0 100644 --- a/tests/cases/standalone/common/promql/round_fn.result +++ b/tests/cases/standalone/common/promql/round_fn.result @@ -18,62 +18,62 @@ Affected Rows: 4 -- SQLNESS SORT_RESULT 3 1 tql eval (3, 4, '1s') round(cache_hit, 0.01); -+---------------------+----------------------------+-------+ -| ts | prom_round(greptime_value) | job | -+---------------------+----------------------------+-------+ -| 1970-01-01T00:00:03 | 123.45 | read | -| 1970-01-01T00:00:03 | 234.57 | write | -| 1970-01-01T00:00:04 | 345.68 | read | -| 1970-01-01T00:00:04 | 456.79 | write | -+---------------------+----------------------------+-------+ ++---------------------+------------------------------------------+-------+ +| ts | prom_round(greptime_value,Float64(0.01)) | job | ++---------------------+------------------------------------------+-------+ +| 1970-01-01T00:00:03 | 123.45 | read | +| 1970-01-01T00:00:03 | 234.57 | write | +| 1970-01-01T00:00:04 | 345.68 | read | +| 1970-01-01T00:00:04 | 456.79 | write | ++---------------------+------------------------------------------+-------+ -- SQLNESS SORT_RESULT 3 1 tql eval (3, 4, '1s') round(cache_hit, 0.1); -+---------------------+----------------------------+-------+ -| ts | prom_round(greptime_value) | job | -+---------------------+----------------------------+-------+ -| 1970-01-01T00:00:03 | 123.5 | read | -| 1970-01-01T00:00:03 | 234.60000000000002 | write | -| 1970-01-01T00:00:04 | 345.70000000000005 | read | -| 1970-01-01T00:00:04 | 456.8 | write | -+---------------------+----------------------------+-------+ ++---------------------+-----------------------------------------+-------+ +| ts | prom_round(greptime_value,Float64(0.1)) | job | ++---------------------+-----------------------------------------+-------+ +| 1970-01-01T00:00:03 | 123.5 | read | +| 1970-01-01T00:00:03 | 234.60000000000002 | write | +| 1970-01-01T00:00:04 | 345.70000000000005 | read | +| 1970-01-01T00:00:04 | 456.8 | write | ++---------------------+-----------------------------------------+-------+ -- SQLNESS SORT_RESULT 3 1 tql eval (3, 4, '1s') round(cache_hit, 1.0); -+---------------------+----------------------------+-------+ -| ts | prom_round(greptime_value) | job | -+---------------------+----------------------------+-------+ -| 1970-01-01T00:00:03 | 123.0 | read | -| 1970-01-01T00:00:03 | 235.0 | write | -| 1970-01-01T00:00:04 | 346.0 | read | -| 1970-01-01T00:00:04 | 457.0 | write | -+---------------------+----------------------------+-------+ ++---------------------+---------------------------------------+-------+ +| ts | prom_round(greptime_value,Float64(1)) | job | ++---------------------+---------------------------------------+-------+ +| 1970-01-01T00:00:03 | 123.0 | read | +| 1970-01-01T00:00:03 | 235.0 | write | +| 1970-01-01T00:00:04 | 346.0 | read | +| 1970-01-01T00:00:04 | 457.0 | write | ++---------------------+---------------------------------------+-------+ -- SQLNESS SORT_RESULT 3 1 tql eval (3, 4, '1s') round(cache_hit); -+---------------------+----------------------------+-------+ -| ts | prom_round(greptime_value) | job | -+---------------------+----------------------------+-------+ -| 1970-01-01T00:00:03 | 123.0 | read | -| 1970-01-01T00:00:03 | 235.0 | write | -| 1970-01-01T00:00:04 | 346.0 | read | -| 1970-01-01T00:00:04 | 457.0 | write | -+---------------------+----------------------------+-------+ ++---------------------+---------------------------------------+-------+ +| ts | prom_round(greptime_value,Float64(0)) | job | ++---------------------+---------------------------------------+-------+ +| 1970-01-01T00:00:03 | 123.0 | read | +| 1970-01-01T00:00:03 | 235.0 | write | +| 1970-01-01T00:00:04 | 346.0 | read | +| 1970-01-01T00:00:04 | 457.0 | write | ++---------------------+---------------------------------------+-------+ -- SQLNESS SORT_RESULT 3 1 tql eval (3, 4, '1s') round(cache_hit, 10.0); -+---------------------+----------------------------+-------+ -| ts | prom_round(greptime_value) | job | -+---------------------+----------------------------+-------+ -| 1970-01-01T00:00:03 | 120.0 | read | -| 1970-01-01T00:00:03 | 230.0 | write | -| 1970-01-01T00:00:04 | 350.0 | read | -| 1970-01-01T00:00:04 | 460.0 | write | -+---------------------+----------------------------+-------+ ++---------------------+----------------------------------------+-------+ +| ts | prom_round(greptime_value,Float64(10)) | job | ++---------------------+----------------------------------------+-------+ +| 1970-01-01T00:00:03 | 120.0 | read | +| 1970-01-01T00:00:03 | 230.0 | write | +| 1970-01-01T00:00:04 | 350.0 | read | +| 1970-01-01T00:00:04 | 460.0 | write | ++---------------------+----------------------------------------+-------+ drop table cache_hit; diff --git a/tests/cases/standalone/common/promql/simple_histogram.result b/tests/cases/standalone/common/promql/simple_histogram.result index 1b4e35e934..0ca95f02c3 100644 --- a/tests/cases/standalone/common/promql/simple_histogram.result +++ b/tests/cases/standalone/common/promql/simple_histogram.result @@ -228,27 +228,27 @@ tql eval (420, 420, '1s') histogram_quantile(0.833, histogram2_bucket); tql eval (2820, 2820, '1s') histogram_quantile(0.166, rate(histogram2_bucket[15m])); -+---------------------+----------------------------+ -| ts | prom_rate(ts_range,val,ts) | -+---------------------+----------------------------+ -| 1970-01-01T00:47:00 | 0.996 | -+---------------------+----------------------------+ ++---------------------+------------------------------------------+ +| ts | prom_rate(ts_range,val,ts,Int64(900000)) | ++---------------------+------------------------------------------+ +| 1970-01-01T00:47:00 | 0.996 | ++---------------------+------------------------------------------+ tql eval (2820, 2820, '1s') histogram_quantile(0.5, rate(histogram2_bucket[15m])); -+---------------------+----------------------------+ -| ts | prom_rate(ts_range,val,ts) | -+---------------------+----------------------------+ -| 1970-01-01T00:47:00 | 3.0 | -+---------------------+----------------------------+ ++---------------------+------------------------------------------+ +| ts | prom_rate(ts_range,val,ts,Int64(900000)) | ++---------------------+------------------------------------------+ +| 1970-01-01T00:47:00 | 3.0 | ++---------------------+------------------------------------------+ tql eval (2820, 2820, '1s') histogram_quantile(0.833, rate(histogram2_bucket[15m])); -+---------------------+----------------------------+ -| ts | prom_rate(ts_range,val,ts) | -+---------------------+----------------------------+ -| 1970-01-01T00:47:00 | 4.998 | -+---------------------+----------------------------+ ++---------------------+------------------------------------------+ +| ts | prom_rate(ts_range,val,ts,Int64(900000)) | ++---------------------+------------------------------------------+ +| 1970-01-01T00:47:00 | 4.998 | ++---------------------+------------------------------------------+ drop table histogram2_bucket; @@ -284,12 +284,12 @@ Affected Rows: 12 tql eval (3000, 3005, '3s') histogram_quantile(0.5, sum by(le, s) (rate(histogram3_bucket[5m]))); -+---+---------------------+---------------------------------+ -| s | ts | sum(prom_rate(ts_range,val,ts)) | -+---+---------------------+---------------------------------+ -| a | 1970-01-01T00:50:00 | 0.55 | -| a | 1970-01-01T00:50:03 | 0.5500000000000002 | -+---+---------------------+---------------------------------+ ++---+---------------------+-----------------------------------------------+ +| s | ts | sum(prom_rate(ts_range,val,ts,Int64(300000))) | ++---+---------------------+-----------------------------------------------+ +| a | 1970-01-01T00:50:00 | 0.55 | +| a | 1970-01-01T00:50:03 | 0.5500000000000002 | ++---+---------------------+-----------------------------------------------+ drop table histogram3_bucket; diff --git a/tests/cases/standalone/common/promql/subquery.result b/tests/cases/standalone/common/promql/subquery.result index d088468b17..12e65c4310 100644 --- a/tests/cases/standalone/common/promql/subquery.result +++ b/tests/cases/standalone/common/promql/subquery.result @@ -45,19 +45,19 @@ tql eval (359, 359, '1s') sum_over_time(metric_total[60s:10s]); tql eval (10, 10, '1s') rate(metric_total[20s:10s]); -+---------------------+----------------------------+ -| ts | prom_rate(ts_range,val,ts) | -+---------------------+----------------------------+ -| 1970-01-01T00:00:10 | 0.1 | -+---------------------+----------------------------+ ++---------------------+-----------------------------------------+ +| ts | prom_rate(ts_range,val,ts,Int64(20000)) | ++---------------------+-----------------------------------------+ +| 1970-01-01T00:00:10 | 0.1 | ++---------------------+-----------------------------------------+ tql eval (20, 20, '1s') rate(metric_total[20s:5s]); -+---------------------+----------------------------+ -| ts | prom_rate(ts_range,val,ts) | -+---------------------+----------------------------+ -| 1970-01-01T00:00:20 | 0.06666666666666667 | -+---------------------+----------------------------+ ++---------------------+-----------------------------------------+ +| ts | prom_rate(ts_range,val,ts,Int64(20000)) | ++---------------------+-----------------------------------------+ +| 1970-01-01T00:00:20 | 0.06666666666666667 | ++---------------------+-----------------------------------------+ drop table metric_total; diff --git a/tests/cases/standalone/common/tql-explain-analyze/analyze.result b/tests/cases/standalone/common/tql-explain-analyze/analyze.result index af1a3d4fce..63491df951 100644 --- a/tests/cases/standalone/common/tql-explain-analyze/analyze.result +++ b/tests/cases/standalone/common/tql-explain-analyze/analyze.result @@ -137,8 +137,8 @@ TQL ANALYZE (0, 10, '5s') rate(test[10s]); | stage | node | plan_| +-+-+-+ | 0_| 0_|_CoalesceBatchesExec: target_batch_size=8192 REDACTED -|_|_|_FilterExec: prom_rate(j_range,i,j)@1 IS NOT NULL REDACTED -|_|_|_ProjectionExec: expr=[j@1 as j, prom_rate(j_range@4, i@0, j@1) as prom_rate(j_range,i,j), k@2 as k, l@3 as l] REDACTED +|_|_|_FilterExec: prom_rate(j_range,i,j,Int64(10000))@1 IS NOT NULL REDACTED +|_|_|_ProjectionExec: expr=[j@1 as j, prom_rate(j_range@4, i@0, j@1, 10000) as prom_rate(j_range,i,j,Int64(10000)), k@2 as k, l@3 as l] REDACTED |_|_|_PromRangeManipulateExec: req range=[0..10000], interval=[5000], eval range=[10000], time index=[j] REDACTED |_|_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [true] REDACTED |_|_|_PromSeriesDivideExec: tags=["k", "l"] REDACTED From e30753fc31e166af2afbfc585a26fbd284bb8aae Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Thu, 24 Apr 2025 16:11:45 +0800 Subject: [PATCH 75/82] feat: allow forced region failover for local WAL (#5972) * feat: allow forced region failover for local WAL * chore: upgrade config.md * chore: apply suggestions from CR --- config/config.md | 1 + config/metasrv.example.toml | 4 ++++ src/meta-srv/src/metasrv.rs | 6 ++++++ src/meta-srv/src/metasrv/builder.rs | 20 ++++++++++++++------ 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/config/config.md b/config/config.md index f34a41d861..f3230190c9 100644 --- a/config/config.md +++ b/config/config.md @@ -319,6 +319,7 @@ | `selector` | String | `round_robin` | Datanode selector type.
- `round_robin` (default value)
- `lease_based`
- `load_based`
For details, please see "https://docs.greptime.com/developer-guide/metasrv/selector". | | `use_memory_store` | Bool | `false` | Store data in memory. | | `enable_region_failover` | Bool | `false` | Whether to enable region failover.
This feature is only available on GreptimeDB running on cluster mode and
- Using Remote WAL
- Using shared storage (e.g., s3). | +| `allow_region_failover_on_local_wal` | Bool | `false` | Whether to allow region failover on local WAL.
**This option is not recommended to be set to true, because it may lead to data loss during failover.** | | `node_max_idle_time` | String | `24hours` | Max allowed idle time before removing node info from metasrv memory. | | `enable_telemetry` | Bool | `true` | Whether to enable greptimedb telemetry. Enabled by default. | | `runtime` | -- | -- | The runtime options. | diff --git a/config/metasrv.example.toml b/config/metasrv.example.toml index 89c92352b2..0e7f9b74f0 100644 --- a/config/metasrv.example.toml +++ b/config/metasrv.example.toml @@ -50,6 +50,10 @@ use_memory_store = false ## - Using shared storage (e.g., s3). enable_region_failover = false +## Whether to allow region failover on local WAL. +## **This option is not recommended to be set to true, because it may lead to data loss during failover.** +allow_region_failover_on_local_wal = false + ## Max allowed idle time before removing node info from metasrv memory. node_max_idle_time = "24hours" diff --git a/src/meta-srv/src/metasrv.rs b/src/meta-srv/src/metasrv.rs index 34b3cac25e..6c9111dd9c 100644 --- a/src/meta-srv/src/metasrv.rs +++ b/src/meta-srv/src/metasrv.rs @@ -111,6 +111,11 @@ pub struct MetasrvOptions { pub use_memory_store: bool, /// Whether to enable region failover. pub enable_region_failover: bool, + /// Whether to allow region failover on local WAL. + /// + /// If it's true, the region failover will be allowed even if the local WAL is used. + /// Note that this option is not recommended to be set to true, because it may lead to data loss during failover. + pub allow_region_failover_on_local_wal: bool, /// The HTTP server options. pub http: HttpOptions, /// The logging options. @@ -173,6 +178,7 @@ impl Default for MetasrvOptions { selector: SelectorType::default(), use_memory_store: false, enable_region_failover: false, + allow_region_failover_on_local_wal: false, http: HttpOptions::default(), logging: LoggingOptions { dir: format!("{METASRV_HOME}/logs"), diff --git a/src/meta-srv/src/metasrv/builder.rs b/src/meta-srv/src/metasrv/builder.rs index 02f835e226..0c93e4e4c7 100644 --- a/src/meta-srv/src/metasrv/builder.rs +++ b/src/meta-srv/src/metasrv/builder.rs @@ -40,7 +40,8 @@ use common_meta::state_store::KvStateStore; use common_meta::wal_options_allocator::{build_kafka_client, build_wal_options_allocator}; use common_procedure::local::{LocalManager, ManagerConfig}; use common_procedure::ProcedureManagerRef; -use snafu::ResultExt; +use common_telemetry::warn; +use snafu::{ensure, ResultExt}; use crate::cache_invalidator::MetasrvCacheInvalidator; use crate::cluster::{MetaPeerClientBuilder, MetaPeerClientRef}; @@ -276,18 +277,25 @@ impl MetasrvBuilder { }, )); let peer_lookup_service = Arc::new(MetaPeerLookupService::new(meta_peer_client.clone())); + if !is_remote_wal && options.enable_region_failover { - return error::UnexpectedSnafu { - violated: "Region failover is not supported in the local WAL implementation!", + ensure!( + options.allow_region_failover_on_local_wal, + error::UnexpectedSnafu { + violated: "Region failover is not supported in the local WAL implementation! + If you want to enable region failover for local WAL, please set `allow_region_failover_on_local_wal` to true.", + } + ); + if options.allow_region_failover_on_local_wal { + warn!("Region failover is force enabled in the local WAL implementation! This may lead to data loss during failover!"); } - .fail(); } let (tx, rx) = RegionSupervisor::channel(); let (region_failure_detector_controller, region_supervisor_ticker): ( RegionFailureDetectorControllerRef, Option>, - ) = if options.enable_region_failover && is_remote_wal { + ) = if options.enable_region_failover { ( Arc::new(RegionFailureDetectorControl::new(tx.clone())) as _, Some(Arc::new(RegionSupervisorTicker::new( @@ -313,7 +321,7 @@ impl MetasrvBuilder { )); region_migration_manager.try_start()?; - let region_failover_handler = if options.enable_region_failover && is_remote_wal { + let region_failover_handler = if options.enable_region_failover { let region_supervisor = RegionSupervisor::new( rx, options.failure_detector, From d5026f3491859847242b9a548d4f8d69aa964978 Mon Sep 17 00:00:00 2001 From: Zhenchi Date: Fri, 25 Apr 2025 07:31:26 +0800 Subject: [PATCH 76/82] perf: optimize fulltext zh tokenizer for ascii-only text (#5975) Signed-off-by: Zhenchi --- src/index/src/fulltext_index/tokenizer.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/index/src/fulltext_index/tokenizer.rs b/src/index/src/fulltext_index/tokenizer.rs index b00e7fda9c..54aa33edc8 100644 --- a/src/index/src/fulltext_index/tokenizer.rs +++ b/src/index/src/fulltext_index/tokenizer.rs @@ -46,7 +46,11 @@ pub struct ChineseTokenizer; impl Tokenizer for ChineseTokenizer { fn tokenize<'a>(&self, text: &'a str) -> Vec<&'a str> { - JIEBA.cut(text, false) + if text.is_ascii() { + EnglishTokenizer {}.tokenize(text) + } else { + JIEBA.cut(text, false) + } } } From 85d564b0fbbd7cbfcee8a6a7401f7e2b0f3c1530 Mon Sep 17 00:00:00 2001 From: dennis zhuang Date: Fri, 25 Apr 2025 10:34:49 -0700 Subject: [PATCH 77/82] fix: upgrade sqlparse and validate align in range query (#5958) * fix: upgrade sqlparse and validate align in range query * update sqlparser to the merged commit Signed-off-by: Ruihang Xia --------- Signed-off-by: Ruihang Xia Co-authored-by: Ruihang Xia Co-authored-by: Zhenchi --- Cargo.lock | 22 +++++++++---------- Cargo.toml | 2 +- .../standalone/common/range/error.result | 6 ++++- tests/cases/standalone/common/range/error.sql | 2 ++ 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 839ceafddc..f980f4fcbf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2510,7 +2510,7 @@ dependencies = [ "futures-util", "serde", "snafu 0.8.5", - "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=e98e6b322426a9d397a71efef17075966223c089)", + "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=0cf6c04490d59435ee965edd2078e8855bd8471e)", "sqlparser_derive 0.1.1", "statrs", "store-api", @@ -3656,7 +3656,7 @@ dependencies = [ "serde", "serde_json", "snafu 0.8.5", - "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=e98e6b322426a9d397a71efef17075966223c089)", + "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=0cf6c04490d59435ee965edd2078e8855bd8471e)", "sqlparser_derive 0.1.1", ] @@ -4553,7 +4553,7 @@ dependencies = [ "session", "snafu 0.8.5", "sql", - "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=e98e6b322426a9d397a71efef17075966223c089)", + "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=0cf6c04490d59435ee965edd2078e8855bd8471e)", "store-api", "strfmt", "substrait 0.14.0", @@ -8166,7 +8166,7 @@ dependencies = [ "session", "snafu 0.8.5", "sql", - "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=e98e6b322426a9d397a71efef17075966223c089)", + "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=0cf6c04490d59435ee965edd2078e8855bd8471e)", "store-api", "substrait 0.14.0", "table", @@ -8443,7 +8443,7 @@ dependencies = [ "session", "snafu 0.8.5", "sql", - "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=e98e6b322426a9d397a71efef17075966223c089)", + "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=0cf6c04490d59435ee965edd2078e8855bd8471e)", "store-api", "table", ] @@ -9477,7 +9477,7 @@ dependencies = [ "session", "snafu 0.8.5", "sql", - "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=e98e6b322426a9d397a71efef17075966223c089)", + "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=0cf6c04490d59435ee965edd2078e8855bd8471e)", "statrs", "store-api", "substrait 0.14.0", @@ -11304,7 +11304,7 @@ dependencies = [ "serde", "serde_json", "snafu 0.8.5", - "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=e98e6b322426a9d397a71efef17075966223c089)", + "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=0cf6c04490d59435ee965edd2078e8855bd8471e)", "sqlparser_derive 0.1.1", "store-api", "table", @@ -11373,7 +11373,7 @@ dependencies = [ [[package]] name = "sqlparser" version = "0.54.0" -source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=e98e6b322426a9d397a71efef17075966223c089#e98e6b322426a9d397a71efef17075966223c089" +source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=0cf6c04490d59435ee965edd2078e8855bd8471e#0cf6c04490d59435ee965edd2078e8855bd8471e" dependencies = [ "lazy_static", "log", @@ -11381,7 +11381,7 @@ dependencies = [ "regex", "serde", "sqlparser 0.54.0 (registry+https://github.com/rust-lang/crates.io-index)", - "sqlparser_derive 0.3.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=e98e6b322426a9d397a71efef17075966223c089)", + "sqlparser_derive 0.3.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=0cf6c04490d59435ee965edd2078e8855bd8471e)", ] [[package]] @@ -11409,7 +11409,7 @@ dependencies = [ [[package]] name = "sqlparser_derive" version = "0.3.0" -source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=e98e6b322426a9d397a71efef17075966223c089#e98e6b322426a9d397a71efef17075966223c089" +source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=0cf6c04490d59435ee965edd2078e8855bd8471e#0cf6c04490d59435ee965edd2078e8855bd8471e" dependencies = [ "proc-macro2", "quote", @@ -12264,7 +12264,7 @@ dependencies = [ "serde_yaml", "snafu 0.8.5", "sql", - "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=e98e6b322426a9d397a71efef17075966223c089)", + "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=0cf6c04490d59435ee965edd2078e8855bd8471e)", "sqlx", "store-api", "strum 0.27.1", diff --git a/Cargo.toml b/Cargo.toml index 92dba96d00..aa8793abb6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -191,7 +191,7 @@ simd-json = "0.15" similar-asserts = "1.6.0" smallvec = { version = "1", features = ["serde"] } snafu = "0.8" -sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "e98e6b322426a9d397a71efef17075966223c089", features = [ +sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "0cf6c04490d59435ee965edd2078e8855bd8471e", features = [ "visitor", "serde", ] } # branch = "v0.54.x" diff --git a/tests/cases/standalone/common/range/error.result b/tests/cases/standalone/common/range/error.result index f7236d6096..e3f12646e7 100644 --- a/tests/cases/standalone/common/range/error.result +++ b/tests/cases/standalone/common/range/error.result @@ -54,7 +54,11 @@ Error: 2000(InvalidSyntax), Invalid SQL syntax: sql parser error: Can't use the -- 2.2 no align param SELECT min(val) RANGE '5s' FROM host; -Error: 3000(PlanQuery), Error during planning: Missing argument in range select query +Error: 2000(InvalidSyntax), Invalid SQL syntax: sql parser error: ALIGN argument cannot be omitted in the range select query + +SELECT min(val) RANGE '5s' FILL PREV FROM host; + +Error: 2000(InvalidSyntax), Invalid SQL syntax: sql parser error: ALIGN argument cannot be omitted in the range select query -- 2.3 type mismatch SELECT covar(ceil(val), floor(val)) RANGE '20s' FROM host ALIGN '10s'; diff --git a/tests/cases/standalone/common/range/error.sql b/tests/cases/standalone/common/range/error.sql index ba3d1f63e2..3659be1c79 100644 --- a/tests/cases/standalone/common/range/error.sql +++ b/tests/cases/standalone/common/range/error.sql @@ -40,6 +40,8 @@ SELECT 1 RANGE '10s' FILL NULL FROM host ALIGN '1h' FILL NULL; SELECT min(val) RANGE '5s' FROM host; +SELECT min(val) RANGE '5s' FILL PREV FROM host; + -- 2.3 type mismatch SELECT covar(ceil(val), floor(val)) RANGE '20s' FROM host ALIGN '10s'; From 489b16ae3033ecd4a2fb1482f3ccf30df81f1447 Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Sat, 26 Apr 2025 02:11:09 +0800 Subject: [PATCH 78/82] fix: security update (#5982) --- Cargo.lock | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f980f4fcbf..a9c2179ede 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1619,9 +1619,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.24" +version = "1.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812acba72f0a070b003d3697490d2b55b837230ae7c6c6497f05cc2ddbb8d938" +checksum = "04da6a0d40b948dfc4fa8f5bbf402b0fc1a64a28dbf7d12ffd683550f2c1b63a" dependencies = [ "jobserver", "libc", @@ -2946,9 +2946,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.13" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" dependencies = [ "crossbeam-utils", ] @@ -6509,7 +6509,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.48.5", ] [[package]] @@ -10005,15 +10005,14 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.8" +version = "0.17.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", "getrandom 0.2.15", "libc", - "spin", "untrusted", "windows-sys 0.52.0", ] From 66e2242e46b87efa5f91321638bc8759d076d25b Mon Sep 17 00:00:00 2001 From: discord9 <55937128+discord9@users.noreply.github.com> Date: Sat, 26 Apr 2025 03:12:30 +0800 Subject: [PATCH 79/82] fix: conn timeout&refactor: better err msg (#5974) * fix: conn timeout&refactor: better err msg * chore: clippy * chore: make test work * chore: comment * todo: fix null cast * fix: retry conn&udd_calc * chore: comment * chore: apply suggestion --------- Co-authored-by: dennis zhuang --- Cargo.lock | 48 ++-- Cargo.toml | 18 +- .../function/src/scalars/uddsketch_calc.rs | 7 + src/flow/src/batching_mode.rs | 6 + src/flow/src/batching_mode/frontend_client.rs | 41 ++- src/flow/src/batching_mode/task.rs | 17 +- src/flow/src/batching_mode/time_window.rs | 22 ++ src/flow/src/batching_mode/utils.rs | 20 +- src/flow/src/df_optimizer.rs | 24 +- src/query/src/lib.rs | 2 +- .../common/flow/flow_step_aggr.result | 266 ++++++++++++++++++ .../standalone/common/flow/flow_step_aggr.sql | 161 +++++++++++ 12 files changed, 565 insertions(+), 67 deletions(-) create mode 100644 tests/cases/standalone/common/flow/flow_step_aggr.result create mode 100644 tests/cases/standalone/common/flow/flow_step_aggr.sql diff --git a/Cargo.lock b/Cargo.lock index a9c2179ede..1e1349709a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3117,7 +3117,7 @@ checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" [[package]] name = "datafusion" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "arrow 54.2.1", "arrow-array 54.2.1", @@ -3168,7 +3168,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "arrow 54.2.1", "async-trait", @@ -3188,7 +3188,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "arrow 54.2.1", "arrow-schema 54.3.1", @@ -3211,7 +3211,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "ahash 0.8.11", "arrow 54.2.1", @@ -3236,7 +3236,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "log", "tokio", @@ -3245,12 +3245,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" [[package]] name = "datafusion-execution" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "arrow 54.2.1", "dashmap", @@ -3268,7 +3268,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "arrow 54.2.1", "chrono", @@ -3288,7 +3288,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "arrow 54.2.1", "datafusion-common", @@ -3299,7 +3299,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "arrow 54.2.1", "arrow-buffer 54.3.1", @@ -3328,7 +3328,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "ahash 0.8.11", "arrow 54.2.1", @@ -3349,7 +3349,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "ahash 0.8.11", "arrow 54.2.1", @@ -3361,7 +3361,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "arrow 54.2.1", "arrow-array 54.2.1", @@ -3383,7 +3383,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "arrow 54.2.1", "async-trait", @@ -3398,7 +3398,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "datafusion-common", "datafusion-doc", @@ -3414,7 +3414,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -3423,7 +3423,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "datafusion-expr", "quote", @@ -3433,7 +3433,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "arrow 54.2.1", "chrono", @@ -3451,7 +3451,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "ahash 0.8.11", "arrow 54.2.1", @@ -3474,7 +3474,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "ahash 0.8.11", "arrow 54.2.1", @@ -3487,7 +3487,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "arrow 54.2.1", "arrow-schema 54.3.1", @@ -3508,7 +3508,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "ahash 0.8.11", "arrow 54.2.1", @@ -3538,7 +3538,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "arrow 54.2.1", "arrow-array 54.2.1", @@ -3556,7 +3556,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "45.0.0" -source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=5bbedc6704162afb03478f56ffb629405a4e1220#5bbedc6704162afb03478f56ffb629405a4e1220" +source = "git+https://github.com/waynexia/arrow-datafusion.git?rev=e104c7cf62b11dd5fe41461b82514978234326b4#e104c7cf62b11dd5fe41461b82514978234326b4" dependencies = [ "async-recursion", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index aa8793abb6..e51d50ee2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -112,15 +112,15 @@ clap = { version = "4.4", features = ["derive"] } config = "0.13.0" crossbeam-utils = "0.8" dashmap = "6.1" -datafusion = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "5bbedc6704162afb03478f56ffb629405a4e1220" } -datafusion-common = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "5bbedc6704162afb03478f56ffb629405a4e1220" } -datafusion-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "5bbedc6704162afb03478f56ffb629405a4e1220" } -datafusion-functions = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "5bbedc6704162afb03478f56ffb629405a4e1220" } -datafusion-optimizer = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "5bbedc6704162afb03478f56ffb629405a4e1220" } -datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "5bbedc6704162afb03478f56ffb629405a4e1220" } -datafusion-physical-plan = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "5bbedc6704162afb03478f56ffb629405a4e1220" } -datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "5bbedc6704162afb03478f56ffb629405a4e1220" } -datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "5bbedc6704162afb03478f56ffb629405a4e1220" } +datafusion = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "e104c7cf62b11dd5fe41461b82514978234326b4" } +datafusion-common = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "e104c7cf62b11dd5fe41461b82514978234326b4" } +datafusion-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "e104c7cf62b11dd5fe41461b82514978234326b4" } +datafusion-functions = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "e104c7cf62b11dd5fe41461b82514978234326b4" } +datafusion-optimizer = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "e104c7cf62b11dd5fe41461b82514978234326b4" } +datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "e104c7cf62b11dd5fe41461b82514978234326b4" } +datafusion-physical-plan = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "e104c7cf62b11dd5fe41461b82514978234326b4" } +datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "e104c7cf62b11dd5fe41461b82514978234326b4" } +datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "e104c7cf62b11dd5fe41461b82514978234326b4" } deadpool = "0.12" deadpool-postgres = "0.14" derive_builder = "0.20" diff --git a/src/common/function/src/scalars/uddsketch_calc.rs b/src/common/function/src/scalars/uddsketch_calc.rs index 5c0beb4fec..f429766eb7 100644 --- a/src/common/function/src/scalars/uddsketch_calc.rs +++ b/src/common/function/src/scalars/uddsketch_calc.rs @@ -115,6 +115,13 @@ impl Function for UddSketchCalcFunction { } }; + // Check if the sketch is empty, if so, return null + // This is important to avoid panics when calling estimate_quantile on an empty sketch + // In practice, this will happen if input is all null + if sketch.bucket_iter().count() == 0 { + builder.push_null(); + continue; + } // Compute the estimated quantile from the sketch let result = sketch.estimate_quantile(perc); builder.push(Some(result)); diff --git a/src/flow/src/batching_mode.rs b/src/flow/src/batching_mode.rs index 152ad5781c..031c7aad4b 100644 --- a/src/flow/src/batching_mode.rs +++ b/src/flow/src/batching_mode.rs @@ -32,3 +32,9 @@ pub const SLOW_QUERY_THRESHOLD: Duration = Duration::from_secs(60); /// The minimum duration between two queries execution by batching mode task const MIN_REFRESH_DURATION: Duration = Duration::new(5, 0); + +/// Grpc connection timeout +const GRPC_CONN_TIMEOUT: Duration = Duration::from_secs(5); + +/// Grpc max retry number +const GRPC_MAX_RETRIES: u32 = 3; diff --git a/src/flow/src/batching_mode/frontend_client.rs b/src/flow/src/batching_mode/frontend_client.rs index 2454e86251..9f16ea07fa 100644 --- a/src/flow/src/batching_mode/frontend_client.rs +++ b/src/flow/src/batching_mode/frontend_client.rs @@ -25,12 +25,15 @@ use common_meta::cluster::{NodeInfo, NodeInfoKey, Role}; use common_meta::peer::Peer; use common_meta::rpc::store::RangeRequest; use common_query::Output; +use common_telemetry::warn; use meta_client::client::MetaClient; use servers::query_handler::grpc::GrpcQueryHandler; use session::context::{QueryContextBuilder, QueryContextRef}; use snafu::{OptionExt, ResultExt}; -use crate::batching_mode::DEFAULT_BATCHING_ENGINE_QUERY_TIMEOUT; +use crate::batching_mode::{ + DEFAULT_BATCHING_ENGINE_QUERY_TIMEOUT, GRPC_CONN_TIMEOUT, GRPC_MAX_RETRIES, +}; use crate::error::{ExternalSnafu, InvalidRequestSnafu, UnexpectedSnafu}; use crate::Error; @@ -99,7 +102,9 @@ impl FrontendClient { Self::Distributed { meta_client, chnl_mgr: { - let cfg = ChannelConfig::new().timeout(DEFAULT_BATCHING_ENGINE_QUERY_TIMEOUT); + let cfg = ChannelConfig::new() + .connect_timeout(GRPC_CONN_TIMEOUT) + .timeout(DEFAULT_BATCHING_ENGINE_QUERY_TIMEOUT); ChannelManager::with_config(cfg) }, } @@ -223,12 +228,32 @@ impl FrontendClient { peer: db.peer.clone(), }); - db.database - .handle(req.clone()) - .await - .with_context(|_| InvalidRequestSnafu { - context: format!("Failed to handle request: {:?}", req), - }) + let mut retry = 0; + + loop { + let ret = db.database.handle(req.clone()).await.with_context(|_| { + InvalidRequestSnafu { + context: format!("Failed to handle request: {:?}", req), + } + }); + if let Err(err) = ret { + if retry < GRPC_MAX_RETRIES { + retry += 1; + warn!( + "Failed to send request to grpc handle at Peer={:?}, retry = {}, error = {:?}", + db.peer, retry, err + ); + continue; + } else { + common_telemetry::error!( + "Failed to send request to grpc handle at Peer={:?} after {} retries, error = {:?}", + db.peer, retry, err + ); + return Err(err); + } + } + return ret; + } } FrontendClient::Standalone { database_client } => { let ctx = QueryContextBuilder::default() diff --git a/src/flow/src/batching_mode/task.rs b/src/flow/src/batching_mode/task.rs index 1547faae11..bb1f296c90 100644 --- a/src/flow/src/batching_mode/task.rs +++ b/src/flow/src/batching_mode/task.rs @@ -53,6 +53,7 @@ use crate::batching_mode::utils::{ use crate::batching_mode::{ DEFAULT_BATCHING_ENGINE_QUERY_TIMEOUT, MIN_REFRESH_DURATION, SLOW_QUERY_THRESHOLD, }; +use crate::df_optimizer::apply_df_optimizer; use crate::error::{ ConvertColumnSchemaSnafu, DatafusionSnafu, ExternalSnafu, InvalidQuerySnafu, SubstraitEncodeLogicalPlanSnafu, UnexpectedSnafu, @@ -541,7 +542,10 @@ impl BatchingTask { .clone() .rewrite(&mut add_auto_column) .with_context(|_| DatafusionSnafu { - context: format!("Failed to rewrite plan {:?}", self.config.plan), + context: format!( + "Failed to rewrite plan:\n {}\n", + self.config.plan + ), })? .data; let schema_len = plan.schema().fields().len(); @@ -573,16 +577,19 @@ impl BatchingTask { let mut add_filter = AddFilterRewriter::new(expr); let mut add_auto_column = AddAutoColumnRewriter::new(sink_table_schema.clone()); - // make a not optimized plan for clearer unparse + let plan = sql_to_df_plan(query_ctx.clone(), engine.clone(), &self.config.query, false) .await?; - plan.clone() + let rewrite = plan + .clone() .rewrite(&mut add_filter) .and_then(|p| p.data.rewrite(&mut add_auto_column)) .with_context(|_| DatafusionSnafu { - context: format!("Failed to rewrite plan {plan:?}"), + context: format!("Failed to rewrite plan:\n {}\n", plan), })? - .data + .data; + // only apply optimize after complex rewrite is done + apply_df_optimizer(rewrite).await? }; Ok(Some((new_plan, schema_len))) diff --git a/src/flow/src/batching_mode/time_window.rs b/src/flow/src/batching_mode/time_window.rs index e6a0d6ad8c..398250fc8b 100644 --- a/src/flow/src/batching_mode/time_window.rs +++ b/src/flow/src/batching_mode/time_window.rs @@ -704,6 +704,28 @@ mod test { ), "SELECT arrow_cast(date_bin(INTERVAL '1 MINS', numbers_with_ts.ts), 'Timestamp(Second, None)') AS time_window FROM numbers_with_ts WHERE ((ts >= CAST('2025-02-24 10:48:00' AS TIMESTAMP)) AND (ts <= CAST('2025-02-24 10:49:00' AS TIMESTAMP))) GROUP BY arrow_cast(date_bin(INTERVAL '1 MINS', numbers_with_ts.ts), 'Timestamp(Second, None)')" ), + // complex time window index with where + ( + "SELECT arrow_cast(date_bin(INTERVAL '1 MINS', numbers_with_ts.ts), 'Timestamp(Second, None)') AS time_window FROM numbers_with_ts WHERE number in (2, 3, 4) GROUP BY time_window;", + Timestamp::new(1740394109, TimeUnit::Second), + ( + "ts".to_string(), + Some(Timestamp::new(1740394080, TimeUnit::Second)), + Some(Timestamp::new(1740394140, TimeUnit::Second)), + ), + "SELECT arrow_cast(date_bin(INTERVAL '1 MINS', numbers_with_ts.ts), 'Timestamp(Second, None)') AS time_window FROM numbers_with_ts WHERE numbers_with_ts.number IN (2, 3, 4) AND ((ts >= CAST('2025-02-24 10:48:00' AS TIMESTAMP)) AND (ts <= CAST('2025-02-24 10:49:00' AS TIMESTAMP))) GROUP BY arrow_cast(date_bin(INTERVAL '1 MINS', numbers_with_ts.ts), 'Timestamp(Second, None)')" + ), + // complex time window index with between and + ( + "SELECT arrow_cast(date_bin(INTERVAL '1 MINS', numbers_with_ts.ts), 'Timestamp(Second, None)') AS time_window FROM numbers_with_ts WHERE number BETWEEN 2 AND 4 GROUP BY time_window;", + Timestamp::new(1740394109, TimeUnit::Second), + ( + "ts".to_string(), + Some(Timestamp::new(1740394080, TimeUnit::Second)), + Some(Timestamp::new(1740394140, TimeUnit::Second)), + ), + "SELECT arrow_cast(date_bin(INTERVAL '1 MINS', numbers_with_ts.ts), 'Timestamp(Second, None)') AS time_window FROM numbers_with_ts WHERE (numbers_with_ts.number BETWEEN 2 AND 4) AND ((ts >= CAST('2025-02-24 10:48:00' AS TIMESTAMP)) AND (ts <= CAST('2025-02-24 10:49:00' AS TIMESTAMP))) GROUP BY arrow_cast(date_bin(INTERVAL '1 MINS', numbers_with_ts.ts), 'Timestamp(Second, None)')" + ), // no time index ( "SELECT date_bin('5 minutes', ts) FROM numbers_with_ts;", diff --git a/src/flow/src/batching_mode/utils.rs b/src/flow/src/batching_mode/utils.rs index 7aa6a8b12f..117db03665 100644 --- a/src/flow/src/batching_mode/utils.rs +++ b/src/flow/src/batching_mode/utils.rs @@ -342,8 +342,8 @@ impl TreeNodeRewriter for AddAutoColumnRewriter { } } else { return Err(DataFusionError::Plan(format!( - "Expect table have 0,1 or 2 columns more than query columns, found {} query columns {:?}, {} table columns {:?} at node {:?}", - query_col_cnt, exprs, table_col_cnt, self.schema.column_schemas(), node + "Expect table have 0,1 or 2 columns more than query columns, found {} query columns {:?}, {} table columns {:?}", + query_col_cnt, exprs, table_col_cnt, self.schema.column_schemas() ))); } @@ -406,7 +406,9 @@ mod test { use datatypes::prelude::ConcreteDataType; use datatypes::schema::{ColumnSchema, Schema}; use pretty_assertions::assert_eq; + use query::query_engine::DefaultSerializer; use session::context::QueryContext; + use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan}; use super::*; use crate::test_utils::create_test_query_engine; @@ -701,4 +703,18 @@ mod test { ); } } + + #[tokio::test] + async fn test_null_cast() { + let query_engine = create_test_query_engine(); + let ctx = QueryContext::arc(); + let sql = "SELECT NULL::DOUBLE FROM numbers_with_ts"; + let plan = sql_to_df_plan(ctx, query_engine.clone(), sql, false) + .await + .unwrap(); + + let _sub_plan = DFLogicalSubstraitConvertor {} + .encode(&plan, DefaultSerializer) + .unwrap(); + } } diff --git a/src/flow/src/df_optimizer.rs b/src/flow/src/df_optimizer.rs index d83bb77718..bef5b3ed79 100644 --- a/src/flow/src/df_optimizer.rs +++ b/src/flow/src/df_optimizer.rs @@ -25,7 +25,6 @@ use datafusion::config::ConfigOptions; use datafusion::error::DataFusionError; use datafusion::functions_aggregate::count::count_udaf; use datafusion::functions_aggregate::sum::sum_udaf; -use datafusion::optimizer::analyzer::count_wildcard_rule::CountWildcardRule; use datafusion::optimizer::analyzer::type_coercion::TypeCoercion; use datafusion::optimizer::common_subexpr_eliminate::CommonSubexprEliminate; use datafusion::optimizer::optimize_projections::OptimizeProjections; @@ -42,6 +41,7 @@ use datafusion_expr::{ BinaryExpr, ColumnarValue, Expr, Operator, Projection, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility, }; +use query::optimizer::count_wildcard::CountWildcardToTimeIndexRule; use query::parser::QueryLanguageParser; use query::query_engine::DefaultSerializer; use query::QueryEngine; @@ -61,9 +61,9 @@ pub async fn apply_df_optimizer( ) -> Result { let cfg = ConfigOptions::new(); let analyzer = Analyzer::with_rules(vec![ - Arc::new(CountWildcardRule::new()), - Arc::new(AvgExpandRule::new()), - Arc::new(TumbleExpandRule::new()), + Arc::new(CountWildcardToTimeIndexRule), + Arc::new(AvgExpandRule), + Arc::new(TumbleExpandRule), Arc::new(CheckGroupByRule::new()), Arc::new(TypeCoercion::new()), ]); @@ -128,13 +128,7 @@ pub async fn sql_to_flow_plan( } #[derive(Debug)] -struct AvgExpandRule {} - -impl AvgExpandRule { - pub fn new() -> Self { - Self {} - } -} +struct AvgExpandRule; impl AnalyzerRule for AvgExpandRule { fn analyze( @@ -331,13 +325,7 @@ impl TreeNodeRewriter for ExpandAvgRewriter<'_> { /// expand tumble in aggr expr to tumble_start and tumble_end with column name like `window_start` #[derive(Debug)] -struct TumbleExpandRule {} - -impl TumbleExpandRule { - pub fn new() -> Self { - Self {} - } -} +struct TumbleExpandRule; impl AnalyzerRule for TumbleExpandRule { fn analyze( diff --git a/src/query/src/lib.rs b/src/query/src/lib.rs index 26fbfb27cd..0aed2860e9 100644 --- a/src/query/src/lib.rs +++ b/src/query/src/lib.rs @@ -28,7 +28,7 @@ pub mod error; pub mod executor; pub mod log_query; pub mod metrics; -mod optimizer; +pub mod optimizer; pub mod options; pub mod parser; mod part_sort; diff --git a/tests/cases/standalone/common/flow/flow_step_aggr.result b/tests/cases/standalone/common/flow/flow_step_aggr.result new file mode 100644 index 0000000000..ab76a67617 --- /dev/null +++ b/tests/cases/standalone/common/flow/flow_step_aggr.result @@ -0,0 +1,266 @@ +CREATE TABLE access_log ( + "url" STRING, + user_id BIGINT, + ts TIMESTAMP TIME INDEX, + PRIMARY KEY ("url", user_id) +); + +Affected Rows: 0 + +CREATE TABLE access_log_10s ( + "url" STRING, + time_window timestamp time INDEX, + state BINARY, + PRIMARY KEY ("url") +); + +Affected Rows: 0 + +CREATE FLOW calc_access_log_10s SINK TO access_log_10s +AS +SELECT + "url", + date_bin('10s'::INTERVAL, ts) AS time_window, + hll(user_id) AS state +FROM + access_log +GROUP BY + "url", + time_window; + +Affected Rows: 0 + +-- insert 4 rows of data +INSERT INTO access_log VALUES + ("/dashboard", 1, "2025-03-04 00:00:00"), + ("/dashboard", 1, "2025-03-04 00:00:01"), + ("/dashboard", 2, "2025-03-04 00:00:05"), + ("/not_found", 3, "2025-03-04 00:00:11"), + ("/dashboard", 4, "2025-03-04 00:00:15"); + +Affected Rows: 5 + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('calc_access_log_10s'); + ++-----------------------------------------+ +| ADMIN FLUSH_FLOW('calc_access_log_10s') | ++-----------------------------------------+ +| FLOW_FLUSHED | ++-----------------------------------------+ + +-- query should return 3 rows +SELECT "url", time_window FROM access_log_10s +ORDER BY + time_window; + ++------------+---------------------+ +| url | time_window | ++------------+---------------------+ +| /dashboard | 2025-03-04T00:00:00 | +| /dashboard | 2025-03-04T00:00:10 | +| /not_found | 2025-03-04T00:00:10 | ++------------+---------------------+ + +-- use hll_count to query the approximate data in access_log_10s +SELECT "url", time_window, hll_count(state) FROM access_log_10s +ORDER BY + time_window; + ++------------+---------------------+---------------------------------+ +| url | time_window | hll_count(access_log_10s.state) | ++------------+---------------------+---------------------------------+ +| /dashboard | 2025-03-04T00:00:00 | 2 | +| /dashboard | 2025-03-04T00:00:10 | 1 | +| /not_found | 2025-03-04T00:00:10 | 1 | ++------------+---------------------+---------------------------------+ + +-- further, we can aggregate 10 seconds of data to every minute, by using hll_merge to merge 10 seconds of hyperloglog state +SELECT + "url", + date_bin('1 minute'::INTERVAL, time_window) AS time_window_1m, + hll_count(hll_merge(state)) as uv_per_min +FROM + access_log_10s +GROUP BY + "url", + time_window_1m +ORDER BY + time_window_1m; + ++------------+---------------------+------------+ +| url | time_window_1m | uv_per_min | ++------------+---------------------+------------+ +| /not_found | 2025-03-04T00:00:00 | 1 | +| /dashboard | 2025-03-04T00:00:00 | 3 | ++------------+---------------------+------------+ + +DROP FLOW calc_access_log_10s; + +Affected Rows: 0 + +DROP TABLE access_log_10s; + +Affected Rows: 0 + +DROP TABLE access_log; + +Affected Rows: 0 + +CREATE TABLE percentile_base ( + "id" INT PRIMARY KEY, + "value" DOUBLE, + ts timestamp(0) time index +); + +Affected Rows: 0 + +CREATE TABLE percentile_5s ( + "percentile_state" BINARY, + time_window timestamp(0) time index +); + +Affected Rows: 0 + +CREATE FLOW calc_percentile_5s SINK TO percentile_5s +AS +SELECT + uddsketch_state(128, 0.01, "value") AS "value", + date_bin('5 seconds'::INTERVAL, ts) AS time_window +FROM + percentile_base +WHERE + "value" > 0 AND "value" < 70 +GROUP BY + time_window; + +Affected Rows: 0 + +INSERT INTO percentile_base ("id", "value", ts) VALUES + (1, 10.0, 1), + (2, 20.0, 2), + (3, 30.0, 3), + (4, 40.0, 4), + (5, 50.0, 5), + (6, 60.0, 6), + (7, 70.0, 7), + (8, 80.0, 8), + (9, 90.0, 9), + (10, 100.0, 10); + +Affected Rows: 10 + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('calc_percentile_5s'); + ++----------------------------------------+ +| ADMIN FLUSH_FLOW('calc_percentile_5s') | ++----------------------------------------+ +| FLOW_FLUSHED | ++----------------------------------------+ + +SELECT + time_window, + uddsketch_calc(0.99, `percentile_state`) AS p99 +FROM + percentile_5s +ORDER BY + time_window; + ++---------------------+--------------------+ +| time_window | p99 | ++---------------------+--------------------+ +| 1970-01-01T00:00:00 | 40.04777053326359 | +| 1970-01-01T00:00:05 | 59.745049810145126 | ++---------------------+--------------------+ + +DROP FLOW calc_percentile_5s; + +Affected Rows: 0 + +DROP TABLE percentile_5s; + +Affected Rows: 0 + +DROP TABLE percentile_base; + +Affected Rows: 0 + +CREATE TABLE percentile_base ( + "id" INT PRIMARY KEY, + "value" DOUBLE, + ts timestamp(0) time index +); + +Affected Rows: 0 + +CREATE TABLE percentile_5s ( + "percentile_state" BINARY, + time_window timestamp(0) time index +); + +Affected Rows: 0 + +CREATE FLOW calc_percentile_5s SINK TO percentile_5s +AS +SELECT + uddsketch_state(128, 0.01, CASE WHEN "value" > 0 AND "value" < 70 THEN "value" ELSE NULL END) AS "value", + date_bin('5 seconds'::INTERVAL, ts) AS time_window +FROM + percentile_base +GROUP BY + time_window; + +Affected Rows: 0 + +INSERT INTO percentile_base ("id", "value", ts) VALUES + (1, 10.0, 1), + (2, 20.0, 2), + (3, 30.0, 3), + (4, 40.0, 4), + (5, 50.0, 5), + (6, 60.0, 6), + (7, 70.0, 7), + (8, 80.0, 8), + (9, 90.0, 9), + (10, 100.0, 10); + +Affected Rows: 10 + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('calc_percentile_5s'); + ++----------------------------------------+ +| ADMIN FLUSH_FLOW('calc_percentile_5s') | ++----------------------------------------+ +| FLOW_FLUSHED | ++----------------------------------------+ + +SELECT + time_window, + uddsketch_calc(0.99, percentile_state) AS p99 +FROM + percentile_5s +ORDER BY + time_window; + ++---------------------+--------------------+ +| time_window | p99 | ++---------------------+--------------------+ +| 1970-01-01T00:00:00 | 40.04777053326359 | +| 1970-01-01T00:00:05 | 59.745049810145126 | +| 1970-01-01T00:00:10 | | ++---------------------+--------------------+ + +DROP FLOW calc_percentile_5s; + +Affected Rows: 0 + +DROP TABLE percentile_5s; + +Affected Rows: 0 + +DROP TABLE percentile_base; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/flow/flow_step_aggr.sql b/tests/cases/standalone/common/flow/flow_step_aggr.sql new file mode 100644 index 0000000000..44dde88912 --- /dev/null +++ b/tests/cases/standalone/common/flow/flow_step_aggr.sql @@ -0,0 +1,161 @@ +CREATE TABLE access_log ( + "url" STRING, + user_id BIGINT, + ts TIMESTAMP TIME INDEX, + PRIMARY KEY ("url", user_id) +); + +CREATE TABLE access_log_10s ( + "url" STRING, + time_window timestamp time INDEX, + state BINARY, + PRIMARY KEY ("url") +); + +CREATE FLOW calc_access_log_10s SINK TO access_log_10s +AS +SELECT + "url", + date_bin('10s'::INTERVAL, ts) AS time_window, + hll(user_id) AS state +FROM + access_log +GROUP BY + "url", + time_window; + +-- insert 4 rows of data +INSERT INTO access_log VALUES + ("/dashboard", 1, "2025-03-04 00:00:00"), + ("/dashboard", 1, "2025-03-04 00:00:01"), + ("/dashboard", 2, "2025-03-04 00:00:05"), + ("/not_found", 3, "2025-03-04 00:00:11"), + ("/dashboard", 4, "2025-03-04 00:00:15"); + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('calc_access_log_10s'); + +-- query should return 3 rows +SELECT "url", time_window FROM access_log_10s +ORDER BY + time_window; + +-- use hll_count to query the approximate data in access_log_10s +SELECT "url", time_window, hll_count(state) FROM access_log_10s +ORDER BY + time_window; + +-- further, we can aggregate 10 seconds of data to every minute, by using hll_merge to merge 10 seconds of hyperloglog state +SELECT + "url", + date_bin('1 minute'::INTERVAL, time_window) AS time_window_1m, + hll_count(hll_merge(state)) as uv_per_min +FROM + access_log_10s +GROUP BY + "url", + time_window_1m +ORDER BY + time_window_1m; + +DROP FLOW calc_access_log_10s; +DROP TABLE access_log_10s; +DROP TABLE access_log; + +CREATE TABLE percentile_base ( + "id" INT PRIMARY KEY, + "value" DOUBLE, + ts timestamp(0) time index +); + +CREATE TABLE percentile_5s ( + "percentile_state" BINARY, + time_window timestamp(0) time index +); + +CREATE FLOW calc_percentile_5s SINK TO percentile_5s +AS +SELECT + uddsketch_state(128, 0.01, "value") AS "value", + date_bin('5 seconds'::INTERVAL, ts) AS time_window +FROM + percentile_base +WHERE + "value" > 0 AND "value" < 70 +GROUP BY + time_window; + +INSERT INTO percentile_base ("id", "value", ts) VALUES + (1, 10.0, 1), + (2, 20.0, 2), + (3, 30.0, 3), + (4, 40.0, 4), + (5, 50.0, 5), + (6, 60.0, 6), + (7, 70.0, 7), + (8, 80.0, 8), + (9, 90.0, 9), + (10, 100.0, 10); + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('calc_percentile_5s'); + +SELECT + time_window, + uddsketch_calc(0.99, `percentile_state`) AS p99 +FROM + percentile_5s +ORDER BY + time_window; + +DROP FLOW calc_percentile_5s; +DROP TABLE percentile_5s; +DROP TABLE percentile_base; + +CREATE TABLE percentile_base ( + "id" INT PRIMARY KEY, + "value" DOUBLE, + ts timestamp(0) time index +); + +CREATE TABLE percentile_5s ( + "percentile_state" BINARY, + time_window timestamp(0) time index +); + +CREATE FLOW calc_percentile_5s SINK TO percentile_5s +AS +SELECT + uddsketch_state(128, 0.01, CASE WHEN "value" > 0 AND "value" < 70 THEN "value" ELSE NULL END) AS "value", + date_bin('5 seconds'::INTERVAL, ts) AS time_window +FROM + percentile_base +GROUP BY + time_window; + +INSERT INTO percentile_base ("id", "value", ts) VALUES + (1, 10.0, 1), + (2, 20.0, 2), + (3, 30.0, 3), + (4, 40.0, 4), + (5, 50.0, 5), + (6, 60.0, 6), + (7, 70.0, 7), + (8, 80.0, 8), + (9, 90.0, 9), + (10, 100.0, 10); + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('calc_percentile_5s'); + +SELECT + time_window, + uddsketch_calc(0.99, percentile_state) AS p99 +FROM + percentile_5s +ORDER BY + time_window; + +DROP FLOW calc_percentile_5s; +DROP TABLE percentile_5s; +DROP TABLE percentile_base; From 2ff54486d33fe70888089efc37990c415c74c949 Mon Sep 17 00:00:00 2001 From: Zhenchi Date: Sun, 27 Apr 2025 09:39:44 +0800 Subject: [PATCH 80/82] chore: bump main branch version to 0.15 (#5984) Signed-off-by: Zhenchi --- .github/workflows/release.yml | 2 +- Cargo.lock | 146 +++++++++++++++++----------------- Cargo.toml | 2 +- 3 files changed, 75 insertions(+), 75 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fe85a6f2c8..1225235899 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -91,7 +91,7 @@ env: # The scheduled version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-YYYYMMDD', like v0.2.0-nigthly-20230313; NIGHTLY_RELEASE_PREFIX: nightly # Note: The NEXT_RELEASE_VERSION should be modified manually by every formal release. - NEXT_RELEASE_VERSION: v0.14.0 + NEXT_RELEASE_VERSION: v0.15.0 jobs: allocate-runners: diff --git a/Cargo.lock b/Cargo.lock index 1e1349709a..f889410aa9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -185,7 +185,7 @@ checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c" [[package]] name = "api" -version = "0.14.0" +version = "0.15.0" dependencies = [ "common-base", "common-decimal", @@ -915,7 +915,7 @@ dependencies = [ [[package]] name = "auth" -version = "0.14.0" +version = "0.15.0" dependencies = [ "api", "async-trait", @@ -1537,7 +1537,7 @@ dependencies = [ [[package]] name = "cache" -version = "0.14.0" +version = "0.15.0" dependencies = [ "catalog", "common-error", @@ -1561,7 +1561,7 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "catalog" -version = "0.14.0" +version = "0.15.0" dependencies = [ "api", "arrow 54.2.1", @@ -1874,7 +1874,7 @@ checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" [[package]] name = "cli" -version = "0.14.0" +version = "0.15.0" dependencies = [ "async-trait", "auth", @@ -1917,7 +1917,7 @@ dependencies = [ "session", "snafu 0.8.5", "store-api", - "substrait 0.14.0", + "substrait 0.15.0", "table", "tempfile", "tokio", @@ -1926,7 +1926,7 @@ dependencies = [ [[package]] name = "client" -version = "0.14.0" +version = "0.15.0" dependencies = [ "api", "arc-swap", @@ -1955,7 +1955,7 @@ dependencies = [ "rand 0.9.0", "serde_json", "snafu 0.8.5", - "substrait 0.14.0", + "substrait 0.15.0", "substrait 0.37.3", "tokio", "tokio-stream", @@ -1996,7 +1996,7 @@ dependencies = [ [[package]] name = "cmd" -version = "0.14.0" +version = "0.15.0" dependencies = [ "async-trait", "auth", @@ -2056,7 +2056,7 @@ dependencies = [ "similar-asserts", "snafu 0.8.5", "store-api", - "substrait 0.14.0", + "substrait 0.15.0", "table", "temp-env", "tempfile", @@ -2102,7 +2102,7 @@ checksum = "55b672471b4e9f9e95499ea597ff64941a309b2cdbffcc46f2cc5e2d971fd335" [[package]] name = "common-base" -version = "0.14.0" +version = "0.15.0" dependencies = [ "anymap2", "async-trait", @@ -2124,11 +2124,11 @@ dependencies = [ [[package]] name = "common-catalog" -version = "0.14.0" +version = "0.15.0" [[package]] name = "common-config" -version = "0.14.0" +version = "0.15.0" dependencies = [ "common-base", "common-error", @@ -2153,7 +2153,7 @@ dependencies = [ [[package]] name = "common-datasource" -version = "0.14.0" +version = "0.15.0" dependencies = [ "arrow 54.2.1", "arrow-schema 54.3.1", @@ -2190,7 +2190,7 @@ dependencies = [ [[package]] name = "common-decimal" -version = "0.14.0" +version = "0.15.0" dependencies = [ "bigdecimal 0.4.8", "common-error", @@ -2203,7 +2203,7 @@ dependencies = [ [[package]] name = "common-error" -version = "0.14.0" +version = "0.15.0" dependencies = [ "common-macro", "http 1.1.0", @@ -2214,7 +2214,7 @@ dependencies = [ [[package]] name = "common-frontend" -version = "0.14.0" +version = "0.15.0" dependencies = [ "async-trait", "common-error", @@ -2224,7 +2224,7 @@ dependencies = [ [[package]] name = "common-function" -version = "0.14.0" +version = "0.15.0" dependencies = [ "ahash 0.8.11", "api", @@ -2277,7 +2277,7 @@ dependencies = [ [[package]] name = "common-greptimedb-telemetry" -version = "0.14.0" +version = "0.15.0" dependencies = [ "async-trait", "common-runtime", @@ -2294,7 +2294,7 @@ dependencies = [ [[package]] name = "common-grpc" -version = "0.14.0" +version = "0.15.0" dependencies = [ "api", "arrow-flight", @@ -2325,7 +2325,7 @@ dependencies = [ [[package]] name = "common-grpc-expr" -version = "0.14.0" +version = "0.15.0" dependencies = [ "api", "common-base", @@ -2344,7 +2344,7 @@ dependencies = [ [[package]] name = "common-macro" -version = "0.14.0" +version = "0.15.0" dependencies = [ "arc-swap", "common-query", @@ -2358,7 +2358,7 @@ dependencies = [ [[package]] name = "common-mem-prof" -version = "0.14.0" +version = "0.15.0" dependencies = [ "common-error", "common-macro", @@ -2371,7 +2371,7 @@ dependencies = [ [[package]] name = "common-meta" -version = "0.14.0" +version = "0.15.0" dependencies = [ "anymap2", "api", @@ -2432,7 +2432,7 @@ dependencies = [ [[package]] name = "common-options" -version = "0.14.0" +version = "0.15.0" dependencies = [ "common-grpc", "humantime-serde", @@ -2441,11 +2441,11 @@ dependencies = [ [[package]] name = "common-plugins" -version = "0.14.0" +version = "0.15.0" [[package]] name = "common-pprof" -version = "0.14.0" +version = "0.15.0" dependencies = [ "common-error", "common-macro", @@ -2457,7 +2457,7 @@ dependencies = [ [[package]] name = "common-procedure" -version = "0.14.0" +version = "0.15.0" dependencies = [ "async-stream", "async-trait", @@ -2484,7 +2484,7 @@ dependencies = [ [[package]] name = "common-procedure-test" -version = "0.14.0" +version = "0.15.0" dependencies = [ "async-trait", "common-procedure", @@ -2493,7 +2493,7 @@ dependencies = [ [[package]] name = "common-query" -version = "0.14.0" +version = "0.15.0" dependencies = [ "api", "async-trait", @@ -2519,7 +2519,7 @@ dependencies = [ [[package]] name = "common-recordbatch" -version = "0.14.0" +version = "0.15.0" dependencies = [ "arc-swap", "common-error", @@ -2539,7 +2539,7 @@ dependencies = [ [[package]] name = "common-runtime" -version = "0.14.0" +version = "0.15.0" dependencies = [ "async-trait", "clap 4.5.19", @@ -2569,14 +2569,14 @@ dependencies = [ [[package]] name = "common-session" -version = "0.14.0" +version = "0.15.0" dependencies = [ "strum 0.27.1", ] [[package]] name = "common-telemetry" -version = "0.14.0" +version = "0.15.0" dependencies = [ "atty", "backtrace", @@ -2604,7 +2604,7 @@ dependencies = [ [[package]] name = "common-test-util" -version = "0.14.0" +version = "0.15.0" dependencies = [ "client", "common-query", @@ -2616,7 +2616,7 @@ dependencies = [ [[package]] name = "common-time" -version = "0.14.0" +version = "0.15.0" dependencies = [ "arrow 54.2.1", "chrono", @@ -2634,7 +2634,7 @@ dependencies = [ [[package]] name = "common-version" -version = "0.14.0" +version = "0.15.0" dependencies = [ "build-data", "const_format", @@ -2644,7 +2644,7 @@ dependencies = [ [[package]] name = "common-wal" -version = "0.14.0" +version = "0.15.0" dependencies = [ "common-base", "common-error", @@ -3572,7 +3572,7 @@ dependencies = [ [[package]] name = "datanode" -version = "0.14.0" +version = "0.15.0" dependencies = [ "api", "arrow-flight", @@ -3624,7 +3624,7 @@ dependencies = [ "session", "snafu 0.8.5", "store-api", - "substrait 0.14.0", + "substrait 0.15.0", "table", "tokio", "toml 0.8.19", @@ -3633,7 +3633,7 @@ dependencies = [ [[package]] name = "datatypes" -version = "0.14.0" +version = "0.15.0" dependencies = [ "arrow 54.2.1", "arrow-array 54.2.1", @@ -4259,7 +4259,7 @@ dependencies = [ [[package]] name = "file-engine" -version = "0.14.0" +version = "0.15.0" dependencies = [ "api", "async-trait", @@ -4382,7 +4382,7 @@ checksum = "8bf7cc16383c4b8d58b9905a8509f02926ce3058053c056376248d958c9df1e8" [[package]] name = "flow" -version = "0.14.0" +version = "0.15.0" dependencies = [ "api", "arrow 54.2.1", @@ -4444,7 +4444,7 @@ dependencies = [ "snafu 0.8.5", "store-api", "strum 0.27.1", - "substrait 0.14.0", + "substrait 0.15.0", "table", "tokio", "tonic 0.12.3", @@ -4499,7 +4499,7 @@ checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" [[package]] name = "frontend" -version = "0.14.0" +version = "0.15.0" dependencies = [ "api", "arc-swap", @@ -4556,7 +4556,7 @@ dependencies = [ "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=0cf6c04490d59435ee965edd2078e8855bd8471e)", "store-api", "strfmt", - "substrait 0.14.0", + "substrait 0.15.0", "table", "tokio", "toml 0.8.19", @@ -5795,7 +5795,7 @@ dependencies = [ [[package]] name = "index" -version = "0.14.0" +version = "0.15.0" dependencies = [ "async-trait", "asynchronous-codec", @@ -6605,7 +6605,7 @@ checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "log-query" -version = "0.14.0" +version = "0.15.0" dependencies = [ "chrono", "common-error", @@ -6617,7 +6617,7 @@ dependencies = [ [[package]] name = "log-store" -version = "0.14.0" +version = "0.15.0" dependencies = [ "async-stream", "async-trait", @@ -6911,7 +6911,7 @@ dependencies = [ [[package]] name = "meta-client" -version = "0.14.0" +version = "0.15.0" dependencies = [ "api", "async-trait", @@ -6939,7 +6939,7 @@ dependencies = [ [[package]] name = "meta-srv" -version = "0.14.0" +version = "0.15.0" dependencies = [ "api", "async-trait", @@ -7029,7 +7029,7 @@ dependencies = [ [[package]] name = "metric-engine" -version = "0.14.0" +version = "0.15.0" dependencies = [ "api", "aquamarine", @@ -7118,7 +7118,7 @@ dependencies = [ [[package]] name = "mito2" -version = "0.14.0" +version = "0.15.0" dependencies = [ "api", "aquamarine", @@ -7824,7 +7824,7 @@ dependencies = [ [[package]] name = "object-store" -version = "0.14.0" +version = "0.15.0" dependencies = [ "anyhow", "bytes", @@ -8119,7 +8119,7 @@ dependencies = [ [[package]] name = "operator" -version = "0.14.0" +version = "0.15.0" dependencies = [ "ahash 0.8.11", "api", @@ -8168,7 +8168,7 @@ dependencies = [ "sql", "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=0cf6c04490d59435ee965edd2078e8855bd8471e)", "store-api", - "substrait 0.14.0", + "substrait 0.15.0", "table", "tokio", "tokio-util", @@ -8423,7 +8423,7 @@ dependencies = [ [[package]] name = "partition" -version = "0.14.0" +version = "0.15.0" dependencies = [ "api", "async-trait", @@ -8705,7 +8705,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pipeline" -version = "0.14.0" +version = "0.15.0" dependencies = [ "ahash 0.8.11", "api", @@ -8847,7 +8847,7 @@ dependencies = [ [[package]] name = "plugins" -version = "0.14.0" +version = "0.15.0" dependencies = [ "auth", "clap 4.5.19", @@ -9127,7 +9127,7 @@ dependencies = [ [[package]] name = "promql" -version = "0.14.0" +version = "0.15.0" dependencies = [ "ahash 0.8.11", "async-trait", @@ -9373,7 +9373,7 @@ dependencies = [ [[package]] name = "puffin" -version = "0.14.0" +version = "0.15.0" dependencies = [ "async-compression 0.4.13", "async-trait", @@ -9414,7 +9414,7 @@ dependencies = [ [[package]] name = "query" -version = "0.14.0" +version = "0.15.0" dependencies = [ "ahash 0.8.11", "api", @@ -9480,7 +9480,7 @@ dependencies = [ "sqlparser 0.54.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=0cf6c04490d59435ee965edd2078e8855bd8471e)", "statrs", "store-api", - "substrait 0.14.0", + "substrait 0.15.0", "table", "tokio", "tokio-stream", @@ -10830,7 +10830,7 @@ dependencies = [ [[package]] name = "servers" -version = "0.14.0" +version = "0.15.0" dependencies = [ "ahash 0.8.11", "api", @@ -10950,7 +10950,7 @@ dependencies = [ [[package]] name = "session" -version = "0.14.0" +version = "0.15.0" dependencies = [ "api", "arc-swap", @@ -11275,7 +11275,7 @@ dependencies = [ [[package]] name = "sql" -version = "0.14.0" +version = "0.15.0" dependencies = [ "api", "chrono", @@ -11330,7 +11330,7 @@ dependencies = [ [[package]] name = "sqlness-runner" -version = "0.14.0" +version = "0.15.0" dependencies = [ "async-trait", "clap 4.5.19", @@ -11649,7 +11649,7 @@ dependencies = [ [[package]] name = "store-api" -version = "0.14.0" +version = "0.15.0" dependencies = [ "api", "aquamarine", @@ -11798,7 +11798,7 @@ dependencies = [ [[package]] name = "substrait" -version = "0.14.0" +version = "0.15.0" dependencies = [ "async-trait", "bytes", @@ -11978,7 +11978,7 @@ dependencies = [ [[package]] name = "table" -version = "0.14.0" +version = "0.15.0" dependencies = [ "api", "async-trait", @@ -12229,7 +12229,7 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" [[package]] name = "tests-fuzz" -version = "0.14.0" +version = "0.15.0" dependencies = [ "arbitrary", "async-trait", @@ -12273,7 +12273,7 @@ dependencies = [ [[package]] name = "tests-integration" -version = "0.14.0" +version = "0.15.0" dependencies = [ "api", "arrow-flight", @@ -12340,7 +12340,7 @@ dependencies = [ "sql", "sqlx", "store-api", - "substrait 0.14.0", + "substrait 0.15.0", "table", "tempfile", "time", diff --git a/Cargo.toml b/Cargo.toml index e51d50ee2b..5e3374108a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,7 +68,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.14.0" +version = "0.15.0" edition = "2021" license = "Apache-2.0" From eeba46671715191d72c47827178d2468b0b11cde Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Sun, 27 Apr 2025 12:43:44 +0800 Subject: [PATCH 81/82] ci: read next release version from toml by default (#5986) * ci: read next release version from toml by default * ci: send error message to stderr * ci: take the first version only --- .github/scripts/create-version.sh | 14 +++++++------- .github/workflows/release.yml | 3 --- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/scripts/create-version.sh b/.github/scripts/create-version.sh index 1de37df190..0e8218ba01 100755 --- a/.github/scripts/create-version.sh +++ b/.github/scripts/create-version.sh @@ -10,17 +10,17 @@ set -e function create_version() { # Read from envrionment variables. if [ -z "$GITHUB_EVENT_NAME" ]; then - echo "GITHUB_EVENT_NAME is empty" + echo "GITHUB_EVENT_NAME is empty" >&2 exit 1 fi if [ -z "$NEXT_RELEASE_VERSION" ]; then - echo "NEXT_RELEASE_VERSION is empty" - exit 1 + echo "NEXT_RELEASE_VERSION is empty, use version from Cargo.toml" >&2 + export NEXT_RELEASE_VERSION=$(grep '^version = ' Cargo.toml | cut -d '"' -f 2 | head -n 1) fi if [ -z "$NIGHTLY_RELEASE_PREFIX" ]; then - echo "NIGHTLY_RELEASE_PREFIX is empty" + echo "NIGHTLY_RELEASE_PREFIX is empty" >&2 exit 1 fi @@ -35,7 +35,7 @@ function create_version() { # It will be like 'dev-2023080819-f0e7216c'. if [ "$NEXT_RELEASE_VERSION" = dev ]; then if [ -z "$COMMIT_SHA" ]; then - echo "COMMIT_SHA is empty in dev build" + echo "COMMIT_SHA is empty in dev build" >&2 exit 1 fi echo "dev-$(date "+%Y%m%d-%s")-$(echo "$COMMIT_SHA" | cut -c1-8)" @@ -45,7 +45,7 @@ function create_version() { # Note: Only output 'version=xxx' to stdout when everything is ok, so that it can be used in GitHub Actions Outputs. if [ "$GITHUB_EVENT_NAME" = push ]; then if [ -z "$GITHUB_REF_NAME" ]; then - echo "GITHUB_REF_NAME is empty in push event" + echo "GITHUB_REF_NAME is empty in push event" >&2 exit 1 fi echo "$GITHUB_REF_NAME" @@ -54,7 +54,7 @@ function create_version() { elif [ "$GITHUB_EVENT_NAME" = schedule ]; then echo "$NEXT_RELEASE_VERSION-$NIGHTLY_RELEASE_PREFIX-$(date "+%Y%m%d")" else - echo "Unsupported GITHUB_EVENT_NAME: $GITHUB_EVENT_NAME" + echo "Unsupported GITHUB_EVENT_NAME: $GITHUB_EVENT_NAME" >&2 exit 1 fi } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1225235899..b3c7ee4cdd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -90,8 +90,6 @@ env: # The scheduled version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-YYYYMMDD', like v0.2.0-nigthly-20230313; NIGHTLY_RELEASE_PREFIX: nightly - # Note: The NEXT_RELEASE_VERSION should be modified manually by every formal release. - NEXT_RELEASE_VERSION: v0.15.0 jobs: allocate-runners: @@ -135,7 +133,6 @@ jobs: env: GITHUB_EVENT_NAME: ${{ github.event_name }} GITHUB_REF_NAME: ${{ github.ref_name }} - NEXT_RELEASE_VERSION: ${{ env.NEXT_RELEASE_VERSION }} NIGHTLY_RELEASE_PREFIX: ${{ env.NIGHTLY_RELEASE_PREFIX }} - name: Allocate linux-amd64 runner From 3c943be1896b6d6971f27f6f2e37cbdcf91514f6 Mon Sep 17 00:00:00 2001 From: shuiyisong <113876041+shuiyisong@users.noreply.github.com> Date: Sun, 27 Apr 2025 17:02:36 +0800 Subject: [PATCH 82/82] chore: update rust toolchain (#5818) * chore: update nightly version * chore: sort lint lines * chore: minor fix * chore: update nix * chore: update toolchain to 2024-04-14 * chore: update toolchain to 2024-04-15 * chore: remove unnecessory test * chore: do not assert oid in sqlness test * chore: fix margin issue * chore: fix cr issues * chore: fix cr issues --------- Co-authored-by: Ning Sun --- Cargo.lock | 34 +++++++++---------- Cargo.toml | 5 +-- flake.lock | 18 +++++----- flake.nix | 2 +- rust-toolchain.toml | 2 +- .../pg_catalog/pg_namespace/oid_map.rs | 6 ---- .../function/src/scalars/matches_term.rs | 12 ++++--- src/common/meta/src/lib.rs | 2 -- src/common/meta/src/rpc/router.rs | 7 ++-- .../query/src/logical_plan/accumulator.rs | 5 ++- src/datatypes/src/schema/constraint.rs | 7 ++-- src/datatypes/src/vectors/constant.rs | 3 +- src/datatypes/src/vectors/decimal.rs | 2 +- src/datatypes/src/vectors/null.rs | 4 +-- src/datatypes/src/vectors/primitive.rs | 2 +- .../create/sort/external_sort.rs | 2 +- src/meta-srv/src/lib.rs | 1 - src/meta-srv/src/mocks.rs | 5 +-- src/meta-srv/src/service/store/cached_kv.rs | 2 +- src/mito2/src/read/projection.rs | 6 ++-- .../src/sst/index/bloom_filter/creator.rs | 19 +++++------ .../src/sst/index/fulltext_index/creator.rs | 12 +++---- .../src/sst/index/inverted_index/creator.rs | 19 +++++------ src/mito2/src/sst/parquet/format.rs | 2 +- .../req_convert/insert/fill_impure_default.rs | 8 ++--- src/pipeline/src/etl/processor/dissect.rs | 2 +- src/query/src/lib.rs | 1 - src/query/src/part_sort.rs | 2 +- src/servers/src/http/timeout.rs | 2 +- src/servers/src/lib.rs | 1 - src/sql/src/statements/transform.rs | 2 +- src/store-api/src/metadata.rs | 2 +- src/table/src/table/adapter.rs | 2 +- tests-fuzz/src/utils.rs | 6 +--- .../unstable/fuzz_create_table_standalone.rs | 2 +- tests-integration/src/cluster.rs | 5 +-- .../common/system/pg_catalog.result | 19 ++++++----- .../standalone/common/system/pg_catalog.sql | 7 ++-- 38 files changed, 109 insertions(+), 131 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f889410aa9..1603528b22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -173,9 +173,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.89" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" [[package]] name = "anymap2" @@ -1597,7 +1597,7 @@ dependencies = [ "partition", "paste", "prometheus", - "rustc-hash 2.0.0", + "rustc-hash 2.1.1", "serde_json", "session", "snafu 0.8.5", @@ -3110,9 +3110,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.6.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" [[package]] name = "datafusion" @@ -6509,7 +6509,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -6599,9 +6599,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.22" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] name = "log-query" @@ -7780,7 +7780,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" dependencies = [ - "proc-macro-crate 1.3.1", + "proc-macro-crate 3.2.0", "proc-macro2", "quote", "syn 2.0.100", @@ -9527,7 +9527,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.0.0", + "rustc-hash 2.1.1", "rustls", "socket2", "thiserror 1.0.64", @@ -9544,7 +9544,7 @@ dependencies = [ "bytes", "rand 0.8.5", "ring", - "rustc-hash 2.0.0", + "rustc-hash 2.1.1", "rustls", "slab", "thiserror 1.0.64", @@ -9821,9 +9821,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -10333,9 +10333,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-hash" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustc_version" @@ -11158,9 +11158,9 @@ dependencies = [ [[package]] name = "smallbitvec" -version = "2.5.3" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcc3fc564a4b53fd1e8589628efafe57602d91bde78be18186b5f61e8faea470" +checksum = "d31d263dd118560e1a492922182ab6ca6dc1d03a3bf54e7699993f31a4150e3f" [[package]] name = "smallvec" diff --git a/Cargo.toml b/Cargo.toml index 5e3374108a..b0a049bbd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,10 +73,11 @@ edition = "2021" license = "Apache-2.0" [workspace.lints] -clippy.print_stdout = "warn" -clippy.print_stderr = "warn" clippy.dbg_macro = "warn" clippy.implicit_clone = "warn" +clippy.result_large_err = "allow" +clippy.large_enum_variant = "allow" +clippy.doc_overindented_list_items = "allow" rust.unknown_lints = "deny" rust.unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tokio_unstable)'] } diff --git a/flake.lock b/flake.lock index cfea27d34b..f2b2521130 100644 --- a/flake.lock +++ b/flake.lock @@ -8,11 +8,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1737613896, - "narHash": "sha256-ldqXIglq74C7yKMFUzrS9xMT/EVs26vZpOD68Sh7OcU=", + "lastModified": 1742452566, + "narHash": "sha256-sVuLDQ2UIWfXUBbctzrZrXM2X05YjX08K7XHMztt36E=", "owner": "nix-community", "repo": "fenix", - "rev": "303a062fdd8e89f233db05868468975d17855d80", + "rev": "7d9ba794daf5e8cc7ee728859bc688d8e26d5f06", "type": "github" }, "original": { @@ -41,11 +41,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1737569578, - "narHash": "sha256-6qY0pk2QmUtBT9Mywdvif0i/CLVgpCjMUn6g9vB+f3M=", + "lastModified": 1743576891, + "narHash": "sha256-vXiKURtntURybE6FMNFAVpRPr8+e8KoLPrYs9TGuAKc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "47addd76727f42d351590c905d9d1905ca895b82", + "rev": "44a69ed688786e98a101f02b712c313f1ade37ab", "type": "github" }, "original": { @@ -65,11 +65,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1737581772, - "narHash": "sha256-t1P2Pe3FAX9TlJsCZbmJ3wn+C4qr6aSMypAOu8WNsN0=", + "lastModified": 1742296961, + "narHash": "sha256-gCpvEQOrugHWLimD1wTFOJHagnSEP6VYBDspq96Idu0=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "582af7ee9c8d84f5d534272fc7de9f292bd849be", + "rev": "15d87419f1a123d8f888d608129c3ce3ff8f13d4", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index a6d9fbc0df..225f631721 100644 --- a/flake.nix +++ b/flake.nix @@ -21,7 +21,7 @@ lib = nixpkgs.lib; rustToolchain = fenix.packages.${system}.fromToolchainName { name = (lib.importTOML ./rust-toolchain.toml).toolchain.channel; - sha256 = "sha256-f/CVA1EC61EWbh0SjaRNhLL0Ypx2ObupbzigZp8NmL4="; + sha256 = "sha256-i0Sh/ZFFsHlZ3oFZFc24qdk6Cd8Do8OPU4HJQsrKOeM="; }; in { diff --git a/rust-toolchain.toml b/rust-toolchain.toml index eb2546003b..5d547223f2 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "nightly-2024-12-25" +channel = "nightly-2025-04-15" diff --git a/src/catalog/src/system_schema/pg_catalog/pg_namespace/oid_map.rs b/src/catalog/src/system_schema/pg_catalog/pg_namespace/oid_map.rs index edbdac25c7..a2165d731c 100644 --- a/src/catalog/src/system_schema/pg_catalog/pg_namespace/oid_map.rs +++ b/src/catalog/src/system_schema/pg_catalog/pg_namespace/oid_map.rs @@ -84,12 +84,6 @@ mod tests { let key1 = "3178510"; let key2 = "4215648"; - // have collision - assert_eq!( - oid_map.hasher.hash_one(key1) as u32, - oid_map.hasher.hash_one(key2) as u32 - ); - // insert them into oid_map let oid1 = oid_map.get_oid(key1); let oid2 = oid_map.get_oid(key2); diff --git a/src/common/function/src/scalars/matches_term.rs b/src/common/function/src/scalars/matches_term.rs index c99c5ca572..54cf556e85 100644 --- a/src/common/function/src/scalars/matches_term.rs +++ b/src/common/function/src/scalars/matches_term.rs @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::fmt; +use std::iter::repeat_n; use std::sync::Arc; -use std::{fmt, iter}; use common_query::error::{InvalidFuncArgsSnafu, Result}; use common_query::prelude::Volatility; @@ -126,9 +127,10 @@ impl Function for MatchesTermFunction { let term = term_column.get_ref(0).as_string().unwrap(); match term { None => { - return Ok(Arc::new(BooleanVector::from_iter( - iter::repeat(None).take(text_column.len()), - ))); + return Ok(Arc::new(BooleanVector::from_iter(repeat_n( + None, + text_column.len(), + )))); } Some(term) => Some(MatchesTermFinder::new(term)), } @@ -217,7 +219,7 @@ impl MatchesTermFinder { } let mut pos = 0; - while let Some(found_pos) = self.finder.find(text[pos..].as_bytes()) { + while let Some(found_pos) = self.finder.find(&text.as_bytes()[pos..]) { let actual_pos = pos + found_pos; let prev_ok = self.starts_with_non_alnum diff --git a/src/common/meta/src/lib.rs b/src/common/meta/src/lib.rs index b1cc18d5e4..7bfbd78f9c 100644 --- a/src/common/meta/src/lib.rs +++ b/src/common/meta/src/lib.rs @@ -15,8 +15,6 @@ #![feature(assert_matches)] #![feature(btree_extract_if)] #![feature(let_chains)] -#![feature(extract_if)] -#![feature(hash_extract_if)] pub mod cache; pub mod cache_invalidator; diff --git a/src/common/meta/src/rpc/router.rs b/src/common/meta/src/rpc/router.rs index 7ddd104c61..2386ca73a7 100644 --- a/src/common/meta/src/rpc/router.rs +++ b/src/common/meta/src/rpc/router.rs @@ -176,15 +176,12 @@ impl TableRoute { })? .into(); - let leader_peer = peers - .get(region_route.leader_peer_index as usize) - .cloned() - .map(Into::into); + let leader_peer = peers.get(region_route.leader_peer_index as usize).cloned(); let follower_peers = region_route .follower_peer_indexes .into_iter() - .filter_map(|x| peers.get(x as usize).cloned().map(Into::into)) + .filter_map(|x| peers.get(x as usize).cloned()) .collect::>(); region_routes.push(RegionRoute { diff --git a/src/common/query/src/logical_plan/accumulator.rs b/src/common/query/src/logical_plan/accumulator.rs index 32f1b4587c..a9c499d323 100644 --- a/src/common/query/src/logical_plan/accumulator.rs +++ b/src/common/query/src/logical_plan/accumulator.rs @@ -24,7 +24,7 @@ use datatypes::prelude::*; use datatypes::vectors::{Helper as VectorHelper, VectorRef}; use snafu::ResultExt; -use crate::error::{self, Error, FromScalarValueSnafu, IntoVectorSnafu, Result}; +use crate::error::{self, FromScalarValueSnafu, IntoVectorSnafu, Result}; use crate::prelude::*; pub type AggregateFunctionCreatorRef = Arc; @@ -166,8 +166,7 @@ impl DfAccumulator for DfAccumulatorAdaptor { let output_type = self.creator.output_type()?; let scalar_value = value .try_to_scalar_value(&output_type) - .context(error::ToScalarValueSnafu) - .map_err(Error::from)?; + .context(error::ToScalarValueSnafu)?; Ok(scalar_value) } diff --git a/src/datatypes/src/schema/constraint.rs b/src/datatypes/src/schema/constraint.rs index 1a2128c200..560500810f 100644 --- a/src/datatypes/src/schema/constraint.rs +++ b/src/datatypes/src/schema/constraint.rs @@ -253,9 +253,10 @@ fn create_current_timestamp_vector( data_type: &ConcreteDataType, num_rows: usize, ) -> Result { - let current_timestamp_vector = TimestampMillisecondVector::from_values( - std::iter::repeat(util::current_time_millis()).take(num_rows), - ); + let current_timestamp_vector = TimestampMillisecondVector::from_values(std::iter::repeat_n( + util::current_time_millis(), + num_rows, + )); if data_type.is_timestamp() { current_timestamp_vector.cast(data_type) } else { diff --git a/src/datatypes/src/vectors/constant.rs b/src/datatypes/src/vectors/constant.rs index 66587cf1d7..3ccade1392 100644 --- a/src/datatypes/src/vectors/constant.rs +++ b/src/datatypes/src/vectors/constant.rs @@ -198,8 +198,7 @@ impl fmt::Debug for ConstantVector { impl Serializable for ConstantVector { fn serialize_to_json(&self) -> Result> { - std::iter::repeat(self.get(0)) - .take(self.len()) + std::iter::repeat_n(self.get(0), self.len()) .map(serde_json::Value::try_from) .collect::>() .context(SerializeSnafu) diff --git a/src/datatypes/src/vectors/decimal.rs b/src/datatypes/src/vectors/decimal.rs index cce26e3e3e..e446b36de3 100644 --- a/src/datatypes/src/vectors/decimal.rs +++ b/src/datatypes/src/vectors/decimal.rs @@ -412,7 +412,7 @@ pub(crate) fn replicate_decimal128( // Safety: std::iter::Repeat and std::iter::Take implement TrustedLen. builder .mutable_array - .append_trusted_len_iter(std::iter::repeat(data).take(repeat_times)); + .append_trusted_len_iter(std::iter::repeat_n(data, repeat_times)); } } None => { diff --git a/src/datatypes/src/vectors/null.rs b/src/datatypes/src/vectors/null.rs index 292e2c5e33..e745ee13d6 100644 --- a/src/datatypes/src/vectors/null.rs +++ b/src/datatypes/src/vectors/null.rs @@ -120,9 +120,7 @@ impl fmt::Debug for NullVector { impl Serializable for NullVector { fn serialize_to_json(&self) -> Result> { - Ok(std::iter::repeat(serde_json::Value::Null) - .take(self.len()) - .collect()) + Ok(std::iter::repeat_n(serde_json::Value::Null, self.len()).collect()) } } diff --git a/src/datatypes/src/vectors/primitive.rs b/src/datatypes/src/vectors/primitive.rs index 7b059e0d07..f3e49183f5 100644 --- a/src/datatypes/src/vectors/primitive.rs +++ b/src/datatypes/src/vectors/primitive.rs @@ -388,7 +388,7 @@ pub(crate) fn replicate_primitive( // Safety: std::iter::Repeat and std::iter::Take implement TrustedLen. builder .mutable_array - .append_trusted_len_iter(std::iter::repeat(data).take(repeat_times)); + .append_trusted_len_iter(std::iter::repeat_n(data, repeat_times)); } } None => { diff --git a/src/index/src/inverted_index/create/sort/external_sort.rs b/src/index/src/inverted_index/create/sort/external_sort.rs index e8f67b7b7b..3b4eaebc5c 100644 --- a/src/index/src/inverted_index/create/sort/external_sort.rs +++ b/src/index/src/inverted_index/create/sort/external_sort.rs @@ -481,7 +481,7 @@ mod tests { let mock_values = dic_values .iter() - .flat_map(|(value, size)| iter::repeat(value.clone()).take(*size)) + .flat_map(|(value, size)| std::iter::repeat_n(value.clone(), *size)) .collect::>(); let sorted_result = sorted_result(&mock_values, segment_row_count); diff --git a/src/meta-srv/src/lib.rs b/src/meta-srv/src/lib.rs index ebd3b7b54f..20b9285723 100644 --- a/src/meta-srv/src/lib.rs +++ b/src/meta-srv/src/lib.rs @@ -14,7 +14,6 @@ #![feature(result_flattening)] #![feature(assert_matches)] -#![feature(extract_if)] #![feature(hash_set_entry)] pub mod bootstrap; diff --git a/src/meta-srv/src/mocks.rs b/src/meta-srv/src/mocks.rs index 656ceeb3d9..29ed3a5ae8 100644 --- a/src/meta-srv/src/mocks.rs +++ b/src/meta-srv/src/mocks.rs @@ -141,10 +141,7 @@ pub async fn mock( if let Some(client) = client { Ok(TokioIo::new(client)) } else { - Err(std::io::Error::new( - std::io::ErrorKind::Other, - "Client already taken", - )) + Err(std::io::Error::other("Client already taken")) } } }), diff --git a/src/meta-srv/src/service/store/cached_kv.rs b/src/meta-srv/src/service/store/cached_kv.rs index b26c2a558f..f86b42a9e2 100644 --- a/src/meta-srv/src/service/store/cached_kv.rs +++ b/src/meta-srv/src/service/store/cached_kv.rs @@ -278,7 +278,7 @@ impl KvBackend for LeaderCachedKvBackend { let remote_res = self.store.batch_get(remote_req).await?; let put_req = BatchPutRequest { - kvs: remote_res.kvs.clone().into_iter().map(Into::into).collect(), + kvs: remote_res.kvs.clone().into_iter().collect(), ..Default::default() }; let _ = self.cache.batch_put(put_req).await?; diff --git a/src/mito2/src/read/projection.rs b/src/mito2/src/read/projection.rs index 883f554066..6d6de78a74 100644 --- a/src/mito2/src/read/projection.rs +++ b/src/mito2/src/read/projection.rs @@ -363,9 +363,9 @@ mod tests { builder .push_field_array( *column_id, - Arc::new(Int64Array::from_iter_values( - std::iter::repeat(*field).take(num_rows), - )), + Arc::new(Int64Array::from_iter_values(std::iter::repeat_n( + *field, num_rows, + ))), ) .unwrap(); } diff --git a/src/mito2/src/sst/index/bloom_filter/creator.rs b/src/mito2/src/sst/index/bloom_filter/creator.rs index 5c00b1a19c..53821c7cf2 100644 --- a/src/mito2/src/sst/index/bloom_filter/creator.rs +++ b/src/mito2/src/sst/index/bloom_filter/creator.rs @@ -346,7 +346,6 @@ impl BloomFilterIndexer { #[cfg(test)] pub(crate) mod tests { - use std::iter; use api::v1::SemanticType; use datatypes::data_type::ConcreteDataType; @@ -461,15 +460,15 @@ pub(crate) mod tests { Batch::new( primary_key, - Arc::new(UInt64Vector::from_iter_values( - iter::repeat(0).take(num_rows), - )), - Arc::new(UInt64Vector::from_iter_values( - iter::repeat(0).take(num_rows), - )), - Arc::new(UInt8Vector::from_iter_values( - iter::repeat(1).take(num_rows), - )), + Arc::new(UInt64Vector::from_iter_values(std::iter::repeat_n( + 0, num_rows, + ))), + Arc::new(UInt64Vector::from_iter_values(std::iter::repeat_n( + 0, num_rows, + ))), + Arc::new(UInt8Vector::from_iter_values(std::iter::repeat_n( + 1, num_rows, + ))), vec![u64_field], ) .unwrap() diff --git a/src/mito2/src/sst/index/fulltext_index/creator.rs b/src/mito2/src/sst/index/fulltext_index/creator.rs index 1d884ac3a5..fc9aae9f42 100644 --- a/src/mito2/src/sst/index/fulltext_index/creator.rs +++ b/src/mito2/src/sst/index/fulltext_index/creator.rs @@ -489,12 +489,12 @@ mod tests { Arc::new(UInt64Vector::from_iter_values( (0..num_rows).map(|n| n as u64), )), - Arc::new(UInt64Vector::from_iter_values( - std::iter::repeat(0).take(num_rows), - )), - Arc::new(UInt8Vector::from_iter_values( - std::iter::repeat(1).take(num_rows), - )), + Arc::new(UInt64Vector::from_iter_values(std::iter::repeat_n( + 0, num_rows, + ))), + Arc::new(UInt8Vector::from_iter_values(std::iter::repeat_n( + 1, num_rows, + ))), vec![ BatchColumn { column_id: 1, diff --git a/src/mito2/src/sst/index/inverted_index/creator.rs b/src/mito2/src/sst/index/inverted_index/creator.rs index 8991b72aec..6f44979c78 100644 --- a/src/mito2/src/sst/index/inverted_index/creator.rs +++ b/src/mito2/src/sst/index/inverted_index/creator.rs @@ -326,7 +326,6 @@ impl InvertedIndexer { #[cfg(test)] mod tests { use std::collections::BTreeSet; - use std::iter; use api::v1::SemanticType; use datafusion_expr::{binary_expr, col, lit, Expr as DfExpr, Operator}; @@ -424,15 +423,15 @@ mod tests { Batch::new( primary_key, - Arc::new(UInt64Vector::from_iter_values( - iter::repeat(0).take(num_rows), - )), - Arc::new(UInt64Vector::from_iter_values( - iter::repeat(0).take(num_rows), - )), - Arc::new(UInt8Vector::from_iter_values( - iter::repeat(1).take(num_rows), - )), + Arc::new(UInt64Vector::from_iter_values(std::iter::repeat_n( + 0, num_rows, + ))), + Arc::new(UInt64Vector::from_iter_values(std::iter::repeat_n( + 0, num_rows, + ))), + Arc::new(UInt8Vector::from_iter_values(std::iter::repeat_n( + 1, num_rows, + ))), vec![u64_field], ) .unwrap() diff --git a/src/mito2/src/sst/parquet/format.rs b/src/mito2/src/sst/parquet/format.rs index c90907f0eb..005e276bbd 100644 --- a/src/mito2/src/sst/parquet/format.rs +++ b/src/mito2/src/sst/parquet/format.rs @@ -755,7 +755,7 @@ mod tests { )); let mut keys = vec![]; for (index, num_rows) in pk_row_nums.iter().map(|v| v.1).enumerate() { - keys.extend(std::iter::repeat(index as u32).take(num_rows)); + keys.extend(std::iter::repeat_n(index as u32, num_rows)); } let keys = UInt32Array::from(keys); Arc::new(DictionaryArray::new(keys, values)) diff --git a/src/operator/src/req_convert/insert/fill_impure_default.rs b/src/operator/src/req_convert/insert/fill_impure_default.rs index a60138c6e5..cf1e1565a8 100644 --- a/src/operator/src/req_convert/insert/fill_impure_default.rs +++ b/src/operator/src/req_convert/insert/fill_impure_default.rs @@ -85,11 +85,9 @@ impl ImpureDefaultFiller { .schema .iter() .filter_map(|schema| { - if self.impure_columns.contains_key(&schema.column_name) { - Some(&schema.column_name) - } else { - None - } + self.impure_columns + .contains_key(&schema.column_name) + .then_some(&schema.column_name) }) .collect(); diff --git a/src/pipeline/src/etl/processor/dissect.rs b/src/pipeline/src/etl/processor/dissect.rs index 8c31f42ace..8034d984d2 100644 --- a/src/pipeline/src/etl/processor/dissect.rs +++ b/src/pipeline/src/etl/processor/dissect.rs @@ -325,7 +325,7 @@ impl std::str::FromStr for Pattern { impl Pattern { fn check(&self) -> Result<()> { - if self.len() == 0 { + if self.is_empty() { return DissectEmptyPatternSnafu.fail(); } diff --git a/src/query/src/lib.rs b/src/query/src/lib.rs index 0aed2860e9..428aa5cb45 100644 --- a/src/query/src/lib.rs +++ b/src/query/src/lib.rs @@ -14,7 +14,6 @@ #![feature(let_chains)] #![feature(int_roundings)] -#![feature(trait_upcasting)] #![feature(try_blocks)] #![feature(stmt_expr_attributes)] #![feature(iterator_try_collect)] diff --git a/src/query/src/part_sort.rs b/src/query/src/part_sort.rs index 1c784c8b33..cd35fb66fb 100644 --- a/src/query/src/part_sort.rs +++ b/src/query/src/part_sort.rs @@ -348,7 +348,7 @@ impl PartSortStream { &self, sort_column: &ArrayRef, ) -> datafusion_common::Result> { - if sort_column.len() == 0 { + if sort_column.is_empty() { return Ok(Some(0)); } diff --git a/src/servers/src/http/timeout.rs b/src/servers/src/http/timeout.rs index 050ec492e0..adfc29cd95 100644 --- a/src/servers/src/http/timeout.rs +++ b/src/servers/src/http/timeout.rs @@ -117,7 +117,7 @@ where fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { match self.inner.poll_ready(cx) { Poll::Pending => Poll::Pending, - Poll::Ready(r) => Poll::Ready(r.map_err(Into::into)), + Poll::Ready(r) => Poll::Ready(r), } } diff --git a/src/servers/src/lib.rs b/src/servers/src/lib.rs index a13cd0ce1f..f55bc76e17 100644 --- a/src/servers/src/lib.rs +++ b/src/servers/src/lib.rs @@ -17,7 +17,6 @@ #![feature(exclusive_wrapper)] #![feature(let_chains)] #![feature(if_let_guard)] -#![feature(trait_upcasting)] use datafusion_expr::LogicalPlan; use datatypes::schema::Schema; diff --git a/src/sql/src/statements/transform.rs b/src/sql/src/statements/transform.rs index 2ca642cd11..7bd4218d2f 100644 --- a/src/sql/src/statements/transform.rs +++ b/src/sql/src/statements/transform.rs @@ -55,7 +55,7 @@ pub fn transform_statements(stmts: &mut Vec) -> Result<()> { } } - visit_expressions_mut(stmts, |expr| { + let _ = visit_expressions_mut(stmts, |expr| { for rule in RULES.iter() { rule.visit_expr(expr)?; } diff --git a/src/store-api/src/metadata.rs b/src/store-api/src/metadata.rs index 426e3f69ca..e8372a8df2 100644 --- a/src/store-api/src/metadata.rs +++ b/src/store-api/src/metadata.rs @@ -290,7 +290,7 @@ impl RegionMetadata { pub fn project(&self, projection: &[ColumnId]) -> Result { // check time index ensure!( - projection.iter().any(|id| *id == self.time_index), + projection.contains(&self.time_index), TimeIndexNotFoundSnafu ); diff --git a/src/table/src/table/adapter.rs b/src/table/src/table/adapter.rs index 2cf9a9647f..4ba880b1eb 100644 --- a/src/table/src/table/adapter.rs +++ b/src/table/src/table/adapter.rs @@ -95,7 +95,7 @@ impl TableProvider for DfTableProviderAdapter { filters: &[Expr], limit: Option, ) -> DfResult> { - let filters: Vec = filters.iter().map(Clone::clone).map(Into::into).collect(); + let filters: Vec = filters.iter().map(Clone::clone).collect(); let request = { let mut request = self.scan_req.lock().unwrap(); request.filters = filters; diff --git a/tests-fuzz/src/utils.rs b/tests-fuzz/src/utils.rs index f52d75f4da..ec7f1d8b27 100644 --- a/tests-fuzz/src/utils.rs +++ b/tests-fuzz/src/utils.rs @@ -85,11 +85,7 @@ pub struct UnstableTestVariables { pub fn load_unstable_test_env_variables() -> UnstableTestVariables { let _ = dotenv::dotenv(); let binary_path = env::var(GT_FUZZ_BINARY_PATH).expect("GT_FUZZ_BINARY_PATH not found"); - let root_dir = if let Ok(root) = env::var(GT_FUZZ_INSTANCE_ROOT_DIR) { - Some(root) - } else { - None - }; + let root_dir = env::var(GT_FUZZ_INSTANCE_ROOT_DIR).ok(); UnstableTestVariables { binary_path, diff --git a/tests-fuzz/targets/unstable/fuzz_create_table_standalone.rs b/tests-fuzz/targets/unstable/fuzz_create_table_standalone.rs index 575659ab8e..53369e9792 100644 --- a/tests-fuzz/targets/unstable/fuzz_create_table_standalone.rs +++ b/tests-fuzz/targets/unstable/fuzz_create_table_standalone.rs @@ -157,7 +157,7 @@ async fn execute_unstable_create_table( } Err(err) => { // FIXME(weny): support to retry it later. - if matches!(err, sqlx::Error::PoolTimedOut { .. }) { + if matches!(err, sqlx::Error::PoolTimedOut) { warn!("ignore pool timeout, sql: {sql}"); continue; } diff --git a/tests-integration/src/cluster.rs b/tests-integration/src/cluster.rs index 836c3a5483..0b18a71d3a 100644 --- a/tests-integration/src/cluster.rs +++ b/tests-integration/src/cluster.rs @@ -489,10 +489,7 @@ async fn create_datanode_client(datanode: &Datanode) -> (String, Client) { if let Some(client) = client { Ok(TokioIo::new(client)) } else { - Err(std::io::Error::new( - std::io::ErrorKind::Other, - "Client already taken", - )) + Err(std::io::Error::other("Client already taken")) } } }), diff --git a/tests/cases/standalone/common/system/pg_catalog.result b/tests/cases/standalone/common/system/pg_catalog.result index 092e9cab06..9e154b115c 100644 --- a/tests/cases/standalone/common/system/pg_catalog.result +++ b/tests/cases/standalone/common/system/pg_catalog.result @@ -3,7 +3,7 @@ create database pg_catalog; Error: 1004(InvalidArguments), Schema pg_catalog already exists --- session_user because session_user is based on the current user so is not null is for test +-- session_user because session_user is based on the current user so is not null is for test -- SQLNESS PROTOCOL POSTGRES SELECT session_user is not null; @@ -107,12 +107,13 @@ select * from pg_catalog.pg_type order by oid; +-----+-----------+--------+ -- SQLNESS PROTOCOL POSTGRES +-- SQLNESS REPLACE (\d+\s*) OID select * from pg_catalog.pg_database where datname = 'public'; +------------+---------+ | oid | datname | +------------+---------+ -| 3927743705 | public | +| OID| public | +------------+---------+ -- \d @@ -159,15 +160,16 @@ ORDER BY 1,2; -- make sure oid of namespace keep stable -- SQLNESS PROTOCOL POSTGRES -SELECT * FROM pg_namespace ORDER BY oid; +-- SQLNESS REPLACE (\d+\s*) OID +SELECT * FROM pg_namespace ORDER BY nspname; +------------+--------------------+ | oid | nspname | +------------+--------------------+ -| 667359454 | pg_catalog | -| 3174397350 | information_schema | -| 3338153620 | greptime_private | -| 3927743705 | public | +| OID| greptime_private | +| OID| information_schema | +| OID| pg_catalog | +| OID| public | +------------+--------------------+ -- SQLNESS PROTOCOL POSTGRES @@ -260,6 +262,7 @@ where relnamespace in ( +---------+ -- SQLNESS PROTOCOL POSTGRES +-- SQLNESS REPLACE (\d+\s*) OID select relnamespace, relname, relkind from pg_catalog.pg_class where relnamespace in ( @@ -274,7 +277,7 @@ order by relnamespace, relname; +--------------+---------+---------+ | relnamespace | relname | relkind | +--------------+---------+---------+ -| 434869349 | foo | r | +| OID| foo | r | +--------------+---------+---------+ -- SQLNESS PROTOCOL POSTGRES diff --git a/tests/cases/standalone/common/system/pg_catalog.sql b/tests/cases/standalone/common/system/pg_catalog.sql index 4a110a8f07..0b79c62afe 100644 --- a/tests/cases/standalone/common/system/pg_catalog.sql +++ b/tests/cases/standalone/common/system/pg_catalog.sql @@ -1,7 +1,7 @@ -- should not able to create pg_catalog create database pg_catalog; --- session_user because session_user is based on the current user so is not null is for test +-- session_user because session_user is based on the current user so is not null is for test -- SQLNESS PROTOCOL POSTGRES SELECT session_user is not null; @@ -34,6 +34,7 @@ select * from pg_catalog.pg_database; select * from pg_catalog.pg_type order by oid; -- SQLNESS PROTOCOL POSTGRES +-- SQLNESS REPLACE (\d+\s*) OID select * from pg_catalog.pg_database where datname = 'public'; -- \d @@ -68,7 +69,8 @@ ORDER BY 1,2; -- make sure oid of namespace keep stable -- SQLNESS PROTOCOL POSTGRES -SELECT * FROM pg_namespace ORDER BY oid; +-- SQLNESS REPLACE (\d+\s*) OID +SELECT * FROM pg_namespace ORDER BY nspname; -- SQLNESS PROTOCOL POSTGRES create database my_db; @@ -128,6 +130,7 @@ where relnamespace in ( ); -- SQLNESS PROTOCOL POSTGRES +-- SQLNESS REPLACE (\d+\s*) OID select relnamespace, relname, relkind from pg_catalog.pg_class where relnamespace in (