mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-10-30 07:26:56 +00:00
feat!: switch prom remote write to metric engine (#3198)
* feat: switch prom remote write to metric engine Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * Apply suggestions from code review Co-authored-by: dennis zhuang <killme2008@gmail.com> * fix compile Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * read physical table name from url Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * remove physical table from header Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * fix merge error Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * fix format Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * add with_metric_engine option to config remote write behavior Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * check parameter Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * add specific config param Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * default with_metric_engine to true Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * update UT Signed-off-by: Ruihang Xia <waynestxia@gmail.com> --------- Signed-off-by: Ruihang Xia <waynestxia@gmail.com> Co-authored-by: dennis zhuang <killme2008@gmail.com>
This commit is contained in:
Ruihang Xia
co-authored by
dennis zhuang
dennis zhuang
parent
6a12c27e78
commit
31787f4bfd
+124
-8
@@ -18,13 +18,15 @@ use std::sync::Arc;
|
||||
use api::v1::alter_expr::Kind;
|
||||
use api::v1::region::{InsertRequests as RegionInsertRequests, RegionRequestHeader};
|
||||
use api::v1::{
|
||||
AlterExpr, ColumnSchema, CreateTableExpr, InsertRequests, RowInsertRequest, RowInsertRequests,
|
||||
AlterExpr, ColumnDataType, ColumnSchema, CreateTableExpr, InsertRequests, RowInsertRequest,
|
||||
RowInsertRequests, SemanticType,
|
||||
};
|
||||
use catalog::CatalogManagerRef;
|
||||
use common_catalog::consts::default_engine;
|
||||
use common_grpc_expr::util::{extract_new_columns, ColumnExpr};
|
||||
use common_meta::datanode_manager::{AffectedRows, DatanodeManagerRef};
|
||||
use common_meta::peer::Peer;
|
||||
use common_query::prelude::{GREPTIME_TIMESTAMP, GREPTIME_VALUE};
|
||||
use common_query::Output;
|
||||
use common_telemetry::tracing_context::TracingContext;
|
||||
use common_telemetry::{error, info};
|
||||
@@ -35,6 +37,9 @@ use partition::manager::PartitionRuleManagerRef;
|
||||
use session::context::QueryContextRef;
|
||||
use snafu::prelude::*;
|
||||
use sql::statements::insert::Insert;
|
||||
use store_api::metric_engine_consts::{
|
||||
LOGICAL_TABLE_METADATA_KEY, METRIC_ENGINE_NAME, PHYSICAL_TABLE_METADATA_KEY,
|
||||
};
|
||||
use table::requests::InsertRequest as TableInsertRequest;
|
||||
use table::table_reference::TableReference;
|
||||
use table::TableRef;
|
||||
@@ -95,7 +100,7 @@ impl Inserter {
|
||||
});
|
||||
validate_column_count_match(&requests)?;
|
||||
|
||||
self.create_or_alter_tables_on_demand(&requests, &ctx, statement_executor)
|
||||
self.create_or_alter_tables_on_demand(&requests, &ctx, None, statement_executor)
|
||||
.await?;
|
||||
let inserts = RowToRegion::new(
|
||||
self.catalog_manager.as_ref(),
|
||||
@@ -109,6 +114,44 @@ impl Inserter {
|
||||
Ok(Output::AffectedRows(affected_rows as _))
|
||||
}
|
||||
|
||||
/// Handle row inserts request with metric engine.
|
||||
pub async fn handle_metric_row_inserts(
|
||||
&self,
|
||||
mut requests: RowInsertRequests,
|
||||
ctx: QueryContextRef,
|
||||
statement_executor: &StatementExecutor,
|
||||
physical_table: String,
|
||||
) -> Result<Output> {
|
||||
// remove empty requests
|
||||
requests.inserts.retain(|req| {
|
||||
req.rows
|
||||
.as_ref()
|
||||
.map(|r| !r.rows.is_empty())
|
||||
.unwrap_or_default()
|
||||
});
|
||||
validate_column_count_match(&requests)?;
|
||||
|
||||
// check and create physical table
|
||||
self.create_physical_table_on_demand(&ctx, physical_table.clone(), statement_executor)
|
||||
.await?;
|
||||
|
||||
// check and create logical tables
|
||||
self.create_or_alter_tables_on_demand(
|
||||
&requests,
|
||||
&ctx,
|
||||
Some(physical_table.to_string()),
|
||||
statement_executor,
|
||||
)
|
||||
.await?;
|
||||
let inserts =
|
||||
RowToRegion::new(self.catalog_manager.as_ref(), &self.partition_manager, &ctx)
|
||||
.convert(requests)
|
||||
.await?;
|
||||
|
||||
let affected_rows = self.do_request(inserts, &ctx).await?;
|
||||
Ok(Output::AffectedRows(affected_rows as _))
|
||||
}
|
||||
|
||||
pub async fn handle_table_insert(
|
||||
&self,
|
||||
request: TableInsertRequest,
|
||||
@@ -206,9 +249,10 @@ impl Inserter {
|
||||
&self,
|
||||
requests: &RowInsertRequests,
|
||||
ctx: &QueryContextRef,
|
||||
on_physical_table: Option<String>,
|
||||
statement_executor: &StatementExecutor,
|
||||
) -> Result<()> {
|
||||
// TODO(jeremy): create and alter in batch?
|
||||
// TODO(jeremy): create and alter in batch? (from `handle_metric_row_inserts`)
|
||||
for req in &requests.inserts {
|
||||
let catalog = ctx.current_catalog();
|
||||
let schema = ctx.current_schema();
|
||||
@@ -219,13 +263,76 @@ impl Inserter {
|
||||
self.alter_table_on_demand(req, table, ctx, statement_executor)
|
||||
.await?
|
||||
}
|
||||
None => self.create_table(req, ctx, statement_executor).await?,
|
||||
None => {
|
||||
self.create_table(req, ctx, &on_physical_table, statement_executor)
|
||||
.await?
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn create_physical_table_on_demand(
|
||||
&self,
|
||||
ctx: &QueryContextRef,
|
||||
physical_table: String,
|
||||
statement_executor: &StatementExecutor,
|
||||
) -> Result<()> {
|
||||
let catalog_name = ctx.current_catalog();
|
||||
let schema_name = ctx.current_schema();
|
||||
|
||||
// check if exist
|
||||
if self
|
||||
.get_table(catalog_name, schema_name, &physical_table)
|
||||
.await?
|
||||
.is_some()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let table_reference = TableReference::full(catalog_name, schema_name, &physical_table);
|
||||
info!("Physical metric table `{table_reference}` does not exist, try creating table");
|
||||
|
||||
// schema with timestamp and field column
|
||||
let default_schema = vec![
|
||||
ColumnSchema {
|
||||
column_name: GREPTIME_TIMESTAMP.to_string(),
|
||||
datatype: ColumnDataType::TimestampMillisecond as _,
|
||||
semantic_type: SemanticType::Timestamp as _,
|
||||
datatype_extension: None,
|
||||
},
|
||||
ColumnSchema {
|
||||
column_name: GREPTIME_VALUE.to_string(),
|
||||
datatype: ColumnDataType::Float64 as _,
|
||||
semantic_type: SemanticType::Field as _,
|
||||
datatype_extension: None,
|
||||
},
|
||||
];
|
||||
let create_table_expr = &mut build_create_table_expr(&table_reference, &default_schema)?;
|
||||
|
||||
create_table_expr.engine = METRIC_ENGINE_NAME.to_string();
|
||||
create_table_expr
|
||||
.table_options
|
||||
.insert(PHYSICAL_TABLE_METADATA_KEY.to_string(), "true".to_string());
|
||||
|
||||
// create physical table
|
||||
let res = statement_executor
|
||||
.create_table_inner(create_table_expr, None)
|
||||
.await;
|
||||
|
||||
match res {
|
||||
Ok(_) => {
|
||||
info!("Successfully created table {table_reference}",);
|
||||
Ok(())
|
||||
}
|
||||
Err(err) => {
|
||||
error!("Failed to create table {table_reference}: {err}",);
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_table(
|
||||
&self,
|
||||
catalog: &str,
|
||||
@@ -289,10 +396,14 @@ impl Inserter {
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a table with schema from insert request.
|
||||
///
|
||||
/// To create a metric engine logical table, specify the `on_physical_table` parameter.
|
||||
async fn create_table(
|
||||
&self,
|
||||
req: &RowInsertRequest,
|
||||
ctx: &QueryContextRef,
|
||||
on_physical_table: &Option<String>,
|
||||
statement_executor: &StatementExecutor,
|
||||
) -> Result<()> {
|
||||
let table_ref =
|
||||
@@ -301,10 +412,15 @@ impl Inserter {
|
||||
let request_schema = req.rows.as_ref().unwrap().schema.as_slice();
|
||||
let create_table_expr = &mut build_create_table_expr(&table_ref, request_schema)?;
|
||||
|
||||
info!(
|
||||
"Table {}.{}.{} does not exist, try create table",
|
||||
table_ref.catalog, table_ref.schema, table_ref.table,
|
||||
);
|
||||
if let Some(physical_table) = on_physical_table {
|
||||
create_table_expr.engine = METRIC_ENGINE_NAME.to_string();
|
||||
create_table_expr.table_options.insert(
|
||||
LOGICAL_TABLE_METADATA_KEY.to_string(),
|
||||
physical_table.clone(),
|
||||
);
|
||||
}
|
||||
|
||||
info!("Table `{table_ref}` does not exist, try creating table",);
|
||||
|
||||
// TODO(weny): multiple regions table.
|
||||
let res = statement_executor
|
||||
|
||||
Reference in New Issue
Block a user