1use std::collections::BTreeMap;
16use std::sync::Arc;
17
18use ahash::{HashMap, HashMapExt, HashSet, HashSetExt};
19use api::v1::alter_table_expr::Kind;
20use api::v1::column_def::options_from_skipping;
21use api::v1::region::{
22 InsertRequest as RegionInsertRequest, InsertRequests as RegionInsertRequests,
23 RegionRequestHeader,
24};
25use api::v1::{
26 AlterTableExpr, ColumnDataType, ColumnSchema, CreateTableExpr, InsertRequests,
27 RowInsertRequest, RowInsertRequests, SemanticType,
28};
29use catalog::CatalogManagerRef;
30use client::{OutputData, OutputMeta};
31use common_catalog::consts::{
32 PARENT_SPAN_ID_COLUMN, SERVICE_NAME_COLUMN, TRACE_ID_COLUMN, TRACE_TABLE_NAME,
33 TRACE_TABLE_NAME_SESSION_KEY, default_engine, is_ddl_reserved_table,
34 trace_operations_table_name, trace_services_table_name,
35};
36use common_grpc_expr::util::ColumnExpr;
37use common_meta::cache::TableFlownodeSetCacheRef;
38use common_meta::node_manager::{AffectedRows, NodeManagerRef};
39use common_meta::peer::Peer;
40use common_meta::rpc::ddl::TriggerReason;
41use common_query::Output;
42use common_query::native_histogram::{is_native_histogram_value_type, native_histogram_value_type};
43use common_query::prelude::{greptime_timestamp, greptime_value};
44use common_telemetry::tracing_context::TracingContext;
45use common_telemetry::{error, info, warn};
46use datatypes::schema::SkippingIndexOptions;
47use futures_util::future;
48use meter_macros::write_meter;
49use partition::manager::PartitionRuleManagerRef;
50use session::context::QueryContextRef;
51use snafu::ResultExt;
52use snafu::prelude::*;
53use sql::partition::partition_rule_for_hexstring;
54use sql::statements::create::Partitions;
55use sql::statements::insert::Insert;
56use store_api::metric_engine_consts::{
57 LOGICAL_TABLE_METADATA_KEY, METRIC_ENGINE_NAME, PHYSICAL_TABLE_METADATA_KEY,
58};
59use store_api::mito_engine_options::{
60 APPEND_MODE_KEY, COMPACTION_TYPE, COMPACTION_TYPE_TWCS, MERGE_MODE_KEY, TTL_KEY,
61 TWCS_TIME_WINDOW,
62};
63use store_api::storage::{RegionId, TableId};
64use table::TableRef;
65use table::metadata::TableInfo;
66use table::requests::{
67 AUTO_CREATE_TABLE_KEY, InsertRequest as TableInsertRequest, SEMANTIC_PER_TABLE_INDEX_KEY,
68 TABLE_DATA_MODEL, TABLE_DATA_MODEL_TRACE_V1, TRACE_TABLE_PARTITIONS_HINT_KEY,
69 VALID_TABLE_OPTION_KEYS, is_semantic_option_key, validate_semantic_option,
70};
71use table::table_reference::TableReference;
72
73use crate::error::{
74 CatalogSnafu, ColumnOptionsSnafu, CreatePartitionRulesSnafu, FindRegionLeaderSnafu,
75 InvalidInsertRequestSnafu, JoinTaskSnafu, RequestInsertsSnafu, Result, TableNotFoundSnafu,
76};
77use crate::expr_helper;
78use crate::region_req_factory::RegionRequestFactory;
79use crate::req_convert::common::preprocess_row_insert_requests;
80use crate::req_convert::insert::{
81 ColumnToRow, RowToRegion, StatementToRegion, TableToRegion, fill_reqs_with_impure_default,
82};
83use crate::statement::StatementExecutor;
84
85pub struct Inserter {
86 catalog_manager: CatalogManagerRef,
87 pub(crate) partition_manager: PartitionRuleManagerRef,
88 pub(crate) node_manager: NodeManagerRef,
89 pub(crate) table_flownode_set_cache: TableFlownodeSetCacheRef,
90 auto_create_table: bool,
94}
95
96pub type InserterRef = Arc<Inserter>;
97
98#[derive(Clone)]
100pub enum AutoCreateTableType {
101 Logical(String),
103 Physical,
105 Log,
107 LastNonNull,
109 Trace { alter_existing: bool },
111}
112
113impl AutoCreateTableType {
114 pub fn as_str(&self) -> &'static str {
115 match self {
116 AutoCreateTableType::Logical(_) => "logical",
117 AutoCreateTableType::Physical => "physical",
118 AutoCreateTableType::Log => "log",
119 AutoCreateTableType::LastNonNull => "last_non_null",
120 AutoCreateTableType::Trace { .. } => "trace",
121 }
122 }
123
124 fn alter_existing(&self) -> bool {
125 !matches!(
126 self,
127 Self::Trace {
128 alter_existing: false
129 }
130 )
131 }
132}
133
134#[derive(Clone)]
141pub struct InstantAndNormalInsertRequests {
142 pub normal_requests: RegionInsertRequests,
144 pub instant_requests: RegionInsertRequests,
147}
148
149impl Inserter {
150 pub fn new(
151 catalog_manager: CatalogManagerRef,
152 partition_manager: PartitionRuleManagerRef,
153 node_manager: NodeManagerRef,
154 table_flownode_set_cache: TableFlownodeSetCacheRef,
155 auto_create_table: bool,
156 ) -> Self {
157 Self {
158 catalog_manager,
159 partition_manager,
160 node_manager,
161 table_flownode_set_cache,
162 auto_create_table,
163 }
164 }
165
166 pub async fn handle_column_inserts(
167 &self,
168 requests: InsertRequests,
169 ctx: QueryContextRef,
170 statement_executor: &StatementExecutor,
171 ) -> Result<Output> {
172 let row_inserts = ColumnToRow::convert(requests)?;
173 self.handle_row_inserts(row_inserts, ctx, statement_executor, false, false)
174 .await
175 }
176
177 pub async fn handle_row_inserts(
179 &self,
180 mut requests: RowInsertRequests,
181 ctx: QueryContextRef,
182 statement_executor: &StatementExecutor,
183 accommodate_existing_schema: bool,
184 is_single_value: bool,
185 ) -> Result<Output> {
186 preprocess_row_insert_requests(&mut requests.inserts)?;
187 self.handle_row_inserts_with_create_type(
188 requests,
189 ctx,
190 statement_executor,
191 AutoCreateTableType::Physical,
192 accommodate_existing_schema,
193 is_single_value,
194 )
195 .await
196 }
197
198 pub async fn handle_log_inserts(
200 &self,
201 requests: RowInsertRequests,
202 ctx: QueryContextRef,
203 statement_executor: &StatementExecutor,
204 ) -> Result<Output> {
205 self.handle_row_inserts_with_create_type(
206 requests,
207 ctx,
208 statement_executor,
209 AutoCreateTableType::Log,
210 false,
211 false,
212 )
213 .await
214 }
215
216 pub async fn handle_trace_inserts(
217 &self,
218 requests: RowInsertRequests,
219 ctx: QueryContextRef,
220 statement_executor: &StatementExecutor,
221 ) -> Result<Output> {
222 self.handle_row_inserts_with_create_type(
223 requests,
224 ctx,
225 statement_executor,
226 AutoCreateTableType::Trace {
227 alter_existing: true,
228 },
229 false,
230 false,
231 )
232 .await
233 }
234
235 pub async fn handle_last_non_null_inserts(
237 &self,
238 requests: RowInsertRequests,
239 ctx: QueryContextRef,
240 statement_executor: &StatementExecutor,
241 accommodate_existing_schema: bool,
242 is_single_value: bool,
243 ) -> Result<Output> {
244 self.handle_row_inserts_with_create_type(
245 requests,
246 ctx,
247 statement_executor,
248 AutoCreateTableType::LastNonNull,
249 accommodate_existing_schema,
250 is_single_value,
251 )
252 .await
253 }
254
255 async fn handle_row_inserts_with_create_type(
257 &self,
258 mut requests: RowInsertRequests,
259 ctx: QueryContextRef,
260 statement_executor: &StatementExecutor,
261 create_type: AutoCreateTableType,
262 accommodate_existing_schema: bool,
263 is_single_value: bool,
264 ) -> Result<Output> {
265 requests.inserts.retain(|req| {
267 req.rows
268 .as_ref()
269 .map(|r| !r.rows.is_empty())
270 .unwrap_or_default()
271 });
272 validate_column_count_match(&requests)?;
273
274 let CreateAlterTableResult {
275 instant_table_ids,
276 table_infos,
277 } = self
278 .create_or_alter_tables_on_demand(
279 &mut requests,
280 &ctx,
281 create_type,
282 statement_executor,
283 accommodate_existing_schema,
284 is_single_value,
285 )
286 .await?;
287
288 let name_to_info = table_infos
289 .values()
290 .map(|info| (info.name.clone(), info.clone()))
291 .collect::<HashMap<_, _>>();
292 let inserts = RowToRegion::new(
293 name_to_info,
294 instant_table_ids,
295 self.partition_manager.as_ref(),
296 )
297 .convert(requests)
298 .await?;
299
300 self.do_request(inserts, &table_infos, &ctx).await
301 }
302
303 pub async fn handle_metric_row_inserts(
305 &self,
306 mut requests: RowInsertRequests,
307 ctx: QueryContextRef,
308 statement_executor: &StatementExecutor,
309 physical_table: String,
310 ) -> Result<Output> {
311 requests.inserts.retain(|req| {
313 req.rows
314 .as_ref()
315 .map(|r| !r.rows.is_empty())
316 .unwrap_or_default()
317 });
318 validate_column_count_match(&requests)?;
319
320 self.create_physical_table_on_demand(&ctx, physical_table.clone(), statement_executor)
322 .await?;
323
324 let CreateAlterTableResult {
326 instant_table_ids,
327 table_infos,
328 } = self
329 .create_or_alter_tables_on_demand(
330 &mut requests,
331 &ctx,
332 AutoCreateTableType::Logical(physical_table.clone()),
333 statement_executor,
334 true,
335 true,
336 )
337 .await?;
338 let name_to_info = table_infos
339 .values()
340 .map(|info| (info.name.clone(), info.clone()))
341 .collect::<HashMap<_, _>>();
342 let inserts = RowToRegion::new(name_to_info, instant_table_ids, &self.partition_manager)
343 .convert(requests)
344 .await?;
345
346 self.do_request(inserts, &table_infos, &ctx).await
347 }
348
349 pub async fn handle_table_insert(
350 &self,
351 request: TableInsertRequest,
352 ctx: QueryContextRef,
353 ) -> Result<Output> {
354 let catalog = request.catalog_name.as_str();
355 let schema = request.schema_name.as_str();
356 let table_name = request.table_name.as_str();
357 let table = self.get_table(catalog, schema, table_name).await?;
358 let table = table.with_context(|| TableNotFoundSnafu {
359 table_name: common_catalog::format_full_table_name(catalog, schema, table_name),
360 })?;
361 let table_info = table.table_info();
362
363 let inserts = TableToRegion::new(&table_info, &self.partition_manager)
364 .convert(request)
365 .await?;
366
367 let table_infos = HashMap::from_iter([(table_info.table_id(), table_info.clone())]);
368
369 self.do_request(inserts, &table_infos, &ctx).await
370 }
371
372 pub async fn handle_statement_insert(
373 &self,
374 insert: &Insert,
375 ctx: &QueryContextRef,
376 ) -> Result<Output> {
377 let (inserts, table_info) =
378 StatementToRegion::new(self.catalog_manager.as_ref(), &self.partition_manager, ctx)
379 .convert(insert, ctx)
380 .await?;
381
382 let table_infos = HashMap::from_iter([(table_info.table_id(), table_info.clone())]);
383
384 self.do_request(inserts, &table_infos, ctx).await
385 }
386}
387
388impl Inserter {
389 async fn do_request(
390 &self,
391 requests: InstantAndNormalInsertRequests,
392 table_infos: &HashMap<TableId, Arc<TableInfo>>,
393 ctx: &QueryContextRef,
394 ) -> Result<Output> {
395 let requests = fill_reqs_with_impure_default(table_infos, requests)?;
397
398 let write_cost = write_meter!(
399 ctx.current_catalog(),
400 ctx.current_schema(),
401 requests,
402 ctx.channel() as u8
403 );
404 let request_factory = RegionRequestFactory::new(RegionRequestHeader {
405 tracing_context: TracingContext::from_current_span().to_w3c(),
406 dbname: ctx.get_db_string(),
407 ..Default::default()
408 });
409
410 let InstantAndNormalInsertRequests {
411 normal_requests,
412 instant_requests,
413 } = requests;
414
415 let flow_mirror_task = FlowMirrorTask::new(
417 &self.table_flownode_set_cache,
418 normal_requests
419 .requests
420 .iter()
421 .chain(instant_requests.requests.iter()),
422 )
423 .await?;
424 flow_mirror_task.detach(self.node_manager.clone())?;
425
426 let write_tasks = self
428 .group_requests_by_peer(normal_requests)
429 .await?
430 .into_iter()
431 .map(|(peer, inserts)| {
432 let node_manager = self.node_manager.clone();
433 let request = request_factory.build_insert(inserts);
434 common_runtime::spawn_global(async move {
435 node_manager
436 .datanode(&peer)
437 .await
438 .handle(request)
439 .await
440 .context(RequestInsertsSnafu)
441 })
442 });
443 let results = future::try_join_all(write_tasks)
444 .await
445 .context(JoinTaskSnafu)?;
446 let affected_rows = results
447 .into_iter()
448 .map(|resp| resp.map(|r| r.affected_rows))
449 .sum::<Result<AffectedRows>>()?;
450 crate::metrics::DIST_INGEST_ROW_COUNT
451 .with_label_values(&[ctx.get_db_string().as_str()])
452 .inc_by(affected_rows as u64);
453 Ok(Output::new(
454 OutputData::AffectedRows(affected_rows),
455 OutputMeta::new_with_cost(write_cost as _),
456 ))
457 }
458
459 async fn group_requests_by_peer(
460 &self,
461 requests: RegionInsertRequests,
462 ) -> Result<HashMap<Peer, RegionInsertRequests>> {
463 let mut requests_per_region: HashMap<RegionId, RegionInsertRequests> = HashMap::new();
466 for req in requests.requests {
467 let region_id = RegionId::from_u64(req.region_id);
468 requests_per_region
469 .entry(region_id)
470 .or_default()
471 .requests
472 .push(req);
473 }
474
475 let mut inserts: HashMap<Peer, RegionInsertRequests> = HashMap::new();
476
477 for (region_id, reqs) in requests_per_region {
478 let peer = self
479 .partition_manager
480 .find_region_leader(region_id)
481 .await
482 .context(FindRegionLeaderSnafu)?;
483 inserts
484 .entry(peer)
485 .or_default()
486 .requests
487 .extend(reqs.requests);
488 }
489
490 Ok(inserts)
491 }
492
493 fn auto_create_disabled_reason(&self, ctx: &QueryContextRef) -> Result<Option<&'static str>> {
497 let auto_create_table_hint = ctx
498 .extension(AUTO_CREATE_TABLE_KEY)
499 .map(|v| v.parse::<bool>())
500 .transpose()
501 .map_err(|_| {
502 InvalidInsertRequestSnafu {
503 reason: "`auto_create_table` hint must be a boolean",
504 }
505 .build()
506 })?
507 .unwrap_or(true);
508 Ok(if !self.auto_create_table {
509 Some("auto-create table is disabled by frontend config")
510 } else if !auto_create_table_hint {
511 Some("`auto_create_table` hint is disabled")
512 } else {
513 None
514 })
515 }
516
517 pub async fn ensure_trace_table_on_demand(
522 &self,
523 table_name: &str,
524 request_schema: Vec<ColumnSchema>,
525 alter_existing: bool,
526 ctx: &QueryContextRef,
527 statement_executor: &StatementExecutor,
528 ) -> Result<()> {
529 let mut requests = RowInsertRequests {
530 inserts: vec![RowInsertRequest {
531 table_name: table_name.to_string(),
532 rows: Some(api::v1::Rows {
533 schema: request_schema,
534 rows: Vec::new(),
535 }),
536 }],
537 };
538 self.create_or_alter_tables_on_demand(
539 &mut requests,
540 ctx,
541 AutoCreateTableType::Trace { alter_existing },
542 statement_executor,
543 false,
544 false,
545 )
546 .await?;
547 Ok(())
548 }
549
550 async fn create_or_alter_tables_on_demand(
563 &self,
564 requests: &mut RowInsertRequests,
565 ctx: &QueryContextRef,
566 auto_create_table_type: AutoCreateTableType,
567 statement_executor: &StatementExecutor,
568 accommodate_existing_schema: bool,
569 is_single_value: bool,
570 ) -> Result<CreateAlterTableResult> {
571 let _timer = crate::metrics::CREATE_ALTER_ON_DEMAND
572 .with_label_values(&[auto_create_table_type.as_str()])
573 .start_timer();
574
575 let catalog = ctx.current_catalog();
576 let schema = ctx.current_schema();
577
578 let mut table_infos = HashMap::new();
579 if let Some(disabled_reason) = self.auto_create_disabled_reason(ctx)? {
580 let mut instant_table_ids = HashSet::new();
581 for req in &requests.inserts {
582 let table = match self.get_table(catalog, &schema, &req.table_name).await? {
583 Some(table) => table,
584 None if is_ddl_reserved_table(&schema, &req.table_name) => {
587 statement_executor
588 .create_declared_relationships_table(catalog, ctx.clone())
589 .await?
590 }
591 None => {
592 return InvalidInsertRequestSnafu {
593 reason: format!(
594 "Table `{}` does not exist, and {}",
595 req.table_name, disabled_reason
596 ),
597 }
598 .fail();
599 }
600 };
601 let table_info = table.table_info();
602 if table_info.is_ttl_instant_table() {
603 instant_table_ids.insert(table_info.table_id());
604 }
605 table_infos.insert(table_info.table_id(), table.table_info());
606 }
607 let ret = CreateAlterTableResult {
608 instant_table_ids,
609 table_infos,
610 };
611 return Ok(ret);
612 }
613
614 let mut create_tables = vec![];
615 let mut alter_tables = vec![];
616 let mut need_refresh_table_infos = HashSet::new();
617 let mut instant_table_ids = HashSet::new();
618 let mut per_table_semantics: Option<Option<PerTableSemanticIndex>> = None;
619
620 for req in &mut requests.inserts {
621 match self.get_table(catalog, &schema, &req.table_name).await? {
622 Some(table) => {
623 let table_info = table.table_info();
624 if table_info.is_ttl_instant_table() {
625 instant_table_ids.insert(table_info.table_id());
626 }
627 if let Some(alter_expr) = self.get_alter_table_expr_on_demand(
628 req,
629 &table,
630 ctx,
631 accommodate_existing_schema,
632 is_single_value,
633 auto_create_table_type.alter_existing(),
634 )? {
635 alter_tables.push(alter_expr);
636 need_refresh_table_infos.insert((
637 catalog.to_string(),
638 schema.clone(),
639 req.table_name.clone(),
640 ));
641 } else {
642 table_infos.insert(table_info.table_id(), table.table_info());
643 }
644 }
645 None if is_ddl_reserved_table(&schema, &req.table_name) => {
649 let table = statement_executor
650 .create_declared_relationships_table(catalog, ctx.clone())
651 .await?;
652 let table_info = table.table_info();
653 if table_info.is_ttl_instant_table() {
654 instant_table_ids.insert(table_info.table_id());
655 }
656 table_infos.insert(table_info.table_id(), table_info);
657 }
658 None => {
659 let semantic_index = per_table_semantics
660 .get_or_insert_with(|| parse_per_table_semantic_index(ctx))
661 .as_ref();
662 let create_expr = self.get_create_table_expr_on_demand(
663 req,
664 &auto_create_table_type,
665 ctx,
666 semantic_index,
667 )?;
668 create_tables.push(create_expr);
669 }
670 }
671 }
672
673 match auto_create_table_type {
674 AutoCreateTableType::Logical(_) => {
675 if !create_tables.is_empty() {
676 let tables = self
678 .create_logical_tables(create_tables, ctx, statement_executor)
679 .await?;
680
681 for table in tables {
682 let table_info = table.table_info();
683 if table_info.is_ttl_instant_table() {
684 instant_table_ids.insert(table_info.table_id());
685 }
686 table_infos.insert(table_info.table_id(), table.table_info());
687 }
688 }
689 if !alter_tables.is_empty() {
690 statement_executor
692 .alter_logical_tables(alter_tables, ctx.clone(), TriggerReason::AutoAlter)
693 .await?;
694 }
695 }
696 AutoCreateTableType::Physical
697 | AutoCreateTableType::Log
698 | AutoCreateTableType::LastNonNull => {
699 for create_table in create_tables {
702 let table = self
703 .create_physical_table(create_table, None, ctx, statement_executor)
704 .await?;
705 let table_info = table.table_info();
706 if table_info.is_ttl_instant_table() {
707 instant_table_ids.insert(table_info.table_id());
708 }
709 table_infos.insert(table_info.table_id(), table.table_info());
710 }
711 for alter_expr in alter_tables.into_iter() {
712 statement_executor
713 .alter_table_inner(alter_expr, ctx.clone(), TriggerReason::AutoAlter)
714 .await?;
715 }
716 }
717
718 AutoCreateTableType::Trace { .. } => {
719 let trace_table_name = ctx
720 .extension(TRACE_TABLE_NAME_SESSION_KEY)
721 .unwrap_or(TRACE_TABLE_NAME);
722
723 let trace_table_partitions = if let Some(trace_table_partitions) =
724 ctx.extension(TRACE_TABLE_PARTITIONS_HINT_KEY)
725 {
726 let p = trace_table_partitions.parse::<u32>().map_err(|_| {
727 InvalidInsertRequestSnafu {
728 reason: format!(
729 "Failed to parse trace_table_partitions: {}",
730 trace_table_partitions
731 ),
732 }
733 .build()
734 })?;
735 Some(p)
736 } else {
737 None
738 };
739
740 for mut create_table in create_tables {
743 if create_table.table_name == trace_services_table_name(trace_table_name)
744 || create_table.table_name == trace_operations_table_name(trace_table_name)
745 {
746 create_table
748 .table_options
749 .insert(APPEND_MODE_KEY.to_string(), "false".to_string());
750 create_table.table_options.remove(TTL_KEY);
752
753 let table = self
754 .create_physical_table(create_table, None, ctx, statement_executor)
755 .await?;
756 let table_info = table.table_info();
757 if table_info.is_ttl_instant_table() {
758 instant_table_ids.insert(table_info.table_id());
759 }
760 table_infos.insert(table_info.table_id(), table.table_info());
761 } else {
762 let partitions = if matches!(trace_table_partitions, Some(0) | Some(1)) {
765 None
767 } else {
768 let p = partition_rule_for_hexstring(
769 TRACE_ID_COLUMN,
770 trace_table_partitions,
771 )
772 .context(CreatePartitionRulesSnafu)?;
773 Some(p)
774 };
775
776 let index_columns =
781 [TRACE_ID_COLUMN, PARENT_SPAN_ID_COLUMN, SERVICE_NAME_COLUMN];
782 for index_column in index_columns {
783 if let Some(col) = create_table
784 .column_defs
785 .iter_mut()
786 .find(|c| c.name == index_column)
787 {
788 col.options =
789 options_from_skipping(&SkippingIndexOptions::default())
790 .context(ColumnOptionsSnafu)?;
791 } else {
792 warn!(
793 "Column {} not found when creating index for trace table: {}.",
794 index_column, create_table.table_name
795 );
796 }
797 }
798
799 create_table.table_options.insert(
801 TABLE_DATA_MODEL.to_string(),
802 TABLE_DATA_MODEL_TRACE_V1.to_string(),
803 );
804
805 let table = self
806 .create_physical_table(
807 create_table,
808 partitions,
809 ctx,
810 statement_executor,
811 )
812 .await?;
813 let table_info = table.table_info();
814 if table_info.is_ttl_instant_table() {
815 instant_table_ids.insert(table_info.table_id());
816 }
817 table_infos.insert(table_info.table_id(), table.table_info());
818 }
819 }
820 for alter_expr in alter_tables.into_iter() {
821 statement_executor
822 .alter_table_inner(alter_expr, ctx.clone(), TriggerReason::AutoAlter)
823 .await?;
824 }
825 }
826 }
827
828 for (catalog, schema, table_name) in need_refresh_table_infos {
830 let table = self
831 .get_table(&catalog, &schema, &table_name)
832 .await?
833 .context(TableNotFoundSnafu {
834 table_name: common_catalog::format_full_table_name(
835 &catalog,
836 &schema,
837 &table_name,
838 ),
839 })?;
840 let table_info = table.table_info();
841 table_infos.insert(table_info.table_id(), table.table_info());
842 }
843
844 Ok(CreateAlterTableResult {
845 instant_table_ids,
846 table_infos,
847 })
848 }
849
850 async fn create_physical_table_on_demand(
851 &self,
852 ctx: &QueryContextRef,
853 physical_table: String,
854 statement_executor: &StatementExecutor,
855 ) -> Result<()> {
856 let catalog_name = ctx.current_catalog();
857 let schema_name = ctx.current_schema();
858
859 if self
861 .get_table(catalog_name, &schema_name, &physical_table)
862 .await?
863 .is_some()
864 {
865 return Ok(());
866 }
867
868 if let Some(disabled_reason) = self.auto_create_disabled_reason(ctx)? {
870 return InvalidInsertRequestSnafu {
871 reason: format!(
872 "Physical table `{physical_table}` does not exist, and {disabled_reason}"
873 ),
874 }
875 .fail();
876 }
877
878 let table_reference = TableReference::full(catalog_name, &schema_name, &physical_table);
879 info!("Physical metric table `{table_reference}` does not exist, try creating table");
880
881 let default_schema = vec![
883 ColumnSchema {
884 column_name: greptime_timestamp().to_string(),
885 datatype: ColumnDataType::TimestampMillisecond as _,
886 semantic_type: SemanticType::Timestamp as _,
887 datatype_extension: None,
888 options: None,
889 },
890 ColumnSchema {
891 column_name: greptime_value().to_string(),
892 datatype: ColumnDataType::Float64 as _,
893 semantic_type: SemanticType::Field as _,
894 datatype_extension: None,
895 options: None,
896 },
897 ];
898 let create_table_expr =
899 &mut build_create_table_expr(&table_reference, &default_schema, default_engine())?;
900
901 create_table_expr.engine = METRIC_ENGINE_NAME.to_string();
902 create_table_expr
903 .table_options
904 .insert(PHYSICAL_TABLE_METADATA_KEY.to_string(), "true".to_string());
905
906 let res = statement_executor
908 .create_table_inner(
909 create_table_expr,
910 None,
911 ctx.clone(),
912 TriggerReason::AutoCreate,
913 )
914 .await;
915
916 match res {
917 Ok(_) => {
918 info!("Successfully created table {table_reference}",);
919 Ok(())
920 }
921 Err(err) => {
922 error!(err; "Failed to create table {table_reference}");
923 Err(err)
924 }
925 }
926 }
927
928 async fn get_table(
929 &self,
930 catalog: &str,
931 schema: &str,
932 table: &str,
933 ) -> Result<Option<TableRef>> {
934 self.catalog_manager
935 .table(catalog, schema, table, None)
936 .await
937 .context(CatalogSnafu)
938 }
939
940 fn get_create_table_expr_on_demand(
941 &self,
942 req: &RowInsertRequest,
943 create_type: &AutoCreateTableType,
944 ctx: &QueryContextRef,
945 semantic_index: Option<&PerTableSemanticIndex>,
946 ) -> Result<CreateTableExpr> {
947 let schema = ctx.current_schema();
948 let mut table_options = std::collections::HashMap::with_capacity(4);
949 fill_table_options_for_create(&mut table_options, create_type, ctx);
950 apply_per_table_semantic_options(
951 &mut table_options,
952 semantic_index,
953 ctx.current_schema().as_str(),
954 &req.table_name,
955 );
956
957 let engine_name = if let AutoCreateTableType::Logical(_) = create_type {
958 METRIC_ENGINE_NAME
960 } else {
961 default_engine()
962 };
963
964 let table_ref = TableReference::full(ctx.current_catalog(), &schema, &req.table_name);
965 let request_schema = req.rows.as_ref().unwrap().schema.as_slice();
967 let mut create_table_expr =
968 build_create_table_expr(&table_ref, request_schema, engine_name)?;
969
970 if ctx.extension(SPLUNK_PK_METADATA_ORDER_KEY).is_some() {
972 reorder_splunk_primary_keys(&mut create_table_expr.primary_keys);
973 }
974
975 info!("Table `{table_ref}` does not exist, try creating table");
976 create_table_expr.table_options.extend(table_options);
977 Ok(create_table_expr)
978 }
979
980 fn get_alter_table_expr_on_demand(
988 &self,
989 req: &mut RowInsertRequest,
990 table: &TableRef,
991 ctx: &QueryContextRef,
992 accommodate_existing_schema: bool,
993 is_single_value: bool,
994 alter_existing: bool,
995 ) -> Result<Option<AlterTableExpr>> {
996 if !alter_existing {
997 return Ok(None);
998 }
999
1000 let catalog_name = ctx.current_catalog();
1001 let schema_name = ctx.current_schema();
1002 let table_name = table.table_info().name.clone();
1003
1004 if is_ddl_reserved_table(&schema_name, &table_name) {
1007 return Ok(None);
1008 }
1009
1010 let request_schema = req.rows.as_ref().unwrap().schema.as_slice();
1011 let request_field_count = request_schema
1012 .iter()
1013 .filter(|col| col.semantic_type == SemanticType::Field as i32)
1014 .count();
1015 let column_exprs = ColumnExpr::from_column_schemas(request_schema);
1016 let add_columns = expr_helper::extract_add_columns_expr(&table.schema(), column_exprs)?;
1017 let Some(mut add_columns) = add_columns else {
1018 return Ok(None);
1019 };
1020
1021 if is_single_value {
1022 let request_is_native_histogram = request_is_native_histogram(request_schema);
1023 let table_is_native_histogram = table_is_native_histogram(table);
1024 ensure!(
1025 request_is_native_histogram == table_is_native_histogram,
1026 InvalidInsertRequestSnafu {
1027 reason: format!(
1028 "Table `{table_name}` cannot mix native histogram and float sample fields"
1029 ),
1030 }
1031 );
1032 }
1033
1034 if accommodate_existing_schema {
1036 let table_schema = table.schema();
1037 let ts_col_name = table_schema.timestamp_column().map(|c| c.name.clone());
1039 let mut field_col_name = None;
1041 if is_single_value && request_field_count <= 1 {
1042 let mut multiple_field_cols = false;
1043 table.field_columns().for_each(|col| {
1044 if field_col_name.is_none() {
1045 field_col_name = Some(col.name.clone());
1046 } else {
1047 multiple_field_cols = true;
1048 }
1049 });
1050 if multiple_field_cols {
1051 field_col_name = None;
1052 }
1053 }
1054
1055 if let Some(rows) = req.rows.as_mut() {
1057 for col in &mut rows.schema {
1058 match col.semantic_type {
1059 x if x == SemanticType::Timestamp as i32 => {
1060 if let Some(ref ts_name) = ts_col_name
1061 && col.column_name != *ts_name
1062 {
1063 col.column_name = ts_name.clone();
1064 }
1065 }
1066 x if x == SemanticType::Field as i32 => {
1067 if let Some(ref field_name) = field_col_name
1068 && col.column_name != *field_name
1069 {
1070 col.column_name = field_name.clone();
1071 }
1072 }
1073 _ => {}
1074 }
1075 }
1076 }
1077
1078 add_columns.add_columns.retain(|col| {
1080 let def = col.column_def.as_ref().unwrap();
1081 def.semantic_type == SemanticType::Tag as i32
1082 || (def.semantic_type == SemanticType::Field as i32 && field_col_name.is_none())
1083 });
1084
1085 if add_columns.add_columns.is_empty() {
1086 return Ok(None);
1087 }
1088 }
1089
1090 Ok(Some(AlterTableExpr {
1091 catalog_name: catalog_name.to_string(),
1092 schema_name: schema_name.clone(),
1093 table_name: table_name.clone(),
1094 kind: Some(Kind::AddColumns(add_columns)),
1095 }))
1096 }
1097
1098 async fn create_physical_table(
1100 &self,
1101 mut create_table_expr: CreateTableExpr,
1102 partitions: Option<Partitions>,
1103 ctx: &QueryContextRef,
1104 statement_executor: &StatementExecutor,
1105 ) -> Result<TableRef> {
1106 {
1107 let table_ref = TableReference::full(
1108 &create_table_expr.catalog_name,
1109 &create_table_expr.schema_name,
1110 &create_table_expr.table_name,
1111 );
1112
1113 info!("Table `{table_ref}` does not exist, try creating table");
1114 }
1115 let res = statement_executor
1116 .create_table_inner(
1117 &mut create_table_expr,
1118 partitions,
1119 ctx.clone(),
1120 TriggerReason::AutoCreate,
1121 )
1122 .await;
1123
1124 let table_ref = TableReference::full(
1125 &create_table_expr.catalog_name,
1126 &create_table_expr.schema_name,
1127 &create_table_expr.table_name,
1128 );
1129
1130 match res {
1131 Ok(table) => {
1132 info!(
1133 "Successfully created table {} with options: {:?}",
1134 table_ref, create_table_expr.table_options,
1135 );
1136 Ok(table)
1137 }
1138 Err(err) => {
1139 error!(err; "Failed to create table {}", table_ref);
1140 Err(err)
1141 }
1142 }
1143 }
1144
1145 async fn create_logical_tables(
1146 &self,
1147 create_table_exprs: Vec<CreateTableExpr>,
1148 ctx: &QueryContextRef,
1149 statement_executor: &StatementExecutor,
1150 ) -> Result<Vec<TableRef>> {
1151 let res = statement_executor
1152 .create_logical_tables(&create_table_exprs, ctx.clone(), TriggerReason::AutoCreate)
1153 .await;
1154
1155 match res {
1156 Ok(res) => {
1157 info!("Successfully created logical tables");
1158 Ok(res)
1159 }
1160 Err(err) => {
1161 let failed_tables = create_table_exprs
1162 .into_iter()
1163 .map(|expr| {
1164 format!(
1165 "{}.{}.{}",
1166 expr.catalog_name, expr.schema_name, expr.table_name
1167 )
1168 })
1169 .collect::<Vec<_>>();
1170 error!(
1171 err;
1172 "Failed to create logical tables {:?}",
1173 failed_tables
1174 );
1175 Err(err)
1176 }
1177 }
1178 }
1179
1180 pub fn node_manager(&self) -> &NodeManagerRef {
1181 &self.node_manager
1182 }
1183
1184 pub fn partition_manager(&self) -> &PartitionRuleManagerRef {
1185 &self.partition_manager
1186 }
1187
1188 pub fn table_flownode_set_cache(&self) -> &TableFlownodeSetCacheRef {
1189 &self.table_flownode_set_cache
1190 }
1191}
1192
1193fn request_is_native_histogram(request_schema: &[ColumnSchema]) -> bool {
1194 let mut fields = request_schema
1195 .iter()
1196 .filter(|col| col.semantic_type == SemanticType::Field as i32);
1197 let Some(col) = fields.next() else {
1198 return false;
1199 };
1200
1201 fields.next().is_none()
1202 && api::helper::is_column_type_value_eq(
1203 col.datatype,
1204 col.datatype_extension.clone(),
1205 native_histogram_value_type(),
1206 )
1207}
1208
1209fn table_is_native_histogram(table: &TableRef) -> bool {
1210 let mut fields = table.field_columns();
1211 let Some(col) = fields.next() else {
1212 return false;
1213 };
1214
1215 fields.next().is_none() && is_native_histogram_value_type(&col.data_type)
1216}
1217
1218fn validate_column_count_match(requests: &RowInsertRequests) -> Result<()> {
1219 for request in &requests.inserts {
1220 let rows = request.rows.as_ref().unwrap();
1221 let column_count = rows.schema.len();
1222 rows.rows.iter().try_for_each(|r| {
1223 ensure!(
1224 r.values.len() == column_count,
1225 InvalidInsertRequestSnafu {
1226 reason: format!(
1227 "column count mismatch, columns: {}, values: {}",
1228 column_count,
1229 r.values.len()
1230 )
1231 }
1232 );
1233 Ok(())
1234 })?;
1235 }
1236 Ok(())
1237}
1238
1239pub fn fill_table_options_for_create(
1241 table_options: &mut std::collections::HashMap<String, String>,
1242 create_type: &AutoCreateTableType,
1243 ctx: &QueryContextRef,
1244) {
1245 for key in VALID_TABLE_OPTION_KEYS {
1246 if let Some(value) = ctx.extension(key) {
1247 table_options.insert(key.to_string(), value.to_string());
1248 }
1249 }
1250
1251 for (key, value) in ctx.extensions() {
1253 if is_semantic_option_key(&key) && validate_semantic_option(&key, &value) {
1254 table_options.insert(key, value);
1255 }
1256 }
1257
1258 match create_type {
1259 AutoCreateTableType::Logical(physical_table) => {
1260 table_options.insert(
1261 LOGICAL_TABLE_METADATA_KEY.to_string(),
1262 physical_table.clone(),
1263 );
1264 }
1265 AutoCreateTableType::Physical => {
1266 if let Some(append_mode) = ctx.extension(APPEND_MODE_KEY) {
1267 table_options.insert(APPEND_MODE_KEY.to_string(), append_mode.to_string());
1268 }
1269 if let Some(merge_mode) = ctx.extension(MERGE_MODE_KEY) {
1270 table_options.insert(MERGE_MODE_KEY.to_string(), merge_mode.to_string());
1271 }
1272 if let Some(time_window) = ctx.extension(TWCS_TIME_WINDOW) {
1273 table_options.insert(TWCS_TIME_WINDOW.to_string(), time_window.to_string());
1274 table_options.insert(
1276 COMPACTION_TYPE.to_string(),
1277 COMPACTION_TYPE_TWCS.to_string(),
1278 );
1279 }
1280 }
1281 AutoCreateTableType::Log => {
1284 table_options.insert(APPEND_MODE_KEY.to_string(), "true".to_string());
1285 }
1286 AutoCreateTableType::LastNonNull => {
1287 if ctx
1288 .extension(APPEND_MODE_KEY)
1289 .is_some_and(|value| value.eq_ignore_ascii_case("true"))
1290 {
1291 table_options.insert(APPEND_MODE_KEY.to_string(), "true".to_string());
1292 table_options.insert(MERGE_MODE_KEY.to_string(), "last_row".to_string());
1293 } else if let Some(merge_mode) = ctx.extension(MERGE_MODE_KEY) {
1294 table_options.insert(MERGE_MODE_KEY.to_string(), merge_mode.to_string());
1295 } else {
1296 table_options.insert(MERGE_MODE_KEY.to_string(), "last_non_null".to_string());
1297 }
1298 }
1299 AutoCreateTableType::Trace { .. } => {
1300 table_options.insert(APPEND_MODE_KEY.to_string(), "true".to_string());
1301 }
1302 }
1303}
1304
1305pub type PerTableSemanticIndex = BTreeMap<String, BTreeMap<String, BTreeMap<String, String>>>;
1311
1312pub fn parse_per_table_semantic_index(ctx: &QueryContextRef) -> Option<PerTableSemanticIndex> {
1317 let raw = ctx.extension(SEMANTIC_PER_TABLE_INDEX_KEY)?;
1318 match serde_json::from_str(raw) {
1319 Ok(index) => Some(index),
1320 Err(_) => {
1321 warn!("failed to parse semantic per-table index, skipping per-table options");
1322 None
1323 }
1324 }
1325}
1326
1327pub fn apply_per_table_semantic_options(
1335 table_options: &mut std::collections::HashMap<String, String>,
1336 index: Option<&PerTableSemanticIndex>,
1337 schema: &str,
1338 table_name: &str,
1339) {
1340 let Some(entry) = index
1341 .and_then(|index| index.get(schema))
1342 .and_then(|tables| tables.get(table_name))
1343 else {
1344 return;
1345 };
1346 for (key, value) in entry {
1347 if is_semantic_option_key(key) && validate_semantic_option(key, value) {
1348 table_options.insert(key.clone(), value.clone());
1349 }
1350 }
1351}
1352
1353pub fn build_create_table_expr(
1354 table: &TableReference,
1355 request_schema: &[ColumnSchema],
1356 engine: &str,
1357) -> Result<CreateTableExpr> {
1358 expr_helper::create_table_expr_by_column_schemas(table, request_schema, engine, None)
1359}
1360
1361pub const SPLUNK_PK_METADATA_ORDER_KEY: &str = "splunk_pk_metadata_order";
1365
1366fn reorder_splunk_primary_keys(primary_keys: &mut [String]) {
1369 const LEAD: [&str; 3] = ["host", "source", "sourcetype"];
1370 primary_keys.sort_by_key(|name| {
1373 LEAD.iter()
1374 .position(|&lead| lead == name.as_str())
1375 .unwrap_or(LEAD.len())
1376 });
1377}
1378
1379struct CreateAlterTableResult {
1381 instant_table_ids: HashSet<TableId>,
1383 table_infos: HashMap<TableId, Arc<TableInfo>>,
1385}
1386
1387struct FlowMirrorTask {
1388 requests: HashMap<Peer, RegionInsertRequests>,
1389 num_rows: usize,
1390}
1391
1392impl FlowMirrorTask {
1393 async fn new(
1394 cache: &TableFlownodeSetCacheRef,
1395 requests: impl Iterator<Item = &RegionInsertRequest>,
1396 ) -> Result<Self> {
1397 let mut src_table_reqs: HashMap<TableId, Option<(Vec<Peer>, RegionInsertRequests)>> =
1398 HashMap::new();
1399 let mut num_rows = 0;
1400
1401 for req in requests {
1402 let table_id = RegionId::from_u64(req.region_id).table_id();
1403 match src_table_reqs.get_mut(&table_id) {
1404 Some(Some((_peers, reqs))) => reqs.requests.push(req.clone()),
1405 Some(None) => continue,
1407 _ => {
1408 let peers = cache
1410 .get(table_id)
1411 .await
1412 .context(RequestInsertsSnafu)?
1413 .unwrap_or_default()
1414 .values()
1415 .cloned()
1416 .collect::<HashSet<_>>()
1417 .into_iter()
1418 .collect::<Vec<_>>();
1419
1420 if !peers.is_empty() {
1421 let mut reqs = RegionInsertRequests::default();
1422 reqs.requests.push(req.clone());
1423 num_rows += reqs
1424 .requests
1425 .iter()
1426 .map(|r| r.rows.as_ref().unwrap().rows.len())
1427 .sum::<usize>();
1428 src_table_reqs.insert(table_id, Some((peers, reqs)));
1429 } else {
1430 src_table_reqs.insert(table_id, None);
1432 }
1433 }
1434 }
1435 }
1436
1437 let mut inserts: HashMap<Peer, RegionInsertRequests> = HashMap::new();
1438
1439 for (_table_id, (peers, reqs)) in src_table_reqs
1440 .into_iter()
1441 .filter_map(|(k, v)| v.map(|v| (k, v)))
1442 {
1443 if peers.len() == 1 {
1444 inserts
1446 .entry(peers[0].clone())
1447 .or_default()
1448 .requests
1449 .extend(reqs.requests);
1450 continue;
1451 } else {
1452 for flownode in peers {
1454 inserts
1455 .entry(flownode.clone())
1456 .or_default()
1457 .requests
1458 .extend(reqs.requests.clone());
1459 }
1460 }
1461 }
1462
1463 Ok(Self {
1464 requests: inserts,
1465 num_rows,
1466 })
1467 }
1468
1469 fn detach(self, node_manager: NodeManagerRef) -> Result<()> {
1470 crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.add(self.num_rows as i64);
1471 for (peer, inserts) in self.requests {
1472 let node_manager = node_manager.clone();
1473 common_runtime::spawn_global(async move {
1474 let result = node_manager
1475 .flownode(&peer)
1476 .await
1477 .handle_inserts(inserts)
1478 .await
1479 .context(RequestInsertsSnafu);
1480
1481 match result {
1482 Ok(resp) => {
1483 let affected_rows = resp.affected_rows;
1484 crate::metrics::DIST_MIRROR_ROW_COUNT.inc_by(affected_rows);
1485 crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.sub(affected_rows as _);
1486 }
1487 Err(err) => {
1488 error!(err; "Failed to insert data into flownode {}", peer);
1489 }
1490 }
1491 });
1492 }
1493
1494 Ok(())
1495 }
1496}
1497
1498#[cfg(test)]
1499mod tests {
1500 use std::sync::Arc;
1501
1502 use api::helper::ColumnDataTypeWrapper;
1503 use api::v1::helper::{field_column_schema, time_index_column_schema};
1504 use api::v1::{RowInsertRequest, Rows, Value};
1505 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
1506 use common_meta::cache::new_table_flownode_set_cache;
1507 use common_meta::ddl::test_util::datanode_handler::NaiveDatanodeHandler;
1508 use common_meta::test_util::MockDatanodeManager;
1509 use common_query::native_histogram::NATIVE_HISTOGRAM_FIELD;
1510 use common_query::prelude::{greptime_native_histogram, set_default_prefix};
1511 use datatypes::data_type::ConcreteDataType;
1512 use datatypes::schema::ColumnSchema;
1513 use moka::future::Cache;
1514 use session::context::QueryContext;
1515 use table::TableRef;
1516 use table::dist_table::DummyDataSource;
1517 use table::metadata::{TableInfoBuilder, TableMetaBuilder, TableType};
1518
1519 use super::*;
1520 use crate::tests::{create_partition_rule_manager, prepare_mocked_backend};
1521
1522 fn make_table_ref_with_schema(
1523 ts_name: &str,
1524 field_name: &str,
1525 field_type: ConcreteDataType,
1526 ) -> TableRef {
1527 let schema = datatypes::schema::SchemaBuilder::try_from_columns(vec![
1528 ColumnSchema::new(
1529 ts_name,
1530 ConcreteDataType::timestamp_millisecond_datatype(),
1531 false,
1532 )
1533 .with_time_index(true),
1534 ColumnSchema::new(field_name, field_type, true),
1535 ])
1536 .unwrap()
1537 .build()
1538 .unwrap();
1539 let meta = TableMetaBuilder::empty()
1540 .schema(Arc::new(schema))
1541 .primary_key_indices(vec![])
1542 .value_indices(vec![1])
1543 .engine("mito")
1544 .next_column_id(0)
1545 .options(Default::default())
1546 .created_on(Default::default())
1547 .build()
1548 .unwrap();
1549 let info = Arc::new(
1550 TableInfoBuilder::default()
1551 .table_id(1)
1552 .table_version(0)
1553 .name("test_table")
1554 .schema_name(DEFAULT_SCHEMA_NAME)
1555 .catalog_name(DEFAULT_CATALOG_NAME)
1556 .desc(None)
1557 .table_type(TableType::Base)
1558 .meta(meta)
1559 .build()
1560 .unwrap(),
1561 );
1562 Arc::new(table::Table::new(
1563 info,
1564 table::metadata::FilterPushDownType::Unsupported,
1565 Arc::new(DummyDataSource),
1566 ))
1567 }
1568
1569 #[tokio::test]
1570 async fn test_accommodate_existing_schema_and_reject_kind_changes() {
1571 let ts_name = "my_ts";
1572 let field_name = "my_field";
1573 let table =
1574 make_table_ref_with_schema(ts_name, field_name, ConcreteDataType::float64_datatype());
1575
1576 let mut req = RowInsertRequest {
1578 table_name: "test_table".to_string(),
1579 rows: Some(Rows {
1580 schema: vec![
1581 time_index_column_schema("ts_wrong", ColumnDataType::TimestampMillisecond),
1582 field_column_schema("field_wrong", ColumnDataType::Float64),
1583 ],
1584 rows: vec![api::v1::Row {
1585 values: vec![Value::default(), Value::default()],
1586 }],
1587 }),
1588 };
1589 let ctx = Arc::new(QueryContext::with(
1590 DEFAULT_CATALOG_NAME,
1591 DEFAULT_SCHEMA_NAME,
1592 ));
1593
1594 let kv_backend = prepare_mocked_backend().await;
1595 let inserter = Inserter::new(
1596 catalog::memory::MemoryCatalogManager::new(),
1597 create_partition_rule_manager(kv_backend.clone()).await,
1598 Arc::new(MockDatanodeManager::new(NaiveDatanodeHandler)),
1599 Arc::new(new_table_flownode_set_cache(
1600 String::new(),
1601 Cache::new(100),
1602 kv_backend.clone(),
1603 )),
1604 true,
1605 );
1606 assert!(
1608 inserter
1609 .get_alter_table_expr_on_demand(&mut req, &table, &ctx, false, false, false)
1610 .unwrap()
1611 .is_none()
1612 );
1613 let alter_expr = inserter
1614 .get_alter_table_expr_on_demand(&mut req, &table, &ctx, true, true, true)
1615 .unwrap();
1616 assert!(alter_expr.is_none());
1617
1618 let req_schema = req.rows.as_ref().unwrap().schema.clone();
1620 assert_eq!(req_schema[0].column_name, ts_name);
1621 assert_eq!(req_schema[1].column_name, field_name);
1622
1623 let (datatype, datatype_extension) =
1624 ColumnDataTypeWrapper::try_from(native_histogram_value_type().clone())
1625 .unwrap()
1626 .into_parts();
1627 let mut histogram_req = RowInsertRequest {
1628 table_name: "test_table".to_string(),
1629 rows: Some(Rows {
1630 schema: vec![
1631 time_index_column_schema("ts", ColumnDataType::TimestampMillisecond),
1632 api::v1::ColumnSchema {
1633 column_name: greptime_native_histogram().to_string(),
1634 datatype: datatype as i32,
1635 semantic_type: SemanticType::Field as i32,
1636 datatype_extension,
1637 options: None,
1638 },
1639 ],
1640 rows: vec![],
1641 }),
1642 };
1643 let error = inserter
1644 .get_alter_table_expr_on_demand(&mut histogram_req, &table, &ctx, false, true, true)
1645 .unwrap_err();
1646 assert!(
1647 error
1648 .to_string()
1649 .contains("cannot mix native histogram and float sample fields")
1650 );
1651
1652 let histogram_table = make_table_ref_with_schema(
1653 "ts",
1654 greptime_native_histogram(),
1655 native_histogram_value_type().clone(),
1656 );
1657 let mut sample_req = RowInsertRequest {
1658 table_name: "test_table".to_string(),
1659 rows: Some(Rows {
1660 schema: vec![
1661 time_index_column_schema("ts", ColumnDataType::TimestampMillisecond),
1662 field_column_schema(greptime_value(), ColumnDataType::Float64),
1663 ],
1664 rows: vec![],
1665 }),
1666 };
1667 let error = inserter
1668 .get_alter_table_expr_on_demand(
1669 &mut sample_req,
1670 &histogram_table,
1671 &ctx,
1672 false,
1673 true,
1674 true,
1675 )
1676 .unwrap_err();
1677 assert!(
1678 error
1679 .to_string()
1680 .contains("cannot mix native histogram and float sample fields")
1681 );
1682 }
1683
1684 #[test]
1685 fn test_native_histogram_detection_survives_prefix_change() {
1686 set_default_prefix(Some("custom")).unwrap();
1687 let table = make_table_ref_with_schema(
1688 "custom_timestamp",
1689 NATIVE_HISTOGRAM_FIELD,
1690 native_histogram_value_type().clone(),
1691 );
1692 let (datatype, datatype_extension) =
1693 ColumnDataTypeWrapper::try_from(native_histogram_value_type().clone())
1694 .unwrap()
1695 .into_parts();
1696 let request_schema = [api::v1::ColumnSchema {
1697 column_name: greptime_native_histogram().to_string(),
1698 datatype: datatype as i32,
1699 semantic_type: SemanticType::Field as i32,
1700 datatype_extension,
1701 options: None,
1702 }];
1703
1704 assert!(request_is_native_histogram(&request_schema));
1705 assert!(table_is_native_histogram(&table));
1706 }
1707
1708 #[test]
1709 fn test_last_non_null_create_options_preserve_default_without_append_mode() {
1710 let ctx = Arc::new(QueryContext::with(
1711 DEFAULT_CATALOG_NAME,
1712 DEFAULT_SCHEMA_NAME,
1713 ));
1714 let mut table_options = Default::default();
1715
1716 fill_table_options_for_create(&mut table_options, &AutoCreateTableType::LastNonNull, &ctx);
1717
1718 assert_eq!(
1719 Some("last_non_null"),
1720 table_options.get(MERGE_MODE_KEY).map(String::as_str)
1721 );
1722 assert!(!table_options.contains_key(APPEND_MODE_KEY));
1723 }
1724
1725 #[test]
1726 fn test_fill_table_options_copies_semantic_extensions() {
1727 use table::requests::{
1728 SEMANTIC_METRIC_TYPE, SEMANTIC_PER_TABLE_INDEX_KEY, SEMANTIC_SIGNAL_TYPE,
1729 SEMANTIC_SOURCE, SEMANTIC_SOURCE_VERSION, SIGNAL_TYPE_METRIC, SOURCE_OPENTELEMETRY,
1730 };
1731
1732 let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
1733 ctx.set_extension(SEMANTIC_SIGNAL_TYPE, SIGNAL_TYPE_METRIC);
1734 ctx.set_extension(SEMANTIC_SOURCE, SOURCE_OPENTELEMETRY);
1735 ctx.set_extension(SEMANTIC_SOURCE_VERSION, "2.0");
1736 ctx.set_extension(SEMANTIC_METRIC_TYPE, "bogus");
1737 ctx.set_extension(SEMANTIC_PER_TABLE_INDEX_KEY, "{}");
1739 let ctx = Arc::new(ctx);
1740 let mut table_options = Default::default();
1741
1742 fill_table_options_for_create(&mut table_options, &AutoCreateTableType::Physical, &ctx);
1743
1744 assert_eq!(
1745 Some(SIGNAL_TYPE_METRIC),
1746 table_options.get(SEMANTIC_SIGNAL_TYPE).map(String::as_str)
1747 );
1748 assert_eq!(
1749 Some(SOURCE_OPENTELEMETRY),
1750 table_options.get(SEMANTIC_SOURCE).map(String::as_str)
1751 );
1752 assert_eq!(
1753 Some("2.0"),
1754 table_options
1755 .get(SEMANTIC_SOURCE_VERSION)
1756 .map(String::as_str)
1757 );
1758 assert!(!table_options.contains_key(SEMANTIC_METRIC_TYPE));
1759 assert!(!table_options.contains_key(SEMANTIC_PER_TABLE_INDEX_KEY));
1760 }
1761
1762 #[test]
1763 fn test_apply_per_table_semantic_options() {
1764 use table::requests::{
1765 SEMANTIC_METRIC_TYPE, SEMANTIC_METRIC_UNIT, SEMANTIC_PER_TABLE_INDEX_KEY,
1766 };
1767
1768 let index = format!(
1769 r#"{{
1770 "{DEFAULT_SCHEMA_NAME}": {{
1771 "http_requests_total": {{
1772 "greptime.semantic.metric.type": "counter",
1773 "greptime.semantic.metric.unit": "By",
1774 "greptime.semantic.metric.type_BOGUS": "x"
1775 }},
1776 "other_table": {{
1777 "greptime.semantic.metric.type": "gauge"
1778 }}
1779 }},
1780 "other_schema": {{
1781 "http_requests_total": {{
1782 "greptime.semantic.metric.type": "gauge"
1783 }}
1784 }}
1785 }}"#
1786 );
1787 let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
1788 ctx.set_extension(SEMANTIC_PER_TABLE_INDEX_KEY, index);
1789 let ctx = Arc::new(ctx);
1790
1791 let index = parse_per_table_semantic_index(&ctx);
1792 assert!(index.is_some());
1793 let index = index.as_ref();
1794
1795 let mut table_options = std::collections::HashMap::new();
1796 apply_per_table_semantic_options(
1797 &mut table_options,
1798 index,
1799 DEFAULT_SCHEMA_NAME,
1800 "http_requests_total",
1801 );
1802 assert_eq!(
1804 table_options.get(SEMANTIC_METRIC_TYPE).map(String::as_str),
1805 Some("counter")
1806 );
1807 assert_eq!(
1808 table_options.get(SEMANTIC_METRIC_UNIT).map(String::as_str),
1809 Some("By")
1810 );
1811 assert!(!table_options.contains_key("greptime.semantic.metric.type_BOGUS"));
1814 assert_eq!(table_options.len(), 2);
1815
1816 let mut empty = std::collections::HashMap::new();
1817 apply_per_table_semantic_options(&mut empty, index, DEFAULT_SCHEMA_NAME, "not_in_index");
1818 assert!(empty.is_empty());
1819
1820 let mut opts = std::collections::HashMap::new();
1823 apply_per_table_semantic_options(
1824 &mut opts,
1825 index,
1826 "schema_without_entry",
1827 "http_requests_total",
1828 );
1829 assert!(opts.is_empty());
1830
1831 let bare = Arc::new(QueryContext::with(
1833 DEFAULT_CATALOG_NAME,
1834 DEFAULT_SCHEMA_NAME,
1835 ));
1836 assert!(parse_per_table_semantic_index(&bare).is_none());
1837 let mut opts = std::collections::HashMap::new();
1838 apply_per_table_semantic_options(
1839 &mut opts,
1840 None,
1841 DEFAULT_SCHEMA_NAME,
1842 "http_requests_total",
1843 );
1844 assert!(opts.is_empty());
1845 }
1846
1847 #[test]
1848 fn test_last_non_null_create_options_preserve_default_with_append_mode_false() {
1849 let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
1850 ctx.set_extension(APPEND_MODE_KEY, "false");
1851 let ctx = Arc::new(ctx);
1852 let mut table_options = Default::default();
1853
1854 fill_table_options_for_create(&mut table_options, &AutoCreateTableType::LastNonNull, &ctx);
1855
1856 assert!(!table_options.contains_key(APPEND_MODE_KEY));
1857 assert_eq!(
1858 Some("last_non_null"),
1859 table_options.get(MERGE_MODE_KEY).map(String::as_str)
1860 );
1861 }
1862
1863 #[test]
1864 fn test_last_non_null_create_options_use_configured_merge_mode() {
1865 let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
1866 ctx.set_extension(MERGE_MODE_KEY, "last_row");
1867 let ctx = Arc::new(ctx);
1868 let mut table_options = Default::default();
1869
1870 fill_table_options_for_create(&mut table_options, &AutoCreateTableType::LastNonNull, &ctx);
1871
1872 assert_eq!(
1873 Some("last_row"),
1874 table_options.get(MERGE_MODE_KEY).map(String::as_str)
1875 );
1876 assert!(!table_options.contains_key(APPEND_MODE_KEY));
1877 }
1878
1879 #[test]
1880 fn test_last_non_null_create_options_use_last_row_with_append_mode_true() {
1881 let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
1882 ctx.set_extension(APPEND_MODE_KEY, "true");
1883 let ctx = Arc::new(ctx);
1884 let mut table_options = Default::default();
1885
1886 fill_table_options_for_create(&mut table_options, &AutoCreateTableType::LastNonNull, &ctx);
1887
1888 assert_eq!(
1889 Some("true"),
1890 table_options.get(APPEND_MODE_KEY).map(String::as_str)
1891 );
1892 assert_eq!(
1893 Some("last_row"),
1894 table_options.get(MERGE_MODE_KEY).map(String::as_str)
1895 );
1896 }
1897}