Skip to main content

operator/
insert.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::BTreeMap;
16use std::sync::Arc;
17
18use ahash::{HashMap, HashMapExt, HashSet, HashSetExt};
19use api::v1::alter_table_expr::Kind;
20use api::v1::column_def::options_from_skipping;
21use api::v1::region::{
22    InsertRequest as RegionInsertRequest, InsertRequests as RegionInsertRequests,
23    RegionRequestHeader,
24};
25use api::v1::{
26    AlterTableExpr, ColumnDataType, ColumnSchema, CreateTableExpr, InsertRequests,
27    RowInsertRequest, RowInsertRequests, Rows, SemanticType,
28};
29use catalog::CatalogManagerRef;
30use client::{OutputData, OutputMeta};
31use common_catalog::consts::{
32    DEFAULT_PRIVATE_SCHEMA_NAME, PARENT_SPAN_ID_COLUMN, SERVICE_NAME_COLUMN, TRACE_ID_COLUMN,
33    TRACE_TABLE_NAME, TRACE_TABLE_NAME_SESSION_KEY, default_engine, is_ddl_reserved_table,
34    trace_operations_table_name, trace_services_table_name,
35};
36use common_event_recorder::DEFAULT_EVENTS_TABLE_NAME;
37use common_frontend::slow_query_event::SLOW_QUERY_TABLE_NAME;
38use common_grpc_expr::util::ColumnExpr;
39use common_meta::cache::TableFlownodeSetCacheRef;
40use common_meta::datanode::REGION_STATS_HISTORY_TABLE_NAME;
41use common_meta::node_manager::{AffectedRows, NodeManagerRef};
42use common_meta::peer::Peer;
43use common_meta::rpc::ddl::TriggerReason;
44use common_query::Output;
45use common_query::native_histogram::{is_native_histogram_value_type, native_histogram_value_type};
46use common_query::prelude::{greptime_timestamp, greptime_value};
47use common_telemetry::tracing_context::TracingContext;
48use common_telemetry::{debug, error, warn};
49use datatypes::schema::SkippingIndexOptions;
50use futures_util::future;
51use meter_macros::write_meter;
52use partition::manager::PartitionRuleManagerRef;
53use session::context::QueryContextRef;
54use snafu::ResultExt;
55use snafu::prelude::*;
56use sql::partition::partition_rule_for_hexstring;
57use sql::statements::create::Partitions;
58use sql::statements::insert::Insert;
59use store_api::metric_engine_consts::{
60    LOGICAL_TABLE_METADATA_KEY, METRIC_ENGINE_NAME, PHYSICAL_TABLE_METADATA_KEY,
61};
62use store_api::mito_engine_options::{
63    APPEND_MODE_KEY, COMPACTION_TYPE, COMPACTION_TYPE_TWCS, MERGE_MODE_KEY, TTL_KEY,
64    TWCS_TIME_WINDOW,
65};
66use store_api::storage::{RegionId, TableId};
67use table::TableRef;
68use table::metadata::{TableInfo, TableInfoRef};
69use table::requests::{
70    AUTO_CREATE_TABLE_KEY, InsertRequest as TableInsertRequest, SEMANTIC_PER_TABLE_INDEX_KEY,
71    SEMANTIC_PIPELINE, TABLE_DATA_MODEL, TABLE_DATA_MODEL_TRACE_V1, TABLE_DATA_MODEL_TRACE_V2,
72    TRACE_TABLE_PARTITIONS_HINT_KEY, VALID_TABLE_OPTION_KEYS, is_semantic_option_key,
73    validate_semantic_option,
74};
75use table::table_reference::TableReference;
76
77use crate::batcher::PendingRowsBatcher;
78use crate::error::{
79    CatalogSnafu, ColumnOptionsSnafu, CreatePartitionRulesSnafu, FindRegionLeaderSnafu,
80    InvalidInsertRequestSnafu, JoinTaskSnafu, RequestInsertsSnafu, Result, TableNotFoundSnafu,
81};
82use crate::expr_helper;
83use crate::region_req_factory::RegionRequestFactory;
84use crate::req_convert::common::preprocess_row_insert_requests;
85use crate::req_convert::insert::{
86    ColumnToRow, ImpureDefaultFiller, RowToRegion, StatementToRegion, TableToRegion,
87    fill_reqs_with_impure_default, rows_to_record_batch,
88};
89use crate::statement::StatementExecutor;
90
91pub struct Inserter {
92    catalog_manager: CatalogManagerRef,
93    pub(crate) partition_manager: PartitionRuleManagerRef,
94    pub(crate) node_manager: NodeManagerRef,
95    pub(crate) table_flownode_set_cache: TableFlownodeSetCacheRef,
96    /// Server-side upper bound for auto table creation on write.
97    /// When `false`, missing tables are never auto-created regardless of the
98    /// per-request `auto_create_table` hint. When `true`, the hint still applies.
99    auto_create_table: bool,
100    pending_rows_batcher: Option<Arc<dyn PendingRowsBatcher>>,
101}
102
103pub type InserterRef = Arc<Inserter>;
104
105/// Hint for the table type to create automatically.
106#[derive(Clone)]
107pub enum AutoCreateTableType {
108    /// A logical table with the physical table name.
109    Logical(String),
110    /// A physical table.
111    Physical,
112    /// A log table which is append-only.
113    Log,
114    /// A table that merges rows by `last_non_null` strategy.
115    LastNonNull,
116    /// Create table that build index and default partition rules on trace_id
117    Trace { alter_existing: bool },
118}
119
120impl AutoCreateTableType {
121    pub fn as_str(&self) -> &'static str {
122        match self {
123            AutoCreateTableType::Logical(_) => "logical",
124            AutoCreateTableType::Physical => "physical",
125            AutoCreateTableType::Log => "log",
126            AutoCreateTableType::LastNonNull => "last_non_null",
127            AutoCreateTableType::Trace { .. } => "trace",
128        }
129    }
130
131    fn alter_existing(&self) -> bool {
132        !matches!(
133            self,
134            Self::Trace {
135                alter_existing: false
136            }
137        )
138    }
139}
140
141/// Split insert requests into normal and instant requests.
142///
143/// Where instant requests are requests with ttl=instant,
144/// and normal requests are requests with ttl set to other values.
145///
146/// This is used to split requests for different processing.
147#[derive(Clone)]
148pub struct InstantAndNormalInsertRequests {
149    /// Requests with normal ttl.
150    pub normal_requests: RegionInsertRequests,
151    /// Requests with ttl=instant.
152    /// Will be discarded immediately at frontend, wouldn't even insert into memtable, and only sent to flow node if needed.
153    pub instant_requests: RegionInsertRequests,
154}
155
156impl Inserter {
157    pub fn new(
158        catalog_manager: CatalogManagerRef,
159        partition_manager: PartitionRuleManagerRef,
160        node_manager: NodeManagerRef,
161        table_flownode_set_cache: TableFlownodeSetCacheRef,
162        auto_create_table: bool,
163    ) -> Self {
164        Self {
165            catalog_manager,
166            partition_manager,
167            node_manager,
168            table_flownode_set_cache,
169            auto_create_table,
170            pending_rows_batcher: None,
171        }
172    }
173
174    /// Installs the shared batcher; callers explicitly select its ingestion entry point.
175    pub fn with_pending_rows_batcher(
176        mut self,
177        batcher: Option<Arc<dyn PendingRowsBatcher>>,
178    ) -> Self {
179        self.pending_rows_batcher = batcher;
180        self
181    }
182
183    pub async fn handle_column_inserts(
184        &self,
185        requests: InsertRequests,
186        ctx: QueryContextRef,
187        statement_executor: &StatementExecutor,
188    ) -> Result<Output> {
189        let row_inserts = ColumnToRow::convert(requests)?;
190        self.handle_row_inserts(row_inserts, ctx, statement_executor, false, false)
191            .await
192    }
193
194    /// Handles row inserts request and creates a physical table on demand.
195    pub async fn handle_row_inserts(
196        &self,
197        mut requests: RowInsertRequests,
198        ctx: QueryContextRef,
199        statement_executor: &StatementExecutor,
200        accommodate_existing_schema: bool,
201        is_single_value: bool,
202    ) -> Result<Output> {
203        preprocess_row_insert_requests(&mut requests.inserts)?;
204        self.handle_row_inserts_with_create_type(
205            requests,
206            ctx,
207            statement_executor,
208            AutoCreateTableType::Physical,
209            accommodate_existing_schema,
210            is_single_value,
211        )
212        .await
213    }
214
215    /// Handles row inserts request and creates a log table on demand.
216    pub async fn handle_log_inserts(
217        &self,
218        requests: RowInsertRequests,
219        ctx: QueryContextRef,
220        statement_executor: &StatementExecutor,
221    ) -> Result<Output> {
222        self.handle_row_inserts_with_create_type(
223            requests,
224            ctx,
225            statement_executor,
226            AutoCreateTableType::Log,
227            false,
228            false,
229        )
230        .await
231    }
232
233    pub async fn handle_trace_inserts(
234        &self,
235        requests: RowInsertRequests,
236        ctx: QueryContextRef,
237        statement_executor: &StatementExecutor,
238    ) -> Result<Output> {
239        self.handle_row_inserts_with_create_type(
240            requests,
241            ctx,
242            statement_executor,
243            AutoCreateTableType::Trace {
244                alter_existing: true,
245            },
246            false,
247            false,
248        )
249        .await
250    }
251
252    /// Handles row inserts request and creates a table with `last_non_null` merge mode on demand.
253    pub async fn handle_last_non_null_inserts(
254        &self,
255        requests: RowInsertRequests,
256        ctx: QueryContextRef,
257        statement_executor: &StatementExecutor,
258        accommodate_existing_schema: bool,
259        is_single_value: bool,
260    ) -> Result<Output> {
261        self.handle_row_inserts_with_create_type(
262            requests,
263            ctx,
264            statement_executor,
265            AutoCreateTableType::LastNonNull,
266            accommodate_existing_schema,
267            is_single_value,
268        )
269        .await
270    }
271
272    /// Handles row inserts request with specified [AutoCreateTableType].
273    async fn handle_row_inserts_with_create_type(
274        &self,
275        mut requests: RowInsertRequests,
276        ctx: QueryContextRef,
277        statement_executor: &StatementExecutor,
278        create_type: AutoCreateTableType,
279        accommodate_existing_schema: bool,
280        is_single_value: bool,
281    ) -> Result<Output> {
282        let skip_wal = ctx.skip_wal();
283
284        let batcher = self
285            .pending_rows_batcher
286            .as_ref()
287            .filter(|_| ctx.batching_enabled());
288
289        // remove empty requests
290        requests.inserts.retain(|req| {
291            req.rows
292                .as_ref()
293                .map(|r| !r.rows.is_empty())
294                .unwrap_or_default()
295        });
296        validate_column_count_match(&requests)?;
297
298        let CreateAlterTableResult {
299            instant_table_ids,
300            table_infos,
301        } = self
302            .create_or_alter_tables_on_demand(
303                &mut requests,
304                &ctx,
305                create_type,
306                statement_executor,
307                accommodate_existing_schema,
308                is_single_value,
309            )
310            .await?;
311
312        // Instant tables have no persisted data for dirty-window Flow to read.
313        // Metric tables keep their existing dedicated ingestion path.
314        if let Some(batcher) = batcher
315            && instant_table_ids.is_empty()
316            && table_infos
317                .values()
318                .all(|info| info.meta.engine == default_engine())
319        {
320            return self
321                .submit_pending_rows(requests, table_infos, ctx, batcher)
322                .await;
323        }
324
325        let name_to_info = table_infos
326            .values()
327            .map(|info| (info.name.clone(), info.clone()))
328            .collect::<HashMap<_, _>>();
329        let inserts = RowToRegion::new(
330            name_to_info,
331            instant_table_ids,
332            self.partition_manager.as_ref(),
333        )
334        .convert(requests, skip_wal)
335        .await?;
336
337        self.do_request(inserts, &table_infos, &ctx).await
338    }
339
340    async fn submit_pending_rows(
341        &self,
342        mut requests: RowInsertRequests,
343        table_infos: HashMap<TableId, Arc<TableInfo>>,
344        ctx: QueryContextRef,
345        batcher: &Arc<dyn PendingRowsBatcher>,
346    ) -> Result<Output> {
347        // All entry points, including single-table and SQL writes, skip empty input
348        // before evaluating defaults or converting prepared rows.
349        requests.inserts.retain(|request| {
350            request
351                .rows
352                .as_ref()
353                .is_some_and(|rows| !rows.rows.is_empty())
354        });
355        let by_name = table_infos
356            .values()
357            .map(|info| (info.name.as_str(), info))
358            .collect::<HashMap<_, _>>();
359        let mut prepared = Vec::with_capacity(requests.inserts.len());
360        for request in &mut requests.inserts {
361            let table_info =
362                by_name
363                    .get(request.table_name.as_str())
364                    .context(TableNotFoundSnafu {
365                        table_name: &request.table_name,
366                    })?;
367            let Some(rows) = &mut request.rows else {
368                continue;
369            };
370            ImpureDefaultFiller::new((*table_info).clone())?.fill_rows(rows);
371            let batch = rows_to_record_batch(rows, table_info)?;
372            prepared.push(((*table_info).clone(), batch));
373        }
374        // Preserve the existing meter input and original request boundary. These
375        // envelopes are only for accounting; routing happens after batching.
376        let metered = InstantAndNormalInsertRequests {
377            normal_requests: RegionInsertRequests {
378                requests: requests
379                    .inserts
380                    .into_iter()
381                    .map(|request| RegionInsertRequest {
382                        rows: request.rows,
383                        ..Default::default()
384                    })
385                    .collect(),
386            },
387            instant_requests: RegionInsertRequests::default(),
388        };
389        let write_cost = write_meter!(
390            ctx.current_catalog(),
391            ctx.current_schema(),
392            metered,
393            ctx.channel() as u8
394        );
395        prepared.retain(|(_, batch)| batch.num_rows() != 0);
396        let results = if prepared.is_empty() {
397            Vec::new()
398        } else {
399            // One original request shares admission across all table submissions.
400            let permit = batcher.acquire().await?;
401            let submissions = prepared.into_iter().map(|(info, batch)| {
402                // Routing uses the target database; metering above retains the
403                // original request context, including fully qualified SQL writes.
404                let mut target_ctx = ctx.fork();
405                target_ctx.set_current_catalog(&info.catalog_name);
406                target_ctx.set_current_schema(&info.schema_name);
407                batcher.submit(info, batch, Arc::new(target_ctx), permit.clone())
408            });
409            // Observe every table completion even when another table fails.
410            future::join_all(submissions).await
411        };
412        let affected_rows = results.into_iter().sum::<Result<usize>>()?;
413        Ok(Output::new(
414            OutputData::AffectedRows(affected_rows),
415            OutputMeta::new_with_cost(write_cost as _),
416        ))
417    }
418
419    /// Handles row inserts request with metric engine.
420    pub async fn handle_metric_row_inserts(
421        &self,
422        mut requests: RowInsertRequests,
423        ctx: QueryContextRef,
424        statement_executor: &StatementExecutor,
425        physical_table: String,
426    ) -> Result<Output> {
427        let skip_wal = ctx.skip_wal();
428
429        // remove empty requests
430        requests.inserts.retain(|req| {
431            req.rows
432                .as_ref()
433                .map(|r| !r.rows.is_empty())
434                .unwrap_or_default()
435        });
436        validate_column_count_match(&requests)?;
437
438        // check and create physical table
439        self.create_physical_table_on_demand(&ctx, physical_table.clone(), statement_executor)
440            .await?;
441
442        // check and create logical tables
443        let CreateAlterTableResult {
444            instant_table_ids,
445            table_infos,
446        } = self
447            .create_or_alter_tables_on_demand(
448                &mut requests,
449                &ctx,
450                AutoCreateTableType::Logical(physical_table.clone()),
451                statement_executor,
452                true,
453                true,
454            )
455            .await?;
456        let name_to_info = table_infos
457            .values()
458            .map(|info| (info.name.clone(), info.clone()))
459            .collect::<HashMap<_, _>>();
460        let inserts = RowToRegion::new(name_to_info, instant_table_ids, &self.partition_manager)
461            .convert(requests, skip_wal)
462            .await?;
463
464        self.do_request(inserts, &table_infos, &ctx).await
465    }
466
467    fn table_batcher(
468        &self,
469        table_info: &TableInfoRef,
470        ctx: &QueryContextRef,
471    ) -> Option<&Arc<dyn PendingRowsBatcher>> {
472        self.pending_rows_batcher.as_ref().filter(|_| {
473            ctx.batching_enabled()
474                && !table_info.is_ttl_instant_table()
475                && table_info.meta.engine == default_engine()
476        })
477    }
478
479    async fn submit_table_rows(
480        &self,
481        rows: Rows,
482        table_info: TableInfoRef,
483        ctx: QueryContextRef,
484        batcher: &Arc<dyn PendingRowsBatcher>,
485    ) -> Result<Output> {
486        let requests = RowInsertRequests {
487            inserts: vec![RowInsertRequest {
488                table_name: table_info.name.clone(),
489                rows: Some(rows),
490            }],
491        };
492        let table_infos = HashMap::from_iter([(table_info.table_id(), table_info)]);
493        self.submit_pending_rows(requests, table_infos, ctx, batcher)
494            .await
495    }
496
497    pub async fn handle_table_insert(
498        &self,
499        request: TableInsertRequest,
500        ctx: QueryContextRef,
501    ) -> Result<Output> {
502        let catalog = request.catalog_name.as_str();
503        let schema = request.schema_name.as_str();
504        let table_name = request.table_name.as_str();
505        let table = self.get_table(catalog, schema, table_name).await?;
506        let table = table.with_context(|| TableNotFoundSnafu {
507            table_name: common_catalog::format_full_table_name(catalog, schema, table_name),
508        })?;
509        let table_info = table.table_info();
510
511        let converter = TableToRegion::new(&table_info, &self.partition_manager);
512        let skip_wal = request.skip_wal;
513        let rows = converter.prepare(request)?;
514        if let Some(batcher) = self.table_batcher(&table_info, &ctx) {
515            return self.submit_table_rows(rows, table_info, ctx, batcher).await;
516        }
517        let inserts = converter.partition(rows, skip_wal).await?;
518
519        let table_infos = HashMap::from_iter([(table_info.table_id(), table_info.clone())]);
520
521        self.do_request(inserts, &table_infos, &ctx).await
522    }
523
524    pub async fn handle_statement_insert(
525        &self,
526        insert: &Insert,
527        ctx: &QueryContextRef,
528    ) -> Result<Output> {
529        let converter =
530            StatementToRegion::new(self.catalog_manager.as_ref(), &self.partition_manager, ctx);
531        let (rows, table_info) = converter.prepare(insert, ctx).await?;
532        if let Some(batcher) = self.table_batcher(&table_info, ctx) {
533            return self
534                .submit_table_rows(rows, table_info, ctx.clone(), batcher)
535                .await;
536        }
537        let inserts = converter.partition(rows, table_info.clone(), ctx).await?;
538
539        let table_infos = HashMap::from_iter([(table_info.table_id(), table_info.clone())]);
540
541        self.do_request(inserts, &table_infos, ctx).await
542    }
543}
544
545impl Inserter {
546    async fn do_request(
547        &self,
548        requests: InstantAndNormalInsertRequests,
549        table_infos: &HashMap<TableId, Arc<TableInfo>>,
550        ctx: &QueryContextRef,
551    ) -> Result<Output> {
552        // Fill impure default values in the request
553        let requests = fill_reqs_with_impure_default(table_infos, requests)?;
554
555        let write_cost = write_meter!(
556            ctx.current_catalog(),
557            ctx.current_schema(),
558            requests,
559            ctx.channel() as u8
560        );
561        let request_factory = RegionRequestFactory::new(RegionRequestHeader {
562            tracing_context: TracingContext::from_current_span().to_w3c(),
563            dbname: ctx.get_db_string(),
564            ..Default::default()
565        });
566
567        let InstantAndNormalInsertRequests {
568            normal_requests,
569            instant_requests,
570        } = requests;
571
572        // Mirror requests for source table to flownode asynchronously
573        let flow_mirror_task = FlowMirrorTask::new(
574            &self.table_flownode_set_cache,
575            normal_requests
576                .requests
577                .iter()
578                .chain(instant_requests.requests.iter()),
579        )
580        .await?;
581        flow_mirror_task.detach(self.node_manager.clone())?;
582
583        // Write requests to datanode and wait for response
584        let write_tasks = self
585            .group_requests_by_peer(normal_requests)
586            .await?
587            .into_iter()
588            .map(|(peer, inserts)| {
589                let node_manager = self.node_manager.clone();
590                let request = request_factory.build_insert(inserts);
591                common_runtime::spawn_global(async move {
592                    node_manager
593                        .datanode(&peer)
594                        .await
595                        .handle(request)
596                        .await
597                        .context(RequestInsertsSnafu)
598                })
599            });
600        let results = future::try_join_all(write_tasks)
601            .await
602            .context(JoinTaskSnafu)?;
603        let affected_rows = results
604            .into_iter()
605            .map(|resp| resp.map(|r| r.affected_rows))
606            .sum::<Result<AffectedRows>>()?;
607        crate::metrics::DIST_INGEST_ROW_COUNT
608            .with_label_values(&[ctx.get_db_string().as_str()])
609            .inc_by(affected_rows as u64);
610        Ok(Output::new(
611            OutputData::AffectedRows(affected_rows),
612            OutputMeta::new_with_cost(write_cost as _),
613        ))
614    }
615
616    async fn group_requests_by_peer(
617        &self,
618        requests: RegionInsertRequests,
619    ) -> Result<HashMap<Peer, RegionInsertRequests>> {
620        // group by region ids first to reduce repeatedly call `find_region_leader`
621        // TODO(discord9): determine if a addition clone is worth it
622        let mut requests_per_region: HashMap<RegionId, RegionInsertRequests> = HashMap::new();
623        for req in requests.requests {
624            let region_id = RegionId::from_u64(req.region_id);
625            requests_per_region
626                .entry(region_id)
627                .or_default()
628                .requests
629                .push(req);
630        }
631
632        let mut inserts: HashMap<Peer, RegionInsertRequests> = HashMap::new();
633
634        for (region_id, reqs) in requests_per_region {
635            let peer = self
636                .partition_manager
637                .find_region_leader(region_id)
638                .await
639                .context(FindRegionLeaderSnafu)?;
640            inserts
641                .entry(peer)
642                .or_default()
643                .requests
644                .extend(reqs.requests);
645        }
646
647        Ok(inserts)
648    }
649
650    /// Returns `Some(reason)` if the config or request hint disables automatic
651    /// table creation. Exempt private system tables are handled by
652    /// [`Self::is_auto_create_exempt_private_table`].
653    fn auto_create_disabled_reason(&self, ctx: &QueryContextRef) -> Result<Option<&'static str>> {
654        let auto_create_table_hint = ctx
655            .extension(AUTO_CREATE_TABLE_KEY)
656            .map(|v| v.parse::<bool>())
657            .transpose()
658            .map_err(|_| {
659                InvalidInsertRequestSnafu {
660                    reason: "`auto_create_table` hint must be a boolean",
661                }
662                .build()
663            })?
664            .unwrap_or(true);
665        Ok(if !self.auto_create_table {
666            Some("auto-create table is disabled by frontend config")
667        } else if !auto_create_table_hint {
668            Some("`auto_create_table` hint is disabled")
669        } else {
670            None
671        })
672    }
673
674    /// Returns whether a private system table may infer and reconcile its schema
675    /// even when automatic table creation is disabled.
676    fn is_auto_create_exempt_private_table(schema: &str, table: &str) -> bool {
677        schema == DEFAULT_PRIVATE_SCHEMA_NAME
678            && matches!(
679                table,
680                DEFAULT_EVENTS_TABLE_NAME | SLOW_QUERY_TABLE_NAME | REGION_STATS_HISTORY_TABLE_NAME
681            )
682    }
683
684    /// Ensures a trace table has the request-global schema without requiring a
685    /// padded data row to drive on-demand creation or alteration. When
686    /// `alter_existing` is false, a table created after planning is left for the
687    /// caller to re-plan.
688    pub async fn ensure_trace_table_on_demand(
689        &self,
690        table_name: &str,
691        request_schema: Vec<ColumnSchema>,
692        alter_existing: bool,
693        ctx: &QueryContextRef,
694        statement_executor: &StatementExecutor,
695    ) -> Result<()> {
696        let mut requests = RowInsertRequests {
697            inserts: vec![RowInsertRequest {
698                table_name: table_name.to_string(),
699                rows: Some(api::v1::Rows {
700                    schema: request_schema,
701                    rows: Vec::new(),
702                }),
703            }],
704        };
705        self.create_or_alter_tables_on_demand(
706            &mut requests,
707            ctx,
708            AutoCreateTableType::Trace { alter_existing },
709            statement_executor,
710            false,
711            false,
712        )
713        .await?;
714        Ok(())
715    }
716
717    /// Creates or alter tables on demand:
718    /// - if table does not exist, create table by inferred CreateExpr
719    /// - if table exist, check if schema matches. If any new column found, alter table by inferred `AlterExpr`
720    ///
721    /// Returns a mapping from table name to table id, where table name is the table name involved in the requests.
722    /// This mapping is used in the conversion of RowToRegion.
723    ///
724    /// `accommodate_existing_schema` is used to determine if the existing schema should override the new schema.
725    /// It only works for TIME_INDEX and single VALUE columns. This is for the case where the user creates a table with
726    /// custom schema, and then inserts data with endpoints that have default schema setting, like prometheus
727    /// remote write. This will modify the `RowInsertRequests` in place.
728    /// `is_single_value` indicates whether the default schema only contains single value column so we can accommodate it.
729    async fn create_or_alter_tables_on_demand(
730        &self,
731        requests: &mut RowInsertRequests,
732        ctx: &QueryContextRef,
733        auto_create_table_type: AutoCreateTableType,
734        statement_executor: &StatementExecutor,
735        accommodate_existing_schema: bool,
736        is_single_value: bool,
737    ) -> Result<CreateAlterTableResult> {
738        let _timer = crate::metrics::CREATE_ALTER_ON_DEMAND
739            .with_label_values(&[auto_create_table_type.as_str()])
740            .start_timer();
741        let catalog = ctx.current_catalog();
742        let schema = ctx.current_schema();
743
744        let auto_create_disabled_reason = self.auto_create_disabled_reason(ctx)?;
745        // Enabled batches permit every table, so only disabled batches need a whitelist scan.
746        let has_auto_create_exempt_table = auto_create_disabled_reason.is_some()
747            && requests
748                .inserts
749                .iter()
750                .any(|req| Self::is_auto_create_exempt_private_table(&schema, &req.table_name));
751        let mut table_infos = HashMap::new();
752        // Without exempt tables, verify existing tables and reject missing ones without inferring schemas.
753        if let Some(disabled_reason) = auto_create_disabled_reason
754            && !has_auto_create_exempt_table
755        {
756            let mut instant_table_ids = HashSet::new();
757            for req in &requests.inserts {
758                let table = match self.get_table(catalog, &schema, &req.table_name).await? {
759                    Some(table) => table,
760                    // System-defined table: created canonically by the system,
761                    // so the auto-create config/hint does not apply.
762                    None if is_ddl_reserved_table(&schema, &req.table_name) => {
763                        statement_executor
764                            .create_declared_relationships_table(catalog, ctx.clone())
765                            .await?
766                    }
767                    None => {
768                        return InvalidInsertRequestSnafu {
769                            reason: format!(
770                                "Table `{}` does not exist, and {}",
771                                req.table_name, disabled_reason
772                            ),
773                        }
774                        .fail();
775                    }
776                };
777                let table_info = table.table_info();
778                if matches!(auto_create_table_type, AutoCreateTableType::Trace { .. }) {
779                    validate_trace_table_model(&table_info, ctx)?;
780                }
781                if table_info.is_ttl_instant_table() {
782                    instant_table_ids.insert(table_info.table_id());
783                }
784                table_infos.insert(table_info.table_id(), table.table_info());
785            }
786            let ret = CreateAlterTableResult {
787                instant_table_ids,
788                table_infos,
789            };
790            return Ok(ret);
791        }
792
793        let mut create_tables = vec![];
794        let mut alter_tables = vec![];
795        let mut need_refresh_table_infos = HashSet::new();
796        let mut instant_table_ids = HashSet::new();
797        let mut per_table_semantics: Option<Option<PerTableSemanticIndex>> = None;
798
799        for req in &mut requests.inserts {
800            // Mixed batches need a per-table decision so an exempt table cannot authorize others.
801            let auto_create_allowed = auto_create_disabled_reason.is_none()
802                || Self::is_auto_create_exempt_private_table(&schema, &req.table_name);
803            match self.get_table(catalog, &schema, &req.table_name).await? {
804                Some(table) => {
805                    let table_info = table.table_info();
806                    if matches!(auto_create_table_type, AutoCreateTableType::Trace { .. }) {
807                        validate_trace_table_model(&table_info, ctx)?;
808                    }
809                    if table_info.is_ttl_instant_table() {
810                        instant_table_ids.insert(table_info.table_id());
811                    }
812                    if auto_create_allowed
813                        && let Some(alter_expr) = self.get_alter_table_expr_on_demand(
814                            req,
815                            &table,
816                            ctx,
817                            accommodate_existing_schema,
818                            is_single_value,
819                            auto_create_table_type.alter_existing(),
820                        )?
821                    {
822                        alter_tables.push(alter_expr);
823                        need_refresh_table_infos.insert((
824                            catalog.to_string(),
825                            schema.clone(),
826                            req.table_name.clone(),
827                        ));
828                    } else {
829                        table_infos.insert(table_info.table_id(), table.table_info());
830                    }
831                }
832                // A DDL-reserved table's definition never derives from the
833                // write request; the system creates it canonically, below the
834                // user-DDL guard that rejects the generic create path.
835                None if is_ddl_reserved_table(&schema, &req.table_name) => {
836                    let table = statement_executor
837                        .create_declared_relationships_table(catalog, ctx.clone())
838                        .await?;
839                    let table_info = table.table_info();
840                    if table_info.is_ttl_instant_table() {
841                        instant_table_ids.insert(table_info.table_id());
842                    }
843                    table_infos.insert(table_info.table_id(), table_info);
844                }
845                None if !auto_create_allowed
846                    && let Some(disabled_reason) = auto_create_disabled_reason =>
847                {
848                    return InvalidInsertRequestSnafu {
849                        reason: format!(
850                            "Table `{}` does not exist, and {}",
851                            req.table_name, disabled_reason,
852                        ),
853                    }
854                    .fail();
855                }
856                None => {
857                    let semantic_index = per_table_semantics
858                        .get_or_insert_with(|| parse_per_table_semantic_index(ctx))
859                        .as_ref();
860                    let create_expr = self.get_create_table_expr_on_demand(
861                        req,
862                        &auto_create_table_type,
863                        ctx,
864                        semantic_index,
865                    )?;
866                    create_tables.push(create_expr);
867                }
868            }
869        }
870
871        match auto_create_table_type {
872            AutoCreateTableType::Logical(_) => {
873                if !create_tables.is_empty() {
874                    // Creates logical tables in batch.
875                    let tables = self
876                        .create_logical_tables(create_tables, ctx, statement_executor)
877                        .await?;
878
879                    for table in tables {
880                        let table_info = table.table_info();
881                        if table_info.is_ttl_instant_table() {
882                            instant_table_ids.insert(table_info.table_id());
883                        }
884                        table_infos.insert(table_info.table_id(), table.table_info());
885                    }
886                }
887                if !alter_tables.is_empty() {
888                    // Alter logical tables in batch.
889                    statement_executor
890                        .alter_logical_tables(alter_tables, ctx.clone(), TriggerReason::AutoAlter)
891                        .await?;
892                }
893            }
894            AutoCreateTableType::Physical
895            | AutoCreateTableType::Log
896            | AutoCreateTableType::LastNonNull => {
897                // note that auto create table shouldn't be ttl instant table
898                // for it's a very unexpected behavior and should be set by user explicitly
899                for create_table in create_tables {
900                    let table = self
901                        .create_physical_table(create_table, None, ctx, statement_executor)
902                        .await?;
903                    let table_info = table.table_info();
904                    if table_info.is_ttl_instant_table() {
905                        instant_table_ids.insert(table_info.table_id());
906                    }
907                    table_infos.insert(table_info.table_id(), table.table_info());
908                }
909                for alter_expr in alter_tables.into_iter() {
910                    statement_executor
911                        .alter_table_inner(alter_expr, ctx.clone(), TriggerReason::AutoAlter)
912                        .await?;
913                }
914            }
915
916            AutoCreateTableType::Trace { .. } => {
917                let trace_table_name = ctx
918                    .extension(TRACE_TABLE_NAME_SESSION_KEY)
919                    .unwrap_or(TRACE_TABLE_NAME);
920
921                let trace_table_partitions = if let Some(trace_table_partitions) =
922                    ctx.extension(TRACE_TABLE_PARTITIONS_HINT_KEY)
923                {
924                    let p = trace_table_partitions.parse::<u32>().map_err(|_| {
925                        InvalidInsertRequestSnafu {
926                            reason: format!(
927                                "Failed to parse trace_table_partitions: {}",
928                                trace_table_partitions
929                            ),
930                        }
931                        .build()
932                    })?;
933                    Some(p)
934                } else {
935                    None
936                };
937
938                // note that auto create table shouldn't be ttl instant table
939                // for it's a very unexpected behavior and should be set by user explicitly
940                for mut create_table in create_tables {
941                    if create_table.table_name == trace_services_table_name(trace_table_name)
942                        || create_table.table_name == trace_operations_table_name(trace_table_name)
943                    {
944                        // Disable append mode for auxiliary tables (services/operations) since they require upsert behavior.
945                        create_table
946                            .table_options
947                            .insert(APPEND_MODE_KEY.to_string(), "false".to_string());
948                        // Remove `ttl` key from table options if it exists
949                        create_table.table_options.remove(TTL_KEY);
950
951                        let table = self
952                            .create_physical_table(create_table, None, ctx, statement_executor)
953                            .await?;
954                        let table_info = table.table_info();
955                        if table_info.is_ttl_instant_table() {
956                            instant_table_ids.insert(table_info.table_id());
957                        }
958                        table_infos.insert(table_info.table_id(), table.table_info());
959                    } else {
960                        // prebuilt partition rules for uuid data: see the function
961                        // for more information
962                        let partitions = if matches!(trace_table_partitions, Some(0) | Some(1)) {
963                            // disable partitions
964                            None
965                        } else {
966                            let p = partition_rule_for_hexstring(
967                                TRACE_ID_COLUMN,
968                                trace_table_partitions,
969                            )
970                            .context(CreatePartitionRulesSnafu)?;
971                            Some(p)
972                        };
973
974                        // add skip index to
975                        // - trace_id: when searching by trace id
976                        // - parent_span_id: when searching root span
977                        // - span_name: when searching certain types of span
978                        let index_columns =
979                            [TRACE_ID_COLUMN, PARENT_SPAN_ID_COLUMN, SERVICE_NAME_COLUMN];
980                        for index_column in index_columns {
981                            if let Some(col) = create_table
982                                .column_defs
983                                .iter_mut()
984                                .find(|c| c.name == index_column)
985                            {
986                                col.options =
987                                    options_from_skipping(&SkippingIndexOptions::default())
988                                        .context(ColumnOptionsSnafu)?;
989                            } else {
990                                warn!(
991                                    "Column {} not found when creating index for trace table: {}.",
992                                    index_column, create_table.table_name
993                                );
994                            }
995                        }
996
997                        // use table_options to mark table model version
998                        create_table.table_options.insert(
999                            TABLE_DATA_MODEL.to_string(),
1000                            ctx.extension(SEMANTIC_PIPELINE)
1001                                .unwrap_or(TABLE_DATA_MODEL_TRACE_V1)
1002                                .to_string(),
1003                        );
1004
1005                        let table = self
1006                            .create_physical_table(
1007                                create_table,
1008                                partitions,
1009                                ctx,
1010                                statement_executor,
1011                            )
1012                            .await?;
1013                        let table_info = table.table_info();
1014                        if table_info.is_ttl_instant_table() {
1015                            instant_table_ids.insert(table_info.table_id());
1016                        }
1017                        table_infos.insert(table_info.table_id(), table.table_info());
1018                    }
1019                }
1020                for alter_expr in alter_tables.into_iter() {
1021                    statement_executor
1022                        .alter_table_inner(alter_expr, ctx.clone(), TriggerReason::AutoAlter)
1023                        .await?;
1024                }
1025            }
1026        }
1027
1028        // refresh table infos for altered tables
1029        for (catalog, schema, table_name) in need_refresh_table_infos {
1030            let table = self
1031                .get_table(&catalog, &schema, &table_name)
1032                .await?
1033                .context(TableNotFoundSnafu {
1034                    table_name: common_catalog::format_full_table_name(
1035                        &catalog,
1036                        &schema,
1037                        &table_name,
1038                    ),
1039                })?;
1040            let table_info = table.table_info();
1041            table_infos.insert(table_info.table_id(), table.table_info());
1042        }
1043
1044        Ok(CreateAlterTableResult {
1045            instant_table_ids,
1046            table_infos,
1047        })
1048    }
1049
1050    async fn create_physical_table_on_demand(
1051        &self,
1052        ctx: &QueryContextRef,
1053        physical_table: String,
1054        statement_executor: &StatementExecutor,
1055    ) -> Result<()> {
1056        let catalog_name = ctx.current_catalog();
1057        let schema_name = ctx.current_schema();
1058
1059        // check if exist
1060        if self
1061            .get_table(catalog_name, &schema_name, &physical_table)
1062            .await?
1063            .is_some()
1064        {
1065            return Ok(());
1066        }
1067
1068        // Gate here too, otherwise a disabled switch would still leak the physical table.
1069        if let Some(disabled_reason) = self.auto_create_disabled_reason(ctx)? {
1070            return InvalidInsertRequestSnafu {
1071                reason: format!(
1072                    "Physical table `{physical_table}` does not exist, and {disabled_reason}"
1073                ),
1074            }
1075            .fail();
1076        }
1077
1078        let table_reference = TableReference::full(catalog_name, &schema_name, &physical_table);
1079        debug!("Ensuring physical metric table `{table_reference}` exists for insert");
1080
1081        // schema with timestamp and field column
1082        let default_schema = vec![
1083            ColumnSchema {
1084                column_name: greptime_timestamp().to_string(),
1085                datatype: ColumnDataType::TimestampMillisecond as _,
1086                semantic_type: SemanticType::Timestamp as _,
1087                datatype_extension: None,
1088                options: None,
1089            },
1090            ColumnSchema {
1091                column_name: greptime_value().to_string(),
1092                datatype: ColumnDataType::Float64 as _,
1093                semantic_type: SemanticType::Field as _,
1094                datatype_extension: None,
1095                options: None,
1096            },
1097        ];
1098        let create_table_expr =
1099            &mut build_create_table_expr(&table_reference, &default_schema, default_engine())?;
1100
1101        create_table_expr.engine = METRIC_ENGINE_NAME.to_string();
1102        create_table_expr
1103            .table_options
1104            .insert(PHYSICAL_TABLE_METADATA_KEY.to_string(), "true".to_string());
1105
1106        // create physical table
1107        let res = statement_executor
1108            .create_table_inner(
1109                create_table_expr,
1110                None,
1111                ctx.clone(),
1112                TriggerReason::AutoCreate,
1113            )
1114            .await;
1115
1116        match res {
1117            Ok(_) => Ok(()),
1118            Err(err) => {
1119                error!(err; "Failed to create table {table_reference}");
1120                Err(err)
1121            }
1122        }
1123    }
1124
1125    async fn get_table(
1126        &self,
1127        catalog: &str,
1128        schema: &str,
1129        table: &str,
1130    ) -> Result<Option<TableRef>> {
1131        self.catalog_manager
1132            .table(catalog, schema, table, None)
1133            .await
1134            .context(CatalogSnafu)
1135    }
1136
1137    fn get_create_table_expr_on_demand(
1138        &self,
1139        req: &RowInsertRequest,
1140        create_type: &AutoCreateTableType,
1141        ctx: &QueryContextRef,
1142        semantic_index: Option<&PerTableSemanticIndex>,
1143    ) -> Result<CreateTableExpr> {
1144        let schema = ctx.current_schema();
1145        let mut table_options = std::collections::HashMap::with_capacity(4);
1146        fill_table_options_for_create(&mut table_options, create_type, ctx);
1147        apply_per_table_semantic_options(
1148            &mut table_options,
1149            semantic_index,
1150            ctx.current_schema().as_str(),
1151            &req.table_name,
1152        );
1153
1154        let engine_name = if let AutoCreateTableType::Logical(_) = create_type {
1155            // engine should be metric engine when creating logical tables.
1156            METRIC_ENGINE_NAME
1157        } else {
1158            default_engine()
1159        };
1160
1161        let table_ref = TableReference::full(ctx.current_catalog(), &schema, &req.table_name);
1162        // SAFETY: `req.rows` is guaranteed to be `Some` by `handle_row_inserts_with_create_type()`.
1163        let request_schema = req.rows.as_ref().unwrap().schema.as_slice();
1164        let mut create_table_expr =
1165            build_create_table_expr(&table_ref, request_schema, engine_name)?;
1166
1167        // extension set by the Splunk HEC handler for identity path
1168        if ctx.extension(SPLUNK_PK_METADATA_ORDER_KEY).is_some() {
1169            reorder_splunk_primary_keys(&mut create_table_expr.primary_keys);
1170        }
1171
1172        debug!("Ensuring table `{table_ref}` exists for insert");
1173        create_table_expr.table_options.extend(table_options);
1174        Ok(create_table_expr)
1175    }
1176
1177    /// Returns an alter table expression if it finds new columns in the request.
1178    /// When `accommodate_existing_schema` is false, it always adds columns if not exist.
1179    /// When `accommodate_existing_schema` is true, it may modify the input `req` to
1180    /// accommodate it with existing schema. See [`create_or_alter_tables_on_demand`](Self::create_or_alter_tables_on_demand)
1181    /// for more details.
1182    /// When `is_single_value` is true, it also rejects native-histogram/float kind changes.
1183    /// When both options are true, it considers fields when modifying the input `req`.
1184    fn get_alter_table_expr_on_demand(
1185        &self,
1186        req: &mut RowInsertRequest,
1187        table: &TableRef,
1188        ctx: &QueryContextRef,
1189        accommodate_existing_schema: bool,
1190        is_single_value: bool,
1191        alter_existing: bool,
1192    ) -> Result<Option<AlterTableExpr>> {
1193        if !alter_existing {
1194            return Ok(None);
1195        }
1196
1197        let catalog_name = ctx.current_catalog();
1198        let schema_name = ctx.current_schema();
1199        let table_name = table.table_info().name.clone();
1200
1201        // Never auto-alter a system-defined table to fit a write; a request
1202        // with unknown columns fails instead.
1203        if is_ddl_reserved_table(&schema_name, &table_name) {
1204            return Ok(None);
1205        }
1206
1207        let request_schema = req.rows.as_ref().unwrap().schema.as_slice();
1208        let request_field_count = request_schema
1209            .iter()
1210            .filter(|col| col.semantic_type == SemanticType::Field as i32)
1211            .count();
1212        let column_exprs = ColumnExpr::from_column_schemas(request_schema);
1213        let add_columns = expr_helper::extract_add_columns_expr(&table.schema(), column_exprs)?;
1214        let Some(mut add_columns) = add_columns else {
1215            return Ok(None);
1216        };
1217
1218        if is_single_value {
1219            let request_is_native_histogram = request_is_native_histogram(request_schema);
1220            let table_is_native_histogram = table_is_native_histogram(table);
1221            ensure!(
1222                request_is_native_histogram == table_is_native_histogram,
1223                InvalidInsertRequestSnafu {
1224                    reason: format!(
1225                        "Table `{table_name}` cannot mix native histogram and float sample fields"
1226                    ),
1227                }
1228            );
1229        }
1230
1231        // If accommodate_existing_schema is true, update request schema for Timestamp/Field columns
1232        if accommodate_existing_schema {
1233            let table_schema = table.schema();
1234            // Find timestamp column name
1235            let ts_col_name = table_schema.timestamp_column().map(|c| c.name.clone());
1236            // Find field column name if there is only one and `is_single_value` is true.
1237            let mut field_col_name = None;
1238            if is_single_value && request_field_count <= 1 {
1239                let mut multiple_field_cols = false;
1240                table.field_columns().for_each(|col| {
1241                    if field_col_name.is_none() {
1242                        field_col_name = Some(col.name.clone());
1243                    } else {
1244                        multiple_field_cols = true;
1245                    }
1246                });
1247                if multiple_field_cols {
1248                    field_col_name = None;
1249                }
1250            }
1251
1252            // Update column name in request schema for Timestamp/Field columns
1253            if let Some(rows) = req.rows.as_mut() {
1254                for col in &mut rows.schema {
1255                    match col.semantic_type {
1256                        x if x == SemanticType::Timestamp as i32 => {
1257                            if let Some(ref ts_name) = ts_col_name
1258                                && col.column_name != *ts_name
1259                            {
1260                                col.column_name = ts_name.clone();
1261                            }
1262                        }
1263                        x if x == SemanticType::Field as i32 => {
1264                            if let Some(ref field_name) = field_col_name
1265                                && col.column_name != *field_name
1266                            {
1267                                col.column_name = field_name.clone();
1268                            }
1269                        }
1270                        _ => {}
1271                    }
1272                }
1273            }
1274
1275            // Only keep columns that are tags or non-single field.
1276            add_columns.add_columns.retain(|col| {
1277                let def = col.column_def.as_ref().unwrap();
1278                def.semantic_type == SemanticType::Tag as i32
1279                    || (def.semantic_type == SemanticType::Field as i32 && field_col_name.is_none())
1280            });
1281
1282            if add_columns.add_columns.is_empty() {
1283                return Ok(None);
1284            }
1285        }
1286
1287        Ok(Some(AlterTableExpr {
1288            catalog_name: catalog_name.to_string(),
1289            schema_name: schema_name.clone(),
1290            table_name: table_name.clone(),
1291            kind: Some(Kind::AddColumns(add_columns)),
1292        }))
1293    }
1294
1295    /// Creates a table with options.
1296    async fn create_physical_table(
1297        &self,
1298        mut create_table_expr: CreateTableExpr,
1299        partitions: Option<Partitions>,
1300        ctx: &QueryContextRef,
1301        statement_executor: &StatementExecutor,
1302    ) -> Result<TableRef> {
1303        let res = statement_executor
1304            .create_table_inner(
1305                &mut create_table_expr,
1306                partitions,
1307                ctx.clone(),
1308                TriggerReason::AutoCreate,
1309            )
1310            .await;
1311
1312        let table_ref = TableReference::full(
1313            &create_table_expr.catalog_name,
1314            &create_table_expr.schema_name,
1315            &create_table_expr.table_name,
1316        );
1317
1318        match res {
1319            Ok(table) => {
1320                validate_trace_table_model(&table.table_info(), ctx)?;
1321                Ok(table)
1322            }
1323            Err(err) => {
1324                error!(err; "Failed to create table {}", table_ref);
1325                Err(err)
1326            }
1327        }
1328    }
1329
1330    async fn create_logical_tables(
1331        &self,
1332        create_table_exprs: Vec<CreateTableExpr>,
1333        ctx: &QueryContextRef,
1334        statement_executor: &StatementExecutor,
1335    ) -> Result<Vec<TableRef>> {
1336        let res = statement_executor
1337            .create_logical_tables(&create_table_exprs, ctx.clone(), TriggerReason::AutoCreate)
1338            .await;
1339
1340        match res {
1341            Ok(res) => Ok(res),
1342            Err(err) => {
1343                let failed_tables = create_table_exprs
1344                    .into_iter()
1345                    .map(|expr| {
1346                        format!(
1347                            "{}.{}.{}",
1348                            expr.catalog_name, expr.schema_name, expr.table_name
1349                        )
1350                    })
1351                    .collect::<Vec<_>>();
1352                error!(
1353                    err;
1354                    "Failed to create logical tables {:?}",
1355                    failed_tables
1356                );
1357                Err(err)
1358            }
1359        }
1360    }
1361
1362    pub fn node_manager(&self) -> &NodeManagerRef {
1363        &self.node_manager
1364    }
1365
1366    pub fn partition_manager(&self) -> &PartitionRuleManagerRef {
1367        &self.partition_manager
1368    }
1369
1370    pub fn table_flownode_set_cache(&self) -> &TableFlownodeSetCacheRef {
1371        &self.table_flownode_set_cache
1372    }
1373}
1374
1375fn request_is_native_histogram(request_schema: &[ColumnSchema]) -> bool {
1376    let mut fields = request_schema
1377        .iter()
1378        .filter(|col| col.semantic_type == SemanticType::Field as i32);
1379    let Some(col) = fields.next() else {
1380        return false;
1381    };
1382
1383    fields.next().is_none()
1384        && api::helper::is_column_type_value_eq(
1385            col.datatype,
1386            col.datatype_extension.clone(),
1387            native_histogram_value_type(),
1388        )
1389}
1390
1391fn table_is_native_histogram(table: &TableRef) -> bool {
1392    let mut fields = table.field_columns();
1393    let Some(col) = fields.next() else {
1394        return false;
1395    };
1396
1397    fields.next().is_none() && is_native_histogram_value_type(&col.data_type)
1398}
1399
1400fn validate_column_count_match(requests: &RowInsertRequests) -> Result<()> {
1401    for request in &requests.inserts {
1402        let rows = request.rows.as_ref().unwrap();
1403        let column_count = rows.schema.len();
1404        rows.rows.iter().try_for_each(|r| {
1405            ensure!(
1406                r.values.len() == column_count,
1407                InvalidInsertRequestSnafu {
1408                    reason: format!(
1409                        "column count mismatch, columns: {}, values: {}",
1410                        column_count,
1411                        r.values.len()
1412                    )
1413                }
1414            );
1415            Ok(())
1416        })?;
1417    }
1418    Ok(())
1419}
1420
1421/// Rejects writes from a different built-in trace model before schema mutation.
1422/// Unstamped, explicitly created tables remain subject to normal schema validation.
1423pub fn validate_trace_table_model(table_info: &TableInfo, ctx: &QueryContextRef) -> Result<()> {
1424    let Some(expected @ (TABLE_DATA_MODEL_TRACE_V1 | TABLE_DATA_MODEL_TRACE_V2)) =
1425        ctx.extension(SEMANTIC_PIPELINE)
1426    else {
1427        return Ok(());
1428    };
1429    if let Some(actual) = table_info.meta.options.extra_options.get(TABLE_DATA_MODEL) {
1430        ensure!(
1431            actual == expected,
1432            InvalidInsertRequestSnafu {
1433                reason: format!(
1434                    "Trace table `{}` uses {actual}, but the request uses {expected}",
1435                    table_info.name,
1436                ),
1437            }
1438        );
1439    }
1440    Ok(())
1441}
1442
1443/// Fill table options for a new table by create type.
1444pub fn fill_table_options_for_create(
1445    table_options: &mut std::collections::HashMap<String, String>,
1446    create_type: &AutoCreateTableType,
1447    ctx: &QueryContextRef,
1448) {
1449    for key in VALID_TABLE_OPTION_KEYS {
1450        if let Some(value) = ctx.extension(key) {
1451            table_options.insert(key.to_string(), value.to_string());
1452        }
1453    }
1454
1455    // Semantic keys use their own vocabulary instead of the fixed option list.
1456    for (key, value) in ctx.extensions() {
1457        if is_semantic_option_key(&key) && validate_semantic_option(&key, &value) {
1458            table_options.insert(key, value);
1459        }
1460    }
1461
1462    match create_type {
1463        AutoCreateTableType::Logical(physical_table) => {
1464            table_options.insert(
1465                LOGICAL_TABLE_METADATA_KEY.to_string(),
1466                physical_table.clone(),
1467            );
1468        }
1469        AutoCreateTableType::Physical => {
1470            if let Some(append_mode) = ctx.extension(APPEND_MODE_KEY) {
1471                table_options.insert(APPEND_MODE_KEY.to_string(), append_mode.to_string());
1472            }
1473            if let Some(merge_mode) = ctx.extension(MERGE_MODE_KEY) {
1474                table_options.insert(MERGE_MODE_KEY.to_string(), merge_mode.to_string());
1475            }
1476            if let Some(time_window) = ctx.extension(TWCS_TIME_WINDOW) {
1477                table_options.insert(TWCS_TIME_WINDOW.to_string(), time_window.to_string());
1478                // We need to set the compaction type explicitly.
1479                table_options.insert(
1480                    COMPACTION_TYPE.to_string(),
1481                    COMPACTION_TYPE_TWCS.to_string(),
1482                );
1483            }
1484        }
1485        // Set append_mode to true for log table.
1486        // because log tables should keep rows with the same ts and tags.
1487        AutoCreateTableType::Log => {
1488            table_options.insert(APPEND_MODE_KEY.to_string(), "true".to_string());
1489        }
1490        AutoCreateTableType::LastNonNull => {
1491            if ctx
1492                .extension(APPEND_MODE_KEY)
1493                .is_some_and(|value| value.eq_ignore_ascii_case("true"))
1494            {
1495                table_options.insert(APPEND_MODE_KEY.to_string(), "true".to_string());
1496                table_options.insert(MERGE_MODE_KEY.to_string(), "last_row".to_string());
1497            } else if let Some(merge_mode) = ctx.extension(MERGE_MODE_KEY) {
1498                table_options.insert(MERGE_MODE_KEY.to_string(), merge_mode.to_string());
1499            } else {
1500                table_options.insert(MERGE_MODE_KEY.to_string(), "last_non_null".to_string());
1501            }
1502        }
1503        AutoCreateTableType::Trace { .. } => {
1504            table_options.insert(APPEND_MODE_KEY.to_string(), "true".to_string());
1505        }
1506    }
1507}
1508
1509/// The parsed per-table semantic index: `{schema -> {table -> {key -> value}}}`,
1510/// produced by the OTLP metrics encode path (where one metric can fan out into
1511/// several tables with distinct keys) and the Prometheus remote write v2 path
1512/// (where per-series metadata declares type/unit, and a series may override its
1513/// target schema).
1514pub type PerTableSemanticIndex = BTreeMap<String, BTreeMap<String, BTreeMap<String, String>>>;
1515
1516/// Parses the per-table semantic index off the context extension. Call once per
1517/// create-planning round: a first write creating N tables would otherwise
1518/// re-parse the whole index N times. `None` when the request carries no index
1519/// (logs, traces, Prom RW v1) or it fails to parse.
1520pub fn parse_per_table_semantic_index(ctx: &QueryContextRef) -> Option<PerTableSemanticIndex> {
1521    let raw = ctx.extension(SEMANTIC_PER_TABLE_INDEX_KEY)?;
1522    match serde_json::from_str(raw) {
1523        Ok(index) => Some(index),
1524        Err(_) => {
1525            warn!("failed to parse semantic per-table index, skipping per-table options");
1526            None
1527        }
1528    }
1529}
1530
1531/// Folds the semantic keys of the table being created into `table_options`.
1532///
1533/// Common keys shared by every table in a request travel as plain semantic
1534/// extensions and are handled by [`fill_table_options_for_create`]; this
1535/// carries only the per-table tail and is applied after it, so a per-table
1536/// value (e.g. `declared` quality) wins. Keys are re-checked against the
1537/// vocabulary defensively.
1538pub fn apply_per_table_semantic_options(
1539    table_options: &mut std::collections::HashMap<String, String>,
1540    index: Option<&PerTableSemanticIndex>,
1541    schema: &str,
1542    table_name: &str,
1543) {
1544    let Some(entry) = index
1545        .and_then(|index| index.get(schema))
1546        .and_then(|tables| tables.get(table_name))
1547    else {
1548        return;
1549    };
1550    for (key, value) in entry {
1551        if is_semantic_option_key(key) && validate_semantic_option(key, value) {
1552            table_options.insert(key.clone(), value.clone());
1553        }
1554    }
1555}
1556
1557pub fn build_create_table_expr(
1558    table: &TableReference,
1559    request_schema: &[ColumnSchema],
1560    engine: &str,
1561) -> Result<CreateTableExpr> {
1562    expr_helper::create_table_expr_by_column_schemas(table, request_schema, engine, None)
1563}
1564
1565/// `QueryContext` extension key the Splunk HEC handler sets (to `"true"`) on its identity
1566/// path to request metadata-first primary-key ordering at table creation. It is absent for
1567/// user-supplied pipelines, so their primary-key order is left untouched.
1568pub const SPLUNK_PK_METADATA_ORDER_KEY: &str = "splunk_pk_metadata_order";
1569
1570/// Moves Splunk's metadata tags (`host`, `source`, `sourcetype`) to the front of the
1571/// primary key, keeping the relative order of the remaining tags.
1572fn reorder_splunk_primary_keys(primary_keys: &mut [String]) {
1573    const LEAD: [&str; 3] = ["host", "source", "sourcetype"];
1574    // Stable sort: `LEAD` columns move to the front in `host`/`source`/`sourcetype` order;
1575    // every other column keeps its existing relative position.
1576    primary_keys.sort_by_key(|name| {
1577        LEAD.iter()
1578            .position(|&lead| lead == name.as_str())
1579            .unwrap_or(LEAD.len())
1580    });
1581}
1582
1583/// Result of `create_or_alter_tables_on_demand`.
1584struct CreateAlterTableResult {
1585    /// table ids of ttl=instant tables.
1586    instant_table_ids: HashSet<TableId>,
1587    /// Table Info of the created tables.
1588    table_infos: HashMap<TableId, Arc<TableInfo>>,
1589}
1590
1591struct FlowMirrorTask {
1592    requests: HashMap<Peer, RegionInsertRequests>,
1593    num_rows: usize,
1594}
1595
1596impl FlowMirrorTask {
1597    async fn new(
1598        cache: &TableFlownodeSetCacheRef,
1599        requests: impl Iterator<Item = &RegionInsertRequest>,
1600    ) -> Result<Self> {
1601        let mut src_table_reqs: HashMap<TableId, Option<(Vec<Peer>, RegionInsertRequests)>> =
1602            HashMap::new();
1603        let mut num_rows = 0;
1604
1605        for req in requests {
1606            let table_id = RegionId::from_u64(req.region_id).table_id();
1607            match src_table_reqs.get_mut(&table_id) {
1608                Some(Some((_peers, reqs))) => reqs.requests.push(req.clone()),
1609                // already know this is not source table
1610                Some(None) => continue,
1611                _ => {
1612                    // dedup peers
1613                    let peers = cache
1614                        .get(table_id)
1615                        .await
1616                        .context(RequestInsertsSnafu)?
1617                        .unwrap_or_default()
1618                        .values()
1619                        .cloned()
1620                        .collect::<HashSet<_>>()
1621                        .into_iter()
1622                        .collect::<Vec<_>>();
1623
1624                    if !peers.is_empty() {
1625                        let mut reqs = RegionInsertRequests::default();
1626                        reqs.requests.push(req.clone());
1627                        num_rows += reqs
1628                            .requests
1629                            .iter()
1630                            .map(|r| r.rows.as_ref().unwrap().rows.len())
1631                            .sum::<usize>();
1632                        src_table_reqs.insert(table_id, Some((peers, reqs)));
1633                    } else {
1634                        // insert a empty entry to avoid repeat query
1635                        src_table_reqs.insert(table_id, None);
1636                    }
1637                }
1638            }
1639        }
1640
1641        let mut inserts: HashMap<Peer, RegionInsertRequests> = HashMap::new();
1642
1643        for (_table_id, (peers, reqs)) in src_table_reqs
1644            .into_iter()
1645            .filter_map(|(k, v)| v.map(|v| (k, v)))
1646        {
1647            if peers.len() == 1 {
1648                // fast path, zero copy
1649                inserts
1650                    .entry(peers[0].clone())
1651                    .or_default()
1652                    .requests
1653                    .extend(reqs.requests);
1654                continue;
1655            } else {
1656                // TODO(discord9): need to split requests to multiple flownodes
1657                for flownode in peers {
1658                    inserts
1659                        .entry(flownode.clone())
1660                        .or_default()
1661                        .requests
1662                        .extend(reqs.requests.clone());
1663                }
1664            }
1665        }
1666
1667        Ok(Self {
1668            requests: inserts,
1669            num_rows,
1670        })
1671    }
1672
1673    fn detach(self, node_manager: NodeManagerRef) -> Result<()> {
1674        crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.add(self.num_rows as i64);
1675        for (peer, inserts) in self.requests {
1676            let node_manager = node_manager.clone();
1677            common_runtime::spawn_global(async move {
1678                let result = node_manager
1679                    .flownode(&peer)
1680                    .await
1681                    .handle_inserts(inserts)
1682                    .await
1683                    .context(RequestInsertsSnafu);
1684
1685                match result {
1686                    Ok(resp) => {
1687                        let affected_rows = resp.affected_rows;
1688                        crate::metrics::DIST_MIRROR_ROW_COUNT.inc_by(affected_rows);
1689                        crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.sub(affected_rows as _);
1690                    }
1691                    Err(err) => {
1692                        error!(err; "Failed to insert data into flownode {}", peer);
1693                    }
1694                }
1695            });
1696        }
1697
1698        Ok(())
1699    }
1700}
1701
1702#[cfg(test)]
1703mod tests {
1704    use std::sync::Arc;
1705
1706    use api::helper::ColumnDataTypeWrapper;
1707    use api::v1::helper::{field_column_schema, time_index_column_schema};
1708    use api::v1::{RowInsertRequest, Rows, Value};
1709    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
1710    use common_meta::cache::new_table_flownode_set_cache;
1711    use common_meta::ddl::test_util::datanode_handler::NaiveDatanodeHandler;
1712    use common_meta::test_util::MockDatanodeManager;
1713    use common_query::native_histogram::NATIVE_HISTOGRAM_FIELD;
1714    use common_query::prelude::{greptime_native_histogram, set_default_prefix};
1715    use datatypes::data_type::ConcreteDataType;
1716    use datatypes::schema::ColumnSchema;
1717    use moka::future::Cache;
1718    use session::context::QueryContext;
1719    use table::TableRef;
1720    use table::dist_table::DummyDataSource;
1721    use table::metadata::{TableInfoBuilder, TableMetaBuilder, TableType};
1722
1723    use crate::insert::*;
1724    use crate::test_util::{create_partition_rule_manager, prepare_mocked_backend};
1725
1726    fn make_table_ref_with_schema(
1727        ts_name: &str,
1728        field_name: &str,
1729        field_type: ConcreteDataType,
1730    ) -> TableRef {
1731        let schema = datatypes::schema::SchemaBuilder::try_from_columns(vec![
1732            ColumnSchema::new(
1733                ts_name,
1734                ConcreteDataType::timestamp_millisecond_datatype(),
1735                false,
1736            )
1737            .with_time_index(true),
1738            ColumnSchema::new(field_name, field_type, true),
1739        ])
1740        .unwrap()
1741        .build()
1742        .unwrap();
1743        let meta = TableMetaBuilder::empty()
1744            .schema(Arc::new(schema))
1745            .primary_key_indices(vec![])
1746            .value_indices(vec![1])
1747            .engine("mito")
1748            .next_column_id(0)
1749            .options(Default::default())
1750            .created_on(Default::default())
1751            .build()
1752            .unwrap();
1753        let info = Arc::new(
1754            TableInfoBuilder::default()
1755                .table_id(1)
1756                .table_version(0)
1757                .name("test_table")
1758                .schema_name(DEFAULT_SCHEMA_NAME)
1759                .catalog_name(DEFAULT_CATALOG_NAME)
1760                .desc(None)
1761                .table_type(TableType::Base)
1762                .meta(meta)
1763                .build()
1764                .unwrap(),
1765        );
1766        Arc::new(table::Table::new(
1767            info,
1768            table::metadata::FilterPushDownType::Unsupported,
1769            Arc::new(DummyDataSource),
1770        ))
1771    }
1772
1773    #[tokio::test]
1774    async fn test_accommodate_existing_schema_and_reject_kind_changes() {
1775        let ts_name = "my_ts";
1776        let field_name = "my_field";
1777        let table =
1778            make_table_ref_with_schema(ts_name, field_name, ConcreteDataType::float64_datatype());
1779
1780        // The request uses different names for timestamp and field columns
1781        let mut req = RowInsertRequest {
1782            table_name: "test_table".to_string(),
1783            rows: Some(Rows {
1784                schema: vec![
1785                    time_index_column_schema("ts_wrong", ColumnDataType::TimestampMillisecond),
1786                    field_column_schema("field_wrong", ColumnDataType::Float64),
1787                ],
1788                rows: vec![api::v1::Row {
1789                    values: vec![Value::default(), Value::default()],
1790                }],
1791            }),
1792        };
1793        let ctx = Arc::new(QueryContext::with(
1794            DEFAULT_CATALOG_NAME,
1795            DEFAULT_SCHEMA_NAME,
1796        ));
1797
1798        let kv_backend = prepare_mocked_backend().await;
1799        let inserter = Inserter::new(
1800            catalog::memory::MemoryCatalogManager::new(),
1801            create_partition_rule_manager(kv_backend.clone()).await,
1802            Arc::new(MockDatanodeManager::new(NaiveDatanodeHandler)),
1803            Arc::new(new_table_flownode_set_cache(
1804                String::new(),
1805                Cache::new(100),
1806                kv_backend.clone(),
1807            )),
1808            true,
1809        );
1810        // Do not apply an absent-table plan to a table that appeared concurrently.
1811        assert!(
1812            inserter
1813                .get_alter_table_expr_on_demand(&mut req, &table, &ctx, false, false, false)
1814                .unwrap()
1815                .is_none()
1816        );
1817        let alter_expr = inserter
1818            .get_alter_table_expr_on_demand(&mut req, &table, &ctx, true, true, true)
1819            .unwrap();
1820        assert!(alter_expr.is_none());
1821
1822        // The request's schema should have updated names for timestamp and field columns
1823        let req_schema = req.rows.as_ref().unwrap().schema.clone();
1824        assert_eq!(req_schema[0].column_name, ts_name);
1825        assert_eq!(req_schema[1].column_name, field_name);
1826
1827        let (datatype, datatype_extension) =
1828            ColumnDataTypeWrapper::try_from(native_histogram_value_type().clone())
1829                .unwrap()
1830                .into_parts();
1831        let mut histogram_req = RowInsertRequest {
1832            table_name: "test_table".to_string(),
1833            rows: Some(Rows {
1834                schema: vec![
1835                    time_index_column_schema("ts", ColumnDataType::TimestampMillisecond),
1836                    api::v1::ColumnSchema {
1837                        column_name: greptime_native_histogram().to_string(),
1838                        datatype: datatype as i32,
1839                        semantic_type: SemanticType::Field as i32,
1840                        datatype_extension,
1841                        options: None,
1842                    },
1843                ],
1844                rows: vec![],
1845            }),
1846        };
1847        let error = inserter
1848            .get_alter_table_expr_on_demand(&mut histogram_req, &table, &ctx, false, true, true)
1849            .unwrap_err();
1850        assert!(
1851            error
1852                .to_string()
1853                .contains("cannot mix native histogram and float sample fields")
1854        );
1855
1856        let histogram_table = make_table_ref_with_schema(
1857            "ts",
1858            greptime_native_histogram(),
1859            native_histogram_value_type().clone(),
1860        );
1861        let mut sample_req = RowInsertRequest {
1862            table_name: "test_table".to_string(),
1863            rows: Some(Rows {
1864                schema: vec![
1865                    time_index_column_schema("ts", ColumnDataType::TimestampMillisecond),
1866                    field_column_schema(greptime_value(), ColumnDataType::Float64),
1867                ],
1868                rows: vec![],
1869            }),
1870        };
1871        let error = inserter
1872            .get_alter_table_expr_on_demand(
1873                &mut sample_req,
1874                &histogram_table,
1875                &ctx,
1876                false,
1877                true,
1878                true,
1879            )
1880            .unwrap_err();
1881        assert!(
1882            error
1883                .to_string()
1884                .contains("cannot mix native histogram and float sample fields")
1885        );
1886    }
1887
1888    #[test]
1889    fn test_native_histogram_detection_survives_prefix_change() {
1890        set_default_prefix(Some("custom")).unwrap();
1891        let table = make_table_ref_with_schema(
1892            "custom_timestamp",
1893            NATIVE_HISTOGRAM_FIELD,
1894            native_histogram_value_type().clone(),
1895        );
1896        let (datatype, datatype_extension) =
1897            ColumnDataTypeWrapper::try_from(native_histogram_value_type().clone())
1898                .unwrap()
1899                .into_parts();
1900        let request_schema = [api::v1::ColumnSchema {
1901            column_name: greptime_native_histogram().to_string(),
1902            datatype: datatype as i32,
1903            semantic_type: SemanticType::Field as i32,
1904            datatype_extension,
1905            options: None,
1906        }];
1907
1908        assert!(request_is_native_histogram(&request_schema));
1909        assert!(table_is_native_histogram(&table));
1910    }
1911
1912    #[test]
1913    fn test_skip_wal_does_not_change_table_options() {
1914        check_skip_wal_does_not_change_table_options(false);
1915        check_skip_wal_does_not_change_table_options(true);
1916    }
1917
1918    fn check_skip_wal_does_not_change_table_options(skip_wal: bool) {
1919        let ctx = Arc::new(QueryContext::with(
1920            DEFAULT_CATALOG_NAME,
1921            DEFAULT_SCHEMA_NAME,
1922        ));
1923        ctx.set_skip_wal(skip_wal);
1924        let mut options = Default::default();
1925        fill_table_options_for_create(&mut options, &AutoCreateTableType::Physical, &ctx);
1926        assert!(!options.contains_key(session::hints::INSERT_SKIP_WAL_HINT));
1927        assert!(!options.contains_key("skip_wal"));
1928    }
1929
1930    #[test]
1931    fn test_last_non_null_create_options_preserve_default_without_append_mode() {
1932        let ctx = Arc::new(QueryContext::with(
1933            DEFAULT_CATALOG_NAME,
1934            DEFAULT_SCHEMA_NAME,
1935        ));
1936        let mut table_options = Default::default();
1937
1938        fill_table_options_for_create(&mut table_options, &AutoCreateTableType::LastNonNull, &ctx);
1939
1940        assert_eq!(
1941            Some("last_non_null"),
1942            table_options.get(MERGE_MODE_KEY).map(String::as_str)
1943        );
1944        assert!(!table_options.contains_key(APPEND_MODE_KEY));
1945    }
1946
1947    #[test]
1948    fn test_fill_table_options_copies_semantic_extensions() {
1949        use table::requests::{
1950            SEMANTIC_METRIC_TYPE, SEMANTIC_PER_TABLE_INDEX_KEY, SEMANTIC_SIGNAL_TYPE,
1951            SEMANTIC_SOURCE, SEMANTIC_SOURCE_VERSION, SIGNAL_TYPE_METRIC, SOURCE_OPENTELEMETRY,
1952        };
1953
1954        let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
1955        ctx.set_extension(SEMANTIC_SIGNAL_TYPE, SIGNAL_TYPE_METRIC);
1956        ctx.set_extension(SEMANTIC_SOURCE, SOURCE_OPENTELEMETRY);
1957        ctx.set_extension(SEMANTIC_SOURCE_VERSION, "2.0");
1958        ctx.set_extension(SEMANTIC_METRIC_TYPE, "bogus");
1959        // The internal transport key must NOT be copied into table options.
1960        ctx.set_extension(SEMANTIC_PER_TABLE_INDEX_KEY, "{}");
1961        let ctx = Arc::new(ctx);
1962        let mut table_options = Default::default();
1963
1964        fill_table_options_for_create(&mut table_options, &AutoCreateTableType::Physical, &ctx);
1965
1966        assert_eq!(
1967            Some(SIGNAL_TYPE_METRIC),
1968            table_options.get(SEMANTIC_SIGNAL_TYPE).map(String::as_str)
1969        );
1970        assert_eq!(
1971            Some(SOURCE_OPENTELEMETRY),
1972            table_options.get(SEMANTIC_SOURCE).map(String::as_str)
1973        );
1974        assert_eq!(
1975            Some("2.0"),
1976            table_options
1977                .get(SEMANTIC_SOURCE_VERSION)
1978                .map(String::as_str)
1979        );
1980        assert!(!table_options.contains_key(SEMANTIC_METRIC_TYPE));
1981        assert!(!table_options.contains_key(SEMANTIC_PER_TABLE_INDEX_KEY));
1982    }
1983
1984    #[test]
1985    fn test_apply_per_table_semantic_options() {
1986        use table::requests::{
1987            SEMANTIC_METRIC_TYPE, SEMANTIC_METRIC_UNIT, SEMANTIC_PER_TABLE_INDEX_KEY,
1988        };
1989
1990        let index = format!(
1991            r#"{{
1992            "{DEFAULT_SCHEMA_NAME}": {{
1993                "http_requests_total": {{
1994                    "greptime.semantic.metric.type": "counter",
1995                    "greptime.semantic.metric.unit": "By",
1996                    "greptime.semantic.metric.type_BOGUS": "x"
1997                }},
1998                "other_table": {{
1999                    "greptime.semantic.metric.type": "gauge"
2000                }}
2001            }},
2002            "other_schema": {{
2003                "http_requests_total": {{
2004                    "greptime.semantic.metric.type": "gauge"
2005                }}
2006            }}
2007        }}"#
2008        );
2009        let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
2010        ctx.set_extension(SEMANTIC_PER_TABLE_INDEX_KEY, index);
2011        let ctx = Arc::new(ctx);
2012
2013        let index = parse_per_table_semantic_index(&ctx);
2014        assert!(index.is_some());
2015        let index = index.as_ref();
2016
2017        let mut table_options = std::collections::HashMap::new();
2018        apply_per_table_semantic_options(
2019            &mut table_options,
2020            index,
2021            DEFAULT_SCHEMA_NAME,
2022            "http_requests_total",
2023        );
2024        // The write schema's entry applies — not other_schema's `gauge`.
2025        assert_eq!(
2026            table_options.get(SEMANTIC_METRIC_TYPE).map(String::as_str),
2027            Some("counter")
2028        );
2029        assert_eq!(
2030            table_options.get(SEMANTIC_METRIC_UNIT).map(String::as_str),
2031            Some("By")
2032        );
2033        // The unknown key is rejected by the vocabulary check; other tables' keys
2034        // never appear.
2035        assert!(!table_options.contains_key("greptime.semantic.metric.type_BOGUS"));
2036        assert_eq!(table_options.len(), 2);
2037
2038        let mut empty = std::collections::HashMap::new();
2039        apply_per_table_semantic_options(&mut empty, index, DEFAULT_SCHEMA_NAME, "not_in_index");
2040        assert!(empty.is_empty());
2041
2042        // A schema with no entry is a no-op even when the table name matches
2043        // elsewhere.
2044        let mut opts = std::collections::HashMap::new();
2045        apply_per_table_semantic_options(
2046            &mut opts,
2047            index,
2048            "schema_without_entry",
2049            "http_requests_total",
2050        );
2051        assert!(opts.is_empty());
2052
2053        // No extension at all parses to no index (e.g. logs / Prom RW v1).
2054        let bare = Arc::new(QueryContext::with(
2055            DEFAULT_CATALOG_NAME,
2056            DEFAULT_SCHEMA_NAME,
2057        ));
2058        assert!(parse_per_table_semantic_index(&bare).is_none());
2059        let mut opts = std::collections::HashMap::new();
2060        apply_per_table_semantic_options(
2061            &mut opts,
2062            None,
2063            DEFAULT_SCHEMA_NAME,
2064            "http_requests_total",
2065        );
2066        assert!(opts.is_empty());
2067    }
2068
2069    #[test]
2070    fn test_last_non_null_create_options_preserve_default_with_append_mode_false() {
2071        let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
2072        ctx.set_extension(APPEND_MODE_KEY, "false");
2073        let ctx = Arc::new(ctx);
2074        let mut table_options = Default::default();
2075
2076        fill_table_options_for_create(&mut table_options, &AutoCreateTableType::LastNonNull, &ctx);
2077
2078        assert!(!table_options.contains_key(APPEND_MODE_KEY));
2079        assert_eq!(
2080            Some("last_non_null"),
2081            table_options.get(MERGE_MODE_KEY).map(String::as_str)
2082        );
2083    }
2084
2085    #[test]
2086    fn test_last_non_null_create_options_use_configured_merge_mode() {
2087        let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
2088        ctx.set_extension(MERGE_MODE_KEY, "last_row");
2089        let ctx = Arc::new(ctx);
2090        let mut table_options = Default::default();
2091
2092        fill_table_options_for_create(&mut table_options, &AutoCreateTableType::LastNonNull, &ctx);
2093
2094        assert_eq!(
2095            Some("last_row"),
2096            table_options.get(MERGE_MODE_KEY).map(String::as_str)
2097        );
2098        assert!(!table_options.contains_key(APPEND_MODE_KEY));
2099    }
2100
2101    #[test]
2102    fn test_last_non_null_create_options_use_last_row_with_append_mode_true() {
2103        let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
2104        ctx.set_extension(APPEND_MODE_KEY, "true");
2105        let ctx = Arc::new(ctx);
2106        let mut table_options = Default::default();
2107
2108        fill_table_options_for_create(&mut table_options, &AutoCreateTableType::LastNonNull, &ctx);
2109
2110        assert_eq!(
2111            Some("true"),
2112            table_options.get(APPEND_MODE_KEY).map(String::as_str)
2113        );
2114        assert_eq!(
2115            Some("last_row"),
2116            table_options.get(MERGE_MODE_KEY).map(String::as_str)
2117        );
2118    }
2119
2120    struct UnexpectedBatcher;
2121
2122    #[async_trait::async_trait]
2123    impl PendingRowsBatcher for UnexpectedBatcher {
2124        async fn acquire(&self) -> Result<Arc<tokio::sync::OwnedSemaphorePermit>> {
2125            panic!("empty writes must not acquire batch admission")
2126        }
2127
2128        async fn submit(
2129            &self,
2130            _table_info: TableInfoRef,
2131            _batch: arrow::record_batch::RecordBatch,
2132            _ctx: QueryContextRef,
2133            _permit: Arc<tokio::sync::OwnedSemaphorePermit>,
2134        ) -> Result<usize> {
2135            panic!("empty writes must not submit a batch")
2136        }
2137    }
2138
2139    async fn batcher_test_inserter() -> Inserter {
2140        let kv_backend = prepare_mocked_backend().await;
2141        Inserter::new(
2142            catalog::memory::MemoryCatalogManager::new(),
2143            create_partition_rule_manager(kv_backend.clone()).await,
2144            Arc::new(MockDatanodeManager::new(NaiveDatanodeHandler)),
2145            Arc::new(new_table_flownode_set_cache(
2146                String::new(),
2147                Cache::new(100),
2148                kv_backend,
2149            )),
2150            true,
2151        )
2152    }
2153
2154    #[tokio::test]
2155    async fn test_instant_table_bypasses_batcher() {
2156        let batcher: Arc<dyn PendingRowsBatcher> = Arc::new(UnexpectedBatcher);
2157        let inserter = batcher_test_inserter()
2158            .await
2159            .with_pending_rows_batcher(Some(batcher));
2160        let mut ctx = session::context::QueryContextBuilder::default().build();
2161        ctx.set_batching_enabled(true);
2162        let ctx = Arc::new(ctx);
2163        let table = make_table_ref_with_schema("ts", "value", ConcreteDataType::float64_datatype())
2164            .table_info();
2165        assert!(inserter.table_batcher(&table, &ctx).is_some());
2166        let mut instant = (*table).clone();
2167        instant.meta.options.ttl = Some(common_time::ttl::TimeToLive::Instant);
2168        assert!(inserter.table_batcher(&Arc::new(instant), &ctx).is_none());
2169    }
2170
2171    #[tokio::test]
2172    async fn test_empty_prepared_rows_skip_batcher() {
2173        let inserter = batcher_test_inserter().await;
2174        let table = make_table_ref_with_schema("ts", "value", ConcreteDataType::float64_datatype())
2175            .table_info();
2176        let batcher: Arc<dyn PendingRowsBatcher> = Arc::new(UnexpectedBatcher);
2177        let ctx = QueryContext::arc();
2178        let output = inserter
2179            .submit_table_rows(
2180                Rows {
2181                    schema: vec![],
2182                    rows: vec![],
2183                },
2184                table.clone(),
2185                ctx.clone(),
2186                &batcher,
2187            )
2188            .await
2189            .unwrap();
2190        assert!(matches!(output.data, OutputData::AffectedRows(0)));
2191        let output = inserter
2192            .submit_pending_rows(
2193                RowInsertRequests {
2194                    inserts: vec![RowInsertRequest {
2195                        table_name: table.name.clone(),
2196                        rows: None,
2197                    }],
2198                },
2199                HashMap::from_iter([(table.table_id(), table)]),
2200                ctx,
2201                &batcher,
2202            )
2203            .await
2204            .unwrap();
2205        assert!(matches!(output.data, OutputData::AffectedRows(0)));
2206    }
2207}