operator/insert/
otlp.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Side-effect-free destination validation for prepared OTLP batches. Catalog
16//! failures propagate; row errors let the converter reject complete source-point fan-outs.
17
18use std::collections::HashSet;
19
20use ahash::{HashMap as AHashMap, HashMapExt};
21use api::helper::{ColumnDataTypeWrapper, to_grpc_value};
22use api::v1::alter_table_expr::Kind;
23use api::v1::column_def::try_as_column_schema;
24use api::v1::{
25    ColumnDataType, ColumnDef, ColumnSchema, Row, RowInsertRequest, Rows, SemanticType, Value,
26};
27use common_query::native_histogram::native_histogram_value_type;
28use common_query::prelude::{greptime_timestamp, greptime_value};
29use common_time::timestamp::TimeUnit;
30use datatypes::schema::ColumnSchema as DataColumn;
31use session::context::QueryContextRef;
32use snafu::prelude::*;
33use store_api::metric_engine_consts::{LOGICAL_TABLE_METADATA_KEY, PHYSICAL_TABLE_METADATA_KEY};
34use table::TableRef;
35
36use crate::error::{self, InvalidInsertRequestSnafu, Result};
37use crate::insert::{
38    AutoCreateTableType, Inserter, convert_rows_time_unit, convert_timestamp_value_data,
39    request_is_native_histogram, table_is_native_histogram, table_time_index_unit,
40};
41use crate::req_convert::insert::{ImpureDefaultFiller, value_matches_type};
42
43pub struct MetricInsertPreflight {
44    /// Rejections in batch order, then row order.
45    pub rejections: Vec<Vec<Option<String>>>,
46    pub physical_tables: std::collections::HashMap<String, String>,
47}
48
49struct PhysicalColumns {
50    by_name: AHashMap<String, (DataColumn, SemanticType)>,
51    fields: Vec<String>,
52}
53
54impl PhysicalColumns {
55    fn new(columns: Vec<(DataColumn, SemanticType)>) -> Self {
56        let fields = columns
57            .iter()
58            .filter(|(_, semantic)| *semantic == SemanticType::Field)
59            .map(|(column, _)| column.name.clone())
60            .collect();
61        let by_name = columns
62            .into_iter()
63            .map(|(column, semantic)| (column.name.clone(), (column, semantic)))
64            .collect();
65        Self { by_name, fields }
66    }
67}
68
69impl Inserter {
70    /// Validate each row independently, sharing schema planning within each batch.
71    /// Fresh cross-point conflicts are resolved by the converter after this pass.
72    pub async fn validate_and_normalize_otlp_metrics(
73        &self,
74        batches: &mut [RowInsertRequest],
75        ctx: &QueryContextRef,
76        metric_engine: bool,
77        physical_table: &str,
78    ) -> Result<MetricInsertPreflight> {
79        let disabled = self.auto_create_disabled_reason(ctx)?;
80        let mut tables = AHashMap::new();
81        let mut physical_tables = std::collections::HashMap::new();
82        for request in batches.iter() {
83            if !tables.contains_key(&request.table_name) {
84                let table = self
85                    .get_table(
86                        ctx.current_catalog(),
87                        &ctx.current_schema(),
88                        &request.table_name,
89                    )
90                    .await?;
91                tables.insert(request.table_name.clone(), table);
92            }
93            let backing = match tables[&request.table_name].as_ref() {
94                Some(table) => table
95                    .table_info()
96                    .meta
97                    .options
98                    .extra_options
99                    .get(LOGICAL_TABLE_METADATA_KEY)
100                    .cloned(),
101                None if metric_engine => Some(physical_table.to_string()),
102                None => None,
103            };
104            if let Some(backing) = backing {
105                physical_tables.insert(request.table_name.clone(), backing);
106            }
107        }
108        // Shared schemas accumulate tags across logical tables; index them once per request.
109        let mut physical_columns = AHashMap::new();
110        for backing in physical_tables.values() {
111            if !tables.contains_key(backing) {
112                let table = self
113                    .get_table(ctx.current_catalog(), &ctx.current_schema(), backing)
114                    .await?;
115                tables.insert(backing.clone(), table);
116            }
117            physical_columns.entry(backing.as_str()).or_insert_with(|| {
118                PhysicalColumns::new(
119                    tables[backing]
120                        .as_ref()
121                        .map(table_columns)
122                        .unwrap_or_else(default_physical_columns),
123                )
124            });
125        }
126        // Match ordinary insertion: impure defaults are evaluated once per table
127        // and request, while supplied nulls remain supplied nulls.
128        let mut fillers = AHashMap::new();
129        let mut build_plan = |source: &mut RowInsertRequest| {
130            let rows = source.rows.as_mut().context(InvalidInsertRequestSnafu {
131                reason: "missing metric rows",
132            })?;
133            let input_columns = rows.schema.len();
134            let source_timestamp = timestamp_column(&rows.schema);
135            let request = &mut RowInsertRequest {
136                table_name: source.table_name.clone(),
137                rows: Some(Rows {
138                    schema: std::mem::take(&mut rows.schema),
139                    rows: Vec::new(),
140                }),
141            };
142            let auto_create = disabled.is_none()
143                || Self::is_auto_create_exempt_private_table(
144                    &ctx.current_schema(),
145                    &request.table_name,
146                );
147            let target = tables[&request.table_name].as_ref();
148            let backing = physical_tables.get(&request.table_name);
149            let physical = backing.and_then(|name| tables[name].as_ref());
150            if let Some(physical) = physical {
151                ensure!(
152                    physical
153                        .table_info()
154                        .meta
155                        .options
156                        .extra_options
157                        .contains_key(PHYSICAL_TABLE_METADATA_KEY),
158                    InvalidInsertRequestSnafu {
159                        reason: "selected backing table is not a physical metric table"
160                    }
161                );
162            }
163            if metric_engine {
164                ensure!(
165                    backing.is_none() || physical.is_some() || auto_create,
166                    InvalidInsertRequestSnafu {
167                        reason: format!(
168                            "missing physical table and {}",
169                            disabled.unwrap_or_default()
170                        ),
171                    }
172                );
173                let unit = target
174                    .and_then(table_time_index_unit)
175                    .or_else(|| physical.and_then(table_time_index_unit))
176                    .unwrap_or(TimeUnit::Millisecond);
177                if let Some(rows) = &mut request.rows {
178                    convert_rows_time_unit(rows, unit)?;
179                }
180            }
181            let mut columns = if let Some(table) = target {
182                ensure!(
183                    !table
184                        .table_info()
185                        .meta
186                        .options
187                        .extra_options
188                        .contains_key(PHYSICAL_TABLE_METADATA_KEY),
189                    InvalidInsertRequestSnafu {
190                        reason: "direct writes to a physical metric table are forbidden",
191                    }
192                );
193                ensure!(
194                    request_is_native_histogram(&request.rows.as_ref().unwrap().schema)
195                        == table_is_native_histogram(table),
196                    InvalidInsertRequestSnafu {
197                        reason: format!(
198                            "Table `{}` cannot mix native histogram and float sample fields",
199                            request.table_name
200                        ),
201                    }
202                );
203                let mut columns = table_columns(table);
204                if auto_create {
205                    // Ordinary OTLP insertion does not rename columns.
206                    if let Some(alter) = self.get_alter_table_expr_on_demand(
207                        request,
208                        table,
209                        ctx,
210                        metric_engine,
211                        true,
212                        true,
213                    )? && let Some(Kind::AddColumns(add)) = alter.kind
214                    {
215                        for column in add.add_columns {
216                            let def =
217                                column
218                                    .column_def
219                                    .with_context(|| InvalidInsertRequestSnafu {
220                                        reason: "missing column definition",
221                                    })?;
222                            columns.push(column_definition(&def)?);
223                        }
224                    }
225                }
226                columns
227            } else {
228                ensure!(
229                    auto_create,
230                    InvalidInsertRequestSnafu {
231                        reason: format!(
232                            "missing table `{}` and {}",
233                            request.table_name,
234                            disabled.unwrap_or_default()
235                        ),
236                    }
237                );
238                let create_type = if metric_engine {
239                    AutoCreateTableType::Logical(physical_table.to_string())
240                } else {
241                    AutoCreateTableType::Physical
242                };
243                let create =
244                    self.get_create_table_expr_on_demand(request, &create_type, ctx, None)?;
245                if metric_engine {
246                    ensure!(
247                        create
248                            .column_defs
249                            .iter()
250                            .filter(|c| c.semantic_type == SemanticType::Field as i32)
251                            .count()
252                            == 1,
253                        InvalidInsertRequestSnafu {
254                            reason: "logical metrics require exactly one sample field"
255                        }
256                    );
257                }
258                create
259                    .column_defs
260                    .iter()
261                    .map(column_definition)
262                    .collect::<Result<Vec<_>>>()?
263            };
264            let rows = request.rows.as_mut().context(InvalidInsertRequestSnafu {
265                reason: "missing metric rows",
266            })?;
267            if let Some(backing) = backing {
268                // Logical tag nullability follows the shared physical schema.
269                for (column, semantic) in &mut columns {
270                    if *semantic == SemanticType::Tag {
271                        column.set_nullable();
272                    }
273                }
274                validate_physical_columns(rows, &physical_columns[backing.as_str()], auto_create)?;
275            }
276            // Only defaults need values in the planning template.
277            rows.rows.push(Row::default());
278            if let Some(table) = target {
279                if !fillers.contains_key(&request.table_name) {
280                    fillers.insert(
281                        request.table_name.clone(),
282                        ImpureDefaultFiller::new(table.table_info())?,
283                    );
284                }
285                fillers[&request.table_name].fill_rows(rows);
286            }
287            RowValidationPlan::new(
288                input_columns,
289                source_timestamp,
290                std::mem::take(rows),
291                &columns,
292            )
293        };
294        let mut rejections = Vec::with_capacity(batches.len());
295        for request in batches {
296            let num_rows = request
297                .rows
298                .as_ref()
299                .context(InvalidInsertRequestSnafu {
300                    reason: "missing metric rows",
301                })?
302                .rows
303                .len();
304            let plan = match build_plan(request) {
305                Ok(plan) => plan,
306                Err(err) if attributable(&err) => {
307                    rejections.push(vec![Some(err.to_string()); num_rows]);
308                    continue;
309                }
310                Err(err) => return Err(err),
311            };
312            let rows = request.rows.as_mut().context(InvalidInsertRequestSnafu {
313                reason: "missing metric rows",
314            })?;
315            let mut errors = Vec::with_capacity(num_rows);
316            for row in &mut rows.rows {
317                match plan.apply_row(row) {
318                    Ok(()) => errors.push(None),
319                    Err(err) if attributable(&err) => errors.push(Some(err.to_string())),
320                    Err(err) => return Err(err),
321                }
322            }
323            rows.schema = plan.schema;
324            rejections.push(errors);
325        }
326        Ok(MetricInsertPreflight {
327            rejections,
328            physical_tables,
329        })
330    }
331}
332
333fn attributable(err: &error::Error) -> bool {
334    matches!(
335        err,
336        error::Error::InvalidInsertRequest { .. }
337            | error::Error::BuildCreateExprOnInsertion { .. }
338            | error::Error::FindNewColumnsOnInsertion { .. }
339            | error::Error::InvalidColumnDef { .. }
340    )
341}
342
343fn invalid(reason: impl Into<String>) -> error::Error {
344    InvalidInsertRequestSnafu {
345        reason: reason.into(),
346    }
347    .build()
348}
349
350fn column_definition(def: &ColumnDef) -> Result<(DataColumn, SemanticType)> {
351    let column = try_as_column_schema(def).map_err(|err| invalid(err.to_string()))?;
352    let semantic =
353        SemanticType::try_from(def.semantic_type).map_err(|err| invalid(err.to_string()))?;
354    Ok((column, semantic))
355}
356
357fn table_columns(table: &TableRef) -> Vec<(DataColumn, SemanticType)> {
358    let info = table.table_info();
359    let primary_keys: HashSet<_> = info.meta.primary_key_indices.iter().copied().collect();
360    info.meta
361        .schema
362        .column_schemas()
363        .iter()
364        .enumerate()
365        .map(|(index, column)| {
366            let semantic = if column.is_time_index() {
367                SemanticType::Timestamp
368            } else if primary_keys.contains(&index) {
369                SemanticType::Tag
370            } else {
371                SemanticType::Field
372            };
373            (column.clone(), semantic)
374        })
375        .collect()
376}
377
378fn default_physical_columns() -> Vec<(DataColumn, SemanticType)> {
379    use datatypes::prelude::ConcreteDataType;
380    vec![
381        (
382            DataColumn::new(
383                greptime_timestamp(),
384                ConcreteDataType::timestamp_millisecond_datatype(),
385                false,
386            )
387            .with_time_index(true),
388            SemanticType::Timestamp,
389        ),
390        (
391            DataColumn::new(greptime_value(), ConcreteDataType::float64_datatype(), true),
392            SemanticType::Field,
393        ),
394    ]
395}
396
397fn check_definition(
398    source: &ColumnSchema,
399    target: &DataColumn,
400    semantic: SemanticType,
401) -> Result<()> {
402    ensure!(
403        api::helper::is_column_type_value_eq(
404            source.datatype,
405            source.datatype_extension.clone(),
406            &target.data_type
407        ) && source.semantic_type == semantic as i32,
408        InvalidInsertRequestSnafu {
409            reason: format!(
410                "incompatible datatype or semantic type for column `{}`",
411                source.column_name
412            ),
413        }
414    );
415    Ok(())
416}
417
418fn validate_physical_columns(
419    rows: &Rows,
420    columns: &PhysicalColumns,
421    auto_create: bool,
422) -> Result<()> {
423    for source in &rows.schema {
424        if let Some((target, semantic)) = columns.by_name.get(&source.column_name) {
425            check_definition(source, target, *semantic)?;
426        } else {
427            ensure!(
428                auto_create,
429                InvalidInsertRequestSnafu {
430                    reason: format!(
431                        "missing physical column `{}` while auto-DDL is disabled",
432                        source.column_name
433                    )
434                }
435            );
436            ensure!(
437                source.semantic_type == SemanticType::Tag as i32
438                    || (source.semantic_type == SemanticType::Field as i32
439                        && api::helper::is_column_type_value_eq(
440                            source.datatype,
441                            source.datatype_extension.clone(),
442                            native_histogram_value_type()
443                        )),
444                InvalidInsertRequestSnafu {
445                    reason: format!("cannot add physical column `{}`", source.column_name)
446                }
447            );
448        }
449    }
450    for name in &columns.fields {
451        if !rows.schema.iter().any(|c| c.column_name == *name) {
452            let (column, _) = &columns.by_name[name];
453            ensure!(
454                !column.is_default_impure(),
455                InvalidInsertRequestSnafu {
456                    reason: format!("impure default for physical field `{}`", column.name)
457                }
458            );
459            ensure!(
460                column
461                    .create_default()
462                    .map_err(|e| invalid(e.to_string()))?
463                    .is_some(),
464                InvalidInsertRequestSnafu {
465                    reason: format!("missing required physical field `{}`", column.name)
466                }
467            );
468        }
469    }
470    Ok(())
471}
472
473/// Schema decisions are shared; row values are always checked independently.
474struct RowValidationPlan {
475    input_columns: usize,
476    schema: Vec<ColumnSchema>,
477    nullable: Vec<bool>,
478    defaults: Vec<Value>,
479    timestamp: Option<(usize, TimeUnit, TimeUnit)>,
480}
481
482impl RowValidationPlan {
483    fn new(
484        input_columns: usize,
485        source_timestamp: Option<(usize, TimeUnit)>,
486        mut template: Rows,
487        columns: &[(DataColumn, SemanticType)],
488    ) -> Result<Self> {
489        let by_name: AHashMap<_, _> = columns
490            .iter()
491            .map(|(column, semantic)| (column.name.as_str(), (column, *semantic)))
492            .collect();
493        let mut names = HashSet::new();
494        let mut nullable = Vec::with_capacity(columns.len());
495        for source in &template.schema {
496            ensure!(
497                names.insert(source.column_name.as_str()),
498                InvalidInsertRequestSnafu {
499                    reason: format!("duplicate column `{}`", source.column_name),
500                }
501            );
502            let (target, semantic) = by_name
503                .get(source.column_name.as_str())
504                .ok_or_else(|| invalid(format!("unknown column `{}`", source.column_name)))?;
505            check_definition(source, target, *semantic)?;
506            nullable.push(target.is_nullable());
507        }
508        let missing = columns
509            .iter()
510            .filter(|(column, _)| !names.contains(column.name.as_str()))
511            .collect::<Vec<_>>();
512        let mut row = template.rows.pop().context(InvalidInsertRequestSnafu {
513            reason: "missing normalization template",
514        })?;
515        // Resolve omissions before schemas are merged; an explicit null never uses a default.
516        for (column, semantic) in missing {
517            let value = column
518                .create_default()
519                .map_err(|err| invalid(err.to_string()))?
520                .ok_or_else(|| invalid(format!("missing required column `{}`", column.name)))?;
521            let (datatype, datatype_extension) =
522                ColumnDataTypeWrapper::try_from(column.data_type.clone())
523                    .map_err(|err| invalid(err.to_string()))?
524                    .into_parts();
525            template.schema.push(ColumnSchema {
526                column_name: column.name.clone(),
527                datatype: datatype as i32,
528                semantic_type: *semantic as i32,
529                datatype_extension,
530                options: None,
531            });
532            row.values.push(to_grpc_value(value));
533            nullable.push(column.is_nullable());
534        }
535        ensure!(
536            row.values.len() + input_columns == template.schema.len(),
537            InvalidInsertRequestSnafu {
538                reason: "invalid normalization template"
539            }
540        );
541        let defaults = row.values;
542        for ((source, nullable), value) in template.schema[input_columns..]
543            .iter()
544            .zip(&nullable[input_columns..])
545            .zip(&defaults)
546        {
547            validate_value(value, source, *nullable)?;
548        }
549        let timestamp = source_timestamp.and_then(|(index, source)| {
550            let target = ColumnDataType::try_from(template.schema[index].datatype)
551                .ok()
552                .and_then(api::helper::timestamp_unit)?;
553            (source != target).then_some((index, source, target))
554        });
555        Ok(Self {
556            input_columns,
557            schema: template.schema,
558            nullable,
559            defaults,
560            timestamp,
561        })
562    }
563
564    fn apply_row(&self, row: &mut Row) -> Result<()> {
565        ensure!(
566            row.values.len() == self.input_columns,
567            InvalidInsertRequestSnafu {
568                reason: "column count mismatch",
569            }
570        );
571        if let Some((index, source, target)) = self.timestamp
572            && let Some(value) = row.values[index].value_data.take()
573        {
574            row.values[index].value_data =
575                convert_timestamp_value_data(value, source, target, index)?;
576        }
577        for ((source, nullable), value) in self.schema.iter().zip(&self.nullable).zip(&row.values) {
578            validate_value(value, source, *nullable)?;
579        }
580        row.values.extend_from_slice(&self.defaults);
581        Ok(())
582    }
583}
584
585fn timestamp_column(schema: &[ColumnSchema]) -> Option<(usize, TimeUnit)> {
586    let index = schema
587        .iter()
588        .position(|column| column.semantic_type == SemanticType::Timestamp as i32)?;
589    let unit = ColumnDataType::try_from(schema[index].datatype)
590        .ok()
591        .and_then(api::helper::timestamp_unit)?;
592    Some((index, unit))
593}
594
595fn validate_value(value: &Value, source: &ColumnSchema, nullable: bool) -> Result<()> {
596    ensure!(
597        value_matches_type(value, source.datatype, source.datatype_extension.as_ref()),
598        InvalidInsertRequestSnafu {
599            reason: format!("value datatype differs for `{}`", source.column_name)
600        }
601    );
602    ensure!(
603        value.value_data.is_some() || nullable,
604        InvalidInsertRequestSnafu {
605            reason: format!(
606                "null supplied for non-nullable column `{}`",
607                source.column_name
608            ),
609        }
610    );
611    Ok(())
612}
613
614#[cfg(test)]
615fn validate_and_fill_rows(rows: &mut Rows, columns: &[(DataColumn, SemanticType)]) -> Result<()> {
616    let template = Rows {
617        schema: rows.schema.clone(),
618        rows: vec![Row::default()],
619    };
620    let plan = RowValidationPlan::new(
621        rows.schema.len(),
622        timestamp_column(&rows.schema),
623        template,
624        columns,
625    )?;
626    for row in &mut rows.rows {
627        plan.apply_row(row)?;
628    }
629    rows.schema = plan.schema;
630    Ok(())
631}
632
633#[cfg(test)]
634mod tests {
635    use api::v1::value::ValueData;
636    use api::v1::{ColumnDataType, Row};
637    use datatypes::prelude::ConcreteDataType;
638    use datatypes::schema::ColumnDefaultConstraint;
639    use datatypes::value::Value as DataValue;
640
641    use super::*;
642
643    #[test]
644    fn test_reused_plan_checks_each_value_and_timestamp() {
645        let schema = vec![api::v1::helper::time_index_column_schema(
646            greptime_timestamp(),
647            ColumnDataType::TimestampSecond,
648        )];
649        let mut template = Rows {
650            schema: schema.clone(),
651            rows: Vec::new(),
652        };
653        convert_rows_time_unit(&mut template, TimeUnit::Nanosecond).unwrap();
654        let columns = vec![(
655            DataColumn::new(
656                greptime_timestamp(),
657                ConcreteDataType::timestamp_nanosecond_datatype(),
658                false,
659            )
660            .with_time_index(true),
661            SemanticType::Timestamp,
662        )];
663        template.rows.push(Row::default());
664        let plan =
665            RowValidationPlan::new(schema.len(), timestamp_column(&schema), template, &columns)
666                .unwrap();
667        for (value, valid) in [
668            (Some(ValueData::TimestampSecondValue(1)), true),
669            (Some(ValueData::StringValue("invalid".into())), false),
670            (None, false),
671            (Some(ValueData::TimestampSecondValue(i64::MAX)), false),
672            (Some(ValueData::TimestampSecondValue(2)), true),
673        ] {
674            let mut rows = Rows {
675                schema: schema.clone(),
676                rows: vec![Row {
677                    values: vec![Value { value_data: value }],
678                }],
679            };
680            assert_eq!(plan.apply_row(&mut rows.rows[0]).is_ok(), valid);
681            if valid {
682                assert_eq!(
683                    plan.schema[0].datatype,
684                    ColumnDataType::TimestampNanosecond as i32
685                );
686                assert!(matches!(
687                    rows.rows[0].values[0].value_data,
688                    Some(ValueData::TimestampNanosecondValue(
689                        1_000_000_000 | 2_000_000_000
690                    ))
691                ));
692            }
693        }
694    }
695
696    #[test]
697    fn test_preflight_omissions_defaults_and_explicit_nulls() {
698        let mut columns = default_physical_columns();
699        columns.push((
700            DataColumn::new("host", ConcreteDataType::string_datatype(), false)
701                .with_default_constraint(Some(ColumnDefaultConstraint::Value(DataValue::String(
702                    "unknown".into(),
703                ))))
704                .unwrap(),
705            SemanticType::Tag,
706        ));
707        let row = || Rows {
708            schema: vec![ColumnSchema {
709                column_name: greptime_timestamp().into(),
710                datatype: ColumnDataType::TimestampMillisecond as i32,
711                semantic_type: SemanticType::Timestamp as i32,
712                ..Default::default()
713            }],
714            rows: vec![Row {
715                values: vec![Value {
716                    value_data: Some(ValueData::TimestampMillisecondValue(1)),
717                }],
718            }],
719        };
720        let mut omitted = row();
721        validate_and_fill_rows(&mut omitted, &columns).unwrap();
722        assert_eq!(omitted.schema[2].column_name, "host");
723        assert_eq!(
724            omitted.rows[0].values[2].value_data,
725            Some(ValueData::StringValue("unknown".into()))
726        );
727        omitted.rows[0].values[2] = Value::default();
728        assert!(validate_and_fill_rows(&mut omitted, &columns).is_err());
729        columns[2].0 = DataColumn::new("host", ConcreteDataType::string_datatype(), false);
730        assert!(validate_and_fill_rows(&mut row(), &columns).is_err());
731        columns[2].0.set_nullable();
732        let mut nullable = row();
733        validate_and_fill_rows(&mut nullable, &columns).unwrap();
734        assert_eq!(nullable.rows[0].values[2].value_data, None);
735    }
736
737    #[test]
738    fn test_preflight_existing_columns_and_physical_definitions() {
739        let columns = default_physical_columns();
740        let mut rows = Rows {
741            schema: vec![ColumnSchema {
742                column_name: greptime_timestamp().into(),
743                datatype: ColumnDataType::TimestampMillisecond as i32,
744                semantic_type: SemanticType::Timestamp as i32,
745                ..Default::default()
746            }],
747            rows: vec![Row {
748                values: vec![Value {
749                    value_data: Some(ValueData::TimestampMillisecondValue(1)),
750                }],
751            }],
752        };
753        validate_and_fill_rows(&mut rows, &columns).unwrap();
754        for case in 0..3 {
755            let mut invalid = rows.clone();
756            match case {
757                0 => invalid.schema[0].semantic_type = SemanticType::Tag as i32,
758                1 => invalid.schema[0].datatype = ColumnDataType::TimestampNanosecond as i32,
759                _ => {
760                    invalid.rows[0].values[0].value_data =
761                        Some(ValueData::StringValue("bad".into()))
762                }
763            }
764            assert!(validate_and_fill_rows(&mut invalid, &columns).is_err());
765        }
766        let columns = PhysicalColumns::new(columns);
767        rows.schema[1].column_name = "new_field".into();
768        assert!(validate_physical_columns(&rows, &columns, true).is_err());
769        rows.schema[1].semantic_type = SemanticType::Tag as i32;
770        rows.schema[1].datatype = ColumnDataType::String as i32;
771        assert!(validate_physical_columns(&rows, &columns, false).is_err());
772        assert!(validate_physical_columns(&rows, &columns, true).is_ok());
773    }
774
775    #[test]
776    fn test_preflight_shared_physical_field_defaults() {
777        let rows = Rows {
778            schema: vec![api::v1::helper::time_index_column_schema(
779                greptime_timestamp(),
780                ColumnDataType::TimestampMillisecond,
781            )],
782            ..Default::default()
783        };
784        let required = DataColumn::new("required", ConcreteDataType::float64_datatype(), false);
785        let defaulted = required
786            .clone()
787            .with_default_constraint(Some(ColumnDefaultConstraint::Value(DataValue::Float64(
788                1.0.into(),
789            ))))
790            .unwrap();
791        let impure = DataColumn::new(
792            "impure",
793            ConcreteDataType::timestamp_millisecond_datatype(),
794            false,
795        )
796        .with_default_constraint(Some(ColumnDefaultConstraint::Function(
797            "current_timestamp()".into(),
798        )))
799        .unwrap();
800        for (field, error) in [
801            (required, Some("missing required physical field `required`")),
802            (defaulted, None),
803            (impure, Some("impure default for physical field `impure`")),
804        ] {
805            let mut columns = default_physical_columns();
806            columns.extend((0..1024).map(|index| {
807                (
808                    DataColumn::new(
809                        format!("unrelated_tag_{index}"),
810                        ConcreteDataType::string_datatype(),
811                        true,
812                    ),
813                    SemanticType::Tag,
814                )
815            }));
816            columns.push((field, SemanticType::Field));
817            let columns = PhysicalColumns::new(columns);
818            for auto_create in [false, true] {
819                let result = validate_physical_columns(&rows, &columns, auto_create);
820                if let Some(error) = error {
821                    assert!(result.unwrap_err().to_string().contains(error));
822                } else {
823                    result.unwrap();
824                }
825            }
826        }
827    }
828}
operator/insert/otlp.rs

operator/insert/
otlp.rs
