Skip to main content

table/
metadata.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::{HashMap, HashSet};
16use std::sync::Arc;
17
18use chrono::{DateTime, Utc};
19use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
20use common_macro::ToMetaBuilder;
21use common_query::AddColumnLocation;
22use datafusion_expr::TableProviderFilterPushDown;
23use datatypes::error::time_index_not_widening_error;
24pub use datatypes::error::{Error as ConvertError, Result as ConvertResult};
25use datatypes::schema::{
26    ColumnSchema, FulltextOptions, Schema, SchemaBuilder, SchemaRef, SkippingIndexOptions,
27};
28use derive_builder::Builder;
29use serde::{Deserialize, Deserializer, Serialize};
30use snafu::{OptionExt, ResultExt, ensure};
31use store_api::metric_engine_consts::PHYSICAL_TABLE_METADATA_KEY;
32use store_api::mito_engine_options::{
33    APPEND_MODE_KEY, AUTO_FLUSH_INTERVAL_KEY, COMPACTION_TYPE, COMPACTION_TYPE_TWCS,
34    MAX_ROW_GROUP_ROW_COUNT, MERGE_MODE_KEY, PRESERVE_ROW_SEQUENCE, SKIP_WAL_KEY, SST_FORMAT_KEY,
35};
36use store_api::region_request::{SetRegionOption, UnsetRegionOption};
37use store_api::storage::{ColumnDescriptor, ColumnDescriptorBuilder, ColumnId};
38
39use crate::error::{self, Result};
40use crate::requests::{
41    AddColumnRequest, AlterKind, AnnotationContext, AnnotationFamily, AnnotationValidationError,
42    ModifyColumnTypeRequest, REPARTITION_COLUMN_HINT_KEY, SetDefaultRequest, SetIndexOption,
43    TableOptions, UnsetIndexOption, has_stable_string_form, parse_entity_columns,
44    parse_entity_option_key, validate_and_normalize_annotation,
45};
46use crate::table_reference::TableReference;
47
48pub type TableId = u32;
49pub type TableVersion = u64;
50
51/// Indicates whether and how a filter expression can be handled by a
52/// Table for table scans.
53#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
54pub enum FilterPushDownType {
55    /// The expression cannot be used by the provider.
56    Unsupported,
57    /// The expression can be used to help minimise the data retrieved,
58    /// but the provider cannot guarantee that all returned tuples
59    /// satisfy the filter. The Filter plan node containing this expression
60    /// will be preserved.
61    Inexact,
62    /// The provider guarantees that all returned data satisfies this
63    /// filter expression. The Filter plan node containing this expression
64    /// will be removed.
65    Exact,
66}
67
68impl From<TableProviderFilterPushDown> for FilterPushDownType {
69    fn from(value: TableProviderFilterPushDown) -> Self {
70        match value {
71            TableProviderFilterPushDown::Unsupported => FilterPushDownType::Unsupported,
72            TableProviderFilterPushDown::Inexact => FilterPushDownType::Inexact,
73            TableProviderFilterPushDown::Exact => FilterPushDownType::Exact,
74        }
75    }
76}
77
78impl From<FilterPushDownType> for TableProviderFilterPushDown {
79    fn from(value: FilterPushDownType) -> Self {
80        match value {
81            FilterPushDownType::Unsupported => TableProviderFilterPushDown::Unsupported,
82            FilterPushDownType::Inexact => TableProviderFilterPushDown::Inexact,
83            FilterPushDownType::Exact => TableProviderFilterPushDown::Exact,
84        }
85    }
86}
87
88/// Indicates the type of this table for metadata/catalog purposes.
89#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
90pub enum TableType {
91    /// An ordinary physical table.
92    Base,
93    /// A non-materialised table that itself uses a query internally to provide data.
94    View,
95    /// A transient table.
96    Temporary,
97}
98
99impl std::fmt::Display for TableType {
100    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
101        match self {
102            TableType::Base => f.write_str("BASE TABLE"),
103            TableType::Temporary => f.write_str("TEMPORARY"),
104            TableType::View => f.write_str("VIEW"),
105        }
106    }
107}
108
109impl From<TableType> for datafusion::datasource::TableType {
110    fn from(t: TableType) -> datafusion::datasource::TableType {
111        match t {
112            TableType::Base => datafusion::datasource::TableType::Base,
113            TableType::View => datafusion::datasource::TableType::View,
114            TableType::Temporary => datafusion::datasource::TableType::Temporary,
115        }
116    }
117}
118
119/// Identifier of the table.
120#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Default)]
121pub struct TableIdent {
122    /// Unique id of this table.
123    pub table_id: TableId,
124    /// Version of the table, bumped when metadata (such as schema) of the table
125    /// being changed.
126    pub version: TableVersion,
127}
128
129/// The table metadata.
130///
131/// Note: if you add new fields to this struct, please ensure 'new_meta_builder' function works.
132#[derive(Clone, Debug, Builder, PartialEq, Eq, ToMetaBuilder, Serialize)]
133#[builder(pattern = "mutable", custom_constructor)]
134pub struct TableMeta {
135    pub schema: SchemaRef,
136    /// The indices of columns in primary key. Note that the index of timestamp column
137    /// is not included in these indices.
138    pub primary_key_indices: Vec<usize>,
139    #[builder(default = "self.default_value_indices()?")]
140    pub value_indices: Vec<usize>,
141    #[builder(default, setter(into))]
142    pub engine: String,
143    pub next_column_id: ColumnId,
144    /// Table options.
145    #[builder(default)]
146    pub options: TableOptions,
147    #[builder(default = "Utc::now()")]
148    pub created_on: DateTime<Utc>,
149    #[builder(default = "self.default_updated_on()")]
150    pub updated_on: DateTime<Utc>,
151    #[builder(default = "Vec::new()")]
152    pub partition_key_indices: Vec<usize>,
153    #[builder(default = "Vec::new()")]
154    pub column_ids: Vec<ColumnId>,
155}
156
157impl TableMeta {
158    pub fn empty() -> Self {
159        Self {
160            schema: Arc::new(Schema::new(vec![])),
161            primary_key_indices: vec![],
162            value_indices: vec![],
163            engine: "".to_string(),
164            next_column_id: 0,
165            options: TableOptions::default(),
166            created_on: Utc::now(),
167            updated_on: Utc::now(),
168            partition_key_indices: vec![],
169            column_ids: vec![],
170        }
171    }
172}
173
174impl<'de> Deserialize<'de> for TableMeta {
175    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
176    where
177        D: Deserializer<'de>,
178    {
179        #[derive(Deserialize)]
180        struct RawTableMeta {
181            schema: SchemaRef,
182            primary_key_indices: Vec<usize>,
183            value_indices: Vec<usize>,
184            engine: String,
185            next_column_id: ColumnId,
186            options: TableOptions,
187            created_on: DateTime<Utc>,
188            updated_on: Option<DateTime<Utc>>,
189            #[serde(default)]
190            partition_key_indices: Vec<usize>,
191            #[serde(default)]
192            column_ids: Vec<ColumnId>,
193        }
194
195        let RawTableMeta {
196            schema,
197            primary_key_indices,
198            value_indices,
199            engine,
200            next_column_id,
201            options,
202            created_on,
203            updated_on,
204            partition_key_indices,
205            column_ids,
206        } = RawTableMeta::deserialize(deserializer)?;
207
208        Ok(Self {
209            schema,
210            primary_key_indices,
211            value_indices,
212            engine,
213            next_column_id,
214            options,
215            created_on,
216            updated_on: updated_on.unwrap_or(created_on),
217            partition_key_indices,
218            column_ids,
219        })
220    }
221}
222
223impl TableMetaBuilder {
224    /// Note: Please always use [new_meta_builder] to create new [TableMetaBuilder].
225    #[cfg(any(test, feature = "testing"))]
226    pub fn empty() -> Self {
227        Self {
228            schema: None,
229            primary_key_indices: None,
230            value_indices: None,
231            engine: None,
232            next_column_id: None,
233            options: None,
234            created_on: None,
235            updated_on: None,
236            partition_key_indices: None,
237            column_ids: None,
238        }
239    }
240}
241
242impl TableMetaBuilder {
243    fn default_value_indices(&self) -> std::result::Result<Vec<usize>, String> {
244        match (&self.primary_key_indices, &self.schema) {
245            (Some(v), Some(schema)) => {
246                let column_schemas = schema.column_schemas();
247                Ok((0..column_schemas.len())
248                    .filter(|idx| !v.contains(idx))
249                    .collect())
250            }
251            _ => Err("Missing primary_key_indices or schema to create value_indices".to_string()),
252        }
253    }
254
255    fn default_updated_on(&self) -> DateTime<Utc> {
256        self.created_on.unwrap_or_default()
257    }
258
259    pub fn new_external_table() -> Self {
260        Self {
261            schema: None,
262            primary_key_indices: Some(Vec::new()),
263            value_indices: Some(Vec::new()),
264            engine: None,
265            next_column_id: Some(0),
266            options: None,
267            created_on: None,
268            updated_on: None,
269            partition_key_indices: None,
270            column_ids: None,
271        }
272    }
273}
274
275/// The result after splitting requests by column location info.
276struct SplitResult<'a> {
277    /// column requests should be added at first place.
278    columns_at_first: Vec<&'a AddColumnRequest>,
279    /// column requests should be added after already exist columns.
280    columns_at_after: HashMap<String, Vec<&'a AddColumnRequest>>,
281    /// column requests should be added at last place.
282    columns_at_last: Vec<&'a AddColumnRequest>,
283    /// all column names should be added.
284    column_names: Vec<String>,
285}
286
287impl TableMeta {
288    pub fn row_key_column_names(&self) -> impl Iterator<Item = &String> {
289        let columns_schemas = &self.schema.column_schemas();
290        self.primary_key_indices
291            .iter()
292            .map(|idx| &columns_schemas[*idx].name)
293    }
294
295    pub fn field_column_names(&self) -> impl Iterator<Item = &String> {
296        // `value_indices` is wrong under distributed mode. Use the logic copied from DESC TABLE
297        let columns_schemas = self.schema.column_schemas();
298        let primary_key_indices = &self.primary_key_indices;
299        columns_schemas
300            .iter()
301            .enumerate()
302            .filter(|(i, cs)| !primary_key_indices.contains(i) && !cs.is_time_index())
303            .map(|(_, cs)| &cs.name)
304    }
305
306    pub fn partition_column_names(&self) -> impl Iterator<Item = &String> {
307        let columns_schemas = &self.schema.column_schemas();
308        self.partition_key_indices
309            .iter()
310            .map(|idx| &columns_schemas[*idx].name)
311    }
312
313    pub fn partition_columns(&self) -> impl Iterator<Item = &ColumnSchema> {
314        self.partition_key_indices
315            .iter()
316            .map(|idx| &self.schema.column_schemas()[*idx])
317    }
318
319    /// Returns the new [TableMetaBuilder] after applying given `alter_kind`.
320    ///
321    /// The returned builder would derive the next column id of this meta.
322    pub fn builder_with_alter_kind(
323        &self,
324        table_name: &str,
325        alter_kind: &AlterKind,
326    ) -> Result<TableMetaBuilder> {
327        let mut builder = match alter_kind {
328            AlterKind::AddColumns { columns } => self.add_columns(table_name, columns),
329            AlterKind::DropColumns { names } => self.remove_columns(table_name, names),
330            AlterKind::ModifyColumnTypes { columns } => {
331                self.modify_column_types(table_name, columns)
332            }
333            // No need to rebuild table meta when renaming tables.
334            AlterKind::RenameTable { .. } => Ok(self.new_meta_builder()),
335            AlterKind::SetTableOptions { options } => self.set_table_options(options),
336            AlterKind::UnsetTableOptions { keys } => self.unset_table_options(keys),
337            AlterKind::SetAnnotations { family, options } => {
338                self.set_annotations(table_name, *family, options)
339            }
340            AlterKind::UnsetAnnotations { family, keys } => {
341                self.unset_annotations(table_name, *family, keys)
342            }
343            AlterKind::SetIndexes { options } => self.set_indexes(table_name, options),
344            AlterKind::UnsetIndexes { options } => self.unset_indexes(table_name, options),
345            AlterKind::DropDefaults { names } => self.drop_defaults(table_name, names),
346            AlterKind::SetDefaults { defaults } => self.set_defaults(table_name, defaults),
347        }?;
348        let _ = builder.updated_on(Utc::now());
349        Ok(builder)
350    }
351
352    /// Creates a [TableMetaBuilder] with modified table options.
353    fn set_table_options(&self, requests: &[SetRegionOption]) -> Result<TableMetaBuilder> {
354        let mut new_options = self.options.clone();
355
356        for request in requests {
357            match request {
358                SetRegionOption::WriteBufferSize(new_write_buffer_size) => {
359                    new_options.write_buffer_size = *new_write_buffer_size;
360                }
361                SetRegionOption::Ttl(new_ttl) => {
362                    new_options.ttl = *new_ttl;
363                }
364                SetRegionOption::Twsc(key, value) => {
365                    if !value.is_empty() {
366                        new_options.extra_options.insert(key.clone(), value.clone());
367                        // Ensure node restart correctly.
368                        new_options.extra_options.insert(
369                            COMPACTION_TYPE.to_string(),
370                            COMPACTION_TYPE_TWCS.to_string(),
371                        );
372                    } else {
373                        // Invalidate the previous change option if an empty value has been set.
374                        new_options.extra_options.remove(key.as_str());
375                    }
376                }
377                SetRegionOption::Format(value) => {
378                    new_options
379                        .extra_options
380                        .insert(SST_FORMAT_KEY.to_string(), value.clone());
381                }
382                SetRegionOption::AppendMode(value) => {
383                    new_options
384                        .extra_options
385                        .insert(APPEND_MODE_KEY.to_string(), value.to_string());
386                    if *value {
387                        new_options.extra_options.remove(MERGE_MODE_KEY);
388                    }
389                }
390                SetRegionOption::AutoFlushInterval(new_interval) => {
391                    if let Some(interval) = new_interval {
392                        new_options.extra_options.insert(
393                            AUTO_FLUSH_INTERVAL_KEY.to_string(),
394                            humantime::format_duration(*interval).to_string(),
395                        );
396                    } else {
397                        new_options.extra_options.remove(AUTO_FLUSH_INTERVAL_KEY);
398                    }
399                }
400                SetRegionOption::MaxRowGroupRowCount(row_count) => {
401                    if let Some(row_count) = row_count {
402                        new_options
403                            .extra_options
404                            .insert(MAX_ROW_GROUP_ROW_COUNT.to_string(), row_count.to_string());
405                    } else {
406                        new_options.extra_options.remove(MAX_ROW_GROUP_ROW_COUNT);
407                    }
408                }
409                SetRegionOption::PreserveRowSequence(preserve) => {
410                    if *preserve {
411                        new_options
412                            .extra_options
413                            .insert(PRESERVE_ROW_SEQUENCE.to_string(), preserve.to_string());
414                    } else {
415                        new_options.extra_options.remove(PRESERVE_ROW_SEQUENCE);
416                    }
417                }
418                SetRegionOption::SkipWal => {
419                    new_options.skip_wal = true;
420                    // Keep the explicit table option so it remains distinguishable
421                    // from a value inherited from the schema.
422                    new_options
423                        .extra_options
424                        .insert(SKIP_WAL_KEY.to_string(), true.to_string());
425                }
426            }
427        }
428        let mut builder = self.new_meta_builder();
429        builder.options(new_options);
430
431        Ok(builder)
432    }
433
434    fn unset_table_options(&self, requests: &[UnsetRegionOption]) -> Result<TableMetaBuilder> {
435        let requests = requests.iter().map(Into::into).collect::<Vec<_>>();
436        self.set_table_options(&requests)
437    }
438
439    /// Applies an annotation SET. Validation lives here, on the mutation
440    /// path, so it runs both at frontend verification and again inside the
441    /// alter procedure's prepare step — under the table lock, against fresh
442    /// metadata.
443    fn set_annotations(
444        &self,
445        table_name: &str,
446        family: AnnotationFamily,
447        options: &[(String, String)],
448    ) -> Result<TableMetaBuilder> {
449        ensure!(
450            !family.requires_single_key() || options.len() == 1,
451            error::InvalidAlterRequestSnafu {
452                table: table_name,
453                err: family.mixed_batch_error(),
454            }
455        );
456        let cx = AnnotationContext {
457            schema: &self.schema,
458            partition_key_indices: &self.partition_key_indices,
459        };
460        let mut new_options = self.options.clone();
461        for (key, value) in options {
462            ensure!(
463                AnnotationFamily::of_key(key) == Some(family),
464                error::InvalidAlterRequestSnafu {
465                    table: table_name,
466                    err: format!(
467                        "`{key}` is outside the `{}` annotation namespace",
468                        family.namespace()
469                    ),
470                }
471            );
472            let checked = validate_and_normalize_annotation(family, &cx, key, value).map_err(
473                |e| match e {
474                    AnnotationValidationError::ColumnNotFound { column } => {
475                        error::ColumnNotExistsSnafu {
476                            column_name: column,
477                            table_name,
478                        }
479                        .build()
480                    }
481                    other => error::InvalidAlterRequestSnafu {
482                        table: table_name,
483                        err: other.to_string(),
484                    }
485                    .build(),
486                },
487            )?;
488            new_options.extra_options.insert(key.clone(), checked);
489        }
490        let mut builder = self.new_meta_builder();
491        builder.options(new_options);
492        Ok(builder)
493    }
494
495    /// Applies an annotation UNSET. Deliberately lenient inside the family's
496    /// namespace: keys this version does not recognise may still be removed,
497    /// so options left behind by other versions can be cleaned up.
498    fn unset_annotations(
499        &self,
500        table_name: &str,
501        family: AnnotationFamily,
502        keys: &[String],
503    ) -> Result<TableMetaBuilder> {
504        ensure!(
505            !family.requires_single_key() || keys.len() == 1,
506            error::InvalidAlterRequestSnafu {
507                table: table_name,
508                err: family.mixed_batch_error(),
509            }
510        );
511        let mut new_options = self.options.clone();
512        for key in keys {
513            ensure!(
514                AnnotationFamily::of_key(key) == Some(family),
515                error::InvalidAlterRequestSnafu {
516                    table: table_name,
517                    err: format!(
518                        "`{key}` is outside the `{}` annotation namespace",
519                        family.namespace()
520                    ),
521                }
522            );
523            new_options.extra_options.remove(key);
524        }
525        let mut builder = self.new_meta_builder();
526        builder.options(new_options);
527        Ok(builder)
528    }
529
530    fn set_indexes(
531        &self,
532        table_name: &str,
533        requests: &[SetIndexOption],
534    ) -> Result<TableMetaBuilder> {
535        let table_schema = &self.schema;
536        let mut set_index_options: HashMap<&str, Vec<_>> = HashMap::new();
537        for request in requests {
538            let column_name = request.column_name();
539            table_schema
540                .column_index_by_name(column_name)
541                .with_context(|| error::ColumnNotExistsSnafu {
542                    column_name,
543                    table_name,
544                })?;
545            set_index_options
546                .entry(column_name)
547                .or_default()
548                .push(request);
549        }
550
551        let mut meta_builder = self.new_meta_builder();
552        let mut columns: Vec<_> = Vec::with_capacity(table_schema.column_schemas().len());
553        for mut column in table_schema.column_schemas().iter().cloned() {
554            if let Some(request) = set_index_options.get(column.name.as_str()) {
555                for request in request {
556                    self.set_index(&mut column, request)?;
557                }
558            }
559            columns.push(column);
560        }
561
562        let mut builder = SchemaBuilder::try_from_columns(columns)
563            .with_context(|_| error::SchemaBuildSnafu {
564                msg: format!("Failed to convert column schemas into schema for table {table_name}"),
565            })?
566            .version(table_schema.version() + 1);
567
568        for (k, v) in table_schema.metadata().iter() {
569            builder = builder.add_metadata(k, v);
570        }
571
572        let new_schema = builder.build().with_context(|_| {
573            let column_names = requests
574                .iter()
575                .map(|request| request.column_name())
576                .collect::<Vec<_>>();
577            error::SchemaBuildSnafu {
578                msg: format!(
579                    "Table {table_name} cannot set index options with columns {column_names:?}",
580                ),
581            }
582        })?;
583        let _ = meta_builder
584            .schema(Arc::new(new_schema))
585            .primary_key_indices(self.primary_key_indices.clone());
586
587        Ok(meta_builder)
588    }
589
590    fn unset_indexes(
591        &self,
592        table_name: &str,
593        requests: &[UnsetIndexOption],
594    ) -> Result<TableMetaBuilder> {
595        let table_schema = &self.schema;
596        let mut set_index_options: HashMap<&str, Vec<_>> = HashMap::new();
597        for request in requests {
598            let column_name = request.column_name();
599            table_schema
600                .column_index_by_name(column_name)
601                .with_context(|| error::ColumnNotExistsSnafu {
602                    column_name,
603                    table_name,
604                })?;
605            set_index_options
606                .entry(column_name)
607                .or_default()
608                .push(request);
609        }
610
611        let mut meta_builder = self.new_meta_builder();
612        let mut columns: Vec<_> = Vec::with_capacity(table_schema.column_schemas().len());
613        for mut column in table_schema.column_schemas().iter().cloned() {
614            if let Some(request) = set_index_options.get(column.name.as_str()) {
615                for request in request {
616                    self.unset_index(&mut column, request)?;
617                }
618            }
619            columns.push(column);
620        }
621
622        let mut builder = SchemaBuilder::try_from_columns(columns)
623            .with_context(|_| error::SchemaBuildSnafu {
624                msg: format!("Failed to convert column schemas into schema for table {table_name}"),
625            })?
626            .version(table_schema.version() + 1);
627
628        for (k, v) in table_schema.metadata().iter() {
629            builder = builder.add_metadata(k, v);
630        }
631
632        let new_schema = builder.build().with_context(|_| {
633            let column_names = requests
634                .iter()
635                .map(|request| request.column_name())
636                .collect::<Vec<_>>();
637            error::SchemaBuildSnafu {
638                msg: format!(
639                    "Table {table_name} cannot set index options with columns {column_names:?}",
640                ),
641            }
642        })?;
643        let _ = meta_builder
644            .schema(Arc::new(new_schema))
645            .primary_key_indices(self.primary_key_indices.clone());
646
647        Ok(meta_builder)
648    }
649
650    fn set_index(&self, column_schema: &mut ColumnSchema, request: &SetIndexOption) -> Result<()> {
651        match request {
652            SetIndexOption::Fulltext {
653                column_name,
654                options,
655            } => {
656                ensure!(
657                    column_schema.data_type.is_string(),
658                    error::InvalidColumnOptionSnafu {
659                        column_name,
660                        msg: "FULLTEXT index only supports string type",
661                    }
662                );
663
664                let current_fulltext_options = column_schema
665                    .fulltext_options()
666                    .context(error::SetFulltextOptionsSnafu { column_name })?;
667                set_column_fulltext_options(
668                    column_schema,
669                    column_name,
670                    options,
671                    current_fulltext_options,
672                )?;
673            }
674            SetIndexOption::Inverted { column_name } => {
675                debug_assert_eq!(column_schema.name, *column_name);
676                column_schema.set_inverted_index(true);
677            }
678            SetIndexOption::Skipping {
679                column_name,
680                options,
681            } => {
682                set_column_skipping_index_options(column_schema, column_name, options)?;
683            }
684        }
685
686        Ok(())
687    }
688
689    fn unset_index(
690        &self,
691        column_schema: &mut ColumnSchema,
692        request: &UnsetIndexOption,
693    ) -> Result<()> {
694        match request {
695            UnsetIndexOption::Fulltext { column_name } => {
696                let current_fulltext_options = column_schema
697                    .fulltext_options()
698                    .context(error::SetFulltextOptionsSnafu { column_name })?;
699                unset_column_fulltext_options(
700                    column_schema,
701                    column_name,
702                    current_fulltext_options.clone(),
703                )?
704            }
705            UnsetIndexOption::Inverted { .. } => {
706                column_schema.set_inverted_index(false);
707            }
708            UnsetIndexOption::Skipping { column_name } => {
709                unset_column_skipping_index_options(column_schema, column_name)?;
710            }
711        }
712
713        Ok(())
714    }
715
716    // TODO(yingwen): Remove this.
717    /// Allocate a new column for the table.
718    ///
719    /// This method would bump the `next_column_id` of the meta.
720    pub fn alloc_new_column(
721        &mut self,
722        table_name: &str,
723        new_column: &ColumnSchema,
724    ) -> Result<ColumnDescriptor> {
725        let desc = ColumnDescriptorBuilder::new(
726            self.next_column_id as ColumnId,
727            &new_column.name,
728            new_column.data_type.clone(),
729        )
730        .is_nullable(new_column.is_nullable())
731        .default_constraint(new_column.default_constraint().cloned())
732        .build()
733        .context(error::BuildColumnDescriptorSnafu {
734            table_name,
735            column_name: &new_column.name,
736        })?;
737
738        // Bump next column id.
739        self.next_column_id += 1;
740
741        Ok(desc)
742    }
743
744    /// Create a [`TableMetaBuilder`] from the current TableMeta.
745    fn new_meta_builder(&self) -> TableMetaBuilder {
746        let mut builder = TableMetaBuilder::from(self);
747        // Manually remove value_indices.
748        builder.value_indices = None;
749        builder
750    }
751
752    // TODO(yingwen): Tests add if not exists.
753    fn add_columns(
754        &self,
755        table_name: &str,
756        requests: &[AddColumnRequest],
757    ) -> Result<TableMetaBuilder> {
758        let table_schema = &self.schema;
759        let mut meta_builder = self.new_meta_builder();
760        let original_primary_key_indices: HashSet<&usize> =
761            self.primary_key_indices.iter().collect();
762
763        let mut names = HashSet::with_capacity(requests.len());
764        let mut new_columns = Vec::with_capacity(requests.len());
765        for col_to_add in requests {
766            if let Some(column_schema) =
767                table_schema.column_schema_by_name(&col_to_add.column_schema.name)
768            {
769                // If the column already exists.
770                ensure!(
771                    col_to_add.add_if_not_exists,
772                    error::ColumnExistsSnafu {
773                        table_name,
774                        column_name: &col_to_add.column_schema.name
775                    },
776                );
777
778                // Checks if the type is the same
779                ensure!(
780                    column_schema.data_type == col_to_add.column_schema.data_type,
781                    error::InvalidAlterRequestSnafu {
782                        table: table_name,
783                        err: format!(
784                            "column {} already exists with different type {:?}",
785                            col_to_add.column_schema.name, column_schema.data_type,
786                        ),
787                    }
788                );
789            } else {
790                // A new column.
791                // Ensures we only add a column once.
792                ensure!(
793                    names.insert(&col_to_add.column_schema.name),
794                    error::InvalidAlterRequestSnafu {
795                        table: table_name,
796                        err: format!(
797                            "add column {} more than once",
798                            col_to_add.column_schema.name
799                        ),
800                    }
801                );
802
803                ensure!(
804                    col_to_add.column_schema.is_nullable()
805                        || col_to_add.column_schema.default_constraint().is_some(),
806                    error::InvalidAlterRequestSnafu {
807                        table: table_name,
808                        err: format!(
809                            "no default value for column {}",
810                            col_to_add.column_schema.name
811                        ),
812                    },
813                );
814
815                // A dropped column may leave a stale entity declaration behind;
816                // re-adding it must not hand the declaration a type without a
817                // stable string form.
818                if !has_stable_string_form(&col_to_add.column_schema.data_type)
819                    && let Some(key) =
820                        entity_option_referencing(&self.options, &col_to_add.column_schema.name)
821                {
822                    return error::InvalidAlterRequestSnafu {
823                        table: table_name,
824                        err: format!(
825                            "column `{}` is referenced by entity option `{key}` and must \
826                             keep a type that renders as a string, got `{}`",
827                            col_to_add.column_schema.name, col_to_add.column_schema.data_type
828                        ),
829                    }
830                    .fail();
831                }
832
833                new_columns.push(col_to_add.clone());
834            }
835        }
836        let requests = &new_columns[..];
837
838        let SplitResult {
839            columns_at_first,
840            columns_at_after,
841            columns_at_last,
842            column_names,
843        } = self.split_requests_by_column_location(table_name, requests)?;
844        let mut primary_key_indices = Vec::with_capacity(self.primary_key_indices.len());
845        let mut columns = Vec::with_capacity(table_schema.num_columns() + requests.len());
846        // add new columns with FIRST, and in reverse order of requests.
847        columns_at_first.iter().rev().for_each(|request| {
848            if request.is_key {
849                // If a key column is added, we also need to store its index in primary_key_indices.
850                primary_key_indices.push(columns.len());
851            }
852            columns.push(request.column_schema.clone());
853        });
854        // add existed columns in original order and handle new columns with AFTER.
855        for (index, column_schema) in table_schema.column_schemas().iter().enumerate() {
856            if original_primary_key_indices.contains(&index) {
857                primary_key_indices.push(columns.len());
858            }
859            columns.push(column_schema.clone());
860            if let Some(requests) = columns_at_after.get(&column_schema.name) {
861                requests.iter().rev().for_each(|request| {
862                    if request.is_key {
863                        // If a key column is added, we also need to store its index in primary_key_indices.
864                        primary_key_indices.push(columns.len());
865                    }
866                    columns.push(request.column_schema.clone());
867                });
868            }
869        }
870        // add new columns without location info to last.
871        columns_at_last.iter().for_each(|request| {
872            if request.is_key {
873                // If a key column is added, we also need to store its index in primary_key_indices.
874                primary_key_indices.push(columns.len());
875            }
876            columns.push(request.column_schema.clone());
877        });
878
879        let mut builder = SchemaBuilder::try_from(columns)
880            .with_context(|_| error::SchemaBuildSnafu {
881                msg: format!("Failed to convert column schemas into schema for table {table_name}"),
882            })?
883            // Also bump the schema version.
884            .version(table_schema.version() + 1);
885        for (k, v) in table_schema.metadata().iter() {
886            builder = builder.add_metadata(k, v);
887        }
888        let new_schema = builder.build().with_context(|_| error::SchemaBuildSnafu {
889            msg: format!("Table {table_name} cannot add new columns {column_names:?}"),
890        })?;
891
892        let partition_key_indices = self
893            .partition_key_indices
894            .iter()
895            .map(|idx| table_schema.column_name_by_index(*idx))
896            // This unwrap is safe since we only add new columns.
897            .map(|name| new_schema.column_index_by_name(name).unwrap())
898            .collect();
899
900        // value_indices would be generated automatically.
901        let _ = meta_builder
902            .schema(Arc::new(new_schema))
903            .primary_key_indices(primary_key_indices)
904            .partition_key_indices(partition_key_indices);
905
906        Ok(meta_builder)
907    }
908
909    fn remove_columns(
910        &self,
911        table_name: &str,
912        column_names: &[String],
913    ) -> Result<TableMetaBuilder> {
914        let table_schema = &self.schema;
915        let column_names: HashSet<_> = column_names.iter().collect();
916        let mut meta_builder = self.new_meta_builder();
917
918        let timestamp_index = table_schema.timestamp_index();
919        // Check whether columns are existing and not in primary key index.
920        for column_name in &column_names {
921            if let Some(index) = table_schema.column_index_by_name(column_name) {
922                // This is a linear search, but since there won't be too much columns, the performance should
923                // be acceptable.
924                ensure!(
925                    !self.primary_key_indices.contains(&index),
926                    error::RemoveColumnInIndexSnafu {
927                        column_name: *column_name,
928                        table_name,
929                    }
930                );
931
932                ensure!(
933                    !self.partition_key_indices.contains(&index),
934                    error::RemovePartitionColumnSnafu {
935                        column_name: *column_name,
936                        table_name,
937                    }
938                );
939
940                if let Some(ts_index) = timestamp_index {
941                    // Not allowed to remove column in timestamp index.
942                    ensure!(
943                        index != ts_index,
944                        error::RemoveColumnInIndexSnafu {
945                            column_name: table_schema.column_name_by_index(ts_index),
946                            table_name,
947                        }
948                    );
949                }
950            } else {
951                return error::ColumnNotExistsSnafu {
952                    column_name: *column_name,
953                    table_name,
954                }
955                .fail()?;
956            }
957        }
958
959        // Collect columns after removal.
960        let columns: Vec<_> = table_schema
961            .column_schemas()
962            .iter()
963            .filter(|column_schema| !column_names.contains(&column_schema.name))
964            .cloned()
965            .collect();
966
967        let mut builder = SchemaBuilder::try_from_columns(columns)
968            .with_context(|_| error::SchemaBuildSnafu {
969                msg: format!("Failed to convert column schemas into schema for table {table_name}"),
970            })?
971            // Also bump the schema version.
972            .version(table_schema.version() + 1);
973        for (k, v) in table_schema.metadata().iter() {
974            builder = builder.add_metadata(k, v);
975        }
976        let new_schema = builder.build().with_context(|_| error::SchemaBuildSnafu {
977            msg: format!("Table {table_name} cannot add remove columns {column_names:?}"),
978        })?;
979
980        // Rebuild the indices of primary key columns.
981        let primary_key_indices = self
982            .primary_key_indices
983            .iter()
984            .map(|idx| table_schema.column_name_by_index(*idx))
985            // This unwrap is safe since we don't allow removing a primary key column.
986            .map(|name| new_schema.column_index_by_name(name).unwrap())
987            .collect();
988
989        let partition_key_indices = self
990            .partition_key_indices
991            .iter()
992            .map(|idx| table_schema.column_name_by_index(*idx))
993            // This unwrap is safe since we don't allow removing a partition key column.
994            .map(|name| new_schema.column_index_by_name(name).unwrap())
995            .collect();
996
997        let _ = meta_builder
998            .schema(Arc::new(new_schema))
999            .primary_key_indices(primary_key_indices)
1000            .partition_key_indices(partition_key_indices);
1001
1002        Ok(meta_builder)
1003    }
1004
1005    fn modify_column_types(
1006        &self,
1007        table_name: &str,
1008        requests: &[ModifyColumnTypeRequest],
1009    ) -> Result<TableMetaBuilder> {
1010        let table_schema = &self.schema;
1011        let mut meta_builder = self.new_meta_builder();
1012
1013        let mut modify_column_types = HashMap::with_capacity(requests.len());
1014        let timestamp_index = table_schema.timestamp_index();
1015
1016        for col_to_change in requests {
1017            let change_column_name = &col_to_change.column_name;
1018
1019            let index = table_schema
1020                .column_index_by_name(change_column_name)
1021                .with_context(|| error::ColumnNotExistsSnafu {
1022                    column_name: change_column_name,
1023                    table_name,
1024                })?;
1025
1026            // A column referenced by an entity declaration may be dropped (the
1027            // read-time derivation skips the stale declaration), but must not
1028            // change to a type without a stable string form. Checked after the
1029            // existence lookup so a missing column keeps reporting
1030            // `ColumnNotExists` regardless of stale declarations.
1031            if !has_stable_string_form(&col_to_change.target_type)
1032                && let Some(key) = entity_option_referencing(&self.options, change_column_name)
1033            {
1034                return error::InvalidAlterRequestSnafu {
1035                    table: table_name,
1036                    err: format!(
1037                        "column `{change_column_name}` is referenced by entity option \
1038                         `{key}` and must keep a type that renders as a string, got `{}`",
1039                        col_to_change.target_type
1040                    ),
1041                }
1042                .fail();
1043            }
1044
1045            let column = &table_schema.column_schemas()[index];
1046
1047            ensure!(
1048                !self.primary_key_indices.contains(&index),
1049                error::InvalidAlterRequestSnafu {
1050                    table: table_name,
1051                    err: format!(
1052                        "Not allowed to change primary key index column '{}'",
1053                        column.name
1054                    )
1055                }
1056            );
1057
1058            let is_time_index = timestamp_index == Some(index);
1059            if is_time_index {
1060                // The time index column is NOT NULL by construction and only
1061                // supports widening its unit; historical data in SSTs is cast
1062                // to the new unit on read, so no backfill is needed.
1063                ensure!(
1064                    column
1065                        .data_type
1066                        .is_timestamp_unit_widening_to(&col_to_change.target_type),
1067                    error::InvalidAlterRequestSnafu {
1068                        table: table_name,
1069                        err: time_index_not_widening_error(
1070                            &column.name,
1071                            &column.data_type,
1072                            &col_to_change.target_type,
1073                        ),
1074                    }
1075                );
1076            } else {
1077                ensure!(
1078                    column
1079                        .data_type
1080                        .can_arrow_type_cast_to(&col_to_change.target_type),
1081                    error::InvalidAlterRequestSnafu {
1082                        table: table_name,
1083                        err: format!(
1084                            "column '{}' cannot be cast automatically to type '{}'",
1085                            col_to_change.column_name, col_to_change.target_type,
1086                        ),
1087                    }
1088                );
1089
1090                ensure!(
1091                    column.is_nullable(),
1092                    error::InvalidAlterRequestSnafu {
1093                        table: table_name,
1094                        err: format!(
1095                            "column '{}' must be nullable to ensure safe conversion.",
1096                            col_to_change.column_name,
1097                        ),
1098                    }
1099                );
1100            }
1101            ensure!(
1102                modify_column_types
1103                    .insert(&col_to_change.column_name, col_to_change)
1104                    .is_none(),
1105                error::InvalidAlterRequestSnafu {
1106                    table: table_name,
1107                    err: format!(
1108                        "change column datatype {} more than once",
1109                        col_to_change.column_name
1110                    ),
1111                }
1112            );
1113        }
1114        // Collect columns after changed.
1115
1116        let mut columns: Vec<_> = Vec::with_capacity(table_schema.column_schemas().len());
1117        for mut column in table_schema.column_schemas().iter().cloned() {
1118            if let Some(change_column) = modify_column_types.get(&column.name) {
1119                column.data_type = change_column.target_type.clone();
1120                let new_default = if let Some(default_value) = column.default_constraint() {
1121                    Some(
1122                        default_value
1123                            .cast_to_datatype(&change_column.target_type)
1124                            .with_context(|_| error::CastDefaultValueSnafu {
1125                                reason: format!(
1126                                    "Failed to cast default value from {:?} to type {:?}",
1127                                    default_value, &change_column.target_type
1128                                ),
1129                            })?,
1130                    )
1131                } else {
1132                    None
1133                };
1134                column = column
1135                    .clone()
1136                    .with_default_constraint(new_default.clone())
1137                    .with_context(|_| error::CastDefaultValueSnafu {
1138                        reason: format!("Failed to set new default: {:?}", new_default),
1139                    })?;
1140            }
1141            columns.push(column)
1142        }
1143
1144        let mut builder = SchemaBuilder::try_from_columns(columns)
1145            .with_context(|_| error::SchemaBuildSnafu {
1146                msg: format!("Failed to convert column schemas into schema for table {table_name}"),
1147            })?
1148            // Also bump the schema version.
1149            .version(table_schema.version() + 1);
1150        for (k, v) in table_schema.metadata().iter() {
1151            builder = builder.add_metadata(k, v);
1152        }
1153        let new_schema = builder.build().with_context(|_| {
1154            let column_names: Vec<_> = requests
1155                .iter()
1156                .map(|request| &request.column_name)
1157                .collect();
1158
1159            error::SchemaBuildSnafu {
1160                msg: format!(
1161                    "Table {table_name} cannot change datatype with columns {column_names:?}"
1162                ),
1163            }
1164        })?;
1165
1166        let _ = meta_builder
1167            .schema(Arc::new(new_schema))
1168            .primary_key_indices(self.primary_key_indices.clone());
1169
1170        Ok(meta_builder)
1171    }
1172
1173    /// Split requests into different groups using column location info.
1174    fn split_requests_by_column_location<'a>(
1175        &self,
1176        table_name: &str,
1177        requests: &'a [AddColumnRequest],
1178    ) -> Result<SplitResult<'a>> {
1179        let table_schema = &self.schema;
1180        let mut columns_at_first = Vec::new();
1181        let mut columns_at_after = HashMap::new();
1182        let mut columns_at_last = Vec::new();
1183        let mut column_names = Vec::with_capacity(requests.len());
1184        for request in requests {
1185            // Check whether columns to add are already existing.
1186            let column_name = &request.column_schema.name;
1187            column_names.push(column_name.clone());
1188            ensure!(
1189                table_schema.column_schema_by_name(column_name).is_none(),
1190                error::ColumnExistsSnafu {
1191                    column_name,
1192                    table_name,
1193                }
1194            );
1195            match request.location.as_ref() {
1196                Some(AddColumnLocation::First) => {
1197                    columns_at_first.push(request);
1198                }
1199                Some(AddColumnLocation::After { column_name }) => {
1200                    ensure!(
1201                        table_schema.column_schema_by_name(column_name).is_some(),
1202                        error::ColumnNotExistsSnafu {
1203                            column_name,
1204                            table_name,
1205                        }
1206                    );
1207                    columns_at_after
1208                        .entry(column_name.clone())
1209                        .or_insert(Vec::new())
1210                        .push(request);
1211                }
1212                None => {
1213                    columns_at_last.push(request);
1214                }
1215            }
1216        }
1217        Ok(SplitResult {
1218            columns_at_first,
1219            columns_at_after,
1220            columns_at_last,
1221            column_names,
1222        })
1223    }
1224
1225    fn drop_defaults(&self, table_name: &str, column_names: &[String]) -> Result<TableMetaBuilder> {
1226        let table_schema = &self.schema;
1227        let mut meta_builder = self.new_meta_builder();
1228        let mut columns = Vec::with_capacity(table_schema.num_columns());
1229        for column_schema in table_schema.column_schemas() {
1230            if let Some(name) = column_names.iter().find(|s| **s == column_schema.name) {
1231                // Drop default constraint.
1232                ensure!(
1233                    column_schema.default_constraint().is_some(),
1234                    error::InvalidAlterRequestSnafu {
1235                        table: table_name,
1236                        err: format!("column {name} does not have a default value"),
1237                    }
1238                );
1239                if !column_schema.is_nullable() {
1240                    return error::InvalidAlterRequestSnafu {
1241                        table: table_name,
1242                        err: format!(
1243                            "column {name} is not nullable and `default` cannot be dropped",
1244                        ),
1245                    }
1246                    .fail();
1247                }
1248                let new_column_schema = column_schema.clone();
1249                let new_column_schema = new_column_schema
1250                    .with_default_constraint(None)
1251                    .with_context(|_| error::SchemaBuildSnafu {
1252                        msg: format!("Table {table_name} cannot drop default values"),
1253                    })?;
1254                columns.push(new_column_schema);
1255            } else {
1256                columns.push(column_schema.clone());
1257            }
1258        }
1259
1260        let mut builder = SchemaBuilder::try_from_columns(columns)
1261            .with_context(|_| error::SchemaBuildSnafu {
1262                msg: format!("Failed to convert column schemas into schema for table {table_name}"),
1263            })?
1264            // Also bump the schema version.
1265            .version(table_schema.version() + 1);
1266        for (k, v) in table_schema.metadata().iter() {
1267            builder = builder.add_metadata(k, v);
1268        }
1269        let new_schema = builder.build().with_context(|_| error::SchemaBuildSnafu {
1270            msg: format!("Table {table_name} cannot drop default values"),
1271        })?;
1272
1273        let _ = meta_builder.schema(Arc::new(new_schema));
1274
1275        Ok(meta_builder)
1276    }
1277
1278    fn set_defaults(
1279        &self,
1280        table_name: &str,
1281        set_defaults: &[SetDefaultRequest],
1282    ) -> Result<TableMetaBuilder> {
1283        let table_schema = &self.schema;
1284        let mut meta_builder = self.new_meta_builder();
1285        let mut columns = Vec::with_capacity(table_schema.num_columns());
1286        for column_schema in table_schema.column_schemas() {
1287            if let Some(set_default) = set_defaults
1288                .iter()
1289                .find(|s| s.column_name == column_schema.name)
1290            {
1291                let new_column_schema = column_schema.clone();
1292                let new_column_schema = new_column_schema
1293                    .with_default_constraint(set_default.default_constraint.clone())
1294                    .with_context(|_| error::SchemaBuildSnafu {
1295                        msg: format!("Table {table_name} cannot set default values"),
1296                    })?;
1297                columns.push(new_column_schema);
1298            } else {
1299                columns.push(column_schema.clone());
1300            }
1301        }
1302
1303        let mut builder = SchemaBuilder::try_from_columns(columns)
1304            .with_context(|_| error::SchemaBuildSnafu {
1305                msg: format!("Failed to convert column schemas into schema for table {table_name}"),
1306            })?
1307            // Also bump the schema version.
1308            .version(table_schema.version() + 1);
1309        for (k, v) in table_schema.metadata().iter() {
1310            builder = builder.add_metadata(k, v);
1311        }
1312        let new_schema = builder.build().with_context(|_| error::SchemaBuildSnafu {
1313            msg: format!("Table {table_name} cannot set default values"),
1314        })?;
1315
1316        let _ = meta_builder.schema(Arc::new(new_schema));
1317
1318        Ok(meta_builder)
1319    }
1320}
1321
1322#[derive(Clone, Debug, PartialEq, Eq, Builder, Serialize, Deserialize)]
1323#[builder(pattern = "owned")]
1324pub struct TableInfo {
1325    /// Id and version of the table.
1326    #[builder(default, setter(into))]
1327    pub ident: TableIdent,
1328    /// Name of the table.
1329    #[builder(setter(into))]
1330    pub name: String,
1331    /// Comment of the table.
1332    #[builder(default, setter(into))]
1333    pub desc: Option<String>,
1334    #[builder(default = "DEFAULT_CATALOG_NAME.to_string()", setter(into))]
1335    pub catalog_name: String,
1336    #[builder(default = "DEFAULT_SCHEMA_NAME.to_string()", setter(into))]
1337    pub schema_name: String,
1338    pub meta: TableMeta,
1339    #[builder(default = "TableType::Base")]
1340    pub table_type: TableType,
1341}
1342
1343pub type TableInfoRef = Arc<TableInfo>;
1344
1345impl TableInfo {
1346    pub fn table_id(&self) -> TableId {
1347        self.ident.table_id
1348    }
1349
1350    /// Returns the full table name in the form of `{catalog}.{schema}.{table}`.
1351    pub fn full_table_name(&self) -> String {
1352        common_catalog::format_full_table_name(&self.catalog_name, &self.schema_name, &self.name)
1353    }
1354
1355    pub fn get_db_string(&self) -> String {
1356        common_catalog::build_db_string(&self.catalog_name, &self.schema_name)
1357    }
1358
1359    /// Returns true when the table is the metric engine's physical table.
1360    pub fn is_physical_table(&self) -> bool {
1361        self.meta
1362            .options
1363            .extra_options
1364            .contains_key(PHYSICAL_TABLE_METADATA_KEY)
1365    }
1366
1367    /// Return true if the table's TTL is `instant`.
1368    pub fn is_ttl_instant_table(&self) -> bool {
1369        self.meta
1370            .options
1371            .ttl
1372            .map(|t| t.is_instant())
1373            .unwrap_or(false)
1374    }
1375}
1376
1377impl TableInfoBuilder {
1378    pub fn new<S: Into<String>>(name: S, meta: TableMeta) -> Self {
1379        Self {
1380            name: Some(name.into()),
1381            meta: Some(meta),
1382            ..Default::default()
1383        }
1384    }
1385
1386    pub fn table_id(mut self, id: TableId) -> Self {
1387        let ident = self.ident.get_or_insert_with(TableIdent::default);
1388        ident.table_id = id;
1389        self
1390    }
1391
1392    pub fn table_version(mut self, version: TableVersion) -> Self {
1393        let ident = self.ident.get_or_insert_with(TableIdent::default);
1394        ident.version = version;
1395        self
1396    }
1397}
1398
1399impl TableIdent {
1400    pub fn new(table_id: TableId) -> Self {
1401        Self {
1402            table_id,
1403            version: 0,
1404        }
1405    }
1406}
1407
1408impl From<TableId> for TableIdent {
1409    fn from(table_id: TableId) -> Self {
1410        Self::new(table_id)
1411    }
1412}
1413
1414impl TableInfo {
1415    /// Returns the map of column name to column id.
1416    ///
1417    /// Note: This method may return an empty map for older versions that did not include this field.
1418    pub fn name_to_ids(&self) -> Option<HashMap<String, ColumnId>> {
1419        let column_schemas = self.meta.schema.column_schemas();
1420        if self.meta.column_ids.len() != column_schemas.len() {
1421            None
1422        } else {
1423            Some(
1424                self.meta
1425                    .column_ids
1426                    .iter()
1427                    .enumerate()
1428                    .map(|(index, id)| (column_schemas[index].name.clone(), *id))
1429                    .collect(),
1430            )
1431        }
1432    }
1433
1434    /// Sort the columns in [TableInfo], logical tables require it.
1435    pub fn sort_columns(&mut self) {
1436        let column_schemas = self.meta.schema.column_schemas();
1437        let primary_keys = self
1438            .meta
1439            .primary_key_indices
1440            .iter()
1441            .map(|index| column_schemas[*index].name.clone())
1442            .collect::<HashSet<_>>();
1443
1444        let name_to_ids = self.name_to_ids().unwrap_or_default();
1445        let mut column_schemas = column_schemas.to_vec();
1446        column_schemas.sort_unstable_by(|a, b| a.name.cmp(&b.name));
1447
1448        // Compute new indices of sorted columns
1449        let mut primary_key_indices = Vec::with_capacity(primary_keys.len());
1450        let mut value_indices = Vec::with_capacity(column_schemas.len() - primary_keys.len());
1451        let mut column_ids = Vec::with_capacity(column_schemas.len());
1452        for (index, column_schema) in column_schemas.iter().enumerate() {
1453            if primary_keys.contains(&column_schema.name) {
1454                primary_key_indices.push(index);
1455            } else {
1456                value_indices.push(index);
1457            }
1458            if let Some(id) = name_to_ids.get(&column_schema.name) {
1459                column_ids.push(*id);
1460            }
1461        }
1462
1463        // Overwrite table meta
1464        self.meta.schema = Arc::new(Schema::new_with_version(
1465            column_schemas,
1466            self.meta.schema.version(),
1467        ));
1468        self.meta.primary_key_indices = primary_key_indices;
1469        self.meta.value_indices = value_indices;
1470        self.meta.column_ids = column_ids;
1471    }
1472
1473    /// Extracts region options from table info.
1474    ///
1475    /// All "region options" are actually a copy of table options for redundancy.
1476    pub fn to_region_options(&self) -> HashMap<String, String> {
1477        let mut options = HashMap::from(&self.meta.options);
1478        options.remove(REPARTITION_COLUMN_HINT_KEY);
1479        options
1480    }
1481
1482    /// Returns the table reference.
1483    pub fn table_ref(&self) -> TableReference<'_> {
1484        TableReference::full(
1485            self.catalog_name.as_str(),
1486            self.schema_name.as_str(),
1487            self.name.as_str(),
1488        )
1489    }
1490}
1491
1492fn entity_option_referencing<'a>(options: &'a TableOptions, column: &str) -> Option<&'a str> {
1493    options.extra_options.iter().find_map(|(key, value)| {
1494        (parse_entity_option_key(key).is_some()
1495            && parse_entity_columns(value).iter().any(|c| c == column))
1496        .then_some(key.as_str())
1497    })
1498}
1499
1500/// Set column fulltext options if it passed the validation.
1501///
1502/// Options allowed to modify:
1503/// * backend
1504///
1505/// Options not allowed to modify:
1506/// * analyzer
1507/// * case_sensitive
1508fn set_column_fulltext_options(
1509    column_schema: &mut ColumnSchema,
1510    column_name: &str,
1511    options: &FulltextOptions,
1512    current_options: Option<FulltextOptions>,
1513) -> Result<()> {
1514    if let Some(current_options) = current_options {
1515        ensure!(
1516            current_options.analyzer == options.analyzer
1517                && current_options.case_sensitive == options.case_sensitive,
1518            error::InvalidColumnOptionSnafu {
1519                column_name,
1520                msg: format!(
1521                    "Cannot change analyzer or case_sensitive if FULLTEXT index is set before. Previous analyzer: {}, previous case_sensitive: {}",
1522                    current_options.analyzer, current_options.case_sensitive
1523                ),
1524            }
1525        );
1526    }
1527
1528    column_schema
1529        .set_fulltext_options(options)
1530        .context(error::SetFulltextOptionsSnafu { column_name })?;
1531
1532    Ok(())
1533}
1534
1535fn unset_column_fulltext_options(
1536    column_schema: &mut ColumnSchema,
1537    column_name: &str,
1538    current_options: Option<FulltextOptions>,
1539) -> Result<()> {
1540    ensure!(
1541        current_options
1542            .as_ref()
1543            .is_some_and(|options| options.enable),
1544        error::InvalidColumnOptionSnafu {
1545            column_name,
1546            msg: "FULLTEXT index already disabled".to_string(),
1547        }
1548    );
1549
1550    let mut options = current_options.unwrap();
1551    options.enable = false;
1552    column_schema
1553        .set_fulltext_options(&options)
1554        .context(error::SetFulltextOptionsSnafu { column_name })?;
1555
1556    Ok(())
1557}
1558
1559fn set_column_skipping_index_options(
1560    column_schema: &mut ColumnSchema,
1561    column_name: &str,
1562    options: &SkippingIndexOptions,
1563) -> Result<()> {
1564    column_schema
1565        .set_skipping_options(options)
1566        .context(error::SetSkippingOptionsSnafu { column_name })?;
1567
1568    Ok(())
1569}
1570
1571fn unset_column_skipping_index_options(
1572    column_schema: &mut ColumnSchema,
1573    column_name: &str,
1574) -> Result<()> {
1575    column_schema
1576        .unset_skipping_options()
1577        .context(error::UnsetSkippingOptionsSnafu { column_name })?;
1578    Ok(())
1579}
1580
1581#[cfg(test)]
1582mod tests {
1583    use std::assert_matches;
1584
1585    use common_error::ext::ErrorExt;
1586    use common_error::status_code::StatusCode;
1587    use datatypes::data_type::ConcreteDataType;
1588    use datatypes::schema::{
1589        ColumnSchema, FulltextAnalyzer, FulltextBackend, Schema, SchemaBuilder,
1590    };
1591
1592    use super::*;
1593    use crate::Error;
1594
1595    /// Create a test schema with 3 columns: `[col1 int32, ts timestampmills, col2 int32]`.
1596    fn new_test_schema() -> Schema {
1597        let column_schemas = vec![
1598            ColumnSchema::new("col1", ConcreteDataType::int32_datatype(), true),
1599            ColumnSchema::new(
1600                "ts",
1601                ConcreteDataType::timestamp_millisecond_datatype(),
1602                false,
1603            )
1604            .with_time_index(true),
1605            ColumnSchema::new("col2", ConcreteDataType::int32_datatype(), true),
1606        ];
1607        SchemaBuilder::try_from(column_schemas)
1608            .unwrap()
1609            .version(123)
1610            .build()
1611            .unwrap()
1612    }
1613
1614    fn add_columns_to_meta(meta: &TableMeta) -> TableMeta {
1615        let new_tag = ColumnSchema::new("my_tag", ConcreteDataType::string_datatype(), true);
1616        let new_field = ColumnSchema::new("my_field", ConcreteDataType::string_datatype(), true);
1617        let alter_kind = AlterKind::AddColumns {
1618            columns: vec![
1619                AddColumnRequest {
1620                    column_schema: new_tag,
1621                    is_key: true,
1622                    location: None,
1623                    add_if_not_exists: false,
1624                },
1625                AddColumnRequest {
1626                    column_schema: new_field,
1627                    is_key: false,
1628                    location: None,
1629                    add_if_not_exists: false,
1630                },
1631            ],
1632        };
1633
1634        let builder = meta
1635            .builder_with_alter_kind("my_table", &alter_kind)
1636            .unwrap();
1637        builder.build().unwrap()
1638    }
1639
1640    fn add_columns_to_meta_with_location(meta: &TableMeta) -> TableMeta {
1641        let new_tag = ColumnSchema::new("my_tag_first", ConcreteDataType::string_datatype(), true);
1642        let new_field = ColumnSchema::new(
1643            "my_field_after_ts",
1644            ConcreteDataType::string_datatype(),
1645            true,
1646        );
1647        let yet_another_field = ColumnSchema::new(
1648            "yet_another_field_after_ts",
1649            ConcreteDataType::int64_datatype(),
1650            true,
1651        );
1652        let alter_kind = AlterKind::AddColumns {
1653            columns: vec![
1654                AddColumnRequest {
1655                    column_schema: new_tag,
1656                    is_key: true,
1657                    location: Some(AddColumnLocation::First),
1658                    add_if_not_exists: false,
1659                },
1660                AddColumnRequest {
1661                    column_schema: new_field,
1662                    is_key: false,
1663                    location: Some(AddColumnLocation::After {
1664                        column_name: "ts".to_string(),
1665                    }),
1666                    add_if_not_exists: false,
1667                },
1668                AddColumnRequest {
1669                    column_schema: yet_another_field,
1670                    is_key: true,
1671                    location: Some(AddColumnLocation::After {
1672                        column_name: "ts".to_string(),
1673                    }),
1674                    add_if_not_exists: false,
1675                },
1676            ],
1677        };
1678
1679        let builder = meta
1680            .builder_with_alter_kind("my_table", &alter_kind)
1681            .unwrap();
1682        builder.build().unwrap()
1683    }
1684
1685    #[test]
1686    fn test_modify_time_index_column_type() {
1687        let schema = Arc::new(new_test_schema());
1688        let meta = TableMetaBuilder::empty()
1689            .schema(schema)
1690            .primary_key_indices(vec![0])
1691            .engine("engine")
1692            .next_column_id(3)
1693            .build()
1694            .unwrap();
1695
1696        // Widening the time index unit is allowed.
1697        let alter_kind = AlterKind::ModifyColumnTypes {
1698            columns: vec![ModifyColumnTypeRequest {
1699                column_name: "ts".to_string(),
1700                target_type: ConcreteDataType::timestamp_microsecond_datatype(),
1701            }],
1702        };
1703        let new_meta = meta
1704            .builder_with_alter_kind("my_table", &alter_kind)
1705            .unwrap()
1706            .build()
1707            .unwrap();
1708        let ts_column = new_meta.schema.column_schema_by_name("ts").unwrap();
1709        assert_eq!(
1710            ConcreteDataType::timestamp_microsecond_datatype(),
1711            ts_column.data_type
1712        );
1713        assert!(ts_column.is_time_index());
1714        assert!(!ts_column.is_nullable());
1715        assert_eq!(new_meta.schema.version(), 124);
1716        assert_eq!(&[0], &new_meta.primary_key_indices[..]);
1717
1718        // Any non-widening change (narrowing, same type, non-timestamp) is
1719        // rejected.
1720        for target in [
1721            ConcreteDataType::timestamp_second_datatype(),
1722            ConcreteDataType::timestamp_millisecond_datatype(),
1723            ConcreteDataType::string_datatype(),
1724        ] {
1725            let alter_kind = AlterKind::ModifyColumnTypes {
1726                columns: vec![ModifyColumnTypeRequest {
1727                    column_name: "ts".to_string(),
1728                    target_type: target.clone(),
1729                }],
1730            };
1731            let res = meta.builder_with_alter_kind("my_table", &alter_kind);
1732            assert!(res.is_err(), "expected rejection for {target}");
1733        }
1734    }
1735
1736    #[test]
1737    fn test_add_columns() {
1738        let schema = Arc::new(new_test_schema());
1739        let meta = TableMetaBuilder::empty()
1740            .schema(schema)
1741            .primary_key_indices(vec![0])
1742            .engine("engine")
1743            .next_column_id(3)
1744            .build()
1745            .unwrap();
1746
1747        let new_meta = add_columns_to_meta(&meta);
1748        let names: Vec<String> = new_meta
1749            .schema
1750            .column_schemas()
1751            .iter()
1752            .map(|column_schema| column_schema.name.clone())
1753            .collect();
1754        assert_eq!(&["col1", "ts", "col2", "my_tag", "my_field"], &names[..]);
1755        assert_eq!(&[0, 3], &new_meta.primary_key_indices[..]);
1756        assert_eq!(&[1, 2, 4], &new_meta.value_indices[..]);
1757    }
1758
1759    #[test]
1760    fn test_set_append_mode_true_clears_merge_mode_option() {
1761        let schema = Arc::new(new_test_schema());
1762        let mut table_options = TableOptions::default();
1763        table_options
1764            .extra_options
1765            .insert(MERGE_MODE_KEY.to_string(), "last_non_null".to_string());
1766        let meta = TableMetaBuilder::empty()
1767            .schema(schema)
1768            .primary_key_indices(vec![0])
1769            .engine("engine")
1770            .next_column_id(3)
1771            .options(table_options)
1772            .build()
1773            .unwrap();
1774
1775        let alter_kind = AlterKind::SetTableOptions {
1776            options: vec![SetRegionOption::AppendMode(true)],
1777        };
1778        let new_meta = meta
1779            .builder_with_alter_kind("my_table", &alter_kind)
1780            .unwrap()
1781            .build()
1782            .unwrap();
1783
1784        assert_eq!(
1785            Some("true"),
1786            new_meta
1787                .options
1788                .extra_options
1789                .get(APPEND_MODE_KEY)
1790                .map(String::as_str)
1791        );
1792        assert!(!new_meta.options.extra_options.contains_key(MERGE_MODE_KEY));
1793    }
1794
1795    #[test]
1796    fn test_set_append_mode_false_keeps_merge_mode_option() {
1797        let schema = Arc::new(new_test_schema());
1798        let mut table_options = TableOptions::default();
1799        table_options
1800            .extra_options
1801            .insert(MERGE_MODE_KEY.to_string(), "last_non_null".to_string());
1802        let meta = TableMetaBuilder::empty()
1803            .schema(schema)
1804            .primary_key_indices(vec![0])
1805            .engine("engine")
1806            .next_column_id(3)
1807            .options(table_options)
1808            .build()
1809            .unwrap();
1810
1811        let alter_kind = AlterKind::SetTableOptions {
1812            options: vec![SetRegionOption::AppendMode(false)],
1813        };
1814        let new_meta = meta
1815            .builder_with_alter_kind("my_table", &alter_kind)
1816            .unwrap()
1817            .build()
1818            .unwrap();
1819
1820        assert_eq!(
1821            Some("false"),
1822            new_meta
1823                .options
1824                .extra_options
1825                .get(APPEND_MODE_KEY)
1826                .map(String::as_str)
1827        );
1828        assert_eq!(
1829            Some("last_non_null"),
1830            new_meta
1831                .options
1832                .extra_options
1833                .get(MERGE_MODE_KEY)
1834                .map(String::as_str)
1835        );
1836    }
1837
1838    #[test]
1839    fn test_set_preserve_row_sequence_option() {
1840        let schema = Arc::new(new_test_schema());
1841        let meta = TableMetaBuilder::empty()
1842            .schema(schema)
1843            .primary_key_indices(vec![0])
1844            .engine("engine")
1845            .next_column_id(3)
1846            .build()
1847            .unwrap();
1848
1849        let apply = |meta: &TableMeta, kind: &AlterKind| {
1850            meta.builder_with_alter_kind("my_table", kind)
1851                .unwrap()
1852                .build()
1853                .unwrap()
1854        };
1855
1856        let with_true = apply(
1857            &meta,
1858            &AlterKind::SetTableOptions {
1859                options: vec![SetRegionOption::PreserveRowSequence(true)],
1860            },
1861        );
1862        assert_eq!(
1863            Some("true"),
1864            with_true
1865                .options
1866                .extra_options
1867                .get(PRESERVE_ROW_SEQUENCE)
1868                .map(String::as_str)
1869        );
1870
1871        let set_false = apply(
1872            &with_true,
1873            &AlterKind::SetTableOptions {
1874                options: vec![SetRegionOption::PreserveRowSequence(false)],
1875            },
1876        );
1877        assert!(
1878            !set_false
1879                .options
1880                .extra_options
1881                .contains_key(PRESERVE_ROW_SEQUENCE)
1882        );
1883
1884        let unset = apply(
1885            &with_true,
1886            &AlterKind::UnsetTableOptions {
1887                keys: vec![UnsetRegionOption::PreserveRowSequence],
1888            },
1889        );
1890        assert!(
1891            !unset
1892                .options
1893                .extra_options
1894                .contains_key(PRESERVE_ROW_SEQUENCE)
1895        );
1896    }
1897
1898    #[test]
1899    fn test_set_skip_wal_updates_typed_and_extra_options() {
1900        let mut meta = TableMetaBuilder::empty()
1901            .schema(Arc::new(new_test_schema()))
1902            .primary_key_indices(vec![0])
1903            .engine("engine")
1904            .next_column_id(3)
1905            .build()
1906            .unwrap();
1907        meta.options
1908            .extra_options
1909            .insert(SKIP_WAL_KEY.to_string(), false.to_string());
1910
1911        let alter_kind = AlterKind::SetTableOptions {
1912            options: vec![SetRegionOption::SkipWal],
1913        };
1914        let new_meta = meta
1915            .builder_with_alter_kind("my_table", &alter_kind)
1916            .unwrap()
1917            .build()
1918            .unwrap();
1919
1920        assert!(new_meta.options.skip_wal);
1921        assert_eq!(
1922            Some("true"),
1923            new_meta
1924                .options
1925                .extra_options
1926                .get(SKIP_WAL_KEY)
1927                .map(String::as_str)
1928        );
1929    }
1930
1931    #[test]
1932    fn test_set_repartition_column_hint() {
1933        let meta = TableMetaBuilder::empty()
1934            .schema(Arc::new(new_test_schema()))
1935            .primary_key_indices(vec![0])
1936            .engine("engine")
1937            .next_column_id(3)
1938            .build()
1939            .unwrap();
1940
1941        let alter_kind = AlterKind::SetAnnotations {
1942            family: AnnotationFamily::RepartitionHint,
1943            options: vec![(
1944                REPARTITION_COLUMN_HINT_KEY.to_string(),
1945                " col1 ".to_string(),
1946            )],
1947        };
1948        let new_meta = meta
1949            .builder_with_alter_kind("my_table", &alter_kind)
1950            .unwrap()
1951            .build()
1952            .unwrap();
1953
1954        assert_eq!(
1955            Some("col1"),
1956            new_meta
1957                .options
1958                .extra_options
1959                .get(REPARTITION_COLUMN_HINT_KEY)
1960                .map(String::as_str)
1961        );
1962    }
1963
1964    #[test]
1965    fn test_set_repartition_column_hint_rejects_empty_column() {
1966        let meta = TableMetaBuilder::empty()
1967            .schema(Arc::new(new_test_schema()))
1968            .primary_key_indices(vec![0])
1969            .engine("engine")
1970            .next_column_id(3)
1971            .build()
1972            .unwrap();
1973
1974        let alter_kind = AlterKind::SetAnnotations {
1975            family: AnnotationFamily::RepartitionHint,
1976            options: vec![(REPARTITION_COLUMN_HINT_KEY.to_string(), " ".to_string())],
1977        };
1978        let err = meta
1979            .builder_with_alter_kind("my_table", &alter_kind)
1980            .err()
1981            .unwrap();
1982
1983        assert!(
1984            err.to_string()
1985                .contains("repartition.column.hint expects exactly one column name")
1986        );
1987    }
1988
1989    #[test]
1990    fn test_set_repartition_column_hint_rejects_multiple_columns() {
1991        let meta = TableMetaBuilder::empty()
1992            .schema(Arc::new(new_test_schema()))
1993            .primary_key_indices(vec![0])
1994            .engine("engine")
1995            .next_column_id(3)
1996            .build()
1997            .unwrap();
1998
1999        let alter_kind = AlterKind::SetAnnotations {
2000            family: AnnotationFamily::RepartitionHint,
2001            options: vec![(
2002                REPARTITION_COLUMN_HINT_KEY.to_string(),
2003                "col1,col2".to_string(),
2004            )],
2005        };
2006        let err = meta
2007            .builder_with_alter_kind("my_table", &alter_kind)
2008            .err()
2009            .unwrap();
2010
2011        assert!(
2012            err.to_string()
2013                .contains("repartition.column.hint expects exactly one column name")
2014        );
2015    }
2016
2017    #[test]
2018    fn test_set_repartition_column_hint_rejects_missing_column() {
2019        let meta = TableMetaBuilder::empty()
2020            .schema(Arc::new(new_test_schema()))
2021            .primary_key_indices(vec![0])
2022            .engine("engine")
2023            .next_column_id(3)
2024            .build()
2025            .unwrap();
2026
2027        let alter_kind = AlterKind::SetAnnotations {
2028            family: AnnotationFamily::RepartitionHint,
2029            options: vec![(
2030                REPARTITION_COLUMN_HINT_KEY.to_string(),
2031                "missing".to_string(),
2032            )],
2033        };
2034        let err = meta
2035            .builder_with_alter_kind("my_table", &alter_kind)
2036            .err()
2037            .unwrap();
2038
2039        assert!(err.to_string().contains("Column missing not exists"));
2040    }
2041
2042    #[test]
2043    fn test_set_repartition_column_hint_rejects_time_index_column() {
2044        let meta = TableMetaBuilder::empty()
2045            .schema(Arc::new(new_test_schema()))
2046            .primary_key_indices(vec![0])
2047            .engine("engine")
2048            .next_column_id(3)
2049            .build()
2050            .unwrap();
2051
2052        let alter_kind = AlterKind::SetAnnotations {
2053            family: AnnotationFamily::RepartitionHint,
2054            options: vec![(REPARTITION_COLUMN_HINT_KEY.to_string(), "ts".to_string())],
2055        };
2056        let err = meta
2057            .builder_with_alter_kind("my_table", &alter_kind)
2058            .err()
2059            .unwrap();
2060
2061        assert!(
2062            err.to_string()
2063                .contains("cannot set repartition.column.hint to the time index column")
2064        );
2065    }
2066
2067    #[test]
2068    fn test_set_repartition_column_hint_rejects_partitioned_table() {
2069        let meta = TableMetaBuilder::empty()
2070            .schema(Arc::new(new_test_schema()))
2071            .primary_key_indices(vec![0])
2072            .engine("engine")
2073            .next_column_id(3)
2074            .partition_key_indices(vec![0])
2075            .build()
2076            .unwrap();
2077
2078        let alter_kind = AlterKind::SetAnnotations {
2079            family: AnnotationFamily::RepartitionHint,
2080            options: vec![(REPARTITION_COLUMN_HINT_KEY.to_string(), "col1".to_string())],
2081        };
2082        let err = meta
2083            .builder_with_alter_kind("my_table", &alter_kind)
2084            .err()
2085            .unwrap();
2086
2087        assert!(
2088            err.to_string()
2089                .contains("cannot set repartition.column.hint on a table with partition metadata")
2090        );
2091    }
2092
2093    #[test]
2094    fn test_unset_repartition_column_hint() {
2095        let mut table_options = TableOptions::default();
2096        table_options
2097            .extra_options
2098            .insert(REPARTITION_COLUMN_HINT_KEY.to_string(), "col1".to_string());
2099        let meta = TableMetaBuilder::empty()
2100            .schema(Arc::new(new_test_schema()))
2101            .primary_key_indices(vec![0])
2102            .engine("engine")
2103            .next_column_id(3)
2104            .options(table_options)
2105            .build()
2106            .unwrap();
2107
2108        let new_meta = meta
2109            .builder_with_alter_kind(
2110                "my_table",
2111                &AlterKind::UnsetAnnotations {
2112                    family: AnnotationFamily::RepartitionHint,
2113                    keys: vec![REPARTITION_COLUMN_HINT_KEY.to_string()],
2114                },
2115            )
2116            .unwrap()
2117            .build()
2118            .unwrap();
2119
2120        assert!(
2121            !new_meta
2122                .options
2123                .extra_options
2124                .contains_key(REPARTITION_COLUMN_HINT_KEY)
2125        );
2126    }
2127
2128    #[test]
2129    fn test_repartition_column_hint_is_not_region_option() {
2130        let mut table_options = TableOptions::default();
2131        table_options
2132            .extra_options
2133            .insert(REPARTITION_COLUMN_HINT_KEY.to_string(), "col1".to_string());
2134        let table_info = TableInfoBuilder::default()
2135            .table_id(1)
2136            .table_version(0)
2137            .name("my_table")
2138            .catalog_name(DEFAULT_CATALOG_NAME)
2139            .schema_name(DEFAULT_SCHEMA_NAME)
2140            .meta(
2141                TableMetaBuilder::empty()
2142                    .schema(Arc::new(new_test_schema()))
2143                    .primary_key_indices(vec![0])
2144                    .engine("engine")
2145                    .next_column_id(3)
2146                    .options(table_options)
2147                    .build()
2148                    .unwrap(),
2149            )
2150            .build()
2151            .unwrap();
2152
2153        assert!(
2154            !table_info
2155                .to_region_options()
2156                .contains_key(REPARTITION_COLUMN_HINT_KEY)
2157        );
2158    }
2159
2160    #[test]
2161    fn test_set_auto_flush_interval() {
2162        let schema = Arc::new(new_test_schema());
2163        let table_options = TableOptions::default();
2164        let meta = TableMetaBuilder::empty()
2165            .schema(schema)
2166            .primary_key_indices(vec![0])
2167            .engine("engine")
2168            .next_column_id(3)
2169            .options(table_options)
2170            .build()
2171            .unwrap();
2172
2173        let alter_kind = AlterKind::SetTableOptions {
2174            options: vec![SetRegionOption::AutoFlushInterval(Some(
2175                std::time::Duration::from_secs(300),
2176            ))],
2177        };
2178        let new_meta = meta
2179            .builder_with_alter_kind("my_table", &alter_kind)
2180            .unwrap()
2181            .build()
2182            .unwrap();
2183
2184        assert_eq!(
2185            Some("5m"),
2186            new_meta
2187                .options
2188                .extra_options
2189                .get(AUTO_FLUSH_INTERVAL_KEY)
2190                .map(String::as_str)
2191        );
2192    }
2193
2194    #[test]
2195    fn test_set_auto_flush_interval_none_removes_existing() {
2196        let schema = Arc::new(new_test_schema());
2197        let mut table_options = TableOptions::default();
2198        table_options
2199            .extra_options
2200            .insert(AUTO_FLUSH_INTERVAL_KEY.to_string(), "5m".to_string());
2201        let meta = TableMetaBuilder::empty()
2202            .schema(schema)
2203            .primary_key_indices(vec![0])
2204            .engine("engine")
2205            .next_column_id(3)
2206            .options(table_options)
2207            .build()
2208            .unwrap();
2209
2210        let alter_kind = AlterKind::SetTableOptions {
2211            options: vec![SetRegionOption::AutoFlushInterval(None)],
2212        };
2213        let new_meta = meta
2214            .builder_with_alter_kind("my_table", &alter_kind)
2215            .unwrap()
2216            .build()
2217            .unwrap();
2218
2219        assert!(
2220            !new_meta
2221                .options
2222                .extra_options
2223                .contains_key(AUTO_FLUSH_INTERVAL_KEY)
2224        );
2225    }
2226
2227    #[test]
2228    fn test_set_and_unset_max_row_group_row_count() {
2229        let meta = TableMetaBuilder::empty()
2230            .schema(Arc::new(new_test_schema()))
2231            .primary_key_indices(vec![0])
2232            .engine("engine")
2233            .next_column_id(3)
2234            .options(TableOptions::default())
2235            .build()
2236            .unwrap();
2237
2238        let alter_kind = AlterKind::SetTableOptions {
2239            options: vec![SetRegionOption::MaxRowGroupRowCount(Some(512))],
2240        };
2241        let new_meta = meta
2242            .builder_with_alter_kind("my_table", &alter_kind)
2243            .unwrap()
2244            .build()
2245            .unwrap();
2246        assert_eq!(
2247            Some("512"),
2248            new_meta
2249                .options
2250                .extra_options
2251                .get(MAX_ROW_GROUP_ROW_COUNT)
2252                .map(String::as_str)
2253        );
2254
2255        let alter_kind = AlterKind::UnsetTableOptions {
2256            keys: vec![UnsetRegionOption::MaxRowGroupRowCount],
2257        };
2258        let new_meta = new_meta
2259            .builder_with_alter_kind("my_table", &alter_kind)
2260            .unwrap()
2261            .build()
2262            .unwrap();
2263        assert!(
2264            !new_meta
2265                .options
2266                .extra_options
2267                .contains_key(MAX_ROW_GROUP_ROW_COUNT)
2268        );
2269    }
2270
2271    #[test]
2272    fn test_add_columns_multiple_times() {
2273        let schema = Arc::new(new_test_schema());
2274        let meta = TableMetaBuilder::empty()
2275            .schema(schema)
2276            .primary_key_indices(vec![0])
2277            .engine("engine")
2278            .next_column_id(3)
2279            .build()
2280            .unwrap();
2281
2282        let alter_kind = AlterKind::AddColumns {
2283            columns: vec![
2284                AddColumnRequest {
2285                    column_schema: ColumnSchema::new(
2286                        "col3",
2287                        ConcreteDataType::int32_datatype(),
2288                        true,
2289                    ),
2290                    is_key: true,
2291                    location: None,
2292                    add_if_not_exists: true,
2293                },
2294                AddColumnRequest {
2295                    column_schema: ColumnSchema::new(
2296                        "col3",
2297                        ConcreteDataType::int32_datatype(),
2298                        true,
2299                    ),
2300                    is_key: true,
2301                    location: None,
2302                    add_if_not_exists: true,
2303                },
2304            ],
2305        };
2306        let err = meta
2307            .builder_with_alter_kind("my_table", &alter_kind)
2308            .err()
2309            .unwrap();
2310        assert_eq!(StatusCode::InvalidArguments, err.status_code());
2311    }
2312
2313    #[test]
2314    fn test_remove_columns() {
2315        let schema = Arc::new(new_test_schema());
2316        let meta = TableMetaBuilder::empty()
2317            .schema(schema.clone())
2318            .primary_key_indices(vec![0])
2319            .engine("engine")
2320            .next_column_id(3)
2321            .build()
2322            .unwrap();
2323        // Add more columns so we have enough candidate columns to remove.
2324        let meta = add_columns_to_meta(&meta);
2325
2326        let alter_kind = AlterKind::DropColumns {
2327            names: vec![String::from("col2"), String::from("my_field")],
2328        };
2329        let new_meta = meta
2330            .builder_with_alter_kind("my_table", &alter_kind)
2331            .unwrap()
2332            .build()
2333            .unwrap();
2334
2335        let names: Vec<String> = new_meta
2336            .schema
2337            .column_schemas()
2338            .iter()
2339            .map(|column_schema| column_schema.name.clone())
2340            .collect();
2341        assert_eq!(&["col1", "ts", "my_tag"], &names[..]);
2342        assert_eq!(&[0, 2], &new_meta.primary_key_indices[..]);
2343        assert_eq!(&[1], &new_meta.value_indices[..]);
2344        assert_eq!(
2345            schema.timestamp_column(),
2346            new_meta.schema.timestamp_column()
2347        );
2348    }
2349
2350    #[test]
2351    fn test_remove_multiple_columns_before_timestamp() {
2352        let column_schemas = vec![
2353            ColumnSchema::new("col1", ConcreteDataType::int32_datatype(), true),
2354            ColumnSchema::new("col2", ConcreteDataType::int32_datatype(), true),
2355            ColumnSchema::new("col3", ConcreteDataType::int32_datatype(), true),
2356            ColumnSchema::new(
2357                "ts",
2358                ConcreteDataType::timestamp_millisecond_datatype(),
2359                false,
2360            )
2361            .with_time_index(true),
2362        ];
2363        let schema = Arc::new(
2364            SchemaBuilder::try_from(column_schemas)
2365                .unwrap()
2366                .version(123)
2367                .build()
2368                .unwrap(),
2369        );
2370        let meta = TableMetaBuilder::empty()
2371            .schema(schema.clone())
2372            .primary_key_indices(vec![1])
2373            .engine("engine")
2374            .next_column_id(4)
2375            .build()
2376            .unwrap();
2377
2378        // Remove columns in reverse order to test whether timestamp index is valid.
2379        let alter_kind = AlterKind::DropColumns {
2380            names: vec![String::from("col3"), String::from("col1")],
2381        };
2382        let new_meta = meta
2383            .builder_with_alter_kind("my_table", &alter_kind)
2384            .unwrap()
2385            .build()
2386            .unwrap();
2387
2388        let names: Vec<String> = new_meta
2389            .schema
2390            .column_schemas()
2391            .iter()
2392            .map(|column_schema| column_schema.name.clone())
2393            .collect();
2394        assert_eq!(&["col2", "ts"], &names[..]);
2395        assert_eq!(&[0], &new_meta.primary_key_indices[..]);
2396        assert_eq!(&[1], &new_meta.value_indices[..]);
2397        assert_eq!(
2398            schema.timestamp_column(),
2399            new_meta.schema.timestamp_column()
2400        );
2401    }
2402
2403    #[test]
2404    fn test_add_existing_column() {
2405        let schema = Arc::new(new_test_schema());
2406        let meta = TableMetaBuilder::empty()
2407            .schema(schema)
2408            .primary_key_indices(vec![0])
2409            .engine("engine")
2410            .next_column_id(3)
2411            .build()
2412            .unwrap();
2413
2414        let alter_kind = AlterKind::AddColumns {
2415            columns: vec![AddColumnRequest {
2416                column_schema: ColumnSchema::new("col1", ConcreteDataType::string_datatype(), true),
2417                is_key: false,
2418                location: None,
2419                add_if_not_exists: false,
2420            }],
2421        };
2422
2423        let err = meta
2424            .builder_with_alter_kind("my_table", &alter_kind)
2425            .err()
2426            .unwrap();
2427        assert_eq!(StatusCode::TableColumnExists, err.status_code());
2428
2429        // Add if not exists
2430        let alter_kind = AlterKind::AddColumns {
2431            columns: vec![AddColumnRequest {
2432                column_schema: ColumnSchema::new("col1", ConcreteDataType::int32_datatype(), true),
2433                is_key: true,
2434                location: None,
2435                add_if_not_exists: true,
2436            }],
2437        };
2438        let new_meta = meta
2439            .builder_with_alter_kind("my_table", &alter_kind)
2440            .unwrap()
2441            .build()
2442            .unwrap();
2443        assert_eq!(
2444            meta.schema.column_schemas(),
2445            new_meta.schema.column_schemas()
2446        );
2447        assert_eq!(meta.schema.version() + 1, new_meta.schema.version());
2448    }
2449
2450    #[test]
2451    fn test_add_different_type_column() {
2452        let schema = Arc::new(new_test_schema());
2453        let meta = TableMetaBuilder::empty()
2454            .schema(schema)
2455            .primary_key_indices(vec![0])
2456            .engine("engine")
2457            .next_column_id(3)
2458            .build()
2459            .unwrap();
2460
2461        // Add if not exists, but different type.
2462        let alter_kind = AlterKind::AddColumns {
2463            columns: vec![AddColumnRequest {
2464                column_schema: ColumnSchema::new("col1", ConcreteDataType::string_datatype(), true),
2465                is_key: false,
2466                location: None,
2467                add_if_not_exists: true,
2468            }],
2469        };
2470        let err = meta
2471            .builder_with_alter_kind("my_table", &alter_kind)
2472            .err()
2473            .unwrap();
2474        assert_eq!(StatusCode::InvalidArguments, err.status_code());
2475    }
2476
2477    #[test]
2478    fn test_add_invalid_column() {
2479        let schema = Arc::new(new_test_schema());
2480        let meta = TableMetaBuilder::empty()
2481            .schema(schema)
2482            .primary_key_indices(vec![0])
2483            .engine("engine")
2484            .next_column_id(3)
2485            .build()
2486            .unwrap();
2487
2488        // Not nullable and no default value.
2489        let alter_kind = AlterKind::AddColumns {
2490            columns: vec![AddColumnRequest {
2491                column_schema: ColumnSchema::new(
2492                    "weny",
2493                    ConcreteDataType::string_datatype(),
2494                    false,
2495                ),
2496                is_key: false,
2497                location: None,
2498                add_if_not_exists: false,
2499            }],
2500        };
2501
2502        let err = meta
2503            .builder_with_alter_kind("my_table", &alter_kind)
2504            .err()
2505            .unwrap();
2506        assert_eq!(StatusCode::InvalidArguments, err.status_code());
2507    }
2508
2509    #[test]
2510    fn test_remove_unknown_column() {
2511        let schema = Arc::new(new_test_schema());
2512        let meta = TableMetaBuilder::empty()
2513            .schema(schema)
2514            .primary_key_indices(vec![0])
2515            .engine("engine")
2516            .next_column_id(3)
2517            .build()
2518            .unwrap();
2519
2520        let alter_kind = AlterKind::DropColumns {
2521            names: vec![String::from("unknown")],
2522        };
2523
2524        let err = meta
2525            .builder_with_alter_kind("my_table", &alter_kind)
2526            .err()
2527            .unwrap();
2528        assert_eq!(StatusCode::TableColumnNotFound, err.status_code());
2529    }
2530
2531    #[test]
2532    fn test_change_unknown_column_data_type() {
2533        let schema = Arc::new(new_test_schema());
2534        let meta = TableMetaBuilder::empty()
2535            .schema(schema)
2536            .primary_key_indices(vec![0])
2537            .engine("engine")
2538            .next_column_id(3)
2539            .build()
2540            .unwrap();
2541
2542        let alter_kind = AlterKind::ModifyColumnTypes {
2543            columns: vec![ModifyColumnTypeRequest {
2544                column_name: "unknown".to_string(),
2545                target_type: ConcreteDataType::string_datatype(),
2546            }],
2547        };
2548
2549        let err = meta
2550            .builder_with_alter_kind("my_table", &alter_kind)
2551            .err()
2552            .unwrap();
2553        assert_eq!(StatusCode::TableColumnNotFound, err.status_code());
2554    }
2555
2556    #[test]
2557    fn test_remove_key_column() {
2558        let schema = Arc::new(new_test_schema());
2559        let meta = TableMetaBuilder::empty()
2560            .schema(schema)
2561            .primary_key_indices(vec![0])
2562            .engine("engine")
2563            .next_column_id(3)
2564            .build()
2565            .unwrap();
2566
2567        // Remove column in primary key.
2568        let alter_kind = AlterKind::DropColumns {
2569            names: vec![String::from("col1")],
2570        };
2571
2572        let err = meta
2573            .builder_with_alter_kind("my_table", &alter_kind)
2574            .err()
2575            .unwrap();
2576        assert_eq!(StatusCode::InvalidArguments, err.status_code());
2577
2578        // Remove timestamp column.
2579        let alter_kind = AlterKind::DropColumns {
2580            names: vec![String::from("ts")],
2581        };
2582
2583        let err = meta
2584            .builder_with_alter_kind("my_table", &alter_kind)
2585            .err()
2586            .unwrap();
2587        assert_eq!(StatusCode::InvalidArguments, err.status_code());
2588    }
2589
2590    #[test]
2591    fn test_remove_partition_column() {
2592        let schema = Arc::new(new_test_schema());
2593        let meta = TableMetaBuilder::empty()
2594            .schema(schema)
2595            .primary_key_indices(vec![])
2596            .partition_key_indices(vec![0])
2597            .engine("engine")
2598            .next_column_id(3)
2599            .build()
2600            .unwrap();
2601        // Remove column in primary key.
2602        let alter_kind = AlterKind::DropColumns {
2603            names: vec![String::from("col1")],
2604        };
2605
2606        let err = meta
2607            .builder_with_alter_kind("my_table", &alter_kind)
2608            .err()
2609            .unwrap();
2610        assert_matches!(err, Error::RemovePartitionColumn { .. });
2611    }
2612
2613    #[test]
2614    fn test_change_key_column_data_type() {
2615        let schema = Arc::new(new_test_schema());
2616        let meta = TableMetaBuilder::empty()
2617            .schema(schema)
2618            .primary_key_indices(vec![0])
2619            .engine("engine")
2620            .next_column_id(3)
2621            .build()
2622            .unwrap();
2623
2624        // Remove column in primary key.
2625        let alter_kind = AlterKind::ModifyColumnTypes {
2626            columns: vec![ModifyColumnTypeRequest {
2627                column_name: "col1".to_string(),
2628                target_type: ConcreteDataType::string_datatype(),
2629            }],
2630        };
2631
2632        let err = meta
2633            .builder_with_alter_kind("my_table", &alter_kind)
2634            .err()
2635            .unwrap();
2636        assert_eq!(StatusCode::InvalidArguments, err.status_code());
2637
2638        // Remove timestamp column.
2639        let alter_kind = AlterKind::ModifyColumnTypes {
2640            columns: vec![ModifyColumnTypeRequest {
2641                column_name: "ts".to_string(),
2642                target_type: ConcreteDataType::string_datatype(),
2643            }],
2644        };
2645
2646        let err = meta
2647            .builder_with_alter_kind("my_table", &alter_kind)
2648            .err()
2649            .unwrap();
2650        assert_eq!(StatusCode::InvalidArguments, err.status_code());
2651    }
2652
2653    #[test]
2654    fn test_alloc_new_column() {
2655        let schema = Arc::new(new_test_schema());
2656        let mut meta = TableMetaBuilder::empty()
2657            .schema(schema)
2658            .primary_key_indices(vec![0])
2659            .engine("engine")
2660            .next_column_id(3)
2661            .build()
2662            .unwrap();
2663        assert_eq!(3, meta.next_column_id);
2664
2665        let column_schema = ColumnSchema::new("col1", ConcreteDataType::int32_datatype(), true);
2666        let desc = meta.alloc_new_column("test_table", &column_schema).unwrap();
2667
2668        assert_eq!(4, meta.next_column_id);
2669        assert_eq!(column_schema.name, desc.name);
2670    }
2671
2672    #[test]
2673    fn test_add_columns_with_location() {
2674        let schema = Arc::new(new_test_schema());
2675        let meta = TableMetaBuilder::empty()
2676            .schema(schema)
2677            .primary_key_indices(vec![0])
2678            // partition col: col1, col2
2679            .partition_key_indices(vec![0, 2])
2680            .engine("engine")
2681            .next_column_id(3)
2682            .build()
2683            .unwrap();
2684
2685        let new_meta = add_columns_to_meta_with_location(&meta);
2686        let names: Vec<String> = new_meta
2687            .schema
2688            .column_schemas()
2689            .iter()
2690            .map(|column_schema| column_schema.name.clone())
2691            .collect();
2692        assert_eq!(
2693            &[
2694                "my_tag_first",               // primary key column
2695                "col1",                       // partition column
2696                "ts",                         // timestamp column
2697                "yet_another_field_after_ts", // primary key column
2698                "my_field_after_ts",          // value column
2699                "col2",                       // partition column
2700            ],
2701            &names[..]
2702        );
2703        assert_eq!(&[0, 1, 3], &new_meta.primary_key_indices[..]);
2704        assert_eq!(&[2, 4, 5], &new_meta.value_indices[..]);
2705        assert_eq!(&[1, 5], &new_meta.partition_key_indices[..]);
2706    }
2707
2708    #[test]
2709    fn test_modify_column_fulltext_options() {
2710        let schema = Arc::new(new_test_schema());
2711        let meta = TableMetaBuilder::empty()
2712            .schema(schema)
2713            .primary_key_indices(vec![0])
2714            .engine("engine")
2715            .next_column_id(3)
2716            .build()
2717            .unwrap();
2718
2719        let alter_kind = AlterKind::SetIndexes {
2720            options: vec![SetIndexOption::Fulltext {
2721                column_name: "col1".to_string(),
2722                options: FulltextOptions::default(),
2723            }],
2724        };
2725        let err = meta
2726            .builder_with_alter_kind("my_table", &alter_kind)
2727            .err()
2728            .unwrap();
2729        assert_eq!(
2730            "Invalid column option, column name: col1, error: FULLTEXT index only supports string type",
2731            err.to_string()
2732        );
2733
2734        // Add a string column and make it fulltext indexed
2735        let new_meta = add_columns_to_meta_with_location(&meta);
2736        let alter_kind = AlterKind::SetIndexes {
2737            options: vec![SetIndexOption::Fulltext {
2738                column_name: "my_tag_first".to_string(),
2739                options: FulltextOptions::new_unchecked(
2740                    true,
2741                    FulltextAnalyzer::Chinese,
2742                    true,
2743                    FulltextBackend::Bloom,
2744                    1000,
2745                    0.01,
2746                ),
2747            }],
2748        };
2749        let new_meta = new_meta
2750            .builder_with_alter_kind("my_table", &alter_kind)
2751            .unwrap()
2752            .build()
2753            .unwrap();
2754        let column_schema = new_meta
2755            .schema
2756            .column_schema_by_name("my_tag_first")
2757            .unwrap();
2758        let fulltext_options = column_schema.fulltext_options().unwrap().unwrap();
2759        assert!(fulltext_options.enable);
2760        assert_eq!(
2761            datatypes::schema::FulltextAnalyzer::Chinese,
2762            fulltext_options.analyzer
2763        );
2764        assert!(fulltext_options.case_sensitive);
2765
2766        let alter_kind = AlterKind::UnsetIndexes {
2767            options: vec![UnsetIndexOption::Fulltext {
2768                column_name: "my_tag_first".to_string(),
2769            }],
2770        };
2771        let new_meta = new_meta
2772            .builder_with_alter_kind("my_table", &alter_kind)
2773            .unwrap()
2774            .build()
2775            .unwrap();
2776        let column_schema = new_meta
2777            .schema
2778            .column_schema_by_name("my_tag_first")
2779            .unwrap();
2780        let fulltext_options = column_schema.fulltext_options().unwrap().unwrap();
2781        assert!(!fulltext_options.enable);
2782    }
2783
2784    #[test]
2785    fn test_table_info_serde_compatibility() {
2786        // "serialized" is generated by the following codes before this refactor (PR 7626):
2787        //
2788        // ```Rust
2789        // serde_json::to_string(&RawTableInfo::from(TableInfo {
2790        //     ident: TableIdent {
2791        //         table_id: 1024,
2792        //         version: 1,
2793        //     },
2794        //     name: "foo".to_string(),
2795        //     desc: Some("my table".to_string()),
2796        //     catalog_name: "greptime".to_string(),
2797        //     schema_name: "public".to_string(),
2798        //     meta: TableMeta {
2799        //         schema: Arc::new(new_test_schema()),
2800        //         primary_key_indices: vec![0],
2801        //         value_indices: vec![1, 2],
2802        //         engine: "mito".to_string(),
2803        //         next_column_id: 3,
2804        //         options: TableOptions {
2805        //             ttl: Some(common_time::TimeToLive::Duration(
2806        //                 std::time::Duration::from_secs(3600),
2807        //             )),
2808        //             ..Default::default()
2809        //         },
2810        //         created_on: DateTime::<Utc>::MIN_UTC,
2811        //         updated_on: DateTime::<Utc>::MAX_UTC,
2812        //         partition_key_indices: vec![2],
2813        //         column_ids: vec![0, 1, 2],
2814        //     },
2815        //     table_type: TableType::Base,
2816        // }))
2817        // ```
2818        let serialized = r#"{"ident":{"table_id":1024,"version":1},"name":"foo","desc":"my table","catalog_name":"greptime","schema_name":"public","meta":{"schema":{"column_schemas":[{"name":"col1","data_type":{"Int32":{}},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"ts","data_type":{"Timestamp":{"Millisecond":null}},"is_nullable":false,"is_time_index":true,"default_constraint":null,"metadata":{"greptime:time_index":"true"}},{"name":"col2","data_type":{"Int32":{}},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}}],"timestamp_index":1,"version":123},"primary_key_indices":[0],"value_indices":[1,2],"engine":"mito","next_column_id":3,"options":{"write_buffer_size":null,"ttl":"1h","skip_wal":false,"extra_options":{}},"created_on":"-262143-01-01T00:00:00Z","updated_on":"+262142-12-31T23:59:59.999999999Z","partition_key_indices":[2],"column_ids":[0,1,2]},"table_type":"Base"}"#;
2819
2820        let actual: TableInfo = serde_json::from_str(serialized).unwrap();
2821        let expected = TableInfo {
2822            ident: TableIdent {
2823                table_id: 1024,
2824                version: 1,
2825            },
2826            name: "foo".to_string(),
2827            desc: Some("my table".to_string()),
2828            catalog_name: "greptime".to_string(),
2829            schema_name: "public".to_string(),
2830            meta: TableMeta {
2831                schema: Arc::new(new_test_schema()),
2832                primary_key_indices: vec![0],
2833                value_indices: vec![1, 2],
2834                engine: "mito".to_string(),
2835                next_column_id: 3,
2836                options: TableOptions {
2837                    ttl: Some(common_time::TimeToLive::Duration(
2838                        std::time::Duration::from_secs(3600),
2839                    )),
2840                    ..Default::default()
2841                },
2842                created_on: DateTime::<Utc>::MIN_UTC,
2843                updated_on: DateTime::<Utc>::MAX_UTC,
2844                partition_key_indices: vec![2],
2845                column_ids: vec![0, 1, 2],
2846            },
2847            table_type: TableType::Base,
2848        };
2849        assert_eq!(actual, expected);
2850    }
2851
2852    use crate::requests::{SEMANTIC_METRIC_UNIT, SEMANTIC_SIGNAL_TYPE};
2853
2854    /// `host` is a tag, `service` and `note` are string fields, `payload` is a
2855    /// binary field.
2856    fn semantic_test_meta() -> TableMeta {
2857        let column_schemas = vec![
2858            ColumnSchema::new("host", ConcreteDataType::string_datatype(), true),
2859            ColumnSchema::new(
2860                "ts",
2861                ConcreteDataType::timestamp_millisecond_datatype(),
2862                false,
2863            )
2864            .with_time_index(true),
2865            ColumnSchema::new("payload", ConcreteDataType::binary_datatype(), true),
2866            ColumnSchema::new("service", ConcreteDataType::string_datatype(), true),
2867            ColumnSchema::new("note", ConcreteDataType::string_datatype(), true),
2868        ];
2869        let schema = Arc::new(
2870            SchemaBuilder::try_from(column_schemas)
2871                .unwrap()
2872                .build()
2873                .unwrap(),
2874        );
2875        TableMetaBuilder::empty()
2876            .schema(schema)
2877            .primary_key_indices(vec![0])
2878            .engine("engine")
2879            .next_column_id(5)
2880            .build()
2881            .unwrap()
2882    }
2883
2884    #[test]
2885    fn test_set_semantic_annotations() {
2886        let meta = semantic_test_meta();
2887        // `service` is a plain field: entity columns may be tags or fields.
2888        let alter_kind = AlterKind::SetAnnotations {
2889            family: AnnotationFamily::Semantic,
2890            options: vec![
2891                (SEMANTIC_SIGNAL_TYPE.to_string(), "trace".to_string()),
2892                (
2893                    "greptime.semantic.entity.service.id".to_string(),
2894                    "service".to_string(),
2895                ),
2896            ],
2897        };
2898        let new_meta = meta
2899            .builder_with_alter_kind("my_table", &alter_kind)
2900            .unwrap()
2901            .build()
2902            .unwrap();
2903        assert_eq!(
2904            new_meta.options.extra_options.get(SEMANTIC_SIGNAL_TYPE),
2905            Some(&"trace".to_string())
2906        );
2907        assert_eq!(
2908            new_meta
2909                .options
2910                .extra_options
2911                .get("greptime.semantic.entity.service.id"),
2912            Some(&"service".to_string())
2913        );
2914    }
2915
2916    #[test]
2917    fn test_repartition_hint_batch_must_be_single_key() {
2918        let meta = TableMetaBuilder::empty()
2919            .schema(Arc::new(new_test_schema()))
2920            .primary_key_indices(vec![0])
2921            .engine("engine")
2922            .next_column_id(3)
2923            .build()
2924            .unwrap();
2925
2926        // Direct gRPC can hand the mutation layer a duplicated batch the
2927        // converter never saw; last-write-wins must not silently apply.
2928        let dup_set = AlterKind::SetAnnotations {
2929            family: AnnotationFamily::RepartitionHint,
2930            options: vec![
2931                (REPARTITION_COLUMN_HINT_KEY.to_string(), "col1".to_string()),
2932                (REPARTITION_COLUMN_HINT_KEY.to_string(), "col2".to_string()),
2933            ],
2934        };
2935        let err = meta
2936            .builder_with_alter_kind("my_table", &dup_set)
2937            .err()
2938            .unwrap();
2939        assert!(
2940            err.to_string().contains("must be altered separately"),
2941            "{err}"
2942        );
2943
2944        let dup_unset = AlterKind::UnsetAnnotations {
2945            family: AnnotationFamily::RepartitionHint,
2946            keys: vec![
2947                REPARTITION_COLUMN_HINT_KEY.to_string(),
2948                REPARTITION_COLUMN_HINT_KEY.to_string(),
2949            ],
2950        };
2951        let err = meta
2952            .builder_with_alter_kind("my_table", &dup_unset)
2953            .err()
2954            .unwrap();
2955        assert!(
2956            err.to_string().contains("must be altered separately"),
2957            "{err}"
2958        );
2959    }
2960
2961    #[test]
2962    fn test_set_semantic_annotations_rejects_invalid() {
2963        let meta = semantic_test_meta();
2964        let cases = [
2965            (
2966                "greptime.semantic.unknown_key",
2967                "x",
2968                "unknown semantic option",
2969            ),
2970            (SEMANTIC_SIGNAL_TYPE, "garbage", "invalid value"),
2971            (
2972                "greptime.semantic.entity.host.id",
2973                "no_such_column",
2974                "no_such_column",
2975            ),
2976            (
2977                "greptime.semantic.entity.host.id",
2978                "payload",
2979                "cannot render as a string",
2980            ),
2981        ];
2982        for (key, value, needle) in cases {
2983            let alter_kind = AlterKind::SetAnnotations {
2984                family: AnnotationFamily::Semantic,
2985                options: vec![(key.to_string(), value.to_string())],
2986            };
2987            let err = meta
2988                .builder_with_alter_kind("my_table", &alter_kind)
2989                .err()
2990                .unwrap();
2991            assert!(
2992                err.to_string().contains(needle),
2993                "key `{key}`: unexpected error `{err}`"
2994            );
2995        }
2996
2997        // ALTER keeps missing columns on the 4002 contract (pinned by sqlness);
2998        // the shared validator must not collapse it into InvalidArguments.
2999        let missing = meta
3000            .builder_with_alter_kind(
3001                "my_table",
3002                &AlterKind::SetAnnotations {
3003                    family: AnnotationFamily::Semantic,
3004                    options: vec![(
3005                        "greptime.semantic.entity.host.id".to_string(),
3006                        "no_such_column".to_string(),
3007                    )],
3008                },
3009            )
3010            .err()
3011            .unwrap();
3012        assert_eq!(
3013            common_error::status_code::StatusCode::TableColumnNotFound,
3014            common_error::ext::ErrorExt::status_code(&missing)
3015        );
3016    }
3017
3018    #[test]
3019    fn test_unset_semantic_annotations() {
3020        let mut meta = semantic_test_meta();
3021        meta.options
3022            .extra_options
3023            .insert(SEMANTIC_SIGNAL_TYPE.to_string(), "trace".to_string());
3024        // A key this version does not recognise, still inside the namespace.
3025        meta.options
3026            .extra_options
3027            .insert("greptime.semantic.future_key".to_string(), "x".to_string());
3028
3029        let alter_kind = AlterKind::UnsetAnnotations {
3030            family: AnnotationFamily::Semantic,
3031            keys: vec![
3032                SEMANTIC_SIGNAL_TYPE.to_string(),
3033                "greptime.semantic.future_key".to_string(),
3034                // Absent key: removal is a no-op, not an error.
3035                SEMANTIC_METRIC_UNIT.to_string(),
3036            ],
3037        };
3038        let new_meta = meta
3039            .builder_with_alter_kind("my_table", &alter_kind)
3040            .unwrap()
3041            .build()
3042            .unwrap();
3043        assert!(
3044            !new_meta
3045                .options
3046                .extra_options
3047                .contains_key(SEMANTIC_SIGNAL_TYPE)
3048        );
3049        assert!(
3050            !new_meta
3051                .options
3052                .extra_options
3053                .contains_key("greptime.semantic.future_key")
3054        );
3055
3056        let outside = AlterKind::UnsetAnnotations {
3057            family: AnnotationFamily::Semantic,
3058            keys: vec!["ttl".to_string()],
3059        };
3060        let err = meta
3061            .builder_with_alter_kind("my_table", &outside)
3062            .err()
3063            .unwrap();
3064        assert!(err.to_string().contains("annotation namespace"), "{err}");
3065    }
3066
3067    #[test]
3068    fn test_modify_entity_column_type_keeps_string_form() {
3069        let mut meta = semantic_test_meta();
3070        meta.options.extra_options.insert(
3071            "greptime.semantic.entity.service.id".to_string(),
3072            "service".to_string(),
3073        );
3074
3075        let alter_kind = AlterKind::ModifyColumnTypes {
3076            columns: vec![ModifyColumnTypeRequest {
3077                column_name: "service".to_string(),
3078                target_type: ConcreteDataType::binary_datatype(),
3079            }],
3080        };
3081        let err = meta
3082            .builder_with_alter_kind("my_table", &alter_kind)
3083            .err()
3084            .unwrap();
3085        assert!(
3086            err.to_string()
3087                .contains("must keep a type that renders as a string"),
3088            "{err}"
3089        );
3090
3091        // An unreferenced column may still change to a non-string form.
3092        let alter_kind = AlterKind::ModifyColumnTypes {
3093            columns: vec![ModifyColumnTypeRequest {
3094                column_name: "note".to_string(),
3095                target_type: ConcreteDataType::binary_datatype(),
3096            }],
3097        };
3098        meta.builder_with_alter_kind("my_table", &alter_kind)
3099            .unwrap();
3100    }
3101
3102    #[test]
3103    fn test_stale_entity_declaration_guards_readd_and_reports_missing_column() {
3104        // A stale declaration: `gone` was dropped after being declared.
3105        let mut meta = semantic_test_meta();
3106        meta.options.extra_options.insert(
3107            "greptime.semantic.entity.service.id".to_string(),
3108            "gone".to_string(),
3109        );
3110
3111        // MODIFY on the missing column keeps the ColumnNotExists contract
3112        // (4002), stale declaration or not.
3113        let modify = AlterKind::ModifyColumnTypes {
3114            columns: vec![ModifyColumnTypeRequest {
3115                column_name: "gone".to_string(),
3116                target_type: ConcreteDataType::binary_datatype(),
3117            }],
3118        };
3119        let err = meta
3120            .builder_with_alter_kind("my_table", &modify)
3121            .err()
3122            .unwrap();
3123        assert_eq!(
3124            common_error::status_code::StatusCode::TableColumnNotFound,
3125            common_error::ext::ErrorExt::status_code(&err)
3126        );
3127
3128        // Re-adding the declared column must not hand the declaration a
3129        // non-string type...
3130        let add = |ty: ConcreteDataType| AlterKind::AddColumns {
3131            columns: vec![AddColumnRequest {
3132                column_schema: ColumnSchema::new("gone", ty, true),
3133                is_key: false,
3134                location: None,
3135                add_if_not_exists: false,
3136            }],
3137        };
3138        let err = meta
3139            .builder_with_alter_kind("my_table", &add(ConcreteDataType::binary_datatype()))
3140            .err()
3141            .unwrap();
3142        assert!(
3143            err.to_string()
3144                .contains("must keep a type that renders as a string"),
3145            "{err}"
3146        );
3147
3148        // ...while a string re-add re-satisfies it.
3149        meta.builder_with_alter_kind("my_table", &add(ConcreteDataType::string_datatype()))
3150            .unwrap();
3151    }
3152}