Skip to main content

table/
metadata.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::{HashMap, HashSet};
16use std::sync::Arc;
17
18use chrono::{DateTime, Utc};
19use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
20use common_macro::ToMetaBuilder;
21use common_query::AddColumnLocation;
22use datafusion_expr::TableProviderFilterPushDown;
23pub use datatypes::error::{Error as ConvertError, Result as ConvertResult};
24use datatypes::schema::{
25    ColumnSchema, FulltextOptions, Schema, SchemaBuilder, SchemaRef, SkippingIndexOptions,
26};
27use derive_builder::Builder;
28use serde::{Deserialize, Deserializer, Serialize};
29use snafu::{OptionExt, ResultExt, ensure};
30use store_api::metric_engine_consts::PHYSICAL_TABLE_METADATA_KEY;
31use store_api::mito_engine_options::{
32    APPEND_MODE_KEY, AUTO_FLUSH_INTERVAL_KEY, COMPACTION_TYPE, COMPACTION_TYPE_TWCS,
33    MAX_ROW_GROUP_ROW_COUNT, MERGE_MODE_KEY, SKIP_WAL_KEY, SST_FORMAT_KEY,
34};
35use store_api::region_request::{SetRegionOption, UnsetRegionOption};
36use store_api::storage::{ColumnDescriptor, ColumnDescriptorBuilder, ColumnId};
37
38use crate::error::{self, Result};
39use crate::requests::{
40    AddColumnRequest, AlterKind, AnnotationContext, AnnotationFamily, AnnotationValidationError,
41    ModifyColumnTypeRequest, REPARTITION_COLUMN_HINT_KEY, SetDefaultRequest, SetIndexOption,
42    TableOptions, UnsetIndexOption, has_stable_string_form, parse_entity_columns,
43    parse_entity_option_key, validate_and_normalize_annotation,
44};
45use crate::table_reference::TableReference;
46
47pub type TableId = u32;
48pub type TableVersion = u64;
49
50/// Indicates whether and how a filter expression can be handled by a
51/// Table for table scans.
52#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
53pub enum FilterPushDownType {
54    /// The expression cannot be used by the provider.
55    Unsupported,
56    /// The expression can be used to help minimise the data retrieved,
57    /// but the provider cannot guarantee that all returned tuples
58    /// satisfy the filter. The Filter plan node containing this expression
59    /// will be preserved.
60    Inexact,
61    /// The provider guarantees that all returned data satisfies this
62    /// filter expression. The Filter plan node containing this expression
63    /// will be removed.
64    Exact,
65}
66
67impl From<TableProviderFilterPushDown> for FilterPushDownType {
68    fn from(value: TableProviderFilterPushDown) -> Self {
69        match value {
70            TableProviderFilterPushDown::Unsupported => FilterPushDownType::Unsupported,
71            TableProviderFilterPushDown::Inexact => FilterPushDownType::Inexact,
72            TableProviderFilterPushDown::Exact => FilterPushDownType::Exact,
73        }
74    }
75}
76
77impl From<FilterPushDownType> for TableProviderFilterPushDown {
78    fn from(value: FilterPushDownType) -> Self {
79        match value {
80            FilterPushDownType::Unsupported => TableProviderFilterPushDown::Unsupported,
81            FilterPushDownType::Inexact => TableProviderFilterPushDown::Inexact,
82            FilterPushDownType::Exact => TableProviderFilterPushDown::Exact,
83        }
84    }
85}
86
87/// Indicates the type of this table for metadata/catalog purposes.
88#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
89pub enum TableType {
90    /// An ordinary physical table.
91    Base,
92    /// A non-materialised table that itself uses a query internally to provide data.
93    View,
94    /// A transient table.
95    Temporary,
96}
97
98impl std::fmt::Display for TableType {
99    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
100        match self {
101            TableType::Base => f.write_str("BASE TABLE"),
102            TableType::Temporary => f.write_str("TEMPORARY"),
103            TableType::View => f.write_str("VIEW"),
104        }
105    }
106}
107
108impl From<TableType> for datafusion::datasource::TableType {
109    fn from(t: TableType) -> datafusion::datasource::TableType {
110        match t {
111            TableType::Base => datafusion::datasource::TableType::Base,
112            TableType::View => datafusion::datasource::TableType::View,
113            TableType::Temporary => datafusion::datasource::TableType::Temporary,
114        }
115    }
116}
117
118/// Identifier of the table.
119#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Default)]
120pub struct TableIdent {
121    /// Unique id of this table.
122    pub table_id: TableId,
123    /// Version of the table, bumped when metadata (such as schema) of the table
124    /// being changed.
125    pub version: TableVersion,
126}
127
128/// The table metadata.
129///
130/// Note: if you add new fields to this struct, please ensure 'new_meta_builder' function works.
131#[derive(Clone, Debug, Builder, PartialEq, Eq, ToMetaBuilder, Serialize)]
132#[builder(pattern = "mutable", custom_constructor)]
133pub struct TableMeta {
134    pub schema: SchemaRef,
135    /// The indices of columns in primary key. Note that the index of timestamp column
136    /// is not included in these indices.
137    pub primary_key_indices: Vec<usize>,
138    #[builder(default = "self.default_value_indices()?")]
139    pub value_indices: Vec<usize>,
140    #[builder(default, setter(into))]
141    pub engine: String,
142    pub next_column_id: ColumnId,
143    /// Table options.
144    #[builder(default)]
145    pub options: TableOptions,
146    #[builder(default = "Utc::now()")]
147    pub created_on: DateTime<Utc>,
148    #[builder(default = "self.default_updated_on()")]
149    pub updated_on: DateTime<Utc>,
150    #[builder(default = "Vec::new()")]
151    pub partition_key_indices: Vec<usize>,
152    #[builder(default = "Vec::new()")]
153    pub column_ids: Vec<ColumnId>,
154}
155
156impl TableMeta {
157    pub fn empty() -> Self {
158        Self {
159            schema: Arc::new(Schema::new(vec![])),
160            primary_key_indices: vec![],
161            value_indices: vec![],
162            engine: "".to_string(),
163            next_column_id: 0,
164            options: TableOptions::default(),
165            created_on: Utc::now(),
166            updated_on: Utc::now(),
167            partition_key_indices: vec![],
168            column_ids: vec![],
169        }
170    }
171}
172
173impl<'de> Deserialize<'de> for TableMeta {
174    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
175    where
176        D: Deserializer<'de>,
177    {
178        #[derive(Deserialize)]
179        struct RawTableMeta {
180            schema: SchemaRef,
181            primary_key_indices: Vec<usize>,
182            value_indices: Vec<usize>,
183            engine: String,
184            next_column_id: ColumnId,
185            options: TableOptions,
186            created_on: DateTime<Utc>,
187            updated_on: Option<DateTime<Utc>>,
188            #[serde(default)]
189            partition_key_indices: Vec<usize>,
190            #[serde(default)]
191            column_ids: Vec<ColumnId>,
192        }
193
194        let RawTableMeta {
195            schema,
196            primary_key_indices,
197            value_indices,
198            engine,
199            next_column_id,
200            options,
201            created_on,
202            updated_on,
203            partition_key_indices,
204            column_ids,
205        } = RawTableMeta::deserialize(deserializer)?;
206
207        Ok(Self {
208            schema,
209            primary_key_indices,
210            value_indices,
211            engine,
212            next_column_id,
213            options,
214            created_on,
215            updated_on: updated_on.unwrap_or(created_on),
216            partition_key_indices,
217            column_ids,
218        })
219    }
220}
221
222impl TableMetaBuilder {
223    /// Note: Please always use [new_meta_builder] to create new [TableMetaBuilder].
224    #[cfg(any(test, feature = "testing"))]
225    pub fn empty() -> Self {
226        Self {
227            schema: None,
228            primary_key_indices: None,
229            value_indices: None,
230            engine: None,
231            next_column_id: None,
232            options: None,
233            created_on: None,
234            updated_on: None,
235            partition_key_indices: None,
236            column_ids: None,
237        }
238    }
239}
240
241impl TableMetaBuilder {
242    fn default_value_indices(&self) -> std::result::Result<Vec<usize>, String> {
243        match (&self.primary_key_indices, &self.schema) {
244            (Some(v), Some(schema)) => {
245                let column_schemas = schema.column_schemas();
246                Ok((0..column_schemas.len())
247                    .filter(|idx| !v.contains(idx))
248                    .collect())
249            }
250            _ => Err("Missing primary_key_indices or schema to create value_indices".to_string()),
251        }
252    }
253
254    fn default_updated_on(&self) -> DateTime<Utc> {
255        self.created_on.unwrap_or_default()
256    }
257
258    pub fn new_external_table() -> Self {
259        Self {
260            schema: None,
261            primary_key_indices: Some(Vec::new()),
262            value_indices: Some(Vec::new()),
263            engine: None,
264            next_column_id: Some(0),
265            options: None,
266            created_on: None,
267            updated_on: None,
268            partition_key_indices: None,
269            column_ids: None,
270        }
271    }
272}
273
274/// The result after splitting requests by column location info.
275struct SplitResult<'a> {
276    /// column requests should be added at first place.
277    columns_at_first: Vec<&'a AddColumnRequest>,
278    /// column requests should be added after already exist columns.
279    columns_at_after: HashMap<String, Vec<&'a AddColumnRequest>>,
280    /// column requests should be added at last place.
281    columns_at_last: Vec<&'a AddColumnRequest>,
282    /// all column names should be added.
283    column_names: Vec<String>,
284}
285
286impl TableMeta {
287    pub fn row_key_column_names(&self) -> impl Iterator<Item = &String> {
288        let columns_schemas = &self.schema.column_schemas();
289        self.primary_key_indices
290            .iter()
291            .map(|idx| &columns_schemas[*idx].name)
292    }
293
294    pub fn field_column_names(&self) -> impl Iterator<Item = &String> {
295        // `value_indices` is wrong under distributed mode. Use the logic copied from DESC TABLE
296        let columns_schemas = self.schema.column_schemas();
297        let primary_key_indices = &self.primary_key_indices;
298        columns_schemas
299            .iter()
300            .enumerate()
301            .filter(|(i, cs)| !primary_key_indices.contains(i) && !cs.is_time_index())
302            .map(|(_, cs)| &cs.name)
303    }
304
305    pub fn partition_column_names(&self) -> impl Iterator<Item = &String> {
306        let columns_schemas = &self.schema.column_schemas();
307        self.partition_key_indices
308            .iter()
309            .map(|idx| &columns_schemas[*idx].name)
310    }
311
312    pub fn partition_columns(&self) -> impl Iterator<Item = &ColumnSchema> {
313        self.partition_key_indices
314            .iter()
315            .map(|idx| &self.schema.column_schemas()[*idx])
316    }
317
318    /// Returns the new [TableMetaBuilder] after applying given `alter_kind`.
319    ///
320    /// The returned builder would derive the next column id of this meta.
321    pub fn builder_with_alter_kind(
322        &self,
323        table_name: &str,
324        alter_kind: &AlterKind,
325    ) -> Result<TableMetaBuilder> {
326        let mut builder = match alter_kind {
327            AlterKind::AddColumns { columns } => self.add_columns(table_name, columns),
328            AlterKind::DropColumns { names } => self.remove_columns(table_name, names),
329            AlterKind::ModifyColumnTypes { columns } => {
330                self.modify_column_types(table_name, columns)
331            }
332            // No need to rebuild table meta when renaming tables.
333            AlterKind::RenameTable { .. } => Ok(self.new_meta_builder()),
334            AlterKind::SetTableOptions { options } => self.set_table_options(options),
335            AlterKind::UnsetTableOptions { keys } => self.unset_table_options(keys),
336            AlterKind::SetAnnotations { family, options } => {
337                self.set_annotations(table_name, *family, options)
338            }
339            AlterKind::UnsetAnnotations { family, keys } => {
340                self.unset_annotations(table_name, *family, keys)
341            }
342            AlterKind::SetIndexes { options } => self.set_indexes(table_name, options),
343            AlterKind::UnsetIndexes { options } => self.unset_indexes(table_name, options),
344            AlterKind::DropDefaults { names } => self.drop_defaults(table_name, names),
345            AlterKind::SetDefaults { defaults } => self.set_defaults(table_name, defaults),
346        }?;
347        let _ = builder.updated_on(Utc::now());
348        Ok(builder)
349    }
350
351    /// Creates a [TableMetaBuilder] with modified table options.
352    fn set_table_options(&self, requests: &[SetRegionOption]) -> Result<TableMetaBuilder> {
353        let mut new_options = self.options.clone();
354
355        for request in requests {
356            match request {
357                SetRegionOption::WriteBufferSize(new_write_buffer_size) => {
358                    new_options.write_buffer_size = *new_write_buffer_size;
359                }
360                SetRegionOption::Ttl(new_ttl) => {
361                    new_options.ttl = *new_ttl;
362                }
363                SetRegionOption::Twsc(key, value) => {
364                    if !value.is_empty() {
365                        new_options.extra_options.insert(key.clone(), value.clone());
366                        // Ensure node restart correctly.
367                        new_options.extra_options.insert(
368                            COMPACTION_TYPE.to_string(),
369                            COMPACTION_TYPE_TWCS.to_string(),
370                        );
371                    } else {
372                        // Invalidate the previous change option if an empty value has been set.
373                        new_options.extra_options.remove(key.as_str());
374                    }
375                }
376                SetRegionOption::Format(value) => {
377                    new_options
378                        .extra_options
379                        .insert(SST_FORMAT_KEY.to_string(), value.clone());
380                }
381                SetRegionOption::AppendMode(value) => {
382                    new_options
383                        .extra_options
384                        .insert(APPEND_MODE_KEY.to_string(), value.to_string());
385                    if *value {
386                        new_options.extra_options.remove(MERGE_MODE_KEY);
387                    }
388                }
389                SetRegionOption::AutoFlushInterval(new_interval) => {
390                    if let Some(interval) = new_interval {
391                        new_options.extra_options.insert(
392                            AUTO_FLUSH_INTERVAL_KEY.to_string(),
393                            humantime::format_duration(*interval).to_string(),
394                        );
395                    } else {
396                        new_options.extra_options.remove(AUTO_FLUSH_INTERVAL_KEY);
397                    }
398                }
399                SetRegionOption::MaxRowGroupRowCount(row_count) => {
400                    if let Some(row_count) = row_count {
401                        new_options
402                            .extra_options
403                            .insert(MAX_ROW_GROUP_ROW_COUNT.to_string(), row_count.to_string());
404                    } else {
405                        new_options.extra_options.remove(MAX_ROW_GROUP_ROW_COUNT);
406                    }
407                }
408                SetRegionOption::SkipWal => {
409                    new_options.skip_wal = true;
410                    // Keep the explicit table option so it remains distinguishable
411                    // from a value inherited from the schema.
412                    new_options
413                        .extra_options
414                        .insert(SKIP_WAL_KEY.to_string(), true.to_string());
415                }
416            }
417        }
418        let mut builder = self.new_meta_builder();
419        builder.options(new_options);
420
421        Ok(builder)
422    }
423
424    fn unset_table_options(&self, requests: &[UnsetRegionOption]) -> Result<TableMetaBuilder> {
425        let requests = requests.iter().map(Into::into).collect::<Vec<_>>();
426        self.set_table_options(&requests)
427    }
428
429    /// Applies an annotation SET. Validation lives here, on the mutation
430    /// path, so it runs both at frontend verification and again inside the
431    /// alter procedure's prepare step — under the table lock, against fresh
432    /// metadata.
433    fn set_annotations(
434        &self,
435        table_name: &str,
436        family: AnnotationFamily,
437        options: &[(String, String)],
438    ) -> Result<TableMetaBuilder> {
439        ensure!(
440            !family.requires_single_key() || options.len() == 1,
441            error::InvalidAlterRequestSnafu {
442                table: table_name,
443                err: family.mixed_batch_error(),
444            }
445        );
446        let cx = AnnotationContext {
447            schema: &self.schema,
448            partition_key_indices: &self.partition_key_indices,
449        };
450        let mut new_options = self.options.clone();
451        for (key, value) in options {
452            ensure!(
453                AnnotationFamily::of_key(key) == Some(family),
454                error::InvalidAlterRequestSnafu {
455                    table: table_name,
456                    err: format!(
457                        "`{key}` is outside the `{}` annotation namespace",
458                        family.namespace()
459                    ),
460                }
461            );
462            let checked = validate_and_normalize_annotation(family, &cx, key, value).map_err(
463                |e| match e {
464                    AnnotationValidationError::ColumnNotFound { column } => {
465                        error::ColumnNotExistsSnafu {
466                            column_name: column,
467                            table_name,
468                        }
469                        .build()
470                    }
471                    other => error::InvalidAlterRequestSnafu {
472                        table: table_name,
473                        err: other.to_string(),
474                    }
475                    .build(),
476                },
477            )?;
478            new_options.extra_options.insert(key.clone(), checked);
479        }
480        let mut builder = self.new_meta_builder();
481        builder.options(new_options);
482        Ok(builder)
483    }
484
485    /// Applies an annotation UNSET. Deliberately lenient inside the family's
486    /// namespace: keys this version does not recognise may still be removed,
487    /// so options left behind by other versions can be cleaned up.
488    fn unset_annotations(
489        &self,
490        table_name: &str,
491        family: AnnotationFamily,
492        keys: &[String],
493    ) -> Result<TableMetaBuilder> {
494        ensure!(
495            !family.requires_single_key() || keys.len() == 1,
496            error::InvalidAlterRequestSnafu {
497                table: table_name,
498                err: family.mixed_batch_error(),
499            }
500        );
501        let mut new_options = self.options.clone();
502        for key in keys {
503            ensure!(
504                AnnotationFamily::of_key(key) == Some(family),
505                error::InvalidAlterRequestSnafu {
506                    table: table_name,
507                    err: format!(
508                        "`{key}` is outside the `{}` annotation namespace",
509                        family.namespace()
510                    ),
511                }
512            );
513            new_options.extra_options.remove(key);
514        }
515        let mut builder = self.new_meta_builder();
516        builder.options(new_options);
517        Ok(builder)
518    }
519
520    fn set_indexes(
521        &self,
522        table_name: &str,
523        requests: &[SetIndexOption],
524    ) -> Result<TableMetaBuilder> {
525        let table_schema = &self.schema;
526        let mut set_index_options: HashMap<&str, Vec<_>> = HashMap::new();
527        for request in requests {
528            let column_name = request.column_name();
529            table_schema
530                .column_index_by_name(column_name)
531                .with_context(|| error::ColumnNotExistsSnafu {
532                    column_name,
533                    table_name,
534                })?;
535            set_index_options
536                .entry(column_name)
537                .or_default()
538                .push(request);
539        }
540
541        let mut meta_builder = self.new_meta_builder();
542        let mut columns: Vec<_> = Vec::with_capacity(table_schema.column_schemas().len());
543        for mut column in table_schema.column_schemas().iter().cloned() {
544            if let Some(request) = set_index_options.get(column.name.as_str()) {
545                for request in request {
546                    self.set_index(&mut column, request)?;
547                }
548            }
549            columns.push(column);
550        }
551
552        let mut builder = SchemaBuilder::try_from_columns(columns)
553            .with_context(|_| error::SchemaBuildSnafu {
554                msg: format!("Failed to convert column schemas into schema for table {table_name}"),
555            })?
556            .version(table_schema.version() + 1);
557
558        for (k, v) in table_schema.metadata().iter() {
559            builder = builder.add_metadata(k, v);
560        }
561
562        let new_schema = builder.build().with_context(|_| {
563            let column_names = requests
564                .iter()
565                .map(|request| request.column_name())
566                .collect::<Vec<_>>();
567            error::SchemaBuildSnafu {
568                msg: format!(
569                    "Table {table_name} cannot set index options with columns {column_names:?}",
570                ),
571            }
572        })?;
573        let _ = meta_builder
574            .schema(Arc::new(new_schema))
575            .primary_key_indices(self.primary_key_indices.clone());
576
577        Ok(meta_builder)
578    }
579
580    fn unset_indexes(
581        &self,
582        table_name: &str,
583        requests: &[UnsetIndexOption],
584    ) -> Result<TableMetaBuilder> {
585        let table_schema = &self.schema;
586        let mut set_index_options: HashMap<&str, Vec<_>> = HashMap::new();
587        for request in requests {
588            let column_name = request.column_name();
589            table_schema
590                .column_index_by_name(column_name)
591                .with_context(|| error::ColumnNotExistsSnafu {
592                    column_name,
593                    table_name,
594                })?;
595            set_index_options
596                .entry(column_name)
597                .or_default()
598                .push(request);
599        }
600
601        let mut meta_builder = self.new_meta_builder();
602        let mut columns: Vec<_> = Vec::with_capacity(table_schema.column_schemas().len());
603        for mut column in table_schema.column_schemas().iter().cloned() {
604            if let Some(request) = set_index_options.get(column.name.as_str()) {
605                for request in request {
606                    self.unset_index(&mut column, request)?;
607                }
608            }
609            columns.push(column);
610        }
611
612        let mut builder = SchemaBuilder::try_from_columns(columns)
613            .with_context(|_| error::SchemaBuildSnafu {
614                msg: format!("Failed to convert column schemas into schema for table {table_name}"),
615            })?
616            .version(table_schema.version() + 1);
617
618        for (k, v) in table_schema.metadata().iter() {
619            builder = builder.add_metadata(k, v);
620        }
621
622        let new_schema = builder.build().with_context(|_| {
623            let column_names = requests
624                .iter()
625                .map(|request| request.column_name())
626                .collect::<Vec<_>>();
627            error::SchemaBuildSnafu {
628                msg: format!(
629                    "Table {table_name} cannot set index options with columns {column_names:?}",
630                ),
631            }
632        })?;
633        let _ = meta_builder
634            .schema(Arc::new(new_schema))
635            .primary_key_indices(self.primary_key_indices.clone());
636
637        Ok(meta_builder)
638    }
639
640    fn set_index(&self, column_schema: &mut ColumnSchema, request: &SetIndexOption) -> Result<()> {
641        match request {
642            SetIndexOption::Fulltext {
643                column_name,
644                options,
645            } => {
646                ensure!(
647                    column_schema.data_type.is_string(),
648                    error::InvalidColumnOptionSnafu {
649                        column_name,
650                        msg: "FULLTEXT index only supports string type",
651                    }
652                );
653
654                let current_fulltext_options = column_schema
655                    .fulltext_options()
656                    .context(error::SetFulltextOptionsSnafu { column_name })?;
657                set_column_fulltext_options(
658                    column_schema,
659                    column_name,
660                    options,
661                    current_fulltext_options,
662                )?;
663            }
664            SetIndexOption::Inverted { column_name } => {
665                debug_assert_eq!(column_schema.name, *column_name);
666                column_schema.set_inverted_index(true);
667            }
668            SetIndexOption::Skipping {
669                column_name,
670                options,
671            } => {
672                set_column_skipping_index_options(column_schema, column_name, options)?;
673            }
674        }
675
676        Ok(())
677    }
678
679    fn unset_index(
680        &self,
681        column_schema: &mut ColumnSchema,
682        request: &UnsetIndexOption,
683    ) -> Result<()> {
684        match request {
685            UnsetIndexOption::Fulltext { column_name } => {
686                let current_fulltext_options = column_schema
687                    .fulltext_options()
688                    .context(error::SetFulltextOptionsSnafu { column_name })?;
689                unset_column_fulltext_options(
690                    column_schema,
691                    column_name,
692                    current_fulltext_options.clone(),
693                )?
694            }
695            UnsetIndexOption::Inverted { .. } => {
696                column_schema.set_inverted_index(false);
697            }
698            UnsetIndexOption::Skipping { column_name } => {
699                unset_column_skipping_index_options(column_schema, column_name)?;
700            }
701        }
702
703        Ok(())
704    }
705
706    // TODO(yingwen): Remove this.
707    /// Allocate a new column for the table.
708    ///
709    /// This method would bump the `next_column_id` of the meta.
710    pub fn alloc_new_column(
711        &mut self,
712        table_name: &str,
713        new_column: &ColumnSchema,
714    ) -> Result<ColumnDescriptor> {
715        let desc = ColumnDescriptorBuilder::new(
716            self.next_column_id as ColumnId,
717            &new_column.name,
718            new_column.data_type.clone(),
719        )
720        .is_nullable(new_column.is_nullable())
721        .default_constraint(new_column.default_constraint().cloned())
722        .build()
723        .context(error::BuildColumnDescriptorSnafu {
724            table_name,
725            column_name: &new_column.name,
726        })?;
727
728        // Bump next column id.
729        self.next_column_id += 1;
730
731        Ok(desc)
732    }
733
734    /// Create a [`TableMetaBuilder`] from the current TableMeta.
735    fn new_meta_builder(&self) -> TableMetaBuilder {
736        let mut builder = TableMetaBuilder::from(self);
737        // Manually remove value_indices.
738        builder.value_indices = None;
739        builder
740    }
741
742    // TODO(yingwen): Tests add if not exists.
743    fn add_columns(
744        &self,
745        table_name: &str,
746        requests: &[AddColumnRequest],
747    ) -> Result<TableMetaBuilder> {
748        let table_schema = &self.schema;
749        let mut meta_builder = self.new_meta_builder();
750        let original_primary_key_indices: HashSet<&usize> =
751            self.primary_key_indices.iter().collect();
752
753        let mut names = HashSet::with_capacity(requests.len());
754        let mut new_columns = Vec::with_capacity(requests.len());
755        for col_to_add in requests {
756            if let Some(column_schema) =
757                table_schema.column_schema_by_name(&col_to_add.column_schema.name)
758            {
759                // If the column already exists.
760                ensure!(
761                    col_to_add.add_if_not_exists,
762                    error::ColumnExistsSnafu {
763                        table_name,
764                        column_name: &col_to_add.column_schema.name
765                    },
766                );
767
768                // Checks if the type is the same
769                ensure!(
770                    column_schema.data_type == col_to_add.column_schema.data_type,
771                    error::InvalidAlterRequestSnafu {
772                        table: table_name,
773                        err: format!(
774                            "column {} already exists with different type {:?}",
775                            col_to_add.column_schema.name, column_schema.data_type,
776                        ),
777                    }
778                );
779            } else {
780                // A new column.
781                // Ensures we only add a column once.
782                ensure!(
783                    names.insert(&col_to_add.column_schema.name),
784                    error::InvalidAlterRequestSnafu {
785                        table: table_name,
786                        err: format!(
787                            "add column {} more than once",
788                            col_to_add.column_schema.name
789                        ),
790                    }
791                );
792
793                ensure!(
794                    col_to_add.column_schema.is_nullable()
795                        || col_to_add.column_schema.default_constraint().is_some(),
796                    error::InvalidAlterRequestSnafu {
797                        table: table_name,
798                        err: format!(
799                            "no default value for column {}",
800                            col_to_add.column_schema.name
801                        ),
802                    },
803                );
804
805                // A dropped column may leave a stale entity declaration behind;
806                // re-adding it must not hand the declaration a type without a
807                // stable string form.
808                if !has_stable_string_form(&col_to_add.column_schema.data_type)
809                    && let Some(key) =
810                        entity_option_referencing(&self.options, &col_to_add.column_schema.name)
811                {
812                    return error::InvalidAlterRequestSnafu {
813                        table: table_name,
814                        err: format!(
815                            "column `{}` is referenced by entity option `{key}` and must \
816                             keep a type that renders as a string, got `{}`",
817                            col_to_add.column_schema.name, col_to_add.column_schema.data_type
818                        ),
819                    }
820                    .fail();
821                }
822
823                new_columns.push(col_to_add.clone());
824            }
825        }
826        let requests = &new_columns[..];
827
828        let SplitResult {
829            columns_at_first,
830            columns_at_after,
831            columns_at_last,
832            column_names,
833        } = self.split_requests_by_column_location(table_name, requests)?;
834        let mut primary_key_indices = Vec::with_capacity(self.primary_key_indices.len());
835        let mut columns = Vec::with_capacity(table_schema.num_columns() + requests.len());
836        // add new columns with FIRST, and in reverse order of requests.
837        columns_at_first.iter().rev().for_each(|request| {
838            if request.is_key {
839                // If a key column is added, we also need to store its index in primary_key_indices.
840                primary_key_indices.push(columns.len());
841            }
842            columns.push(request.column_schema.clone());
843        });
844        // add existed columns in original order and handle new columns with AFTER.
845        for (index, column_schema) in table_schema.column_schemas().iter().enumerate() {
846            if original_primary_key_indices.contains(&index) {
847                primary_key_indices.push(columns.len());
848            }
849            columns.push(column_schema.clone());
850            if let Some(requests) = columns_at_after.get(&column_schema.name) {
851                requests.iter().rev().for_each(|request| {
852                    if request.is_key {
853                        // If a key column is added, we also need to store its index in primary_key_indices.
854                        primary_key_indices.push(columns.len());
855                    }
856                    columns.push(request.column_schema.clone());
857                });
858            }
859        }
860        // add new columns without location info to last.
861        columns_at_last.iter().for_each(|request| {
862            if request.is_key {
863                // If a key column is added, we also need to store its index in primary_key_indices.
864                primary_key_indices.push(columns.len());
865            }
866            columns.push(request.column_schema.clone());
867        });
868
869        let mut builder = SchemaBuilder::try_from(columns)
870            .with_context(|_| error::SchemaBuildSnafu {
871                msg: format!("Failed to convert column schemas into schema for table {table_name}"),
872            })?
873            // Also bump the schema version.
874            .version(table_schema.version() + 1);
875        for (k, v) in table_schema.metadata().iter() {
876            builder = builder.add_metadata(k, v);
877        }
878        let new_schema = builder.build().with_context(|_| error::SchemaBuildSnafu {
879            msg: format!("Table {table_name} cannot add new columns {column_names:?}"),
880        })?;
881
882        let partition_key_indices = self
883            .partition_key_indices
884            .iter()
885            .map(|idx| table_schema.column_name_by_index(*idx))
886            // This unwrap is safe since we only add new columns.
887            .map(|name| new_schema.column_index_by_name(name).unwrap())
888            .collect();
889
890        // value_indices would be generated automatically.
891        let _ = meta_builder
892            .schema(Arc::new(new_schema))
893            .primary_key_indices(primary_key_indices)
894            .partition_key_indices(partition_key_indices);
895
896        Ok(meta_builder)
897    }
898
899    fn remove_columns(
900        &self,
901        table_name: &str,
902        column_names: &[String],
903    ) -> Result<TableMetaBuilder> {
904        let table_schema = &self.schema;
905        let column_names: HashSet<_> = column_names.iter().collect();
906        let mut meta_builder = self.new_meta_builder();
907
908        let timestamp_index = table_schema.timestamp_index();
909        // Check whether columns are existing and not in primary key index.
910        for column_name in &column_names {
911            if let Some(index) = table_schema.column_index_by_name(column_name) {
912                // This is a linear search, but since there won't be too much columns, the performance should
913                // be acceptable.
914                ensure!(
915                    !self.primary_key_indices.contains(&index),
916                    error::RemoveColumnInIndexSnafu {
917                        column_name: *column_name,
918                        table_name,
919                    }
920                );
921
922                ensure!(
923                    !self.partition_key_indices.contains(&index),
924                    error::RemovePartitionColumnSnafu {
925                        column_name: *column_name,
926                        table_name,
927                    }
928                );
929
930                if let Some(ts_index) = timestamp_index {
931                    // Not allowed to remove column in timestamp index.
932                    ensure!(
933                        index != ts_index,
934                        error::RemoveColumnInIndexSnafu {
935                            column_name: table_schema.column_name_by_index(ts_index),
936                            table_name,
937                        }
938                    );
939                }
940            } else {
941                return error::ColumnNotExistsSnafu {
942                    column_name: *column_name,
943                    table_name,
944                }
945                .fail()?;
946            }
947        }
948
949        // Collect columns after removal.
950        let columns: Vec<_> = table_schema
951            .column_schemas()
952            .iter()
953            .filter(|column_schema| !column_names.contains(&column_schema.name))
954            .cloned()
955            .collect();
956
957        let mut builder = SchemaBuilder::try_from_columns(columns)
958            .with_context(|_| error::SchemaBuildSnafu {
959                msg: format!("Failed to convert column schemas into schema for table {table_name}"),
960            })?
961            // Also bump the schema version.
962            .version(table_schema.version() + 1);
963        for (k, v) in table_schema.metadata().iter() {
964            builder = builder.add_metadata(k, v);
965        }
966        let new_schema = builder.build().with_context(|_| error::SchemaBuildSnafu {
967            msg: format!("Table {table_name} cannot add remove columns {column_names:?}"),
968        })?;
969
970        // Rebuild the indices of primary key columns.
971        let primary_key_indices = self
972            .primary_key_indices
973            .iter()
974            .map(|idx| table_schema.column_name_by_index(*idx))
975            // This unwrap is safe since we don't allow removing a primary key column.
976            .map(|name| new_schema.column_index_by_name(name).unwrap())
977            .collect();
978
979        let partition_key_indices = self
980            .partition_key_indices
981            .iter()
982            .map(|idx| table_schema.column_name_by_index(*idx))
983            // This unwrap is safe since we don't allow removing a partition key column.
984            .map(|name| new_schema.column_index_by_name(name).unwrap())
985            .collect();
986
987        let _ = meta_builder
988            .schema(Arc::new(new_schema))
989            .primary_key_indices(primary_key_indices)
990            .partition_key_indices(partition_key_indices);
991
992        Ok(meta_builder)
993    }
994
995    fn modify_column_types(
996        &self,
997        table_name: &str,
998        requests: &[ModifyColumnTypeRequest],
999    ) -> Result<TableMetaBuilder> {
1000        let table_schema = &self.schema;
1001        let mut meta_builder = self.new_meta_builder();
1002
1003        let mut modify_column_types = HashMap::with_capacity(requests.len());
1004        let timestamp_index = table_schema.timestamp_index();
1005
1006        for col_to_change in requests {
1007            let change_column_name = &col_to_change.column_name;
1008
1009            let index = table_schema
1010                .column_index_by_name(change_column_name)
1011                .with_context(|| error::ColumnNotExistsSnafu {
1012                    column_name: change_column_name,
1013                    table_name,
1014                })?;
1015
1016            // A column referenced by an entity declaration may be dropped (the
1017            // read-time derivation skips the stale declaration), but must not
1018            // change to a type without a stable string form. Checked after the
1019            // existence lookup so a missing column keeps reporting
1020            // `ColumnNotExists` regardless of stale declarations.
1021            if !has_stable_string_form(&col_to_change.target_type)
1022                && let Some(key) = entity_option_referencing(&self.options, change_column_name)
1023            {
1024                return error::InvalidAlterRequestSnafu {
1025                    table: table_name,
1026                    err: format!(
1027                        "column `{change_column_name}` is referenced by entity option \
1028                         `{key}` and must keep a type that renders as a string, got `{}`",
1029                        col_to_change.target_type
1030                    ),
1031                }
1032                .fail();
1033            }
1034
1035            let column = &table_schema.column_schemas()[index];
1036
1037            ensure!(
1038                !self.primary_key_indices.contains(&index),
1039                error::InvalidAlterRequestSnafu {
1040                    table: table_name,
1041                    err: format!(
1042                        "Not allowed to change primary key index column '{}'",
1043                        column.name
1044                    )
1045                }
1046            );
1047
1048            if let Some(ts_index) = timestamp_index {
1049                // Not allowed to change column datatype in timestamp index.
1050                ensure!(
1051                    index != ts_index,
1052                    error::InvalidAlterRequestSnafu {
1053                        table: table_name,
1054                        err: format!(
1055                            "Not allowed to change timestamp index column '{}' datatype",
1056                            column.name
1057                        )
1058                    }
1059                );
1060            }
1061
1062            ensure!(
1063                modify_column_types
1064                    .insert(&col_to_change.column_name, col_to_change)
1065                    .is_none(),
1066                error::InvalidAlterRequestSnafu {
1067                    table: table_name,
1068                    err: format!(
1069                        "change column datatype {} more than once",
1070                        col_to_change.column_name
1071                    ),
1072                }
1073            );
1074
1075            ensure!(
1076                column
1077                    .data_type
1078                    .can_arrow_type_cast_to(&col_to_change.target_type),
1079                error::InvalidAlterRequestSnafu {
1080                    table: table_name,
1081                    err: format!(
1082                        "column '{}' cannot be cast automatically to type '{}'",
1083                        col_to_change.column_name, col_to_change.target_type,
1084                    ),
1085                }
1086            );
1087
1088            ensure!(
1089                column.is_nullable(),
1090                error::InvalidAlterRequestSnafu {
1091                    table: table_name,
1092                    err: format!(
1093                        "column '{}' must be nullable to ensure safe conversion.",
1094                        col_to_change.column_name,
1095                    ),
1096                }
1097            );
1098        }
1099        // Collect columns after changed.
1100
1101        let mut columns: Vec<_> = Vec::with_capacity(table_schema.column_schemas().len());
1102        for mut column in table_schema.column_schemas().iter().cloned() {
1103            if let Some(change_column) = modify_column_types.get(&column.name) {
1104                column.data_type = change_column.target_type.clone();
1105                let new_default = if let Some(default_value) = column.default_constraint() {
1106                    Some(
1107                        default_value
1108                            .cast_to_datatype(&change_column.target_type)
1109                            .with_context(|_| error::CastDefaultValueSnafu {
1110                                reason: format!(
1111                                    "Failed to cast default value from {:?} to type {:?}",
1112                                    default_value, &change_column.target_type
1113                                ),
1114                            })?,
1115                    )
1116                } else {
1117                    None
1118                };
1119                column = column
1120                    .clone()
1121                    .with_default_constraint(new_default.clone())
1122                    .with_context(|_| error::CastDefaultValueSnafu {
1123                        reason: format!("Failed to set new default: {:?}", new_default),
1124                    })?;
1125            }
1126            columns.push(column)
1127        }
1128
1129        let mut builder = SchemaBuilder::try_from_columns(columns)
1130            .with_context(|_| error::SchemaBuildSnafu {
1131                msg: format!("Failed to convert column schemas into schema for table {table_name}"),
1132            })?
1133            // Also bump the schema version.
1134            .version(table_schema.version() + 1);
1135        for (k, v) in table_schema.metadata().iter() {
1136            builder = builder.add_metadata(k, v);
1137        }
1138        let new_schema = builder.build().with_context(|_| {
1139            let column_names: Vec<_> = requests
1140                .iter()
1141                .map(|request| &request.column_name)
1142                .collect();
1143
1144            error::SchemaBuildSnafu {
1145                msg: format!(
1146                    "Table {table_name} cannot change datatype with columns {column_names:?}"
1147                ),
1148            }
1149        })?;
1150
1151        let _ = meta_builder
1152            .schema(Arc::new(new_schema))
1153            .primary_key_indices(self.primary_key_indices.clone());
1154
1155        Ok(meta_builder)
1156    }
1157
1158    /// Split requests into different groups using column location info.
1159    fn split_requests_by_column_location<'a>(
1160        &self,
1161        table_name: &str,
1162        requests: &'a [AddColumnRequest],
1163    ) -> Result<SplitResult<'a>> {
1164        let table_schema = &self.schema;
1165        let mut columns_at_first = Vec::new();
1166        let mut columns_at_after = HashMap::new();
1167        let mut columns_at_last = Vec::new();
1168        let mut column_names = Vec::with_capacity(requests.len());
1169        for request in requests {
1170            // Check whether columns to add are already existing.
1171            let column_name = &request.column_schema.name;
1172            column_names.push(column_name.clone());
1173            ensure!(
1174                table_schema.column_schema_by_name(column_name).is_none(),
1175                error::ColumnExistsSnafu {
1176                    column_name,
1177                    table_name,
1178                }
1179            );
1180            match request.location.as_ref() {
1181                Some(AddColumnLocation::First) => {
1182                    columns_at_first.push(request);
1183                }
1184                Some(AddColumnLocation::After { column_name }) => {
1185                    ensure!(
1186                        table_schema.column_schema_by_name(column_name).is_some(),
1187                        error::ColumnNotExistsSnafu {
1188                            column_name,
1189                            table_name,
1190                        }
1191                    );
1192                    columns_at_after
1193                        .entry(column_name.clone())
1194                        .or_insert(Vec::new())
1195                        .push(request);
1196                }
1197                None => {
1198                    columns_at_last.push(request);
1199                }
1200            }
1201        }
1202        Ok(SplitResult {
1203            columns_at_first,
1204            columns_at_after,
1205            columns_at_last,
1206            column_names,
1207        })
1208    }
1209
1210    fn drop_defaults(&self, table_name: &str, column_names: &[String]) -> Result<TableMetaBuilder> {
1211        let table_schema = &self.schema;
1212        let mut meta_builder = self.new_meta_builder();
1213        let mut columns = Vec::with_capacity(table_schema.num_columns());
1214        for column_schema in table_schema.column_schemas() {
1215            if let Some(name) = column_names.iter().find(|s| **s == column_schema.name) {
1216                // Drop default constraint.
1217                ensure!(
1218                    column_schema.default_constraint().is_some(),
1219                    error::InvalidAlterRequestSnafu {
1220                        table: table_name,
1221                        err: format!("column {name} does not have a default value"),
1222                    }
1223                );
1224                if !column_schema.is_nullable() {
1225                    return error::InvalidAlterRequestSnafu {
1226                        table: table_name,
1227                        err: format!(
1228                            "column {name} is not nullable and `default` cannot be dropped",
1229                        ),
1230                    }
1231                    .fail();
1232                }
1233                let new_column_schema = column_schema.clone();
1234                let new_column_schema = new_column_schema
1235                    .with_default_constraint(None)
1236                    .with_context(|_| error::SchemaBuildSnafu {
1237                        msg: format!("Table {table_name} cannot drop default values"),
1238                    })?;
1239                columns.push(new_column_schema);
1240            } else {
1241                columns.push(column_schema.clone());
1242            }
1243        }
1244
1245        let mut builder = SchemaBuilder::try_from_columns(columns)
1246            .with_context(|_| error::SchemaBuildSnafu {
1247                msg: format!("Failed to convert column schemas into schema for table {table_name}"),
1248            })?
1249            // Also bump the schema version.
1250            .version(table_schema.version() + 1);
1251        for (k, v) in table_schema.metadata().iter() {
1252            builder = builder.add_metadata(k, v);
1253        }
1254        let new_schema = builder.build().with_context(|_| error::SchemaBuildSnafu {
1255            msg: format!("Table {table_name} cannot drop default values"),
1256        })?;
1257
1258        let _ = meta_builder.schema(Arc::new(new_schema));
1259
1260        Ok(meta_builder)
1261    }
1262
1263    fn set_defaults(
1264        &self,
1265        table_name: &str,
1266        set_defaults: &[SetDefaultRequest],
1267    ) -> Result<TableMetaBuilder> {
1268        let table_schema = &self.schema;
1269        let mut meta_builder = self.new_meta_builder();
1270        let mut columns = Vec::with_capacity(table_schema.num_columns());
1271        for column_schema in table_schema.column_schemas() {
1272            if let Some(set_default) = set_defaults
1273                .iter()
1274                .find(|s| s.column_name == column_schema.name)
1275            {
1276                let new_column_schema = column_schema.clone();
1277                let new_column_schema = new_column_schema
1278                    .with_default_constraint(set_default.default_constraint.clone())
1279                    .with_context(|_| error::SchemaBuildSnafu {
1280                        msg: format!("Table {table_name} cannot set default values"),
1281                    })?;
1282                columns.push(new_column_schema);
1283            } else {
1284                columns.push(column_schema.clone());
1285            }
1286        }
1287
1288        let mut builder = SchemaBuilder::try_from_columns(columns)
1289            .with_context(|_| error::SchemaBuildSnafu {
1290                msg: format!("Failed to convert column schemas into schema for table {table_name}"),
1291            })?
1292            // Also bump the schema version.
1293            .version(table_schema.version() + 1);
1294        for (k, v) in table_schema.metadata().iter() {
1295            builder = builder.add_metadata(k, v);
1296        }
1297        let new_schema = builder.build().with_context(|_| error::SchemaBuildSnafu {
1298            msg: format!("Table {table_name} cannot set default values"),
1299        })?;
1300
1301        let _ = meta_builder.schema(Arc::new(new_schema));
1302
1303        Ok(meta_builder)
1304    }
1305}
1306
1307#[derive(Clone, Debug, PartialEq, Eq, Builder, Serialize, Deserialize)]
1308#[builder(pattern = "owned")]
1309pub struct TableInfo {
1310    /// Id and version of the table.
1311    #[builder(default, setter(into))]
1312    pub ident: TableIdent,
1313    /// Name of the table.
1314    #[builder(setter(into))]
1315    pub name: String,
1316    /// Comment of the table.
1317    #[builder(default, setter(into))]
1318    pub desc: Option<String>,
1319    #[builder(default = "DEFAULT_CATALOG_NAME.to_string()", setter(into))]
1320    pub catalog_name: String,
1321    #[builder(default = "DEFAULT_SCHEMA_NAME.to_string()", setter(into))]
1322    pub schema_name: String,
1323    pub meta: TableMeta,
1324    #[builder(default = "TableType::Base")]
1325    pub table_type: TableType,
1326}
1327
1328pub type TableInfoRef = Arc<TableInfo>;
1329
1330impl TableInfo {
1331    pub fn table_id(&self) -> TableId {
1332        self.ident.table_id
1333    }
1334
1335    /// Returns the full table name in the form of `{catalog}.{schema}.{table}`.
1336    pub fn full_table_name(&self) -> String {
1337        common_catalog::format_full_table_name(&self.catalog_name, &self.schema_name, &self.name)
1338    }
1339
1340    pub fn get_db_string(&self) -> String {
1341        common_catalog::build_db_string(&self.catalog_name, &self.schema_name)
1342    }
1343
1344    /// Returns true when the table is the metric engine's physical table.
1345    pub fn is_physical_table(&self) -> bool {
1346        self.meta
1347            .options
1348            .extra_options
1349            .contains_key(PHYSICAL_TABLE_METADATA_KEY)
1350    }
1351
1352    /// Return true if the table's TTL is `instant`.
1353    pub fn is_ttl_instant_table(&self) -> bool {
1354        self.meta
1355            .options
1356            .ttl
1357            .map(|t| t.is_instant())
1358            .unwrap_or(false)
1359    }
1360}
1361
1362impl TableInfoBuilder {
1363    pub fn new<S: Into<String>>(name: S, meta: TableMeta) -> Self {
1364        Self {
1365            name: Some(name.into()),
1366            meta: Some(meta),
1367            ..Default::default()
1368        }
1369    }
1370
1371    pub fn table_id(mut self, id: TableId) -> Self {
1372        let ident = self.ident.get_or_insert_with(TableIdent::default);
1373        ident.table_id = id;
1374        self
1375    }
1376
1377    pub fn table_version(mut self, version: TableVersion) -> Self {
1378        let ident = self.ident.get_or_insert_with(TableIdent::default);
1379        ident.version = version;
1380        self
1381    }
1382}
1383
1384impl TableIdent {
1385    pub fn new(table_id: TableId) -> Self {
1386        Self {
1387            table_id,
1388            version: 0,
1389        }
1390    }
1391}
1392
1393impl From<TableId> for TableIdent {
1394    fn from(table_id: TableId) -> Self {
1395        Self::new(table_id)
1396    }
1397}
1398
1399impl TableInfo {
1400    /// Returns the map of column name to column id.
1401    ///
1402    /// Note: This method may return an empty map for older versions that did not include this field.
1403    pub fn name_to_ids(&self) -> Option<HashMap<String, ColumnId>> {
1404        let column_schemas = self.meta.schema.column_schemas();
1405        if self.meta.column_ids.len() != column_schemas.len() {
1406            None
1407        } else {
1408            Some(
1409                self.meta
1410                    .column_ids
1411                    .iter()
1412                    .enumerate()
1413                    .map(|(index, id)| (column_schemas[index].name.clone(), *id))
1414                    .collect(),
1415            )
1416        }
1417    }
1418
1419    /// Sort the columns in [TableInfo], logical tables require it.
1420    pub fn sort_columns(&mut self) {
1421        let column_schemas = self.meta.schema.column_schemas();
1422        let primary_keys = self
1423            .meta
1424            .primary_key_indices
1425            .iter()
1426            .map(|index| column_schemas[*index].name.clone())
1427            .collect::<HashSet<_>>();
1428
1429        let name_to_ids = self.name_to_ids().unwrap_or_default();
1430        let mut column_schemas = column_schemas.to_vec();
1431        column_schemas.sort_unstable_by(|a, b| a.name.cmp(&b.name));
1432
1433        // Compute new indices of sorted columns
1434        let mut primary_key_indices = Vec::with_capacity(primary_keys.len());
1435        let mut value_indices = Vec::with_capacity(column_schemas.len() - primary_keys.len());
1436        let mut column_ids = Vec::with_capacity(column_schemas.len());
1437        for (index, column_schema) in column_schemas.iter().enumerate() {
1438            if primary_keys.contains(&column_schema.name) {
1439                primary_key_indices.push(index);
1440            } else {
1441                value_indices.push(index);
1442            }
1443            if let Some(id) = name_to_ids.get(&column_schema.name) {
1444                column_ids.push(*id);
1445            }
1446        }
1447
1448        // Overwrite table meta
1449        self.meta.schema = Arc::new(Schema::new_with_version(
1450            column_schemas,
1451            self.meta.schema.version(),
1452        ));
1453        self.meta.primary_key_indices = primary_key_indices;
1454        self.meta.value_indices = value_indices;
1455        self.meta.column_ids = column_ids;
1456    }
1457
1458    /// Extracts region options from table info.
1459    ///
1460    /// All "region options" are actually a copy of table options for redundancy.
1461    pub fn to_region_options(&self) -> HashMap<String, String> {
1462        let mut options = HashMap::from(&self.meta.options);
1463        options.remove(REPARTITION_COLUMN_HINT_KEY);
1464        options
1465    }
1466
1467    /// Returns the table reference.
1468    pub fn table_ref(&self) -> TableReference<'_> {
1469        TableReference::full(
1470            self.catalog_name.as_str(),
1471            self.schema_name.as_str(),
1472            self.name.as_str(),
1473        )
1474    }
1475}
1476
1477fn entity_option_referencing<'a>(options: &'a TableOptions, column: &str) -> Option<&'a str> {
1478    options.extra_options.iter().find_map(|(key, value)| {
1479        (parse_entity_option_key(key).is_some()
1480            && parse_entity_columns(value).iter().any(|c| c == column))
1481        .then_some(key.as_str())
1482    })
1483}
1484
1485/// Set column fulltext options if it passed the validation.
1486///
1487/// Options allowed to modify:
1488/// * backend
1489///
1490/// Options not allowed to modify:
1491/// * analyzer
1492/// * case_sensitive
1493fn set_column_fulltext_options(
1494    column_schema: &mut ColumnSchema,
1495    column_name: &str,
1496    options: &FulltextOptions,
1497    current_options: Option<FulltextOptions>,
1498) -> Result<()> {
1499    if let Some(current_options) = current_options {
1500        ensure!(
1501            current_options.analyzer == options.analyzer
1502                && current_options.case_sensitive == options.case_sensitive,
1503            error::InvalidColumnOptionSnafu {
1504                column_name,
1505                msg: format!(
1506                    "Cannot change analyzer or case_sensitive if FULLTEXT index is set before. Previous analyzer: {}, previous case_sensitive: {}",
1507                    current_options.analyzer, current_options.case_sensitive
1508                ),
1509            }
1510        );
1511    }
1512
1513    column_schema
1514        .set_fulltext_options(options)
1515        .context(error::SetFulltextOptionsSnafu { column_name })?;
1516
1517    Ok(())
1518}
1519
1520fn unset_column_fulltext_options(
1521    column_schema: &mut ColumnSchema,
1522    column_name: &str,
1523    current_options: Option<FulltextOptions>,
1524) -> Result<()> {
1525    ensure!(
1526        current_options
1527            .as_ref()
1528            .is_some_and(|options| options.enable),
1529        error::InvalidColumnOptionSnafu {
1530            column_name,
1531            msg: "FULLTEXT index already disabled".to_string(),
1532        }
1533    );
1534
1535    let mut options = current_options.unwrap();
1536    options.enable = false;
1537    column_schema
1538        .set_fulltext_options(&options)
1539        .context(error::SetFulltextOptionsSnafu { column_name })?;
1540
1541    Ok(())
1542}
1543
1544fn set_column_skipping_index_options(
1545    column_schema: &mut ColumnSchema,
1546    column_name: &str,
1547    options: &SkippingIndexOptions,
1548) -> Result<()> {
1549    column_schema
1550        .set_skipping_options(options)
1551        .context(error::SetSkippingOptionsSnafu { column_name })?;
1552
1553    Ok(())
1554}
1555
1556fn unset_column_skipping_index_options(
1557    column_schema: &mut ColumnSchema,
1558    column_name: &str,
1559) -> Result<()> {
1560    column_schema
1561        .unset_skipping_options()
1562        .context(error::UnsetSkippingOptionsSnafu { column_name })?;
1563    Ok(())
1564}
1565
1566#[cfg(test)]
1567mod tests {
1568    use std::assert_matches;
1569
1570    use common_error::ext::ErrorExt;
1571    use common_error::status_code::StatusCode;
1572    use datatypes::data_type::ConcreteDataType;
1573    use datatypes::schema::{
1574        ColumnSchema, FulltextAnalyzer, FulltextBackend, Schema, SchemaBuilder,
1575    };
1576
1577    use super::*;
1578    use crate::Error;
1579
1580    /// Create a test schema with 3 columns: `[col1 int32, ts timestampmills, col2 int32]`.
1581    fn new_test_schema() -> Schema {
1582        let column_schemas = vec![
1583            ColumnSchema::new("col1", ConcreteDataType::int32_datatype(), true),
1584            ColumnSchema::new(
1585                "ts",
1586                ConcreteDataType::timestamp_millisecond_datatype(),
1587                false,
1588            )
1589            .with_time_index(true),
1590            ColumnSchema::new("col2", ConcreteDataType::int32_datatype(), true),
1591        ];
1592        SchemaBuilder::try_from(column_schemas)
1593            .unwrap()
1594            .version(123)
1595            .build()
1596            .unwrap()
1597    }
1598
1599    fn add_columns_to_meta(meta: &TableMeta) -> TableMeta {
1600        let new_tag = ColumnSchema::new("my_tag", ConcreteDataType::string_datatype(), true);
1601        let new_field = ColumnSchema::new("my_field", ConcreteDataType::string_datatype(), true);
1602        let alter_kind = AlterKind::AddColumns {
1603            columns: vec![
1604                AddColumnRequest {
1605                    column_schema: new_tag,
1606                    is_key: true,
1607                    location: None,
1608                    add_if_not_exists: false,
1609                },
1610                AddColumnRequest {
1611                    column_schema: new_field,
1612                    is_key: false,
1613                    location: None,
1614                    add_if_not_exists: false,
1615                },
1616            ],
1617        };
1618
1619        let builder = meta
1620            .builder_with_alter_kind("my_table", &alter_kind)
1621            .unwrap();
1622        builder.build().unwrap()
1623    }
1624
1625    fn add_columns_to_meta_with_location(meta: &TableMeta) -> TableMeta {
1626        let new_tag = ColumnSchema::new("my_tag_first", ConcreteDataType::string_datatype(), true);
1627        let new_field = ColumnSchema::new(
1628            "my_field_after_ts",
1629            ConcreteDataType::string_datatype(),
1630            true,
1631        );
1632        let yet_another_field = ColumnSchema::new(
1633            "yet_another_field_after_ts",
1634            ConcreteDataType::int64_datatype(),
1635            true,
1636        );
1637        let alter_kind = AlterKind::AddColumns {
1638            columns: vec![
1639                AddColumnRequest {
1640                    column_schema: new_tag,
1641                    is_key: true,
1642                    location: Some(AddColumnLocation::First),
1643                    add_if_not_exists: false,
1644                },
1645                AddColumnRequest {
1646                    column_schema: new_field,
1647                    is_key: false,
1648                    location: Some(AddColumnLocation::After {
1649                        column_name: "ts".to_string(),
1650                    }),
1651                    add_if_not_exists: false,
1652                },
1653                AddColumnRequest {
1654                    column_schema: yet_another_field,
1655                    is_key: true,
1656                    location: Some(AddColumnLocation::After {
1657                        column_name: "ts".to_string(),
1658                    }),
1659                    add_if_not_exists: false,
1660                },
1661            ],
1662        };
1663
1664        let builder = meta
1665            .builder_with_alter_kind("my_table", &alter_kind)
1666            .unwrap();
1667        builder.build().unwrap()
1668    }
1669
1670    #[test]
1671    fn test_add_columns() {
1672        let schema = Arc::new(new_test_schema());
1673        let meta = TableMetaBuilder::empty()
1674            .schema(schema)
1675            .primary_key_indices(vec![0])
1676            .engine("engine")
1677            .next_column_id(3)
1678            .build()
1679            .unwrap();
1680
1681        let new_meta = add_columns_to_meta(&meta);
1682        let names: Vec<String> = new_meta
1683            .schema
1684            .column_schemas()
1685            .iter()
1686            .map(|column_schema| column_schema.name.clone())
1687            .collect();
1688        assert_eq!(&["col1", "ts", "col2", "my_tag", "my_field"], &names[..]);
1689        assert_eq!(&[0, 3], &new_meta.primary_key_indices[..]);
1690        assert_eq!(&[1, 2, 4], &new_meta.value_indices[..]);
1691    }
1692
1693    #[test]
1694    fn test_set_append_mode_true_clears_merge_mode_option() {
1695        let schema = Arc::new(new_test_schema());
1696        let mut table_options = TableOptions::default();
1697        table_options
1698            .extra_options
1699            .insert(MERGE_MODE_KEY.to_string(), "last_non_null".to_string());
1700        let meta = TableMetaBuilder::empty()
1701            .schema(schema)
1702            .primary_key_indices(vec![0])
1703            .engine("engine")
1704            .next_column_id(3)
1705            .options(table_options)
1706            .build()
1707            .unwrap();
1708
1709        let alter_kind = AlterKind::SetTableOptions {
1710            options: vec![SetRegionOption::AppendMode(true)],
1711        };
1712        let new_meta = meta
1713            .builder_with_alter_kind("my_table", &alter_kind)
1714            .unwrap()
1715            .build()
1716            .unwrap();
1717
1718        assert_eq!(
1719            Some("true"),
1720            new_meta
1721                .options
1722                .extra_options
1723                .get(APPEND_MODE_KEY)
1724                .map(String::as_str)
1725        );
1726        assert!(!new_meta.options.extra_options.contains_key(MERGE_MODE_KEY));
1727    }
1728
1729    #[test]
1730    fn test_set_append_mode_false_keeps_merge_mode_option() {
1731        let schema = Arc::new(new_test_schema());
1732        let mut table_options = TableOptions::default();
1733        table_options
1734            .extra_options
1735            .insert(MERGE_MODE_KEY.to_string(), "last_non_null".to_string());
1736        let meta = TableMetaBuilder::empty()
1737            .schema(schema)
1738            .primary_key_indices(vec![0])
1739            .engine("engine")
1740            .next_column_id(3)
1741            .options(table_options)
1742            .build()
1743            .unwrap();
1744
1745        let alter_kind = AlterKind::SetTableOptions {
1746            options: vec![SetRegionOption::AppendMode(false)],
1747        };
1748        let new_meta = meta
1749            .builder_with_alter_kind("my_table", &alter_kind)
1750            .unwrap()
1751            .build()
1752            .unwrap();
1753
1754        assert_eq!(
1755            Some("false"),
1756            new_meta
1757                .options
1758                .extra_options
1759                .get(APPEND_MODE_KEY)
1760                .map(String::as_str)
1761        );
1762        assert_eq!(
1763            Some("last_non_null"),
1764            new_meta
1765                .options
1766                .extra_options
1767                .get(MERGE_MODE_KEY)
1768                .map(String::as_str)
1769        );
1770    }
1771
1772    #[test]
1773    fn test_set_skip_wal_updates_typed_and_extra_options() {
1774        let mut meta = TableMetaBuilder::empty()
1775            .schema(Arc::new(new_test_schema()))
1776            .primary_key_indices(vec![0])
1777            .engine("engine")
1778            .next_column_id(3)
1779            .build()
1780            .unwrap();
1781        meta.options
1782            .extra_options
1783            .insert(SKIP_WAL_KEY.to_string(), false.to_string());
1784
1785        let alter_kind = AlterKind::SetTableOptions {
1786            options: vec![SetRegionOption::SkipWal],
1787        };
1788        let new_meta = meta
1789            .builder_with_alter_kind("my_table", &alter_kind)
1790            .unwrap()
1791            .build()
1792            .unwrap();
1793
1794        assert!(new_meta.options.skip_wal);
1795        assert_eq!(
1796            Some("true"),
1797            new_meta
1798                .options
1799                .extra_options
1800                .get(SKIP_WAL_KEY)
1801                .map(String::as_str)
1802        );
1803    }
1804
1805    #[test]
1806    fn test_set_repartition_column_hint() {
1807        let meta = TableMetaBuilder::empty()
1808            .schema(Arc::new(new_test_schema()))
1809            .primary_key_indices(vec![0])
1810            .engine("engine")
1811            .next_column_id(3)
1812            .build()
1813            .unwrap();
1814
1815        let alter_kind = AlterKind::SetAnnotations {
1816            family: AnnotationFamily::RepartitionHint,
1817            options: vec![(
1818                REPARTITION_COLUMN_HINT_KEY.to_string(),
1819                " col1 ".to_string(),
1820            )],
1821        };
1822        let new_meta = meta
1823            .builder_with_alter_kind("my_table", &alter_kind)
1824            .unwrap()
1825            .build()
1826            .unwrap();
1827
1828        assert_eq!(
1829            Some("col1"),
1830            new_meta
1831                .options
1832                .extra_options
1833                .get(REPARTITION_COLUMN_HINT_KEY)
1834                .map(String::as_str)
1835        );
1836    }
1837
1838    #[test]
1839    fn test_set_repartition_column_hint_rejects_empty_column() {
1840        let meta = TableMetaBuilder::empty()
1841            .schema(Arc::new(new_test_schema()))
1842            .primary_key_indices(vec![0])
1843            .engine("engine")
1844            .next_column_id(3)
1845            .build()
1846            .unwrap();
1847
1848        let alter_kind = AlterKind::SetAnnotations {
1849            family: AnnotationFamily::RepartitionHint,
1850            options: vec![(REPARTITION_COLUMN_HINT_KEY.to_string(), " ".to_string())],
1851        };
1852        let err = meta
1853            .builder_with_alter_kind("my_table", &alter_kind)
1854            .err()
1855            .unwrap();
1856
1857        assert!(
1858            err.to_string()
1859                .contains("repartition.column.hint expects exactly one column name")
1860        );
1861    }
1862
1863    #[test]
1864    fn test_set_repartition_column_hint_rejects_multiple_columns() {
1865        let meta = TableMetaBuilder::empty()
1866            .schema(Arc::new(new_test_schema()))
1867            .primary_key_indices(vec![0])
1868            .engine("engine")
1869            .next_column_id(3)
1870            .build()
1871            .unwrap();
1872
1873        let alter_kind = AlterKind::SetAnnotations {
1874            family: AnnotationFamily::RepartitionHint,
1875            options: vec![(
1876                REPARTITION_COLUMN_HINT_KEY.to_string(),
1877                "col1,col2".to_string(),
1878            )],
1879        };
1880        let err = meta
1881            .builder_with_alter_kind("my_table", &alter_kind)
1882            .err()
1883            .unwrap();
1884
1885        assert!(
1886            err.to_string()
1887                .contains("repartition.column.hint expects exactly one column name")
1888        );
1889    }
1890
1891    #[test]
1892    fn test_set_repartition_column_hint_rejects_missing_column() {
1893        let meta = TableMetaBuilder::empty()
1894            .schema(Arc::new(new_test_schema()))
1895            .primary_key_indices(vec![0])
1896            .engine("engine")
1897            .next_column_id(3)
1898            .build()
1899            .unwrap();
1900
1901        let alter_kind = AlterKind::SetAnnotations {
1902            family: AnnotationFamily::RepartitionHint,
1903            options: vec![(
1904                REPARTITION_COLUMN_HINT_KEY.to_string(),
1905                "missing".to_string(),
1906            )],
1907        };
1908        let err = meta
1909            .builder_with_alter_kind("my_table", &alter_kind)
1910            .err()
1911            .unwrap();
1912
1913        assert!(err.to_string().contains("Column missing not exists"));
1914    }
1915
1916    #[test]
1917    fn test_set_repartition_column_hint_rejects_time_index_column() {
1918        let meta = TableMetaBuilder::empty()
1919            .schema(Arc::new(new_test_schema()))
1920            .primary_key_indices(vec![0])
1921            .engine("engine")
1922            .next_column_id(3)
1923            .build()
1924            .unwrap();
1925
1926        let alter_kind = AlterKind::SetAnnotations {
1927            family: AnnotationFamily::RepartitionHint,
1928            options: vec![(REPARTITION_COLUMN_HINT_KEY.to_string(), "ts".to_string())],
1929        };
1930        let err = meta
1931            .builder_with_alter_kind("my_table", &alter_kind)
1932            .err()
1933            .unwrap();
1934
1935        assert!(
1936            err.to_string()
1937                .contains("cannot set repartition.column.hint to the time index column")
1938        );
1939    }
1940
1941    #[test]
1942    fn test_set_repartition_column_hint_rejects_partitioned_table() {
1943        let meta = TableMetaBuilder::empty()
1944            .schema(Arc::new(new_test_schema()))
1945            .primary_key_indices(vec![0])
1946            .engine("engine")
1947            .next_column_id(3)
1948            .partition_key_indices(vec![0])
1949            .build()
1950            .unwrap();
1951
1952        let alter_kind = AlterKind::SetAnnotations {
1953            family: AnnotationFamily::RepartitionHint,
1954            options: vec![(REPARTITION_COLUMN_HINT_KEY.to_string(), "col1".to_string())],
1955        };
1956        let err = meta
1957            .builder_with_alter_kind("my_table", &alter_kind)
1958            .err()
1959            .unwrap();
1960
1961        assert!(
1962            err.to_string()
1963                .contains("cannot set repartition.column.hint on a table with partition metadata")
1964        );
1965    }
1966
1967    #[test]
1968    fn test_unset_repartition_column_hint() {
1969        let mut table_options = TableOptions::default();
1970        table_options
1971            .extra_options
1972            .insert(REPARTITION_COLUMN_HINT_KEY.to_string(), "col1".to_string());
1973        let meta = TableMetaBuilder::empty()
1974            .schema(Arc::new(new_test_schema()))
1975            .primary_key_indices(vec![0])
1976            .engine("engine")
1977            .next_column_id(3)
1978            .options(table_options)
1979            .build()
1980            .unwrap();
1981
1982        let new_meta = meta
1983            .builder_with_alter_kind(
1984                "my_table",
1985                &AlterKind::UnsetAnnotations {
1986                    family: AnnotationFamily::RepartitionHint,
1987                    keys: vec![REPARTITION_COLUMN_HINT_KEY.to_string()],
1988                },
1989            )
1990            .unwrap()
1991            .build()
1992            .unwrap();
1993
1994        assert!(
1995            !new_meta
1996                .options
1997                .extra_options
1998                .contains_key(REPARTITION_COLUMN_HINT_KEY)
1999        );
2000    }
2001
2002    #[test]
2003    fn test_repartition_column_hint_is_not_region_option() {
2004        let mut table_options = TableOptions::default();
2005        table_options
2006            .extra_options
2007            .insert(REPARTITION_COLUMN_HINT_KEY.to_string(), "col1".to_string());
2008        let table_info = TableInfoBuilder::default()
2009            .table_id(1)
2010            .table_version(0)
2011            .name("my_table")
2012            .catalog_name(DEFAULT_CATALOG_NAME)
2013            .schema_name(DEFAULT_SCHEMA_NAME)
2014            .meta(
2015                TableMetaBuilder::empty()
2016                    .schema(Arc::new(new_test_schema()))
2017                    .primary_key_indices(vec![0])
2018                    .engine("engine")
2019                    .next_column_id(3)
2020                    .options(table_options)
2021                    .build()
2022                    .unwrap(),
2023            )
2024            .build()
2025            .unwrap();
2026
2027        assert!(
2028            !table_info
2029                .to_region_options()
2030                .contains_key(REPARTITION_COLUMN_HINT_KEY)
2031        );
2032    }
2033
2034    #[test]
2035    fn test_set_auto_flush_interval() {
2036        let schema = Arc::new(new_test_schema());
2037        let table_options = TableOptions::default();
2038        let meta = TableMetaBuilder::empty()
2039            .schema(schema)
2040            .primary_key_indices(vec![0])
2041            .engine("engine")
2042            .next_column_id(3)
2043            .options(table_options)
2044            .build()
2045            .unwrap();
2046
2047        let alter_kind = AlterKind::SetTableOptions {
2048            options: vec![SetRegionOption::AutoFlushInterval(Some(
2049                std::time::Duration::from_secs(300),
2050            ))],
2051        };
2052        let new_meta = meta
2053            .builder_with_alter_kind("my_table", &alter_kind)
2054            .unwrap()
2055            .build()
2056            .unwrap();
2057
2058        assert_eq!(
2059            Some("5m"),
2060            new_meta
2061                .options
2062                .extra_options
2063                .get(AUTO_FLUSH_INTERVAL_KEY)
2064                .map(String::as_str)
2065        );
2066    }
2067
2068    #[test]
2069    fn test_set_auto_flush_interval_none_removes_existing() {
2070        let schema = Arc::new(new_test_schema());
2071        let mut table_options = TableOptions::default();
2072        table_options
2073            .extra_options
2074            .insert(AUTO_FLUSH_INTERVAL_KEY.to_string(), "5m".to_string());
2075        let meta = TableMetaBuilder::empty()
2076            .schema(schema)
2077            .primary_key_indices(vec![0])
2078            .engine("engine")
2079            .next_column_id(3)
2080            .options(table_options)
2081            .build()
2082            .unwrap();
2083
2084        let alter_kind = AlterKind::SetTableOptions {
2085            options: vec![SetRegionOption::AutoFlushInterval(None)],
2086        };
2087        let new_meta = meta
2088            .builder_with_alter_kind("my_table", &alter_kind)
2089            .unwrap()
2090            .build()
2091            .unwrap();
2092
2093        assert!(
2094            !new_meta
2095                .options
2096                .extra_options
2097                .contains_key(AUTO_FLUSH_INTERVAL_KEY)
2098        );
2099    }
2100
2101    #[test]
2102    fn test_set_and_unset_max_row_group_row_count() {
2103        let meta = TableMetaBuilder::empty()
2104            .schema(Arc::new(new_test_schema()))
2105            .primary_key_indices(vec![0])
2106            .engine("engine")
2107            .next_column_id(3)
2108            .options(TableOptions::default())
2109            .build()
2110            .unwrap();
2111
2112        let alter_kind = AlterKind::SetTableOptions {
2113            options: vec![SetRegionOption::MaxRowGroupRowCount(Some(512))],
2114        };
2115        let new_meta = meta
2116            .builder_with_alter_kind("my_table", &alter_kind)
2117            .unwrap()
2118            .build()
2119            .unwrap();
2120        assert_eq!(
2121            Some("512"),
2122            new_meta
2123                .options
2124                .extra_options
2125                .get(MAX_ROW_GROUP_ROW_COUNT)
2126                .map(String::as_str)
2127        );
2128
2129        let alter_kind = AlterKind::UnsetTableOptions {
2130            keys: vec![UnsetRegionOption::MaxRowGroupRowCount],
2131        };
2132        let new_meta = new_meta
2133            .builder_with_alter_kind("my_table", &alter_kind)
2134            .unwrap()
2135            .build()
2136            .unwrap();
2137        assert!(
2138            !new_meta
2139                .options
2140                .extra_options
2141                .contains_key(MAX_ROW_GROUP_ROW_COUNT)
2142        );
2143    }
2144
2145    #[test]
2146    fn test_add_columns_multiple_times() {
2147        let schema = Arc::new(new_test_schema());
2148        let meta = TableMetaBuilder::empty()
2149            .schema(schema)
2150            .primary_key_indices(vec![0])
2151            .engine("engine")
2152            .next_column_id(3)
2153            .build()
2154            .unwrap();
2155
2156        let alter_kind = AlterKind::AddColumns {
2157            columns: vec![
2158                AddColumnRequest {
2159                    column_schema: ColumnSchema::new(
2160                        "col3",
2161                        ConcreteDataType::int32_datatype(),
2162                        true,
2163                    ),
2164                    is_key: true,
2165                    location: None,
2166                    add_if_not_exists: true,
2167                },
2168                AddColumnRequest {
2169                    column_schema: ColumnSchema::new(
2170                        "col3",
2171                        ConcreteDataType::int32_datatype(),
2172                        true,
2173                    ),
2174                    is_key: true,
2175                    location: None,
2176                    add_if_not_exists: true,
2177                },
2178            ],
2179        };
2180        let err = meta
2181            .builder_with_alter_kind("my_table", &alter_kind)
2182            .err()
2183            .unwrap();
2184        assert_eq!(StatusCode::InvalidArguments, err.status_code());
2185    }
2186
2187    #[test]
2188    fn test_remove_columns() {
2189        let schema = Arc::new(new_test_schema());
2190        let meta = TableMetaBuilder::empty()
2191            .schema(schema.clone())
2192            .primary_key_indices(vec![0])
2193            .engine("engine")
2194            .next_column_id(3)
2195            .build()
2196            .unwrap();
2197        // Add more columns so we have enough candidate columns to remove.
2198        let meta = add_columns_to_meta(&meta);
2199
2200        let alter_kind = AlterKind::DropColumns {
2201            names: vec![String::from("col2"), String::from("my_field")],
2202        };
2203        let new_meta = meta
2204            .builder_with_alter_kind("my_table", &alter_kind)
2205            .unwrap()
2206            .build()
2207            .unwrap();
2208
2209        let names: Vec<String> = new_meta
2210            .schema
2211            .column_schemas()
2212            .iter()
2213            .map(|column_schema| column_schema.name.clone())
2214            .collect();
2215        assert_eq!(&["col1", "ts", "my_tag"], &names[..]);
2216        assert_eq!(&[0, 2], &new_meta.primary_key_indices[..]);
2217        assert_eq!(&[1], &new_meta.value_indices[..]);
2218        assert_eq!(
2219            schema.timestamp_column(),
2220            new_meta.schema.timestamp_column()
2221        );
2222    }
2223
2224    #[test]
2225    fn test_remove_multiple_columns_before_timestamp() {
2226        let column_schemas = vec![
2227            ColumnSchema::new("col1", ConcreteDataType::int32_datatype(), true),
2228            ColumnSchema::new("col2", ConcreteDataType::int32_datatype(), true),
2229            ColumnSchema::new("col3", ConcreteDataType::int32_datatype(), true),
2230            ColumnSchema::new(
2231                "ts",
2232                ConcreteDataType::timestamp_millisecond_datatype(),
2233                false,
2234            )
2235            .with_time_index(true),
2236        ];
2237        let schema = Arc::new(
2238            SchemaBuilder::try_from(column_schemas)
2239                .unwrap()
2240                .version(123)
2241                .build()
2242                .unwrap(),
2243        );
2244        let meta = TableMetaBuilder::empty()
2245            .schema(schema.clone())
2246            .primary_key_indices(vec![1])
2247            .engine("engine")
2248            .next_column_id(4)
2249            .build()
2250            .unwrap();
2251
2252        // Remove columns in reverse order to test whether timestamp index is valid.
2253        let alter_kind = AlterKind::DropColumns {
2254            names: vec![String::from("col3"), String::from("col1")],
2255        };
2256        let new_meta = meta
2257            .builder_with_alter_kind("my_table", &alter_kind)
2258            .unwrap()
2259            .build()
2260            .unwrap();
2261
2262        let names: Vec<String> = new_meta
2263            .schema
2264            .column_schemas()
2265            .iter()
2266            .map(|column_schema| column_schema.name.clone())
2267            .collect();
2268        assert_eq!(&["col2", "ts"], &names[..]);
2269        assert_eq!(&[0], &new_meta.primary_key_indices[..]);
2270        assert_eq!(&[1], &new_meta.value_indices[..]);
2271        assert_eq!(
2272            schema.timestamp_column(),
2273            new_meta.schema.timestamp_column()
2274        );
2275    }
2276
2277    #[test]
2278    fn test_add_existing_column() {
2279        let schema = Arc::new(new_test_schema());
2280        let meta = TableMetaBuilder::empty()
2281            .schema(schema)
2282            .primary_key_indices(vec![0])
2283            .engine("engine")
2284            .next_column_id(3)
2285            .build()
2286            .unwrap();
2287
2288        let alter_kind = AlterKind::AddColumns {
2289            columns: vec![AddColumnRequest {
2290                column_schema: ColumnSchema::new("col1", ConcreteDataType::string_datatype(), true),
2291                is_key: false,
2292                location: None,
2293                add_if_not_exists: false,
2294            }],
2295        };
2296
2297        let err = meta
2298            .builder_with_alter_kind("my_table", &alter_kind)
2299            .err()
2300            .unwrap();
2301        assert_eq!(StatusCode::TableColumnExists, err.status_code());
2302
2303        // Add if not exists
2304        let alter_kind = AlterKind::AddColumns {
2305            columns: vec![AddColumnRequest {
2306                column_schema: ColumnSchema::new("col1", ConcreteDataType::int32_datatype(), true),
2307                is_key: true,
2308                location: None,
2309                add_if_not_exists: true,
2310            }],
2311        };
2312        let new_meta = meta
2313            .builder_with_alter_kind("my_table", &alter_kind)
2314            .unwrap()
2315            .build()
2316            .unwrap();
2317        assert_eq!(
2318            meta.schema.column_schemas(),
2319            new_meta.schema.column_schemas()
2320        );
2321        assert_eq!(meta.schema.version() + 1, new_meta.schema.version());
2322    }
2323
2324    #[test]
2325    fn test_add_different_type_column() {
2326        let schema = Arc::new(new_test_schema());
2327        let meta = TableMetaBuilder::empty()
2328            .schema(schema)
2329            .primary_key_indices(vec![0])
2330            .engine("engine")
2331            .next_column_id(3)
2332            .build()
2333            .unwrap();
2334
2335        // Add if not exists, but different type.
2336        let alter_kind = AlterKind::AddColumns {
2337            columns: vec![AddColumnRequest {
2338                column_schema: ColumnSchema::new("col1", ConcreteDataType::string_datatype(), true),
2339                is_key: false,
2340                location: None,
2341                add_if_not_exists: true,
2342            }],
2343        };
2344        let err = meta
2345            .builder_with_alter_kind("my_table", &alter_kind)
2346            .err()
2347            .unwrap();
2348        assert_eq!(StatusCode::InvalidArguments, err.status_code());
2349    }
2350
2351    #[test]
2352    fn test_add_invalid_column() {
2353        let schema = Arc::new(new_test_schema());
2354        let meta = TableMetaBuilder::empty()
2355            .schema(schema)
2356            .primary_key_indices(vec![0])
2357            .engine("engine")
2358            .next_column_id(3)
2359            .build()
2360            .unwrap();
2361
2362        // Not nullable and no default value.
2363        let alter_kind = AlterKind::AddColumns {
2364            columns: vec![AddColumnRequest {
2365                column_schema: ColumnSchema::new(
2366                    "weny",
2367                    ConcreteDataType::string_datatype(),
2368                    false,
2369                ),
2370                is_key: false,
2371                location: None,
2372                add_if_not_exists: false,
2373            }],
2374        };
2375
2376        let err = meta
2377            .builder_with_alter_kind("my_table", &alter_kind)
2378            .err()
2379            .unwrap();
2380        assert_eq!(StatusCode::InvalidArguments, err.status_code());
2381    }
2382
2383    #[test]
2384    fn test_remove_unknown_column() {
2385        let schema = Arc::new(new_test_schema());
2386        let meta = TableMetaBuilder::empty()
2387            .schema(schema)
2388            .primary_key_indices(vec![0])
2389            .engine("engine")
2390            .next_column_id(3)
2391            .build()
2392            .unwrap();
2393
2394        let alter_kind = AlterKind::DropColumns {
2395            names: vec![String::from("unknown")],
2396        };
2397
2398        let err = meta
2399            .builder_with_alter_kind("my_table", &alter_kind)
2400            .err()
2401            .unwrap();
2402        assert_eq!(StatusCode::TableColumnNotFound, err.status_code());
2403    }
2404
2405    #[test]
2406    fn test_change_unknown_column_data_type() {
2407        let schema = Arc::new(new_test_schema());
2408        let meta = TableMetaBuilder::empty()
2409            .schema(schema)
2410            .primary_key_indices(vec![0])
2411            .engine("engine")
2412            .next_column_id(3)
2413            .build()
2414            .unwrap();
2415
2416        let alter_kind = AlterKind::ModifyColumnTypes {
2417            columns: vec![ModifyColumnTypeRequest {
2418                column_name: "unknown".to_string(),
2419                target_type: ConcreteDataType::string_datatype(),
2420            }],
2421        };
2422
2423        let err = meta
2424            .builder_with_alter_kind("my_table", &alter_kind)
2425            .err()
2426            .unwrap();
2427        assert_eq!(StatusCode::TableColumnNotFound, err.status_code());
2428    }
2429
2430    #[test]
2431    fn test_remove_key_column() {
2432        let schema = Arc::new(new_test_schema());
2433        let meta = TableMetaBuilder::empty()
2434            .schema(schema)
2435            .primary_key_indices(vec![0])
2436            .engine("engine")
2437            .next_column_id(3)
2438            .build()
2439            .unwrap();
2440
2441        // Remove column in primary key.
2442        let alter_kind = AlterKind::DropColumns {
2443            names: vec![String::from("col1")],
2444        };
2445
2446        let err = meta
2447            .builder_with_alter_kind("my_table", &alter_kind)
2448            .err()
2449            .unwrap();
2450        assert_eq!(StatusCode::InvalidArguments, err.status_code());
2451
2452        // Remove timestamp column.
2453        let alter_kind = AlterKind::DropColumns {
2454            names: vec![String::from("ts")],
2455        };
2456
2457        let err = meta
2458            .builder_with_alter_kind("my_table", &alter_kind)
2459            .err()
2460            .unwrap();
2461        assert_eq!(StatusCode::InvalidArguments, err.status_code());
2462    }
2463
2464    #[test]
2465    fn test_remove_partition_column() {
2466        let schema = Arc::new(new_test_schema());
2467        let meta = TableMetaBuilder::empty()
2468            .schema(schema)
2469            .primary_key_indices(vec![])
2470            .partition_key_indices(vec![0])
2471            .engine("engine")
2472            .next_column_id(3)
2473            .build()
2474            .unwrap();
2475        // Remove column in primary key.
2476        let alter_kind = AlterKind::DropColumns {
2477            names: vec![String::from("col1")],
2478        };
2479
2480        let err = meta
2481            .builder_with_alter_kind("my_table", &alter_kind)
2482            .err()
2483            .unwrap();
2484        assert_matches!(err, Error::RemovePartitionColumn { .. });
2485    }
2486
2487    #[test]
2488    fn test_change_key_column_data_type() {
2489        let schema = Arc::new(new_test_schema());
2490        let meta = TableMetaBuilder::empty()
2491            .schema(schema)
2492            .primary_key_indices(vec![0])
2493            .engine("engine")
2494            .next_column_id(3)
2495            .build()
2496            .unwrap();
2497
2498        // Remove column in primary key.
2499        let alter_kind = AlterKind::ModifyColumnTypes {
2500            columns: vec![ModifyColumnTypeRequest {
2501                column_name: "col1".to_string(),
2502                target_type: ConcreteDataType::string_datatype(),
2503            }],
2504        };
2505
2506        let err = meta
2507            .builder_with_alter_kind("my_table", &alter_kind)
2508            .err()
2509            .unwrap();
2510        assert_eq!(StatusCode::InvalidArguments, err.status_code());
2511
2512        // Remove timestamp column.
2513        let alter_kind = AlterKind::ModifyColumnTypes {
2514            columns: vec![ModifyColumnTypeRequest {
2515                column_name: "ts".to_string(),
2516                target_type: ConcreteDataType::string_datatype(),
2517            }],
2518        };
2519
2520        let err = meta
2521            .builder_with_alter_kind("my_table", &alter_kind)
2522            .err()
2523            .unwrap();
2524        assert_eq!(StatusCode::InvalidArguments, err.status_code());
2525    }
2526
2527    #[test]
2528    fn test_alloc_new_column() {
2529        let schema = Arc::new(new_test_schema());
2530        let mut meta = TableMetaBuilder::empty()
2531            .schema(schema)
2532            .primary_key_indices(vec![0])
2533            .engine("engine")
2534            .next_column_id(3)
2535            .build()
2536            .unwrap();
2537        assert_eq!(3, meta.next_column_id);
2538
2539        let column_schema = ColumnSchema::new("col1", ConcreteDataType::int32_datatype(), true);
2540        let desc = meta.alloc_new_column("test_table", &column_schema).unwrap();
2541
2542        assert_eq!(4, meta.next_column_id);
2543        assert_eq!(column_schema.name, desc.name);
2544    }
2545
2546    #[test]
2547    fn test_add_columns_with_location() {
2548        let schema = Arc::new(new_test_schema());
2549        let meta = TableMetaBuilder::empty()
2550            .schema(schema)
2551            .primary_key_indices(vec![0])
2552            // partition col: col1, col2
2553            .partition_key_indices(vec![0, 2])
2554            .engine("engine")
2555            .next_column_id(3)
2556            .build()
2557            .unwrap();
2558
2559        let new_meta = add_columns_to_meta_with_location(&meta);
2560        let names: Vec<String> = new_meta
2561            .schema
2562            .column_schemas()
2563            .iter()
2564            .map(|column_schema| column_schema.name.clone())
2565            .collect();
2566        assert_eq!(
2567            &[
2568                "my_tag_first",               // primary key column
2569                "col1",                       // partition column
2570                "ts",                         // timestamp column
2571                "yet_another_field_after_ts", // primary key column
2572                "my_field_after_ts",          // value column
2573                "col2",                       // partition column
2574            ],
2575            &names[..]
2576        );
2577        assert_eq!(&[0, 1, 3], &new_meta.primary_key_indices[..]);
2578        assert_eq!(&[2, 4, 5], &new_meta.value_indices[..]);
2579        assert_eq!(&[1, 5], &new_meta.partition_key_indices[..]);
2580    }
2581
2582    #[test]
2583    fn test_modify_column_fulltext_options() {
2584        let schema = Arc::new(new_test_schema());
2585        let meta = TableMetaBuilder::empty()
2586            .schema(schema)
2587            .primary_key_indices(vec![0])
2588            .engine("engine")
2589            .next_column_id(3)
2590            .build()
2591            .unwrap();
2592
2593        let alter_kind = AlterKind::SetIndexes {
2594            options: vec![SetIndexOption::Fulltext {
2595                column_name: "col1".to_string(),
2596                options: FulltextOptions::default(),
2597            }],
2598        };
2599        let err = meta
2600            .builder_with_alter_kind("my_table", &alter_kind)
2601            .err()
2602            .unwrap();
2603        assert_eq!(
2604            "Invalid column option, column name: col1, error: FULLTEXT index only supports string type",
2605            err.to_string()
2606        );
2607
2608        // Add a string column and make it fulltext indexed
2609        let new_meta = add_columns_to_meta_with_location(&meta);
2610        let alter_kind = AlterKind::SetIndexes {
2611            options: vec![SetIndexOption::Fulltext {
2612                column_name: "my_tag_first".to_string(),
2613                options: FulltextOptions::new_unchecked(
2614                    true,
2615                    FulltextAnalyzer::Chinese,
2616                    true,
2617                    FulltextBackend::Bloom,
2618                    1000,
2619                    0.01,
2620                ),
2621            }],
2622        };
2623        let new_meta = new_meta
2624            .builder_with_alter_kind("my_table", &alter_kind)
2625            .unwrap()
2626            .build()
2627            .unwrap();
2628        let column_schema = new_meta
2629            .schema
2630            .column_schema_by_name("my_tag_first")
2631            .unwrap();
2632        let fulltext_options = column_schema.fulltext_options().unwrap().unwrap();
2633        assert!(fulltext_options.enable);
2634        assert_eq!(
2635            datatypes::schema::FulltextAnalyzer::Chinese,
2636            fulltext_options.analyzer
2637        );
2638        assert!(fulltext_options.case_sensitive);
2639
2640        let alter_kind = AlterKind::UnsetIndexes {
2641            options: vec![UnsetIndexOption::Fulltext {
2642                column_name: "my_tag_first".to_string(),
2643            }],
2644        };
2645        let new_meta = new_meta
2646            .builder_with_alter_kind("my_table", &alter_kind)
2647            .unwrap()
2648            .build()
2649            .unwrap();
2650        let column_schema = new_meta
2651            .schema
2652            .column_schema_by_name("my_tag_first")
2653            .unwrap();
2654        let fulltext_options = column_schema.fulltext_options().unwrap().unwrap();
2655        assert!(!fulltext_options.enable);
2656    }
2657
2658    #[test]
2659    fn test_table_info_serde_compatibility() {
2660        // "serialized" is generated by the following codes before this refactor (PR 7626):
2661        //
2662        // ```Rust
2663        // serde_json::to_string(&RawTableInfo::from(TableInfo {
2664        //     ident: TableIdent {
2665        //         table_id: 1024,
2666        //         version: 1,
2667        //     },
2668        //     name: "foo".to_string(),
2669        //     desc: Some("my table".to_string()),
2670        //     catalog_name: "greptime".to_string(),
2671        //     schema_name: "public".to_string(),
2672        //     meta: TableMeta {
2673        //         schema: Arc::new(new_test_schema()),
2674        //         primary_key_indices: vec![0],
2675        //         value_indices: vec![1, 2],
2676        //         engine: "mito".to_string(),
2677        //         next_column_id: 3,
2678        //         options: TableOptions {
2679        //             ttl: Some(common_time::TimeToLive::Duration(
2680        //                 std::time::Duration::from_secs(3600),
2681        //             )),
2682        //             ..Default::default()
2683        //         },
2684        //         created_on: DateTime::<Utc>::MIN_UTC,
2685        //         updated_on: DateTime::<Utc>::MAX_UTC,
2686        //         partition_key_indices: vec![2],
2687        //         column_ids: vec![0, 1, 2],
2688        //     },
2689        //     table_type: TableType::Base,
2690        // }))
2691        // ```
2692        let serialized = r#"{"ident":{"table_id":1024,"version":1},"name":"foo","desc":"my table","catalog_name":"greptime","schema_name":"public","meta":{"schema":{"column_schemas":[{"name":"col1","data_type":{"Int32":{}},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"ts","data_type":{"Timestamp":{"Millisecond":null}},"is_nullable":false,"is_time_index":true,"default_constraint":null,"metadata":{"greptime:time_index":"true"}},{"name":"col2","data_type":{"Int32":{}},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}}],"timestamp_index":1,"version":123},"primary_key_indices":[0],"value_indices":[1,2],"engine":"mito","next_column_id":3,"options":{"write_buffer_size":null,"ttl":"1h","skip_wal":false,"extra_options":{}},"created_on":"-262143-01-01T00:00:00Z","updated_on":"+262142-12-31T23:59:59.999999999Z","partition_key_indices":[2],"column_ids":[0,1,2]},"table_type":"Base"}"#;
2693
2694        let actual: TableInfo = serde_json::from_str(serialized).unwrap();
2695        let expected = TableInfo {
2696            ident: TableIdent {
2697                table_id: 1024,
2698                version: 1,
2699            },
2700            name: "foo".to_string(),
2701            desc: Some("my table".to_string()),
2702            catalog_name: "greptime".to_string(),
2703            schema_name: "public".to_string(),
2704            meta: TableMeta {
2705                schema: Arc::new(new_test_schema()),
2706                primary_key_indices: vec![0],
2707                value_indices: vec![1, 2],
2708                engine: "mito".to_string(),
2709                next_column_id: 3,
2710                options: TableOptions {
2711                    ttl: Some(common_time::TimeToLive::Duration(
2712                        std::time::Duration::from_secs(3600),
2713                    )),
2714                    ..Default::default()
2715                },
2716                created_on: DateTime::<Utc>::MIN_UTC,
2717                updated_on: DateTime::<Utc>::MAX_UTC,
2718                partition_key_indices: vec![2],
2719                column_ids: vec![0, 1, 2],
2720            },
2721            table_type: TableType::Base,
2722        };
2723        assert_eq!(actual, expected);
2724    }
2725
2726    use crate::requests::{SEMANTIC_METRIC_UNIT, SEMANTIC_SIGNAL_TYPE};
2727
2728    /// `host` is a tag, `service` and `note` are string fields, `payload` is a
2729    /// binary field.
2730    fn semantic_test_meta() -> TableMeta {
2731        let column_schemas = vec![
2732            ColumnSchema::new("host", ConcreteDataType::string_datatype(), true),
2733            ColumnSchema::new(
2734                "ts",
2735                ConcreteDataType::timestamp_millisecond_datatype(),
2736                false,
2737            )
2738            .with_time_index(true),
2739            ColumnSchema::new("payload", ConcreteDataType::binary_datatype(), true),
2740            ColumnSchema::new("service", ConcreteDataType::string_datatype(), true),
2741            ColumnSchema::new("note", ConcreteDataType::string_datatype(), true),
2742        ];
2743        let schema = Arc::new(
2744            SchemaBuilder::try_from(column_schemas)
2745                .unwrap()
2746                .build()
2747                .unwrap(),
2748        );
2749        TableMetaBuilder::empty()
2750            .schema(schema)
2751            .primary_key_indices(vec![0])
2752            .engine("engine")
2753            .next_column_id(5)
2754            .build()
2755            .unwrap()
2756    }
2757
2758    #[test]
2759    fn test_set_semantic_annotations() {
2760        let meta = semantic_test_meta();
2761        // `service` is a plain field: entity columns may be tags or fields.
2762        let alter_kind = AlterKind::SetAnnotations {
2763            family: AnnotationFamily::Semantic,
2764            options: vec![
2765                (SEMANTIC_SIGNAL_TYPE.to_string(), "trace".to_string()),
2766                (
2767                    "greptime.semantic.entity.service.id".to_string(),
2768                    "service".to_string(),
2769                ),
2770            ],
2771        };
2772        let new_meta = meta
2773            .builder_with_alter_kind("my_table", &alter_kind)
2774            .unwrap()
2775            .build()
2776            .unwrap();
2777        assert_eq!(
2778            new_meta.options.extra_options.get(SEMANTIC_SIGNAL_TYPE),
2779            Some(&"trace".to_string())
2780        );
2781        assert_eq!(
2782            new_meta
2783                .options
2784                .extra_options
2785                .get("greptime.semantic.entity.service.id"),
2786            Some(&"service".to_string())
2787        );
2788    }
2789
2790    #[test]
2791    fn test_repartition_hint_batch_must_be_single_key() {
2792        let meta = TableMetaBuilder::empty()
2793            .schema(Arc::new(new_test_schema()))
2794            .primary_key_indices(vec![0])
2795            .engine("engine")
2796            .next_column_id(3)
2797            .build()
2798            .unwrap();
2799
2800        // Direct gRPC can hand the mutation layer a duplicated batch the
2801        // converter never saw; last-write-wins must not silently apply.
2802        let dup_set = AlterKind::SetAnnotations {
2803            family: AnnotationFamily::RepartitionHint,
2804            options: vec![
2805                (REPARTITION_COLUMN_HINT_KEY.to_string(), "col1".to_string()),
2806                (REPARTITION_COLUMN_HINT_KEY.to_string(), "col2".to_string()),
2807            ],
2808        };
2809        let err = meta
2810            .builder_with_alter_kind("my_table", &dup_set)
2811            .err()
2812            .unwrap();
2813        assert!(
2814            err.to_string().contains("must be altered separately"),
2815            "{err}"
2816        );
2817
2818        let dup_unset = AlterKind::UnsetAnnotations {
2819            family: AnnotationFamily::RepartitionHint,
2820            keys: vec![
2821                REPARTITION_COLUMN_HINT_KEY.to_string(),
2822                REPARTITION_COLUMN_HINT_KEY.to_string(),
2823            ],
2824        };
2825        let err = meta
2826            .builder_with_alter_kind("my_table", &dup_unset)
2827            .err()
2828            .unwrap();
2829        assert!(
2830            err.to_string().contains("must be altered separately"),
2831            "{err}"
2832        );
2833    }
2834
2835    #[test]
2836    fn test_set_semantic_annotations_rejects_invalid() {
2837        let meta = semantic_test_meta();
2838        let cases = [
2839            (
2840                "greptime.semantic.unknown_key",
2841                "x",
2842                "unknown semantic option",
2843            ),
2844            (SEMANTIC_SIGNAL_TYPE, "garbage", "invalid value"),
2845            (
2846                "greptime.semantic.entity.host.id",
2847                "no_such_column",
2848                "no_such_column",
2849            ),
2850            (
2851                "greptime.semantic.entity.host.id",
2852                "payload",
2853                "cannot render as a string",
2854            ),
2855        ];
2856        for (key, value, needle) in cases {
2857            let alter_kind = AlterKind::SetAnnotations {
2858                family: AnnotationFamily::Semantic,
2859                options: vec![(key.to_string(), value.to_string())],
2860            };
2861            let err = meta
2862                .builder_with_alter_kind("my_table", &alter_kind)
2863                .err()
2864                .unwrap();
2865            assert!(
2866                err.to_string().contains(needle),
2867                "key `{key}`: unexpected error `{err}`"
2868            );
2869        }
2870
2871        // ALTER keeps missing columns on the 4002 contract (pinned by sqlness);
2872        // the shared validator must not collapse it into InvalidArguments.
2873        let missing = meta
2874            .builder_with_alter_kind(
2875                "my_table",
2876                &AlterKind::SetAnnotations {
2877                    family: AnnotationFamily::Semantic,
2878                    options: vec![(
2879                        "greptime.semantic.entity.host.id".to_string(),
2880                        "no_such_column".to_string(),
2881                    )],
2882                },
2883            )
2884            .err()
2885            .unwrap();
2886        assert_eq!(
2887            common_error::status_code::StatusCode::TableColumnNotFound,
2888            common_error::ext::ErrorExt::status_code(&missing)
2889        );
2890    }
2891
2892    #[test]
2893    fn test_unset_semantic_annotations() {
2894        let mut meta = semantic_test_meta();
2895        meta.options
2896            .extra_options
2897            .insert(SEMANTIC_SIGNAL_TYPE.to_string(), "trace".to_string());
2898        // A key this version does not recognise, still inside the namespace.
2899        meta.options
2900            .extra_options
2901            .insert("greptime.semantic.future_key".to_string(), "x".to_string());
2902
2903        let alter_kind = AlterKind::UnsetAnnotations {
2904            family: AnnotationFamily::Semantic,
2905            keys: vec![
2906                SEMANTIC_SIGNAL_TYPE.to_string(),
2907                "greptime.semantic.future_key".to_string(),
2908                // Absent key: removal is a no-op, not an error.
2909                SEMANTIC_METRIC_UNIT.to_string(),
2910            ],
2911        };
2912        let new_meta = meta
2913            .builder_with_alter_kind("my_table", &alter_kind)
2914            .unwrap()
2915            .build()
2916            .unwrap();
2917        assert!(
2918            !new_meta
2919                .options
2920                .extra_options
2921                .contains_key(SEMANTIC_SIGNAL_TYPE)
2922        );
2923        assert!(
2924            !new_meta
2925                .options
2926                .extra_options
2927                .contains_key("greptime.semantic.future_key")
2928        );
2929
2930        let outside = AlterKind::UnsetAnnotations {
2931            family: AnnotationFamily::Semantic,
2932            keys: vec!["ttl".to_string()],
2933        };
2934        let err = meta
2935            .builder_with_alter_kind("my_table", &outside)
2936            .err()
2937            .unwrap();
2938        assert!(err.to_string().contains("annotation namespace"), "{err}");
2939    }
2940
2941    #[test]
2942    fn test_modify_entity_column_type_keeps_string_form() {
2943        let mut meta = semantic_test_meta();
2944        meta.options.extra_options.insert(
2945            "greptime.semantic.entity.service.id".to_string(),
2946            "service".to_string(),
2947        );
2948
2949        let alter_kind = AlterKind::ModifyColumnTypes {
2950            columns: vec![ModifyColumnTypeRequest {
2951                column_name: "service".to_string(),
2952                target_type: ConcreteDataType::binary_datatype(),
2953            }],
2954        };
2955        let err = meta
2956            .builder_with_alter_kind("my_table", &alter_kind)
2957            .err()
2958            .unwrap();
2959        assert!(
2960            err.to_string()
2961                .contains("must keep a type that renders as a string"),
2962            "{err}"
2963        );
2964
2965        // An unreferenced column may still change to a non-string form.
2966        let alter_kind = AlterKind::ModifyColumnTypes {
2967            columns: vec![ModifyColumnTypeRequest {
2968                column_name: "note".to_string(),
2969                target_type: ConcreteDataType::binary_datatype(),
2970            }],
2971        };
2972        meta.builder_with_alter_kind("my_table", &alter_kind)
2973            .unwrap();
2974    }
2975
2976    #[test]
2977    fn test_stale_entity_declaration_guards_readd_and_reports_missing_column() {
2978        // A stale declaration: `gone` was dropped after being declared.
2979        let mut meta = semantic_test_meta();
2980        meta.options.extra_options.insert(
2981            "greptime.semantic.entity.service.id".to_string(),
2982            "gone".to_string(),
2983        );
2984
2985        // MODIFY on the missing column keeps the ColumnNotExists contract
2986        // (4002), stale declaration or not.
2987        let modify = AlterKind::ModifyColumnTypes {
2988            columns: vec![ModifyColumnTypeRequest {
2989                column_name: "gone".to_string(),
2990                target_type: ConcreteDataType::binary_datatype(),
2991            }],
2992        };
2993        let err = meta
2994            .builder_with_alter_kind("my_table", &modify)
2995            .err()
2996            .unwrap();
2997        assert_eq!(
2998            common_error::status_code::StatusCode::TableColumnNotFound,
2999            common_error::ext::ErrorExt::status_code(&err)
3000        );
3001
3002        // Re-adding the declared column must not hand the declaration a
3003        // non-string type...
3004        let add = |ty: ConcreteDataType| AlterKind::AddColumns {
3005            columns: vec![AddColumnRequest {
3006                column_schema: ColumnSchema::new("gone", ty, true),
3007                is_key: false,
3008                location: None,
3009                add_if_not_exists: false,
3010            }],
3011        };
3012        let err = meta
3013            .builder_with_alter_kind("my_table", &add(ConcreteDataType::binary_datatype()))
3014            .err()
3015            .unwrap();
3016        assert!(
3017            err.to_string()
3018                .contains("must keep a type that renders as a string"),
3019            "{err}"
3020        );
3021
3022        // ...while a string re-add re-satisfies it.
3023        meta.builder_with_alter_kind("my_table", &add(ConcreteDataType::string_datatype()))
3024            .unwrap();
3025    }
3026}