Skip to main content

store_api/
metadata.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Metadata of region and column.
16//!
17//! This mod has its own error type [MetadataError] for validation and codec exceptions.
18
19use std::any::Any;
20use std::collections::{HashMap, HashSet};
21use std::sync::Arc;
22use std::{fmt, mem};
23
24use api::v1::SemanticType;
25use api::v1::column_def::try_as_column_schema;
26use api::v1::region::RegionColumnDef;
27use common_base::hash::partition_expr_version;
28use common_error::ext::ErrorExt;
29use common_error::status_code::StatusCode;
30use common_macro::stack_trace_debug;
31use datatypes::arrow;
32use datatypes::arrow::datatypes::FieldRef;
33use datatypes::extension::json::json2_metadata_with_updated_settings;
34use datatypes::prelude::ConcreteDataType;
35use datatypes::schema::{ColumnSchema, FulltextOptions, Schema, SchemaRef};
36use datatypes::types::TimestampType;
37use itertools::Itertools;
38use serde::de::Error;
39use serde::{Deserialize, Deserializer, Serialize};
40use snafu::{Location, OptionExt, ResultExt, Snafu, ensure};
41
42use crate::codec::PrimaryKeyEncoding;
43use crate::metric_engine_consts::{DATA_SCHEMA_TABLE_ID_COLUMN_NAME, DATA_SCHEMA_TSID_COLUMN_NAME};
44use crate::region_request::{
45    AddColumn, AddColumnLocation, AlterKind, ModifyColumnType, SetIndexOption, UnsetIndexOption,
46};
47use crate::storage::consts::{ReservedColumnId, is_internal_column};
48use crate::storage::{ColumnId, RegionId};
49
50pub type Result<T> = std::result::Result<T, MetadataError>;
51
52/// Metadata of a column.
53#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
54pub struct ColumnMetadata {
55    /// Schema of this column. Is the same as `column_schema` in [SchemaRef].
56    pub column_schema: ColumnSchema,
57    /// Semantic type of this column (e.g. tag or timestamp).
58    pub semantic_type: SemanticType,
59    /// Immutable and unique id of a region.
60    pub column_id: ColumnId,
61}
62
63impl fmt::Debug for ColumnMetadata {
64    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
65        write!(
66            f,
67            "[{:?} {:?} {:?}]",
68            self.column_schema, self.semantic_type, self.column_id,
69        )
70    }
71}
72
73impl ColumnMetadata {
74    /// Construct `Self` from protobuf struct [RegionColumnDef]
75    pub fn try_from_column_def(column_def: RegionColumnDef) -> Result<Self> {
76        let column_id = column_def.column_id;
77        let column_def = column_def
78            .column_def
79            .context(InvalidRawRegionRequestSnafu {
80                err: "column_def is absent",
81            })?;
82        let semantic_type = column_def.semantic_type();
83        let column_schema = try_as_column_schema(&column_def).context(ConvertColumnSchemaSnafu)?;
84
85        Ok(Self {
86            column_schema,
87            semantic_type,
88            column_id,
89        })
90    }
91
92    /// Encodes a vector of `ColumnMetadata` into a JSON byte vector.
93    pub fn encode_list(columns: &[Self]) -> serde_json::Result<Vec<u8>> {
94        serde_json::to_vec(columns)
95    }
96
97    /// Decodes a JSON byte vector into a vector of `ColumnMetadata`.
98    pub fn decode_list(bytes: &[u8]) -> serde_json::Result<Vec<Self>> {
99        serde_json::from_slice(bytes)
100    }
101
102    pub fn is_same_datatype(&self, other: &Self) -> bool {
103        self.column_schema.data_type == other.column_schema.data_type
104    }
105
106    /// Returns the estimated memory footprint of this metadata.
107    pub fn estimated_size(&self) -> usize {
108        mem::size_of_val(self) - mem::size_of_val(&self.column_schema)
109            + self.column_schema.estimated_size()
110    }
111}
112
113#[cfg_attr(doc, aquamarine::aquamarine)]
114/// General static metadata of a region.
115///
116/// This struct implements [Serialize] and [Deserialize] traits.
117/// To build a [RegionMetadata] object, use [RegionMetadataBuilder].
118///
119/// ```mermaid
120/// class RegionMetadata {
121///     +RegionId region_id
122///     +SchemaRef schema
123///     +Vec&lt;ColumnMetadata&gt; column_metadatas
124///     +Vec&lt;ColumnId&gt; primary_key
125/// }
126/// class Schema
127/// class ColumnMetadata {
128///     +ColumnSchema column_schema
129///     +SemanticTyle semantic_type
130///     +ColumnId column_id
131/// }
132/// class SemanticType
133/// RegionMetadata o-- Schema
134/// RegionMetadata o-- ColumnMetadata
135/// ColumnMetadata o-- SemanticType
136/// ```
137#[derive(Clone, PartialEq, Eq, Serialize)]
138pub struct RegionMetadata {
139    /// Latest schema constructed from [column_metadatas](RegionMetadata::column_metadatas).
140    #[serde(skip)]
141    pub schema: SchemaRef,
142
143    // We don't pub `time_index` and `id_to_index` and always construct them via [SkippedFields]
144    // so we can assumes they are valid.
145    /// Id of the time index column.
146    #[serde(skip)]
147    time_index: ColumnId,
148    /// Map column id to column's index in [column_metadatas](RegionMetadata::column_metadatas).
149    #[serde(skip)]
150    id_to_index: HashMap<ColumnId, usize>,
151
152    /// Columns in the region. Has the same order as columns
153    /// in [schema](RegionMetadata::schema).
154    pub column_metadatas: Vec<ColumnMetadata>,
155    /// Maintains an ordered list of primary keys
156    pub primary_key: Vec<ColumnId>,
157
158    /// Immutable and unique id of a region.
159    pub region_id: RegionId,
160    /// Current version of the region schema.
161    ///
162    /// The version starts from 0. Altering the schema bumps the version.
163    pub schema_version: u64,
164
165    /// Primary key encoding mode.
166    pub primary_key_encoding: PrimaryKeyEncoding,
167
168    /// Partition expression serialized as a JSON string.
169    /// Compatibility behavior:
170    /// - None: no partition expr was ever set in the manifest (legacy regions).
171    /// - Some(""): an explicit “single-region/no-partition” designation. This is distinct from None and should be preserved as-is.
172    pub partition_expr: Option<String>,
173    #[serde(skip)]
174    pub partition_expr_version: u64,
175}
176
177impl fmt::Debug for RegionMetadata {
178    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
179        f.debug_struct("RegionMetadata")
180            .field("column_metadatas", &self.column_metadatas)
181            .field("time_index", &self.time_index)
182            .field("primary_key", &self.primary_key)
183            .field("region_id", &self.region_id)
184            .field("schema_version", &self.schema_version)
185            .field("partition_expr", &self.partition_expr)
186            .finish()
187    }
188}
189
190pub type RegionMetadataRef = Arc<RegionMetadata>;
191
192impl<'de> Deserialize<'de> for RegionMetadata {
193    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
194    where
195        D: Deserializer<'de>,
196    {
197        // helper internal struct for deserialization
198        #[derive(Deserialize)]
199        struct RegionMetadataWithoutSchema {
200            column_metadatas: Vec<ColumnMetadata>,
201            primary_key: Vec<ColumnId>,
202            region_id: RegionId,
203            schema_version: u64,
204            #[serde(default)]
205            primary_key_encoding: PrimaryKeyEncoding,
206            #[serde(default)]
207            partition_expr: Option<String>,
208        }
209
210        let without_schema = RegionMetadataWithoutSchema::deserialize(deserializer)?;
211        let skipped =
212            SkippedFields::new(&without_schema.column_metadatas).map_err(D::Error::custom)?;
213
214        let partition_expr_version =
215            partition_expr_version(without_schema.partition_expr.as_deref());
216
217        Ok(Self {
218            schema: skipped.schema,
219            time_index: skipped.time_index,
220            id_to_index: skipped.id_to_index,
221            column_metadatas: without_schema.column_metadatas,
222            primary_key: without_schema.primary_key,
223            region_id: without_schema.region_id,
224            schema_version: without_schema.schema_version,
225            primary_key_encoding: without_schema.primary_key_encoding,
226            partition_expr: without_schema.partition_expr,
227            partition_expr_version,
228        })
229    }
230}
231
232impl RegionMetadata {
233    /// Decode the metadata from a JSON str.
234    pub fn from_json(s: &str) -> Result<Self> {
235        serde_json::from_str(s).context(SerdeJsonSnafu)
236    }
237
238    /// Returns the estimated memory footprint of this metadata.
239    pub fn estimated_size(&self) -> usize {
240        mem::size_of_val(self)
241            + mem::size_of::<ColumnMetadata>() * self.column_metadatas.capacity()
242            + self
243                .column_metadatas
244                .iter()
245                .map(|column| column.estimated_size() - mem::size_of::<ColumnMetadata>())
246                .sum::<usize>()
247            + mem::size_of::<ColumnId>() * self.primary_key.capacity()
248            + mem::size_of::<(ColumnId, usize)>() * self.id_to_index.capacity()
249            + self.schema.estimated_size()
250            + self
251                .partition_expr
252                .as_ref()
253                .map(|expr| expr.capacity())
254                .unwrap_or_default()
255    }
256
257    /// Encode the metadata to a JSON string.
258    pub fn to_json(&self) -> Result<String> {
259        serde_json::to_string(&self).context(SerdeJsonSnafu)
260    }
261
262    pub fn set_partition_expr(&mut self, expr: Option<String>) {
263        self.partition_expr_version = partition_expr_version(expr.as_deref());
264        self.partition_expr = expr;
265    }
266
267    /// Find column by id.
268    pub fn column_by_id(&self, column_id: ColumnId) -> Option<&ColumnMetadata> {
269        self.id_to_index
270            .get(&column_id)
271            .map(|index| &self.column_metadatas[*index])
272    }
273
274    /// Find column index by id.
275    pub fn column_index_by_id(&self, column_id: ColumnId) -> Option<usize> {
276        self.id_to_index.get(&column_id).copied()
277    }
278
279    /// Find column index by name.
280    pub fn column_index_by_name(&self, column_name: &str) -> Option<usize> {
281        self.column_metadatas
282            .iter()
283            .position(|col| col.column_schema.name == column_name)
284    }
285
286    /// Returns the time index column
287    ///
288    /// # Panics
289    /// Panics if the time index column id is invalid.
290    pub fn time_index_column(&self) -> &ColumnMetadata {
291        let index = self.id_to_index[&self.time_index];
292        &self.column_metadatas[index]
293    }
294
295    /// Returns timestamp type of time index column
296    ///
297    /// # Panics
298    /// Panics if the time index column id is invalid.
299    pub fn time_index_type(&self) -> TimestampType {
300        let index = self.id_to_index[&self.time_index];
301        self.column_metadatas[index]
302            .column_schema
303            .data_type
304            .as_timestamp()
305            .unwrap()
306    }
307
308    /// Returns the position of the time index.
309    pub fn time_index_column_pos(&self) -> usize {
310        self.id_to_index[&self.time_index]
311    }
312
313    /// Returns the arrow field of the time index column.
314    pub fn time_index_field(&self) -> FieldRef {
315        let index = self.id_to_index[&self.time_index];
316        self.schema.arrow_schema().fields[index].clone()
317    }
318
319    /// Finds a column by name.
320    pub fn column_by_name(&self, name: &str) -> Option<&ColumnMetadata> {
321        self.schema
322            .column_index_by_name(name)
323            .map(|index| &self.column_metadatas[index])
324    }
325
326    /// Returns all primary key columns.
327    pub fn primary_key_columns(&self) -> impl Iterator<Item = &ColumnMetadata> {
328        // safety: RegionMetadata::validate ensures every primary key exists.
329        self.primary_key
330            .iter()
331            .map(|id| self.column_by_id(*id).unwrap())
332    }
333
334    /// Returns all field columns before projection.
335    ///
336    /// **Use with caution**. On read path where might have projection, this method
337    /// can return columns that not present in data batch.
338    pub fn field_columns(&self) -> impl Iterator<Item = &ColumnMetadata> {
339        self.column_metadatas
340            .iter()
341            .filter(|column| column.semantic_type == SemanticType::Field)
342    }
343
344    /// Returns a column's index in primary key if it is a primary key column.
345    ///
346    /// This does a linear search.
347    pub fn primary_key_index(&self, column_id: ColumnId) -> Option<usize> {
348        self.primary_key.iter().position(|id| *id == column_id)
349    }
350
351    /// Project the metadata to a new one using specified column ids.
352    ///
353    /// [RegionId] and schema version are preserved.
354    pub fn project(&self, projection: &[ColumnId]) -> Result<RegionMetadata> {
355        // check time index
356        ensure!(
357            projection.contains(&self.time_index),
358            TimeIndexNotFoundSnafu
359        );
360
361        // prepare new indices
362        let indices_to_preserve = projection
363            .iter()
364            .map(|id| {
365                self.column_index_by_id(*id)
366                    .with_context(|| InvalidRegionRequestSnafu {
367                        region_id: self.region_id,
368                        err: format!("column id {} not found", id),
369                    })
370            })
371            .collect::<Result<Vec<_>>>()?;
372
373        // project schema
374        let projected_schema =
375            self.schema
376                .try_project(&indices_to_preserve)
377                .with_context(|_| SchemaProjectSnafu {
378                    origin_schema: self.schema.clone(),
379                    projection: projection.to_vec(),
380                })?;
381
382        // project columns, generate projected primary key and new id_to_index
383        let mut projected_column_metadatas = Vec::with_capacity(indices_to_preserve.len());
384        let mut projected_primary_key = vec![];
385        let mut projected_id_to_index = HashMap::with_capacity(indices_to_preserve.len());
386        for index in indices_to_preserve {
387            let col = self.column_metadatas[index].clone();
388            if col.semantic_type == SemanticType::Tag {
389                projected_primary_key.push(col.column_id);
390            }
391            projected_id_to_index.insert(col.column_id, projected_column_metadatas.len());
392            projected_column_metadatas.push(col);
393        }
394
395        Ok(RegionMetadata {
396            schema: Arc::new(projected_schema),
397            time_index: self.time_index,
398            id_to_index: projected_id_to_index,
399            column_metadatas: projected_column_metadatas,
400            primary_key: projected_primary_key,
401            region_id: self.region_id,
402            schema_version: self.schema_version,
403            primary_key_encoding: self.primary_key_encoding,
404            partition_expr: self.partition_expr.clone(),
405            partition_expr_version: partition_expr_version(self.partition_expr.as_deref()),
406        })
407    }
408
409    /// Gets the column ids to be indexed by inverted index.
410    pub fn inverted_indexed_column_ids<'a>(
411        &self,
412        ignore_column_ids: impl Iterator<Item = &'a ColumnId>,
413    ) -> HashSet<ColumnId> {
414        let mut inverted_index = self
415            .column_metadatas
416            .iter()
417            .filter(|column| column.column_schema.is_inverted_indexed())
418            .map(|column| column.column_id)
419            .collect::<HashSet<_>>();
420
421        for ignored in ignore_column_ids {
422            inverted_index.remove(ignored);
423        }
424
425        inverted_index
426    }
427
428    /// Checks whether the metadata is valid.
429    fn validate(&self) -> Result<()> {
430        // Id to name.
431        let mut id_names = HashMap::with_capacity(self.column_metadatas.len());
432        for col in &self.column_metadatas {
433            // Validate each column.
434            Self::validate_column_metadata(col)?;
435
436            // Check whether column id is duplicated. We already check column name
437            // is unique in `Schema` so we only check column id here.
438            ensure!(
439                !id_names.contains_key(&col.column_id),
440                InvalidMetaSnafu {
441                    reason: format!(
442                        "column {} and {} have the same column id {}",
443                        id_names[&col.column_id], col.column_schema.name, col.column_id,
444                    ),
445                }
446            );
447            id_names.insert(col.column_id, &col.column_schema.name);
448        }
449
450        // Checks there is only one time index.
451        let time_indexes = self
452            .column_metadatas
453            .iter()
454            .filter(|col| col.semantic_type == SemanticType::Timestamp)
455            .collect::<Vec<_>>();
456        ensure!(
457            time_indexes.len() == 1,
458            InvalidMetaSnafu {
459                reason: format!(
460                    "expect only one time index, found {}: {}",
461                    time_indexes.len(),
462                    time_indexes
463                        .iter()
464                        .map(|c| &c.column_schema.name)
465                        .join(", ")
466                ),
467            }
468        );
469
470        // Checks the time index column is not nullable.
471        ensure!(
472            !self.time_index_column().column_schema.is_nullable(),
473            InvalidMetaSnafu {
474                reason: format!(
475                    "time index column {} must be NOT NULL",
476                    self.time_index_column().column_schema.name
477                ),
478            }
479        );
480
481        if self.primary_key_encoding == PrimaryKeyEncoding::Sparse {
482            ensure!(
483                self.primary_key
484                    .starts_with(&[ReservedColumnId::table_id(), ReservedColumnId::tsid()]),
485                InvalidMetaSnafu {
486                    reason: "sparse primary keys must start with (__table_id, __tsid)",
487                }
488            );
489        }
490
491        if !self.primary_key.is_empty() {
492            let mut pk_ids = HashSet::with_capacity(self.primary_key.len());
493            // Checks column ids in the primary key is valid.
494            for column_id in &self.primary_key {
495                // Checks whether the column id exists.
496                ensure!(
497                    id_names.contains_key(column_id),
498                    InvalidMetaSnafu {
499                        reason: format!("unknown column id {}", column_id),
500                    }
501                );
502
503                // Safety: Column with specific id must exist.
504                let column = self.column_by_id(*column_id).unwrap();
505                // Checks duplicate.
506                ensure!(
507                    !pk_ids.contains(&column_id),
508                    InvalidMetaSnafu {
509                        reason: format!(
510                            "duplicate column {} in primary key",
511                            column.column_schema.name
512                        ),
513                    }
514                );
515
516                // Checks this is not a time index column.
517                ensure!(
518                    *column_id != self.time_index,
519                    InvalidMetaSnafu {
520                        reason: format!(
521                            "column {} is already a time index column",
522                            column.column_schema.name,
523                        ),
524                    }
525                );
526
527                // Checks semantic type.
528                ensure!(
529                    column.semantic_type == SemanticType::Tag,
530                    InvalidMetaSnafu {
531                        reason: format!(
532                            "semantic type of column {} should be Tag, not {:?}",
533                            column.column_schema.name, column.semantic_type
534                        ),
535                    }
536                );
537
538                if self.primary_key_encoding == PrimaryKeyEncoding::Sparse {
539                    let schema = &column.column_schema;
540                    let valid = if *column_id == ReservedColumnId::table_id() {
541                        schema.name == DATA_SCHEMA_TABLE_ID_COLUMN_NAME
542                            && schema.data_type == ConcreteDataType::uint32_datatype()
543                            && !schema.is_nullable()
544                    } else if *column_id == ReservedColumnId::tsid() {
545                        schema.name == DATA_SCHEMA_TSID_COLUMN_NAME
546                            && schema.data_type == ConcreteDataType::uint64_datatype()
547                            && !schema.is_nullable()
548                    } else {
549                        !ReservedColumnId::is_reserved(*column_id) && schema.data_type.is_string()
550                    };
551                    ensure!(
552                        valid,
553                        InvalidMetaSnafu {
554                            reason: format!("invalid sparse primary key column {}", schema.name),
555                        }
556                    );
557                }
558
559                pk_ids.insert(column_id);
560            }
561        }
562
563        // Checks tag semantic type.
564        let num_tag = self
565            .column_metadatas
566            .iter()
567            .filter(|col| col.semantic_type == SemanticType::Tag)
568            .count();
569        ensure!(
570            num_tag == self.primary_key.len(),
571            InvalidMetaSnafu {
572                reason: format!(
573                    "number of primary key columns {} not equal to tag columns {}",
574                    self.primary_key.len(),
575                    num_tag
576                ),
577            }
578        );
579
580        Ok(())
581    }
582
583    /// Checks whether it is a valid column.
584    fn validate_column_metadata(column_metadata: &ColumnMetadata) -> Result<()> {
585        column_metadata
586            .column_schema
587            .compression_options()
588            .context(InvalidSchemaSnafu)?;
589        if column_metadata.semantic_type == SemanticType::Timestamp {
590            ensure!(
591                column_metadata.column_schema.data_type.is_timestamp(),
592                InvalidMetaSnafu {
593                    reason: format!(
594                        "column `{}` is not timestamp type",
595                        column_metadata.column_schema.name
596                    ),
597                }
598            );
599        }
600
601        ensure!(
602            !is_internal_column(&column_metadata.column_schema.name),
603            InvalidMetaSnafu {
604                reason: format!(
605                    "{} is internal column name that can not be used",
606                    column_metadata.column_schema.name
607                ),
608            }
609        );
610
611        Ok(())
612    }
613}
614
615/// Builder to build [RegionMetadata].
616pub struct RegionMetadataBuilder {
617    region_id: RegionId,
618    column_metadatas: Vec<ColumnMetadata>,
619    primary_key: Vec<ColumnId>,
620    schema_version: u64,
621    primary_key_encoding: PrimaryKeyEncoding,
622    partition_expr: Option<String>,
623}
624
625impl RegionMetadataBuilder {
626    /// Returns a new builder.
627    pub fn new(id: RegionId) -> Self {
628        Self {
629            region_id: id,
630            column_metadatas: vec![],
631            primary_key: vec![],
632            schema_version: 0,
633            primary_key_encoding: PrimaryKeyEncoding::Dense,
634            partition_expr: None,
635        }
636    }
637
638    /// Creates a builder from existing [RegionMetadata].
639    pub fn from_existing(existing: RegionMetadata) -> Self {
640        Self {
641            column_metadatas: existing.column_metadatas,
642            primary_key: existing.primary_key,
643            region_id: existing.region_id,
644            schema_version: existing.schema_version,
645            primary_key_encoding: existing.primary_key_encoding,
646            partition_expr: existing.partition_expr,
647        }
648    }
649
650    /// Sets the primary key encoding mode.
651    pub fn primary_key_encoding(&mut self, encoding: PrimaryKeyEncoding) -> &mut Self {
652        self.primary_key_encoding = encoding;
653        self
654    }
655
656    /// Sets the partition expression in JSON string form.
657    pub fn partition_expr_json(&mut self, expr_json: Option<String>) -> &mut Self {
658        self.partition_expr = expr_json;
659        self
660    }
661
662    /// Pushes a new column metadata to this region's metadata.
663    pub fn push_column_metadata(&mut self, column_metadata: ColumnMetadata) -> &mut Self {
664        self.column_metadatas.push(column_metadata);
665        self
666    }
667
668    /// Sets the primary key of the region.
669    pub fn primary_key(&mut self, key: Vec<ColumnId>) -> &mut Self {
670        self.primary_key = key;
671        self
672    }
673
674    /// Increases the schema version by 1.
675    pub fn bump_version(&mut self) -> &mut Self {
676        self.schema_version += 1;
677        self
678    }
679
680    /// Applies the alter `kind` to the builder.
681    ///
682    /// The `kind` should be valid.
683    pub fn alter(&mut self, kind: AlterKind) -> Result<&mut Self> {
684        match kind {
685            AlterKind::AddColumns { columns } => self.add_columns(columns)?,
686            AlterKind::DropColumns { names } => self.drop_columns(&names),
687            AlterKind::ModifyColumnTypes { columns } => self.modify_column_types(columns)?,
688            AlterKind::SetJsonSettings {
689                column_name,
690                settings,
691            } => self.set_json_settings(column_name, settings)?,
692            AlterKind::SetIndexes { options } => self.set_indexes(options)?,
693            AlterKind::UnsetIndexes { options } => self.unset_indexes(options)?,
694            AlterKind::SetRegionOptions { options: _ } => {
695                // nothing to be done with RegionMetadata
696            }
697            AlterKind::UnsetRegionOptions { keys: _ } => {
698                // nothing to be done with RegionMetadata
699            }
700            AlterKind::DropDefaults { names } => {
701                self.drop_defaults(names)?;
702            }
703            AlterKind::SetDefaults { columns } => self.set_defaults(&columns)?,
704            AlterKind::SyncColumns { column_metadatas } => {
705                self.primary_key = column_metadatas
706                    .iter()
707                    .filter_map(|column_metadata| {
708                        if column_metadata.semantic_type == SemanticType::Tag {
709                            Some(column_metadata.column_id)
710                        } else {
711                            None
712                        }
713                    })
714                    .collect::<Vec<_>>();
715                self.column_metadatas = column_metadatas;
716            }
717        }
718        Ok(self)
719    }
720
721    /// Consumes the builder and build a [RegionMetadata].
722    pub fn build(self) -> Result<RegionMetadata> {
723        self.build_with_options(true)
724    }
725
726    /// Builds metadata without running validation.
727    ///
728    /// Intended for file/external engines that should accept arbitrary schemas
729    /// coming from files.
730    pub fn build_without_validation(self) -> Result<RegionMetadata> {
731        self.build_with_options(false)
732    }
733
734    fn build_with_options(self, validate: bool) -> Result<RegionMetadata> {
735        let skipped = SkippedFields::new(&self.column_metadatas)?;
736
737        let partition_expr_version = partition_expr_version(self.partition_expr.as_deref());
738        let meta = RegionMetadata {
739            schema: skipped.schema,
740            time_index: skipped.time_index,
741            id_to_index: skipped.id_to_index,
742            column_metadatas: self.column_metadatas,
743            primary_key: self.primary_key,
744            region_id: self.region_id,
745            schema_version: self.schema_version,
746            primary_key_encoding: self.primary_key_encoding,
747            partition_expr: self.partition_expr,
748            partition_expr_version,
749        };
750
751        if validate {
752            meta.validate()?;
753        }
754
755        Ok(meta)
756    }
757
758    /// Adds columns to the metadata if not exist.
759    fn add_columns(&mut self, columns: Vec<AddColumn>) -> Result<()> {
760        let mut names: HashSet<_> = self
761            .column_metadatas
762            .iter()
763            .map(|col| col.column_schema.name.clone())
764            .collect();
765
766        for add_column in columns {
767            if names.contains(&add_column.column_metadata.column_schema.name) {
768                // Column already exists.
769                continue;
770            }
771
772            let column_id = add_column.column_metadata.column_id;
773            let semantic_type = add_column.column_metadata.semantic_type;
774            let column_name = add_column.column_metadata.column_schema.name.clone();
775            match add_column.location {
776                None => {
777                    self.column_metadatas.push(add_column.column_metadata);
778                }
779                Some(AddColumnLocation::First) => {
780                    self.column_metadatas.insert(0, add_column.column_metadata);
781                }
782                Some(AddColumnLocation::After { column_name }) => {
783                    let pos = self
784                        .column_metadatas
785                        .iter()
786                        .position(|col| col.column_schema.name == column_name)
787                        .context(InvalidRegionRequestSnafu {
788                            region_id: self.region_id,
789                            err: format!(
790                                "column {} not found, failed to add column {} after it",
791                                column_name, add_column.column_metadata.column_schema.name
792                            ),
793                        })?;
794                    // Insert after pos.
795                    self.column_metadatas
796                        .insert(pos + 1, add_column.column_metadata);
797                }
798            }
799            names.insert(column_name);
800            if semantic_type == SemanticType::Tag {
801                // For a new tag, we extend the primary key.
802                self.primary_key.push(column_id);
803            }
804        }
805
806        Ok(())
807    }
808
809    /// Drops columns from the metadata if exist.
810    fn drop_columns(&mut self, names: &[String]) {
811        let name_set: HashSet<_> = names.iter().collect();
812        self.column_metadatas
813            .retain(|col| !name_set.contains(&col.column_schema.name));
814    }
815
816    /// Changes columns type to the metadata if exist.
817    fn modify_column_types(&mut self, columns: Vec<ModifyColumnType>) -> Result<()> {
818        let mut change_type_map: HashMap<_, _> = columns
819            .into_iter()
820            .map(
821                |ModifyColumnType {
822                     column_name,
823                     target_type,
824                 }| (column_name, target_type),
825            )
826            .collect();
827
828        for column_meta in self.column_metadatas.iter_mut() {
829            if let Some(target_type) = change_type_map.remove(&column_meta.column_schema.name) {
830                column_meta.column_schema.data_type = target_type.clone();
831                // also cast default value to target_type if default value exist
832                let new_default =
833                    if let Some(default_value) = column_meta.column_schema.default_constraint() {
834                        Some(
835                            default_value
836                                .cast_to_datatype(&target_type)
837                                .with_context(|_| CastDefaultValueSnafu {
838                                    reason: format!(
839                                        "Failed to cast default value from {:?} to type {:?}",
840                                        default_value, target_type
841                                    ),
842                                })?,
843                        )
844                    } else {
845                        None
846                    };
847                column_meta.column_schema = column_meta
848                    .column_schema
849                    .clone()
850                    .with_default_constraint(new_default.clone())
851                    .with_context(|_| CastDefaultValueSnafu {
852                        reason: format!("Failed to set new default: {:?}", new_default),
853                    })?;
854            }
855        }
856
857        Ok(())
858    }
859
860    fn set_json_settings(
861        &mut self,
862        col_name: String,
863        settings: datatypes::json::JsonSettings,
864    ) -> Result<()> {
865        let Some(col_meta) = self
866            .column_metadatas
867            .iter_mut()
868            .find(|col| col.column_schema.name == col_name)
869        else {
870            return InvalidRegionRequestSnafu {
871                region_id: self.region_id,
872                err: format!("column {col_name} not found"),
873            }
874            .fail();
875        };
876
877        let old_metadata = col_meta.column_schema.metadata();
878        let new_metadata =
879            json2_metadata_with_updated_settings(old_metadata, settings).map_err(|err| {
880                InvalidRegionRequestSnafu {
881                    region_id: self.region_id,
882                    err: err.to_string(),
883                }
884                .build()
885            })?;
886
887        *col_meta.column_schema.mut_metadata() = new_metadata;
888        Ok(())
889    }
890
891    fn set_indexes(&mut self, options: Vec<SetIndexOption>) -> Result<()> {
892        let mut set_index_map: HashMap<_, Vec<_>> = HashMap::new();
893        for option in &options {
894            set_index_map
895                .entry(option.column_name())
896                .or_default()
897                .push(option);
898        }
899
900        for column_metadata in self.column_metadatas.iter_mut() {
901            if let Some(options) = set_index_map.remove(&column_metadata.column_schema.name) {
902                for option in options {
903                    Self::set_index(column_metadata, option)?;
904                }
905            }
906        }
907
908        Ok(())
909    }
910
911    fn unset_indexes(&mut self, options: Vec<UnsetIndexOption>) -> Result<()> {
912        let mut unset_index_map: HashMap<_, Vec<_>> = HashMap::new();
913        for option in &options {
914            unset_index_map
915                .entry(option.column_name())
916                .or_default()
917                .push(option);
918        }
919
920        for column_metadata in self.column_metadatas.iter_mut() {
921            if let Some(options) = unset_index_map.remove(&column_metadata.column_schema.name) {
922                for option in options {
923                    Self::unset_index(column_metadata, option)?;
924                }
925            }
926        }
927
928        Ok(())
929    }
930
931    fn set_index(column_metadata: &mut ColumnMetadata, options: &SetIndexOption) -> Result<()> {
932        match options {
933            SetIndexOption::Fulltext {
934                column_name,
935                options,
936            } => {
937                ensure!(
938                    column_metadata.column_schema.data_type.is_string(),
939                    InvalidColumnOptionSnafu {
940                        column_name,
941                        msg: "FULLTEXT index only supports string type".to_string(),
942                    }
943                );
944                let current_fulltext_options = column_metadata
945                    .column_schema
946                    .fulltext_options()
947                    .with_context(|_| GetFulltextOptionsSnafu {
948                        column_name: column_name.clone(),
949                    })?;
950                set_column_fulltext_options(
951                    column_metadata,
952                    column_name,
953                    options,
954                    current_fulltext_options,
955                )?;
956            }
957            SetIndexOption::Inverted { .. } => {
958                column_metadata.column_schema.set_inverted_index(true)
959            }
960            SetIndexOption::Skipping {
961                column_name,
962                options,
963            } => {
964                column_metadata
965                    .column_schema
966                    .set_skipping_options(options)
967                    .context(UnsetSkippingIndexOptionsSnafu { column_name })?;
968            }
969        }
970
971        Ok(())
972    }
973
974    fn unset_index(column_metadata: &mut ColumnMetadata, options: &UnsetIndexOption) -> Result<()> {
975        match options {
976            UnsetIndexOption::Fulltext { column_name } => {
977                ensure!(
978                    column_metadata.column_schema.data_type.is_string(),
979                    InvalidColumnOptionSnafu {
980                        column_name,
981                        msg: "FULLTEXT index only supports string type".to_string(),
982                    }
983                );
984
985                let current_fulltext_options = column_metadata
986                    .column_schema
987                    .fulltext_options()
988                    .with_context(|_| GetFulltextOptionsSnafu {
989                        column_name: column_name.clone(),
990                    })?;
991
992                unset_column_fulltext_options(
993                    column_metadata,
994                    column_name,
995                    current_fulltext_options,
996                )?;
997            }
998            UnsetIndexOption::Inverted { .. } => {
999                column_metadata.column_schema.set_inverted_index(false)
1000            }
1001            UnsetIndexOption::Skipping { column_name } => {
1002                column_metadata
1003                    .column_schema
1004                    .unset_skipping_options()
1005                    .context(UnsetSkippingIndexOptionsSnafu { column_name })?;
1006            }
1007        }
1008
1009        Ok(())
1010    }
1011
1012    fn drop_defaults(&mut self, column_names: Vec<String>) -> Result<()> {
1013        for name in column_names.iter() {
1014            let meta = self
1015                .column_metadatas
1016                .iter_mut()
1017                .find(|col| col.column_schema.name == *name);
1018            if let Some(meta) = meta {
1019                if !meta.column_schema.is_nullable() {
1020                    return InvalidRegionRequestSnafu {
1021                        region_id: self.region_id,
1022                        err: format!(
1023                            "column {name} is not nullable and `default` cannot be dropped",
1024                        ),
1025                    }
1026                    .fail();
1027                }
1028                meta.column_schema = meta
1029                    .column_schema
1030                    .clone()
1031                    .with_default_constraint(None)
1032                    .with_context(|_| CastDefaultValueSnafu {
1033                        reason: format!("Failed to drop default : {name:?}"),
1034                    })?;
1035            } else {
1036                return InvalidRegionRequestSnafu {
1037                    region_id: self.region_id,
1038                    err: format!("column {name} not found",),
1039                }
1040                .fail();
1041            }
1042        }
1043        Ok(())
1044    }
1045
1046    fn set_defaults(&mut self, set_defaults: &[crate::region_request::SetDefault]) -> Result<()> {
1047        for set_default in set_defaults.iter() {
1048            let meta = self
1049                .column_metadatas
1050                .iter_mut()
1051                .find(|col| col.column_schema.name == set_default.name);
1052            if let Some(meta) = meta {
1053                let default_constraint = common_sql::convert::deserialize_default_constraint(
1054                    set_default.default_constraint.as_slice(),
1055                    &meta.column_schema.name,
1056                    &meta.column_schema.data_type,
1057                )
1058                .context(SqlCommonSnafu)?;
1059
1060                meta.column_schema = meta
1061                    .column_schema
1062                    .clone()
1063                    .with_default_constraint(default_constraint)
1064                    .with_context(|_| CastDefaultValueSnafu {
1065                        reason: format!("Failed to set default : {set_default:?}"),
1066                    })?;
1067            } else {
1068                return InvalidRegionRequestSnafu {
1069                    region_id: self.region_id,
1070                    err: format!("column {} not found", set_default.name),
1071                }
1072                .fail();
1073            }
1074        }
1075        Ok(())
1076    }
1077}
1078
1079/// Fields skipped in serialization.
1080struct SkippedFields {
1081    /// Last schema.
1082    schema: SchemaRef,
1083    /// Id of the time index column.
1084    time_index: ColumnId,
1085    /// Map column id to column's index in [column_metadatas](RegionMetadata::column_metadatas).
1086    id_to_index: HashMap<ColumnId, usize>,
1087}
1088
1089impl SkippedFields {
1090    /// Constructs skipped fields from `column_metadatas`.
1091    fn new(column_metadatas: &[ColumnMetadata]) -> Result<SkippedFields> {
1092        let column_schemas = column_metadatas
1093            .iter()
1094            .map(|column_metadata| column_metadata.column_schema.clone())
1095            .collect();
1096        let schema = Arc::new(Schema::try_new(column_schemas).context(InvalidSchemaSnafu)?);
1097        let time_index = column_metadatas
1098            .iter()
1099            .find_map(|col| {
1100                if col.semantic_type == SemanticType::Timestamp {
1101                    Some(col.column_id)
1102                } else {
1103                    None
1104                }
1105            })
1106            .context(InvalidMetaSnafu {
1107                reason: "time index not found",
1108            })?;
1109        let id_to_index = column_metadatas
1110            .iter()
1111            .enumerate()
1112            .map(|(idx, col)| (col.column_id, idx))
1113            .collect();
1114
1115        Ok(SkippedFields {
1116            schema,
1117            time_index,
1118            id_to_index,
1119        })
1120    }
1121}
1122
1123#[derive(Snafu)]
1124#[snafu(visibility(pub))]
1125#[stack_trace_debug]
1126pub enum MetadataError {
1127    #[snafu(display("Invalid schema"))]
1128    InvalidSchema {
1129        source: datatypes::error::Error,
1130        #[snafu(implicit)]
1131        location: Location,
1132    },
1133
1134    #[snafu(display("Invalid metadata, {}", reason))]
1135    InvalidMeta {
1136        reason: String,
1137        #[snafu(implicit)]
1138        location: Location,
1139    },
1140
1141    #[snafu(display("Failed to ser/de json object"))]
1142    SerdeJson {
1143        #[snafu(implicit)]
1144        location: Location,
1145        #[snafu(source)]
1146        error: serde_json::Error,
1147    },
1148
1149    #[snafu(display("Invalid raw region request, err: {}", err))]
1150    InvalidRawRegionRequest {
1151        err: String,
1152        #[snafu(implicit)]
1153        location: Location,
1154    },
1155
1156    #[snafu(display("Invalid region request, region_id: {}, err: {}", region_id, err))]
1157    InvalidRegionRequest {
1158        region_id: RegionId,
1159        err: String,
1160        #[snafu(implicit)]
1161        location: Location,
1162    },
1163
1164    #[snafu(display("Unexpected schema error during project"))]
1165    SchemaProject {
1166        origin_schema: SchemaRef,
1167        projection: Vec<ColumnId>,
1168        #[snafu(implicit)]
1169        location: Location,
1170        source: datatypes::Error,
1171    },
1172
1173    #[snafu(display("Time index column not found"))]
1174    TimeIndexNotFound {
1175        #[snafu(implicit)]
1176        location: Location,
1177    },
1178
1179    #[snafu(display("Change column {} not exists in region: {}", column_name, region_id))]
1180    ChangeColumnNotFound {
1181        column_name: String,
1182        region_id: RegionId,
1183        #[snafu(implicit)]
1184        location: Location,
1185    },
1186
1187    #[snafu(display("Failed to convert column schema"))]
1188    ConvertColumnSchema {
1189        source: api::error::Error,
1190        #[snafu(implicit)]
1191        location: Location,
1192    },
1193
1194    #[snafu(display("Failed to convert TimeRanges"))]
1195    ConvertTimeRanges {
1196        source: api::error::Error,
1197        #[snafu(implicit)]
1198        location: Location,
1199    },
1200
1201    #[snafu(display("Invalid set region option request, key: {}, value: {}", key, value))]
1202    InvalidSetRegionOptionRequest {
1203        key: String,
1204        value: String,
1205        #[snafu(implicit)]
1206        location: Location,
1207    },
1208
1209    #[snafu(display("Invalid set region option request, key: {}", key))]
1210    InvalidUnsetRegionOptionRequest {
1211        key: String,
1212        #[snafu(implicit)]
1213        location: Location,
1214    },
1215
1216    #[snafu(display("Failed to decode protobuf"))]
1217    DecodeProto {
1218        #[snafu(source)]
1219        error: prost::UnknownEnumValue,
1220        #[snafu(implicit)]
1221        location: Location,
1222    },
1223
1224    #[snafu(display("Invalid column option, column name: {}, error: {}", column_name, msg))]
1225    InvalidColumnOption {
1226        column_name: String,
1227        msg: String,
1228        #[snafu(implicit)]
1229        location: Location,
1230    },
1231
1232    #[snafu(display("Failed to set fulltext options for column {}", column_name))]
1233    SetFulltextOptions {
1234        column_name: String,
1235        source: datatypes::Error,
1236        #[snafu(implicit)]
1237        location: Location,
1238    },
1239
1240    #[snafu(display("Failed to get fulltext options for column {}", column_name))]
1241    GetFulltextOptions {
1242        column_name: String,
1243        source: datatypes::Error,
1244        #[snafu(implicit)]
1245        location: Location,
1246    },
1247
1248    #[snafu(display("Failed to set skipping index options for column {}", column_name))]
1249    SetSkippingIndexOptions {
1250        column_name: String,
1251        source: datatypes::Error,
1252        #[snafu(implicit)]
1253        location: Location,
1254    },
1255
1256    #[snafu(display("Failed to unset skipping index options for column {}", column_name))]
1257    UnsetSkippingIndexOptions {
1258        column_name: String,
1259        source: datatypes::Error,
1260        #[snafu(implicit)]
1261        location: Location,
1262    },
1263
1264    #[snafu(display("Failed to decode arrow ipc record batches"))]
1265    DecodeArrowIpc {
1266        #[snafu(source)]
1267        error: arrow::error::ArrowError,
1268        #[snafu(implicit)]
1269        location: Location,
1270    },
1271
1272    #[snafu(display("Failed to cast default value, reason: {}", reason))]
1273    CastDefaultValue {
1274        reason: String,
1275        source: datatypes::Error,
1276        #[snafu(implicit)]
1277        location: Location,
1278    },
1279
1280    #[snafu(display("Unexpected: {}", reason))]
1281    Unexpected {
1282        reason: String,
1283        #[snafu(implicit)]
1284        location: Location,
1285    },
1286
1287    #[snafu(display("Failed to encode/decode flight message"))]
1288    FlightCodec {
1289        source: common_grpc::Error,
1290        #[snafu(implicit)]
1291        location: Location,
1292    },
1293
1294    #[snafu(display("Invalid index option"))]
1295    InvalidIndexOption {
1296        #[snafu(implicit)]
1297        location: Location,
1298        #[snafu(source)]
1299        error: datatypes::error::Error,
1300    },
1301
1302    #[snafu(display("Sql common error"))]
1303    SqlCommon {
1304        source: common_sql::error::Error,
1305        #[snafu(implicit)]
1306        location: Location,
1307    },
1308}
1309
1310impl ErrorExt for MetadataError {
1311    fn status_code(&self) -> StatusCode {
1312        match self {
1313            Self::SqlCommon { source, .. } => source.status_code(),
1314            _ => StatusCode::InvalidArguments,
1315        }
1316    }
1317
1318    fn as_any(&self) -> &dyn Any {
1319        self
1320    }
1321}
1322
1323/// Set column fulltext options if it passed the validation.
1324///
1325/// Options allowed to modify:
1326/// * backend
1327///
1328/// Options not allowed to modify:
1329/// * analyzer
1330/// * case_sensitive
1331fn set_column_fulltext_options(
1332    column_meta: &mut ColumnMetadata,
1333    column_name: &str,
1334    options: &FulltextOptions,
1335    current_options: Option<FulltextOptions>,
1336) -> Result<()> {
1337    if let Some(current_options) = current_options {
1338        ensure!(
1339            current_options.analyzer == options.analyzer
1340                && current_options.case_sensitive == options.case_sensitive,
1341            InvalidColumnOptionSnafu {
1342                column_name,
1343                msg: format!(
1344                    "Cannot change analyzer or case_sensitive if FULLTEXT index is set before. Previous analyzer: {}, previous case_sensitive: {}",
1345                    current_options.analyzer, current_options.case_sensitive
1346                ),
1347            }
1348        );
1349    }
1350
1351    column_meta
1352        .column_schema
1353        .set_fulltext_options(options)
1354        .context(SetFulltextOptionsSnafu { column_name })?;
1355
1356    Ok(())
1357}
1358
1359fn unset_column_fulltext_options(
1360    column_meta: &mut ColumnMetadata,
1361    column_name: &str,
1362    current_options: Option<FulltextOptions>,
1363) -> Result<()> {
1364    if let Some(mut current_options) = current_options
1365        && current_options.enable
1366    {
1367        current_options.enable = false;
1368        column_meta
1369            .column_schema
1370            .set_fulltext_options(&current_options)
1371            .context(SetFulltextOptionsSnafu { column_name })?;
1372    } else {
1373        return InvalidColumnOptionSnafu {
1374            column_name,
1375            msg: "FULLTEXT index already disabled",
1376        }
1377        .fail();
1378    }
1379
1380    Ok(())
1381}
1382
1383#[cfg(test)]
1384mod test {
1385    use datatypes::prelude::ConcreteDataType;
1386    use datatypes::schema::{
1387        ColumnDefaultConstraint, ColumnSchema, FulltextAnalyzer, FulltextBackend,
1388    };
1389    use datatypes::value::Value;
1390
1391    use super::*;
1392
1393    fn create_builder() -> RegionMetadataBuilder {
1394        RegionMetadataBuilder::new(RegionId::new(1234, 5678))
1395    }
1396
1397    fn build_test_region_metadata() -> RegionMetadata {
1398        let mut builder = create_builder();
1399        builder
1400            .push_column_metadata(ColumnMetadata {
1401                column_schema: ColumnSchema::new("a", ConcreteDataType::int64_datatype(), false),
1402                semantic_type: SemanticType::Tag,
1403                column_id: 1,
1404            })
1405            .push_column_metadata(ColumnMetadata {
1406                column_schema: ColumnSchema::new("b", ConcreteDataType::float64_datatype(), false),
1407                semantic_type: SemanticType::Field,
1408                column_id: 2,
1409            })
1410            .push_column_metadata(ColumnMetadata {
1411                column_schema: ColumnSchema::new(
1412                    "c",
1413                    ConcreteDataType::timestamp_millisecond_datatype(),
1414                    false,
1415                ),
1416                semantic_type: SemanticType::Timestamp,
1417                column_id: 3,
1418            })
1419            .primary_key(vec![1])
1420            .partition_expr_json(Some("".to_string()));
1421        builder.build().unwrap()
1422    }
1423
1424    #[test]
1425    fn test_sparse_primary_key_requires_reserved_columns() {
1426        for primary_key in [vec![], vec![1]] {
1427            let mut metadata = build_test_region_metadata();
1428            if primary_key.is_empty() {
1429                metadata.column_metadatas[0].semantic_type = SemanticType::Field;
1430            }
1431            metadata.primary_key = primary_key;
1432            let mut builder = RegionMetadataBuilder::from_existing(metadata);
1433            builder.primary_key_encoding(PrimaryKeyEncoding::Sparse);
1434            let error = builder.build().unwrap_err();
1435            assert_eq!(error.status_code(), StatusCode::InvalidArguments);
1436            assert!(
1437                error
1438                    .to_string()
1439                    .contains("sparse primary keys must start with (__table_id, __tsid)")
1440            );
1441        }
1442    }
1443
1444    #[test]
1445    fn test_sparse_reserved_primary_key_contract() {
1446        let mut builder = RegionMetadataBuilder::from_existing(build_test_region_metadata());
1447        builder.primary_key_encoding(PrimaryKeyEncoding::Sparse);
1448        builder.column_metadatas[0].column_schema.data_type = ConcreteDataType::string_datatype();
1449        for (name, column_id, data_type) in [
1450            (
1451                DATA_SCHEMA_TABLE_ID_COLUMN_NAME,
1452                ReservedColumnId::table_id(),
1453                ConcreteDataType::uint32_datatype(),
1454            ),
1455            (
1456                DATA_SCHEMA_TSID_COLUMN_NAME,
1457                ReservedColumnId::tsid(),
1458                ConcreteDataType::uint64_datatype(),
1459            ),
1460        ] {
1461            builder.push_column_metadata(ColumnMetadata {
1462                column_schema: ColumnSchema::new(name, data_type, false),
1463                semantic_type: SemanticType::Tag,
1464                column_id,
1465            });
1466        }
1467        builder.primary_key(vec![
1468            ReservedColumnId::table_id(),
1469            ReservedColumnId::tsid(),
1470            1,
1471        ]);
1472        let metadata = builder.build().unwrap();
1473        let json = serde_json::to_string(&metadata).unwrap();
1474        assert_eq!(
1475            metadata,
1476            serde_json::from_str::<RegionMetadata>(&json).unwrap()
1477        );
1478
1479        for case in 0..12 {
1480            let mut builder = RegionMetadataBuilder::from_existing(metadata.clone());
1481            match case {
1482                0 => builder.primary_key.swap(0, 1),
1483                1 => {
1484                    builder.primary_key.remove(0);
1485                    builder.column_metadatas.remove(3);
1486                }
1487                2 => builder.column_metadatas[3].column_schema.set_nullable(),
1488                3 => {
1489                    builder.column_metadatas[3].column_schema.data_type =
1490                        ConcreteDataType::uint64_datatype()
1491                }
1492                4 => builder.column_metadatas[4].column_schema.name = "wrong_name".into(),
1493                5 => {
1494                    builder.column_metadatas[0].column_schema.data_type =
1495                        ConcreteDataType::int64_datatype()
1496                }
1497                6 => {
1498                    builder.primary_key.remove(1);
1499                    builder.column_metadatas.remove(4);
1500                }
1501                7 => builder.column_metadatas[4].column_schema.set_nullable(),
1502                8 => {
1503                    builder.column_metadatas[4].column_schema.data_type =
1504                        ConcreteDataType::uint32_datatype()
1505                }
1506                9 => builder.column_metadatas[3].semantic_type = SemanticType::Field,
1507                10 => builder.column_metadatas[4].semantic_type = SemanticType::Field,
1508                11 => {
1509                    builder.primary_key[0] = 10;
1510                    builder.primary_key[1] = 11;
1511                    builder.column_metadatas[3].column_id = 10;
1512                    builder.column_metadatas[4].column_id = 11;
1513                }
1514                _ => unreachable!(),
1515            }
1516            assert_eq!(
1517                builder.build().unwrap_err().status_code(),
1518                StatusCode::InvalidArguments
1519            );
1520        }
1521    }
1522
1523    #[test]
1524    fn test_region_metadata() {
1525        let region_metadata = build_test_region_metadata();
1526        assert_eq!("c", region_metadata.time_index_column().column_schema.name);
1527        assert_eq!(
1528            "a",
1529            region_metadata.column_by_id(1).unwrap().column_schema.name
1530        );
1531        assert_eq!(None, region_metadata.column_by_id(10));
1532    }
1533
1534    #[test]
1535    fn test_region_metadata_serde() {
1536        let region_metadata = build_test_region_metadata();
1537        let serialized = serde_json::to_string(&region_metadata).unwrap();
1538        let deserialized: RegionMetadata = serde_json::from_str(&serialized).unwrap();
1539        assert_eq!(region_metadata, deserialized);
1540    }
1541
1542    #[test]
1543    fn test_column_metadata_validate() {
1544        let mut builder = create_builder();
1545        let col = ColumnMetadata {
1546            column_schema: ColumnSchema::new("ts", ConcreteDataType::string_datatype(), false),
1547            semantic_type: SemanticType::Timestamp,
1548            column_id: 1,
1549        };
1550
1551        builder.push_column_metadata(col);
1552        let err = builder.build().unwrap_err();
1553        assert!(
1554            err.to_string()
1555                .contains("column `ts` is not timestamp type"),
1556            "unexpected err: {err}",
1557        );
1558    }
1559
1560    #[test]
1561    fn test_empty_region_metadata() {
1562        let builder = create_builder();
1563        let err = builder.build().unwrap_err();
1564        // A region must have a time index.
1565        assert!(
1566            err.to_string().contains("time index not found"),
1567            "unexpected err: {err}",
1568        );
1569    }
1570
1571    #[test]
1572    fn test_same_column_id() {
1573        let mut builder = create_builder();
1574        builder
1575            .push_column_metadata(ColumnMetadata {
1576                column_schema: ColumnSchema::new("a", ConcreteDataType::int64_datatype(), false),
1577                semantic_type: SemanticType::Tag,
1578                column_id: 1,
1579            })
1580            .push_column_metadata(ColumnMetadata {
1581                column_schema: ColumnSchema::new(
1582                    "b",
1583                    ConcreteDataType::timestamp_millisecond_datatype(),
1584                    false,
1585                ),
1586                semantic_type: SemanticType::Timestamp,
1587                column_id: 1,
1588            });
1589        let err = builder.build().unwrap_err();
1590        assert!(
1591            err.to_string()
1592                .contains("column a and b have the same column id"),
1593            "unexpected err: {err}",
1594        );
1595    }
1596
1597    #[test]
1598    fn test_duplicate_time_index() {
1599        let mut builder = create_builder();
1600        builder
1601            .push_column_metadata(ColumnMetadata {
1602                column_schema: ColumnSchema::new(
1603                    "a",
1604                    ConcreteDataType::timestamp_millisecond_datatype(),
1605                    false,
1606                ),
1607                semantic_type: SemanticType::Timestamp,
1608                column_id: 1,
1609            })
1610            .push_column_metadata(ColumnMetadata {
1611                column_schema: ColumnSchema::new(
1612                    "b",
1613                    ConcreteDataType::timestamp_millisecond_datatype(),
1614                    false,
1615                ),
1616                semantic_type: SemanticType::Timestamp,
1617                column_id: 2,
1618            });
1619        let err = builder.build().unwrap_err();
1620        assert!(
1621            err.to_string().contains("expect only one time index"),
1622            "unexpected err: {err}",
1623        );
1624    }
1625
1626    #[test]
1627    fn test_unknown_primary_key() {
1628        let mut builder = create_builder();
1629        builder
1630            .push_column_metadata(ColumnMetadata {
1631                column_schema: ColumnSchema::new("a", ConcreteDataType::string_datatype(), false),
1632                semantic_type: SemanticType::Tag,
1633                column_id: 1,
1634            })
1635            .push_column_metadata(ColumnMetadata {
1636                column_schema: ColumnSchema::new(
1637                    "b",
1638                    ConcreteDataType::timestamp_millisecond_datatype(),
1639                    false,
1640                ),
1641                semantic_type: SemanticType::Timestamp,
1642                column_id: 2,
1643            })
1644            .primary_key(vec![3]);
1645        let err = builder.build().unwrap_err();
1646        assert!(
1647            err.to_string().contains("unknown column id 3"),
1648            "unexpected err: {err}",
1649        );
1650    }
1651
1652    #[test]
1653    fn test_same_primary_key() {
1654        let mut builder = create_builder();
1655        builder
1656            .push_column_metadata(ColumnMetadata {
1657                column_schema: ColumnSchema::new("a", ConcreteDataType::string_datatype(), false),
1658                semantic_type: SemanticType::Tag,
1659                column_id: 1,
1660            })
1661            .push_column_metadata(ColumnMetadata {
1662                column_schema: ColumnSchema::new(
1663                    "b",
1664                    ConcreteDataType::timestamp_millisecond_datatype(),
1665                    false,
1666                ),
1667                semantic_type: SemanticType::Timestamp,
1668                column_id: 2,
1669            })
1670            .primary_key(vec![1, 1]);
1671        let err = builder.build().unwrap_err();
1672        assert!(
1673            err.to_string()
1674                .contains("duplicate column a in primary key"),
1675            "unexpected err: {err}",
1676        );
1677    }
1678
1679    #[test]
1680    fn test_in_time_index() {
1681        let mut builder = create_builder();
1682        builder
1683            .push_column_metadata(ColumnMetadata {
1684                column_schema: ColumnSchema::new(
1685                    "ts",
1686                    ConcreteDataType::timestamp_millisecond_datatype(),
1687                    false,
1688                ),
1689                semantic_type: SemanticType::Timestamp,
1690                column_id: 1,
1691            })
1692            .primary_key(vec![1]);
1693        let err = builder.build().unwrap_err();
1694        assert!(
1695            err.to_string()
1696                .contains("column ts is already a time index column"),
1697            "unexpected err: {err}",
1698        );
1699    }
1700
1701    #[test]
1702    fn test_nullable_time_index() {
1703        let mut builder = create_builder();
1704        builder.push_column_metadata(ColumnMetadata {
1705            column_schema: ColumnSchema::new(
1706                "ts",
1707                ConcreteDataType::timestamp_millisecond_datatype(),
1708                true,
1709            ),
1710            semantic_type: SemanticType::Timestamp,
1711            column_id: 1,
1712        });
1713        let err = builder.build().unwrap_err();
1714        assert!(
1715            err.to_string()
1716                .contains("time index column ts must be NOT NULL"),
1717            "unexpected err: {err}",
1718        );
1719    }
1720
1721    #[test]
1722    fn test_primary_key_semantic_type() {
1723        let mut builder = create_builder();
1724        builder
1725            .push_column_metadata(ColumnMetadata {
1726                column_schema: ColumnSchema::new(
1727                    "ts",
1728                    ConcreteDataType::timestamp_millisecond_datatype(),
1729                    false,
1730                ),
1731                semantic_type: SemanticType::Timestamp,
1732                column_id: 1,
1733            })
1734            .push_column_metadata(ColumnMetadata {
1735                column_schema: ColumnSchema::new("a", ConcreteDataType::float64_datatype(), true),
1736                semantic_type: SemanticType::Field,
1737                column_id: 2,
1738            })
1739            .primary_key(vec![2]);
1740        let err = builder.build().unwrap_err();
1741        assert!(
1742            err.to_string()
1743                .contains("semantic type of column a should be Tag, not Field"),
1744            "unexpected err: {err}",
1745        );
1746    }
1747
1748    #[test]
1749    fn test_primary_key_tag_num() {
1750        let mut builder = create_builder();
1751        builder
1752            .push_column_metadata(ColumnMetadata {
1753                column_schema: ColumnSchema::new(
1754                    "ts",
1755                    ConcreteDataType::timestamp_millisecond_datatype(),
1756                    false,
1757                ),
1758                semantic_type: SemanticType::Timestamp,
1759                column_id: 1,
1760            })
1761            .push_column_metadata(ColumnMetadata {
1762                column_schema: ColumnSchema::new("a", ConcreteDataType::string_datatype(), true),
1763                semantic_type: SemanticType::Tag,
1764                column_id: 2,
1765            })
1766            .push_column_metadata(ColumnMetadata {
1767                column_schema: ColumnSchema::new("b", ConcreteDataType::string_datatype(), true),
1768                semantic_type: SemanticType::Tag,
1769                column_id: 3,
1770            })
1771            .primary_key(vec![2]);
1772        let err = builder.build().unwrap_err();
1773        assert!(
1774            err.to_string()
1775                .contains("number of primary key columns 1 not equal to tag columns 2"),
1776            "unexpected err: {err}",
1777        );
1778    }
1779
1780    #[test]
1781    fn test_bump_version() {
1782        let mut region_metadata = build_test_region_metadata();
1783        let mut builder = RegionMetadataBuilder::from_existing(region_metadata.clone());
1784        builder.bump_version();
1785        let new_meta = builder.build().unwrap();
1786        region_metadata.schema_version += 1;
1787        assert_eq!(region_metadata, new_meta);
1788    }
1789
1790    fn new_column_metadata(name: &str, is_tag: bool, column_id: ColumnId) -> ColumnMetadata {
1791        let semantic_type = if is_tag {
1792            SemanticType::Tag
1793        } else {
1794            SemanticType::Field
1795        };
1796        ColumnMetadata {
1797            column_schema: ColumnSchema::new(name, ConcreteDataType::string_datatype(), true),
1798            semantic_type,
1799            column_id,
1800        }
1801    }
1802
1803    fn check_columns(metadata: &RegionMetadata, names: &[&str]) {
1804        let actual: Vec<_> = metadata
1805            .column_metadatas
1806            .iter()
1807            .map(|col| &col.column_schema.name)
1808            .collect();
1809        assert_eq!(names, actual);
1810    }
1811
1812    fn get_columns_default_constraint(
1813        metadata: &RegionMetadata,
1814        name: String,
1815    ) -> Option<Option<&ColumnDefaultConstraint>> {
1816        metadata.column_metadatas.iter().find_map(|col| {
1817            if col.column_schema.name == name {
1818                Some(col.column_schema.default_constraint())
1819            } else {
1820                None
1821            }
1822        })
1823    }
1824
1825    #[test]
1826    fn test_alter() {
1827        // a (tag), b (field), c (ts)
1828        let metadata = build_test_region_metadata();
1829        let mut builder = RegionMetadataBuilder::from_existing(metadata);
1830        // tag d
1831        builder
1832            .alter(AlterKind::AddColumns {
1833                columns: vec![AddColumn {
1834                    column_metadata: new_column_metadata("d", true, 4),
1835                    location: None,
1836                }],
1837            })
1838            .unwrap();
1839        let metadata = builder.build().unwrap();
1840        check_columns(&metadata, &["a", "b", "c", "d"]);
1841        assert_eq!([1, 4], &metadata.primary_key[..]);
1842
1843        let mut builder = RegionMetadataBuilder::from_existing(metadata);
1844        builder
1845            .alter(AlterKind::AddColumns {
1846                columns: vec![AddColumn {
1847                    column_metadata: new_column_metadata("e", false, 5),
1848                    location: Some(AddColumnLocation::First),
1849                }],
1850            })
1851            .unwrap();
1852        let metadata = builder.build().unwrap();
1853        check_columns(&metadata, &["e", "a", "b", "c", "d"]);
1854
1855        let mut builder = RegionMetadataBuilder::from_existing(metadata);
1856        builder
1857            .alter(AlterKind::AddColumns {
1858                columns: vec![AddColumn {
1859                    column_metadata: new_column_metadata("f", false, 6),
1860                    location: Some(AddColumnLocation::After {
1861                        column_name: "b".to_string(),
1862                    }),
1863                }],
1864            })
1865            .unwrap();
1866        let metadata = builder.build().unwrap();
1867        check_columns(&metadata, &["e", "a", "b", "f", "c", "d"]);
1868
1869        let mut builder = RegionMetadataBuilder::from_existing(metadata);
1870        builder
1871            .alter(AlterKind::AddColumns {
1872                columns: vec![AddColumn {
1873                    column_metadata: new_column_metadata("g", false, 7),
1874                    location: Some(AddColumnLocation::After {
1875                        column_name: "d".to_string(),
1876                    }),
1877                }],
1878            })
1879            .unwrap();
1880        let metadata = builder.build().unwrap();
1881        check_columns(&metadata, &["e", "a", "b", "f", "c", "d", "g"]);
1882
1883        let mut builder = RegionMetadataBuilder::from_existing(metadata);
1884        builder
1885            .alter(AlterKind::DropColumns {
1886                names: vec!["g".to_string(), "e".to_string()],
1887            })
1888            .unwrap();
1889        let metadata = builder.build().unwrap();
1890        check_columns(&metadata, &["a", "b", "f", "c", "d"]);
1891
1892        let mut builder = RegionMetadataBuilder::from_existing(metadata.clone());
1893        builder
1894            .alter(AlterKind::DropColumns {
1895                names: vec!["a".to_string()],
1896            })
1897            .unwrap();
1898        // Build returns error as the primary key contains a.
1899        let err = builder.build().unwrap_err();
1900        assert_eq!(StatusCode::InvalidArguments, err.status_code());
1901
1902        let mut builder: RegionMetadataBuilder = RegionMetadataBuilder::from_existing(metadata);
1903        let mut column_metadata = new_column_metadata("g", false, 8);
1904        let default_constraint = Some(ColumnDefaultConstraint::Value(Value::from("g")));
1905        column_metadata.column_schema = column_metadata
1906            .column_schema
1907            .with_default_constraint(default_constraint.clone())
1908            .unwrap();
1909        builder
1910            .alter(AlterKind::AddColumns {
1911                columns: vec![AddColumn {
1912                    column_metadata,
1913                    location: None,
1914                }],
1915            })
1916            .unwrap();
1917        let metadata = builder.build().unwrap();
1918        assert_eq!(
1919            get_columns_default_constraint(&metadata, "g".to_string()).unwrap(),
1920            default_constraint.as_ref()
1921        );
1922        check_columns(&metadata, &["a", "b", "f", "c", "d", "g"]);
1923
1924        let mut builder: RegionMetadataBuilder = RegionMetadataBuilder::from_existing(metadata);
1925        builder
1926            .alter(AlterKind::DropDefaults {
1927                names: vec!["g".to_string()],
1928            })
1929            .unwrap();
1930        let metadata = builder.build().unwrap();
1931        assert_eq!(
1932            get_columns_default_constraint(&metadata, "g".to_string()).unwrap(),
1933            None
1934        );
1935        check_columns(&metadata, &["a", "b", "f", "c", "d", "g"]);
1936
1937        let mut builder: RegionMetadataBuilder = RegionMetadataBuilder::from_existing(metadata);
1938        builder
1939            .alter(AlterKind::DropColumns {
1940                names: vec!["g".to_string()],
1941            })
1942            .unwrap();
1943        let metadata = builder.build().unwrap();
1944        check_columns(&metadata, &["a", "b", "f", "c", "d"]);
1945
1946        let mut builder = RegionMetadataBuilder::from_existing(metadata);
1947        builder
1948            .alter(AlterKind::ModifyColumnTypes {
1949                columns: vec![ModifyColumnType {
1950                    column_name: "b".to_string(),
1951                    target_type: ConcreteDataType::string_datatype(),
1952                }],
1953            })
1954            .unwrap();
1955        let metadata = builder.build().unwrap();
1956        check_columns(&metadata, &["a", "b", "f", "c", "d"]);
1957        let b_type = &metadata
1958            .column_by_name("b")
1959            .unwrap()
1960            .column_schema
1961            .data_type;
1962        assert_eq!(ConcreteDataType::string_datatype(), *b_type);
1963
1964        let mut builder = RegionMetadataBuilder::from_existing(metadata);
1965        builder
1966            .alter(AlterKind::SetIndexes {
1967                options: vec![SetIndexOption::Fulltext {
1968                    column_name: "b".to_string(),
1969                    options: FulltextOptions::new_unchecked(
1970                        true,
1971                        FulltextAnalyzer::Chinese,
1972                        true,
1973                        FulltextBackend::Bloom,
1974                        1000,
1975                        0.01,
1976                    ),
1977                }],
1978            })
1979            .unwrap();
1980        let metadata = builder.build().unwrap();
1981        let a_fulltext_options = metadata
1982            .column_by_name("b")
1983            .unwrap()
1984            .column_schema
1985            .fulltext_options()
1986            .unwrap()
1987            .unwrap();
1988        assert!(a_fulltext_options.enable);
1989        assert_eq!(
1990            datatypes::schema::FulltextAnalyzer::Chinese,
1991            a_fulltext_options.analyzer
1992        );
1993        assert!(a_fulltext_options.case_sensitive);
1994
1995        let mut builder = RegionMetadataBuilder::from_existing(metadata);
1996        builder
1997            .alter(AlterKind::UnsetIndexes {
1998                options: vec![UnsetIndexOption::Fulltext {
1999                    column_name: "b".to_string(),
2000                }],
2001            })
2002            .unwrap();
2003        let metadata = builder.build().unwrap();
2004        let a_fulltext_options = metadata
2005            .column_by_name("b")
2006            .unwrap()
2007            .column_schema
2008            .fulltext_options()
2009            .unwrap()
2010            .unwrap();
2011        assert!(!a_fulltext_options.enable);
2012        assert_eq!(
2013            datatypes::schema::FulltextAnalyzer::Chinese,
2014            a_fulltext_options.analyzer
2015        );
2016        assert!(a_fulltext_options.case_sensitive);
2017    }
2018
2019    #[test]
2020    fn test_add_if_not_exists() {
2021        // a (tag), b (field), c (ts)
2022        let metadata = build_test_region_metadata();
2023        let mut builder = RegionMetadataBuilder::from_existing(metadata);
2024        // tag d
2025        builder
2026            .alter(AlterKind::AddColumns {
2027                columns: vec![
2028                    AddColumn {
2029                        column_metadata: new_column_metadata("d", true, 4),
2030                        location: None,
2031                    },
2032                    AddColumn {
2033                        column_metadata: new_column_metadata("d", true, 4),
2034                        location: None,
2035                    },
2036                ],
2037            })
2038            .unwrap();
2039        let metadata = builder.build().unwrap();
2040        check_columns(&metadata, &["a", "b", "c", "d"]);
2041        assert_eq!([1, 4], &metadata.primary_key[..]);
2042
2043        let mut builder = RegionMetadataBuilder::from_existing(metadata);
2044        // field b.
2045        builder
2046            .alter(AlterKind::AddColumns {
2047                columns: vec![AddColumn {
2048                    column_metadata: new_column_metadata("b", false, 2),
2049                    location: None,
2050                }],
2051            })
2052            .unwrap();
2053        let metadata = builder.build().unwrap();
2054        check_columns(&metadata, &["a", "b", "c", "d"]);
2055    }
2056
2057    #[test]
2058    fn test_add_column_with_inverted_index() {
2059        // only set inverted index to true explicitly will this column be inverted indexed
2060
2061        // a (tag), b (field), c (ts)
2062        let metadata = build_test_region_metadata();
2063        let mut builder = RegionMetadataBuilder::from_existing(metadata);
2064        // tag d, e
2065        let mut col = new_column_metadata("d", true, 4);
2066        col.column_schema.set_inverted_index(true);
2067        builder
2068            .alter(AlterKind::AddColumns {
2069                columns: vec![
2070                    AddColumn {
2071                        column_metadata: col,
2072                        location: None,
2073                    },
2074                    AddColumn {
2075                        column_metadata: new_column_metadata("e", true, 5),
2076                        location: None,
2077                    },
2078                ],
2079            })
2080            .unwrap();
2081        let metadata = builder.build().unwrap();
2082        check_columns(&metadata, &["a", "b", "c", "d", "e"]);
2083        assert_eq!([1, 4, 5], &metadata.primary_key[..]);
2084        let column_metadata = metadata.column_by_name("a").unwrap();
2085        assert!(!column_metadata.column_schema.is_inverted_indexed());
2086        let column_metadata = metadata.column_by_name("b").unwrap();
2087        assert!(!column_metadata.column_schema.is_inverted_indexed());
2088        let column_metadata = metadata.column_by_name("c").unwrap();
2089        assert!(!column_metadata.column_schema.is_inverted_indexed());
2090        let column_metadata = metadata.column_by_name("d").unwrap();
2091        assert!(column_metadata.column_schema.is_inverted_indexed());
2092        let column_metadata = metadata.column_by_name("e").unwrap();
2093        assert!(!column_metadata.column_schema.is_inverted_indexed());
2094    }
2095
2096    #[test]
2097    fn test_drop_if_exists() {
2098        // a (tag), b (field), c (ts)
2099        let metadata = build_test_region_metadata();
2100        let mut builder = RegionMetadataBuilder::from_existing(metadata);
2101        // field d, e
2102        builder
2103            .alter(AlterKind::AddColumns {
2104                columns: vec![
2105                    AddColumn {
2106                        column_metadata: new_column_metadata("d", false, 4),
2107                        location: None,
2108                    },
2109                    AddColumn {
2110                        column_metadata: new_column_metadata("e", false, 5),
2111                        location: None,
2112                    },
2113                ],
2114            })
2115            .unwrap();
2116        let metadata = builder.build().unwrap();
2117        check_columns(&metadata, &["a", "b", "c", "d", "e"]);
2118
2119        let mut builder = RegionMetadataBuilder::from_existing(metadata);
2120        builder
2121            .alter(AlterKind::DropColumns {
2122                names: vec!["b".to_string(), "b".to_string()],
2123            })
2124            .unwrap();
2125        let metadata = builder.build().unwrap();
2126        check_columns(&metadata, &["a", "c", "d", "e"]);
2127
2128        let mut builder = RegionMetadataBuilder::from_existing(metadata);
2129        builder
2130            .alter(AlterKind::DropColumns {
2131                names: vec!["b".to_string(), "e".to_string()],
2132            })
2133            .unwrap();
2134        let metadata = builder.build().unwrap();
2135        check_columns(&metadata, &["a", "c", "d"]);
2136    }
2137
2138    #[test]
2139    fn test_invalid_column_name() {
2140        let mut builder = create_builder();
2141        builder.push_column_metadata(ColumnMetadata {
2142            column_schema: ColumnSchema::new(
2143                "__sequence",
2144                ConcreteDataType::timestamp_millisecond_datatype(),
2145                false,
2146            ),
2147            semantic_type: SemanticType::Timestamp,
2148            column_id: 1,
2149        });
2150        let err = builder.build().unwrap_err();
2151        assert!(
2152            err.to_string()
2153                .contains("internal column name that can not be used"),
2154            "unexpected err: {err}",
2155        );
2156    }
2157
2158    #[test]
2159    fn test_allow_internal_column_name() {
2160        let mut builder = create_builder();
2161        builder
2162            .push_column_metadata(ColumnMetadata {
2163                column_schema: ColumnSchema::new(
2164                    "__primary_key",
2165                    ConcreteDataType::string_datatype(),
2166                    false,
2167                ),
2168                semantic_type: SemanticType::Tag,
2169                column_id: 1,
2170            })
2171            .push_column_metadata(ColumnMetadata {
2172                column_schema: ColumnSchema::new(
2173                    "ts",
2174                    ConcreteDataType::timestamp_millisecond_datatype(),
2175                    false,
2176                ),
2177                semantic_type: SemanticType::Timestamp,
2178                column_id: 2,
2179            })
2180            .primary_key(vec![1]);
2181
2182        let metadata = builder.build_without_validation().unwrap();
2183        assert_eq!(
2184            "__primary_key",
2185            metadata.column_metadatas[0].column_schema.name
2186        );
2187    }
2188
2189    #[test]
2190    fn test_build_without_validation() {
2191        // Primary key points to a Field column, which would normally fail validation.
2192        let mut builder = create_builder();
2193        builder
2194            .push_column_metadata(ColumnMetadata {
2195                column_schema: ColumnSchema::new(
2196                    "ts",
2197                    ConcreteDataType::timestamp_millisecond_datatype(),
2198                    false,
2199                ),
2200                semantic_type: SemanticType::Timestamp,
2201                column_id: 1,
2202            })
2203            .push_column_metadata(ColumnMetadata {
2204                column_schema: ColumnSchema::new(
2205                    "field",
2206                    ConcreteDataType::string_datatype(),
2207                    true,
2208                ),
2209                semantic_type: SemanticType::Field,
2210                column_id: 2,
2211            })
2212            .primary_key(vec![2]);
2213
2214        // Unvalidated build should succeed.
2215        let metadata = builder.build_without_validation().unwrap();
2216        assert_eq!(vec![2], metadata.primary_key);
2217
2218        // Validated build still rejects it.
2219        let mut builder = create_builder();
2220        builder
2221            .push_column_metadata(ColumnMetadata {
2222                column_schema: ColumnSchema::new(
2223                    "ts",
2224                    ConcreteDataType::timestamp_millisecond_datatype(),
2225                    false,
2226                ),
2227                semantic_type: SemanticType::Timestamp,
2228                column_id: 1,
2229            })
2230            .push_column_metadata(ColumnMetadata {
2231                column_schema: ColumnSchema::new(
2232                    "field",
2233                    ConcreteDataType::string_datatype(),
2234                    true,
2235                ),
2236                semantic_type: SemanticType::Field,
2237                column_id: 2,
2238            })
2239            .primary_key(vec![2]);
2240        let err = builder.build().unwrap_err();
2241        assert!(
2242            err.to_string()
2243                .contains("semantic type of column field should be Tag"),
2244            "unexpected err: {err}"
2245        );
2246    }
2247
2248    #[test]
2249    fn test_debug_for_column_metadata() {
2250        let region_metadata = build_test_region_metadata();
2251        let formatted = format!("{:?}", region_metadata);
2252        assert_eq!(
2253            formatted,
2254            "RegionMetadata { column_metadatas: [[a Int64 not null Tag 1], [b Float64 not null Field 2], [c TimestampMillisecond not null Timestamp 3]], time_index: 3, primary_key: [1], region_id: 5299989648942(1234, 5678), schema_version: 0, partition_expr: Some(\"\") }"
2255        );
2256    }
2257
2258    #[test]
2259    fn test_region_metadata_deserialize_default_primary_key_encoding() {
2260        let serialize = r#"{"column_metadatas":[{"column_schema":{"name":"a","data_type":{"Int64":{}},"is_nullable":false,"is_time_index":false,"default_constraint":null,"metadata":{}},"semantic_type":"Tag","column_id":1},{"column_schema":{"name":"b","data_type":{"Float64":{}},"is_nullable":false,"is_time_index":false,"default_constraint":null,"metadata":{}},"semantic_type":"Field","column_id":2},{"column_schema":{"name":"c","data_type":{"Timestamp":{"Millisecond":null}},"is_nullable":false,"is_time_index":false,"default_constraint":null,"metadata":{}},"semantic_type":"Timestamp","column_id":3}],"primary_key":[1],"region_id":5299989648942,"schema_version":0}"#;
2261        let deserialized: RegionMetadata = serde_json::from_str(serialize).unwrap();
2262        assert_eq!(deserialized.primary_key_encoding, PrimaryKeyEncoding::Dense);
2263
2264        let serialize = r#"{"column_metadatas":[{"column_schema":{"name":"a","data_type":{"Int64":{}},"is_nullable":false,"is_time_index":false,"default_constraint":null,"metadata":{}},"semantic_type":"Tag","column_id":1},{"column_schema":{"name":"b","data_type":{"Float64":{}},"is_nullable":false,"is_time_index":false,"default_constraint":null,"metadata":{}},"semantic_type":"Field","column_id":2},{"column_schema":{"name":"c","data_type":{"Timestamp":{"Millisecond":null}},"is_nullable":false,"is_time_index":false,"default_constraint":null,"metadata":{}},"semantic_type":"Timestamp","column_id":3}],"primary_key":[1],"region_id":5299989648942,"schema_version":0,"primary_key_encoding":"sparse"}"#;
2265        let deserialized: RegionMetadata = serde_json::from_str(serialize).unwrap();
2266        assert_eq!(
2267            deserialized.primary_key_encoding,
2268            PrimaryKeyEncoding::Sparse
2269        );
2270    }
2271}