1use std::collections::{HashMap, HashSet};
16use std::sync::Arc;
17
18use chrono::{DateTime, Utc};
19use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
20use common_macro::ToMetaBuilder;
21use common_query::AddColumnLocation;
22use datafusion_expr::TableProviderFilterPushDown;
23pub use datatypes::error::{Error as ConvertError, Result as ConvertResult};
24use datatypes::schema::{
25 ColumnSchema, FulltextOptions, Schema, SchemaBuilder, SchemaRef, SkippingIndexOptions,
26};
27use derive_builder::Builder;
28use serde::{Deserialize, Deserializer, Serialize};
29use snafu::{OptionExt, ResultExt, ensure};
30use store_api::metric_engine_consts::PHYSICAL_TABLE_METADATA_KEY;
31use store_api::mito_engine_options::{
32 APPEND_MODE_KEY, AUTO_FLUSH_INTERVAL_KEY, COMPACTION_TYPE, COMPACTION_TYPE_TWCS,
33 MAX_ROW_GROUP_ROW_COUNT, MERGE_MODE_KEY, SKIP_WAL_KEY, SST_FORMAT_KEY,
34};
35use store_api::region_request::{SetRegionOption, UnsetRegionOption};
36use store_api::storage::{ColumnDescriptor, ColumnDescriptorBuilder, ColumnId};
37
38use crate::error::{self, Result};
39use crate::requests::{
40 AddColumnRequest, AlterKind, AnnotationContext, AnnotationFamily, AnnotationValidationError,
41 ModifyColumnTypeRequest, REPARTITION_COLUMN_HINT_KEY, SetDefaultRequest, SetIndexOption,
42 TableOptions, UnsetIndexOption, has_stable_string_form, parse_entity_columns,
43 parse_entity_option_key, validate_and_normalize_annotation,
44};
45use crate::table_reference::TableReference;
46
47pub type TableId = u32;
48pub type TableVersion = u64;
49
50#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
53pub enum FilterPushDownType {
54 Unsupported,
56 Inexact,
61 Exact,
65}
66
67impl From<TableProviderFilterPushDown> for FilterPushDownType {
68 fn from(value: TableProviderFilterPushDown) -> Self {
69 match value {
70 TableProviderFilterPushDown::Unsupported => FilterPushDownType::Unsupported,
71 TableProviderFilterPushDown::Inexact => FilterPushDownType::Inexact,
72 TableProviderFilterPushDown::Exact => FilterPushDownType::Exact,
73 }
74 }
75}
76
77impl From<FilterPushDownType> for TableProviderFilterPushDown {
78 fn from(value: FilterPushDownType) -> Self {
79 match value {
80 FilterPushDownType::Unsupported => TableProviderFilterPushDown::Unsupported,
81 FilterPushDownType::Inexact => TableProviderFilterPushDown::Inexact,
82 FilterPushDownType::Exact => TableProviderFilterPushDown::Exact,
83 }
84 }
85}
86
87#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
89pub enum TableType {
90 Base,
92 View,
94 Temporary,
96}
97
98impl std::fmt::Display for TableType {
99 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
100 match self {
101 TableType::Base => f.write_str("BASE TABLE"),
102 TableType::Temporary => f.write_str("TEMPORARY"),
103 TableType::View => f.write_str("VIEW"),
104 }
105 }
106}
107
108impl From<TableType> for datafusion::datasource::TableType {
109 fn from(t: TableType) -> datafusion::datasource::TableType {
110 match t {
111 TableType::Base => datafusion::datasource::TableType::Base,
112 TableType::View => datafusion::datasource::TableType::View,
113 TableType::Temporary => datafusion::datasource::TableType::Temporary,
114 }
115 }
116}
117
118#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Default)]
120pub struct TableIdent {
121 pub table_id: TableId,
123 pub version: TableVersion,
126}
127
128#[derive(Clone, Debug, Builder, PartialEq, Eq, ToMetaBuilder, Serialize)]
132#[builder(pattern = "mutable", custom_constructor)]
133pub struct TableMeta {
134 pub schema: SchemaRef,
135 pub primary_key_indices: Vec<usize>,
138 #[builder(default = "self.default_value_indices()?")]
139 pub value_indices: Vec<usize>,
140 #[builder(default, setter(into))]
141 pub engine: String,
142 pub next_column_id: ColumnId,
143 #[builder(default)]
145 pub options: TableOptions,
146 #[builder(default = "Utc::now()")]
147 pub created_on: DateTime<Utc>,
148 #[builder(default = "self.default_updated_on()")]
149 pub updated_on: DateTime<Utc>,
150 #[builder(default = "Vec::new()")]
151 pub partition_key_indices: Vec<usize>,
152 #[builder(default = "Vec::new()")]
153 pub column_ids: Vec<ColumnId>,
154}
155
156impl TableMeta {
157 pub fn empty() -> Self {
158 Self {
159 schema: Arc::new(Schema::new(vec![])),
160 primary_key_indices: vec![],
161 value_indices: vec![],
162 engine: "".to_string(),
163 next_column_id: 0,
164 options: TableOptions::default(),
165 created_on: Utc::now(),
166 updated_on: Utc::now(),
167 partition_key_indices: vec![],
168 column_ids: vec![],
169 }
170 }
171}
172
173impl<'de> Deserialize<'de> for TableMeta {
174 fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
175 where
176 D: Deserializer<'de>,
177 {
178 #[derive(Deserialize)]
179 struct RawTableMeta {
180 schema: SchemaRef,
181 primary_key_indices: Vec<usize>,
182 value_indices: Vec<usize>,
183 engine: String,
184 next_column_id: ColumnId,
185 options: TableOptions,
186 created_on: DateTime<Utc>,
187 updated_on: Option<DateTime<Utc>>,
188 #[serde(default)]
189 partition_key_indices: Vec<usize>,
190 #[serde(default)]
191 column_ids: Vec<ColumnId>,
192 }
193
194 let RawTableMeta {
195 schema,
196 primary_key_indices,
197 value_indices,
198 engine,
199 next_column_id,
200 options,
201 created_on,
202 updated_on,
203 partition_key_indices,
204 column_ids,
205 } = RawTableMeta::deserialize(deserializer)?;
206
207 Ok(Self {
208 schema,
209 primary_key_indices,
210 value_indices,
211 engine,
212 next_column_id,
213 options,
214 created_on,
215 updated_on: updated_on.unwrap_or(created_on),
216 partition_key_indices,
217 column_ids,
218 })
219 }
220}
221
222impl TableMetaBuilder {
223 #[cfg(any(test, feature = "testing"))]
225 pub fn empty() -> Self {
226 Self {
227 schema: None,
228 primary_key_indices: None,
229 value_indices: None,
230 engine: None,
231 next_column_id: None,
232 options: None,
233 created_on: None,
234 updated_on: None,
235 partition_key_indices: None,
236 column_ids: None,
237 }
238 }
239}
240
241impl TableMetaBuilder {
242 fn default_value_indices(&self) -> std::result::Result<Vec<usize>, String> {
243 match (&self.primary_key_indices, &self.schema) {
244 (Some(v), Some(schema)) => {
245 let column_schemas = schema.column_schemas();
246 Ok((0..column_schemas.len())
247 .filter(|idx| !v.contains(idx))
248 .collect())
249 }
250 _ => Err("Missing primary_key_indices or schema to create value_indices".to_string()),
251 }
252 }
253
254 fn default_updated_on(&self) -> DateTime<Utc> {
255 self.created_on.unwrap_or_default()
256 }
257
258 pub fn new_external_table() -> Self {
259 Self {
260 schema: None,
261 primary_key_indices: Some(Vec::new()),
262 value_indices: Some(Vec::new()),
263 engine: None,
264 next_column_id: Some(0),
265 options: None,
266 created_on: None,
267 updated_on: None,
268 partition_key_indices: None,
269 column_ids: None,
270 }
271 }
272}
273
274struct SplitResult<'a> {
276 columns_at_first: Vec<&'a AddColumnRequest>,
278 columns_at_after: HashMap<String, Vec<&'a AddColumnRequest>>,
280 columns_at_last: Vec<&'a AddColumnRequest>,
282 column_names: Vec<String>,
284}
285
286impl TableMeta {
287 pub fn row_key_column_names(&self) -> impl Iterator<Item = &String> {
288 let columns_schemas = &self.schema.column_schemas();
289 self.primary_key_indices
290 .iter()
291 .map(|idx| &columns_schemas[*idx].name)
292 }
293
294 pub fn field_column_names(&self) -> impl Iterator<Item = &String> {
295 let columns_schemas = self.schema.column_schemas();
297 let primary_key_indices = &self.primary_key_indices;
298 columns_schemas
299 .iter()
300 .enumerate()
301 .filter(|(i, cs)| !primary_key_indices.contains(i) && !cs.is_time_index())
302 .map(|(_, cs)| &cs.name)
303 }
304
305 pub fn partition_column_names(&self) -> impl Iterator<Item = &String> {
306 let columns_schemas = &self.schema.column_schemas();
307 self.partition_key_indices
308 .iter()
309 .map(|idx| &columns_schemas[*idx].name)
310 }
311
312 pub fn partition_columns(&self) -> impl Iterator<Item = &ColumnSchema> {
313 self.partition_key_indices
314 .iter()
315 .map(|idx| &self.schema.column_schemas()[*idx])
316 }
317
318 pub fn builder_with_alter_kind(
322 &self,
323 table_name: &str,
324 alter_kind: &AlterKind,
325 ) -> Result<TableMetaBuilder> {
326 let mut builder = match alter_kind {
327 AlterKind::AddColumns { columns } => self.add_columns(table_name, columns),
328 AlterKind::DropColumns { names } => self.remove_columns(table_name, names),
329 AlterKind::ModifyColumnTypes { columns } => {
330 self.modify_column_types(table_name, columns)
331 }
332 AlterKind::RenameTable { .. } => Ok(self.new_meta_builder()),
334 AlterKind::SetTableOptions { options } => self.set_table_options(options),
335 AlterKind::UnsetTableOptions { keys } => self.unset_table_options(keys),
336 AlterKind::SetAnnotations { family, options } => {
337 self.set_annotations(table_name, *family, options)
338 }
339 AlterKind::UnsetAnnotations { family, keys } => {
340 self.unset_annotations(table_name, *family, keys)
341 }
342 AlterKind::SetIndexes { options } => self.set_indexes(table_name, options),
343 AlterKind::UnsetIndexes { options } => self.unset_indexes(table_name, options),
344 AlterKind::DropDefaults { names } => self.drop_defaults(table_name, names),
345 AlterKind::SetDefaults { defaults } => self.set_defaults(table_name, defaults),
346 }?;
347 let _ = builder.updated_on(Utc::now());
348 Ok(builder)
349 }
350
351 fn set_table_options(&self, requests: &[SetRegionOption]) -> Result<TableMetaBuilder> {
353 let mut new_options = self.options.clone();
354
355 for request in requests {
356 match request {
357 SetRegionOption::WriteBufferSize(new_write_buffer_size) => {
358 new_options.write_buffer_size = *new_write_buffer_size;
359 }
360 SetRegionOption::Ttl(new_ttl) => {
361 new_options.ttl = *new_ttl;
362 }
363 SetRegionOption::Twsc(key, value) => {
364 if !value.is_empty() {
365 new_options.extra_options.insert(key.clone(), value.clone());
366 new_options.extra_options.insert(
368 COMPACTION_TYPE.to_string(),
369 COMPACTION_TYPE_TWCS.to_string(),
370 );
371 } else {
372 new_options.extra_options.remove(key.as_str());
374 }
375 }
376 SetRegionOption::Format(value) => {
377 new_options
378 .extra_options
379 .insert(SST_FORMAT_KEY.to_string(), value.clone());
380 }
381 SetRegionOption::AppendMode(value) => {
382 new_options
383 .extra_options
384 .insert(APPEND_MODE_KEY.to_string(), value.to_string());
385 if *value {
386 new_options.extra_options.remove(MERGE_MODE_KEY);
387 }
388 }
389 SetRegionOption::AutoFlushInterval(new_interval) => {
390 if let Some(interval) = new_interval {
391 new_options.extra_options.insert(
392 AUTO_FLUSH_INTERVAL_KEY.to_string(),
393 humantime::format_duration(*interval).to_string(),
394 );
395 } else {
396 new_options.extra_options.remove(AUTO_FLUSH_INTERVAL_KEY);
397 }
398 }
399 SetRegionOption::MaxRowGroupRowCount(row_count) => {
400 if let Some(row_count) = row_count {
401 new_options
402 .extra_options
403 .insert(MAX_ROW_GROUP_ROW_COUNT.to_string(), row_count.to_string());
404 } else {
405 new_options.extra_options.remove(MAX_ROW_GROUP_ROW_COUNT);
406 }
407 }
408 SetRegionOption::SkipWal => {
409 new_options.skip_wal = true;
410 new_options
413 .extra_options
414 .insert(SKIP_WAL_KEY.to_string(), true.to_string());
415 }
416 }
417 }
418 let mut builder = self.new_meta_builder();
419 builder.options(new_options);
420
421 Ok(builder)
422 }
423
424 fn unset_table_options(&self, requests: &[UnsetRegionOption]) -> Result<TableMetaBuilder> {
425 let requests = requests.iter().map(Into::into).collect::<Vec<_>>();
426 self.set_table_options(&requests)
427 }
428
429 fn set_annotations(
434 &self,
435 table_name: &str,
436 family: AnnotationFamily,
437 options: &[(String, String)],
438 ) -> Result<TableMetaBuilder> {
439 ensure!(
440 !family.requires_single_key() || options.len() == 1,
441 error::InvalidAlterRequestSnafu {
442 table: table_name,
443 err: family.mixed_batch_error(),
444 }
445 );
446 let cx = AnnotationContext {
447 schema: &self.schema,
448 partition_key_indices: &self.partition_key_indices,
449 };
450 let mut new_options = self.options.clone();
451 for (key, value) in options {
452 ensure!(
453 AnnotationFamily::of_key(key) == Some(family),
454 error::InvalidAlterRequestSnafu {
455 table: table_name,
456 err: format!(
457 "`{key}` is outside the `{}` annotation namespace",
458 family.namespace()
459 ),
460 }
461 );
462 let checked = validate_and_normalize_annotation(family, &cx, key, value).map_err(
463 |e| match e {
464 AnnotationValidationError::ColumnNotFound { column } => {
465 error::ColumnNotExistsSnafu {
466 column_name: column,
467 table_name,
468 }
469 .build()
470 }
471 other => error::InvalidAlterRequestSnafu {
472 table: table_name,
473 err: other.to_string(),
474 }
475 .build(),
476 },
477 )?;
478 new_options.extra_options.insert(key.clone(), checked);
479 }
480 let mut builder = self.new_meta_builder();
481 builder.options(new_options);
482 Ok(builder)
483 }
484
485 fn unset_annotations(
489 &self,
490 table_name: &str,
491 family: AnnotationFamily,
492 keys: &[String],
493 ) -> Result<TableMetaBuilder> {
494 ensure!(
495 !family.requires_single_key() || keys.len() == 1,
496 error::InvalidAlterRequestSnafu {
497 table: table_name,
498 err: family.mixed_batch_error(),
499 }
500 );
501 let mut new_options = self.options.clone();
502 for key in keys {
503 ensure!(
504 AnnotationFamily::of_key(key) == Some(family),
505 error::InvalidAlterRequestSnafu {
506 table: table_name,
507 err: format!(
508 "`{key}` is outside the `{}` annotation namespace",
509 family.namespace()
510 ),
511 }
512 );
513 new_options.extra_options.remove(key);
514 }
515 let mut builder = self.new_meta_builder();
516 builder.options(new_options);
517 Ok(builder)
518 }
519
520 fn set_indexes(
521 &self,
522 table_name: &str,
523 requests: &[SetIndexOption],
524 ) -> Result<TableMetaBuilder> {
525 let table_schema = &self.schema;
526 let mut set_index_options: HashMap<&str, Vec<_>> = HashMap::new();
527 for request in requests {
528 let column_name = request.column_name();
529 table_schema
530 .column_index_by_name(column_name)
531 .with_context(|| error::ColumnNotExistsSnafu {
532 column_name,
533 table_name,
534 })?;
535 set_index_options
536 .entry(column_name)
537 .or_default()
538 .push(request);
539 }
540
541 let mut meta_builder = self.new_meta_builder();
542 let mut columns: Vec<_> = Vec::with_capacity(table_schema.column_schemas().len());
543 for mut column in table_schema.column_schemas().iter().cloned() {
544 if let Some(request) = set_index_options.get(column.name.as_str()) {
545 for request in request {
546 self.set_index(&mut column, request)?;
547 }
548 }
549 columns.push(column);
550 }
551
552 let mut builder = SchemaBuilder::try_from_columns(columns)
553 .with_context(|_| error::SchemaBuildSnafu {
554 msg: format!("Failed to convert column schemas into schema for table {table_name}"),
555 })?
556 .version(table_schema.version() + 1);
557
558 for (k, v) in table_schema.metadata().iter() {
559 builder = builder.add_metadata(k, v);
560 }
561
562 let new_schema = builder.build().with_context(|_| {
563 let column_names = requests
564 .iter()
565 .map(|request| request.column_name())
566 .collect::<Vec<_>>();
567 error::SchemaBuildSnafu {
568 msg: format!(
569 "Table {table_name} cannot set index options with columns {column_names:?}",
570 ),
571 }
572 })?;
573 let _ = meta_builder
574 .schema(Arc::new(new_schema))
575 .primary_key_indices(self.primary_key_indices.clone());
576
577 Ok(meta_builder)
578 }
579
580 fn unset_indexes(
581 &self,
582 table_name: &str,
583 requests: &[UnsetIndexOption],
584 ) -> Result<TableMetaBuilder> {
585 let table_schema = &self.schema;
586 let mut set_index_options: HashMap<&str, Vec<_>> = HashMap::new();
587 for request in requests {
588 let column_name = request.column_name();
589 table_schema
590 .column_index_by_name(column_name)
591 .with_context(|| error::ColumnNotExistsSnafu {
592 column_name,
593 table_name,
594 })?;
595 set_index_options
596 .entry(column_name)
597 .or_default()
598 .push(request);
599 }
600
601 let mut meta_builder = self.new_meta_builder();
602 let mut columns: Vec<_> = Vec::with_capacity(table_schema.column_schemas().len());
603 for mut column in table_schema.column_schemas().iter().cloned() {
604 if let Some(request) = set_index_options.get(column.name.as_str()) {
605 for request in request {
606 self.unset_index(&mut column, request)?;
607 }
608 }
609 columns.push(column);
610 }
611
612 let mut builder = SchemaBuilder::try_from_columns(columns)
613 .with_context(|_| error::SchemaBuildSnafu {
614 msg: format!("Failed to convert column schemas into schema for table {table_name}"),
615 })?
616 .version(table_schema.version() + 1);
617
618 for (k, v) in table_schema.metadata().iter() {
619 builder = builder.add_metadata(k, v);
620 }
621
622 let new_schema = builder.build().with_context(|_| {
623 let column_names = requests
624 .iter()
625 .map(|request| request.column_name())
626 .collect::<Vec<_>>();
627 error::SchemaBuildSnafu {
628 msg: format!(
629 "Table {table_name} cannot set index options with columns {column_names:?}",
630 ),
631 }
632 })?;
633 let _ = meta_builder
634 .schema(Arc::new(new_schema))
635 .primary_key_indices(self.primary_key_indices.clone());
636
637 Ok(meta_builder)
638 }
639
640 fn set_index(&self, column_schema: &mut ColumnSchema, request: &SetIndexOption) -> Result<()> {
641 match request {
642 SetIndexOption::Fulltext {
643 column_name,
644 options,
645 } => {
646 ensure!(
647 column_schema.data_type.is_string(),
648 error::InvalidColumnOptionSnafu {
649 column_name,
650 msg: "FULLTEXT index only supports string type",
651 }
652 );
653
654 let current_fulltext_options = column_schema
655 .fulltext_options()
656 .context(error::SetFulltextOptionsSnafu { column_name })?;
657 set_column_fulltext_options(
658 column_schema,
659 column_name,
660 options,
661 current_fulltext_options,
662 )?;
663 }
664 SetIndexOption::Inverted { column_name } => {
665 debug_assert_eq!(column_schema.name, *column_name);
666 column_schema.set_inverted_index(true);
667 }
668 SetIndexOption::Skipping {
669 column_name,
670 options,
671 } => {
672 set_column_skipping_index_options(column_schema, column_name, options)?;
673 }
674 }
675
676 Ok(())
677 }
678
679 fn unset_index(
680 &self,
681 column_schema: &mut ColumnSchema,
682 request: &UnsetIndexOption,
683 ) -> Result<()> {
684 match request {
685 UnsetIndexOption::Fulltext { column_name } => {
686 let current_fulltext_options = column_schema
687 .fulltext_options()
688 .context(error::SetFulltextOptionsSnafu { column_name })?;
689 unset_column_fulltext_options(
690 column_schema,
691 column_name,
692 current_fulltext_options.clone(),
693 )?
694 }
695 UnsetIndexOption::Inverted { .. } => {
696 column_schema.set_inverted_index(false);
697 }
698 UnsetIndexOption::Skipping { column_name } => {
699 unset_column_skipping_index_options(column_schema, column_name)?;
700 }
701 }
702
703 Ok(())
704 }
705
706 pub fn alloc_new_column(
711 &mut self,
712 table_name: &str,
713 new_column: &ColumnSchema,
714 ) -> Result<ColumnDescriptor> {
715 let desc = ColumnDescriptorBuilder::new(
716 self.next_column_id as ColumnId,
717 &new_column.name,
718 new_column.data_type.clone(),
719 )
720 .is_nullable(new_column.is_nullable())
721 .default_constraint(new_column.default_constraint().cloned())
722 .build()
723 .context(error::BuildColumnDescriptorSnafu {
724 table_name,
725 column_name: &new_column.name,
726 })?;
727
728 self.next_column_id += 1;
730
731 Ok(desc)
732 }
733
734 fn new_meta_builder(&self) -> TableMetaBuilder {
736 let mut builder = TableMetaBuilder::from(self);
737 builder.value_indices = None;
739 builder
740 }
741
742 fn add_columns(
744 &self,
745 table_name: &str,
746 requests: &[AddColumnRequest],
747 ) -> Result<TableMetaBuilder> {
748 let table_schema = &self.schema;
749 let mut meta_builder = self.new_meta_builder();
750 let original_primary_key_indices: HashSet<&usize> =
751 self.primary_key_indices.iter().collect();
752
753 let mut names = HashSet::with_capacity(requests.len());
754 let mut new_columns = Vec::with_capacity(requests.len());
755 for col_to_add in requests {
756 if let Some(column_schema) =
757 table_schema.column_schema_by_name(&col_to_add.column_schema.name)
758 {
759 ensure!(
761 col_to_add.add_if_not_exists,
762 error::ColumnExistsSnafu {
763 table_name,
764 column_name: &col_to_add.column_schema.name
765 },
766 );
767
768 ensure!(
770 column_schema.data_type == col_to_add.column_schema.data_type,
771 error::InvalidAlterRequestSnafu {
772 table: table_name,
773 err: format!(
774 "column {} already exists with different type {:?}",
775 col_to_add.column_schema.name, column_schema.data_type,
776 ),
777 }
778 );
779 } else {
780 ensure!(
783 names.insert(&col_to_add.column_schema.name),
784 error::InvalidAlterRequestSnafu {
785 table: table_name,
786 err: format!(
787 "add column {} more than once",
788 col_to_add.column_schema.name
789 ),
790 }
791 );
792
793 ensure!(
794 col_to_add.column_schema.is_nullable()
795 || col_to_add.column_schema.default_constraint().is_some(),
796 error::InvalidAlterRequestSnafu {
797 table: table_name,
798 err: format!(
799 "no default value for column {}",
800 col_to_add.column_schema.name
801 ),
802 },
803 );
804
805 if !has_stable_string_form(&col_to_add.column_schema.data_type)
809 && let Some(key) =
810 entity_option_referencing(&self.options, &col_to_add.column_schema.name)
811 {
812 return error::InvalidAlterRequestSnafu {
813 table: table_name,
814 err: format!(
815 "column `{}` is referenced by entity option `{key}` and must \
816 keep a type that renders as a string, got `{}`",
817 col_to_add.column_schema.name, col_to_add.column_schema.data_type
818 ),
819 }
820 .fail();
821 }
822
823 new_columns.push(col_to_add.clone());
824 }
825 }
826 let requests = &new_columns[..];
827
828 let SplitResult {
829 columns_at_first,
830 columns_at_after,
831 columns_at_last,
832 column_names,
833 } = self.split_requests_by_column_location(table_name, requests)?;
834 let mut primary_key_indices = Vec::with_capacity(self.primary_key_indices.len());
835 let mut columns = Vec::with_capacity(table_schema.num_columns() + requests.len());
836 columns_at_first.iter().rev().for_each(|request| {
838 if request.is_key {
839 primary_key_indices.push(columns.len());
841 }
842 columns.push(request.column_schema.clone());
843 });
844 for (index, column_schema) in table_schema.column_schemas().iter().enumerate() {
846 if original_primary_key_indices.contains(&index) {
847 primary_key_indices.push(columns.len());
848 }
849 columns.push(column_schema.clone());
850 if let Some(requests) = columns_at_after.get(&column_schema.name) {
851 requests.iter().rev().for_each(|request| {
852 if request.is_key {
853 primary_key_indices.push(columns.len());
855 }
856 columns.push(request.column_schema.clone());
857 });
858 }
859 }
860 columns_at_last.iter().for_each(|request| {
862 if request.is_key {
863 primary_key_indices.push(columns.len());
865 }
866 columns.push(request.column_schema.clone());
867 });
868
869 let mut builder = SchemaBuilder::try_from(columns)
870 .with_context(|_| error::SchemaBuildSnafu {
871 msg: format!("Failed to convert column schemas into schema for table {table_name}"),
872 })?
873 .version(table_schema.version() + 1);
875 for (k, v) in table_schema.metadata().iter() {
876 builder = builder.add_metadata(k, v);
877 }
878 let new_schema = builder.build().with_context(|_| error::SchemaBuildSnafu {
879 msg: format!("Table {table_name} cannot add new columns {column_names:?}"),
880 })?;
881
882 let partition_key_indices = self
883 .partition_key_indices
884 .iter()
885 .map(|idx| table_schema.column_name_by_index(*idx))
886 .map(|name| new_schema.column_index_by_name(name).unwrap())
888 .collect();
889
890 let _ = meta_builder
892 .schema(Arc::new(new_schema))
893 .primary_key_indices(primary_key_indices)
894 .partition_key_indices(partition_key_indices);
895
896 Ok(meta_builder)
897 }
898
899 fn remove_columns(
900 &self,
901 table_name: &str,
902 column_names: &[String],
903 ) -> Result<TableMetaBuilder> {
904 let table_schema = &self.schema;
905 let column_names: HashSet<_> = column_names.iter().collect();
906 let mut meta_builder = self.new_meta_builder();
907
908 let timestamp_index = table_schema.timestamp_index();
909 for column_name in &column_names {
911 if let Some(index) = table_schema.column_index_by_name(column_name) {
912 ensure!(
915 !self.primary_key_indices.contains(&index),
916 error::RemoveColumnInIndexSnafu {
917 column_name: *column_name,
918 table_name,
919 }
920 );
921
922 ensure!(
923 !self.partition_key_indices.contains(&index),
924 error::RemovePartitionColumnSnafu {
925 column_name: *column_name,
926 table_name,
927 }
928 );
929
930 if let Some(ts_index) = timestamp_index {
931 ensure!(
933 index != ts_index,
934 error::RemoveColumnInIndexSnafu {
935 column_name: table_schema.column_name_by_index(ts_index),
936 table_name,
937 }
938 );
939 }
940 } else {
941 return error::ColumnNotExistsSnafu {
942 column_name: *column_name,
943 table_name,
944 }
945 .fail()?;
946 }
947 }
948
949 let columns: Vec<_> = table_schema
951 .column_schemas()
952 .iter()
953 .filter(|column_schema| !column_names.contains(&column_schema.name))
954 .cloned()
955 .collect();
956
957 let mut builder = SchemaBuilder::try_from_columns(columns)
958 .with_context(|_| error::SchemaBuildSnafu {
959 msg: format!("Failed to convert column schemas into schema for table {table_name}"),
960 })?
961 .version(table_schema.version() + 1);
963 for (k, v) in table_schema.metadata().iter() {
964 builder = builder.add_metadata(k, v);
965 }
966 let new_schema = builder.build().with_context(|_| error::SchemaBuildSnafu {
967 msg: format!("Table {table_name} cannot add remove columns {column_names:?}"),
968 })?;
969
970 let primary_key_indices = self
972 .primary_key_indices
973 .iter()
974 .map(|idx| table_schema.column_name_by_index(*idx))
975 .map(|name| new_schema.column_index_by_name(name).unwrap())
977 .collect();
978
979 let partition_key_indices = self
980 .partition_key_indices
981 .iter()
982 .map(|idx| table_schema.column_name_by_index(*idx))
983 .map(|name| new_schema.column_index_by_name(name).unwrap())
985 .collect();
986
987 let _ = meta_builder
988 .schema(Arc::new(new_schema))
989 .primary_key_indices(primary_key_indices)
990 .partition_key_indices(partition_key_indices);
991
992 Ok(meta_builder)
993 }
994
995 fn modify_column_types(
996 &self,
997 table_name: &str,
998 requests: &[ModifyColumnTypeRequest],
999 ) -> Result<TableMetaBuilder> {
1000 let table_schema = &self.schema;
1001 let mut meta_builder = self.new_meta_builder();
1002
1003 let mut modify_column_types = HashMap::with_capacity(requests.len());
1004 let timestamp_index = table_schema.timestamp_index();
1005
1006 for col_to_change in requests {
1007 let change_column_name = &col_to_change.column_name;
1008
1009 let index = table_schema
1010 .column_index_by_name(change_column_name)
1011 .with_context(|| error::ColumnNotExistsSnafu {
1012 column_name: change_column_name,
1013 table_name,
1014 })?;
1015
1016 if !has_stable_string_form(&col_to_change.target_type)
1022 && let Some(key) = entity_option_referencing(&self.options, change_column_name)
1023 {
1024 return error::InvalidAlterRequestSnafu {
1025 table: table_name,
1026 err: format!(
1027 "column `{change_column_name}` is referenced by entity option \
1028 `{key}` and must keep a type that renders as a string, got `{}`",
1029 col_to_change.target_type
1030 ),
1031 }
1032 .fail();
1033 }
1034
1035 let column = &table_schema.column_schemas()[index];
1036
1037 ensure!(
1038 !self.primary_key_indices.contains(&index),
1039 error::InvalidAlterRequestSnafu {
1040 table: table_name,
1041 err: format!(
1042 "Not allowed to change primary key index column '{}'",
1043 column.name
1044 )
1045 }
1046 );
1047
1048 if let Some(ts_index) = timestamp_index {
1049 ensure!(
1051 index != ts_index,
1052 error::InvalidAlterRequestSnafu {
1053 table: table_name,
1054 err: format!(
1055 "Not allowed to change timestamp index column '{}' datatype",
1056 column.name
1057 )
1058 }
1059 );
1060 }
1061
1062 ensure!(
1063 modify_column_types
1064 .insert(&col_to_change.column_name, col_to_change)
1065 .is_none(),
1066 error::InvalidAlterRequestSnafu {
1067 table: table_name,
1068 err: format!(
1069 "change column datatype {} more than once",
1070 col_to_change.column_name
1071 ),
1072 }
1073 );
1074
1075 ensure!(
1076 column
1077 .data_type
1078 .can_arrow_type_cast_to(&col_to_change.target_type),
1079 error::InvalidAlterRequestSnafu {
1080 table: table_name,
1081 err: format!(
1082 "column '{}' cannot be cast automatically to type '{}'",
1083 col_to_change.column_name, col_to_change.target_type,
1084 ),
1085 }
1086 );
1087
1088 ensure!(
1089 column.is_nullable(),
1090 error::InvalidAlterRequestSnafu {
1091 table: table_name,
1092 err: format!(
1093 "column '{}' must be nullable to ensure safe conversion.",
1094 col_to_change.column_name,
1095 ),
1096 }
1097 );
1098 }
1099 let mut columns: Vec<_> = Vec::with_capacity(table_schema.column_schemas().len());
1102 for mut column in table_schema.column_schemas().iter().cloned() {
1103 if let Some(change_column) = modify_column_types.get(&column.name) {
1104 column.data_type = change_column.target_type.clone();
1105 let new_default = if let Some(default_value) = column.default_constraint() {
1106 Some(
1107 default_value
1108 .cast_to_datatype(&change_column.target_type)
1109 .with_context(|_| error::CastDefaultValueSnafu {
1110 reason: format!(
1111 "Failed to cast default value from {:?} to type {:?}",
1112 default_value, &change_column.target_type
1113 ),
1114 })?,
1115 )
1116 } else {
1117 None
1118 };
1119 column = column
1120 .clone()
1121 .with_default_constraint(new_default.clone())
1122 .with_context(|_| error::CastDefaultValueSnafu {
1123 reason: format!("Failed to set new default: {:?}", new_default),
1124 })?;
1125 }
1126 columns.push(column)
1127 }
1128
1129 let mut builder = SchemaBuilder::try_from_columns(columns)
1130 .with_context(|_| error::SchemaBuildSnafu {
1131 msg: format!("Failed to convert column schemas into schema for table {table_name}"),
1132 })?
1133 .version(table_schema.version() + 1);
1135 for (k, v) in table_schema.metadata().iter() {
1136 builder = builder.add_metadata(k, v);
1137 }
1138 let new_schema = builder.build().with_context(|_| {
1139 let column_names: Vec<_> = requests
1140 .iter()
1141 .map(|request| &request.column_name)
1142 .collect();
1143
1144 error::SchemaBuildSnafu {
1145 msg: format!(
1146 "Table {table_name} cannot change datatype with columns {column_names:?}"
1147 ),
1148 }
1149 })?;
1150
1151 let _ = meta_builder
1152 .schema(Arc::new(new_schema))
1153 .primary_key_indices(self.primary_key_indices.clone());
1154
1155 Ok(meta_builder)
1156 }
1157
1158 fn split_requests_by_column_location<'a>(
1160 &self,
1161 table_name: &str,
1162 requests: &'a [AddColumnRequest],
1163 ) -> Result<SplitResult<'a>> {
1164 let table_schema = &self.schema;
1165 let mut columns_at_first = Vec::new();
1166 let mut columns_at_after = HashMap::new();
1167 let mut columns_at_last = Vec::new();
1168 let mut column_names = Vec::with_capacity(requests.len());
1169 for request in requests {
1170 let column_name = &request.column_schema.name;
1172 column_names.push(column_name.clone());
1173 ensure!(
1174 table_schema.column_schema_by_name(column_name).is_none(),
1175 error::ColumnExistsSnafu {
1176 column_name,
1177 table_name,
1178 }
1179 );
1180 match request.location.as_ref() {
1181 Some(AddColumnLocation::First) => {
1182 columns_at_first.push(request);
1183 }
1184 Some(AddColumnLocation::After { column_name }) => {
1185 ensure!(
1186 table_schema.column_schema_by_name(column_name).is_some(),
1187 error::ColumnNotExistsSnafu {
1188 column_name,
1189 table_name,
1190 }
1191 );
1192 columns_at_after
1193 .entry(column_name.clone())
1194 .or_insert(Vec::new())
1195 .push(request);
1196 }
1197 None => {
1198 columns_at_last.push(request);
1199 }
1200 }
1201 }
1202 Ok(SplitResult {
1203 columns_at_first,
1204 columns_at_after,
1205 columns_at_last,
1206 column_names,
1207 })
1208 }
1209
1210 fn drop_defaults(&self, table_name: &str, column_names: &[String]) -> Result<TableMetaBuilder> {
1211 let table_schema = &self.schema;
1212 let mut meta_builder = self.new_meta_builder();
1213 let mut columns = Vec::with_capacity(table_schema.num_columns());
1214 for column_schema in table_schema.column_schemas() {
1215 if let Some(name) = column_names.iter().find(|s| **s == column_schema.name) {
1216 ensure!(
1218 column_schema.default_constraint().is_some(),
1219 error::InvalidAlterRequestSnafu {
1220 table: table_name,
1221 err: format!("column {name} does not have a default value"),
1222 }
1223 );
1224 if !column_schema.is_nullable() {
1225 return error::InvalidAlterRequestSnafu {
1226 table: table_name,
1227 err: format!(
1228 "column {name} is not nullable and `default` cannot be dropped",
1229 ),
1230 }
1231 .fail();
1232 }
1233 let new_column_schema = column_schema.clone();
1234 let new_column_schema = new_column_schema
1235 .with_default_constraint(None)
1236 .with_context(|_| error::SchemaBuildSnafu {
1237 msg: format!("Table {table_name} cannot drop default values"),
1238 })?;
1239 columns.push(new_column_schema);
1240 } else {
1241 columns.push(column_schema.clone());
1242 }
1243 }
1244
1245 let mut builder = SchemaBuilder::try_from_columns(columns)
1246 .with_context(|_| error::SchemaBuildSnafu {
1247 msg: format!("Failed to convert column schemas into schema for table {table_name}"),
1248 })?
1249 .version(table_schema.version() + 1);
1251 for (k, v) in table_schema.metadata().iter() {
1252 builder = builder.add_metadata(k, v);
1253 }
1254 let new_schema = builder.build().with_context(|_| error::SchemaBuildSnafu {
1255 msg: format!("Table {table_name} cannot drop default values"),
1256 })?;
1257
1258 let _ = meta_builder.schema(Arc::new(new_schema));
1259
1260 Ok(meta_builder)
1261 }
1262
1263 fn set_defaults(
1264 &self,
1265 table_name: &str,
1266 set_defaults: &[SetDefaultRequest],
1267 ) -> Result<TableMetaBuilder> {
1268 let table_schema = &self.schema;
1269 let mut meta_builder = self.new_meta_builder();
1270 let mut columns = Vec::with_capacity(table_schema.num_columns());
1271 for column_schema in table_schema.column_schemas() {
1272 if let Some(set_default) = set_defaults
1273 .iter()
1274 .find(|s| s.column_name == column_schema.name)
1275 {
1276 let new_column_schema = column_schema.clone();
1277 let new_column_schema = new_column_schema
1278 .with_default_constraint(set_default.default_constraint.clone())
1279 .with_context(|_| error::SchemaBuildSnafu {
1280 msg: format!("Table {table_name} cannot set default values"),
1281 })?;
1282 columns.push(new_column_schema);
1283 } else {
1284 columns.push(column_schema.clone());
1285 }
1286 }
1287
1288 let mut builder = SchemaBuilder::try_from_columns(columns)
1289 .with_context(|_| error::SchemaBuildSnafu {
1290 msg: format!("Failed to convert column schemas into schema for table {table_name}"),
1291 })?
1292 .version(table_schema.version() + 1);
1294 for (k, v) in table_schema.metadata().iter() {
1295 builder = builder.add_metadata(k, v);
1296 }
1297 let new_schema = builder.build().with_context(|_| error::SchemaBuildSnafu {
1298 msg: format!("Table {table_name} cannot set default values"),
1299 })?;
1300
1301 let _ = meta_builder.schema(Arc::new(new_schema));
1302
1303 Ok(meta_builder)
1304 }
1305}
1306
1307#[derive(Clone, Debug, PartialEq, Eq, Builder, Serialize, Deserialize)]
1308#[builder(pattern = "owned")]
1309pub struct TableInfo {
1310 #[builder(default, setter(into))]
1312 pub ident: TableIdent,
1313 #[builder(setter(into))]
1315 pub name: String,
1316 #[builder(default, setter(into))]
1318 pub desc: Option<String>,
1319 #[builder(default = "DEFAULT_CATALOG_NAME.to_string()", setter(into))]
1320 pub catalog_name: String,
1321 #[builder(default = "DEFAULT_SCHEMA_NAME.to_string()", setter(into))]
1322 pub schema_name: String,
1323 pub meta: TableMeta,
1324 #[builder(default = "TableType::Base")]
1325 pub table_type: TableType,
1326}
1327
1328pub type TableInfoRef = Arc<TableInfo>;
1329
1330impl TableInfo {
1331 pub fn table_id(&self) -> TableId {
1332 self.ident.table_id
1333 }
1334
1335 pub fn full_table_name(&self) -> String {
1337 common_catalog::format_full_table_name(&self.catalog_name, &self.schema_name, &self.name)
1338 }
1339
1340 pub fn get_db_string(&self) -> String {
1341 common_catalog::build_db_string(&self.catalog_name, &self.schema_name)
1342 }
1343
1344 pub fn is_physical_table(&self) -> bool {
1346 self.meta
1347 .options
1348 .extra_options
1349 .contains_key(PHYSICAL_TABLE_METADATA_KEY)
1350 }
1351
1352 pub fn is_ttl_instant_table(&self) -> bool {
1354 self.meta
1355 .options
1356 .ttl
1357 .map(|t| t.is_instant())
1358 .unwrap_or(false)
1359 }
1360}
1361
1362impl TableInfoBuilder {
1363 pub fn new<S: Into<String>>(name: S, meta: TableMeta) -> Self {
1364 Self {
1365 name: Some(name.into()),
1366 meta: Some(meta),
1367 ..Default::default()
1368 }
1369 }
1370
1371 pub fn table_id(mut self, id: TableId) -> Self {
1372 let ident = self.ident.get_or_insert_with(TableIdent::default);
1373 ident.table_id = id;
1374 self
1375 }
1376
1377 pub fn table_version(mut self, version: TableVersion) -> Self {
1378 let ident = self.ident.get_or_insert_with(TableIdent::default);
1379 ident.version = version;
1380 self
1381 }
1382}
1383
1384impl TableIdent {
1385 pub fn new(table_id: TableId) -> Self {
1386 Self {
1387 table_id,
1388 version: 0,
1389 }
1390 }
1391}
1392
1393impl From<TableId> for TableIdent {
1394 fn from(table_id: TableId) -> Self {
1395 Self::new(table_id)
1396 }
1397}
1398
1399impl TableInfo {
1400 pub fn name_to_ids(&self) -> Option<HashMap<String, ColumnId>> {
1404 let column_schemas = self.meta.schema.column_schemas();
1405 if self.meta.column_ids.len() != column_schemas.len() {
1406 None
1407 } else {
1408 Some(
1409 self.meta
1410 .column_ids
1411 .iter()
1412 .enumerate()
1413 .map(|(index, id)| (column_schemas[index].name.clone(), *id))
1414 .collect(),
1415 )
1416 }
1417 }
1418
1419 pub fn sort_columns(&mut self) {
1421 let column_schemas = self.meta.schema.column_schemas();
1422 let primary_keys = self
1423 .meta
1424 .primary_key_indices
1425 .iter()
1426 .map(|index| column_schemas[*index].name.clone())
1427 .collect::<HashSet<_>>();
1428
1429 let name_to_ids = self.name_to_ids().unwrap_or_default();
1430 let mut column_schemas = column_schemas.to_vec();
1431 column_schemas.sort_unstable_by(|a, b| a.name.cmp(&b.name));
1432
1433 let mut primary_key_indices = Vec::with_capacity(primary_keys.len());
1435 let mut value_indices = Vec::with_capacity(column_schemas.len() - primary_keys.len());
1436 let mut column_ids = Vec::with_capacity(column_schemas.len());
1437 for (index, column_schema) in column_schemas.iter().enumerate() {
1438 if primary_keys.contains(&column_schema.name) {
1439 primary_key_indices.push(index);
1440 } else {
1441 value_indices.push(index);
1442 }
1443 if let Some(id) = name_to_ids.get(&column_schema.name) {
1444 column_ids.push(*id);
1445 }
1446 }
1447
1448 self.meta.schema = Arc::new(Schema::new_with_version(
1450 column_schemas,
1451 self.meta.schema.version(),
1452 ));
1453 self.meta.primary_key_indices = primary_key_indices;
1454 self.meta.value_indices = value_indices;
1455 self.meta.column_ids = column_ids;
1456 }
1457
1458 pub fn to_region_options(&self) -> HashMap<String, String> {
1462 let mut options = HashMap::from(&self.meta.options);
1463 options.remove(REPARTITION_COLUMN_HINT_KEY);
1464 options
1465 }
1466
1467 pub fn table_ref(&self) -> TableReference<'_> {
1469 TableReference::full(
1470 self.catalog_name.as_str(),
1471 self.schema_name.as_str(),
1472 self.name.as_str(),
1473 )
1474 }
1475}
1476
1477fn entity_option_referencing<'a>(options: &'a TableOptions, column: &str) -> Option<&'a str> {
1478 options.extra_options.iter().find_map(|(key, value)| {
1479 (parse_entity_option_key(key).is_some()
1480 && parse_entity_columns(value).iter().any(|c| c == column))
1481 .then_some(key.as_str())
1482 })
1483}
1484
1485fn set_column_fulltext_options(
1494 column_schema: &mut ColumnSchema,
1495 column_name: &str,
1496 options: &FulltextOptions,
1497 current_options: Option<FulltextOptions>,
1498) -> Result<()> {
1499 if let Some(current_options) = current_options {
1500 ensure!(
1501 current_options.analyzer == options.analyzer
1502 && current_options.case_sensitive == options.case_sensitive,
1503 error::InvalidColumnOptionSnafu {
1504 column_name,
1505 msg: format!(
1506 "Cannot change analyzer or case_sensitive if FULLTEXT index is set before. Previous analyzer: {}, previous case_sensitive: {}",
1507 current_options.analyzer, current_options.case_sensitive
1508 ),
1509 }
1510 );
1511 }
1512
1513 column_schema
1514 .set_fulltext_options(options)
1515 .context(error::SetFulltextOptionsSnafu { column_name })?;
1516
1517 Ok(())
1518}
1519
1520fn unset_column_fulltext_options(
1521 column_schema: &mut ColumnSchema,
1522 column_name: &str,
1523 current_options: Option<FulltextOptions>,
1524) -> Result<()> {
1525 ensure!(
1526 current_options
1527 .as_ref()
1528 .is_some_and(|options| options.enable),
1529 error::InvalidColumnOptionSnafu {
1530 column_name,
1531 msg: "FULLTEXT index already disabled".to_string(),
1532 }
1533 );
1534
1535 let mut options = current_options.unwrap();
1536 options.enable = false;
1537 column_schema
1538 .set_fulltext_options(&options)
1539 .context(error::SetFulltextOptionsSnafu { column_name })?;
1540
1541 Ok(())
1542}
1543
1544fn set_column_skipping_index_options(
1545 column_schema: &mut ColumnSchema,
1546 column_name: &str,
1547 options: &SkippingIndexOptions,
1548) -> Result<()> {
1549 column_schema
1550 .set_skipping_options(options)
1551 .context(error::SetSkippingOptionsSnafu { column_name })?;
1552
1553 Ok(())
1554}
1555
1556fn unset_column_skipping_index_options(
1557 column_schema: &mut ColumnSchema,
1558 column_name: &str,
1559) -> Result<()> {
1560 column_schema
1561 .unset_skipping_options()
1562 .context(error::UnsetSkippingOptionsSnafu { column_name })?;
1563 Ok(())
1564}
1565
1566#[cfg(test)]
1567mod tests {
1568 use std::assert_matches;
1569
1570 use common_error::ext::ErrorExt;
1571 use common_error::status_code::StatusCode;
1572 use datatypes::data_type::ConcreteDataType;
1573 use datatypes::schema::{
1574 ColumnSchema, FulltextAnalyzer, FulltextBackend, Schema, SchemaBuilder,
1575 };
1576
1577 use super::*;
1578 use crate::Error;
1579
1580 fn new_test_schema() -> Schema {
1582 let column_schemas = vec![
1583 ColumnSchema::new("col1", ConcreteDataType::int32_datatype(), true),
1584 ColumnSchema::new(
1585 "ts",
1586 ConcreteDataType::timestamp_millisecond_datatype(),
1587 false,
1588 )
1589 .with_time_index(true),
1590 ColumnSchema::new("col2", ConcreteDataType::int32_datatype(), true),
1591 ];
1592 SchemaBuilder::try_from(column_schemas)
1593 .unwrap()
1594 .version(123)
1595 .build()
1596 .unwrap()
1597 }
1598
1599 fn add_columns_to_meta(meta: &TableMeta) -> TableMeta {
1600 let new_tag = ColumnSchema::new("my_tag", ConcreteDataType::string_datatype(), true);
1601 let new_field = ColumnSchema::new("my_field", ConcreteDataType::string_datatype(), true);
1602 let alter_kind = AlterKind::AddColumns {
1603 columns: vec![
1604 AddColumnRequest {
1605 column_schema: new_tag,
1606 is_key: true,
1607 location: None,
1608 add_if_not_exists: false,
1609 },
1610 AddColumnRequest {
1611 column_schema: new_field,
1612 is_key: false,
1613 location: None,
1614 add_if_not_exists: false,
1615 },
1616 ],
1617 };
1618
1619 let builder = meta
1620 .builder_with_alter_kind("my_table", &alter_kind)
1621 .unwrap();
1622 builder.build().unwrap()
1623 }
1624
1625 fn add_columns_to_meta_with_location(meta: &TableMeta) -> TableMeta {
1626 let new_tag = ColumnSchema::new("my_tag_first", ConcreteDataType::string_datatype(), true);
1627 let new_field = ColumnSchema::new(
1628 "my_field_after_ts",
1629 ConcreteDataType::string_datatype(),
1630 true,
1631 );
1632 let yet_another_field = ColumnSchema::new(
1633 "yet_another_field_after_ts",
1634 ConcreteDataType::int64_datatype(),
1635 true,
1636 );
1637 let alter_kind = AlterKind::AddColumns {
1638 columns: vec![
1639 AddColumnRequest {
1640 column_schema: new_tag,
1641 is_key: true,
1642 location: Some(AddColumnLocation::First),
1643 add_if_not_exists: false,
1644 },
1645 AddColumnRequest {
1646 column_schema: new_field,
1647 is_key: false,
1648 location: Some(AddColumnLocation::After {
1649 column_name: "ts".to_string(),
1650 }),
1651 add_if_not_exists: false,
1652 },
1653 AddColumnRequest {
1654 column_schema: yet_another_field,
1655 is_key: true,
1656 location: Some(AddColumnLocation::After {
1657 column_name: "ts".to_string(),
1658 }),
1659 add_if_not_exists: false,
1660 },
1661 ],
1662 };
1663
1664 let builder = meta
1665 .builder_with_alter_kind("my_table", &alter_kind)
1666 .unwrap();
1667 builder.build().unwrap()
1668 }
1669
1670 #[test]
1671 fn test_add_columns() {
1672 let schema = Arc::new(new_test_schema());
1673 let meta = TableMetaBuilder::empty()
1674 .schema(schema)
1675 .primary_key_indices(vec![0])
1676 .engine("engine")
1677 .next_column_id(3)
1678 .build()
1679 .unwrap();
1680
1681 let new_meta = add_columns_to_meta(&meta);
1682 let names: Vec<String> = new_meta
1683 .schema
1684 .column_schemas()
1685 .iter()
1686 .map(|column_schema| column_schema.name.clone())
1687 .collect();
1688 assert_eq!(&["col1", "ts", "col2", "my_tag", "my_field"], &names[..]);
1689 assert_eq!(&[0, 3], &new_meta.primary_key_indices[..]);
1690 assert_eq!(&[1, 2, 4], &new_meta.value_indices[..]);
1691 }
1692
1693 #[test]
1694 fn test_set_append_mode_true_clears_merge_mode_option() {
1695 let schema = Arc::new(new_test_schema());
1696 let mut table_options = TableOptions::default();
1697 table_options
1698 .extra_options
1699 .insert(MERGE_MODE_KEY.to_string(), "last_non_null".to_string());
1700 let meta = TableMetaBuilder::empty()
1701 .schema(schema)
1702 .primary_key_indices(vec![0])
1703 .engine("engine")
1704 .next_column_id(3)
1705 .options(table_options)
1706 .build()
1707 .unwrap();
1708
1709 let alter_kind = AlterKind::SetTableOptions {
1710 options: vec![SetRegionOption::AppendMode(true)],
1711 };
1712 let new_meta = meta
1713 .builder_with_alter_kind("my_table", &alter_kind)
1714 .unwrap()
1715 .build()
1716 .unwrap();
1717
1718 assert_eq!(
1719 Some("true"),
1720 new_meta
1721 .options
1722 .extra_options
1723 .get(APPEND_MODE_KEY)
1724 .map(String::as_str)
1725 );
1726 assert!(!new_meta.options.extra_options.contains_key(MERGE_MODE_KEY));
1727 }
1728
1729 #[test]
1730 fn test_set_append_mode_false_keeps_merge_mode_option() {
1731 let schema = Arc::new(new_test_schema());
1732 let mut table_options = TableOptions::default();
1733 table_options
1734 .extra_options
1735 .insert(MERGE_MODE_KEY.to_string(), "last_non_null".to_string());
1736 let meta = TableMetaBuilder::empty()
1737 .schema(schema)
1738 .primary_key_indices(vec![0])
1739 .engine("engine")
1740 .next_column_id(3)
1741 .options(table_options)
1742 .build()
1743 .unwrap();
1744
1745 let alter_kind = AlterKind::SetTableOptions {
1746 options: vec![SetRegionOption::AppendMode(false)],
1747 };
1748 let new_meta = meta
1749 .builder_with_alter_kind("my_table", &alter_kind)
1750 .unwrap()
1751 .build()
1752 .unwrap();
1753
1754 assert_eq!(
1755 Some("false"),
1756 new_meta
1757 .options
1758 .extra_options
1759 .get(APPEND_MODE_KEY)
1760 .map(String::as_str)
1761 );
1762 assert_eq!(
1763 Some("last_non_null"),
1764 new_meta
1765 .options
1766 .extra_options
1767 .get(MERGE_MODE_KEY)
1768 .map(String::as_str)
1769 );
1770 }
1771
1772 #[test]
1773 fn test_set_skip_wal_updates_typed_and_extra_options() {
1774 let mut meta = TableMetaBuilder::empty()
1775 .schema(Arc::new(new_test_schema()))
1776 .primary_key_indices(vec![0])
1777 .engine("engine")
1778 .next_column_id(3)
1779 .build()
1780 .unwrap();
1781 meta.options
1782 .extra_options
1783 .insert(SKIP_WAL_KEY.to_string(), false.to_string());
1784
1785 let alter_kind = AlterKind::SetTableOptions {
1786 options: vec![SetRegionOption::SkipWal],
1787 };
1788 let new_meta = meta
1789 .builder_with_alter_kind("my_table", &alter_kind)
1790 .unwrap()
1791 .build()
1792 .unwrap();
1793
1794 assert!(new_meta.options.skip_wal);
1795 assert_eq!(
1796 Some("true"),
1797 new_meta
1798 .options
1799 .extra_options
1800 .get(SKIP_WAL_KEY)
1801 .map(String::as_str)
1802 );
1803 }
1804
1805 #[test]
1806 fn test_set_repartition_column_hint() {
1807 let meta = TableMetaBuilder::empty()
1808 .schema(Arc::new(new_test_schema()))
1809 .primary_key_indices(vec![0])
1810 .engine("engine")
1811 .next_column_id(3)
1812 .build()
1813 .unwrap();
1814
1815 let alter_kind = AlterKind::SetAnnotations {
1816 family: AnnotationFamily::RepartitionHint,
1817 options: vec![(
1818 REPARTITION_COLUMN_HINT_KEY.to_string(),
1819 " col1 ".to_string(),
1820 )],
1821 };
1822 let new_meta = meta
1823 .builder_with_alter_kind("my_table", &alter_kind)
1824 .unwrap()
1825 .build()
1826 .unwrap();
1827
1828 assert_eq!(
1829 Some("col1"),
1830 new_meta
1831 .options
1832 .extra_options
1833 .get(REPARTITION_COLUMN_HINT_KEY)
1834 .map(String::as_str)
1835 );
1836 }
1837
1838 #[test]
1839 fn test_set_repartition_column_hint_rejects_empty_column() {
1840 let meta = TableMetaBuilder::empty()
1841 .schema(Arc::new(new_test_schema()))
1842 .primary_key_indices(vec![0])
1843 .engine("engine")
1844 .next_column_id(3)
1845 .build()
1846 .unwrap();
1847
1848 let alter_kind = AlterKind::SetAnnotations {
1849 family: AnnotationFamily::RepartitionHint,
1850 options: vec![(REPARTITION_COLUMN_HINT_KEY.to_string(), " ".to_string())],
1851 };
1852 let err = meta
1853 .builder_with_alter_kind("my_table", &alter_kind)
1854 .err()
1855 .unwrap();
1856
1857 assert!(
1858 err.to_string()
1859 .contains("repartition.column.hint expects exactly one column name")
1860 );
1861 }
1862
1863 #[test]
1864 fn test_set_repartition_column_hint_rejects_multiple_columns() {
1865 let meta = TableMetaBuilder::empty()
1866 .schema(Arc::new(new_test_schema()))
1867 .primary_key_indices(vec![0])
1868 .engine("engine")
1869 .next_column_id(3)
1870 .build()
1871 .unwrap();
1872
1873 let alter_kind = AlterKind::SetAnnotations {
1874 family: AnnotationFamily::RepartitionHint,
1875 options: vec![(
1876 REPARTITION_COLUMN_HINT_KEY.to_string(),
1877 "col1,col2".to_string(),
1878 )],
1879 };
1880 let err = meta
1881 .builder_with_alter_kind("my_table", &alter_kind)
1882 .err()
1883 .unwrap();
1884
1885 assert!(
1886 err.to_string()
1887 .contains("repartition.column.hint expects exactly one column name")
1888 );
1889 }
1890
1891 #[test]
1892 fn test_set_repartition_column_hint_rejects_missing_column() {
1893 let meta = TableMetaBuilder::empty()
1894 .schema(Arc::new(new_test_schema()))
1895 .primary_key_indices(vec![0])
1896 .engine("engine")
1897 .next_column_id(3)
1898 .build()
1899 .unwrap();
1900
1901 let alter_kind = AlterKind::SetAnnotations {
1902 family: AnnotationFamily::RepartitionHint,
1903 options: vec![(
1904 REPARTITION_COLUMN_HINT_KEY.to_string(),
1905 "missing".to_string(),
1906 )],
1907 };
1908 let err = meta
1909 .builder_with_alter_kind("my_table", &alter_kind)
1910 .err()
1911 .unwrap();
1912
1913 assert!(err.to_string().contains("Column missing not exists"));
1914 }
1915
1916 #[test]
1917 fn test_set_repartition_column_hint_rejects_time_index_column() {
1918 let meta = TableMetaBuilder::empty()
1919 .schema(Arc::new(new_test_schema()))
1920 .primary_key_indices(vec![0])
1921 .engine("engine")
1922 .next_column_id(3)
1923 .build()
1924 .unwrap();
1925
1926 let alter_kind = AlterKind::SetAnnotations {
1927 family: AnnotationFamily::RepartitionHint,
1928 options: vec![(REPARTITION_COLUMN_HINT_KEY.to_string(), "ts".to_string())],
1929 };
1930 let err = meta
1931 .builder_with_alter_kind("my_table", &alter_kind)
1932 .err()
1933 .unwrap();
1934
1935 assert!(
1936 err.to_string()
1937 .contains("cannot set repartition.column.hint to the time index column")
1938 );
1939 }
1940
1941 #[test]
1942 fn test_set_repartition_column_hint_rejects_partitioned_table() {
1943 let meta = TableMetaBuilder::empty()
1944 .schema(Arc::new(new_test_schema()))
1945 .primary_key_indices(vec![0])
1946 .engine("engine")
1947 .next_column_id(3)
1948 .partition_key_indices(vec![0])
1949 .build()
1950 .unwrap();
1951
1952 let alter_kind = AlterKind::SetAnnotations {
1953 family: AnnotationFamily::RepartitionHint,
1954 options: vec![(REPARTITION_COLUMN_HINT_KEY.to_string(), "col1".to_string())],
1955 };
1956 let err = meta
1957 .builder_with_alter_kind("my_table", &alter_kind)
1958 .err()
1959 .unwrap();
1960
1961 assert!(
1962 err.to_string()
1963 .contains("cannot set repartition.column.hint on a table with partition metadata")
1964 );
1965 }
1966
1967 #[test]
1968 fn test_unset_repartition_column_hint() {
1969 let mut table_options = TableOptions::default();
1970 table_options
1971 .extra_options
1972 .insert(REPARTITION_COLUMN_HINT_KEY.to_string(), "col1".to_string());
1973 let meta = TableMetaBuilder::empty()
1974 .schema(Arc::new(new_test_schema()))
1975 .primary_key_indices(vec![0])
1976 .engine("engine")
1977 .next_column_id(3)
1978 .options(table_options)
1979 .build()
1980 .unwrap();
1981
1982 let new_meta = meta
1983 .builder_with_alter_kind(
1984 "my_table",
1985 &AlterKind::UnsetAnnotations {
1986 family: AnnotationFamily::RepartitionHint,
1987 keys: vec![REPARTITION_COLUMN_HINT_KEY.to_string()],
1988 },
1989 )
1990 .unwrap()
1991 .build()
1992 .unwrap();
1993
1994 assert!(
1995 !new_meta
1996 .options
1997 .extra_options
1998 .contains_key(REPARTITION_COLUMN_HINT_KEY)
1999 );
2000 }
2001
2002 #[test]
2003 fn test_repartition_column_hint_is_not_region_option() {
2004 let mut table_options = TableOptions::default();
2005 table_options
2006 .extra_options
2007 .insert(REPARTITION_COLUMN_HINT_KEY.to_string(), "col1".to_string());
2008 let table_info = TableInfoBuilder::default()
2009 .table_id(1)
2010 .table_version(0)
2011 .name("my_table")
2012 .catalog_name(DEFAULT_CATALOG_NAME)
2013 .schema_name(DEFAULT_SCHEMA_NAME)
2014 .meta(
2015 TableMetaBuilder::empty()
2016 .schema(Arc::new(new_test_schema()))
2017 .primary_key_indices(vec![0])
2018 .engine("engine")
2019 .next_column_id(3)
2020 .options(table_options)
2021 .build()
2022 .unwrap(),
2023 )
2024 .build()
2025 .unwrap();
2026
2027 assert!(
2028 !table_info
2029 .to_region_options()
2030 .contains_key(REPARTITION_COLUMN_HINT_KEY)
2031 );
2032 }
2033
2034 #[test]
2035 fn test_set_auto_flush_interval() {
2036 let schema = Arc::new(new_test_schema());
2037 let table_options = TableOptions::default();
2038 let meta = TableMetaBuilder::empty()
2039 .schema(schema)
2040 .primary_key_indices(vec![0])
2041 .engine("engine")
2042 .next_column_id(3)
2043 .options(table_options)
2044 .build()
2045 .unwrap();
2046
2047 let alter_kind = AlterKind::SetTableOptions {
2048 options: vec![SetRegionOption::AutoFlushInterval(Some(
2049 std::time::Duration::from_secs(300),
2050 ))],
2051 };
2052 let new_meta = meta
2053 .builder_with_alter_kind("my_table", &alter_kind)
2054 .unwrap()
2055 .build()
2056 .unwrap();
2057
2058 assert_eq!(
2059 Some("5m"),
2060 new_meta
2061 .options
2062 .extra_options
2063 .get(AUTO_FLUSH_INTERVAL_KEY)
2064 .map(String::as_str)
2065 );
2066 }
2067
2068 #[test]
2069 fn test_set_auto_flush_interval_none_removes_existing() {
2070 let schema = Arc::new(new_test_schema());
2071 let mut table_options = TableOptions::default();
2072 table_options
2073 .extra_options
2074 .insert(AUTO_FLUSH_INTERVAL_KEY.to_string(), "5m".to_string());
2075 let meta = TableMetaBuilder::empty()
2076 .schema(schema)
2077 .primary_key_indices(vec![0])
2078 .engine("engine")
2079 .next_column_id(3)
2080 .options(table_options)
2081 .build()
2082 .unwrap();
2083
2084 let alter_kind = AlterKind::SetTableOptions {
2085 options: vec![SetRegionOption::AutoFlushInterval(None)],
2086 };
2087 let new_meta = meta
2088 .builder_with_alter_kind("my_table", &alter_kind)
2089 .unwrap()
2090 .build()
2091 .unwrap();
2092
2093 assert!(
2094 !new_meta
2095 .options
2096 .extra_options
2097 .contains_key(AUTO_FLUSH_INTERVAL_KEY)
2098 );
2099 }
2100
2101 #[test]
2102 fn test_set_and_unset_max_row_group_row_count() {
2103 let meta = TableMetaBuilder::empty()
2104 .schema(Arc::new(new_test_schema()))
2105 .primary_key_indices(vec![0])
2106 .engine("engine")
2107 .next_column_id(3)
2108 .options(TableOptions::default())
2109 .build()
2110 .unwrap();
2111
2112 let alter_kind = AlterKind::SetTableOptions {
2113 options: vec![SetRegionOption::MaxRowGroupRowCount(Some(512))],
2114 };
2115 let new_meta = meta
2116 .builder_with_alter_kind("my_table", &alter_kind)
2117 .unwrap()
2118 .build()
2119 .unwrap();
2120 assert_eq!(
2121 Some("512"),
2122 new_meta
2123 .options
2124 .extra_options
2125 .get(MAX_ROW_GROUP_ROW_COUNT)
2126 .map(String::as_str)
2127 );
2128
2129 let alter_kind = AlterKind::UnsetTableOptions {
2130 keys: vec![UnsetRegionOption::MaxRowGroupRowCount],
2131 };
2132 let new_meta = new_meta
2133 .builder_with_alter_kind("my_table", &alter_kind)
2134 .unwrap()
2135 .build()
2136 .unwrap();
2137 assert!(
2138 !new_meta
2139 .options
2140 .extra_options
2141 .contains_key(MAX_ROW_GROUP_ROW_COUNT)
2142 );
2143 }
2144
2145 #[test]
2146 fn test_add_columns_multiple_times() {
2147 let schema = Arc::new(new_test_schema());
2148 let meta = TableMetaBuilder::empty()
2149 .schema(schema)
2150 .primary_key_indices(vec![0])
2151 .engine("engine")
2152 .next_column_id(3)
2153 .build()
2154 .unwrap();
2155
2156 let alter_kind = AlterKind::AddColumns {
2157 columns: vec![
2158 AddColumnRequest {
2159 column_schema: ColumnSchema::new(
2160 "col3",
2161 ConcreteDataType::int32_datatype(),
2162 true,
2163 ),
2164 is_key: true,
2165 location: None,
2166 add_if_not_exists: true,
2167 },
2168 AddColumnRequest {
2169 column_schema: ColumnSchema::new(
2170 "col3",
2171 ConcreteDataType::int32_datatype(),
2172 true,
2173 ),
2174 is_key: true,
2175 location: None,
2176 add_if_not_exists: true,
2177 },
2178 ],
2179 };
2180 let err = meta
2181 .builder_with_alter_kind("my_table", &alter_kind)
2182 .err()
2183 .unwrap();
2184 assert_eq!(StatusCode::InvalidArguments, err.status_code());
2185 }
2186
2187 #[test]
2188 fn test_remove_columns() {
2189 let schema = Arc::new(new_test_schema());
2190 let meta = TableMetaBuilder::empty()
2191 .schema(schema.clone())
2192 .primary_key_indices(vec![0])
2193 .engine("engine")
2194 .next_column_id(3)
2195 .build()
2196 .unwrap();
2197 let meta = add_columns_to_meta(&meta);
2199
2200 let alter_kind = AlterKind::DropColumns {
2201 names: vec![String::from("col2"), String::from("my_field")],
2202 };
2203 let new_meta = meta
2204 .builder_with_alter_kind("my_table", &alter_kind)
2205 .unwrap()
2206 .build()
2207 .unwrap();
2208
2209 let names: Vec<String> = new_meta
2210 .schema
2211 .column_schemas()
2212 .iter()
2213 .map(|column_schema| column_schema.name.clone())
2214 .collect();
2215 assert_eq!(&["col1", "ts", "my_tag"], &names[..]);
2216 assert_eq!(&[0, 2], &new_meta.primary_key_indices[..]);
2217 assert_eq!(&[1], &new_meta.value_indices[..]);
2218 assert_eq!(
2219 schema.timestamp_column(),
2220 new_meta.schema.timestamp_column()
2221 );
2222 }
2223
2224 #[test]
2225 fn test_remove_multiple_columns_before_timestamp() {
2226 let column_schemas = vec![
2227 ColumnSchema::new("col1", ConcreteDataType::int32_datatype(), true),
2228 ColumnSchema::new("col2", ConcreteDataType::int32_datatype(), true),
2229 ColumnSchema::new("col3", ConcreteDataType::int32_datatype(), true),
2230 ColumnSchema::new(
2231 "ts",
2232 ConcreteDataType::timestamp_millisecond_datatype(),
2233 false,
2234 )
2235 .with_time_index(true),
2236 ];
2237 let schema = Arc::new(
2238 SchemaBuilder::try_from(column_schemas)
2239 .unwrap()
2240 .version(123)
2241 .build()
2242 .unwrap(),
2243 );
2244 let meta = TableMetaBuilder::empty()
2245 .schema(schema.clone())
2246 .primary_key_indices(vec![1])
2247 .engine("engine")
2248 .next_column_id(4)
2249 .build()
2250 .unwrap();
2251
2252 let alter_kind = AlterKind::DropColumns {
2254 names: vec![String::from("col3"), String::from("col1")],
2255 };
2256 let new_meta = meta
2257 .builder_with_alter_kind("my_table", &alter_kind)
2258 .unwrap()
2259 .build()
2260 .unwrap();
2261
2262 let names: Vec<String> = new_meta
2263 .schema
2264 .column_schemas()
2265 .iter()
2266 .map(|column_schema| column_schema.name.clone())
2267 .collect();
2268 assert_eq!(&["col2", "ts"], &names[..]);
2269 assert_eq!(&[0], &new_meta.primary_key_indices[..]);
2270 assert_eq!(&[1], &new_meta.value_indices[..]);
2271 assert_eq!(
2272 schema.timestamp_column(),
2273 new_meta.schema.timestamp_column()
2274 );
2275 }
2276
2277 #[test]
2278 fn test_add_existing_column() {
2279 let schema = Arc::new(new_test_schema());
2280 let meta = TableMetaBuilder::empty()
2281 .schema(schema)
2282 .primary_key_indices(vec![0])
2283 .engine("engine")
2284 .next_column_id(3)
2285 .build()
2286 .unwrap();
2287
2288 let alter_kind = AlterKind::AddColumns {
2289 columns: vec![AddColumnRequest {
2290 column_schema: ColumnSchema::new("col1", ConcreteDataType::string_datatype(), true),
2291 is_key: false,
2292 location: None,
2293 add_if_not_exists: false,
2294 }],
2295 };
2296
2297 let err = meta
2298 .builder_with_alter_kind("my_table", &alter_kind)
2299 .err()
2300 .unwrap();
2301 assert_eq!(StatusCode::TableColumnExists, err.status_code());
2302
2303 let alter_kind = AlterKind::AddColumns {
2305 columns: vec![AddColumnRequest {
2306 column_schema: ColumnSchema::new("col1", ConcreteDataType::int32_datatype(), true),
2307 is_key: true,
2308 location: None,
2309 add_if_not_exists: true,
2310 }],
2311 };
2312 let new_meta = meta
2313 .builder_with_alter_kind("my_table", &alter_kind)
2314 .unwrap()
2315 .build()
2316 .unwrap();
2317 assert_eq!(
2318 meta.schema.column_schemas(),
2319 new_meta.schema.column_schemas()
2320 );
2321 assert_eq!(meta.schema.version() + 1, new_meta.schema.version());
2322 }
2323
2324 #[test]
2325 fn test_add_different_type_column() {
2326 let schema = Arc::new(new_test_schema());
2327 let meta = TableMetaBuilder::empty()
2328 .schema(schema)
2329 .primary_key_indices(vec![0])
2330 .engine("engine")
2331 .next_column_id(3)
2332 .build()
2333 .unwrap();
2334
2335 let alter_kind = AlterKind::AddColumns {
2337 columns: vec![AddColumnRequest {
2338 column_schema: ColumnSchema::new("col1", ConcreteDataType::string_datatype(), true),
2339 is_key: false,
2340 location: None,
2341 add_if_not_exists: true,
2342 }],
2343 };
2344 let err = meta
2345 .builder_with_alter_kind("my_table", &alter_kind)
2346 .err()
2347 .unwrap();
2348 assert_eq!(StatusCode::InvalidArguments, err.status_code());
2349 }
2350
2351 #[test]
2352 fn test_add_invalid_column() {
2353 let schema = Arc::new(new_test_schema());
2354 let meta = TableMetaBuilder::empty()
2355 .schema(schema)
2356 .primary_key_indices(vec![0])
2357 .engine("engine")
2358 .next_column_id(3)
2359 .build()
2360 .unwrap();
2361
2362 let alter_kind = AlterKind::AddColumns {
2364 columns: vec![AddColumnRequest {
2365 column_schema: ColumnSchema::new(
2366 "weny",
2367 ConcreteDataType::string_datatype(),
2368 false,
2369 ),
2370 is_key: false,
2371 location: None,
2372 add_if_not_exists: false,
2373 }],
2374 };
2375
2376 let err = meta
2377 .builder_with_alter_kind("my_table", &alter_kind)
2378 .err()
2379 .unwrap();
2380 assert_eq!(StatusCode::InvalidArguments, err.status_code());
2381 }
2382
2383 #[test]
2384 fn test_remove_unknown_column() {
2385 let schema = Arc::new(new_test_schema());
2386 let meta = TableMetaBuilder::empty()
2387 .schema(schema)
2388 .primary_key_indices(vec![0])
2389 .engine("engine")
2390 .next_column_id(3)
2391 .build()
2392 .unwrap();
2393
2394 let alter_kind = AlterKind::DropColumns {
2395 names: vec![String::from("unknown")],
2396 };
2397
2398 let err = meta
2399 .builder_with_alter_kind("my_table", &alter_kind)
2400 .err()
2401 .unwrap();
2402 assert_eq!(StatusCode::TableColumnNotFound, err.status_code());
2403 }
2404
2405 #[test]
2406 fn test_change_unknown_column_data_type() {
2407 let schema = Arc::new(new_test_schema());
2408 let meta = TableMetaBuilder::empty()
2409 .schema(schema)
2410 .primary_key_indices(vec![0])
2411 .engine("engine")
2412 .next_column_id(3)
2413 .build()
2414 .unwrap();
2415
2416 let alter_kind = AlterKind::ModifyColumnTypes {
2417 columns: vec![ModifyColumnTypeRequest {
2418 column_name: "unknown".to_string(),
2419 target_type: ConcreteDataType::string_datatype(),
2420 }],
2421 };
2422
2423 let err = meta
2424 .builder_with_alter_kind("my_table", &alter_kind)
2425 .err()
2426 .unwrap();
2427 assert_eq!(StatusCode::TableColumnNotFound, err.status_code());
2428 }
2429
2430 #[test]
2431 fn test_remove_key_column() {
2432 let schema = Arc::new(new_test_schema());
2433 let meta = TableMetaBuilder::empty()
2434 .schema(schema)
2435 .primary_key_indices(vec![0])
2436 .engine("engine")
2437 .next_column_id(3)
2438 .build()
2439 .unwrap();
2440
2441 let alter_kind = AlterKind::DropColumns {
2443 names: vec![String::from("col1")],
2444 };
2445
2446 let err = meta
2447 .builder_with_alter_kind("my_table", &alter_kind)
2448 .err()
2449 .unwrap();
2450 assert_eq!(StatusCode::InvalidArguments, err.status_code());
2451
2452 let alter_kind = AlterKind::DropColumns {
2454 names: vec![String::from("ts")],
2455 };
2456
2457 let err = meta
2458 .builder_with_alter_kind("my_table", &alter_kind)
2459 .err()
2460 .unwrap();
2461 assert_eq!(StatusCode::InvalidArguments, err.status_code());
2462 }
2463
2464 #[test]
2465 fn test_remove_partition_column() {
2466 let schema = Arc::new(new_test_schema());
2467 let meta = TableMetaBuilder::empty()
2468 .schema(schema)
2469 .primary_key_indices(vec![])
2470 .partition_key_indices(vec![0])
2471 .engine("engine")
2472 .next_column_id(3)
2473 .build()
2474 .unwrap();
2475 let alter_kind = AlterKind::DropColumns {
2477 names: vec![String::from("col1")],
2478 };
2479
2480 let err = meta
2481 .builder_with_alter_kind("my_table", &alter_kind)
2482 .err()
2483 .unwrap();
2484 assert_matches!(err, Error::RemovePartitionColumn { .. });
2485 }
2486
2487 #[test]
2488 fn test_change_key_column_data_type() {
2489 let schema = Arc::new(new_test_schema());
2490 let meta = TableMetaBuilder::empty()
2491 .schema(schema)
2492 .primary_key_indices(vec![0])
2493 .engine("engine")
2494 .next_column_id(3)
2495 .build()
2496 .unwrap();
2497
2498 let alter_kind = AlterKind::ModifyColumnTypes {
2500 columns: vec![ModifyColumnTypeRequest {
2501 column_name: "col1".to_string(),
2502 target_type: ConcreteDataType::string_datatype(),
2503 }],
2504 };
2505
2506 let err = meta
2507 .builder_with_alter_kind("my_table", &alter_kind)
2508 .err()
2509 .unwrap();
2510 assert_eq!(StatusCode::InvalidArguments, err.status_code());
2511
2512 let alter_kind = AlterKind::ModifyColumnTypes {
2514 columns: vec![ModifyColumnTypeRequest {
2515 column_name: "ts".to_string(),
2516 target_type: ConcreteDataType::string_datatype(),
2517 }],
2518 };
2519
2520 let err = meta
2521 .builder_with_alter_kind("my_table", &alter_kind)
2522 .err()
2523 .unwrap();
2524 assert_eq!(StatusCode::InvalidArguments, err.status_code());
2525 }
2526
2527 #[test]
2528 fn test_alloc_new_column() {
2529 let schema = Arc::new(new_test_schema());
2530 let mut meta = TableMetaBuilder::empty()
2531 .schema(schema)
2532 .primary_key_indices(vec![0])
2533 .engine("engine")
2534 .next_column_id(3)
2535 .build()
2536 .unwrap();
2537 assert_eq!(3, meta.next_column_id);
2538
2539 let column_schema = ColumnSchema::new("col1", ConcreteDataType::int32_datatype(), true);
2540 let desc = meta.alloc_new_column("test_table", &column_schema).unwrap();
2541
2542 assert_eq!(4, meta.next_column_id);
2543 assert_eq!(column_schema.name, desc.name);
2544 }
2545
2546 #[test]
2547 fn test_add_columns_with_location() {
2548 let schema = Arc::new(new_test_schema());
2549 let meta = TableMetaBuilder::empty()
2550 .schema(schema)
2551 .primary_key_indices(vec![0])
2552 .partition_key_indices(vec![0, 2])
2554 .engine("engine")
2555 .next_column_id(3)
2556 .build()
2557 .unwrap();
2558
2559 let new_meta = add_columns_to_meta_with_location(&meta);
2560 let names: Vec<String> = new_meta
2561 .schema
2562 .column_schemas()
2563 .iter()
2564 .map(|column_schema| column_schema.name.clone())
2565 .collect();
2566 assert_eq!(
2567 &[
2568 "my_tag_first", "col1", "ts", "yet_another_field_after_ts", "my_field_after_ts", "col2", ],
2575 &names[..]
2576 );
2577 assert_eq!(&[0, 1, 3], &new_meta.primary_key_indices[..]);
2578 assert_eq!(&[2, 4, 5], &new_meta.value_indices[..]);
2579 assert_eq!(&[1, 5], &new_meta.partition_key_indices[..]);
2580 }
2581
2582 #[test]
2583 fn test_modify_column_fulltext_options() {
2584 let schema = Arc::new(new_test_schema());
2585 let meta = TableMetaBuilder::empty()
2586 .schema(schema)
2587 .primary_key_indices(vec![0])
2588 .engine("engine")
2589 .next_column_id(3)
2590 .build()
2591 .unwrap();
2592
2593 let alter_kind = AlterKind::SetIndexes {
2594 options: vec![SetIndexOption::Fulltext {
2595 column_name: "col1".to_string(),
2596 options: FulltextOptions::default(),
2597 }],
2598 };
2599 let err = meta
2600 .builder_with_alter_kind("my_table", &alter_kind)
2601 .err()
2602 .unwrap();
2603 assert_eq!(
2604 "Invalid column option, column name: col1, error: FULLTEXT index only supports string type",
2605 err.to_string()
2606 );
2607
2608 let new_meta = add_columns_to_meta_with_location(&meta);
2610 let alter_kind = AlterKind::SetIndexes {
2611 options: vec![SetIndexOption::Fulltext {
2612 column_name: "my_tag_first".to_string(),
2613 options: FulltextOptions::new_unchecked(
2614 true,
2615 FulltextAnalyzer::Chinese,
2616 true,
2617 FulltextBackend::Bloom,
2618 1000,
2619 0.01,
2620 ),
2621 }],
2622 };
2623 let new_meta = new_meta
2624 .builder_with_alter_kind("my_table", &alter_kind)
2625 .unwrap()
2626 .build()
2627 .unwrap();
2628 let column_schema = new_meta
2629 .schema
2630 .column_schema_by_name("my_tag_first")
2631 .unwrap();
2632 let fulltext_options = column_schema.fulltext_options().unwrap().unwrap();
2633 assert!(fulltext_options.enable);
2634 assert_eq!(
2635 datatypes::schema::FulltextAnalyzer::Chinese,
2636 fulltext_options.analyzer
2637 );
2638 assert!(fulltext_options.case_sensitive);
2639
2640 let alter_kind = AlterKind::UnsetIndexes {
2641 options: vec![UnsetIndexOption::Fulltext {
2642 column_name: "my_tag_first".to_string(),
2643 }],
2644 };
2645 let new_meta = new_meta
2646 .builder_with_alter_kind("my_table", &alter_kind)
2647 .unwrap()
2648 .build()
2649 .unwrap();
2650 let column_schema = new_meta
2651 .schema
2652 .column_schema_by_name("my_tag_first")
2653 .unwrap();
2654 let fulltext_options = column_schema.fulltext_options().unwrap().unwrap();
2655 assert!(!fulltext_options.enable);
2656 }
2657
2658 #[test]
2659 fn test_table_info_serde_compatibility() {
2660 let serialized = r#"{"ident":{"table_id":1024,"version":1},"name":"foo","desc":"my table","catalog_name":"greptime","schema_name":"public","meta":{"schema":{"column_schemas":[{"name":"col1","data_type":{"Int32":{}},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}},{"name":"ts","data_type":{"Timestamp":{"Millisecond":null}},"is_nullable":false,"is_time_index":true,"default_constraint":null,"metadata":{"greptime:time_index":"true"}},{"name":"col2","data_type":{"Int32":{}},"is_nullable":true,"is_time_index":false,"default_constraint":null,"metadata":{}}],"timestamp_index":1,"version":123},"primary_key_indices":[0],"value_indices":[1,2],"engine":"mito","next_column_id":3,"options":{"write_buffer_size":null,"ttl":"1h","skip_wal":false,"extra_options":{}},"created_on":"-262143-01-01T00:00:00Z","updated_on":"+262142-12-31T23:59:59.999999999Z","partition_key_indices":[2],"column_ids":[0,1,2]},"table_type":"Base"}"#;
2693
2694 let actual: TableInfo = serde_json::from_str(serialized).unwrap();
2695 let expected = TableInfo {
2696 ident: TableIdent {
2697 table_id: 1024,
2698 version: 1,
2699 },
2700 name: "foo".to_string(),
2701 desc: Some("my table".to_string()),
2702 catalog_name: "greptime".to_string(),
2703 schema_name: "public".to_string(),
2704 meta: TableMeta {
2705 schema: Arc::new(new_test_schema()),
2706 primary_key_indices: vec![0],
2707 value_indices: vec![1, 2],
2708 engine: "mito".to_string(),
2709 next_column_id: 3,
2710 options: TableOptions {
2711 ttl: Some(common_time::TimeToLive::Duration(
2712 std::time::Duration::from_secs(3600),
2713 )),
2714 ..Default::default()
2715 },
2716 created_on: DateTime::<Utc>::MIN_UTC,
2717 updated_on: DateTime::<Utc>::MAX_UTC,
2718 partition_key_indices: vec![2],
2719 column_ids: vec![0, 1, 2],
2720 },
2721 table_type: TableType::Base,
2722 };
2723 assert_eq!(actual, expected);
2724 }
2725
2726 use crate::requests::{SEMANTIC_METRIC_UNIT, SEMANTIC_SIGNAL_TYPE};
2727
2728 fn semantic_test_meta() -> TableMeta {
2731 let column_schemas = vec![
2732 ColumnSchema::new("host", ConcreteDataType::string_datatype(), true),
2733 ColumnSchema::new(
2734 "ts",
2735 ConcreteDataType::timestamp_millisecond_datatype(),
2736 false,
2737 )
2738 .with_time_index(true),
2739 ColumnSchema::new("payload", ConcreteDataType::binary_datatype(), true),
2740 ColumnSchema::new("service", ConcreteDataType::string_datatype(), true),
2741 ColumnSchema::new("note", ConcreteDataType::string_datatype(), true),
2742 ];
2743 let schema = Arc::new(
2744 SchemaBuilder::try_from(column_schemas)
2745 .unwrap()
2746 .build()
2747 .unwrap(),
2748 );
2749 TableMetaBuilder::empty()
2750 .schema(schema)
2751 .primary_key_indices(vec![0])
2752 .engine("engine")
2753 .next_column_id(5)
2754 .build()
2755 .unwrap()
2756 }
2757
2758 #[test]
2759 fn test_set_semantic_annotations() {
2760 let meta = semantic_test_meta();
2761 let alter_kind = AlterKind::SetAnnotations {
2763 family: AnnotationFamily::Semantic,
2764 options: vec![
2765 (SEMANTIC_SIGNAL_TYPE.to_string(), "trace".to_string()),
2766 (
2767 "greptime.semantic.entity.service.id".to_string(),
2768 "service".to_string(),
2769 ),
2770 ],
2771 };
2772 let new_meta = meta
2773 .builder_with_alter_kind("my_table", &alter_kind)
2774 .unwrap()
2775 .build()
2776 .unwrap();
2777 assert_eq!(
2778 new_meta.options.extra_options.get(SEMANTIC_SIGNAL_TYPE),
2779 Some(&"trace".to_string())
2780 );
2781 assert_eq!(
2782 new_meta
2783 .options
2784 .extra_options
2785 .get("greptime.semantic.entity.service.id"),
2786 Some(&"service".to_string())
2787 );
2788 }
2789
2790 #[test]
2791 fn test_repartition_hint_batch_must_be_single_key() {
2792 let meta = TableMetaBuilder::empty()
2793 .schema(Arc::new(new_test_schema()))
2794 .primary_key_indices(vec![0])
2795 .engine("engine")
2796 .next_column_id(3)
2797 .build()
2798 .unwrap();
2799
2800 let dup_set = AlterKind::SetAnnotations {
2803 family: AnnotationFamily::RepartitionHint,
2804 options: vec![
2805 (REPARTITION_COLUMN_HINT_KEY.to_string(), "col1".to_string()),
2806 (REPARTITION_COLUMN_HINT_KEY.to_string(), "col2".to_string()),
2807 ],
2808 };
2809 let err = meta
2810 .builder_with_alter_kind("my_table", &dup_set)
2811 .err()
2812 .unwrap();
2813 assert!(
2814 err.to_string().contains("must be altered separately"),
2815 "{err}"
2816 );
2817
2818 let dup_unset = AlterKind::UnsetAnnotations {
2819 family: AnnotationFamily::RepartitionHint,
2820 keys: vec![
2821 REPARTITION_COLUMN_HINT_KEY.to_string(),
2822 REPARTITION_COLUMN_HINT_KEY.to_string(),
2823 ],
2824 };
2825 let err = meta
2826 .builder_with_alter_kind("my_table", &dup_unset)
2827 .err()
2828 .unwrap();
2829 assert!(
2830 err.to_string().contains("must be altered separately"),
2831 "{err}"
2832 );
2833 }
2834
2835 #[test]
2836 fn test_set_semantic_annotations_rejects_invalid() {
2837 let meta = semantic_test_meta();
2838 let cases = [
2839 (
2840 "greptime.semantic.unknown_key",
2841 "x",
2842 "unknown semantic option",
2843 ),
2844 (SEMANTIC_SIGNAL_TYPE, "garbage", "invalid value"),
2845 (
2846 "greptime.semantic.entity.host.id",
2847 "no_such_column",
2848 "no_such_column",
2849 ),
2850 (
2851 "greptime.semantic.entity.host.id",
2852 "payload",
2853 "cannot render as a string",
2854 ),
2855 ];
2856 for (key, value, needle) in cases {
2857 let alter_kind = AlterKind::SetAnnotations {
2858 family: AnnotationFamily::Semantic,
2859 options: vec![(key.to_string(), value.to_string())],
2860 };
2861 let err = meta
2862 .builder_with_alter_kind("my_table", &alter_kind)
2863 .err()
2864 .unwrap();
2865 assert!(
2866 err.to_string().contains(needle),
2867 "key `{key}`: unexpected error `{err}`"
2868 );
2869 }
2870
2871 let missing = meta
2874 .builder_with_alter_kind(
2875 "my_table",
2876 &AlterKind::SetAnnotations {
2877 family: AnnotationFamily::Semantic,
2878 options: vec![(
2879 "greptime.semantic.entity.host.id".to_string(),
2880 "no_such_column".to_string(),
2881 )],
2882 },
2883 )
2884 .err()
2885 .unwrap();
2886 assert_eq!(
2887 common_error::status_code::StatusCode::TableColumnNotFound,
2888 common_error::ext::ErrorExt::status_code(&missing)
2889 );
2890 }
2891
2892 #[test]
2893 fn test_unset_semantic_annotations() {
2894 let mut meta = semantic_test_meta();
2895 meta.options
2896 .extra_options
2897 .insert(SEMANTIC_SIGNAL_TYPE.to_string(), "trace".to_string());
2898 meta.options
2900 .extra_options
2901 .insert("greptime.semantic.future_key".to_string(), "x".to_string());
2902
2903 let alter_kind = AlterKind::UnsetAnnotations {
2904 family: AnnotationFamily::Semantic,
2905 keys: vec![
2906 SEMANTIC_SIGNAL_TYPE.to_string(),
2907 "greptime.semantic.future_key".to_string(),
2908 SEMANTIC_METRIC_UNIT.to_string(),
2910 ],
2911 };
2912 let new_meta = meta
2913 .builder_with_alter_kind("my_table", &alter_kind)
2914 .unwrap()
2915 .build()
2916 .unwrap();
2917 assert!(
2918 !new_meta
2919 .options
2920 .extra_options
2921 .contains_key(SEMANTIC_SIGNAL_TYPE)
2922 );
2923 assert!(
2924 !new_meta
2925 .options
2926 .extra_options
2927 .contains_key("greptime.semantic.future_key")
2928 );
2929
2930 let outside = AlterKind::UnsetAnnotations {
2931 family: AnnotationFamily::Semantic,
2932 keys: vec!["ttl".to_string()],
2933 };
2934 let err = meta
2935 .builder_with_alter_kind("my_table", &outside)
2936 .err()
2937 .unwrap();
2938 assert!(err.to_string().contains("annotation namespace"), "{err}");
2939 }
2940
2941 #[test]
2942 fn test_modify_entity_column_type_keeps_string_form() {
2943 let mut meta = semantic_test_meta();
2944 meta.options.extra_options.insert(
2945 "greptime.semantic.entity.service.id".to_string(),
2946 "service".to_string(),
2947 );
2948
2949 let alter_kind = AlterKind::ModifyColumnTypes {
2950 columns: vec![ModifyColumnTypeRequest {
2951 column_name: "service".to_string(),
2952 target_type: ConcreteDataType::binary_datatype(),
2953 }],
2954 };
2955 let err = meta
2956 .builder_with_alter_kind("my_table", &alter_kind)
2957 .err()
2958 .unwrap();
2959 assert!(
2960 err.to_string()
2961 .contains("must keep a type that renders as a string"),
2962 "{err}"
2963 );
2964
2965 let alter_kind = AlterKind::ModifyColumnTypes {
2967 columns: vec![ModifyColumnTypeRequest {
2968 column_name: "note".to_string(),
2969 target_type: ConcreteDataType::binary_datatype(),
2970 }],
2971 };
2972 meta.builder_with_alter_kind("my_table", &alter_kind)
2973 .unwrap();
2974 }
2975
2976 #[test]
2977 fn test_stale_entity_declaration_guards_readd_and_reports_missing_column() {
2978 let mut meta = semantic_test_meta();
2980 meta.options.extra_options.insert(
2981 "greptime.semantic.entity.service.id".to_string(),
2982 "gone".to_string(),
2983 );
2984
2985 let modify = AlterKind::ModifyColumnTypes {
2988 columns: vec![ModifyColumnTypeRequest {
2989 column_name: "gone".to_string(),
2990 target_type: ConcreteDataType::binary_datatype(),
2991 }],
2992 };
2993 let err = meta
2994 .builder_with_alter_kind("my_table", &modify)
2995 .err()
2996 .unwrap();
2997 assert_eq!(
2998 common_error::status_code::StatusCode::TableColumnNotFound,
2999 common_error::ext::ErrorExt::status_code(&err)
3000 );
3001
3002 let add = |ty: ConcreteDataType| AlterKind::AddColumns {
3005 columns: vec![AddColumnRequest {
3006 column_schema: ColumnSchema::new("gone", ty, true),
3007 is_key: false,
3008 location: None,
3009 add_if_not_exists: false,
3010 }],
3011 };
3012 let err = meta
3013 .builder_with_alter_kind("my_table", &add(ConcreteDataType::binary_datatype()))
3014 .err()
3015 .unwrap();
3016 assert!(
3017 err.to_string()
3018 .contains("must keep a type that renders as a string"),
3019 "{err}"
3020 );
3021
3022 meta.builder_with_alter_kind("my_table", &add(ConcreteDataType::string_datatype()))
3024 .unwrap();
3025 }
3026}