table/
requests.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Table and TableEngine requests
16
17use std::collections::{HashMap, HashSet};
18use std::fmt;
19use std::str::FromStr;
20
21use common_base::readable_size::ReadableSize;
22use common_datasource::object_store::oss::is_supported_in_oss;
23use common_datasource::object_store::s3::is_supported_in_s3;
24use common_query::AddColumnLocation;
25use common_time::TimeToLive;
26use common_time::range::TimestampRange;
27use datatypes::data_type::ConcreteDataType;
28use datatypes::json::JsonSettings;
29use datatypes::prelude::VectorRef;
30use datatypes::schema::{
31    ColumnDefaultConstraint, ColumnSchema, FulltextOptions, Schema, SkippingIndexOptions,
32};
33use greptime_proto::v1::region::{build_index_request, compact_request};
34use once_cell::sync::Lazy;
35use serde::{Deserialize, Serialize};
36use store_api::metric_engine_consts::{
37    LOGICAL_TABLE_METADATA_KEY, PHYSICAL_TABLE_METADATA_KEY, is_metric_engine_option_key,
38};
39use store_api::mito_engine_options::{
40    APPEND_MODE_KEY, COMPACTION_TYPE, EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, FloatFieldEncoding,
41    MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD, MEMTABLE_BULK_ENCODE_ROW_THRESHOLD,
42    MEMTABLE_BULK_MAX_MERGE_GROUPS, MEMTABLE_BULK_MERGE_THRESHOLD, MEMTABLE_TYPE, MERGE_MODE_KEY,
43    SST_FORMAT_KEY, TWCS_ACTIVE_WINDOW_L1_MERGE_TRIGGER, TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM,
44    TWCS_FALLBACK_TO_LOCAL, TWCS_INACTIVE_WINDOW_L1_MERGE_TRIGGER,
45    TWCS_INACTIVE_WINDOW_TRIGGER_FILE_NUM, TWCS_MAX_OUTPUT_FILE_SIZE, TWCS_TIME_WINDOW,
46    TWCS_TRIGGER_FILE_NUM, is_mito_engine_option_key, normalize_twcs_trigger_options,
47};
48use store_api::region_request::{SetRegionOption, UnsetRegionOption};
49
50use crate::error::{ConflictingTableOptionsSnafu, ParseTableOptionSnafu, Result};
51use crate::metadata::{TableId, TableVersion};
52
53mod semantic;
54pub use semantic::*;
55
56pub const FILE_TABLE_META_KEY: &str = "__private.file_table_meta";
57pub const RELATIONAL_META_KEY: &str = "__private.relational_meta";
58pub const FILE_TABLE_LOCATION_KEY: &str = "location";
59pub const FILE_TABLE_PATTERN_KEY: &str = "pattern";
60pub const FILE_TABLE_FORMAT_KEY: &str = "format";
61
62/// Options captured at `CREATE EXTERNAL TABLE ... ENGINE=relational WITH (...)` time
63/// and persisted in the region manifest.
64///
65/// `connection` is persisted in **plaintext** so the datanode can reconnect to the
66/// remote database when serving queries on the region.
67#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
68pub struct RelationalOptions {
69    /// Full DSN, e.g. `mysql://user:password@host:3306/db`.
70    pub connection: String,
71    pub remote_table: String,
72}
73
74pub const TABLE_DATA_MODEL: &str = "table_data_model";
75pub const TABLE_DATA_MODEL_TRACE_V1: &str = "greptime_trace_v1";
76/// Table data model used by the JSON2-based OTLP trace pipeline.
77pub const TABLE_DATA_MODEL_TRACE_V2: &str = "greptime_trace_v2";
78
79/// Returns true for the Trace V1 and V2 data models supported
80/// by semantic graph derivation.
81pub fn is_trace_table(table_info: &crate::metadata::TableInfo) -> bool {
82    let table_data_model = table_info.meta.options.data_model();
83    matches!(
84        table_data_model,
85        Some(TABLE_DATA_MODEL_TRACE_V1 | TABLE_DATA_MODEL_TRACE_V2)
86    )
87}
88
89/// Retained table metadata; OTLP metrics always use Prometheus-compatible conversion.
90pub const OTLP_METRIC_COMPAT_KEY: &str = "otlp_metric_compat";
91pub const OTLP_METRIC_COMPAT_PROM: &str = "prom";
92
93pub const VALID_TABLE_OPTION_KEYS: [&str; 15] = [
94    // common keys:
95    WRITE_BUFFER_SIZE_KEY,
96    TTL_KEY,
97    STORAGE_KEY,
98    COMMENT_KEY,
99    SKIP_WAL_KEY,
100    SST_FORMAT_KEY,
101    // file engine keys:
102    FILE_TABLE_LOCATION_KEY,
103    FILE_TABLE_FORMAT_KEY,
104    FILE_TABLE_PATTERN_KEY,
105    // metric engine keys:
106    PHYSICAL_TABLE_METADATA_KEY,
107    LOGICAL_TABLE_METADATA_KEY,
108    // table model info
109    TABLE_DATA_MODEL,
110    OTLP_METRIC_COMPAT_KEY,
111    REPARTITION_COLUMN_HINT_KEY,
112    REPARTITION_PARTITION_NUM_HINT_KEY,
113];
114
115pub const DDL_TIMEOUT: &str = "timeout";
116pub const DDL_WAIT: &str = "wait";
117
118pub const VALID_DDL_OPTION_KEYS: [&str; 2] = [DDL_TIMEOUT, DDL_WAIT];
119
120/// The key of ingest rows rate limit option (rows per second, cluster-wide) in database options.
121pub const INGEST_ROWS_RATE_LIMIT_KEY: &str = "ingest_rows_rate_limit";
122
123// Valid option keys when creating a db.
124static VALID_DB_OPT_KEYS: Lazy<HashSet<&str>> = Lazy::new(|| {
125    let mut set = HashSet::new();
126    set.insert(TTL_KEY);
127    set.insert(STORAGE_KEY);
128    set.insert(MEMTABLE_TYPE);
129    set.insert(MEMTABLE_BULK_MERGE_THRESHOLD);
130    set.insert(MEMTABLE_BULK_ENCODE_ROW_THRESHOLD);
131    set.insert(MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD);
132    set.insert(MEMTABLE_BULK_MAX_MERGE_GROUPS);
133    set.insert(APPEND_MODE_KEY);
134    set.insert(MERGE_MODE_KEY);
135    set.insert(SKIP_WAL_KEY);
136    set.insert(COMPACTION_TYPE);
137    set.insert(TWCS_FALLBACK_TO_LOCAL);
138    set.insert(TWCS_TIME_WINDOW);
139    set.insert(TWCS_TRIGGER_FILE_NUM);
140    set.insert(TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM);
141    set.insert(TWCS_ACTIVE_WINDOW_L1_MERGE_TRIGGER);
142    set.insert(TWCS_INACTIVE_WINDOW_TRIGGER_FILE_NUM);
143    set.insert(TWCS_INACTIVE_WINDOW_L1_MERGE_TRIGGER);
144    set.insert(TWCS_MAX_OUTPUT_FILE_SIZE);
145    set.insert(SST_FORMAT_KEY);
146    set.insert(INGEST_ROWS_RATE_LIMIT_KEY);
147    set
148});
149
150/// Returns true if the `key` is a valid key for database.
151pub fn validate_database_option(key: &str) -> bool {
152    VALID_DB_OPT_KEYS.contains(&key)
153}
154
155/// Validates a database option value, returning the violated constraint on error.
156pub fn validate_database_option_value(
157    key: &str,
158    value: Option<&str>,
159) -> std::result::Result<(), &'static str> {
160    if key == INGEST_ROWS_RATE_LIMIT_KEY {
161        return value
162            .and_then(|value| value.parse::<u64>().ok())
163            .map(|_| ())
164            .ok_or("expected a non-negative integer fitting in u64");
165    }
166    let (minimum, constraint) = match key {
167        TWCS_TRIGGER_FILE_NUM
168        | TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM
169        | TWCS_INACTIVE_WINDOW_TRIGGER_FILE_NUM => {
170            (0, "expected a non-negative integer fitting in usize")
171        }
172        TWCS_ACTIVE_WINDOW_L1_MERGE_TRIGGER | TWCS_INACTIVE_WINDOW_L1_MERGE_TRIGGER => {
173            (2, "expected an integer greater than or equal to 2")
174        }
175        _ => return Ok(()),
176    };
177    if value
178        .and_then(|value| value.parse::<usize>().ok())
179        .is_some_and(|files| files >= minimum)
180    {
181        Ok(())
182    } else {
183        Err(constraint)
184    }
185}
186
187/// Returns true if the `key` is a valid key for any engine or storage.
188pub fn validate_table_option(key: &str) -> bool {
189    if is_supported_in_s3(key) {
190        return true;
191    }
192
193    if is_supported_in_oss(key) {
194        return true;
195    }
196
197    if is_mito_engine_option_key(key) {
198        return true;
199    }
200
201    if is_metric_engine_option_key(key) {
202        return true;
203    }
204
205    // Semantic-layer keys share a reserved prefix instead of a fixed allowlist so
206    // the vocabulary can grow without touching this gate. See `semantic` module.
207    if is_semantic_option_key(key) {
208        return true;
209    }
210
211    VALID_TABLE_OPTION_KEYS.contains(&key) || VALID_DDL_OPTION_KEYS.contains(&key)
212}
213
214#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
215#[serde(default)]
216pub struct TableOptions {
217    /// Per-region write buffer stall threshold. Writes are rejected at twice this size.
218    pub write_buffer_size: Option<ReadableSize>,
219    /// Time-to-live of table. Expired data will be automatically purged.
220    pub ttl: Option<TimeToLive>,
221    /// Skip wal write for this table.
222    pub skip_wal: bool,
223    /// Extra options that may not applicable to all table engines.
224    pub extra_options: HashMap<String, String>,
225}
226
227pub const WRITE_BUFFER_SIZE_KEY: &str = store_api::mito_engine_options::WRITE_BUFFER_SIZE_KEY;
228pub const TTL_KEY: &str = store_api::mito_engine_options::TTL_KEY;
229pub const STORAGE_KEY: &str = "storage";
230pub const COMMENT_KEY: &str = "comment";
231pub const AUTO_CREATE_TABLE_KEY: &str = "auto_create_table";
232pub const SKIP_WAL_KEY: &str = store_api::mito_engine_options::SKIP_WAL_KEY;
233pub const TRACE_TABLE_PARTITIONS_HINT_KEY: &str = "trace_table_partitions";
234pub const REPARTITION_COLUMN_HINT_KEY: &str = "repartition.column.hint";
235
236/// Table-level partition count hint consumed by the auto-repartition planner.
237pub const REPARTITION_PARTITION_NUM_HINT_KEY: &str = "repartition.partition.num.hint";
238
239impl TableOptions {
240    /// Returns the table data model, if specified.
241    pub fn data_model(&self) -> Option<&str> {
242        self.extra_options.get(TABLE_DATA_MODEL).map(String::as_str)
243    }
244
245    pub fn try_from_iter<T: ToString, U: IntoIterator<Item = (T, T)>>(
246        iter: U,
247    ) -> Result<TableOptions> {
248        let mut options = TableOptions::default();
249
250        let mut kvs: HashMap<String, String> = iter
251            .into_iter()
252            .map(|(k, v)| (k.to_string(), v.to_string()))
253            .collect();
254
255        normalize_twcs_trigger_options(&mut kvs).map_err(|conflict| {
256            ConflictingTableOptionsSnafu {
257                first_key: TWCS_TRIGGER_FILE_NUM,
258                first_value: conflict.legacy_value,
259                second_key: TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM,
260                second_value: conflict.canonical_value,
261            }
262            .build()
263        })?;
264
265        if let Some(write_buffer_size) = kvs.get(WRITE_BUFFER_SIZE_KEY) {
266            let size = ReadableSize::from_str(write_buffer_size).map_err(|_| {
267                ParseTableOptionSnafu {
268                    key: WRITE_BUFFER_SIZE_KEY,
269                    value: write_buffer_size,
270                }
271                .build()
272            })?;
273            options.write_buffer_size = Some(size)
274        }
275
276        if let Some(ttl) = kvs.get(TTL_KEY) {
277            let ttl_value = TimeToLive::from_humantime_or_str(ttl).map_err(|_| {
278                ParseTableOptionSnafu {
279                    key: TTL_KEY,
280                    value: ttl,
281                }
282                .build()
283            })?;
284            options.ttl = Some(ttl_value);
285        }
286
287        if let Some(encoding) = kvs.get(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING) {
288            encoding.parse::<FloatFieldEncoding>().map_err(|_| {
289                ParseTableOptionSnafu {
290                    key: EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING,
291                    value: encoding,
292                }
293                .build()
294            })?;
295        }
296
297        if let Some(skip_wal) = kvs.get(SKIP_WAL_KEY) {
298            options.skip_wal = skip_wal.parse().map_err(|_| {
299                ParseTableOptionSnafu {
300                    key: SKIP_WAL_KEY,
301                    value: skip_wal,
302                }
303                .build()
304            })?;
305        }
306
307        options.extra_options = HashMap::from_iter(
308            kvs.into_iter()
309                .filter(|(k, _)| k != WRITE_BUFFER_SIZE_KEY && k != TTL_KEY),
310        );
311
312        Ok(options)
313    }
314}
315
316impl fmt::Display for TableOptions {
317    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
318        let mut key_vals = vec![];
319        if let Some(size) = self.write_buffer_size {
320            key_vals.push(format!("{}={}", WRITE_BUFFER_SIZE_KEY, size));
321        }
322
323        if let Some(ttl) = self.ttl.map(|ttl| ttl.to_string()) {
324            key_vals.push(format!("{}={}", TTL_KEY, ttl));
325        }
326
327        if self.skip_wal && !self.extra_options.contains_key(SKIP_WAL_KEY) {
328            key_vals.push(format!("{}={}", SKIP_WAL_KEY, self.skip_wal));
329        }
330
331        for (k, v) in &self.extra_options {
332            key_vals.push(format!("{}={}", k, v));
333        }
334
335        write!(f, "{}", key_vals.join(" "))
336    }
337}
338
339impl From<&TableOptions> for HashMap<String, String> {
340    fn from(opts: &TableOptions) -> Self {
341        let mut res = HashMap::with_capacity(3 + opts.extra_options.len());
342        if let Some(write_buffer_size) = opts.write_buffer_size {
343            let _ = res.insert(
344                WRITE_BUFFER_SIZE_KEY.to_string(),
345                write_buffer_size.to_string(),
346            );
347        }
348        if let Some(ttl_str) = opts.ttl.map(|ttl| ttl.to_string()) {
349            let _ = res.insert(TTL_KEY.to_string(), ttl_str);
350        }
351        if opts.skip_wal {
352            let _ = res.insert(SKIP_WAL_KEY.to_string(), true.to_string());
353        }
354        res.extend(opts.extra_options.clone());
355        res
356    }
357}
358
359/// Alter table request
360#[derive(Debug, Clone, Serialize, Deserialize)]
361pub struct AlterTableRequest {
362    pub catalog_name: String,
363    pub schema_name: String,
364    pub table_name: String,
365    pub table_id: TableId,
366    pub alter_kind: AlterKind,
367    // None in standalone.
368    pub table_version: Option<TableVersion>,
369}
370
371/// Add column request
372#[derive(Debug, Clone, Serialize, Deserialize)]
373pub struct AddColumnRequest {
374    pub column_schema: ColumnSchema,
375    pub is_key: bool,
376    pub location: Option<AddColumnLocation>,
377    /// Add column if not exists.
378    pub add_if_not_exists: bool,
379}
380
381/// Change column datatype request
382#[derive(Debug, Clone, Serialize, Deserialize)]
383pub struct ModifyColumnTypeRequest {
384    pub column_name: String,
385    pub target_type: ConcreteDataType,
386}
387
388/// Set JSON2 settings request.
389#[derive(Debug, Clone, Serialize, Deserialize)]
390pub struct SetJsonSettingsRequest {
391    pub column_name: String,
392    pub settings: JsonSettings,
393}
394
395/// A family of annotation table options: pure metadata markers that no region
396/// consumes. Setting or unsetting them only rewrites the table's
397/// `extra_options`, so the alter skips region dispatch entirely.
398#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
399pub enum AnnotationFamily {
400    /// `greptime.semantic.*` options (see the [`semantic`] module).
401    Semantic,
402    /// Column and partition count hints consumed by the auto-repartition planner.
403    RepartitionHint,
404}
405
406impl AnnotationFamily {
407    /// The key namespace used in diagnostics; accepted keys are classified by [`Self::of_key`].
408    pub fn namespace(self) -> &'static str {
409        match self {
410            Self::Semantic => SEMANTIC_PREFIX,
411            Self::RepartitionHint => "repartition.",
412        }
413    }
414
415    pub fn of_key(key: &str) -> Option<Self> {
416        if key.starts_with(SEMANTIC_PREFIX) {
417            Some(Self::Semantic)
418        } else if matches!(
419            key,
420            REPARTITION_COLUMN_HINT_KEY | REPARTITION_PARTITION_NUM_HINT_KEY
421        ) {
422            Some(Self::RepartitionHint)
423        } else {
424            None
425        }
426    }
427
428    /// Whether this family may be altered on logical metric tables. Only
429    /// families whose values nothing on the physical side consumes qualify;
430    /// the repartition hint drives physical region repartitioning.
431    pub fn allows_logical_tables(self) -> bool {
432        match self {
433            Self::Semantic => true,
434            Self::RepartitionHint => false,
435        }
436    }
437
438    /// Whether duplicate keys are rejected in this family's SET/UNSET batch.
439    pub fn requires_unique_keys(self) -> bool {
440        matches!(self, Self::RepartitionHint)
441    }
442
443    /// The error for a SET/UNSET batch mixing this family with other options.
444    pub fn mixed_batch_error(self) -> String {
445        match self {
446            Self::Semantic => format!(
447                "`{SEMANTIC_PREFIX}*` options must be altered separately from other table options"
448            ),
449            Self::RepartitionHint => {
450                "repartition hints must be altered separately from other table options".to_string()
451            }
452        }
453    }
454}
455
456/// Why an annotation SET/UNSET key batch was rejected.
457#[derive(Debug, Clone, PartialEq, Eq)]
458pub enum AnnotationKeyError {
459    MixedFamilies { family: AnnotationFamily },
460    DuplicateKey,
461}
462
463impl fmt::Display for AnnotationKeyError {
464    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
465        match self {
466            Self::MixedFamilies { family } => f.write_str(&family.mixed_batch_error()),
467            Self::DuplicateKey => f.write_str("duplicate repartition hint keys"),
468        }
469    }
470}
471
472/// Validates a SET/UNSET batch and returns its annotation family.
473///
474/// Empty batches and batches containing only non-annotation keys return `None`.
475/// Annotation keys cannot share a batch with another family or region options.
476/// Duplicate keys are rejected only for families that require unique keys.
477pub fn validate_annotation_keys<'a>(
478    keys: impl IntoIterator<Item = &'a str>,
479) -> std::result::Result<Option<AnnotationFamily>, AnnotationKeyError> {
480    let mut keys = keys.into_iter();
481    let Some(first) = keys.next() else {
482        return Ok(None);
483    };
484    let family = AnnotationFamily::of_key(first);
485    let reject_duplicates = family.is_some_and(|family| family.requires_unique_keys());
486    let mut seen = HashSet::new();
487    if reject_duplicates {
488        seen.insert(first);
489    }
490    for key in keys {
491        let this = AnnotationFamily::of_key(key);
492        if this != family
493            && let Some(family) = family.or(this)
494        {
495            return Err(AnnotationKeyError::MixedFamilies { family });
496        }
497        if reject_duplicates && !seen.insert(key) {
498            return Err(AnnotationKeyError::DuplicateKey);
499        }
500    }
501    Ok(family)
502}
503
504/// Table shape an annotation option is validated against.
505pub struct AnnotationContext<'a> {
506    pub data_model: Option<&'a str>,
507    pub schema: &'a Schema,
508    pub partition_key_indices: &'a [usize],
509}
510
511/// Why an annotation option was rejected. Typed so each DDL entry point maps
512/// rules onto its existing error variants and status codes: ALTER keeps
513/// missing columns as `TableColumnNotFound` (4002), CREATE keeps its
514/// `InvalidArguments` family — the rules converge, the contracts do not.
515#[derive(Debug, Clone, PartialEq, Eq)]
516pub enum AnnotationValidationError {
517    UnknownKey {
518        key: String,
519    },
520    InvalidValue {
521        key: String,
522        value: String,
523    },
524    ColumnNotFound {
525        column: String,
526    },
527    ColumnNotStringForm {
528        key: String,
529        column: String,
530        ty: ConcreteDataType,
531    },
532    InvalidPartitionNumHint {
533        value: String,
534    },
535    NotSingleColumn,
536    PartitionMetadataConflict,
537    TimeIndexConflict,
538}
539
540impl fmt::Display for AnnotationValidationError {
541    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
542        match self {
543            Self::UnknownKey { key } => write!(f, "unknown semantic option `{key}`"),
544            Self::InvalidValue { key, value } => {
545                write!(f, "invalid value `{value}` for semantic option `{key}`")
546            }
547            Self::ColumnNotFound { column } => write!(f, "column `{column}` not found"),
548            Self::ColumnNotStringForm { key, column, ty } => write!(
549                f,
550                "entity column `{column}` (option `{key}`) has type `{ty}`, \
551                 which cannot render as a string"
552            ),
553            Self::InvalidPartitionNumHint { value } => write!(
554                f,
555                "{REPARTITION_PARTITION_NUM_HINT_KEY} expects a positive integer within u32 range, got `{value}`"
556            ),
557            Self::NotSingleColumn => write!(
558                f,
559                "{REPARTITION_COLUMN_HINT_KEY} expects exactly one column name"
560            ),
561            Self::PartitionMetadataConflict => write!(
562                f,
563                "cannot set {REPARTITION_COLUMN_HINT_KEY} on a table with partition metadata"
564            ),
565            Self::TimeIndexConflict => write!(
566                f,
567                "cannot set {REPARTITION_COLUMN_HINT_KEY} to the time index column"
568            ),
569        }
570    }
571}
572
573/// Validates one annotation option and returns the value to store — the
574/// repartition hints are trimmed, semantic values pass
575/// through unchanged.
576pub(crate) fn validate_and_normalize_annotation(
577    family: AnnotationFamily,
578    cx: &AnnotationContext<'_>,
579    key: &str,
580    value: &str,
581) -> std::result::Result<String, AnnotationValidationError> {
582    match family {
583        AnnotationFamily::Semantic => {
584            if !is_semantic_option_key(key) {
585                return Err(AnnotationValidationError::UnknownKey {
586                    key: key.to_string(),
587                });
588            }
589            if !validate_semantic_option(key, value) {
590                return Err(AnnotationValidationError::InvalidValue {
591                    key: key.to_string(),
592                    value: value.to_string(),
593                });
594            }
595            if parse_entity_option_key(key).is_some() {
596                for column in parse_entity_columns(value) {
597                    if trace_v2_attribute(cx.schema, cx.data_model, &column).is_some() {
598                        continue;
599                    }
600                    let schema = cx.schema.column_schema_by_name(&column).ok_or_else(|| {
601                        AnnotationValidationError::ColumnNotFound {
602                            column: column.clone(),
603                        }
604                    })?;
605                    if !has_stable_string_form(&schema.data_type) {
606                        return Err(AnnotationValidationError::ColumnNotStringForm {
607                            key: key.to_string(),
608                            column,
609                            ty: schema.data_type.clone(),
610                        });
611                    }
612                }
613            }
614            Ok(value.to_string())
615        }
616        AnnotationFamily::RepartitionHint if key == REPARTITION_PARTITION_NUM_HINT_KEY => {
617            let value = value.trim();
618            if !matches!(value.parse::<u32>(), Ok(1..)) {
619                return Err(AnnotationValidationError::InvalidPartitionNumHint {
620                    value: value.to_string(),
621                });
622            }
623            Ok(value.to_string())
624        }
625        AnnotationFamily::RepartitionHint => {
626            let column_name = value.trim();
627            if column_name.is_empty() || column_name.contains(',') {
628                return Err(AnnotationValidationError::NotSingleColumn);
629            }
630            if !cx.partition_key_indices.is_empty() {
631                return Err(AnnotationValidationError::PartitionMetadataConflict);
632            }
633            let column_index = cx.schema.column_index_by_name(column_name).ok_or_else(|| {
634                AnnotationValidationError::ColumnNotFound {
635                    column: column_name.to_string(),
636                }
637            })?;
638            if cx.schema.timestamp_index() == Some(column_index) {
639                return Err(AnnotationValidationError::TimeIndexConflict);
640            }
641            Ok(column_name.to_string())
642        }
643    }
644}
645
646/// CREATE-side entry: validates every annotation option present in `options`
647/// and writes normalized values back in place.
648pub fn validate_and_normalize_annotation_options(
649    options: &mut TableOptions,
650    schema: &Schema,
651    partition_key_indices: &[usize],
652) -> std::result::Result<(), AnnotationValidationError> {
653    let cx = AnnotationContext {
654        data_model: options.data_model(),
655        schema,
656        partition_key_indices,
657    };
658    let mut normalized = Vec::new();
659    for (key, value) in &options.extra_options {
660        let Some(family) = AnnotationFamily::of_key(key) else {
661            continue;
662        };
663        let checked = validate_and_normalize_annotation(family, &cx, key, value)?;
664        if checked != *value {
665            normalized.push((key.clone(), checked));
666        }
667    }
668    for (key, value) in normalized {
669        options.extra_options.insert(key, value);
670    }
671    Ok(())
672}
673
674#[derive(Debug, Clone, Serialize, Deserialize)]
675pub enum AlterKind {
676    AddColumns {
677        columns: Vec<AddColumnRequest>,
678    },
679    DropColumns {
680        names: Vec<String>,
681    },
682    ModifyColumnTypes {
683        columns: Vec<ModifyColumnTypeRequest>,
684    },
685    SetJsonSettings {
686        request: SetJsonSettingsRequest,
687    },
688    RenameTable {
689        new_table_name: String,
690    },
691    SetTableOptions {
692        options: Vec<SetRegionOption>,
693    },
694    UnsetTableOptions {
695        keys: Vec<UnsetRegionOption>,
696    },
697    SetAnnotations {
698        family: AnnotationFamily,
699        options: Vec<(String, String)>,
700    },
701    UnsetAnnotations {
702        family: AnnotationFamily,
703        keys: Vec<String>,
704    },
705    SetIndexes {
706        options: Vec<SetIndexOption>,
707    },
708    UnsetIndexes {
709        options: Vec<UnsetIndexOption>,
710    },
711    DropDefaults {
712        names: Vec<String>,
713    },
714    SetDefaults {
715        defaults: Vec<SetDefaultRequest>,
716    },
717}
718
719#[derive(Debug, Clone, Serialize, Deserialize)]
720pub struct SetDefaultRequest {
721    pub column_name: String,
722    pub default_constraint: Option<ColumnDefaultConstraint>,
723}
724
725#[derive(Debug, Clone, Serialize, Deserialize)]
726pub enum SetIndexOption {
727    Fulltext {
728        column_name: String,
729        options: FulltextOptions,
730    },
731    Inverted {
732        column_name: String,
733    },
734    Skipping {
735        column_name: String,
736        options: SkippingIndexOptions,
737    },
738}
739
740impl SetIndexOption {
741    /// Returns the column name of the index option.
742    pub fn column_name(&self) -> &str {
743        match self {
744            SetIndexOption::Fulltext { column_name, .. } => column_name,
745            SetIndexOption::Inverted { column_name, .. } => column_name,
746            SetIndexOption::Skipping { column_name, .. } => column_name,
747        }
748    }
749}
750
751#[derive(Debug, Clone, Serialize, Deserialize)]
752pub enum UnsetIndexOption {
753    Fulltext { column_name: String },
754    Inverted { column_name: String },
755    Skipping { column_name: String },
756}
757
758impl UnsetIndexOption {
759    /// Returns the column name of the index option.
760    pub fn column_name(&self) -> &str {
761        match self {
762            UnsetIndexOption::Fulltext { column_name, .. } => column_name,
763            UnsetIndexOption::Inverted { column_name, .. } => column_name,
764            UnsetIndexOption::Skipping { column_name, .. } => column_name,
765        }
766    }
767}
768
769#[derive(Debug)]
770pub struct InsertRequest {
771    pub catalog_name: String,
772    pub schema_name: String,
773    pub table_name: String,
774    pub columns_values: HashMap<String, VectorRef>,
775    /// Whether this insert should skip WAL.
776    pub skip_wal: bool,
777}
778
779/// Delete (by primary key) request
780#[derive(Debug)]
781pub struct DeleteRequest {
782    pub catalog_name: String,
783    pub schema_name: String,
784    pub table_name: String,
785    /// Values of each column in this table's primary key and time index.
786    ///
787    /// The key is the column name, and the value is the column value.
788    pub key_column_values: HashMap<String, VectorRef>,
789}
790
791#[derive(Debug)]
792pub enum CopyDirection {
793    Export,
794    Import,
795}
796
797/// Copy table request
798#[derive(Debug)]
799pub struct CopyTableRequest {
800    pub catalog_name: String,
801    pub schema_name: String,
802    pub table_name: String,
803    pub location: String,
804    pub with: HashMap<String, String>,
805    pub connection: HashMap<String, String>,
806    pub pattern: Option<String>,
807    pub direction: CopyDirection,
808    pub timestamp_range: Option<TimestampRange>,
809    pub limit: Option<u64>,
810}
811
812#[derive(Debug, Clone, Default)]
813pub struct FlushTableRequest {
814    pub catalog_name: String,
815    pub schema_name: String,
816    pub table_name: String,
817}
818
819#[derive(Debug, Clone, Default)]
820pub struct BuildIndexTableRequest {
821    /// The index build mode. Absent options select SST indexes.
822    pub options: Option<build_index_request::Options>,
823    pub catalog_name: String,
824    pub schema_name: String,
825    pub table_name: String,
826}
827
828#[derive(Debug, Clone, PartialEq)]
829pub struct CompactTableRequest {
830    pub catalog_name: String,
831    pub schema_name: String,
832    pub table_name: String,
833    pub compact_options: compact_request::Options,
834    pub parallelism: u32,
835    pub time_range: Option<TimestampRange>,
836}
837
838impl Default for CompactTableRequest {
839    fn default() -> Self {
840        Self {
841            catalog_name: Default::default(),
842            schema_name: Default::default(),
843            table_name: Default::default(),
844            compact_options: compact_request::Options::Regular(Default::default()),
845            parallelism: 1,
846            time_range: None,
847        }
848    }
849}
850
851#[derive(Debug, Clone, Default, Deserialize, Serialize)]
852pub struct CopyDatabaseRequest {
853    pub catalog_name: String,
854    pub schema_name: String,
855    pub location: String,
856    pub with: HashMap<String, String>,
857    pub connection: HashMap<String, String>,
858    pub time_range: Option<TimestampRange>,
859}
860
861#[derive(Debug, Clone, Default, Deserialize, Serialize)]
862pub struct CopyQueryToRequest {
863    pub location: String,
864    pub with: HashMap<String, String>,
865    pub connection: HashMap<String, String>,
866}
867
868#[cfg(test)]
869mod tests {
870    use std::time::Duration;
871
872    use common_error::ext::ErrorExt;
873    use common_error::status_code::StatusCode;
874
875    use super::*;
876
877    #[test]
878    fn test_validate_annotation_keys() {
879        let column = REPARTITION_COLUMN_HINT_KEY;
880        let count = REPARTITION_PARTITION_NUM_HINT_KEY;
881        for (keys, expected) in [
882            (vec![], None),
883            (vec![TTL_KEY, TTL_KEY], None),
884            (vec!["repartition.unknown.hint"], None),
885            (vec![column], Some(AnnotationFamily::RepartitionHint)),
886            (vec![count], Some(AnnotationFamily::RepartitionHint)),
887            (vec![column, count], Some(AnnotationFamily::RepartitionHint)),
888            (vec![count, column], Some(AnnotationFamily::RepartitionHint)),
889            (
890                vec!["greptime.semantic.source", "greptime.semantic.source"],
891                Some(AnnotationFamily::Semantic),
892            ),
893        ] {
894            assert_eq!(validate_annotation_keys(keys), Ok(expected));
895        }
896        for keys in [
897            vec![column, column],
898            vec![count, count],
899            vec![column, count, column],
900        ] {
901            assert_eq!(
902                validate_annotation_keys(keys),
903                Err(AnnotationKeyError::DuplicateKey)
904            );
905        }
906        for keys in [
907            vec![column, TTL_KEY],
908            vec![TTL_KEY, count],
909            vec![column, "greptime.semantic.source"],
910        ] {
911            assert_eq!(
912                validate_annotation_keys(keys),
913                Err(AnnotationKeyError::MixedFamilies {
914                    family: AnnotationFamily::RepartitionHint,
915                })
916            );
917        }
918    }
919
920    #[test]
921    fn test_validate_table_option() {
922        assert!(validate_table_option(FILE_TABLE_LOCATION_KEY));
923        assert!(validate_table_option(FILE_TABLE_FORMAT_KEY));
924        assert!(validate_table_option(FILE_TABLE_PATTERN_KEY));
925        assert!(validate_table_option(TTL_KEY));
926        assert!(validate_table_option(WRITE_BUFFER_SIZE_KEY));
927        assert!(validate_table_option(STORAGE_KEY));
928        assert!(validate_table_option(MEMTABLE_BULK_MERGE_THRESHOLD));
929        assert!(validate_table_option(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING));
930        assert!(validate_table_option(REPARTITION_COLUMN_HINT_KEY));
931        assert!(validate_table_option(REPARTITION_PARTITION_NUM_HINT_KEY));
932        assert_eq!(AnnotationFamily::of_key("repartition.unknown.hint"), None);
933        assert!(!validate_table_option("foo"));
934
935        // Only whitelisted semantic keys are accepted.
936        assert!(validate_table_option(SEMANTIC_SIGNAL_TYPE));
937        assert!(validate_table_option(SEMANTIC_METRIC_TYPE));
938        // Unknown semantic key, near-miss, and the internal transport key are rejected.
939        assert!(!validate_table_option("greptime.semantic.future.key"));
940        assert!(!validate_table_option("greptime.semanticx"));
941        assert!(!validate_table_option(SEMANTIC_PER_TABLE_INDEX_KEY));
942    }
943
944    #[test]
945    fn test_validate_database_option() {
946        assert!(validate_database_option(MEMTABLE_TYPE));
947        assert!(validate_database_option(MEMTABLE_BULK_MERGE_THRESHOLD));
948        assert!(validate_database_option(MEMTABLE_BULK_ENCODE_ROW_THRESHOLD));
949        assert!(validate_database_option(
950            MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD
951        ));
952        assert!(validate_database_option(MEMTABLE_BULK_MAX_MERGE_GROUPS));
953        assert!(validate_database_option(
954            "compaction.twcs.active_window.trigger_file_num"
955        ));
956        assert!(validate_database_option(
957            "compaction.twcs.active_window.l1_merge_trigger"
958        ));
959        assert!(validate_database_option(
960            "compaction.twcs.inactive_window.trigger_file_num"
961        ));
962        assert!(validate_database_option(
963            "compaction.twcs.inactive_window.l1_merge_trigger"
964        ));
965        assert!(validate_database_option(INGEST_ROWS_RATE_LIMIT_KEY));
966        assert!(validate_database_option("ingest_rows_rate_limit"));
967        assert!(!validate_database_option("foo"));
968    }
969
970    #[test]
971    fn test_parse_float_field_encoding_option() {
972        let options = TableOptions::try_from_iter([(
973            EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING,
974            "byte_stream_split",
975        )])
976        .unwrap();
977        assert_eq!(
978            options
979                .extra_options
980                .get(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING),
981            Some(&"byte_stream_split".to_string())
982        );
983        assert!(
984            TableOptions::try_from_iter([(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, "invalid",)])
985                .is_err()
986        );
987    }
988
989    #[test]
990    fn test_database_trigger_value_boundaries() {
991        let maximum = usize::MAX.to_string();
992        let overflow = format!("{maximum}0");
993        for (key, minimum) in [
994            (TWCS_TRIGGER_FILE_NUM, 0),
995            (TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM, 0),
996            (TWCS_INACTIVE_WINDOW_TRIGGER_FILE_NUM, 0),
997            (TWCS_ACTIVE_WINDOW_L1_MERGE_TRIGGER, 2),
998            (TWCS_INACTIVE_WINDOW_L1_MERGE_TRIGGER, 2),
999        ] {
1000            for invalid in [
1001                None,
1002                Some(""),
1003                Some("invalid"),
1004                Some("-1"),
1005                Some(overflow.as_str()),
1006            ] {
1007                assert!(
1008                    validate_database_option_value(key, invalid).is_err(),
1009                    "{key}: {invalid:?}"
1010                );
1011            }
1012            for valid in ["2", maximum.as_str()] {
1013                assert!(
1014                    validate_database_option_value(key, Some(valid)).is_ok(),
1015                    "{key}: {valid}"
1016                );
1017            }
1018            for boundary in ["0", "1"] {
1019                assert_eq!(
1020                    validate_database_option_value(key, Some(boundary)).is_ok(),
1021                    minimum == 0,
1022                    "{key}: {boundary}"
1023                );
1024            }
1025        }
1026    }
1027
1028    #[test]
1029    fn test_database_ingest_rate_limit_value_boundaries() {
1030        for invalid in [
1031            None,
1032            Some(""),
1033            Some("abc"),
1034            Some("1000/s"),
1035            Some("-1"),
1036            Some("1.5"),
1037            Some("18446744073709551616"),
1038        ] {
1039            assert!(validate_database_option_value(INGEST_ROWS_RATE_LIMIT_KEY, invalid).is_err());
1040        }
1041        let maximum = u64::MAX.to_string();
1042        for valid in ["0", "1", maximum.as_str()] {
1043            assert!(
1044                validate_database_option_value(INGEST_ROWS_RATE_LIMIT_KEY, Some(valid)).is_ok()
1045            );
1046        }
1047    }
1048
1049    #[test]
1050    fn test_serialize_table_options() {
1051        let options = TableOptions {
1052            write_buffer_size: None,
1053            ttl: Some(Duration::from_secs(1000).into()),
1054            extra_options: HashMap::new(),
1055            skip_wal: false,
1056        };
1057        let serialized = serde_json::to_string(&options).unwrap();
1058        let deserialized: TableOptions = serde_json::from_str(&serialized).unwrap();
1059        assert_eq!(options, deserialized);
1060    }
1061
1062    #[test]
1063    fn test_convert_hashmap_between_table_options() {
1064        let options = TableOptions {
1065            write_buffer_size: Some(ReadableSize::mb(128)),
1066            ttl: Some(Duration::from_secs(1000).into()),
1067            extra_options: HashMap::new(),
1068            skip_wal: false,
1069        };
1070        let serialized_map = HashMap::from(&options);
1071        let serialized = TableOptions::try_from_iter(&serialized_map).unwrap();
1072        assert_eq!(options, serialized);
1073
1074        let options = TableOptions {
1075            write_buffer_size: None,
1076            ttl: None,
1077            extra_options: HashMap::from([(SKIP_WAL_KEY.to_string(), true.to_string())]),
1078            skip_wal: true,
1079        };
1080        let serialized_map = HashMap::from(&options);
1081        assert_eq!(
1082            Some("true"),
1083            serialized_map.get(SKIP_WAL_KEY).map(String::as_str)
1084        );
1085        let serialized = TableOptions::try_from_iter(&serialized_map).unwrap();
1086        assert_eq!(options, serialized);
1087
1088        let options = TableOptions {
1089            write_buffer_size: None,
1090            ttl: Default::default(),
1091            extra_options: HashMap::new(),
1092            skip_wal: false,
1093        };
1094        let serialized_map = HashMap::from(&options);
1095        let serialized = TableOptions::try_from_iter(&serialized_map).unwrap();
1096        assert_eq!(options, serialized);
1097
1098        let options = TableOptions {
1099            write_buffer_size: Some(ReadableSize::mb(128)),
1100            ttl: Some(Duration::from_secs(1000).into()),
1101            extra_options: HashMap::from([("a".to_string(), "A".to_string())]),
1102            skip_wal: false,
1103        };
1104        let serialized_map = HashMap::from(&options);
1105        let serialized = TableOptions::try_from_iter(&serialized_map).unwrap();
1106        assert_eq!(options, serialized);
1107
1108        let options = TableOptions {
1109            extra_options: HashMap::from([(SKIP_WAL_KEY.to_string(), false.to_string())]),
1110            skip_wal: false,
1111            ..Default::default()
1112        };
1113        let serialized_map = HashMap::from(&options);
1114        let serialized = TableOptions::try_from_iter(&serialized_map).unwrap();
1115        assert_eq!(options, serialized);
1116    }
1117
1118    #[test]
1119    fn test_table_options_normalizes_twcs_trigger_aliases() {
1120        for options in [
1121            vec![(TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM, "4")],
1122            vec![
1123                (TWCS_TRIGGER_FILE_NUM, "4"),
1124                (TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM, "4"),
1125            ],
1126        ] {
1127            let table_options = TableOptions::try_from_iter(options).unwrap();
1128            assert_eq!(
1129                HashMap::from([(TWCS_TRIGGER_FILE_NUM.to_string(), "4".to_string())]),
1130                table_options.extra_options
1131            );
1132        }
1133    }
1134
1135    #[test]
1136    fn test_table_options_rejects_conflicting_twcs_trigger_aliases() {
1137        let error = TableOptions::try_from_iter([
1138            (TWCS_TRIGGER_FILE_NUM, "4"),
1139            (TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM, "8"),
1140        ])
1141        .unwrap_err();
1142        assert_eq!(StatusCode::InvalidArguments, error.status_code());
1143        assert_eq!(
1144            "Conflicting table options: compaction.twcs.trigger_file_num=4 and compaction.twcs.active_window.trigger_file_num=8",
1145            error.to_string()
1146        );
1147    }
1148
1149    #[test]
1150    fn test_table_options_to_string() {
1151        let options = TableOptions {
1152            write_buffer_size: Some(ReadableSize::mb(128)),
1153            ttl: Some(Duration::from_secs(1000).into()),
1154            extra_options: HashMap::new(),
1155            skip_wal: false,
1156        };
1157
1158        assert_eq!(
1159            "write_buffer_size=128.0MiB ttl=16m 40s",
1160            options.to_string()
1161        );
1162
1163        let options = TableOptions {
1164            write_buffer_size: Some(ReadableSize::mb(128)),
1165            ttl: Some(Duration::from_secs(1000).into()),
1166            extra_options: HashMap::from([("a".to_string(), "A".to_string())]),
1167            skip_wal: false,
1168        };
1169
1170        assert_eq!(
1171            "write_buffer_size=128.0MiB ttl=16m 40s a=A",
1172            options.to_string()
1173        );
1174
1175        let options = TableOptions {
1176            write_buffer_size: Some(ReadableSize::mb(128)),
1177            ttl: Some(Duration::from_secs(1000).into()),
1178            extra_options: HashMap::new(),
1179            skip_wal: true,
1180        };
1181        assert_eq!(
1182            "write_buffer_size=128.0MiB ttl=16m 40s skip_wal=true",
1183            options.to_string()
1184        );
1185
1186        let options = TableOptions {
1187            write_buffer_size: None,
1188            ttl: None,
1189            extra_options: HashMap::from([(SKIP_WAL_KEY.to_string(), "false".to_string())]),
1190            skip_wal: false,
1191        };
1192        assert_eq!("skip_wal=false", options.to_string());
1193    }
1194}
table/requests.rs

table/
requests.rs
