table/
requests.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Table and TableEngine requests
16
17use std::collections::{HashMap, HashSet};
18use std::fmt;
19use std::str::FromStr;
20
21use common_base::readable_size::ReadableSize;
22use common_datasource::object_store::oss::is_supported_in_oss;
23use common_datasource::object_store::s3::is_supported_in_s3;
24use common_query::AddColumnLocation;
25use common_time::TimeToLive;
26use common_time::range::TimestampRange;
27use datatypes::data_type::ConcreteDataType;
28use datatypes::prelude::VectorRef;
29use datatypes::schema::{
30    ColumnDefaultConstraint, ColumnSchema, FulltextOptions, SkippingIndexOptions,
31};
32use greptime_proto::v1::region::compact_request;
33use once_cell::sync::Lazy;
34use serde::{Deserialize, Serialize};
35use store_api::metric_engine_consts::{
36    LOGICAL_TABLE_METADATA_KEY, PHYSICAL_TABLE_METADATA_KEY, is_metric_engine_option_key,
37};
38use store_api::mito_engine_options::{
39    APPEND_MODE_KEY, COMPACTION_TYPE, MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD,
40    MEMTABLE_BULK_ENCODE_ROW_THRESHOLD, MEMTABLE_BULK_MAX_MERGE_GROUPS,
41    MEMTABLE_BULK_MERGE_THRESHOLD, MEMTABLE_TYPE, MERGE_MODE_KEY, SST_FORMAT_KEY,
42    TWCS_FALLBACK_TO_LOCAL, TWCS_MAX_OUTPUT_FILE_SIZE, TWCS_TIME_WINDOW, TWCS_TRIGGER_FILE_NUM,
43    is_mito_engine_option_key,
44};
45use store_api::region_request::{SetRegionOption, UnsetRegionOption};
46
47use crate::error::{ParseTableOptionSnafu, Result};
48use crate::metadata::{TableId, TableVersion};
49use crate::table_reference::TableReference;
50
51mod semantic;
52pub use semantic::*;
53
54pub const FILE_TABLE_META_KEY: &str = "__private.file_table_meta";
55pub const FILE_TABLE_LOCATION_KEY: &str = "location";
56pub const FILE_TABLE_PATTERN_KEY: &str = "pattern";
57pub const FILE_TABLE_FORMAT_KEY: &str = "format";
58
59pub const TABLE_DATA_MODEL: &str = "table_data_model";
60pub const TABLE_DATA_MODEL_TRACE_V1: &str = "greptime_trace_v1";
61
62pub const OTLP_METRIC_COMPAT_KEY: &str = "otlp_metric_compat";
63pub const OTLP_METRIC_COMPAT_PROM: &str = "prom";
64
65pub const VALID_TABLE_OPTION_KEYS: [&str; 13] = [
66    // common keys:
67    WRITE_BUFFER_SIZE_KEY,
68    TTL_KEY,
69    STORAGE_KEY,
70    COMMENT_KEY,
71    SKIP_WAL_KEY,
72    SST_FORMAT_KEY,
73    // file engine keys:
74    FILE_TABLE_LOCATION_KEY,
75    FILE_TABLE_FORMAT_KEY,
76    FILE_TABLE_PATTERN_KEY,
77    // metric engine keys:
78    PHYSICAL_TABLE_METADATA_KEY,
79    LOGICAL_TABLE_METADATA_KEY,
80    // table model info
81    TABLE_DATA_MODEL,
82    OTLP_METRIC_COMPAT_KEY,
83];
84
85pub const DDL_TIMEOUT: &str = "timeout";
86pub const DDL_WAIT: &str = "wait";
87
88pub const VALID_DDL_OPTION_KEYS: [&str; 2] = [DDL_TIMEOUT, DDL_WAIT];
89
90// Valid option keys when creating a db.
91static VALID_DB_OPT_KEYS: Lazy<HashSet<&str>> = Lazy::new(|| {
92    let mut set = HashSet::new();
93    set.insert(TTL_KEY);
94    set.insert(STORAGE_KEY);
95    set.insert(MEMTABLE_TYPE);
96    set.insert(MEMTABLE_BULK_MERGE_THRESHOLD);
97    set.insert(MEMTABLE_BULK_ENCODE_ROW_THRESHOLD);
98    set.insert(MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD);
99    set.insert(MEMTABLE_BULK_MAX_MERGE_GROUPS);
100    set.insert(APPEND_MODE_KEY);
101    set.insert(MERGE_MODE_KEY);
102    set.insert(SKIP_WAL_KEY);
103    set.insert(COMPACTION_TYPE);
104    set.insert(TWCS_FALLBACK_TO_LOCAL);
105    set.insert(TWCS_TIME_WINDOW);
106    set.insert(TWCS_TRIGGER_FILE_NUM);
107    set.insert(TWCS_MAX_OUTPUT_FILE_SIZE);
108    set.insert(SST_FORMAT_KEY);
109    set
110});
111
112/// Returns true if the `key` is a valid key for database.
113pub fn validate_database_option(key: &str) -> bool {
114    VALID_DB_OPT_KEYS.contains(&key)
115}
116
117/// Returns true if the `key` is a valid key for any engine or storage.
118pub fn validate_table_option(key: &str) -> bool {
119    if is_supported_in_s3(key) {
120        return true;
121    }
122
123    if is_supported_in_oss(key) {
124        return true;
125    }
126
127    if is_mito_engine_option_key(key) {
128        return true;
129    }
130
131    if is_metric_engine_option_key(key) {
132        return true;
133    }
134
135    // Semantic-layer keys share a reserved prefix instead of a fixed allowlist so
136    // the vocabulary can grow without touching this gate. See `semantic` module.
137    if is_semantic_option_key(key) {
138        return true;
139    }
140
141    VALID_TABLE_OPTION_KEYS.contains(&key) || VALID_DDL_OPTION_KEYS.contains(&key)
142}
143
144#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
145#[serde(default)]
146pub struct TableOptions {
147    /// Memtable size of memtable.
148    pub write_buffer_size: Option<ReadableSize>,
149    /// Time-to-live of table. Expired data will be automatically purged.
150    pub ttl: Option<TimeToLive>,
151    /// Skip wal write for this table.
152    pub skip_wal: bool,
153    /// Extra options that may not applicable to all table engines.
154    pub extra_options: HashMap<String, String>,
155}
156
157pub const WRITE_BUFFER_SIZE_KEY: &str = "write_buffer_size";
158pub const TTL_KEY: &str = store_api::mito_engine_options::TTL_KEY;
159pub const STORAGE_KEY: &str = "storage";
160pub const COMMENT_KEY: &str = "comment";
161pub const AUTO_CREATE_TABLE_KEY: &str = "auto_create_table";
162pub const SKIP_WAL_KEY: &str = store_api::mito_engine_options::SKIP_WAL_KEY;
163pub const TRACE_TABLE_PARTITIONS_HINT_KEY: &str = "trace_table_partitions";
164
165impl TableOptions {
166    pub fn try_from_iter<T: ToString, U: IntoIterator<Item = (T, T)>>(
167        iter: U,
168    ) -> Result<TableOptions> {
169        let mut options = TableOptions::default();
170
171        let kvs: HashMap<String, String> = iter
172            .into_iter()
173            .map(|(k, v)| (k.to_string(), v.to_string()))
174            .collect();
175
176        if let Some(write_buffer_size) = kvs.get(WRITE_BUFFER_SIZE_KEY) {
177            let size = ReadableSize::from_str(write_buffer_size).map_err(|_| {
178                ParseTableOptionSnafu {
179                    key: WRITE_BUFFER_SIZE_KEY,
180                    value: write_buffer_size,
181                }
182                .build()
183            })?;
184            options.write_buffer_size = Some(size)
185        }
186
187        if let Some(ttl) = kvs.get(TTL_KEY) {
188            let ttl_value = TimeToLive::from_humantime_or_str(ttl).map_err(|_| {
189                ParseTableOptionSnafu {
190                    key: TTL_KEY,
191                    value: ttl,
192                }
193                .build()
194            })?;
195            options.ttl = Some(ttl_value);
196        }
197
198        if let Some(skip_wal) = kvs.get(SKIP_WAL_KEY) {
199            options.skip_wal = skip_wal.parse().map_err(|_| {
200                ParseTableOptionSnafu {
201                    key: SKIP_WAL_KEY,
202                    value: skip_wal,
203                }
204                .build()
205            })?;
206        }
207
208        options.extra_options = HashMap::from_iter(
209            kvs.into_iter()
210                .filter(|(k, _)| k != WRITE_BUFFER_SIZE_KEY && k != TTL_KEY),
211        );
212
213        Ok(options)
214    }
215}
216
217impl fmt::Display for TableOptions {
218    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
219        let mut key_vals = vec![];
220        if let Some(size) = self.write_buffer_size {
221            key_vals.push(format!("{}={}", WRITE_BUFFER_SIZE_KEY, size));
222        }
223
224        if let Some(ttl) = self.ttl.map(|ttl| ttl.to_string()) {
225            key_vals.push(format!("{}={}", TTL_KEY, ttl));
226        }
227
228        if self.skip_wal {
229            key_vals.push(format!("{}={}", SKIP_WAL_KEY, self.skip_wal));
230        }
231
232        for (k, v) in &self.extra_options {
233            key_vals.push(format!("{}={}", k, v));
234        }
235
236        write!(f, "{}", key_vals.join(" "))
237    }
238}
239
240impl From<&TableOptions> for HashMap<String, String> {
241    fn from(opts: &TableOptions) -> Self {
242        let mut res = HashMap::with_capacity(2 + opts.extra_options.len());
243        if let Some(write_buffer_size) = opts.write_buffer_size {
244            let _ = res.insert(
245                WRITE_BUFFER_SIZE_KEY.to_string(),
246                write_buffer_size.to_string(),
247            );
248        }
249        if let Some(ttl_str) = opts.ttl.map(|ttl| ttl.to_string()) {
250            let _ = res.insert(TTL_KEY.to_string(), ttl_str);
251        }
252        res.extend(
253            opts.extra_options
254                .iter()
255                .map(|(k, v)| (k.clone(), v.clone())),
256        );
257        res
258    }
259}
260
261/// Alter table request
262#[derive(Debug, Clone, Serialize, Deserialize)]
263pub struct AlterTableRequest {
264    pub catalog_name: String,
265    pub schema_name: String,
266    pub table_name: String,
267    pub table_id: TableId,
268    pub alter_kind: AlterKind,
269    // None in standalone.
270    pub table_version: Option<TableVersion>,
271}
272
273/// Add column request
274#[derive(Debug, Clone, Serialize, Deserialize)]
275pub struct AddColumnRequest {
276    pub column_schema: ColumnSchema,
277    pub is_key: bool,
278    pub location: Option<AddColumnLocation>,
279    /// Add column if not exists.
280    pub add_if_not_exists: bool,
281}
282
283/// Change column datatype request
284#[derive(Debug, Clone, Serialize, Deserialize)]
285pub struct ModifyColumnTypeRequest {
286    pub column_name: String,
287    pub target_type: ConcreteDataType,
288}
289
290#[derive(Debug, Clone, Serialize, Deserialize)]
291pub enum AlterKind {
292    AddColumns {
293        columns: Vec<AddColumnRequest>,
294    },
295    DropColumns {
296        names: Vec<String>,
297    },
298    ModifyColumnTypes {
299        columns: Vec<ModifyColumnTypeRequest>,
300    },
301    RenameTable {
302        new_table_name: String,
303    },
304    SetTableOptions {
305        options: Vec<SetRegionOption>,
306    },
307    UnsetTableOptions {
308        keys: Vec<UnsetRegionOption>,
309    },
310    SetIndexes {
311        options: Vec<SetIndexOption>,
312    },
313    UnsetIndexes {
314        options: Vec<UnsetIndexOption>,
315    },
316    DropDefaults {
317        names: Vec<String>,
318    },
319    SetDefaults {
320        defaults: Vec<SetDefaultRequest>,
321    },
322}
323
324#[derive(Debug, Clone, Serialize, Deserialize)]
325pub struct SetDefaultRequest {
326    pub column_name: String,
327    pub default_constraint: Option<ColumnDefaultConstraint>,
328}
329
330#[derive(Debug, Clone, Serialize, Deserialize)]
331pub enum SetIndexOption {
332    Fulltext {
333        column_name: String,
334        options: FulltextOptions,
335    },
336    Inverted {
337        column_name: String,
338    },
339    Skipping {
340        column_name: String,
341        options: SkippingIndexOptions,
342    },
343}
344
345impl SetIndexOption {
346    /// Returns the column name of the index option.
347    pub fn column_name(&self) -> &str {
348        match self {
349            SetIndexOption::Fulltext { column_name, .. } => column_name,
350            SetIndexOption::Inverted { column_name, .. } => column_name,
351            SetIndexOption::Skipping { column_name, .. } => column_name,
352        }
353    }
354}
355
356#[derive(Debug, Clone, Serialize, Deserialize)]
357pub enum UnsetIndexOption {
358    Fulltext { column_name: String },
359    Inverted { column_name: String },
360    Skipping { column_name: String },
361}
362
363impl UnsetIndexOption {
364    /// Returns the column name of the index option.
365    pub fn column_name(&self) -> &str {
366        match self {
367            UnsetIndexOption::Fulltext { column_name, .. } => column_name,
368            UnsetIndexOption::Inverted { column_name, .. } => column_name,
369            UnsetIndexOption::Skipping { column_name, .. } => column_name,
370        }
371    }
372}
373
374#[derive(Debug)]
375pub struct InsertRequest {
376    pub catalog_name: String,
377    pub schema_name: String,
378    pub table_name: String,
379    pub columns_values: HashMap<String, VectorRef>,
380}
381
382/// Delete (by primary key) request
383#[derive(Debug)]
384pub struct DeleteRequest {
385    pub catalog_name: String,
386    pub schema_name: String,
387    pub table_name: String,
388    /// Values of each column in this table's primary key and time index.
389    ///
390    /// The key is the column name, and the value is the column value.
391    pub key_column_values: HashMap<String, VectorRef>,
392}
393
394#[derive(Debug)]
395pub enum CopyDirection {
396    Export,
397    Import,
398}
399
400/// Copy table request
401#[derive(Debug)]
402pub struct CopyTableRequest {
403    pub catalog_name: String,
404    pub schema_name: String,
405    pub table_name: String,
406    pub location: String,
407    pub with: HashMap<String, String>,
408    pub connection: HashMap<String, String>,
409    pub pattern: Option<String>,
410    pub direction: CopyDirection,
411    pub timestamp_range: Option<TimestampRange>,
412    pub limit: Option<u64>,
413}
414
415#[derive(Debug, Clone, Default)]
416pub struct FlushTableRequest {
417    pub catalog_name: String,
418    pub schema_name: String,
419    pub table_name: String,
420}
421
422#[derive(Debug, Clone, Default)]
423pub struct BuildIndexTableRequest {
424    pub catalog_name: String,
425    pub schema_name: String,
426    pub table_name: String,
427}
428
429#[derive(Debug, Clone, PartialEq)]
430pub struct CompactTableRequest {
431    pub catalog_name: String,
432    pub schema_name: String,
433    pub table_name: String,
434    pub compact_options: compact_request::Options,
435    pub parallelism: u32,
436}
437
438impl Default for CompactTableRequest {
439    fn default() -> Self {
440        Self {
441            catalog_name: Default::default(),
442            schema_name: Default::default(),
443            table_name: Default::default(),
444            compact_options: compact_request::Options::Regular(Default::default()),
445            parallelism: 1,
446        }
447    }
448}
449
450/// Truncate table request
451#[derive(Debug, Clone, Serialize, Deserialize)]
452pub struct TruncateTableRequest {
453    pub catalog_name: String,
454    pub schema_name: String,
455    pub table_name: String,
456    pub table_id: TableId,
457}
458
459impl TruncateTableRequest {
460    pub fn table_ref(&self) -> TableReference<'_> {
461        TableReference {
462            catalog: &self.catalog_name,
463            schema: &self.schema_name,
464            table: &self.table_name,
465        }
466    }
467}
468
469#[derive(Debug, Clone, Default, Deserialize, Serialize)]
470pub struct CopyDatabaseRequest {
471    pub catalog_name: String,
472    pub schema_name: String,
473    pub location: String,
474    pub with: HashMap<String, String>,
475    pub connection: HashMap<String, String>,
476    pub time_range: Option<TimestampRange>,
477}
478
479#[derive(Debug, Clone, Default, Deserialize, Serialize)]
480pub struct CopyQueryToRequest {
481    pub location: String,
482    pub with: HashMap<String, String>,
483    pub connection: HashMap<String, String>,
484}
485
486#[cfg(test)]
487mod tests {
488    use std::time::Duration;
489
490    use super::*;
491
492    #[test]
493    fn test_validate_table_option() {
494        assert!(validate_table_option(FILE_TABLE_LOCATION_KEY));
495        assert!(validate_table_option(FILE_TABLE_FORMAT_KEY));
496        assert!(validate_table_option(FILE_TABLE_PATTERN_KEY));
497        assert!(validate_table_option(TTL_KEY));
498        assert!(validate_table_option(WRITE_BUFFER_SIZE_KEY));
499        assert!(validate_table_option(STORAGE_KEY));
500        assert!(validate_table_option(MEMTABLE_BULK_MERGE_THRESHOLD));
501        assert!(!validate_table_option("foo"));
502
503        // Only whitelisted semantic keys are accepted.
504        assert!(validate_table_option(SEMANTIC_SIGNAL_TYPE));
505        assert!(validate_table_option(SEMANTIC_METRIC_TYPE));
506        // Unknown semantic key, near-miss, and the internal transport key are rejected.
507        assert!(!validate_table_option("greptime.semantic.future.key"));
508        assert!(!validate_table_option("greptime.semanticx"));
509        assert!(!validate_table_option(SEMANTIC_PER_TABLE_INDEX_KEY));
510    }
511
512    #[test]
513    fn test_validate_database_option() {
514        assert!(validate_database_option(MEMTABLE_TYPE));
515        assert!(validate_database_option(MEMTABLE_BULK_MERGE_THRESHOLD));
516        assert!(validate_database_option(MEMTABLE_BULK_ENCODE_ROW_THRESHOLD));
517        assert!(validate_database_option(
518            MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD
519        ));
520        assert!(validate_database_option(MEMTABLE_BULK_MAX_MERGE_GROUPS));
521        assert!(!validate_database_option("foo"));
522    }
523
524    #[test]
525    fn test_serialize_table_options() {
526        let options = TableOptions {
527            write_buffer_size: None,
528            ttl: Some(Duration::from_secs(1000).into()),
529            extra_options: HashMap::new(),
530            skip_wal: false,
531        };
532        let serialized = serde_json::to_string(&options).unwrap();
533        let deserialized: TableOptions = serde_json::from_str(&serialized).unwrap();
534        assert_eq!(options, deserialized);
535    }
536
537    #[test]
538    fn test_convert_hashmap_between_table_options() {
539        let options = TableOptions {
540            write_buffer_size: Some(ReadableSize::mb(128)),
541            ttl: Some(Duration::from_secs(1000).into()),
542            extra_options: HashMap::new(),
543            skip_wal: false,
544        };
545        let serialized_map = HashMap::from(&options);
546        let serialized = TableOptions::try_from_iter(&serialized_map).unwrap();
547        assert_eq!(options, serialized);
548
549        let options = TableOptions {
550            write_buffer_size: None,
551            ttl: Default::default(),
552            extra_options: HashMap::new(),
553            skip_wal: false,
554        };
555        let serialized_map = HashMap::from(&options);
556        let serialized = TableOptions::try_from_iter(&serialized_map).unwrap();
557        assert_eq!(options, serialized);
558
559        let options = TableOptions {
560            write_buffer_size: Some(ReadableSize::mb(128)),
561            ttl: Some(Duration::from_secs(1000).into()),
562            extra_options: HashMap::from([("a".to_string(), "A".to_string())]),
563            skip_wal: false,
564        };
565        let serialized_map = HashMap::from(&options);
566        let serialized = TableOptions::try_from_iter(&serialized_map).unwrap();
567        assert_eq!(options, serialized);
568    }
569
570    #[test]
571    fn test_table_options_to_string() {
572        let options = TableOptions {
573            write_buffer_size: Some(ReadableSize::mb(128)),
574            ttl: Some(Duration::from_secs(1000).into()),
575            extra_options: HashMap::new(),
576            skip_wal: false,
577        };
578
579        assert_eq!(
580            "write_buffer_size=128.0MiB ttl=16m 40s",
581            options.to_string()
582        );
583
584        let options = TableOptions {
585            write_buffer_size: Some(ReadableSize::mb(128)),
586            ttl: Some(Duration::from_secs(1000).into()),
587            extra_options: HashMap::from([("a".to_string(), "A".to_string())]),
588            skip_wal: false,
589        };
590
591        assert_eq!(
592            "write_buffer_size=128.0MiB ttl=16m 40s a=A",
593            options.to_string()
594        );
595
596        let options = TableOptions {
597            write_buffer_size: Some(ReadableSize::mb(128)),
598            ttl: Some(Duration::from_secs(1000).into()),
599            extra_options: HashMap::new(),
600            skip_wal: true,
601        };
602        assert_eq!(
603            "write_buffer_size=128.0MiB ttl=16m 40s skip_wal=true",
604            options.to_string()
605        );
606    }
607}
table/requests.rs

table/
requests.rs
