table/
requests.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Table and TableEngine requests
16
17use std::collections::{HashMap, HashSet};
18use std::fmt;
19use std::str::FromStr;
20
21use common_base::readable_size::ReadableSize;
22use common_datasource::object_store::oss::is_supported_in_oss;
23use common_datasource::object_store::s3::is_supported_in_s3;
24use common_query::AddColumnLocation;
25use common_time::TimeToLive;
26use common_time::range::TimestampRange;
27use datatypes::data_type::ConcreteDataType;
28use datatypes::prelude::VectorRef;
29use datatypes::schema::{
30    ColumnDefaultConstraint, ColumnSchema, FulltextOptions, SkippingIndexOptions,
31};
32use greptime_proto::v1::region::compact_request;
33use once_cell::sync::Lazy;
34use serde::{Deserialize, Serialize};
35use store_api::metric_engine_consts::{
36    LOGICAL_TABLE_METADATA_KEY, PHYSICAL_TABLE_METADATA_KEY, is_metric_engine_option_key,
37};
38use store_api::mito_engine_options::{
39    APPEND_MODE_KEY, COMPACTION_TYPE, MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD,
40    MEMTABLE_BULK_ENCODE_ROW_THRESHOLD, MEMTABLE_BULK_MAX_MERGE_GROUPS,
41    MEMTABLE_BULK_MERGE_THRESHOLD, MEMTABLE_TYPE, MERGE_MODE_KEY, SST_FORMAT_KEY,
42    TWCS_FALLBACK_TO_LOCAL, TWCS_MAX_OUTPUT_FILE_SIZE, TWCS_TIME_WINDOW, TWCS_TRIGGER_FILE_NUM,
43    is_mito_engine_option_key,
44};
45use store_api::region_request::{SetRegionOption, UnsetRegionOption};
46
47use crate::error::{ParseTableOptionSnafu, Result};
48use crate::metadata::{TableId, TableVersion};
49use crate::table_reference::TableReference;
50
51pub const FILE_TABLE_META_KEY: &str = "__private.file_table_meta";
52pub const FILE_TABLE_LOCATION_KEY: &str = "location";
53pub const FILE_TABLE_PATTERN_KEY: &str = "pattern";
54pub const FILE_TABLE_FORMAT_KEY: &str = "format";
55
56pub const TABLE_DATA_MODEL: &str = "table_data_model";
57pub const TABLE_DATA_MODEL_TRACE_V1: &str = "greptime_trace_v1";
58
59pub const OTLP_METRIC_COMPAT_KEY: &str = "otlp_metric_compat";
60pub const OTLP_METRIC_COMPAT_PROM: &str = "prom";
61
62pub const VALID_TABLE_OPTION_KEYS: [&str; 13] = [
63    // common keys:
64    WRITE_BUFFER_SIZE_KEY,
65    TTL_KEY,
66    STORAGE_KEY,
67    COMMENT_KEY,
68    SKIP_WAL_KEY,
69    SST_FORMAT_KEY,
70    // file engine keys:
71    FILE_TABLE_LOCATION_KEY,
72    FILE_TABLE_FORMAT_KEY,
73    FILE_TABLE_PATTERN_KEY,
74    // metric engine keys:
75    PHYSICAL_TABLE_METADATA_KEY,
76    LOGICAL_TABLE_METADATA_KEY,
77    // table model info
78    TABLE_DATA_MODEL,
79    OTLP_METRIC_COMPAT_KEY,
80];
81
82pub const DDL_TIMEOUT: &str = "timeout";
83pub const DDL_WAIT: &str = "wait";
84
85pub const VALID_DDL_OPTION_KEYS: [&str; 2] = [DDL_TIMEOUT, DDL_WAIT];
86
87// Valid option keys when creating a db.
88static VALID_DB_OPT_KEYS: Lazy<HashSet<&str>> = Lazy::new(|| {
89    let mut set = HashSet::new();
90    set.insert(TTL_KEY);
91    set.insert(STORAGE_KEY);
92    set.insert(MEMTABLE_TYPE);
93    set.insert(MEMTABLE_BULK_MERGE_THRESHOLD);
94    set.insert(MEMTABLE_BULK_ENCODE_ROW_THRESHOLD);
95    set.insert(MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD);
96    set.insert(MEMTABLE_BULK_MAX_MERGE_GROUPS);
97    set.insert(APPEND_MODE_KEY);
98    set.insert(MERGE_MODE_KEY);
99    set.insert(SKIP_WAL_KEY);
100    set.insert(COMPACTION_TYPE);
101    set.insert(TWCS_FALLBACK_TO_LOCAL);
102    set.insert(TWCS_TIME_WINDOW);
103    set.insert(TWCS_TRIGGER_FILE_NUM);
104    set.insert(TWCS_MAX_OUTPUT_FILE_SIZE);
105    set.insert(SST_FORMAT_KEY);
106    set
107});
108
109/// Returns true if the `key` is a valid key for database.
110pub fn validate_database_option(key: &str) -> bool {
111    VALID_DB_OPT_KEYS.contains(&key)
112}
113
114/// Returns true if the `key` is a valid key for any engine or storage.
115pub fn validate_table_option(key: &str) -> bool {
116    if is_supported_in_s3(key) {
117        return true;
118    }
119
120    if is_supported_in_oss(key) {
121        return true;
122    }
123
124    if is_mito_engine_option_key(key) {
125        return true;
126    }
127
128    if is_metric_engine_option_key(key) {
129        return true;
130    }
131
132    VALID_TABLE_OPTION_KEYS.contains(&key) || VALID_DDL_OPTION_KEYS.contains(&key)
133}
134
135#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
136#[serde(default)]
137pub struct TableOptions {
138    /// Memtable size of memtable.
139    pub write_buffer_size: Option<ReadableSize>,
140    /// Time-to-live of table. Expired data will be automatically purged.
141    pub ttl: Option<TimeToLive>,
142    /// Skip wal write for this table.
143    pub skip_wal: bool,
144    /// Extra options that may not applicable to all table engines.
145    pub extra_options: HashMap<String, String>,
146}
147
148pub const WRITE_BUFFER_SIZE_KEY: &str = "write_buffer_size";
149pub const TTL_KEY: &str = store_api::mito_engine_options::TTL_KEY;
150pub const STORAGE_KEY: &str = "storage";
151pub const COMMENT_KEY: &str = "comment";
152pub const AUTO_CREATE_TABLE_KEY: &str = "auto_create_table";
153pub const SKIP_WAL_KEY: &str = store_api::mito_engine_options::SKIP_WAL_KEY;
154pub const TRACE_TABLE_PARTITIONS_HINT_KEY: &str = "trace_table_partitions";
155
156impl TableOptions {
157    pub fn try_from_iter<T: ToString, U: IntoIterator<Item = (T, T)>>(
158        iter: U,
159    ) -> Result<TableOptions> {
160        let mut options = TableOptions::default();
161
162        let kvs: HashMap<String, String> = iter
163            .into_iter()
164            .map(|(k, v)| (k.to_string(), v.to_string()))
165            .collect();
166
167        if let Some(write_buffer_size) = kvs.get(WRITE_BUFFER_SIZE_KEY) {
168            let size = ReadableSize::from_str(write_buffer_size).map_err(|_| {
169                ParseTableOptionSnafu {
170                    key: WRITE_BUFFER_SIZE_KEY,
171                    value: write_buffer_size,
172                }
173                .build()
174            })?;
175            options.write_buffer_size = Some(size)
176        }
177
178        if let Some(ttl) = kvs.get(TTL_KEY) {
179            let ttl_value = TimeToLive::from_humantime_or_str(ttl).map_err(|_| {
180                ParseTableOptionSnafu {
181                    key: TTL_KEY,
182                    value: ttl,
183                }
184                .build()
185            })?;
186            options.ttl = Some(ttl_value);
187        }
188
189        if let Some(skip_wal) = kvs.get(SKIP_WAL_KEY) {
190            options.skip_wal = skip_wal.parse().map_err(|_| {
191                ParseTableOptionSnafu {
192                    key: SKIP_WAL_KEY,
193                    value: skip_wal,
194                }
195                .build()
196            })?;
197        }
198
199        options.extra_options = HashMap::from_iter(
200            kvs.into_iter()
201                .filter(|(k, _)| k != WRITE_BUFFER_SIZE_KEY && k != TTL_KEY),
202        );
203
204        Ok(options)
205    }
206}
207
208impl fmt::Display for TableOptions {
209    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
210        let mut key_vals = vec![];
211        if let Some(size) = self.write_buffer_size {
212            key_vals.push(format!("{}={}", WRITE_BUFFER_SIZE_KEY, size));
213        }
214
215        if let Some(ttl) = self.ttl.map(|ttl| ttl.to_string()) {
216            key_vals.push(format!("{}={}", TTL_KEY, ttl));
217        }
218
219        if self.skip_wal {
220            key_vals.push(format!("{}={}", SKIP_WAL_KEY, self.skip_wal));
221        }
222
223        for (k, v) in &self.extra_options {
224            key_vals.push(format!("{}={}", k, v));
225        }
226
227        write!(f, "{}", key_vals.join(" "))
228    }
229}
230
231impl From<&TableOptions> for HashMap<String, String> {
232    fn from(opts: &TableOptions) -> Self {
233        let mut res = HashMap::with_capacity(2 + opts.extra_options.len());
234        if let Some(write_buffer_size) = opts.write_buffer_size {
235            let _ = res.insert(
236                WRITE_BUFFER_SIZE_KEY.to_string(),
237                write_buffer_size.to_string(),
238            );
239        }
240        if let Some(ttl_str) = opts.ttl.map(|ttl| ttl.to_string()) {
241            let _ = res.insert(TTL_KEY.to_string(), ttl_str);
242        }
243        res.extend(
244            opts.extra_options
245                .iter()
246                .map(|(k, v)| (k.clone(), v.clone())),
247        );
248        res
249    }
250}
251
252/// Alter table request
253#[derive(Debug, Clone, Serialize, Deserialize)]
254pub struct AlterTableRequest {
255    pub catalog_name: String,
256    pub schema_name: String,
257    pub table_name: String,
258    pub table_id: TableId,
259    pub alter_kind: AlterKind,
260    // None in standalone.
261    pub table_version: Option<TableVersion>,
262}
263
264/// Add column request
265#[derive(Debug, Clone, Serialize, Deserialize)]
266pub struct AddColumnRequest {
267    pub column_schema: ColumnSchema,
268    pub is_key: bool,
269    pub location: Option<AddColumnLocation>,
270    /// Add column if not exists.
271    pub add_if_not_exists: bool,
272}
273
274/// Change column datatype request
275#[derive(Debug, Clone, Serialize, Deserialize)]
276pub struct ModifyColumnTypeRequest {
277    pub column_name: String,
278    pub target_type: ConcreteDataType,
279}
280
281#[derive(Debug, Clone, Serialize, Deserialize)]
282pub enum AlterKind {
283    AddColumns {
284        columns: Vec<AddColumnRequest>,
285    },
286    DropColumns {
287        names: Vec<String>,
288    },
289    ModifyColumnTypes {
290        columns: Vec<ModifyColumnTypeRequest>,
291    },
292    RenameTable {
293        new_table_name: String,
294    },
295    SetTableOptions {
296        options: Vec<SetRegionOption>,
297    },
298    UnsetTableOptions {
299        keys: Vec<UnsetRegionOption>,
300    },
301    SetIndexes {
302        options: Vec<SetIndexOption>,
303    },
304    UnsetIndexes {
305        options: Vec<UnsetIndexOption>,
306    },
307    DropDefaults {
308        names: Vec<String>,
309    },
310    SetDefaults {
311        defaults: Vec<SetDefaultRequest>,
312    },
313}
314
315#[derive(Debug, Clone, Serialize, Deserialize)]
316pub struct SetDefaultRequest {
317    pub column_name: String,
318    pub default_constraint: Option<ColumnDefaultConstraint>,
319}
320
321#[derive(Debug, Clone, Serialize, Deserialize)]
322pub enum SetIndexOption {
323    Fulltext {
324        column_name: String,
325        options: FulltextOptions,
326    },
327    Inverted {
328        column_name: String,
329    },
330    Skipping {
331        column_name: String,
332        options: SkippingIndexOptions,
333    },
334}
335
336impl SetIndexOption {
337    /// Returns the column name of the index option.
338    pub fn column_name(&self) -> &str {
339        match self {
340            SetIndexOption::Fulltext { column_name, .. } => column_name,
341            SetIndexOption::Inverted { column_name, .. } => column_name,
342            SetIndexOption::Skipping { column_name, .. } => column_name,
343        }
344    }
345}
346
347#[derive(Debug, Clone, Serialize, Deserialize)]
348pub enum UnsetIndexOption {
349    Fulltext { column_name: String },
350    Inverted { column_name: String },
351    Skipping { column_name: String },
352}
353
354impl UnsetIndexOption {
355    /// Returns the column name of the index option.
356    pub fn column_name(&self) -> &str {
357        match self {
358            UnsetIndexOption::Fulltext { column_name, .. } => column_name,
359            UnsetIndexOption::Inverted { column_name, .. } => column_name,
360            UnsetIndexOption::Skipping { column_name, .. } => column_name,
361        }
362    }
363}
364
365#[derive(Debug)]
366pub struct InsertRequest {
367    pub catalog_name: String,
368    pub schema_name: String,
369    pub table_name: String,
370    pub columns_values: HashMap<String, VectorRef>,
371}
372
373/// Delete (by primary key) request
374#[derive(Debug)]
375pub struct DeleteRequest {
376    pub catalog_name: String,
377    pub schema_name: String,
378    pub table_name: String,
379    /// Values of each column in this table's primary key and time index.
380    ///
381    /// The key is the column name, and the value is the column value.
382    pub key_column_values: HashMap<String, VectorRef>,
383}
384
385#[derive(Debug)]
386pub enum CopyDirection {
387    Export,
388    Import,
389}
390
391/// Copy table request
392#[derive(Debug)]
393pub struct CopyTableRequest {
394    pub catalog_name: String,
395    pub schema_name: String,
396    pub table_name: String,
397    pub location: String,
398    pub with: HashMap<String, String>,
399    pub connection: HashMap<String, String>,
400    pub pattern: Option<String>,
401    pub direction: CopyDirection,
402    pub timestamp_range: Option<TimestampRange>,
403    pub limit: Option<u64>,
404}
405
406#[derive(Debug, Clone, Default)]
407pub struct FlushTableRequest {
408    pub catalog_name: String,
409    pub schema_name: String,
410    pub table_name: String,
411}
412
413#[derive(Debug, Clone, Default)]
414pub struct BuildIndexTableRequest {
415    pub catalog_name: String,
416    pub schema_name: String,
417    pub table_name: String,
418}
419
420#[derive(Debug, Clone, PartialEq)]
421pub struct CompactTableRequest {
422    pub catalog_name: String,
423    pub schema_name: String,
424    pub table_name: String,
425    pub compact_options: compact_request::Options,
426    pub parallelism: u32,
427}
428
429impl Default for CompactTableRequest {
430    fn default() -> Self {
431        Self {
432            catalog_name: Default::default(),
433            schema_name: Default::default(),
434            table_name: Default::default(),
435            compact_options: compact_request::Options::Regular(Default::default()),
436            parallelism: 1,
437        }
438    }
439}
440
441/// Truncate table request
442#[derive(Debug, Clone, Serialize, Deserialize)]
443pub struct TruncateTableRequest {
444    pub catalog_name: String,
445    pub schema_name: String,
446    pub table_name: String,
447    pub table_id: TableId,
448}
449
450impl TruncateTableRequest {
451    pub fn table_ref(&self) -> TableReference<'_> {
452        TableReference {
453            catalog: &self.catalog_name,
454            schema: &self.schema_name,
455            table: &self.table_name,
456        }
457    }
458}
459
460#[derive(Debug, Clone, Default, Deserialize, Serialize)]
461pub struct CopyDatabaseRequest {
462    pub catalog_name: String,
463    pub schema_name: String,
464    pub location: String,
465    pub with: HashMap<String, String>,
466    pub connection: HashMap<String, String>,
467    pub time_range: Option<TimestampRange>,
468}
469
470#[derive(Debug, Clone, Default, Deserialize, Serialize)]
471pub struct CopyQueryToRequest {
472    pub location: String,
473    pub with: HashMap<String, String>,
474    pub connection: HashMap<String, String>,
475}
476
477#[cfg(test)]
478mod tests {
479    use std::time::Duration;
480
481    use super::*;
482
483    #[test]
484    fn test_validate_table_option() {
485        assert!(validate_table_option(FILE_TABLE_LOCATION_KEY));
486        assert!(validate_table_option(FILE_TABLE_FORMAT_KEY));
487        assert!(validate_table_option(FILE_TABLE_PATTERN_KEY));
488        assert!(validate_table_option(TTL_KEY));
489        assert!(validate_table_option(WRITE_BUFFER_SIZE_KEY));
490        assert!(validate_table_option(STORAGE_KEY));
491        assert!(validate_table_option(MEMTABLE_BULK_MERGE_THRESHOLD));
492        assert!(!validate_table_option("foo"));
493    }
494
495    #[test]
496    fn test_validate_database_option() {
497        assert!(validate_database_option(MEMTABLE_TYPE));
498        assert!(validate_database_option(MEMTABLE_BULK_MERGE_THRESHOLD));
499        assert!(validate_database_option(MEMTABLE_BULK_ENCODE_ROW_THRESHOLD));
500        assert!(validate_database_option(
501            MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD
502        ));
503        assert!(validate_database_option(MEMTABLE_BULK_MAX_MERGE_GROUPS));
504        assert!(!validate_database_option("foo"));
505    }
506
507    #[test]
508    fn test_serialize_table_options() {
509        let options = TableOptions {
510            write_buffer_size: None,
511            ttl: Some(Duration::from_secs(1000).into()),
512            extra_options: HashMap::new(),
513            skip_wal: false,
514        };
515        let serialized = serde_json::to_string(&options).unwrap();
516        let deserialized: TableOptions = serde_json::from_str(&serialized).unwrap();
517        assert_eq!(options, deserialized);
518    }
519
520    #[test]
521    fn test_convert_hashmap_between_table_options() {
522        let options = TableOptions {
523            write_buffer_size: Some(ReadableSize::mb(128)),
524            ttl: Some(Duration::from_secs(1000).into()),
525            extra_options: HashMap::new(),
526            skip_wal: false,
527        };
528        let serialized_map = HashMap::from(&options);
529        let serialized = TableOptions::try_from_iter(&serialized_map).unwrap();
530        assert_eq!(options, serialized);
531
532        let options = TableOptions {
533            write_buffer_size: None,
534            ttl: Default::default(),
535            extra_options: HashMap::new(),
536            skip_wal: false,
537        };
538        let serialized_map = HashMap::from(&options);
539        let serialized = TableOptions::try_from_iter(&serialized_map).unwrap();
540        assert_eq!(options, serialized);
541
542        let options = TableOptions {
543            write_buffer_size: Some(ReadableSize::mb(128)),
544            ttl: Some(Duration::from_secs(1000).into()),
545            extra_options: HashMap::from([("a".to_string(), "A".to_string())]),
546            skip_wal: false,
547        };
548        let serialized_map = HashMap::from(&options);
549        let serialized = TableOptions::try_from_iter(&serialized_map).unwrap();
550        assert_eq!(options, serialized);
551    }
552
553    #[test]
554    fn test_table_options_to_string() {
555        let options = TableOptions {
556            write_buffer_size: Some(ReadableSize::mb(128)),
557            ttl: Some(Duration::from_secs(1000).into()),
558            extra_options: HashMap::new(),
559            skip_wal: false,
560        };
561
562        assert_eq!(
563            "write_buffer_size=128.0MiB ttl=16m 40s",
564            options.to_string()
565        );
566
567        let options = TableOptions {
568            write_buffer_size: Some(ReadableSize::mb(128)),
569            ttl: Some(Duration::from_secs(1000).into()),
570            extra_options: HashMap::from([("a".to_string(), "A".to_string())]),
571            skip_wal: false,
572        };
573
574        assert_eq!(
575            "write_buffer_size=128.0MiB ttl=16m 40s a=A",
576            options.to_string()
577        );
578
579        let options = TableOptions {
580            write_buffer_size: Some(ReadableSize::mb(128)),
581            ttl: Some(Duration::from_secs(1000).into()),
582            extra_options: HashMap::new(),
583            skip_wal: true,
584        };
585        assert_eq!(
586            "write_buffer_size=128.0MiB ttl=16m 40s skip_wal=true",
587            options.to_string()
588        );
589    }
590}
table/requests.rs

table/
requests.rs
