Skip to main content

metric_engine/engine/
create.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15mod extract_new_columns;
16
17use std::collections::{HashMap, HashSet};
18
19use api::v1::SemanticType;
20use common_query::native_histogram::is_native_histogram_value_schema;
21use common_telemetry::info;
22use common_time::{FOREVER, Timestamp};
23use datatypes::data_type::ConcreteDataType;
24use datatypes::schema::{ColumnSchema, SkippingIndexOptions};
25use datatypes::value::Value;
26use mito2::engine::MITO_ENGINE_NAME;
27use snafu::{OptionExt, ResultExt, ensure};
28use store_api::metadata::ColumnMetadata;
29use store_api::metric_engine_consts::{
30    ALTER_PHYSICAL_EXTENSION_KEY, DATA_REGION_SUBDIR, DATA_SCHEMA_TABLE_ID_COLUMN_NAME,
31    DATA_SCHEMA_TSID_COLUMN_NAME, LOGICAL_TABLE_METADATA_KEY, METADATA_REGION_SUBDIR,
32    METADATA_SCHEMA_KEY_COLUMN_INDEX, METADATA_SCHEMA_KEY_COLUMN_NAME,
33    METADATA_SCHEMA_TIMESTAMP_COLUMN_INDEX, METADATA_SCHEMA_TIMESTAMP_COLUMN_NAME,
34    METADATA_SCHEMA_VALUE_COLUMN_INDEX, METADATA_SCHEMA_VALUE_COLUMN_NAME,
35    is_metric_engine_internal_column,
36};
37use store_api::mito_engine_options::{TTL_KEY, WAL_OPTIONS_KEY};
38use store_api::region_engine::RegionEngine;
39use store_api::region_request::{AffectedRows, PathType, RegionCreateRequest, RegionRequest};
40use store_api::storage::RegionId;
41use store_api::storage::consts::ReservedColumnId;
42
43use crate::engine::MetricEngineInner;
44use crate::engine::create::extract_new_columns::extract_new_columns;
45use crate::engine::options::{PhysicalRegionOptions, set_data_region_options};
46use crate::error::{
47    ColumnTypeMismatchSnafu, ConflictRegionOptionSnafu, CreateMitoRegionSnafu,
48    InternalColumnOccupiedSnafu, InvalidMetadataSnafu, MissingRegionOptionSnafu,
49    MultipleFieldColumnSnafu, NoFieldColumnSnafu, ParseRegionIdSnafu, PhysicalRegionNotFoundSnafu,
50    Result, SerializeColumnMetadataSnafu, UnexpectedRequestSnafu,
51};
52use crate::metrics::PHYSICAL_REGION_COUNT;
53use crate::utils::{
54    self, append_manifest_info, encode_manifest_info_to_extensions, to_data_region_id,
55    to_metadata_region_id,
56};
57
58const DEFAULT_TABLE_ID_SKIPPING_INDEX_GRANULARITY: u32 = 1024;
59const DEFAULT_TABLE_ID_SKIPPING_INDEX_FALSE_POSITIVE_RATE: f64 = 0.01;
60
61impl MetricEngineInner {
62    pub async fn create_regions(
63        &self,
64        mut requests: Vec<(RegionId, RegionCreateRequest)>,
65        extension_return_value: &mut HashMap<String, Vec<u8>>,
66    ) -> Result<AffectedRows> {
67        if requests.is_empty() {
68            return Ok(0);
69        }
70
71        for (_, request) in requests.iter() {
72            Self::verify_region_create_request(request)?;
73        }
74
75        let first_request = &requests.first().unwrap().1;
76        if first_request.is_physical_table() {
77            ensure!(
78                requests.len() == 1,
79                UnexpectedRequestSnafu {
80                    reason: "Physical table must be created with single request".to_string(),
81                }
82            );
83            let (region_id, request) = requests.pop().unwrap();
84            self.create_physical_region(region_id, request, extension_return_value)
85                .await?;
86
87            return Ok(0);
88        } else if first_request
89            .options
90            .contains_key(LOGICAL_TABLE_METADATA_KEY)
91        {
92            if requests.len() == 1 {
93                let request = &requests.first().unwrap().1;
94                let physical_region_id = parse_physical_region_id(request)?;
95                let mut manifest_infos = Vec::with_capacity(1);
96                self.create_logical_regions(physical_region_id, requests, extension_return_value)
97                    .await?;
98                append_manifest_info(&self.mito, physical_region_id, &mut manifest_infos);
99                encode_manifest_info_to_extensions(&manifest_infos, extension_return_value)?;
100            } else {
101                let grouped_requests =
102                    group_create_logical_region_requests_by_physical_region_id(requests)?;
103                let mut manifest_infos = Vec::with_capacity(grouped_requests.len());
104                for (physical_region_id, requests) in grouped_requests {
105                    self.create_logical_regions(
106                        physical_region_id,
107                        requests,
108                        extension_return_value,
109                    )
110                    .await?;
111                    append_manifest_info(&self.mito, physical_region_id, &mut manifest_infos);
112                }
113                encode_manifest_info_to_extensions(&manifest_infos, extension_return_value)?;
114            }
115        } else {
116            return MissingRegionOptionSnafu {}.fail();
117        }
118
119        Ok(0)
120    }
121
122    /// Initialize a physical metric region at given region id.
123    async fn create_physical_region(
124        &self,
125        region_id: RegionId,
126        request: RegionCreateRequest,
127        extension_return_value: &mut HashMap<String, Vec<u8>>,
128    ) -> Result<()> {
129        let physical_region_options = PhysicalRegionOptions::try_from(&request.options)?;
130        let (data_region_id, metadata_region_id) = Self::transform_region_id(region_id);
131
132        // create metadata region
133        let create_metadata_region_request = self.create_request_for_metadata_region(&request);
134        self.mito
135            .handle_request(
136                metadata_region_id,
137                RegionRequest::Create(create_metadata_region_request),
138            )
139            .await
140            .with_context(|_| CreateMitoRegionSnafu {
141                region_type: METADATA_REGION_SUBDIR,
142            })?;
143
144        // create data region
145        let create_data_region_request = self.create_request_for_data_region(&request);
146        let physical_columns = create_data_region_request
147            .column_metadatas
148            .iter()
149            .map(|metadata| (metadata.column_schema.name.clone(), metadata.clone()))
150            .collect::<HashMap<_, _>>();
151        let time_index_unit = create_data_region_request
152            .column_metadatas
153            .iter()
154            .find_map(|metadata| {
155                if metadata.semantic_type == SemanticType::Timestamp {
156                    metadata
157                        .column_schema
158                        .data_type
159                        .as_timestamp()
160                        .map(|data_type| data_type.unit())
161                } else {
162                    None
163                }
164            })
165            .context(UnexpectedRequestSnafu {
166                reason: "No time index column found",
167            })?;
168        let response = self
169            .mito
170            .handle_request(
171                data_region_id,
172                RegionRequest::Create(create_data_region_request),
173            )
174            .await
175            .with_context(|_| CreateMitoRegionSnafu {
176                region_type: DATA_REGION_SUBDIR,
177            })?;
178        let primary_key_encoding = self.mito.get_primary_key_encoding(data_region_id).context(
179            PhysicalRegionNotFoundSnafu {
180                region_id: data_region_id,
181            },
182        )?;
183        extension_return_value.extend(response.extensions);
184
185        info!(
186            "Created physical metric region {region_id}, primary key encoding={primary_key_encoding}, physical_region_options={physical_region_options:?}"
187        );
188        PHYSICAL_REGION_COUNT.inc();
189
190        // remember this table
191        self.state.write().unwrap().add_physical_region(
192            data_region_id,
193            physical_columns,
194            primary_key_encoding,
195            physical_region_options,
196            time_index_unit,
197        );
198
199        Ok(())
200    }
201
202    /// Create multiple logical regions on the same physical region.
203    async fn create_logical_regions(
204        &self,
205        physical_region_id: RegionId,
206        requests: Vec<(RegionId, RegionCreateRequest)>,
207        extension_return_value: &mut HashMap<String, Vec<u8>>,
208    ) -> Result<()> {
209        let data_region_id = utils::to_data_region_id(physical_region_id);
210
211        let unit = self
212            .state
213            .read()
214            .unwrap()
215            .physical_region_time_index_unit(physical_region_id)
216            .context(PhysicalRegionNotFoundSnafu {
217                region_id: data_region_id,
218            })?;
219        // Checks the time index unit of each request.
220        for (_, request) in &requests {
221            // Safety: verify_region_create_request() ensures that the request is valid.
222            let time_index_column = request
223                .column_metadatas
224                .iter()
225                .find(|col| col.semantic_type == SemanticType::Timestamp)
226                .unwrap();
227            let request_unit = time_index_column
228                .column_schema
229                .data_type
230                .as_timestamp()
231                .unwrap()
232                .unit();
233            ensure!(
234                request_unit == unit,
235                UnexpectedRequestSnafu {
236                    reason: format!(
237                        "Metric has differenttime unit ({:?}) than the physical region ({:?})",
238                        request_unit, unit
239                    ),
240                }
241            );
242        }
243
244        // Filters out the requests that the logical region already exists
245        let requests = {
246            let state = self.state.read().unwrap();
247            let mut skipped = Vec::with_capacity(requests.len());
248            let mut kept_requests = Vec::with_capacity(requests.len());
249
250            for (region_id, request) in requests {
251                if state.is_logical_region_exist(region_id) {
252                    skipped.push(region_id);
253                } else {
254                    kept_requests.push((region_id, request));
255                }
256            }
257
258            // log skipped regions
259            if !skipped.is_empty() {
260                info!(
261                    "Skipped creating logical regions {skipped:?} because they already exist",
262                    skipped = skipped
263                );
264            }
265            kept_requests
266        };
267
268        // Finds new columns to add to physical region
269        let mut new_column_names = HashSet::new();
270        let mut new_columns = Vec::new();
271
272        let index_option = {
273            let state = &self.state.read().unwrap();
274            let region_state = state
275                .physical_region_states()
276                .get(&data_region_id)
277                .with_context(|| PhysicalRegionNotFoundSnafu {
278                    region_id: data_region_id,
279                })?;
280            let physical_columns = region_state.physical_columns();
281
282            extract_new_columns(
283                &requests,
284                physical_columns,
285                &mut new_column_names,
286                &mut new_columns,
287            )?;
288            region_state.options().index
289        };
290
291        // TODO(weny): we dont need to pass a mutable new_columns here.
292        self.data_region
293            .add_columns(data_region_id, new_columns, index_option)
294            .await?;
295
296        let physical_columns = self.data_region.physical_columns(data_region_id).await?;
297        let physical_schema_map = physical_columns
298            .iter()
299            .map(|metadata| (metadata.column_schema.name.as_str(), metadata))
300            .collect::<HashMap<_, _>>();
301        let logical_regions = requests
302            .iter()
303            .map(|(region_id, _)| *region_id)
304            .collect::<Vec<_>>();
305        let logical_region_columns = requests.iter().map(|(region_id, request)| {
306            (
307                *region_id,
308                request
309                    .column_metadatas
310                    .iter()
311                    .map(|metadata| {
312                        // Safety: previous steps ensure the physical region exist
313                        let column_metadata = *physical_schema_map
314                            .get(metadata.column_schema.name.as_str())
315                            .unwrap();
316                        (metadata.column_schema.name.as_str(), column_metadata)
317                    })
318                    .collect::<HashMap<_, _>>(),
319            )
320        });
321
322        let new_add_columns = new_column_names.iter().map(|name| {
323            // Safety: previous steps ensure the physical region exist
324            let column_metadata = *physical_schema_map.get(name).unwrap();
325            (name.to_string(), column_metadata.clone())
326        });
327
328        extension_return_value.insert(
329            ALTER_PHYSICAL_EXTENSION_KEY.to_string(),
330            ColumnMetadata::encode_list(&physical_columns).context(SerializeColumnMetadataSnafu)?,
331        );
332
333        // Writes logical regions metadata to metadata region
334        self.metadata_region
335            .add_logical_regions(physical_region_id, true, logical_region_columns)
336            .await?;
337
338        {
339            let mut state = self.state.write().unwrap();
340            state.add_physical_columns(data_region_id, new_add_columns);
341            state.add_logical_regions(physical_region_id, logical_regions.clone());
342        }
343        for logical_region_id in logical_regions {
344            self.metadata_region
345                .open_logical_region(logical_region_id)
346                .await;
347        }
348
349        Ok(())
350    }
351
352    /// Check if
353    /// - internal columns are not occupied
354    /// - required table option is present ([PHYSICAL_TABLE_METADATA_KEY] or
355    ///   [LOGICAL_TABLE_METADATA_KEY])
356    fn verify_region_create_request(request: &RegionCreateRequest) -> Result<()> {
357        request.validate().context(InvalidMetadataSnafu)?;
358
359        let name_to_index = request
360            .column_metadatas
361            .iter()
362            .enumerate()
363            .map(|(idx, metadata)| (metadata.column_schema.name.clone(), idx))
364            .collect::<HashMap<String, usize>>();
365
366        let table_id_col_def = request.column_metadatas.iter().any(is_metric_name_col);
367        let tsid_col_def = request.column_metadatas.iter().any(is_tsid_col);
368
369        // check if internal columns are not occupied or defined in the request
370        ensure!(
371            !name_to_index.contains_key(DATA_SCHEMA_TABLE_ID_COLUMN_NAME) || table_id_col_def,
372            InternalColumnOccupiedSnafu {
373                column: DATA_SCHEMA_TABLE_ID_COLUMN_NAME,
374            }
375        );
376        ensure!(
377            !name_to_index.contains_key(DATA_SCHEMA_TSID_COLUMN_NAME) || tsid_col_def,
378            InternalColumnOccupiedSnafu {
379                column: DATA_SCHEMA_TSID_COLUMN_NAME,
380            }
381        );
382
383        // check if required table option is present
384        ensure!(
385            request.is_physical_table() || request.options.contains_key(LOGICAL_TABLE_METADATA_KEY),
386            MissingRegionOptionSnafu {}
387        );
388        ensure!(
389            !(request.is_physical_table()
390                && request.options.contains_key(LOGICAL_TABLE_METADATA_KEY)),
391            ConflictRegionOptionSnafu {}
392        );
393
394        // check if field columns are either a normal metric value or a native histogram.
395        let mut field_cols = Vec::new();
396        for col in &request.column_metadatas {
397            // Verified in above steps.
398            if is_metric_engine_internal_column(&col.column_schema.name) {
399                continue;
400            }
401            match col.semantic_type {
402                SemanticType::Tag => ensure!(
403                    col.column_schema.data_type == ConcreteDataType::string_datatype(),
404                    ColumnTypeMismatchSnafu {
405                        expect: ConcreteDataType::string_datatype(),
406                        actual: col.column_schema.data_type.clone(),
407                    }
408                ),
409                SemanticType::Field => {
410                    field_cols.push(col);
411                }
412                SemanticType::Timestamp => {}
413            }
414        }
415        let [field_col] = field_cols.as_slice() else {
416            if field_cols.is_empty() {
417                NoFieldColumnSnafu.fail()?;
418            }
419            return MultipleFieldColumnSnafu {
420                previous: field_cols[0].column_schema.name.clone(),
421                current: field_cols[1].column_schema.name.clone(),
422            }
423            .fail();
424        };
425
426        if is_native_histogram_value_schema(
427            &field_col.column_schema.name,
428            &field_col.column_schema.data_type,
429        ) {
430            return Ok(());
431        }
432
433        // make sure the normal field column is float64 type
434        ensure!(
435            field_col.column_schema.data_type == ConcreteDataType::float64_datatype(),
436            ColumnTypeMismatchSnafu {
437                expect: ConcreteDataType::float64_datatype(),
438                actual: field_col.column_schema.data_type.clone(),
439            }
440        );
441
442        Ok(())
443    }
444
445    /// Build data region id and metadata region id from the given region id.
446    ///
447    /// Return value: (data_region_id, metadata_region_id)
448    fn transform_region_id(region_id: RegionId) -> (RegionId, RegionId) {
449        (
450            to_data_region_id(region_id),
451            to_metadata_region_id(region_id),
452        )
453    }
454
455    /// Build [RegionCreateRequest] for metadata region
456    ///
457    /// This method will append [METADATA_REGION_SUBDIR] to the given `region_dir`.
458    pub fn create_request_for_metadata_region(
459        &self,
460        request: &RegionCreateRequest,
461    ) -> RegionCreateRequest {
462        // ts TIME INDEX DEFAULT 0
463        let timestamp_column_metadata = ColumnMetadata {
464            column_id: METADATA_SCHEMA_TIMESTAMP_COLUMN_INDEX as _,
465            semantic_type: SemanticType::Timestamp,
466            column_schema: ColumnSchema::new(
467                METADATA_SCHEMA_TIMESTAMP_COLUMN_NAME,
468                ConcreteDataType::timestamp_millisecond_datatype(),
469                false,
470            )
471            .with_default_constraint(Some(datatypes::schema::ColumnDefaultConstraint::Value(
472                Value::Timestamp(Timestamp::new_millisecond(0)),
473            )))
474            .unwrap(),
475        };
476        // key STRING PRIMARY KEY
477        let key_column_metadata = ColumnMetadata {
478            column_id: METADATA_SCHEMA_KEY_COLUMN_INDEX as _,
479            semantic_type: SemanticType::Tag,
480            column_schema: ColumnSchema::new(
481                METADATA_SCHEMA_KEY_COLUMN_NAME,
482                ConcreteDataType::string_datatype(),
483                false,
484            ),
485        };
486        // val STRING
487        let value_column_metadata = ColumnMetadata {
488            column_id: METADATA_SCHEMA_VALUE_COLUMN_INDEX as _,
489            semantic_type: SemanticType::Field,
490            column_schema: ColumnSchema::new(
491                METADATA_SCHEMA_VALUE_COLUMN_NAME,
492                ConcreteDataType::string_datatype(),
493                true,
494            ),
495        };
496
497        let options = region_options_for_metadata_region(&request.options);
498        RegionCreateRequest {
499            engine: MITO_ENGINE_NAME.to_string(),
500            column_metadatas: vec![
501                timestamp_column_metadata,
502                key_column_metadata,
503                value_column_metadata,
504            ],
505            primary_key: vec![METADATA_SCHEMA_KEY_COLUMN_INDEX as _],
506            options,
507            table_dir: request.table_dir.clone(),
508            path_type: PathType::Metadata,
509            partition_expr_json: Some("".to_string()),
510            requirements: request.requirements,
511        }
512    }
513
514    /// Convert [RegionCreateRequest] for data region.
515    ///
516    /// All tag columns in the original request will be converted to value columns.
517    /// Those columns real semantic type is stored in metadata region.
518    ///
519    /// This will also add internal columns to the request.
520    pub fn create_request_for_data_region(
521        &self,
522        request: &RegionCreateRequest,
523    ) -> RegionCreateRequest {
524        let mut data_region_request = request.clone();
525        let mut primary_key = vec![ReservedColumnId::table_id(), ReservedColumnId::tsid()];
526
527        data_region_request.table_dir = request.table_dir.clone();
528        data_region_request.path_type = PathType::Data;
529
530        let table_id_col_def = request.column_metadatas.iter().any(is_metric_name_col);
531        let tsid_col_def = request.column_metadatas.iter().any(is_tsid_col);
532
533        // change nullability for tag columns
534        data_region_request
535            .column_metadatas
536            .iter_mut()
537            .for_each(|metadata| {
538                if metadata.semantic_type == SemanticType::Tag
539                    && !is_metric_name_col(metadata)
540                    && !is_tsid_col(metadata)
541                {
542                    metadata.column_schema.set_nullable();
543                    primary_key.push(metadata.column_id);
544                }
545            });
546
547        // add internal columns if not defined in the request
548        if !table_id_col_def {
549            data_region_request.column_metadatas.push(table_id_col());
550        }
551        if !tsid_col_def {
552            data_region_request.column_metadatas.push(tsid_col());
553        }
554        data_region_request.primary_key = primary_key;
555
556        // set data region options
557        set_data_region_options(
558            &mut data_region_request.options,
559            self.config.sparse_primary_key_encoding,
560        );
561
562        data_region_request
563    }
564}
565
566fn table_id_col() -> ColumnMetadata {
567    ColumnMetadata {
568        column_id: ReservedColumnId::table_id(),
569        semantic_type: SemanticType::Tag,
570        column_schema: ColumnSchema::new(
571            DATA_SCHEMA_TABLE_ID_COLUMN_NAME,
572            ConcreteDataType::uint32_datatype(),
573            false,
574        )
575        .with_skipping_options(SkippingIndexOptions::new_unchecked(
576            DEFAULT_TABLE_ID_SKIPPING_INDEX_GRANULARITY,
577            DEFAULT_TABLE_ID_SKIPPING_INDEX_FALSE_POSITIVE_RATE,
578            datatypes::schema::SkippingIndexType::BloomFilter,
579        ))
580        .unwrap(),
581    }
582}
583
584fn tsid_col() -> ColumnMetadata {
585    ColumnMetadata {
586        column_id: ReservedColumnId::tsid(),
587        semantic_type: SemanticType::Tag,
588        column_schema: ColumnSchema::new(
589            DATA_SCHEMA_TSID_COLUMN_NAME,
590            ConcreteDataType::uint64_datatype(),
591            false,
592        )
593        .with_inverted_index(false),
594    }
595}
596
597/// Returns true if the column is the metric name column.
598pub(crate) fn is_metric_name_col(column: &ColumnMetadata) -> bool {
599    column.column_id == ReservedColumnId::table_id()
600        && column.semantic_type == SemanticType::Tag
601        && column.column_schema.data_type == ConcreteDataType::uint32_datatype()
602        && column.column_schema.name == DATA_SCHEMA_TABLE_ID_COLUMN_NAME
603        && !column.column_schema.is_nullable()
604}
605
606/// Returns true if the column is the tsid column.
607pub(crate) fn is_tsid_col(column: &ColumnMetadata) -> bool {
608    column.column_id == ReservedColumnId::tsid()
609        && column.semantic_type == SemanticType::Tag
610        && column.column_schema.data_type == ConcreteDataType::uint64_datatype()
611        && column.column_schema.name == DATA_SCHEMA_TSID_COLUMN_NAME
612        && !column.column_schema.is_nullable()
613}
614
615/// Groups the create logical region requests by physical region id.
616fn group_create_logical_region_requests_by_physical_region_id(
617    requests: Vec<(RegionId, RegionCreateRequest)>,
618) -> Result<HashMap<RegionId, Vec<(RegionId, RegionCreateRequest)>>> {
619    let mut result = HashMap::with_capacity(requests.len());
620    for (region_id, request) in requests {
621        let physical_region_id = parse_physical_region_id(&request)?;
622        result
623            .entry(physical_region_id)
624            .or_insert_with(Vec::new)
625            .push((region_id, request));
626    }
627
628    Ok(result)
629}
630
631/// Parses the physical region id from the request.
632fn parse_physical_region_id(request: &RegionCreateRequest) -> Result<RegionId> {
633    let physical_region_id_raw = request
634        .options
635        .get(LOGICAL_TABLE_METADATA_KEY)
636        .ok_or(MissingRegionOptionSnafu {}.build())?;
637
638    let physical_region_id: RegionId = physical_region_id_raw
639        .parse::<u64>()
640        .with_context(|_| ParseRegionIdSnafu {
641            raw: physical_region_id_raw,
642        })?
643        .into();
644
645    Ok(physical_region_id)
646}
647
648/// Creates the region options for metadata region in metric engine.
649pub(crate) fn region_options_for_metadata_region(
650    original: &HashMap<String, String>,
651) -> HashMap<String, String> {
652    let mut metadata_region_options = HashMap::new();
653    metadata_region_options.insert(TTL_KEY.to_string(), FOREVER.to_string());
654
655    if let Some(wal_options) = original.get(WAL_OPTIONS_KEY) {
656        metadata_region_options.insert(WAL_OPTIONS_KEY.to_string(), wal_options.clone());
657    }
658
659    metadata_region_options
660}
661
662#[cfg(test)]
663mod test {
664    use common_meta::ddl::test_util::assert_column_name_and_id;
665    use common_meta::ddl::utils::{parse_column_metadatas, parse_manifest_infos_from_extensions};
666    use common_query::native_histogram::{NATIVE_HISTOGRAM_FIELD, native_histogram_value_type};
667    use common_query::prelude::{greptime_timestamp, greptime_value};
668    use store_api::metric_engine_consts::{METRIC_ENGINE_NAME, PHYSICAL_TABLE_METADATA_KEY};
669    use store_api::region_request::{BatchRegionDdlRequest, RegionRequirements};
670
671    use super::*;
672    use crate::config::EngineConfig;
673    use crate::engine::MetricEngine;
674    use crate::test_util::{TestEnv, create_logical_region_request};
675
676    #[test]
677    fn test_internal_column_metadata() {
678        let table_id_col = table_id_col();
679        let tsid_col = tsid_col();
680        assert!(is_metric_name_col(&table_id_col));
681        assert!(is_tsid_col(&tsid_col));
682    }
683
684    #[test]
685    fn test_verify_region_create_request() {
686        // internal column is occupied
687        let request = RegionCreateRequest {
688            column_metadatas: vec![
689                ColumnMetadata {
690                    column_id: 0,
691                    semantic_type: SemanticType::Timestamp,
692                    column_schema: ColumnSchema::new(
693                        METADATA_SCHEMA_TIMESTAMP_COLUMN_NAME,
694                        ConcreteDataType::timestamp_millisecond_datatype(),
695                        false,
696                    ),
697                },
698                ColumnMetadata {
699                    column_id: 1,
700                    semantic_type: SemanticType::Tag,
701                    column_schema: ColumnSchema::new(
702                        DATA_SCHEMA_TABLE_ID_COLUMN_NAME,
703                        ConcreteDataType::uint32_datatype(),
704                        false,
705                    ),
706                },
707            ],
708            table_dir: "test_dir".to_string(),
709            path_type: PathType::Bare,
710            engine: METRIC_ENGINE_NAME.to_string(),
711            primary_key: vec![],
712            options: HashMap::new(),
713            partition_expr_json: Some("".to_string()),
714            requirements: RegionRequirements::object_storage(),
715        };
716        let result = MetricEngineInner::verify_region_create_request(&request);
717        assert!(result.is_err());
718        assert_eq!(
719            result.unwrap_err().to_string(),
720            "Internal column __table_id is reserved".to_string()
721        );
722
723        // allow reserved internal columns when defined properly
724        let request = RegionCreateRequest {
725            column_metadatas: vec![
726                ColumnMetadata {
727                    column_id: 0,
728                    semantic_type: SemanticType::Timestamp,
729                    column_schema: ColumnSchema::new(
730                        METADATA_SCHEMA_TIMESTAMP_COLUMN_NAME,
731                        ConcreteDataType::timestamp_millisecond_datatype(),
732                        false,
733                    ),
734                },
735                ColumnMetadata {
736                    column_id: 1,
737                    semantic_type: SemanticType::Tag,
738                    column_schema: ColumnSchema::new(
739                        "column1".to_string(),
740                        ConcreteDataType::string_datatype(),
741                        false,
742                    ),
743                },
744                ColumnMetadata {
745                    column_id: 2,
746                    semantic_type: SemanticType::Field,
747                    column_schema: ColumnSchema::new(
748                        "column2".to_string(),
749                        ConcreteDataType::float64_datatype(),
750                        false,
751                    ),
752                },
753                table_id_col(),
754                tsid_col(),
755            ],
756            table_dir: "test_dir".to_string(),
757            path_type: PathType::Bare,
758            engine: METRIC_ENGINE_NAME.to_string(),
759            primary_key: vec![],
760            options: [(PHYSICAL_TABLE_METADATA_KEY.to_string(), String::new())]
761                .into_iter()
762                .collect(),
763            partition_expr_json: Some("".to_string()),
764            requirements: Default::default(),
765        };
766        MetricEngineInner::verify_region_create_request(&request).unwrap();
767
768        // valid request
769        let request = RegionCreateRequest {
770            column_metadatas: vec![
771                ColumnMetadata {
772                    column_id: 0,
773                    semantic_type: SemanticType::Timestamp,
774                    column_schema: ColumnSchema::new(
775                        METADATA_SCHEMA_TIMESTAMP_COLUMN_NAME,
776                        ConcreteDataType::timestamp_millisecond_datatype(),
777                        false,
778                    ),
779                },
780                ColumnMetadata {
781                    column_id: 1,
782                    semantic_type: SemanticType::Tag,
783                    column_schema: ColumnSchema::new(
784                        "column1".to_string(),
785                        ConcreteDataType::string_datatype(),
786                        false,
787                    ),
788                },
789                ColumnMetadata {
790                    column_id: 2,
791                    semantic_type: SemanticType::Field,
792                    column_schema: ColumnSchema::new(
793                        "column2".to_string(),
794                        ConcreteDataType::float64_datatype(),
795                        false,
796                    ),
797                },
798            ],
799            table_dir: "test_dir".to_string(),
800            path_type: PathType::Bare,
801            engine: METRIC_ENGINE_NAME.to_string(),
802            primary_key: vec![],
803            options: [(PHYSICAL_TABLE_METADATA_KEY.to_string(), String::new())]
804                .into_iter()
805                .collect(),
806            partition_expr_json: Some("".to_string()),
807            requirements: Default::default(),
808        };
809        MetricEngineInner::verify_region_create_request(&request).unwrap();
810    }
811
812    #[test]
813    fn test_verify_region_create_request_options() {
814        let mut request = RegionCreateRequest {
815            column_metadatas: vec![
816                ColumnMetadata {
817                    column_id: 0,
818                    semantic_type: SemanticType::Timestamp,
819                    column_schema: ColumnSchema::new(
820                        METADATA_SCHEMA_TIMESTAMP_COLUMN_NAME,
821                        ConcreteDataType::timestamp_millisecond_datatype(),
822                        false,
823                    ),
824                },
825                ColumnMetadata {
826                    column_id: 1,
827                    semantic_type: SemanticType::Field,
828                    column_schema: ColumnSchema::new(
829                        "val".to_string(),
830                        ConcreteDataType::float64_datatype(),
831                        false,
832                    ),
833                },
834            ],
835            table_dir: "test_dir".to_string(),
836            path_type: PathType::Bare,
837            engine: METRIC_ENGINE_NAME.to_string(),
838            primary_key: vec![],
839            options: HashMap::new(),
840            partition_expr_json: Some("".to_string()),
841            requirements: Default::default(),
842        };
843        MetricEngineInner::verify_region_create_request(&request).unwrap_err();
844
845        let mut options = HashMap::new();
846        options.insert(PHYSICAL_TABLE_METADATA_KEY.to_string(), "value".to_string());
847        request.options.clone_from(&options);
848        MetricEngineInner::verify_region_create_request(&request).unwrap();
849
850        options.insert(LOGICAL_TABLE_METADATA_KEY.to_string(), "value".to_string());
851        request.options.clone_from(&options);
852        MetricEngineInner::verify_region_create_request(&request).unwrap_err();
853
854        options.remove(PHYSICAL_TABLE_METADATA_KEY).unwrap();
855        request.options = options;
856        MetricEngineInner::verify_region_create_request(&request).unwrap();
857    }
858
859    #[test]
860    fn test_verify_region_create_request_native_histogram_fields() {
861        let native_histogram_columns = vec![
862            ColumnMetadata {
863                column_id: 0,
864                semantic_type: SemanticType::Timestamp,
865                column_schema: ColumnSchema::new(
866                    greptime_timestamp(),
867                    ConcreteDataType::timestamp_millisecond_datatype(),
868                    false,
869                ),
870            },
871            ColumnMetadata {
872                column_id: 1,
873                semantic_type: SemanticType::Tag,
874                column_schema: ColumnSchema::new("job", ConcreteDataType::string_datatype(), true),
875            },
876            ColumnMetadata {
877                column_id: 2,
878                semantic_type: SemanticType::Field,
879                column_schema: ColumnSchema::new(
880                    NATIVE_HISTOGRAM_FIELD,
881                    native_histogram_value_type().clone(),
882                    true,
883                ),
884            },
885        ];
886        let request = RegionCreateRequest {
887            column_metadatas: native_histogram_columns,
888            table_dir: "test_dir".to_string(),
889            path_type: PathType::Bare,
890            engine: METRIC_ENGINE_NAME.to_string(),
891            primary_key: vec![],
892            options: [(
893                LOGICAL_TABLE_METADATA_KEY.to_string(),
894                "physical".to_string(),
895            )]
896            .into_iter()
897            .collect(),
898            partition_expr_json: Some("".to_string()),
899            requirements: Default::default(),
900        };
901        MetricEngineInner::verify_region_create_request(&request).unwrap();
902
903        let request = RegionCreateRequest {
904            column_metadatas: vec![
905                ColumnMetadata {
906                    column_id: 0,
907                    semantic_type: SemanticType::Timestamp,
908                    column_schema: ColumnSchema::new(
909                        greptime_timestamp(),
910                        ConcreteDataType::timestamp_millisecond_datatype(),
911                        false,
912                    ),
913                },
914                ColumnMetadata {
915                    column_id: 1,
916                    semantic_type: SemanticType::Field,
917                    column_schema: ColumnSchema::new(
918                        "value_a",
919                        ConcreteDataType::float64_datatype(),
920                        true,
921                    ),
922                },
923                ColumnMetadata {
924                    column_id: 2,
925                    semantic_type: SemanticType::Field,
926                    column_schema: ColumnSchema::new(
927                        "value_b",
928                        ConcreteDataType::float64_datatype(),
929                        true,
930                    ),
931                },
932            ],
933            table_dir: "test_dir".to_string(),
934            path_type: PathType::Bare,
935            engine: METRIC_ENGINE_NAME.to_string(),
936            primary_key: vec![],
937            options: [(
938                LOGICAL_TABLE_METADATA_KEY.to_string(),
939                "physical".to_string(),
940            )]
941            .into_iter()
942            .collect(),
943            partition_expr_json: Some("".to_string()),
944            requirements: Default::default(),
945        };
946        assert!(MetricEngineInner::verify_region_create_request(&request).is_err());
947    }
948
949    #[tokio::test]
950    async fn test_create_request_for_physical_regions() {
951        // original request
952        let options: HashMap<_, _> = [
953            ("ttl".to_string(), "60m".to_string()),
954            ("skip_wal".to_string(), "true".to_string()),
955        ]
956        .into_iter()
957        .collect();
958        let request = RegionCreateRequest {
959            engine: METRIC_ENGINE_NAME.to_string(),
960            column_metadatas: vec![
961                ColumnMetadata {
962                    column_id: 0,
963                    semantic_type: SemanticType::Timestamp,
964                    column_schema: ColumnSchema::new(
965                        "timestamp",
966                        ConcreteDataType::timestamp_millisecond_datatype(),
967                        false,
968                    ),
969                },
970                ColumnMetadata {
971                    column_id: 1,
972                    semantic_type: SemanticType::Tag,
973                    column_schema: ColumnSchema::new(
974                        "tag",
975                        ConcreteDataType::string_datatype(),
976                        false,
977                    ),
978                },
979            ],
980            primary_key: vec![0],
981            options,
982            table_dir: "/test_dir".to_string(),
983            path_type: PathType::Bare,
984            partition_expr_json: Some("".to_string()),
985            requirements: RegionRequirements::object_storage(),
986        };
987
988        // set up
989        let env = TestEnv::new().await;
990        let engine = MetricEngine::try_new(env.mito(), EngineConfig::default()).unwrap();
991        let engine_inner = engine.inner;
992
993        // check create data region request
994        let data_region_request = engine_inner.create_request_for_data_region(&request);
995        assert_eq!(data_region_request.table_dir, "/test_dir".to_string());
996        assert_eq!(data_region_request.path_type, PathType::Data);
997        assert_eq!(data_region_request.column_metadatas.len(), 4);
998        assert_eq!(
999            data_region_request.primary_key,
1000            vec![ReservedColumnId::table_id(), ReservedColumnId::tsid(), 1]
1001        );
1002        assert!(data_region_request.options.contains_key("ttl"));
1003        assert_eq!(
1004            data_region_request.requirements,
1005            RegionRequirements::object_storage()
1006        );
1007
1008        // check create metadata region request
1009        let metadata_region_request = engine_inner.create_request_for_metadata_region(&request);
1010        assert_eq!(metadata_region_request.table_dir, "/test_dir".to_string());
1011        assert_eq!(metadata_region_request.path_type, PathType::Metadata);
1012        assert_eq!(
1013            metadata_region_request.options.get("ttl").unwrap(),
1014            "forever"
1015        );
1016        assert!(!metadata_region_request.options.contains_key("skip_wal"));
1017        assert_eq!(
1018            metadata_region_request.requirements,
1019            RegionRequirements::object_storage()
1020        );
1021    }
1022
1023    #[tokio::test]
1024    async fn test_create_request_for_physical_regions_with_internal_columns() {
1025        let options: HashMap<_, _> = [
1026            ("ttl".to_string(), "60m".to_string()),
1027            ("skip_wal".to_string(), "true".to_string()),
1028        ]
1029        .into_iter()
1030        .collect();
1031        let request = RegionCreateRequest {
1032            engine: METRIC_ENGINE_NAME.to_string(),
1033            column_metadatas: vec![
1034                ColumnMetadata {
1035                    column_id: 0,
1036                    semantic_type: SemanticType::Timestamp,
1037                    column_schema: ColumnSchema::new(
1038                        "timestamp",
1039                        ConcreteDataType::timestamp_millisecond_datatype(),
1040                        false,
1041                    ),
1042                },
1043                ColumnMetadata {
1044                    column_id: 1,
1045                    semantic_type: SemanticType::Tag,
1046                    column_schema: ColumnSchema::new(
1047                        "tag",
1048                        ConcreteDataType::string_datatype(),
1049                        false,
1050                    ),
1051                },
1052                ColumnMetadata {
1053                    column_id: 2,
1054                    semantic_type: SemanticType::Field,
1055                    column_schema: ColumnSchema::new(
1056                        "value",
1057                        ConcreteDataType::float64_datatype(),
1058                        false,
1059                    ),
1060                },
1061                table_id_col(),
1062                tsid_col(),
1063            ],
1064            primary_key: vec![0],
1065            options,
1066            table_dir: "/test_dir".to_string(),
1067            path_type: PathType::Bare,
1068            partition_expr_json: Some("".to_string()),
1069            requirements: Default::default(),
1070        };
1071
1072        let env = TestEnv::new().await;
1073        let engine = MetricEngine::try_new(env.mito(), EngineConfig::default()).unwrap();
1074        let engine_inner = engine.inner;
1075
1076        let data_region_request = engine_inner.create_request_for_data_region(&request);
1077        assert_eq!(data_region_request.column_metadatas.len(), 5);
1078        assert_eq!(
1079            data_region_request.primary_key,
1080            vec![ReservedColumnId::table_id(), ReservedColumnId::tsid(), 1]
1081        );
1082
1083        let table_id_count = data_region_request
1084            .column_metadatas
1085            .iter()
1086            .filter(|metadata| metadata.column_schema.name == DATA_SCHEMA_TABLE_ID_COLUMN_NAME)
1087            .count();
1088        let tsid_count = data_region_request
1089            .column_metadatas
1090            .iter()
1091            .filter(|metadata| metadata.column_schema.name == DATA_SCHEMA_TSID_COLUMN_NAME)
1092            .count();
1093        assert_eq!(table_id_count, 1);
1094        assert_eq!(tsid_count, 1);
1095
1096        let tag_metadata = data_region_request
1097            .column_metadatas
1098            .iter()
1099            .find(|metadata| metadata.column_schema.name == "tag")
1100            .unwrap();
1101        assert!(tag_metadata.column_schema.is_nullable());
1102
1103        let table_id_metadata = data_region_request
1104            .column_metadatas
1105            .iter()
1106            .find(|metadata| metadata.column_schema.name == DATA_SCHEMA_TABLE_ID_COLUMN_NAME)
1107            .unwrap();
1108        assert!(is_metric_name_col(table_id_metadata));
1109
1110        let tsid_metadata = data_region_request
1111            .column_metadatas
1112            .iter()
1113            .find(|metadata| metadata.column_schema.name == DATA_SCHEMA_TSID_COLUMN_NAME)
1114            .unwrap();
1115        assert!(is_tsid_col(tsid_metadata));
1116    }
1117
1118    #[tokio::test]
1119    async fn test_create_logical_regions() {
1120        let env = TestEnv::new().await;
1121        let engine = env.metric();
1122        let physical_region_id1 = RegionId::new(1024, 0);
1123        let physical_region_id2 = RegionId::new(1024, 1);
1124        let logical_region_id1 = RegionId::new(1025, 0);
1125        let logical_region_id2 = RegionId::new(1025, 1);
1126        env.create_physical_region(physical_region_id1, "/test_dir1", vec![])
1127            .await;
1128        env.create_physical_region(physical_region_id2, "/test_dir2", vec![])
1129            .await;
1130
1131        let region_create_request1 =
1132            create_logical_region_request(&["job"], physical_region_id1, "logical1");
1133        let region_create_request2 =
1134            create_logical_region_request(&["job"], physical_region_id2, "logical2");
1135
1136        let response = engine
1137            .handle_batch_ddl_requests(BatchRegionDdlRequest::Create(vec![
1138                (logical_region_id1, region_create_request1),
1139                (logical_region_id2, region_create_request2),
1140            ]))
1141            .await
1142            .unwrap();
1143
1144        let manifest_infos = parse_manifest_infos_from_extensions(&response.extensions).unwrap();
1145        assert_eq!(manifest_infos.len(), 2);
1146        let region_ids = manifest_infos.into_iter().map(|i| i.0).collect::<Vec<_>>();
1147        assert!(region_ids.contains(&physical_region_id1));
1148        assert!(region_ids.contains(&physical_region_id2));
1149
1150        let column_metadatas =
1151            parse_column_metadatas(&response.extensions, ALTER_PHYSICAL_EXTENSION_KEY).unwrap();
1152        assert_column_name_and_id(
1153            &column_metadatas,
1154            &[
1155                (greptime_timestamp(), 0),
1156                (greptime_value(), 1),
1157                ("__table_id", ReservedColumnId::table_id()),
1158                ("__tsid", ReservedColumnId::tsid()),
1159                ("job", 2),
1160            ],
1161        );
1162    }
1163}