1mod extract_new_columns;
16
17use std::collections::{HashMap, HashSet};
18
19use api::v1::SemanticType;
20use common_query::native_histogram::is_native_histogram_value_schema;
21use common_telemetry::info;
22use common_time::{FOREVER, Timestamp};
23use datatypes::data_type::ConcreteDataType;
24use datatypes::schema::{ColumnSchema, SkippingIndexOptions};
25use datatypes::value::Value;
26use mito2::engine::MITO_ENGINE_NAME;
27use snafu::{OptionExt, ResultExt, ensure};
28use store_api::metadata::ColumnMetadata;
29use store_api::metric_engine_consts::{
30 ALTER_PHYSICAL_EXTENSION_KEY, DATA_REGION_SUBDIR, DATA_SCHEMA_TABLE_ID_COLUMN_NAME,
31 DATA_SCHEMA_TSID_COLUMN_NAME, LOGICAL_TABLE_METADATA_KEY, METADATA_REGION_SUBDIR,
32 METADATA_SCHEMA_KEY_COLUMN_INDEX, METADATA_SCHEMA_KEY_COLUMN_NAME,
33 METADATA_SCHEMA_TIMESTAMP_COLUMN_INDEX, METADATA_SCHEMA_TIMESTAMP_COLUMN_NAME,
34 METADATA_SCHEMA_VALUE_COLUMN_INDEX, METADATA_SCHEMA_VALUE_COLUMN_NAME,
35 is_metric_engine_internal_column,
36};
37use store_api::mito_engine_options::{TTL_KEY, WAL_OPTIONS_KEY};
38use store_api::region_engine::RegionEngine;
39use store_api::region_request::{AffectedRows, PathType, RegionCreateRequest, RegionRequest};
40use store_api::storage::RegionId;
41use store_api::storage::consts::ReservedColumnId;
42
43use crate::engine::MetricEngineInner;
44use crate::engine::create::extract_new_columns::extract_new_columns;
45use crate::engine::options::{PhysicalRegionOptions, set_data_region_options};
46use crate::error::{
47 ColumnTypeMismatchSnafu, ConflictRegionOptionSnafu, CreateMitoRegionSnafu,
48 InternalColumnOccupiedSnafu, InvalidMetadataSnafu, MissingRegionOptionSnafu,
49 MultipleFieldColumnSnafu, NoFieldColumnSnafu, ParseRegionIdSnafu, PhysicalRegionNotFoundSnafu,
50 Result, SerializeColumnMetadataSnafu, UnexpectedRequestSnafu,
51};
52use crate::metrics::PHYSICAL_REGION_COUNT;
53use crate::utils::{
54 self, append_manifest_info, encode_manifest_info_to_extensions, to_data_region_id,
55 to_metadata_region_id,
56};
57
58const DEFAULT_TABLE_ID_SKIPPING_INDEX_GRANULARITY: u32 = 1024;
59const DEFAULT_TABLE_ID_SKIPPING_INDEX_FALSE_POSITIVE_RATE: f64 = 0.01;
60
61impl MetricEngineInner {
62 pub async fn create_regions(
63 &self,
64 mut requests: Vec<(RegionId, RegionCreateRequest)>,
65 extension_return_value: &mut HashMap<String, Vec<u8>>,
66 ) -> Result<AffectedRows> {
67 if requests.is_empty() {
68 return Ok(0);
69 }
70
71 for (_, request) in requests.iter() {
72 Self::verify_region_create_request(request)?;
73 }
74
75 let first_request = &requests.first().unwrap().1;
76 if first_request.is_physical_table() {
77 ensure!(
78 requests.len() == 1,
79 UnexpectedRequestSnafu {
80 reason: "Physical table must be created with single request".to_string(),
81 }
82 );
83 let (region_id, request) = requests.pop().unwrap();
84 self.create_physical_region(region_id, request, extension_return_value)
85 .await?;
86
87 return Ok(0);
88 } else if first_request
89 .options
90 .contains_key(LOGICAL_TABLE_METADATA_KEY)
91 {
92 if requests.len() == 1 {
93 let request = &requests.first().unwrap().1;
94 let physical_region_id = parse_physical_region_id(request)?;
95 let mut manifest_infos = Vec::with_capacity(1);
96 self.create_logical_regions(physical_region_id, requests, extension_return_value)
97 .await?;
98 append_manifest_info(&self.mito, physical_region_id, &mut manifest_infos);
99 encode_manifest_info_to_extensions(&manifest_infos, extension_return_value)?;
100 } else {
101 let grouped_requests =
102 group_create_logical_region_requests_by_physical_region_id(requests)?;
103 let mut manifest_infos = Vec::with_capacity(grouped_requests.len());
104 for (physical_region_id, requests) in grouped_requests {
105 self.create_logical_regions(
106 physical_region_id,
107 requests,
108 extension_return_value,
109 )
110 .await?;
111 append_manifest_info(&self.mito, physical_region_id, &mut manifest_infos);
112 }
113 encode_manifest_info_to_extensions(&manifest_infos, extension_return_value)?;
114 }
115 } else {
116 return MissingRegionOptionSnafu {}.fail();
117 }
118
119 Ok(0)
120 }
121
122 async fn create_physical_region(
124 &self,
125 region_id: RegionId,
126 request: RegionCreateRequest,
127 extension_return_value: &mut HashMap<String, Vec<u8>>,
128 ) -> Result<()> {
129 let physical_region_options = PhysicalRegionOptions::try_from(&request.options)?;
130 let (data_region_id, metadata_region_id) = Self::transform_region_id(region_id);
131
132 let create_metadata_region_request = self.create_request_for_metadata_region(&request);
134 self.mito
135 .handle_request(
136 metadata_region_id,
137 RegionRequest::Create(create_metadata_region_request),
138 )
139 .await
140 .with_context(|_| CreateMitoRegionSnafu {
141 region_type: METADATA_REGION_SUBDIR,
142 })?;
143
144 let create_data_region_request = self.create_request_for_data_region(&request);
146 let physical_columns = create_data_region_request
147 .column_metadatas
148 .iter()
149 .map(|metadata| (metadata.column_schema.name.clone(), metadata.clone()))
150 .collect::<HashMap<_, _>>();
151 let time_index_unit = create_data_region_request
152 .column_metadatas
153 .iter()
154 .find_map(|metadata| {
155 if metadata.semantic_type == SemanticType::Timestamp {
156 metadata
157 .column_schema
158 .data_type
159 .as_timestamp()
160 .map(|data_type| data_type.unit())
161 } else {
162 None
163 }
164 })
165 .context(UnexpectedRequestSnafu {
166 reason: "No time index column found",
167 })?;
168 let response = self
169 .mito
170 .handle_request(
171 data_region_id,
172 RegionRequest::Create(create_data_region_request),
173 )
174 .await
175 .with_context(|_| CreateMitoRegionSnafu {
176 region_type: DATA_REGION_SUBDIR,
177 })?;
178 let primary_key_encoding = self.mito.get_primary_key_encoding(data_region_id).context(
179 PhysicalRegionNotFoundSnafu {
180 region_id: data_region_id,
181 },
182 )?;
183 extension_return_value.extend(response.extensions);
184
185 info!(
186 "Created physical metric region {region_id}, primary key encoding={primary_key_encoding}, physical_region_options={physical_region_options:?}"
187 );
188 PHYSICAL_REGION_COUNT.inc();
189
190 self.state.write().unwrap().add_physical_region(
192 data_region_id,
193 physical_columns,
194 primary_key_encoding,
195 physical_region_options,
196 time_index_unit,
197 );
198
199 Ok(())
200 }
201
202 async fn create_logical_regions(
204 &self,
205 physical_region_id: RegionId,
206 requests: Vec<(RegionId, RegionCreateRequest)>,
207 extension_return_value: &mut HashMap<String, Vec<u8>>,
208 ) -> Result<()> {
209 let data_region_id = utils::to_data_region_id(physical_region_id);
210
211 let unit = self
212 .state
213 .read()
214 .unwrap()
215 .physical_region_time_index_unit(physical_region_id)
216 .context(PhysicalRegionNotFoundSnafu {
217 region_id: data_region_id,
218 })?;
219 for (_, request) in &requests {
221 let time_index_column = request
223 .column_metadatas
224 .iter()
225 .find(|col| col.semantic_type == SemanticType::Timestamp)
226 .unwrap();
227 let request_unit = time_index_column
228 .column_schema
229 .data_type
230 .as_timestamp()
231 .unwrap()
232 .unit();
233 ensure!(
234 request_unit == unit,
235 UnexpectedRequestSnafu {
236 reason: format!(
237 "Metric has differenttime unit ({:?}) than the physical region ({:?})",
238 request_unit, unit
239 ),
240 }
241 );
242 }
243
244 let requests = {
246 let state = self.state.read().unwrap();
247 let mut skipped = Vec::with_capacity(requests.len());
248 let mut kept_requests = Vec::with_capacity(requests.len());
249
250 for (region_id, request) in requests {
251 if state.is_logical_region_exist(region_id) {
252 skipped.push(region_id);
253 } else {
254 kept_requests.push((region_id, request));
255 }
256 }
257
258 if !skipped.is_empty() {
260 info!(
261 "Skipped creating logical regions {skipped:?} because they already exist",
262 skipped = skipped
263 );
264 }
265 kept_requests
266 };
267
268 let mut new_column_names = HashSet::new();
270 let mut new_columns = Vec::new();
271
272 let index_option = {
273 let state = &self.state.read().unwrap();
274 let region_state = state
275 .physical_region_states()
276 .get(&data_region_id)
277 .with_context(|| PhysicalRegionNotFoundSnafu {
278 region_id: data_region_id,
279 })?;
280 let physical_columns = region_state.physical_columns();
281
282 extract_new_columns(
283 &requests,
284 physical_columns,
285 &mut new_column_names,
286 &mut new_columns,
287 )?;
288 region_state.options().index
289 };
290
291 self.data_region
293 .add_columns(data_region_id, new_columns, index_option)
294 .await?;
295
296 let physical_columns = self.data_region.physical_columns(data_region_id).await?;
297 let physical_schema_map = physical_columns
298 .iter()
299 .map(|metadata| (metadata.column_schema.name.as_str(), metadata))
300 .collect::<HashMap<_, _>>();
301 let logical_regions = requests
302 .iter()
303 .map(|(region_id, _)| *region_id)
304 .collect::<Vec<_>>();
305 let logical_region_columns = requests.iter().map(|(region_id, request)| {
306 (
307 *region_id,
308 request
309 .column_metadatas
310 .iter()
311 .map(|metadata| {
312 let column_metadata = *physical_schema_map
314 .get(metadata.column_schema.name.as_str())
315 .unwrap();
316 (metadata.column_schema.name.as_str(), column_metadata)
317 })
318 .collect::<HashMap<_, _>>(),
319 )
320 });
321
322 let new_add_columns = new_column_names.iter().map(|name| {
323 let column_metadata = *physical_schema_map.get(name).unwrap();
325 (name.to_string(), column_metadata.clone())
326 });
327
328 extension_return_value.insert(
329 ALTER_PHYSICAL_EXTENSION_KEY.to_string(),
330 ColumnMetadata::encode_list(&physical_columns).context(SerializeColumnMetadataSnafu)?,
331 );
332
333 self.metadata_region
335 .add_logical_regions(physical_region_id, true, logical_region_columns)
336 .await?;
337
338 {
339 let mut state = self.state.write().unwrap();
340 state.add_physical_columns(data_region_id, new_add_columns);
341 state.add_logical_regions(physical_region_id, logical_regions.clone());
342 }
343 for logical_region_id in logical_regions {
344 self.metadata_region
345 .open_logical_region(logical_region_id)
346 .await;
347 }
348
349 Ok(())
350 }
351
352 fn verify_region_create_request(request: &RegionCreateRequest) -> Result<()> {
357 request.validate().context(InvalidMetadataSnafu)?;
358
359 let name_to_index = request
360 .column_metadatas
361 .iter()
362 .enumerate()
363 .map(|(idx, metadata)| (metadata.column_schema.name.clone(), idx))
364 .collect::<HashMap<String, usize>>();
365
366 let table_id_col_def = request.column_metadatas.iter().any(is_metric_name_col);
367 let tsid_col_def = request.column_metadatas.iter().any(is_tsid_col);
368
369 ensure!(
371 !name_to_index.contains_key(DATA_SCHEMA_TABLE_ID_COLUMN_NAME) || table_id_col_def,
372 InternalColumnOccupiedSnafu {
373 column: DATA_SCHEMA_TABLE_ID_COLUMN_NAME,
374 }
375 );
376 ensure!(
377 !name_to_index.contains_key(DATA_SCHEMA_TSID_COLUMN_NAME) || tsid_col_def,
378 InternalColumnOccupiedSnafu {
379 column: DATA_SCHEMA_TSID_COLUMN_NAME,
380 }
381 );
382
383 ensure!(
385 request.is_physical_table() || request.options.contains_key(LOGICAL_TABLE_METADATA_KEY),
386 MissingRegionOptionSnafu {}
387 );
388 ensure!(
389 !(request.is_physical_table()
390 && request.options.contains_key(LOGICAL_TABLE_METADATA_KEY)),
391 ConflictRegionOptionSnafu {}
392 );
393
394 let mut field_cols = Vec::new();
396 for col in &request.column_metadatas {
397 if is_metric_engine_internal_column(&col.column_schema.name) {
399 continue;
400 }
401 match col.semantic_type {
402 SemanticType::Tag => ensure!(
403 col.column_schema.data_type == ConcreteDataType::string_datatype(),
404 ColumnTypeMismatchSnafu {
405 expect: ConcreteDataType::string_datatype(),
406 actual: col.column_schema.data_type.clone(),
407 }
408 ),
409 SemanticType::Field => {
410 field_cols.push(col);
411 }
412 SemanticType::Timestamp => {}
413 }
414 }
415 let [field_col] = field_cols.as_slice() else {
416 if field_cols.is_empty() {
417 NoFieldColumnSnafu.fail()?;
418 }
419 return MultipleFieldColumnSnafu {
420 previous: field_cols[0].column_schema.name.clone(),
421 current: field_cols[1].column_schema.name.clone(),
422 }
423 .fail();
424 };
425
426 if is_native_histogram_value_schema(
427 &field_col.column_schema.name,
428 &field_col.column_schema.data_type,
429 ) {
430 return Ok(());
431 }
432
433 ensure!(
435 field_col.column_schema.data_type == ConcreteDataType::float64_datatype(),
436 ColumnTypeMismatchSnafu {
437 expect: ConcreteDataType::float64_datatype(),
438 actual: field_col.column_schema.data_type.clone(),
439 }
440 );
441
442 Ok(())
443 }
444
445 fn transform_region_id(region_id: RegionId) -> (RegionId, RegionId) {
449 (
450 to_data_region_id(region_id),
451 to_metadata_region_id(region_id),
452 )
453 }
454
455 pub fn create_request_for_metadata_region(
459 &self,
460 request: &RegionCreateRequest,
461 ) -> RegionCreateRequest {
462 let timestamp_column_metadata = ColumnMetadata {
464 column_id: METADATA_SCHEMA_TIMESTAMP_COLUMN_INDEX as _,
465 semantic_type: SemanticType::Timestamp,
466 column_schema: ColumnSchema::new(
467 METADATA_SCHEMA_TIMESTAMP_COLUMN_NAME,
468 ConcreteDataType::timestamp_millisecond_datatype(),
469 false,
470 )
471 .with_default_constraint(Some(datatypes::schema::ColumnDefaultConstraint::Value(
472 Value::Timestamp(Timestamp::new_millisecond(0)),
473 )))
474 .unwrap(),
475 };
476 let key_column_metadata = ColumnMetadata {
478 column_id: METADATA_SCHEMA_KEY_COLUMN_INDEX as _,
479 semantic_type: SemanticType::Tag,
480 column_schema: ColumnSchema::new(
481 METADATA_SCHEMA_KEY_COLUMN_NAME,
482 ConcreteDataType::string_datatype(),
483 false,
484 ),
485 };
486 let value_column_metadata = ColumnMetadata {
488 column_id: METADATA_SCHEMA_VALUE_COLUMN_INDEX as _,
489 semantic_type: SemanticType::Field,
490 column_schema: ColumnSchema::new(
491 METADATA_SCHEMA_VALUE_COLUMN_NAME,
492 ConcreteDataType::string_datatype(),
493 true,
494 ),
495 };
496
497 let options = region_options_for_metadata_region(&request.options);
498 RegionCreateRequest {
499 engine: MITO_ENGINE_NAME.to_string(),
500 column_metadatas: vec![
501 timestamp_column_metadata,
502 key_column_metadata,
503 value_column_metadata,
504 ],
505 primary_key: vec![METADATA_SCHEMA_KEY_COLUMN_INDEX as _],
506 options,
507 table_dir: request.table_dir.clone(),
508 path_type: PathType::Metadata,
509 partition_expr_json: Some("".to_string()),
510 requirements: request.requirements,
511 }
512 }
513
514 pub fn create_request_for_data_region(
521 &self,
522 request: &RegionCreateRequest,
523 ) -> RegionCreateRequest {
524 let mut data_region_request = request.clone();
525 let mut primary_key = vec![ReservedColumnId::table_id(), ReservedColumnId::tsid()];
526
527 data_region_request.table_dir = request.table_dir.clone();
528 data_region_request.path_type = PathType::Data;
529
530 let table_id_col_def = request.column_metadatas.iter().any(is_metric_name_col);
531 let tsid_col_def = request.column_metadatas.iter().any(is_tsid_col);
532
533 data_region_request
535 .column_metadatas
536 .iter_mut()
537 .for_each(|metadata| {
538 if metadata.semantic_type == SemanticType::Tag
539 && !is_metric_name_col(metadata)
540 && !is_tsid_col(metadata)
541 {
542 metadata.column_schema.set_nullable();
543 primary_key.push(metadata.column_id);
544 }
545 });
546
547 if !table_id_col_def {
549 data_region_request.column_metadatas.push(table_id_col());
550 }
551 if !tsid_col_def {
552 data_region_request.column_metadatas.push(tsid_col());
553 }
554 data_region_request.primary_key = primary_key;
555
556 set_data_region_options(
558 &mut data_region_request.options,
559 self.config.sparse_primary_key_encoding,
560 );
561
562 data_region_request
563 }
564}
565
566fn table_id_col() -> ColumnMetadata {
567 ColumnMetadata {
568 column_id: ReservedColumnId::table_id(),
569 semantic_type: SemanticType::Tag,
570 column_schema: ColumnSchema::new(
571 DATA_SCHEMA_TABLE_ID_COLUMN_NAME,
572 ConcreteDataType::uint32_datatype(),
573 false,
574 )
575 .with_skipping_options(SkippingIndexOptions::new_unchecked(
576 DEFAULT_TABLE_ID_SKIPPING_INDEX_GRANULARITY,
577 DEFAULT_TABLE_ID_SKIPPING_INDEX_FALSE_POSITIVE_RATE,
578 datatypes::schema::SkippingIndexType::BloomFilter,
579 ))
580 .unwrap(),
581 }
582}
583
584fn tsid_col() -> ColumnMetadata {
585 ColumnMetadata {
586 column_id: ReservedColumnId::tsid(),
587 semantic_type: SemanticType::Tag,
588 column_schema: ColumnSchema::new(
589 DATA_SCHEMA_TSID_COLUMN_NAME,
590 ConcreteDataType::uint64_datatype(),
591 false,
592 )
593 .with_inverted_index(false),
594 }
595}
596
597pub(crate) fn is_metric_name_col(column: &ColumnMetadata) -> bool {
599 column.column_id == ReservedColumnId::table_id()
600 && column.semantic_type == SemanticType::Tag
601 && column.column_schema.data_type == ConcreteDataType::uint32_datatype()
602 && column.column_schema.name == DATA_SCHEMA_TABLE_ID_COLUMN_NAME
603 && !column.column_schema.is_nullable()
604}
605
606pub(crate) fn is_tsid_col(column: &ColumnMetadata) -> bool {
608 column.column_id == ReservedColumnId::tsid()
609 && column.semantic_type == SemanticType::Tag
610 && column.column_schema.data_type == ConcreteDataType::uint64_datatype()
611 && column.column_schema.name == DATA_SCHEMA_TSID_COLUMN_NAME
612 && !column.column_schema.is_nullable()
613}
614
615fn group_create_logical_region_requests_by_physical_region_id(
617 requests: Vec<(RegionId, RegionCreateRequest)>,
618) -> Result<HashMap<RegionId, Vec<(RegionId, RegionCreateRequest)>>> {
619 let mut result = HashMap::with_capacity(requests.len());
620 for (region_id, request) in requests {
621 let physical_region_id = parse_physical_region_id(&request)?;
622 result
623 .entry(physical_region_id)
624 .or_insert_with(Vec::new)
625 .push((region_id, request));
626 }
627
628 Ok(result)
629}
630
631fn parse_physical_region_id(request: &RegionCreateRequest) -> Result<RegionId> {
633 let physical_region_id_raw = request
634 .options
635 .get(LOGICAL_TABLE_METADATA_KEY)
636 .ok_or(MissingRegionOptionSnafu {}.build())?;
637
638 let physical_region_id: RegionId = physical_region_id_raw
639 .parse::<u64>()
640 .with_context(|_| ParseRegionIdSnafu {
641 raw: physical_region_id_raw,
642 })?
643 .into();
644
645 Ok(physical_region_id)
646}
647
648pub(crate) fn region_options_for_metadata_region(
650 original: &HashMap<String, String>,
651) -> HashMap<String, String> {
652 let mut metadata_region_options = HashMap::new();
653 metadata_region_options.insert(TTL_KEY.to_string(), FOREVER.to_string());
654
655 if let Some(wal_options) = original.get(WAL_OPTIONS_KEY) {
656 metadata_region_options.insert(WAL_OPTIONS_KEY.to_string(), wal_options.clone());
657 }
658
659 metadata_region_options
660}
661
662#[cfg(test)]
663mod test {
664 use common_meta::ddl::test_util::assert_column_name_and_id;
665 use common_meta::ddl::utils::{parse_column_metadatas, parse_manifest_infos_from_extensions};
666 use common_query::native_histogram::{NATIVE_HISTOGRAM_FIELD, native_histogram_value_type};
667 use common_query::prelude::{greptime_timestamp, greptime_value};
668 use store_api::metric_engine_consts::{METRIC_ENGINE_NAME, PHYSICAL_TABLE_METADATA_KEY};
669 use store_api::region_request::{BatchRegionDdlRequest, RegionRequirements};
670
671 use super::*;
672 use crate::config::EngineConfig;
673 use crate::engine::MetricEngine;
674 use crate::test_util::{TestEnv, create_logical_region_request};
675
676 #[test]
677 fn test_internal_column_metadata() {
678 let table_id_col = table_id_col();
679 let tsid_col = tsid_col();
680 assert!(is_metric_name_col(&table_id_col));
681 assert!(is_tsid_col(&tsid_col));
682 }
683
684 #[test]
685 fn test_verify_region_create_request() {
686 let request = RegionCreateRequest {
688 column_metadatas: vec![
689 ColumnMetadata {
690 column_id: 0,
691 semantic_type: SemanticType::Timestamp,
692 column_schema: ColumnSchema::new(
693 METADATA_SCHEMA_TIMESTAMP_COLUMN_NAME,
694 ConcreteDataType::timestamp_millisecond_datatype(),
695 false,
696 ),
697 },
698 ColumnMetadata {
699 column_id: 1,
700 semantic_type: SemanticType::Tag,
701 column_schema: ColumnSchema::new(
702 DATA_SCHEMA_TABLE_ID_COLUMN_NAME,
703 ConcreteDataType::uint32_datatype(),
704 false,
705 ),
706 },
707 ],
708 table_dir: "test_dir".to_string(),
709 path_type: PathType::Bare,
710 engine: METRIC_ENGINE_NAME.to_string(),
711 primary_key: vec![],
712 options: HashMap::new(),
713 partition_expr_json: Some("".to_string()),
714 requirements: RegionRequirements::object_storage(),
715 };
716 let result = MetricEngineInner::verify_region_create_request(&request);
717 assert!(result.is_err());
718 assert_eq!(
719 result.unwrap_err().to_string(),
720 "Internal column __table_id is reserved".to_string()
721 );
722
723 let request = RegionCreateRequest {
725 column_metadatas: vec![
726 ColumnMetadata {
727 column_id: 0,
728 semantic_type: SemanticType::Timestamp,
729 column_schema: ColumnSchema::new(
730 METADATA_SCHEMA_TIMESTAMP_COLUMN_NAME,
731 ConcreteDataType::timestamp_millisecond_datatype(),
732 false,
733 ),
734 },
735 ColumnMetadata {
736 column_id: 1,
737 semantic_type: SemanticType::Tag,
738 column_schema: ColumnSchema::new(
739 "column1".to_string(),
740 ConcreteDataType::string_datatype(),
741 false,
742 ),
743 },
744 ColumnMetadata {
745 column_id: 2,
746 semantic_type: SemanticType::Field,
747 column_schema: ColumnSchema::new(
748 "column2".to_string(),
749 ConcreteDataType::float64_datatype(),
750 false,
751 ),
752 },
753 table_id_col(),
754 tsid_col(),
755 ],
756 table_dir: "test_dir".to_string(),
757 path_type: PathType::Bare,
758 engine: METRIC_ENGINE_NAME.to_string(),
759 primary_key: vec![],
760 options: [(PHYSICAL_TABLE_METADATA_KEY.to_string(), String::new())]
761 .into_iter()
762 .collect(),
763 partition_expr_json: Some("".to_string()),
764 requirements: Default::default(),
765 };
766 MetricEngineInner::verify_region_create_request(&request).unwrap();
767
768 let request = RegionCreateRequest {
770 column_metadatas: vec![
771 ColumnMetadata {
772 column_id: 0,
773 semantic_type: SemanticType::Timestamp,
774 column_schema: ColumnSchema::new(
775 METADATA_SCHEMA_TIMESTAMP_COLUMN_NAME,
776 ConcreteDataType::timestamp_millisecond_datatype(),
777 false,
778 ),
779 },
780 ColumnMetadata {
781 column_id: 1,
782 semantic_type: SemanticType::Tag,
783 column_schema: ColumnSchema::new(
784 "column1".to_string(),
785 ConcreteDataType::string_datatype(),
786 false,
787 ),
788 },
789 ColumnMetadata {
790 column_id: 2,
791 semantic_type: SemanticType::Field,
792 column_schema: ColumnSchema::new(
793 "column2".to_string(),
794 ConcreteDataType::float64_datatype(),
795 false,
796 ),
797 },
798 ],
799 table_dir: "test_dir".to_string(),
800 path_type: PathType::Bare,
801 engine: METRIC_ENGINE_NAME.to_string(),
802 primary_key: vec![],
803 options: [(PHYSICAL_TABLE_METADATA_KEY.to_string(), String::new())]
804 .into_iter()
805 .collect(),
806 partition_expr_json: Some("".to_string()),
807 requirements: Default::default(),
808 };
809 MetricEngineInner::verify_region_create_request(&request).unwrap();
810 }
811
812 #[test]
813 fn test_verify_region_create_request_options() {
814 let mut request = RegionCreateRequest {
815 column_metadatas: vec![
816 ColumnMetadata {
817 column_id: 0,
818 semantic_type: SemanticType::Timestamp,
819 column_schema: ColumnSchema::new(
820 METADATA_SCHEMA_TIMESTAMP_COLUMN_NAME,
821 ConcreteDataType::timestamp_millisecond_datatype(),
822 false,
823 ),
824 },
825 ColumnMetadata {
826 column_id: 1,
827 semantic_type: SemanticType::Field,
828 column_schema: ColumnSchema::new(
829 "val".to_string(),
830 ConcreteDataType::float64_datatype(),
831 false,
832 ),
833 },
834 ],
835 table_dir: "test_dir".to_string(),
836 path_type: PathType::Bare,
837 engine: METRIC_ENGINE_NAME.to_string(),
838 primary_key: vec![],
839 options: HashMap::new(),
840 partition_expr_json: Some("".to_string()),
841 requirements: Default::default(),
842 };
843 MetricEngineInner::verify_region_create_request(&request).unwrap_err();
844
845 let mut options = HashMap::new();
846 options.insert(PHYSICAL_TABLE_METADATA_KEY.to_string(), "value".to_string());
847 request.options.clone_from(&options);
848 MetricEngineInner::verify_region_create_request(&request).unwrap();
849
850 options.insert(LOGICAL_TABLE_METADATA_KEY.to_string(), "value".to_string());
851 request.options.clone_from(&options);
852 MetricEngineInner::verify_region_create_request(&request).unwrap_err();
853
854 options.remove(PHYSICAL_TABLE_METADATA_KEY).unwrap();
855 request.options = options;
856 MetricEngineInner::verify_region_create_request(&request).unwrap();
857 }
858
859 #[test]
860 fn test_verify_region_create_request_native_histogram_fields() {
861 let native_histogram_columns = vec![
862 ColumnMetadata {
863 column_id: 0,
864 semantic_type: SemanticType::Timestamp,
865 column_schema: ColumnSchema::new(
866 greptime_timestamp(),
867 ConcreteDataType::timestamp_millisecond_datatype(),
868 false,
869 ),
870 },
871 ColumnMetadata {
872 column_id: 1,
873 semantic_type: SemanticType::Tag,
874 column_schema: ColumnSchema::new("job", ConcreteDataType::string_datatype(), true),
875 },
876 ColumnMetadata {
877 column_id: 2,
878 semantic_type: SemanticType::Field,
879 column_schema: ColumnSchema::new(
880 NATIVE_HISTOGRAM_FIELD,
881 native_histogram_value_type().clone(),
882 true,
883 ),
884 },
885 ];
886 let request = RegionCreateRequest {
887 column_metadatas: native_histogram_columns,
888 table_dir: "test_dir".to_string(),
889 path_type: PathType::Bare,
890 engine: METRIC_ENGINE_NAME.to_string(),
891 primary_key: vec![],
892 options: [(
893 LOGICAL_TABLE_METADATA_KEY.to_string(),
894 "physical".to_string(),
895 )]
896 .into_iter()
897 .collect(),
898 partition_expr_json: Some("".to_string()),
899 requirements: Default::default(),
900 };
901 MetricEngineInner::verify_region_create_request(&request).unwrap();
902
903 let request = RegionCreateRequest {
904 column_metadatas: vec![
905 ColumnMetadata {
906 column_id: 0,
907 semantic_type: SemanticType::Timestamp,
908 column_schema: ColumnSchema::new(
909 greptime_timestamp(),
910 ConcreteDataType::timestamp_millisecond_datatype(),
911 false,
912 ),
913 },
914 ColumnMetadata {
915 column_id: 1,
916 semantic_type: SemanticType::Field,
917 column_schema: ColumnSchema::new(
918 "value_a",
919 ConcreteDataType::float64_datatype(),
920 true,
921 ),
922 },
923 ColumnMetadata {
924 column_id: 2,
925 semantic_type: SemanticType::Field,
926 column_schema: ColumnSchema::new(
927 "value_b",
928 ConcreteDataType::float64_datatype(),
929 true,
930 ),
931 },
932 ],
933 table_dir: "test_dir".to_string(),
934 path_type: PathType::Bare,
935 engine: METRIC_ENGINE_NAME.to_string(),
936 primary_key: vec![],
937 options: [(
938 LOGICAL_TABLE_METADATA_KEY.to_string(),
939 "physical".to_string(),
940 )]
941 .into_iter()
942 .collect(),
943 partition_expr_json: Some("".to_string()),
944 requirements: Default::default(),
945 };
946 assert!(MetricEngineInner::verify_region_create_request(&request).is_err());
947 }
948
949 #[tokio::test]
950 async fn test_create_request_for_physical_regions() {
951 let options: HashMap<_, _> = [
953 ("ttl".to_string(), "60m".to_string()),
954 ("skip_wal".to_string(), "true".to_string()),
955 ]
956 .into_iter()
957 .collect();
958 let request = RegionCreateRequest {
959 engine: METRIC_ENGINE_NAME.to_string(),
960 column_metadatas: vec![
961 ColumnMetadata {
962 column_id: 0,
963 semantic_type: SemanticType::Timestamp,
964 column_schema: ColumnSchema::new(
965 "timestamp",
966 ConcreteDataType::timestamp_millisecond_datatype(),
967 false,
968 ),
969 },
970 ColumnMetadata {
971 column_id: 1,
972 semantic_type: SemanticType::Tag,
973 column_schema: ColumnSchema::new(
974 "tag",
975 ConcreteDataType::string_datatype(),
976 false,
977 ),
978 },
979 ],
980 primary_key: vec![0],
981 options,
982 table_dir: "/test_dir".to_string(),
983 path_type: PathType::Bare,
984 partition_expr_json: Some("".to_string()),
985 requirements: RegionRequirements::object_storage(),
986 };
987
988 let env = TestEnv::new().await;
990 let engine = MetricEngine::try_new(env.mito(), EngineConfig::default()).unwrap();
991 let engine_inner = engine.inner;
992
993 let data_region_request = engine_inner.create_request_for_data_region(&request);
995 assert_eq!(data_region_request.table_dir, "/test_dir".to_string());
996 assert_eq!(data_region_request.path_type, PathType::Data);
997 assert_eq!(data_region_request.column_metadatas.len(), 4);
998 assert_eq!(
999 data_region_request.primary_key,
1000 vec![ReservedColumnId::table_id(), ReservedColumnId::tsid(), 1]
1001 );
1002 assert!(data_region_request.options.contains_key("ttl"));
1003 assert_eq!(
1004 data_region_request.requirements,
1005 RegionRequirements::object_storage()
1006 );
1007
1008 let metadata_region_request = engine_inner.create_request_for_metadata_region(&request);
1010 assert_eq!(metadata_region_request.table_dir, "/test_dir".to_string());
1011 assert_eq!(metadata_region_request.path_type, PathType::Metadata);
1012 assert_eq!(
1013 metadata_region_request.options.get("ttl").unwrap(),
1014 "forever"
1015 );
1016 assert!(!metadata_region_request.options.contains_key("skip_wal"));
1017 assert_eq!(
1018 metadata_region_request.requirements,
1019 RegionRequirements::object_storage()
1020 );
1021 }
1022
1023 #[tokio::test]
1024 async fn test_create_request_for_physical_regions_with_internal_columns() {
1025 let options: HashMap<_, _> = [
1026 ("ttl".to_string(), "60m".to_string()),
1027 ("skip_wal".to_string(), "true".to_string()),
1028 ]
1029 .into_iter()
1030 .collect();
1031 let request = RegionCreateRequest {
1032 engine: METRIC_ENGINE_NAME.to_string(),
1033 column_metadatas: vec![
1034 ColumnMetadata {
1035 column_id: 0,
1036 semantic_type: SemanticType::Timestamp,
1037 column_schema: ColumnSchema::new(
1038 "timestamp",
1039 ConcreteDataType::timestamp_millisecond_datatype(),
1040 false,
1041 ),
1042 },
1043 ColumnMetadata {
1044 column_id: 1,
1045 semantic_type: SemanticType::Tag,
1046 column_schema: ColumnSchema::new(
1047 "tag",
1048 ConcreteDataType::string_datatype(),
1049 false,
1050 ),
1051 },
1052 ColumnMetadata {
1053 column_id: 2,
1054 semantic_type: SemanticType::Field,
1055 column_schema: ColumnSchema::new(
1056 "value",
1057 ConcreteDataType::float64_datatype(),
1058 false,
1059 ),
1060 },
1061 table_id_col(),
1062 tsid_col(),
1063 ],
1064 primary_key: vec![0],
1065 options,
1066 table_dir: "/test_dir".to_string(),
1067 path_type: PathType::Bare,
1068 partition_expr_json: Some("".to_string()),
1069 requirements: Default::default(),
1070 };
1071
1072 let env = TestEnv::new().await;
1073 let engine = MetricEngine::try_new(env.mito(), EngineConfig::default()).unwrap();
1074 let engine_inner = engine.inner;
1075
1076 let data_region_request = engine_inner.create_request_for_data_region(&request);
1077 assert_eq!(data_region_request.column_metadatas.len(), 5);
1078 assert_eq!(
1079 data_region_request.primary_key,
1080 vec![ReservedColumnId::table_id(), ReservedColumnId::tsid(), 1]
1081 );
1082
1083 let table_id_count = data_region_request
1084 .column_metadatas
1085 .iter()
1086 .filter(|metadata| metadata.column_schema.name == DATA_SCHEMA_TABLE_ID_COLUMN_NAME)
1087 .count();
1088 let tsid_count = data_region_request
1089 .column_metadatas
1090 .iter()
1091 .filter(|metadata| metadata.column_schema.name == DATA_SCHEMA_TSID_COLUMN_NAME)
1092 .count();
1093 assert_eq!(table_id_count, 1);
1094 assert_eq!(tsid_count, 1);
1095
1096 let tag_metadata = data_region_request
1097 .column_metadatas
1098 .iter()
1099 .find(|metadata| metadata.column_schema.name == "tag")
1100 .unwrap();
1101 assert!(tag_metadata.column_schema.is_nullable());
1102
1103 let table_id_metadata = data_region_request
1104 .column_metadatas
1105 .iter()
1106 .find(|metadata| metadata.column_schema.name == DATA_SCHEMA_TABLE_ID_COLUMN_NAME)
1107 .unwrap();
1108 assert!(is_metric_name_col(table_id_metadata));
1109
1110 let tsid_metadata = data_region_request
1111 .column_metadatas
1112 .iter()
1113 .find(|metadata| metadata.column_schema.name == DATA_SCHEMA_TSID_COLUMN_NAME)
1114 .unwrap();
1115 assert!(is_tsid_col(tsid_metadata));
1116 }
1117
1118 #[tokio::test]
1119 async fn test_create_logical_regions() {
1120 let env = TestEnv::new().await;
1121 let engine = env.metric();
1122 let physical_region_id1 = RegionId::new(1024, 0);
1123 let physical_region_id2 = RegionId::new(1024, 1);
1124 let logical_region_id1 = RegionId::new(1025, 0);
1125 let logical_region_id2 = RegionId::new(1025, 1);
1126 env.create_physical_region(physical_region_id1, "/test_dir1", vec![])
1127 .await;
1128 env.create_physical_region(physical_region_id2, "/test_dir2", vec![])
1129 .await;
1130
1131 let region_create_request1 =
1132 create_logical_region_request(&["job"], physical_region_id1, "logical1");
1133 let region_create_request2 =
1134 create_logical_region_request(&["job"], physical_region_id2, "logical2");
1135
1136 let response = engine
1137 .handle_batch_ddl_requests(BatchRegionDdlRequest::Create(vec![
1138 (logical_region_id1, region_create_request1),
1139 (logical_region_id2, region_create_request2),
1140 ]))
1141 .await
1142 .unwrap();
1143
1144 let manifest_infos = parse_manifest_infos_from_extensions(&response.extensions).unwrap();
1145 assert_eq!(manifest_infos.len(), 2);
1146 let region_ids = manifest_infos.into_iter().map(|i| i.0).collect::<Vec<_>>();
1147 assert!(region_ids.contains(&physical_region_id1));
1148 assert!(region_ids.contains(&physical_region_id2));
1149
1150 let column_metadatas =
1151 parse_column_metadatas(&response.extensions, ALTER_PHYSICAL_EXTENSION_KEY).unwrap();
1152 assert_column_name_and_id(
1153 &column_metadatas,
1154 &[
1155 (greptime_timestamp(), 0),
1156 (greptime_value(), 1),
1157 ("__table_id", ReservedColumnId::table_id()),
1158 ("__tsid", ReservedColumnId::tsid()),
1159 ("job", 2),
1160 ],
1161 );
1162 }
1163}