1use common_catalog::consts::{
34 RESOURCE_ATTRIBUTES_COLUMN, SCOPE_ATTRIBUTES_COLUMN, SPAN_ATTRIBUTES_COLUMN,
35};
36use datatypes::prelude::ConcreteDataType;
37use datatypes::schema::{ColumnSchema, Schema};
38
39use crate::requests::TABLE_DATA_MODEL_TRACE_V2;
40
41pub const SEMANTIC_PREFIX: &str = "greptime.semantic.";
43
44pub const SEMANTIC_PER_TABLE_INDEX_KEY: &str = "greptime.internal.semantic.per_table_index";
49
50pub const SEMANTIC_TABLE_SCOPE: &str = "greptime.semantic.table_scope";
57
58pub const SEMANTIC_SIGNAL_TYPE: &str = "greptime.semantic.signal_type";
61pub const SEMANTIC_SOURCE: &str = "greptime.semantic.source";
63pub const SEMANTIC_SOURCE_VERSION: &str = "greptime.semantic.source_version";
65pub const SEMANTIC_PIPELINE: &str = "greptime.semantic.pipeline";
68
69pub const SEMANTIC_TRACE_CONVENTIONS: &str = "greptime.semantic.trace.conventions";
74
75pub const SEMANTIC_METRIC_TYPE: &str = "greptime.semantic.metric.type";
80pub const SEMANTIC_METRIC_UNIT: &str = "greptime.semantic.metric.unit";
83pub const SEMANTIC_METRIC_TEMPORALITY: &str = "greptime.semantic.metric.temporality";
86pub const SEMANTIC_METRIC_METADATA_QUALITY: &str = "greptime.semantic.metric.metadata_quality";
89pub const SEMANTIC_METRIC_ORIGINAL_NAME: &str = "greptime.semantic.metric.original_name";
92
93pub const SEMANTIC_ENTITY_PREFIX: &str = "greptime.semantic.entity.";
102
103pub const SEMANTIC_ENTITY_SERVICE_ID: &str = "greptime.semantic.entity.service.id";
106
107#[derive(Debug, Clone, Copy, PartialEq, Eq)]
112pub enum EntityRole {
113 Id,
118 Descriptive,
119 Scope,
120}
121
122impl EntityRole {
123 fn parse(role: &str) -> Option<Self> {
124 match role {
125 "id" => Some(EntityRole::Id),
126 "descriptive" => Some(EntityRole::Descriptive),
127 "scope" => Some(EntityRole::Scope),
128 _ => None,
129 }
130 }
131}
132
133pub const SIGNAL_TYPE_TRACE: &str = "trace";
136pub const SIGNAL_TYPE_LOG: &str = "log";
137pub const SIGNAL_TYPE_METRIC: &str = "metric";
138pub const SIGNAL_TYPE_EVENT: &str = "event";
139
140pub const SOURCE_OPENTELEMETRY: &str = "opentelemetry";
141pub const SOURCE_PROMETHEUS: &str = "prometheus";
142pub const SOURCE_INFLUXDB: &str = "influxdb";
143pub const SOURCE_OPENTSDB: &str = "opentsdb";
144pub const SOURCE_LOKI: &str = "loki";
145pub const SOURCE_ELASTICSEARCH: &str = "elasticsearch";
146
147pub const METADATA_QUALITY_DECLARED: &str = "declared";
148pub const METADATA_QUALITY_INFERRED: &str = "inferred";
149
150pub const METRIC_TEMPORALITY_CUMULATIVE: &str = "cumulative";
151pub const METRIC_TEMPORALITY_DELTA: &str = "delta";
152
153pub const SEMANTIC_VALUE_UNKNOWN: &str = "unknown";
155pub const SEMANTIC_VALUE_MIXED: &str = "mixed";
157
158pub const SEMANTIC_OPTION_KEYS: &[&str] = &[
163 SEMANTIC_TABLE_SCOPE,
164 SEMANTIC_SIGNAL_TYPE,
165 SEMANTIC_SOURCE,
166 SEMANTIC_SOURCE_VERSION,
167 SEMANTIC_PIPELINE,
168 SEMANTIC_TRACE_CONVENTIONS,
169 SEMANTIC_METRIC_TYPE,
170 SEMANTIC_METRIC_UNIT,
171 SEMANTIC_METRIC_TEMPORALITY,
172 SEMANTIC_METRIC_METADATA_QUALITY,
173 SEMANTIC_METRIC_ORIGINAL_NAME,
174];
175
176fn is_valid_entity_type(ty: &str) -> bool {
181 !ty.is_empty()
182 && ty.split('.').all(|seg| {
183 !seg.is_empty()
184 && seg
185 .bytes()
186 .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_')
187 })
188}
189
190pub fn parse_entity_option_key(key: &str) -> Option<(&str, EntityRole)> {
196 let rest = key.strip_prefix(SEMANTIC_ENTITY_PREFIX)?;
197 let (ty, role) = rest.rsplit_once('.')?;
198 if !is_valid_entity_type(ty) {
199 return None;
200 }
201 Some((ty, EntityRole::parse(role)?))
202}
203
204pub fn is_entity_option_key(key: &str) -> bool {
206 parse_entity_option_key(key).is_some()
207}
208
209pub fn trace_v2_attribute<'a>(
213 schema: &Schema,
214 data_model: Option<&str>,
215 column: &'a str,
216) -> Option<(&'a str, &'a str)> {
217 if data_model != Some(TABLE_DATA_MODEL_TRACE_V2)
218 || schema.column_schema_by_name(column).is_some()
219 {
220 return None;
221 }
222 let (root, key) = column.split_once('.')?;
223 (!key.is_empty()
224 && schema
225 .column_schema_by_name(root)
226 .is_some_and(is_v2_attribute_column))
227 .then_some((root, key))
228}
229
230fn is_v2_attribute_column(schema: &ColumnSchema) -> bool {
231 matches!(
232 schema.name.as_str(),
233 RESOURCE_ATTRIBUTES_COLUMN | SCOPE_ATTRIBUTES_COLUMN | SPAN_ATTRIBUTES_COLUMN
234 ) && schema.data_type.is_json2()
235}
236
237pub fn has_stable_string_form(data_type: &ConcreteDataType) -> bool {
243 !matches!(
244 data_type,
245 ConcreteDataType::Binary(_)
246 | ConcreteDataType::Json(_)
247 | ConcreteDataType::Vector(_)
248 | ConcreteDataType::List(_)
249 | ConcreteDataType::Struct(_)
250 | ConcreteDataType::Dictionary(_)
251 | ConcreteDataType::Null(_)
252 )
253}
254
255pub fn parse_entity_columns(value: &str) -> Vec<String> {
259 value
260 .split(',')
261 .map(str::trim)
262 .filter(|column| !column.is_empty())
263 .map(String::from)
264 .collect()
265}
266
267pub fn is_semantic_option_key(key: &str) -> bool {
274 SEMANTIC_OPTION_KEYS.contains(&key) || is_entity_option_key(key)
275}
276
277pub fn validate_semantic_option(key: &str, value: &str) -> bool {
288 if is_entity_option_key(key) {
289 return !value.is_empty() && value.split(',').all(|column| !column.trim().is_empty());
290 }
291 match key {
292 SEMANTIC_TABLE_SCOPE => matches!(value, "catalog" | "public_global" | "global"),
293 SEMANTIC_PIPELINE
294 | SEMANTIC_SOURCE_VERSION
295 | SEMANTIC_METRIC_UNIT
296 | SEMANTIC_METRIC_ORIGINAL_NAME
297 | SEMANTIC_TRACE_CONVENTIONS => !value.is_empty(),
298
299 SEMANTIC_SIGNAL_TYPE => matches!(value, "trace" | "log" | "metric" | "event" | "unknown"),
300 SEMANTIC_SOURCE => matches!(
301 value,
302 "opentelemetry"
303 | "prometheus"
304 | "influxdb"
305 | "opentsdb"
306 | "elasticsearch"
307 | "loki"
308 | "custom"
309 | "mixed"
310 | "unknown"
311 ),
312 SEMANTIC_METRIC_TYPE => matches!(
313 value,
314 "counter"
315 | "gauge"
316 | "histogram"
317 | "summary"
318 | "updown_counter"
319 | "gauge_histogram"
320 | "info"
321 | "stateset"
322 | "mixed"
323 | "unknown"
324 ),
325 SEMANTIC_METRIC_TEMPORALITY => {
326 matches!(
327 value,
328 METRIC_TEMPORALITY_CUMULATIVE | METRIC_TEMPORALITY_DELTA | "mixed" | "unknown"
329 )
330 }
331 SEMANTIC_METRIC_METADATA_QUALITY => matches!(value, "declared" | "inferred" | "unknown"),
332
333 _ => false,
334 }
335}
336
337#[cfg(test)]
338mod tests {
339 use datatypes::types::JsonType;
340
341 use super::*;
342
343 #[test]
344 fn test_trace_v2_attribute_references() {
345 let schema = Schema::new(vec![
346 ColumnSchema::new(
347 RESOURCE_ATTRIBUTES_COLUMN,
348 ConcreteDataType::Json(JsonType::null()),
349 true,
350 ),
351 ColumnSchema::new(
352 SCOPE_ATTRIBUTES_COLUMN,
353 ConcreteDataType::string_datatype(),
354 true,
355 ),
356 ColumnSchema::new(
357 "resource_attributes.physical",
358 ConcreteDataType::string_datatype(),
359 true,
360 ),
361 ]);
362 let model = Some(TABLE_DATA_MODEL_TRACE_V2);
363 for key in ["host.name", "host..name", "host[0]", "host.*", r#"a"b\c.d"#] {
364 let column = format!("resource_attributes.{key}");
365 assert_eq!(
366 trace_v2_attribute(&schema, model, &column),
367 Some((RESOURCE_ATTRIBUTES_COLUMN, key))
368 );
369 }
370 for column in [
371 "resource_attributes",
372 "resource_attributes.",
373 "scope_attributes.name",
374 "span_attributes.name",
375 "resource_attributes.physical",
376 "missing.key",
377 ] {
378 assert_eq!(trace_v2_attribute(&schema, model, column), None, "{column}");
379 }
380 for model in [None, Some(crate::requests::TABLE_DATA_MODEL_TRACE_V1)] {
381 assert_eq!(
382 trace_v2_attribute(&schema, model, "resource_attributes.host.name"),
383 None
384 );
385 }
386 }
387
388 #[test]
389 fn test_is_semantic_option_key() {
390 assert!(is_semantic_option_key(SEMANTIC_SIGNAL_TYPE));
391 assert!(is_semantic_option_key(SEMANTIC_METRIC_TYPE));
392 assert!(is_semantic_option_key(SEMANTIC_PIPELINE));
393
394 assert!(!is_semantic_option_key("greptime.semantic.future.key"));
396 assert!(!is_semantic_option_key("greptime.semantic.unknown_key"));
397 assert!(!is_semantic_option_key(
399 "greptime.semantic.metric.monotonic"
400 ));
401 assert!(!is_semantic_option_key(
402 "greptime.semantic.resource.attributes_dropped"
403 ));
404 assert!(!is_semantic_option_key("greptime.semanticx"));
406 assert!(!is_semantic_option_key("semantic.signal_type"));
407 assert!(!is_semantic_option_key("table_data_model"));
408 assert!(!is_semantic_option_key(SEMANTIC_PER_TABLE_INDEX_KEY));
410 }
411
412 #[test]
413 fn test_validate_semantic_option() {
414 assert!(validate_semantic_option(SEMANTIC_TABLE_SCOPE, "catalog"));
416 assert!(validate_semantic_option(
417 SEMANTIC_TABLE_SCOPE,
418 "public_global"
419 ));
420 assert!(validate_semantic_option(SEMANTIC_TABLE_SCOPE, "global"));
421 assert!(!validate_semantic_option(SEMANTIC_TABLE_SCOPE, "cluster"));
422 assert!(!validate_semantic_option(SEMANTIC_TABLE_SCOPE, "unknown"));
423 assert!(!validate_semantic_option(SEMANTIC_TABLE_SCOPE, ""));
424 assert!(validate_semantic_option(SEMANTIC_SIGNAL_TYPE, "metric"));
425 assert!(!validate_semantic_option(SEMANTIC_SIGNAL_TYPE, "spans"));
426 assert!(validate_semantic_option(SEMANTIC_METRIC_TYPE, "counter"));
427 assert!(validate_semantic_option(SEMANTIC_METRIC_TYPE, "mixed"));
428 assert!(!validate_semantic_option(SEMANTIC_METRIC_TYPE, "bogus"));
429
430 assert!(validate_semantic_option(
432 SEMANTIC_METRIC_TEMPORALITY,
433 "unknown"
434 ));
435 assert!(validate_semantic_option(SEMANTIC_METRIC_UNIT, "By"));
436 assert!(!validate_semantic_option(SEMANTIC_METRIC_UNIT, ""));
437 assert!(validate_semantic_option(
438 SEMANTIC_PIPELINE,
439 "greptime_trace_v1"
440 ));
441
442 assert!(!validate_semantic_option(
444 "greptime.semantic.metric.monotonic",
445 "true"
446 ));
447 assert!(!validate_semantic_option(
449 "greptime.semantic.future.key",
450 "x"
451 ));
452
453 assert!(validate_semantic_option(
455 SEMANTIC_SIGNAL_TYPE,
456 SIGNAL_TYPE_TRACE
457 ));
458 assert!(validate_semantic_option(
459 SEMANTIC_SIGNAL_TYPE,
460 SIGNAL_TYPE_METRIC
461 ));
462 assert!(validate_semantic_option(
463 SEMANTIC_SIGNAL_TYPE,
464 SIGNAL_TYPE_LOG
465 ));
466 assert!(validate_semantic_option(
467 SEMANTIC_SOURCE,
468 SOURCE_OPENTELEMETRY
469 ));
470 assert!(validate_semantic_option(SEMANTIC_SOURCE, SOURCE_PROMETHEUS));
471 assert!(validate_semantic_option(SEMANTIC_SOURCE_VERSION, "2.0"));
472 assert!(validate_semantic_option(
473 SEMANTIC_METRIC_METADATA_QUALITY,
474 METADATA_QUALITY_INFERRED
475 ));
476 assert!(validate_semantic_option(
477 SEMANTIC_METRIC_METADATA_QUALITY,
478 METADATA_QUALITY_DECLARED
479 ));
480 assert!(validate_semantic_option(
481 SEMANTIC_TRACE_CONVENTIONS,
482 SEMANTIC_VALUE_UNKNOWN
483 ));
484 for key in SEMANTIC_OPTION_KEYS {
486 assert!(
487 !validate_semantic_option(key, ""),
488 "empty value should never validate for {key}"
489 );
490 }
491 }
492
493 #[test]
494 fn test_entity_option_key() {
495 for key in [
498 "greptime.semantic.entity.service.id",
499 "greptime.semantic.entity.k8s.pod.id",
500 "greptime.semantic.entity.service.instance.descriptive",
501 "greptime.semantic.entity.host.scope",
502 "greptime.semantic.entity.agent.id",
503 ] {
504 assert!(is_entity_option_key(key), "should accept {key}");
505 assert!(is_semantic_option_key(key), "should recognise {key}");
506 }
507
508 for key in [
511 "greptime.semantic.entity..id",
512 "greptime.semantic.entity.service.bogusrole",
513 "greptime.semantic.entity.service",
514 "greptime.semantic.entity.",
515 "greptime.semantic.entity.Service.id",
516 ] {
517 assert!(!is_entity_option_key(key), "should reject {key}");
518 assert!(!is_semantic_option_key(key), "should not recognise {key}");
519 }
520
521 assert!(!is_entity_option_key(SEMANTIC_SIGNAL_TYPE));
523
524 assert!(is_entity_option_key(SEMANTIC_ENTITY_SERVICE_ID));
526 assert!(is_semantic_option_key(SEMANTIC_ENTITY_SERVICE_ID));
527 }
528
529 #[test]
530 fn test_validate_entity_option() {
531 assert!(validate_semantic_option(
533 "greptime.semantic.entity.service.id",
534 "service_name"
535 ));
536 assert!(validate_semantic_option(
537 "greptime.semantic.entity.process.id",
538 "pid,start_time"
539 ));
540 assert!(!validate_semantic_option(
542 "greptime.semantic.entity.service.id",
543 ""
544 ));
545 assert!(!validate_semantic_option(
546 "greptime.semantic.entity.process.id",
547 "pid,"
548 ));
549 assert!(!validate_semantic_option(
551 "greptime.semantic.entity.service.bogusrole",
552 "service_name"
553 ));
554 }
555}