1use common_catalog::consts::{
34 RESOURCE_ATTRIBUTES_COLUMN, SCOPE_ATTRIBUTES_COLUMN, SPAN_ATTRIBUTES_COLUMN,
35};
36use datatypes::prelude::ConcreteDataType;
37use datatypes::schema::{ColumnSchema, Schema};
38
39use crate::requests::TABLE_DATA_MODEL_TRACE_V2;
40
41pub const SEMANTIC_PREFIX: &str = "greptime.semantic.";
43
44pub const SEMANTIC_PER_TABLE_INDEX_KEY: &str = "greptime.internal.semantic.per_table_index";
49
50pub const SEMANTIC_TABLE_SCOPE: &str = "greptime.semantic.table_scope";
56
57pub const SEMANTIC_SIGNAL_TYPE: &str = "greptime.semantic.signal_type";
60pub const SEMANTIC_SOURCE: &str = "greptime.semantic.source";
62pub const SEMANTIC_SOURCE_VERSION: &str = "greptime.semantic.source_version";
64pub const SEMANTIC_PIPELINE: &str = "greptime.semantic.pipeline";
67
68pub const SEMANTIC_TRACE_CONVENTIONS: &str = "greptime.semantic.trace.conventions";
73
74pub const SEMANTIC_METRIC_TYPE: &str = "greptime.semantic.metric.type";
79pub const SEMANTIC_METRIC_UNIT: &str = "greptime.semantic.metric.unit";
82pub const SEMANTIC_METRIC_TEMPORALITY: &str = "greptime.semantic.metric.temporality";
85pub const SEMANTIC_METRIC_METADATA_QUALITY: &str = "greptime.semantic.metric.metadata_quality";
88pub const SEMANTIC_METRIC_ORIGINAL_NAME: &str = "greptime.semantic.metric.original_name";
91
92pub const SEMANTIC_ENTITY_PREFIX: &str = "greptime.semantic.entity.";
101
102pub const SEMANTIC_ENTITY_SERVICE_ID: &str = "greptime.semantic.entity.service.id";
105
106#[derive(Debug, Clone, Copy, PartialEq, Eq)]
111pub enum EntityRole {
112 Id,
117 Descriptive,
118 Scope,
119}
120
121impl EntityRole {
122 fn parse(role: &str) -> Option<Self> {
123 match role {
124 "id" => Some(EntityRole::Id),
125 "descriptive" => Some(EntityRole::Descriptive),
126 "scope" => Some(EntityRole::Scope),
127 _ => None,
128 }
129 }
130}
131
132pub const SIGNAL_TYPE_TRACE: &str = "trace";
135pub const SIGNAL_TYPE_LOG: &str = "log";
136pub const SIGNAL_TYPE_METRIC: &str = "metric";
137pub const SIGNAL_TYPE_EVENT: &str = "event";
138
139pub const SOURCE_OPENTELEMETRY: &str = "opentelemetry";
140pub const SOURCE_PROMETHEUS: &str = "prometheus";
141pub const SOURCE_INFLUXDB: &str = "influxdb";
142pub const SOURCE_OPENTSDB: &str = "opentsdb";
143pub const SOURCE_LOKI: &str = "loki";
144pub const SOURCE_ELASTICSEARCH: &str = "elasticsearch";
145
146pub const METADATA_QUALITY_DECLARED: &str = "declared";
147pub const METADATA_QUALITY_INFERRED: &str = "inferred";
148
149pub const METRIC_TEMPORALITY_CUMULATIVE: &str = "cumulative";
150pub const METRIC_TEMPORALITY_DELTA: &str = "delta";
151
152pub const SEMANTIC_VALUE_UNKNOWN: &str = "unknown";
154pub const SEMANTIC_VALUE_MIXED: &str = "mixed";
156
157pub const SEMANTIC_OPTION_KEYS: &[&str] = &[
162 SEMANTIC_TABLE_SCOPE,
163 SEMANTIC_SIGNAL_TYPE,
164 SEMANTIC_SOURCE,
165 SEMANTIC_SOURCE_VERSION,
166 SEMANTIC_PIPELINE,
167 SEMANTIC_TRACE_CONVENTIONS,
168 SEMANTIC_METRIC_TYPE,
169 SEMANTIC_METRIC_UNIT,
170 SEMANTIC_METRIC_TEMPORALITY,
171 SEMANTIC_METRIC_METADATA_QUALITY,
172 SEMANTIC_METRIC_ORIGINAL_NAME,
173];
174
175fn is_valid_entity_type(ty: &str) -> bool {
180 !ty.is_empty()
181 && ty.split('.').all(|seg| {
182 !seg.is_empty()
183 && seg
184 .bytes()
185 .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_')
186 })
187}
188
189pub fn parse_entity_option_key(key: &str) -> Option<(&str, EntityRole)> {
195 let rest = key.strip_prefix(SEMANTIC_ENTITY_PREFIX)?;
196 let (ty, role) = rest.rsplit_once('.')?;
197 if !is_valid_entity_type(ty) {
198 return None;
199 }
200 Some((ty, EntityRole::parse(role)?))
201}
202
203pub fn is_entity_option_key(key: &str) -> bool {
205 parse_entity_option_key(key).is_some()
206}
207
208pub fn trace_v2_attribute<'a>(
212 schema: &Schema,
213 data_model: Option<&str>,
214 column: &'a str,
215) -> Option<(&'a str, &'a str)> {
216 if data_model != Some(TABLE_DATA_MODEL_TRACE_V2)
217 || schema.column_schema_by_name(column).is_some()
218 {
219 return None;
220 }
221 let (root, key) = column.split_once('.')?;
222 (!key.is_empty()
223 && schema
224 .column_schema_by_name(root)
225 .is_some_and(is_v2_attribute_column))
226 .then_some((root, key))
227}
228
229fn is_v2_attribute_column(schema: &ColumnSchema) -> bool {
230 matches!(
231 schema.name.as_str(),
232 RESOURCE_ATTRIBUTES_COLUMN | SCOPE_ATTRIBUTES_COLUMN | SPAN_ATTRIBUTES_COLUMN
233 ) && schema.data_type.is_json2()
234}
235
236pub fn has_stable_string_form(data_type: &ConcreteDataType) -> bool {
242 !matches!(
243 data_type,
244 ConcreteDataType::Binary(_)
245 | ConcreteDataType::Json(_)
246 | ConcreteDataType::Vector(_)
247 | ConcreteDataType::List(_)
248 | ConcreteDataType::Struct(_)
249 | ConcreteDataType::Dictionary(_)
250 | ConcreteDataType::Null(_)
251 )
252}
253
254pub fn parse_entity_columns(value: &str) -> Vec<String> {
258 value
259 .split(',')
260 .map(str::trim)
261 .filter(|column| !column.is_empty())
262 .map(String::from)
263 .collect()
264}
265
266pub fn is_semantic_option_key(key: &str) -> bool {
273 SEMANTIC_OPTION_KEYS.contains(&key) || is_entity_option_key(key)
274}
275
276pub fn validate_semantic_option(key: &str, value: &str) -> bool {
287 if is_entity_option_key(key) {
288 return !value.is_empty() && value.split(',').all(|column| !column.trim().is_empty());
289 }
290 match key {
291 SEMANTIC_TABLE_SCOPE => matches!(value, "catalog" | "global"),
292 SEMANTIC_PIPELINE
293 | SEMANTIC_SOURCE_VERSION
294 | SEMANTIC_METRIC_UNIT
295 | SEMANTIC_METRIC_ORIGINAL_NAME
296 | SEMANTIC_TRACE_CONVENTIONS => !value.is_empty(),
297
298 SEMANTIC_SIGNAL_TYPE => matches!(value, "trace" | "log" | "metric" | "event" | "unknown"),
299 SEMANTIC_SOURCE => matches!(
300 value,
301 "opentelemetry"
302 | "prometheus"
303 | "influxdb"
304 | "opentsdb"
305 | "elasticsearch"
306 | "loki"
307 | "custom"
308 | "mixed"
309 | "unknown"
310 ),
311 SEMANTIC_METRIC_TYPE => matches!(
312 value,
313 "counter"
314 | "gauge"
315 | "histogram"
316 | "summary"
317 | "updown_counter"
318 | "gauge_histogram"
319 | "info"
320 | "stateset"
321 | "mixed"
322 | "unknown"
323 ),
324 SEMANTIC_METRIC_TEMPORALITY => {
325 matches!(
326 value,
327 METRIC_TEMPORALITY_CUMULATIVE | METRIC_TEMPORALITY_DELTA | "mixed" | "unknown"
328 )
329 }
330 SEMANTIC_METRIC_METADATA_QUALITY => matches!(value, "declared" | "inferred" | "unknown"),
331
332 _ => false,
333 }
334}
335
336#[cfg(test)]
337mod tests {
338 use datatypes::types::JsonType;
339
340 use super::*;
341
342 #[test]
343 fn test_trace_v2_attribute_references() {
344 let schema = Schema::new(vec![
345 ColumnSchema::new(
346 RESOURCE_ATTRIBUTES_COLUMN,
347 ConcreteDataType::Json(JsonType::null()),
348 true,
349 ),
350 ColumnSchema::new(
351 SCOPE_ATTRIBUTES_COLUMN,
352 ConcreteDataType::string_datatype(),
353 true,
354 ),
355 ColumnSchema::new(
356 "resource_attributes.physical",
357 ConcreteDataType::string_datatype(),
358 true,
359 ),
360 ]);
361 let model = Some(TABLE_DATA_MODEL_TRACE_V2);
362 for key in ["host.name", "host..name", "host[0]", "host.*", r#"a"b\c.d"#] {
363 let column = format!("resource_attributes.{key}");
364 assert_eq!(
365 trace_v2_attribute(&schema, model, &column),
366 Some((RESOURCE_ATTRIBUTES_COLUMN, key))
367 );
368 }
369 for column in [
370 "resource_attributes",
371 "resource_attributes.",
372 "scope_attributes.name",
373 "span_attributes.name",
374 "resource_attributes.physical",
375 "missing.key",
376 ] {
377 assert_eq!(trace_v2_attribute(&schema, model, column), None, "{column}");
378 }
379 for model in [None, Some(crate::requests::TABLE_DATA_MODEL_TRACE_V1)] {
380 assert_eq!(
381 trace_v2_attribute(&schema, model, "resource_attributes.host.name"),
382 None
383 );
384 }
385 }
386
387 #[test]
388 fn test_is_semantic_option_key() {
389 assert!(is_semantic_option_key(SEMANTIC_SIGNAL_TYPE));
390 assert!(is_semantic_option_key(SEMANTIC_METRIC_TYPE));
391 assert!(is_semantic_option_key(SEMANTIC_PIPELINE));
392
393 assert!(!is_semantic_option_key("greptime.semantic.future.key"));
395 assert!(!is_semantic_option_key("greptime.semantic.unknown_key"));
396 assert!(!is_semantic_option_key(
398 "greptime.semantic.metric.monotonic"
399 ));
400 assert!(!is_semantic_option_key(
401 "greptime.semantic.resource.attributes_dropped"
402 ));
403 assert!(!is_semantic_option_key("greptime.semanticx"));
405 assert!(!is_semantic_option_key("semantic.signal_type"));
406 assert!(!is_semantic_option_key("table_data_model"));
407 assert!(!is_semantic_option_key(SEMANTIC_PER_TABLE_INDEX_KEY));
409 }
410
411 #[test]
412 fn test_validate_semantic_option() {
413 assert!(validate_semantic_option(SEMANTIC_TABLE_SCOPE, "catalog"));
415 assert!(validate_semantic_option(SEMANTIC_TABLE_SCOPE, "global"));
416 assert!(!validate_semantic_option(SEMANTIC_TABLE_SCOPE, "cluster"));
417 assert!(!validate_semantic_option(SEMANTIC_TABLE_SCOPE, "unknown"));
418 assert!(!validate_semantic_option(SEMANTIC_TABLE_SCOPE, ""));
419 assert!(validate_semantic_option(SEMANTIC_SIGNAL_TYPE, "metric"));
420 assert!(!validate_semantic_option(SEMANTIC_SIGNAL_TYPE, "spans"));
421 assert!(validate_semantic_option(SEMANTIC_METRIC_TYPE, "counter"));
422 assert!(validate_semantic_option(SEMANTIC_METRIC_TYPE, "mixed"));
423 assert!(!validate_semantic_option(SEMANTIC_METRIC_TYPE, "bogus"));
424
425 assert!(validate_semantic_option(
427 SEMANTIC_METRIC_TEMPORALITY,
428 "unknown"
429 ));
430 assert!(validate_semantic_option(SEMANTIC_METRIC_UNIT, "By"));
431 assert!(!validate_semantic_option(SEMANTIC_METRIC_UNIT, ""));
432 assert!(validate_semantic_option(
433 SEMANTIC_PIPELINE,
434 "greptime_trace_v1"
435 ));
436
437 assert!(!validate_semantic_option(
439 "greptime.semantic.metric.monotonic",
440 "true"
441 ));
442 assert!(!validate_semantic_option(
444 "greptime.semantic.future.key",
445 "x"
446 ));
447
448 assert!(validate_semantic_option(
450 SEMANTIC_SIGNAL_TYPE,
451 SIGNAL_TYPE_TRACE
452 ));
453 assert!(validate_semantic_option(
454 SEMANTIC_SIGNAL_TYPE,
455 SIGNAL_TYPE_METRIC
456 ));
457 assert!(validate_semantic_option(
458 SEMANTIC_SIGNAL_TYPE,
459 SIGNAL_TYPE_LOG
460 ));
461 assert!(validate_semantic_option(
462 SEMANTIC_SOURCE,
463 SOURCE_OPENTELEMETRY
464 ));
465 assert!(validate_semantic_option(SEMANTIC_SOURCE, SOURCE_PROMETHEUS));
466 assert!(validate_semantic_option(SEMANTIC_SOURCE_VERSION, "2.0"));
467 assert!(validate_semantic_option(
468 SEMANTIC_METRIC_METADATA_QUALITY,
469 METADATA_QUALITY_INFERRED
470 ));
471 assert!(validate_semantic_option(
472 SEMANTIC_METRIC_METADATA_QUALITY,
473 METADATA_QUALITY_DECLARED
474 ));
475 assert!(validate_semantic_option(
476 SEMANTIC_TRACE_CONVENTIONS,
477 SEMANTIC_VALUE_UNKNOWN
478 ));
479 for key in SEMANTIC_OPTION_KEYS {
481 assert!(
482 !validate_semantic_option(key, ""),
483 "empty value should never validate for {key}"
484 );
485 }
486 }
487
488 #[test]
489 fn test_entity_option_key() {
490 for key in [
493 "greptime.semantic.entity.service.id",
494 "greptime.semantic.entity.k8s.pod.id",
495 "greptime.semantic.entity.service.instance.descriptive",
496 "greptime.semantic.entity.host.scope",
497 "greptime.semantic.entity.agent.id",
498 ] {
499 assert!(is_entity_option_key(key), "should accept {key}");
500 assert!(is_semantic_option_key(key), "should recognise {key}");
501 }
502
503 for key in [
506 "greptime.semantic.entity..id",
507 "greptime.semantic.entity.service.bogusrole",
508 "greptime.semantic.entity.service",
509 "greptime.semantic.entity.",
510 "greptime.semantic.entity.Service.id",
511 ] {
512 assert!(!is_entity_option_key(key), "should reject {key}");
513 assert!(!is_semantic_option_key(key), "should not recognise {key}");
514 }
515
516 assert!(!is_entity_option_key(SEMANTIC_SIGNAL_TYPE));
518
519 assert!(is_entity_option_key(SEMANTIC_ENTITY_SERVICE_ID));
521 assert!(is_semantic_option_key(SEMANTIC_ENTITY_SERVICE_ID));
522 }
523
524 #[test]
525 fn test_validate_entity_option() {
526 assert!(validate_semantic_option(
528 "greptime.semantic.entity.service.id",
529 "service_name"
530 ));
531 assert!(validate_semantic_option(
532 "greptime.semantic.entity.process.id",
533 "pid,start_time"
534 ));
535 assert!(!validate_semantic_option(
537 "greptime.semantic.entity.service.id",
538 ""
539 ));
540 assert!(!validate_semantic_option(
541 "greptime.semantic.entity.process.id",
542 "pid,"
543 ));
544 assert!(!validate_semantic_option(
546 "greptime.semantic.entity.service.bogusrole",
547 "service_name"
548 ));
549 }
550}