table/requests/
semantic.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Table semantic layer vocabulary.
16//!
17//! A thin layer of semantic metadata attached to a table via `table_options`, so
18//! machine consumers (LLM agents, alert/dashboard builders, MCP servers, ETL) can
19//! align a table with the observability concept it stands for without guessing
20//! from column names. See `docs/rfcs/2026-05-28-table-semantic-layer.md`.
21//!
22//! The vocabulary is intentionally small: a key earns its place only when it
23//! records something a consumer cannot cheaply and reliably recover from the
24//! schema/data itself. Keys whose value is already in the metric name by
25//! convention, is a constant, or duplicates an existing column are deliberately
26//! omitted rather than stamped for completeness.
27//!
28//! All public keys share the [`SEMANTIC_PREFIX`] namespace and are string-valued.
29//! [`is_semantic_option_key`] gates them through
30//! [`crate::requests::validate_table_option`], so they are accepted both on the
31//! ingestion auto-create path and on explicit `CREATE TABLE ... WITH (...)` DDL.
32
33use common_catalog::consts::{
34    RESOURCE_ATTRIBUTES_COLUMN, SCOPE_ATTRIBUTES_COLUMN, SPAN_ATTRIBUTES_COLUMN,
35};
36use datatypes::prelude::ConcreteDataType;
37use datatypes::schema::{ColumnSchema, Schema};
38
39use crate::requests::TABLE_DATA_MODEL_TRACE_V2;
40
41/// Reserved prefix for every public semantic table-option key.
42pub const SEMANTIC_PREFIX: &str = "greptime.semantic.";
43
44/// Internal `QueryContext` extension key carrying the per-table semantic index
45/// (a `{table_name -> {semantic_key: value}}` JSON blob) from the ingestion
46/// encode path to the auto-create site. Deliberately OUTSIDE [`SEMANTIC_PREFIX`]
47/// so it is not a valid table option and never leaks into a table's options.
48pub const SEMANTIC_PER_TABLE_INDEX_KEY: &str = "greptime.internal.semantic.per_table_index";
49
50// ---- Common keys (all signals) ----
51
52/// Signal kind: one of [`SIGNAL_TYPE_TRACE`] / [`SIGNAL_TYPE_LOG`] /
53/// [`SIGNAL_TYPE_METRIC`] / [`SIGNAL_TYPE_EVENT`].
54pub const SEMANTIC_SIGNAL_TYPE: &str = "greptime.semantic.signal_type";
55/// Ingestion ecosystem, e.g. [`SOURCE_OPENTELEMETRY`] / [`SOURCE_PROMETHEUS`].
56pub const SEMANTIC_SOURCE: &str = "greptime.semantic.source";
57/// Source protocol version, e.g. Prometheus remote write `1.0` / `2.0`.
58pub const SEMANTIC_SOURCE_VERSION: &str = "greptime.semantic.source_version";
59/// Internal ingestion pipeline / data model, e.g. `greptime_trace_v1`. The
60/// signal-agnostic successor to the engine-specific `table_data_model` option.
61pub const SEMANTIC_PIPELINE: &str = "greptime.semantic.pipeline";
62
63// ---- Trace keys ----
64
65/// Semantic-conventions version the rows conform to (e.g. the OTel schema URL),
66/// or [`SEMANTIC_VALUE_UNKNOWN`] / [`SEMANTIC_VALUE_MIXED`] when not single-valued.
67pub const SEMANTIC_TRACE_CONVENTIONS: &str = "greptime.semantic.trace.conventions";
68
69// ---- Metric keys ----
70
71/// Instrument kind: `counter` / `gauge` / `histogram` / `summary` /
72/// `updown_counter` / `gauge_histogram` / `info` / `stateset`.
73pub const SEMANTIC_METRIC_TYPE: &str = "greptime.semantic.metric.type";
74/// UCUM unit, e.g. `s`, `By`, `{request}`. Discarded by the row encoders, so it
75/// is unrecoverable once ingested.
76pub const SEMANTIC_METRIC_UNIT: &str = "greptime.semantic.metric.unit";
77/// Catalog-level `cumulative` / `delta` / `mixed` description for OTLP metrics.
78/// Per-series query behavior is determined from stored row identity instead.
79pub const SEMANTIC_METRIC_TEMPORALITY: &str = "greptime.semantic.metric.temporality";
80/// [`METADATA_QUALITY_DECLARED`] when the protocol stated the type, or
81/// [`METADATA_QUALITY_INFERRED`] when guessed from a name suffix.
82pub const SEMANTIC_METRIC_METADATA_QUALITY: &str = "greptime.semantic.metric.metadata_quality";
83/// Pre-translation OTel name when the table name was Prometheus-ised; the key a
84/// consumer uses to look the metric up in the OTel semantic conventions.
85pub const SEMANTIC_METRIC_ORIGINAL_NAME: &str = "greptime.semantic.metric.original_name";
86
87// ---- Entity keys (open sub-namespace) ----
88
89/// Reserved prefix for the entity-identity sub-namespace:
90/// `greptime.semantic.entity.<type>.{id|descriptive|scope}`. Unlike the rest of
91/// the vocabulary (a closed whitelist), entity types are open-ended (`service`,
92/// `host`, `k8s.pod`, `process`, `agent`, custom, ...), so keys here are validated
93/// by prefix + shape rather than membership. See
94/// `docs/rfcs/2026-06-25-entity-relationships-and-graph-query.md`.
95pub const SEMANTIC_ENTITY_PREFIX: &str = "greptime.semantic.entity.";
96
97/// The well-known entity-identity key auto-stamped on OTLP trace tables; its value
98/// is the `service_name` tag column, declaring the logical `service` entity.
99pub const SEMANTIC_ENTITY_SERVICE_ID: &str = "greptime.semantic.entity.service.id";
100
101/// The role a set of columns plays for an entity: `id` (identifying
102/// attributes), `descriptive`, or `scope`. Columns may be tags or fields; DDL
103/// validation only requires that they exist and render as stable strings
104/// ([`has_stable_string_form`]).
105#[derive(Debug, Clone, Copy, PartialEq, Eq)]
106pub enum EntityRole {
107    /// Identifying columns. Their **order is part of the identity**: the
108    /// entity id is their values joined in the declared order, so two tables
109    /// naming the same entity must list them the same way (broad to narrow)
110    /// or they name two entities.
111    Id,
112    Descriptive,
113    Scope,
114}
115
116impl EntityRole {
117    fn parse(role: &str) -> Option<Self> {
118        match role {
119            "id" => Some(EntityRole::Id),
120            "descriptive" => Some(EntityRole::Descriptive),
121            "scope" => Some(EntityRole::Scope),
122            _ => None,
123        }
124    }
125}
126
127// ---- Value constants ----
128
129pub const SIGNAL_TYPE_TRACE: &str = "trace";
130pub const SIGNAL_TYPE_LOG: &str = "log";
131pub const SIGNAL_TYPE_METRIC: &str = "metric";
132pub const SIGNAL_TYPE_EVENT: &str = "event";
133
134pub const SOURCE_OPENTELEMETRY: &str = "opentelemetry";
135pub const SOURCE_PROMETHEUS: &str = "prometheus";
136pub const SOURCE_INFLUXDB: &str = "influxdb";
137pub const SOURCE_OPENTSDB: &str = "opentsdb";
138pub const SOURCE_LOKI: &str = "loki";
139pub const SOURCE_ELASTICSEARCH: &str = "elasticsearch";
140
141pub const METADATA_QUALITY_DECLARED: &str = "declared";
142pub const METADATA_QUALITY_INFERRED: &str = "inferred";
143
144pub const METRIC_TEMPORALITY_CUMULATIVE: &str = "cumulative";
145pub const METRIC_TEMPORALITY_DELTA: &str = "delta";
146
147/// Sentinel for a key that cannot be determined at stamp time.
148pub const SEMANTIC_VALUE_UNKNOWN: &str = "unknown";
149/// Sentinel for a single-valued key that saw conflicting sources.
150pub const SEMANTIC_VALUE_MIXED: &str = "mixed";
151
152/// Every recognised public semantic table-option key. The set is a closed
153/// whitelist: keys under [`SEMANTIC_PREFIX`] that are not listed here are rejected,
154/// so an unknown key like `greptime.semantic.unknown_key` does not silently land
155/// in a table's options. Adding a key to the vocabulary means adding it here.
156pub const SEMANTIC_OPTION_KEYS: &[&str] = &[
157    SEMANTIC_SIGNAL_TYPE,
158    SEMANTIC_SOURCE,
159    SEMANTIC_SOURCE_VERSION,
160    SEMANTIC_PIPELINE,
161    SEMANTIC_TRACE_CONVENTIONS,
162    SEMANTIC_METRIC_TYPE,
163    SEMANTIC_METRIC_UNIT,
164    SEMANTIC_METRIC_TEMPORALITY,
165    SEMANTIC_METRIC_METADATA_QUALITY,
166    SEMANTIC_METRIC_ORIGINAL_NAME,
167];
168
169/// Returns true if `ty` is a syntactically valid entity type, e.g. `service`,
170/// `host`, `k8s.pod`, `service.instance`. An entity type is one or more
171/// dot-separated segments, each a non-empty `[a-z0-9_]+` token. The dotted form
172/// carries the two-entity-layer convention (`service` vs `service.instance`).
173fn is_valid_entity_type(ty: &str) -> bool {
174    !ty.is_empty()
175        && ty.split('.').all(|seg| {
176            !seg.is_empty()
177                && seg
178                    .bytes()
179                    .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_')
180        })
181}
182
183/// Parses a well-formed entity-identity option key of the shape
184/// `greptime.semantic.entity.<type>.{id|descriptive|scope}` into
185/// `(entity_type, role)`. The `<type>` may itself contain dots; the role is the
186/// final dot-separated segment. This is the single parser of the key format —
187/// DDL validation and the read-time derivation both go through it.
188pub fn parse_entity_option_key(key: &str) -> Option<(&str, EntityRole)> {
189    let rest = key.strip_prefix(SEMANTIC_ENTITY_PREFIX)?;
190    let (ty, role) = rest.rsplit_once('.')?;
191    if !is_valid_entity_type(ty) {
192        return None;
193    }
194    Some((ty, EntityRole::parse(role)?))
195}
196
197/// Returns true if `key` is a well-formed entity-identity option key.
198pub fn is_entity_option_key(key: &str) -> bool {
199    parse_entity_option_key(key).is_some()
200}
201
202/// Resolves a Trace V2 entity reference to a JSON2 root and a literal attribute key.
203/// Everything after the first dot belongs to the key, matching Trace V1's
204/// flattened column names. Existing physical columns take precedence.
205pub fn trace_v2_attribute<'a>(
206    schema: &Schema,
207    data_model: Option<&str>,
208    column: &'a str,
209) -> Option<(&'a str, &'a str)> {
210    if data_model != Some(TABLE_DATA_MODEL_TRACE_V2)
211        || schema.column_schema_by_name(column).is_some()
212    {
213        return None;
214    }
215    let (root, key) = column.split_once('.')?;
216    (!key.is_empty()
217        && schema
218            .column_schema_by_name(root)
219            .is_some_and(is_v2_attribute_column))
220    .then_some((root, key))
221}
222
223fn is_v2_attribute_column(schema: &ColumnSchema) -> bool {
224    matches!(
225        schema.name.as_str(),
226        RESOURCE_ATTRIBUTES_COLUMN | SCOPE_ATTRIBUTES_COLUMN | SPAN_ATTRIBUTES_COLUMN
227    ) && schema.data_type.is_json2()
228}
229
230/// Returns true if a column of `data_type` renders as a stable string — the
231/// requirement for entity id/descriptive/scope columns. The read-time
232/// derivation casts them to strings, so a type without a stable string form
233/// would fail only when the graph is scanned; DDL validation rejects it up
234/// front instead.
235pub fn has_stable_string_form(data_type: &ConcreteDataType) -> bool {
236    !matches!(
237        data_type,
238        ConcreteDataType::Binary(_)
239            | ConcreteDataType::Json(_)
240            | ConcreteDataType::Vector(_)
241            | ConcreteDataType::List(_)
242            | ConcreteDataType::Struct(_)
243            | ConcreteDataType::Dictionary(_)
244            | ConcreteDataType::Null(_)
245    )
246}
247
248/// Tokenizes an entity option's comma-separated column list (trimmed, empty
249/// tokens dropped). [`validate_semantic_option`] rejects empty tokens at DDL
250/// time, so readers only ever drop what validation already refused.
251pub fn parse_entity_columns(value: &str) -> Vec<String> {
252    value
253        .split(',')
254        .map(str::trim)
255        .filter(|column| !column.is_empty())
256        .map(String::from)
257        .collect()
258}
259
260/// Returns true if `key` is a recognised semantic table-option key.
261///
262/// Two acceptance rules: membership in the closed [`SEMANTIC_OPTION_KEYS`]
263/// whitelist, OR the open entity sub-namespace ([`is_entity_option_key`], validated
264/// by prefix + shape). Everything else under [`SEMANTIC_PREFIX`] is rejected, and
265/// the internal [`SEMANTIC_PER_TABLE_INDEX_KEY`] (outside the prefix) never matches.
266pub fn is_semantic_option_key(key: &str) -> bool {
267    SEMANTIC_OPTION_KEYS.contains(&key) || is_entity_option_key(key)
268}
269
270/// Validates a `greptime.semantic.*` option's `value` against its allowed domain.
271///
272/// Open-value keys (unit, original_name, pipeline, conventions) accept any
273/// non-empty string. Closed-domain keys accept a fixed set, plus the `unknown`
274/// sentinel, plus `mixed` for the keys where one long-lived table can
275/// legitimately see multiple values. Entity keys ([`is_entity_option_key`]) take a
276/// comma-separated column-name list (each token non-empty); column existence and
277/// the stable-string-form rule for entity columns are enforced later against
278/// the table schema at DDL time, not here. Keys that are neither whitelisted nor a well-formed entity key
279/// are rejected.
280pub fn validate_semantic_option(key: &str, value: &str) -> bool {
281    if is_entity_option_key(key) {
282        return !value.is_empty() && value.split(',').all(|column| !column.trim().is_empty());
283    }
284    match key {
285        SEMANTIC_PIPELINE
286        | SEMANTIC_SOURCE_VERSION
287        | SEMANTIC_METRIC_UNIT
288        | SEMANTIC_METRIC_ORIGINAL_NAME
289        | SEMANTIC_TRACE_CONVENTIONS => !value.is_empty(),
290
291        SEMANTIC_SIGNAL_TYPE => matches!(value, "trace" | "log" | "metric" | "event" | "unknown"),
292        SEMANTIC_SOURCE => matches!(
293            value,
294            "opentelemetry"
295                | "prometheus"
296                | "influxdb"
297                | "opentsdb"
298                | "elasticsearch"
299                | "loki"
300                | "custom"
301                | "mixed"
302                | "unknown"
303        ),
304        SEMANTIC_METRIC_TYPE => matches!(
305            value,
306            "counter"
307                | "gauge"
308                | "histogram"
309                | "summary"
310                | "updown_counter"
311                | "gauge_histogram"
312                | "info"
313                | "stateset"
314                | "mixed"
315                | "unknown"
316        ),
317        SEMANTIC_METRIC_TEMPORALITY => {
318            matches!(
319                value,
320                METRIC_TEMPORALITY_CUMULATIVE | METRIC_TEMPORALITY_DELTA | "mixed" | "unknown"
321            )
322        }
323        SEMANTIC_METRIC_METADATA_QUALITY => matches!(value, "declared" | "inferred" | "unknown"),
324
325        _ => false,
326    }
327}
328
329#[cfg(test)]
330mod tests {
331    use datatypes::types::JsonType;
332
333    use super::*;
334
335    #[test]
336    fn test_trace_v2_attribute_references() {
337        let schema = Schema::new(vec![
338            ColumnSchema::new(
339                RESOURCE_ATTRIBUTES_COLUMN,
340                ConcreteDataType::Json(JsonType::null()),
341                true,
342            ),
343            ColumnSchema::new(
344                SCOPE_ATTRIBUTES_COLUMN,
345                ConcreteDataType::string_datatype(),
346                true,
347            ),
348            ColumnSchema::new(
349                "resource_attributes.physical",
350                ConcreteDataType::string_datatype(),
351                true,
352            ),
353        ]);
354        let model = Some(TABLE_DATA_MODEL_TRACE_V2);
355        for key in ["host.name", "host..name", "host[0]", "host.*", r#"a"b\c.d"#] {
356            let column = format!("resource_attributes.{key}");
357            assert_eq!(
358                trace_v2_attribute(&schema, model, &column),
359                Some((RESOURCE_ATTRIBUTES_COLUMN, key))
360            );
361        }
362        for column in [
363            "resource_attributes",
364            "resource_attributes.",
365            "scope_attributes.name",
366            "span_attributes.name",
367            "resource_attributes.physical",
368            "missing.key",
369        ] {
370            assert_eq!(trace_v2_attribute(&schema, model, column), None, "{column}");
371        }
372        for model in [None, Some(crate::requests::TABLE_DATA_MODEL_TRACE_V1)] {
373            assert_eq!(
374                trace_v2_attribute(&schema, model, "resource_attributes.host.name"),
375                None
376            );
377        }
378    }
379
380    #[test]
381    fn test_is_semantic_option_key() {
382        assert!(is_semantic_option_key(SEMANTIC_SIGNAL_TYPE));
383        assert!(is_semantic_option_key(SEMANTIC_METRIC_TYPE));
384        assert!(is_semantic_option_key(SEMANTIC_PIPELINE));
385
386        // Unknown keys under the prefix are not whitelisted.
387        assert!(!is_semantic_option_key("greptime.semantic.future.key"));
388        assert!(!is_semantic_option_key("greptime.semantic.unknown_key"));
389        // Keys cut from the vocabulary are no longer accepted.
390        assert!(!is_semantic_option_key(
391            "greptime.semantic.metric.monotonic"
392        ));
393        assert!(!is_semantic_option_key(
394            "greptime.semantic.resource.attributes_dropped"
395        ));
396        // Near-misses must not match.
397        assert!(!is_semantic_option_key("greptime.semanticx"));
398        assert!(!is_semantic_option_key("semantic.signal_type"));
399        assert!(!is_semantic_option_key("table_data_model"));
400        // The internal transport key must never be treated as a table option.
401        assert!(!is_semantic_option_key(SEMANTIC_PER_TABLE_INDEX_KEY));
402    }
403
404    #[test]
405    fn test_validate_semantic_option() {
406        // Enum keys reject out-of-domain values.
407        assert!(validate_semantic_option(SEMANTIC_SIGNAL_TYPE, "metric"));
408        assert!(!validate_semantic_option(SEMANTIC_SIGNAL_TYPE, "spans"));
409        assert!(validate_semantic_option(SEMANTIC_METRIC_TYPE, "counter"));
410        assert!(validate_semantic_option(SEMANTIC_METRIC_TYPE, "mixed"));
411        assert!(!validate_semantic_option(SEMANTIC_METRIC_TYPE, "bogus"));
412
413        // Sentinels and open values.
414        assert!(validate_semantic_option(
415            SEMANTIC_METRIC_TEMPORALITY,
416            "unknown"
417        ));
418        assert!(validate_semantic_option(SEMANTIC_METRIC_UNIT, "By"));
419        assert!(!validate_semantic_option(SEMANTIC_METRIC_UNIT, ""));
420        assert!(validate_semantic_option(
421            SEMANTIC_PIPELINE,
422            "greptime_trace_v1"
423        ));
424
425        // A cut key validates to false regardless of value.
426        assert!(!validate_semantic_option(
427            "greptime.semantic.metric.monotonic",
428            "true"
429        ));
430        // Unknown key is rejected regardless of value.
431        assert!(!validate_semantic_option(
432            "greptime.semantic.future.key",
433            "x"
434        ));
435
436        // Drift guard: every value stamped by the ingestion path must validate.
437        assert!(validate_semantic_option(
438            SEMANTIC_SIGNAL_TYPE,
439            SIGNAL_TYPE_TRACE
440        ));
441        assert!(validate_semantic_option(
442            SEMANTIC_SIGNAL_TYPE,
443            SIGNAL_TYPE_METRIC
444        ));
445        assert!(validate_semantic_option(
446            SEMANTIC_SIGNAL_TYPE,
447            SIGNAL_TYPE_LOG
448        ));
449        assert!(validate_semantic_option(
450            SEMANTIC_SOURCE,
451            SOURCE_OPENTELEMETRY
452        ));
453        assert!(validate_semantic_option(SEMANTIC_SOURCE, SOURCE_PROMETHEUS));
454        assert!(validate_semantic_option(SEMANTIC_SOURCE_VERSION, "2.0"));
455        assert!(validate_semantic_option(
456            SEMANTIC_METRIC_METADATA_QUALITY,
457            METADATA_QUALITY_INFERRED
458        ));
459        assert!(validate_semantic_option(
460            SEMANTIC_METRIC_METADATA_QUALITY,
461            METADATA_QUALITY_DECLARED
462        ));
463        assert!(validate_semantic_option(
464            SEMANTIC_TRACE_CONVENTIONS,
465            SEMANTIC_VALUE_UNKNOWN
466        ));
467        // An empty value never validates, for any whitelisted key.
468        for key in SEMANTIC_OPTION_KEYS {
469            assert!(
470                !validate_semantic_option(key, ""),
471                "empty value should never validate for {key}"
472            );
473        }
474    }
475
476    #[test]
477    fn test_entity_option_key() {
478        // Well-formed entity keys are accepted by the open sub-namespace, and are
479        // therefore recognised as semantic option keys.
480        for key in [
481            "greptime.semantic.entity.service.id",
482            "greptime.semantic.entity.k8s.pod.id",
483            "greptime.semantic.entity.service.instance.descriptive",
484            "greptime.semantic.entity.host.scope",
485            "greptime.semantic.entity.agent.id",
486        ] {
487            assert!(is_entity_option_key(key), "should accept {key}");
488            assert!(is_semantic_option_key(key), "should recognise {key}");
489        }
490
491        // Malformed: empty type segment, bogus role, missing role, bare prefix,
492        // invalid (uppercase) charset in the type.
493        for key in [
494            "greptime.semantic.entity..id",
495            "greptime.semantic.entity.service.bogusrole",
496            "greptime.semantic.entity.service",
497            "greptime.semantic.entity.",
498            "greptime.semantic.entity.Service.id",
499        ] {
500            assert!(!is_entity_option_key(key), "should reject {key}");
501            assert!(!is_semantic_option_key(key), "should not recognise {key}");
502        }
503
504        // A non-entity semantic key is not an entity key.
505        assert!(!is_entity_option_key(SEMANTIC_SIGNAL_TYPE));
506
507        // Drift guard: the auto-stamped constant is a well-formed entity key.
508        assert!(is_entity_option_key(SEMANTIC_ENTITY_SERVICE_ID));
509        assert!(is_semantic_option_key(SEMANTIC_ENTITY_SERVICE_ID));
510    }
511
512    #[test]
513    fn test_validate_entity_option() {
514        // Single- and composite-id column lists validate.
515        assert!(validate_semantic_option(
516            "greptime.semantic.entity.service.id",
517            "service_name"
518        ));
519        assert!(validate_semantic_option(
520            "greptime.semantic.entity.process.id",
521            "pid,start_time"
522        ));
523        // Empty value and blank/empty tokens do not.
524        assert!(!validate_semantic_option(
525            "greptime.semantic.entity.service.id",
526            ""
527        ));
528        assert!(!validate_semantic_option(
529            "greptime.semantic.entity.process.id",
530            "pid,"
531        ));
532        // A malformed entity key validates to false regardless of value.
533        assert!(!validate_semantic_option(
534            "greptime.semantic.entity.service.bogusrole",
535            "service_name"
536        ));
537    }
538}
table/requests/semantic.rs

table/requests/
semantic.rs
