table/requests/
semantic.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Table semantic layer vocabulary.
16//!
17//! A thin layer of semantic metadata attached to a table via `table_options`, so
18//! machine consumers (LLM agents, alert/dashboard builders, MCP servers, ETL) can
19//! align a table with the observability concept it stands for without guessing
20//! from column names. See `docs/rfcs/2026-05-28-table-semantic-layer.md`.
21//!
22//! The vocabulary is intentionally small: a key earns its place only when it
23//! records something a consumer cannot cheaply and reliably recover from the
24//! schema/data itself. Keys whose value is already in the metric name by
25//! convention, is a constant, or duplicates an existing column are deliberately
26//! omitted rather than stamped for completeness.
27//!
28//! All public keys share the [`SEMANTIC_PREFIX`] namespace and are string-valued.
29//! [`is_semantic_option_key`] gates them through
30//! [`crate::requests::validate_table_option`], so they are accepted both on the
31//! ingestion auto-create path and on explicit `CREATE TABLE ... WITH (...)` DDL.
32
33/// Reserved prefix for every public semantic table-option key.
34pub const SEMANTIC_PREFIX: &str = "greptime.semantic.";
35
36/// Internal `QueryContext` extension key carrying the per-table semantic index
37/// (a `{table_name -> {semantic_key: value}}` JSON blob) from the ingestion
38/// encode path to the auto-create site. Deliberately OUTSIDE [`SEMANTIC_PREFIX`]
39/// so it is not a valid table option and never leaks into a table's options.
40pub const SEMANTIC_PER_TABLE_INDEX_KEY: &str = "greptime.internal.semantic.per_table_index";
41
42// ---- Common keys (all signals) ----
43
44/// Signal kind: one of [`SIGNAL_TYPE_TRACE`] / [`SIGNAL_TYPE_LOG`] /
45/// [`SIGNAL_TYPE_METRIC`] / [`SIGNAL_TYPE_EVENT`].
46pub const SEMANTIC_SIGNAL_TYPE: &str = "greptime.semantic.signal_type";
47/// Ingestion ecosystem, e.g. [`SOURCE_OPENTELEMETRY`] / [`SOURCE_PROMETHEUS`].
48pub const SEMANTIC_SOURCE: &str = "greptime.semantic.source";
49/// Source protocol version, e.g. Prometheus remote write `1.0` / `2.0`.
50pub const SEMANTIC_SOURCE_VERSION: &str = "greptime.semantic.source_version";
51/// Internal ingestion pipeline / data model, e.g. `greptime_trace_v1`. The
52/// signal-agnostic successor to the engine-specific `table_data_model` option.
53pub const SEMANTIC_PIPELINE: &str = "greptime.semantic.pipeline";
54
55// ---- Trace keys ----
56
57/// Semantic-conventions version the rows conform to (e.g. the OTel schema URL),
58/// or [`SEMANTIC_VALUE_UNKNOWN`] / [`SEMANTIC_VALUE_MIXED`] when not single-valued.
59pub const SEMANTIC_TRACE_CONVENTIONS: &str = "greptime.semantic.trace.conventions";
60
61// ---- Metric keys ----
62
63/// Instrument kind: `counter` / `gauge` / `histogram` / `summary` /
64/// `updown_counter` / `gauge_histogram` / `info` / `stateset`.
65pub const SEMANTIC_METRIC_TYPE: &str = "greptime.semantic.metric.type";
66/// UCUM unit, e.g. `s`, `By`, `{request}`. Discarded by the row encoders, so it
67/// is unrecoverable once ingested.
68pub const SEMANTIC_METRIC_UNIT: &str = "greptime.semantic.metric.unit";
69/// `cumulative` / `delta` (OTel only). Invisible in the metric name, so it is
70/// unrecoverable from the table alone.
71pub const SEMANTIC_METRIC_TEMPORALITY: &str = "greptime.semantic.metric.temporality";
72/// [`METADATA_QUALITY_DECLARED`] when the protocol stated the type, or
73/// [`METADATA_QUALITY_INFERRED`] when guessed from a name suffix.
74pub const SEMANTIC_METRIC_METADATA_QUALITY: &str = "greptime.semantic.metric.metadata_quality";
75/// Pre-translation OTel name when the table name was Prometheus-ised; the key a
76/// consumer uses to look the metric up in the OTel semantic conventions.
77pub const SEMANTIC_METRIC_ORIGINAL_NAME: &str = "greptime.semantic.metric.original_name";
78
79// ---- Entity keys (open sub-namespace) ----
80
81/// Reserved prefix for the entity-identity sub-namespace:
82/// `greptime.semantic.entity.<type>.{id|descriptive|scope}`. Unlike the rest of
83/// the vocabulary (a closed whitelist), entity types are open-ended (`service`,
84/// `host`, `k8s.pod`, `process`, `agent`, custom, ...), so keys here are validated
85/// by prefix + shape rather than membership. See
86/// `docs/rfcs/2026-06-25-entity-relationships-and-graph-query.md`.
87pub const SEMANTIC_ENTITY_PREFIX: &str = "greptime.semantic.entity.";
88
89/// The well-known entity-identity key auto-stamped on OTLP trace tables; its value
90/// is the `service_name` tag column, declaring the logical `service` entity.
91pub const SEMANTIC_ENTITY_SERVICE_ID: &str = "greptime.semantic.entity.service.id";
92
93/// The role a set of columns plays for an entity: `id` (identifying attributes,
94/// must be tag columns — enforced at DDL time), `descriptive`, or `scope`.
95#[derive(Debug, Clone, Copy, PartialEq, Eq)]
96pub enum EntityRole {
97    Id,
98    Descriptive,
99    Scope,
100}
101
102impl EntityRole {
103    fn parse(role: &str) -> Option<Self> {
104        match role {
105            "id" => Some(EntityRole::Id),
106            "descriptive" => Some(EntityRole::Descriptive),
107            "scope" => Some(EntityRole::Scope),
108            _ => None,
109        }
110    }
111}
112
113// ---- Value constants ----
114
115pub const SIGNAL_TYPE_TRACE: &str = "trace";
116pub const SIGNAL_TYPE_LOG: &str = "log";
117pub const SIGNAL_TYPE_METRIC: &str = "metric";
118pub const SIGNAL_TYPE_EVENT: &str = "event";
119
120pub const SOURCE_OPENTELEMETRY: &str = "opentelemetry";
121pub const SOURCE_PROMETHEUS: &str = "prometheus";
122pub const SOURCE_INFLUXDB: &str = "influxdb";
123pub const SOURCE_OPENTSDB: &str = "opentsdb";
124pub const SOURCE_LOKI: &str = "loki";
125pub const SOURCE_ELASTICSEARCH: &str = "elasticsearch";
126
127pub const METADATA_QUALITY_DECLARED: &str = "declared";
128pub const METADATA_QUALITY_INFERRED: &str = "inferred";
129
130/// Sentinel for a key that cannot be determined at stamp time.
131pub const SEMANTIC_VALUE_UNKNOWN: &str = "unknown";
132/// Sentinel for a single-valued key that saw conflicting sources.
133pub const SEMANTIC_VALUE_MIXED: &str = "mixed";
134
135/// Every recognised public semantic table-option key. The set is a closed
136/// whitelist: keys under [`SEMANTIC_PREFIX`] that are not listed here are rejected,
137/// so an unknown key like `greptime.semantic.unknown_key` does not silently land
138/// in a table's options. Adding a key to the vocabulary means adding it here.
139pub const SEMANTIC_OPTION_KEYS: &[&str] = &[
140    SEMANTIC_SIGNAL_TYPE,
141    SEMANTIC_SOURCE,
142    SEMANTIC_SOURCE_VERSION,
143    SEMANTIC_PIPELINE,
144    SEMANTIC_TRACE_CONVENTIONS,
145    SEMANTIC_METRIC_TYPE,
146    SEMANTIC_METRIC_UNIT,
147    SEMANTIC_METRIC_TEMPORALITY,
148    SEMANTIC_METRIC_METADATA_QUALITY,
149    SEMANTIC_METRIC_ORIGINAL_NAME,
150];
151
152/// Returns true if `ty` is a syntactically valid entity type, e.g. `service`,
153/// `host`, `k8s.pod`, `service.instance`. An entity type is one or more
154/// dot-separated segments, each a non-empty `[a-z0-9_]+` token. The dotted form
155/// carries the two-entity-layer convention (`service` vs `service.instance`).
156fn is_valid_entity_type(ty: &str) -> bool {
157    !ty.is_empty()
158        && ty.split('.').all(|seg| {
159            !seg.is_empty()
160                && seg
161                    .bytes()
162                    .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_')
163        })
164}
165
166/// Parses a well-formed entity-identity option key of the shape
167/// `greptime.semantic.entity.<type>.{id|descriptive|scope}` into
168/// `(entity_type, role)`. The `<type>` may itself contain dots; the role is the
169/// final dot-separated segment. This is the single parser of the key format —
170/// DDL validation and the read-time derivation both go through it.
171pub fn parse_entity_option_key(key: &str) -> Option<(&str, EntityRole)> {
172    let rest = key.strip_prefix(SEMANTIC_ENTITY_PREFIX)?;
173    let (ty, role) = rest.rsplit_once('.')?;
174    if !is_valid_entity_type(ty) {
175        return None;
176    }
177    Some((ty, EntityRole::parse(role)?))
178}
179
180/// Returns true if `key` is a well-formed entity-identity option key.
181pub fn is_entity_option_key(key: &str) -> bool {
182    parse_entity_option_key(key).is_some()
183}
184
185/// Tokenizes an entity option's comma-separated column list (trimmed, empty
186/// tokens dropped). [`validate_semantic_option`] rejects empty tokens at DDL
187/// time, so readers only ever drop what validation already refused.
188pub fn parse_entity_columns(value: &str) -> Vec<String> {
189    value
190        .split(',')
191        .map(|c| c.trim().to_string())
192        .filter(|c| !c.is_empty())
193        .collect()
194}
195
196/// Returns true if `key` is a recognised semantic table-option key.
197///
198/// Two acceptance rules: membership in the closed [`SEMANTIC_OPTION_KEYS`]
199/// whitelist, OR the open entity sub-namespace ([`is_entity_option_key`], validated
200/// by prefix + shape). Everything else under [`SEMANTIC_PREFIX`] is rejected, and
201/// the internal [`SEMANTIC_PER_TABLE_INDEX_KEY`] (outside the prefix) never matches.
202pub fn is_semantic_option_key(key: &str) -> bool {
203    SEMANTIC_OPTION_KEYS.contains(&key) || is_entity_option_key(key)
204}
205
206/// Validates a `greptime.semantic.*` option's `value` against its allowed domain.
207///
208/// Open-value keys (unit, original_name, pipeline, conventions) accept any
209/// non-empty string. Closed-domain keys accept a fixed set, plus the `unknown`
210/// sentinel, plus `mixed` for the keys where one long-lived table can
211/// legitimately see multiple values. Entity keys ([`is_entity_option_key`]) take a
212/// comma-separated column-name list (each token non-empty); column existence and
213/// the stable-string-form rule for entity columns are enforced later against
214/// the table schema at DDL time, not here. Keys that are neither whitelisted nor a well-formed entity key
215/// are rejected.
216pub fn validate_semantic_option(key: &str, value: &str) -> bool {
217    if is_entity_option_key(key) {
218        return !value.is_empty() && value.split(',').all(|col| !col.trim().is_empty());
219    }
220    match key {
221        SEMANTIC_PIPELINE
222        | SEMANTIC_SOURCE_VERSION
223        | SEMANTIC_METRIC_UNIT
224        | SEMANTIC_METRIC_ORIGINAL_NAME
225        | SEMANTIC_TRACE_CONVENTIONS => !value.is_empty(),
226
227        SEMANTIC_SIGNAL_TYPE => matches!(value, "trace" | "log" | "metric" | "event" | "unknown"),
228        SEMANTIC_SOURCE => matches!(
229            value,
230            "opentelemetry"
231                | "prometheus"
232                | "influxdb"
233                | "opentsdb"
234                | "elasticsearch"
235                | "loki"
236                | "custom"
237                | "mixed"
238                | "unknown"
239        ),
240        SEMANTIC_METRIC_TYPE => matches!(
241            value,
242            "counter"
243                | "gauge"
244                | "histogram"
245                | "summary"
246                | "updown_counter"
247                | "gauge_histogram"
248                | "info"
249                | "stateset"
250                | "mixed"
251                | "unknown"
252        ),
253        SEMANTIC_METRIC_TEMPORALITY => {
254            matches!(value, "cumulative" | "delta" | "mixed" | "unknown")
255        }
256        SEMANTIC_METRIC_METADATA_QUALITY => matches!(value, "declared" | "inferred" | "unknown"),
257
258        _ => false,
259    }
260}
261
262#[cfg(test)]
263mod tests {
264    use super::*;
265
266    #[test]
267    fn test_is_semantic_option_key() {
268        assert!(is_semantic_option_key(SEMANTIC_SIGNAL_TYPE));
269        assert!(is_semantic_option_key(SEMANTIC_METRIC_TYPE));
270        assert!(is_semantic_option_key(SEMANTIC_PIPELINE));
271
272        // Unknown keys under the prefix are not whitelisted.
273        assert!(!is_semantic_option_key("greptime.semantic.future.key"));
274        assert!(!is_semantic_option_key("greptime.semantic.unknown_key"));
275        // Keys cut from the vocabulary are no longer accepted.
276        assert!(!is_semantic_option_key(
277            "greptime.semantic.metric.monotonic"
278        ));
279        assert!(!is_semantic_option_key(
280            "greptime.semantic.resource.attributes_dropped"
281        ));
282        // Near-misses must not match.
283        assert!(!is_semantic_option_key("greptime.semanticx"));
284        assert!(!is_semantic_option_key("semantic.signal_type"));
285        assert!(!is_semantic_option_key("table_data_model"));
286        // The internal transport key must never be treated as a table option.
287        assert!(!is_semantic_option_key(SEMANTIC_PER_TABLE_INDEX_KEY));
288    }
289
290    #[test]
291    fn test_validate_semantic_option() {
292        // Enum keys reject out-of-domain values.
293        assert!(validate_semantic_option(SEMANTIC_SIGNAL_TYPE, "metric"));
294        assert!(!validate_semantic_option(SEMANTIC_SIGNAL_TYPE, "spans"));
295        assert!(validate_semantic_option(SEMANTIC_METRIC_TYPE, "counter"));
296        assert!(validate_semantic_option(SEMANTIC_METRIC_TYPE, "mixed"));
297        assert!(!validate_semantic_option(SEMANTIC_METRIC_TYPE, "bogus"));
298
299        // Sentinels and open values.
300        assert!(validate_semantic_option(
301            SEMANTIC_METRIC_TEMPORALITY,
302            "unknown"
303        ));
304        assert!(validate_semantic_option(SEMANTIC_METRIC_UNIT, "By"));
305        assert!(!validate_semantic_option(SEMANTIC_METRIC_UNIT, ""));
306        assert!(validate_semantic_option(
307            SEMANTIC_PIPELINE,
308            "greptime_trace_v1"
309        ));
310
311        // A cut key validates to false regardless of value.
312        assert!(!validate_semantic_option(
313            "greptime.semantic.metric.monotonic",
314            "true"
315        ));
316        // Unknown key is rejected regardless of value.
317        assert!(!validate_semantic_option(
318            "greptime.semantic.future.key",
319            "x"
320        ));
321
322        // Drift guard: every value stamped by the ingestion path must validate.
323        assert!(validate_semantic_option(
324            SEMANTIC_SIGNAL_TYPE,
325            SIGNAL_TYPE_TRACE
326        ));
327        assert!(validate_semantic_option(
328            SEMANTIC_SIGNAL_TYPE,
329            SIGNAL_TYPE_METRIC
330        ));
331        assert!(validate_semantic_option(
332            SEMANTIC_SIGNAL_TYPE,
333            SIGNAL_TYPE_LOG
334        ));
335        assert!(validate_semantic_option(
336            SEMANTIC_SOURCE,
337            SOURCE_OPENTELEMETRY
338        ));
339        assert!(validate_semantic_option(SEMANTIC_SOURCE, SOURCE_PROMETHEUS));
340        assert!(validate_semantic_option(SEMANTIC_SOURCE_VERSION, "2.0"));
341        assert!(validate_semantic_option(
342            SEMANTIC_METRIC_METADATA_QUALITY,
343            METADATA_QUALITY_INFERRED
344        ));
345        assert!(validate_semantic_option(
346            SEMANTIC_METRIC_METADATA_QUALITY,
347            METADATA_QUALITY_DECLARED
348        ));
349        assert!(validate_semantic_option(
350            SEMANTIC_TRACE_CONVENTIONS,
351            SEMANTIC_VALUE_UNKNOWN
352        ));
353        // An empty value never validates, for any whitelisted key.
354        for key in SEMANTIC_OPTION_KEYS {
355            assert!(
356                !validate_semantic_option(key, ""),
357                "empty value should never validate for {key}"
358            );
359        }
360    }
361
362    #[test]
363    fn test_entity_option_key() {
364        // Well-formed entity keys are accepted by the open sub-namespace, and are
365        // therefore recognised as semantic option keys.
366        for key in [
367            "greptime.semantic.entity.service.id",
368            "greptime.semantic.entity.k8s.pod.id",
369            "greptime.semantic.entity.service.instance.descriptive",
370            "greptime.semantic.entity.host.scope",
371            "greptime.semantic.entity.agent.id",
372        ] {
373            assert!(is_entity_option_key(key), "should accept {key}");
374            assert!(is_semantic_option_key(key), "should recognise {key}");
375        }
376
377        // Malformed: empty type segment, bogus role, missing role, bare prefix,
378        // invalid (uppercase) charset in the type.
379        for key in [
380            "greptime.semantic.entity..id",
381            "greptime.semantic.entity.service.bogusrole",
382            "greptime.semantic.entity.service",
383            "greptime.semantic.entity.",
384            "greptime.semantic.entity.Service.id",
385        ] {
386            assert!(!is_entity_option_key(key), "should reject {key}");
387            assert!(!is_semantic_option_key(key), "should not recognise {key}");
388        }
389
390        // A non-entity semantic key is not an entity key.
391        assert!(!is_entity_option_key(SEMANTIC_SIGNAL_TYPE));
392
393        // Drift guard: the auto-stamped constant is a well-formed entity key.
394        assert!(is_entity_option_key(SEMANTIC_ENTITY_SERVICE_ID));
395        assert!(is_semantic_option_key(SEMANTIC_ENTITY_SERVICE_ID));
396    }
397
398    #[test]
399    fn test_validate_entity_option() {
400        // Single- and composite-id column lists validate.
401        assert!(validate_semantic_option(
402            "greptime.semantic.entity.service.id",
403            "service_name"
404        ));
405        assert!(validate_semantic_option(
406            "greptime.semantic.entity.process.id",
407            "pid,start_time"
408        ));
409        // Empty value and blank/empty tokens do not.
410        assert!(!validate_semantic_option(
411            "greptime.semantic.entity.service.id",
412            ""
413        ));
414        assert!(!validate_semantic_option(
415            "greptime.semantic.entity.process.id",
416            "pid,"
417        ));
418        // A malformed entity key validates to false regardless of value.
419        assert!(!validate_semantic_option(
420            "greptime.semantic.entity.service.bogusrole",
421            "service_name"
422        ));
423    }
424}
table/requests/semantic.rs

table/requests/
semantic.rs
