servers/
semantic.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Per-table semantic metadata accumulated during one ingest pass (OTLP
16//! metrics, Prometheus remote write v2).
17//!
18//! Each written table collects the scalar semantic keys its wire metadata
19//! declares (an OTLP histogram fans out into `_bucket`/`_sum`/`_count`
20//! companions; a remote write series names one metric family). The resulting
21//! index is serialized onto the `greptime.internal.semantic.per_table_index`
22//! context extension as `{schema -> {table -> {key -> value}}}` and folded into
23//! each table's options at auto-create time.
24//!
25//! Conflict handling follows the RFC: when two sources disagree on a
26//! single-valued key the value collapses to `mixed` (or `unknown` for keys whose
27//! domain has no `mixed`).
28
29use std::collections::{BTreeMap, HashMap};
30
31use table::requests::{SEMANTIC_VALUE_MIXED, SEMANTIC_VALUE_UNKNOWN, validate_semantic_option};
32
33// `greptime.semantic.metric.type` values stamped per emitted table. Must stay
34// within the domain accepted by `validate_semantic_option`; the drift-guard test
35// asserts this.
36pub const METRIC_TYPE_COUNTER: &str = "counter";
37pub const METRIC_TYPE_UPDOWN_COUNTER: &str = "updown_counter";
38pub const METRIC_TYPE_GAUGE: &str = "gauge";
39pub const METRIC_TYPE_HISTOGRAM: &str = "histogram";
40pub const METRIC_TYPE_GAUGE_HISTOGRAM: &str = "gauge_histogram";
41pub const METRIC_TYPE_SUMMARY: &str = "summary";
42pub const METRIC_TYPE_INFO: &str = "info";
43pub const METRIC_TYPE_STATESET: &str = "stateset";
44
45/// Maps an OpenMetrics unit name (open vocabulary full words: `seconds`,
46/// `bytes`) to the UCUM code the `greptime.semantic.metric.unit` option is
47/// defined in. Only the OpenMetrics base units are mapped; anything else is
48/// dropped — a missing unit beats a corrupted cross-protocol vocabulary.
49pub fn openmetrics_unit_to_ucum(unit: &str) -> Option<&'static str> {
50    Some(match unit {
51        "seconds" => "s",
52        "celsius" => "Cel",
53        "meters" => "m",
54        "bytes" => "By",
55        "ratios" => "1",
56        "volts" => "V",
57        "amperes" => "A",
58        "joules" => "J",
59        "grams" => "g",
60        _ => return None,
61    })
62}
63
64/// Index of `{table_name -> {semantic_key -> value}}` for one target schema.
65#[derive(Debug, Default)]
66pub struct SemanticIndex {
67    /// Per-table scalar keys; conflicting values collapse to `mixed`/`unknown`.
68    tables: HashMap<String, BTreeMap<&'static str, String>>,
69}
70
71impl SemanticIndex {
72    pub fn is_empty(&self) -> bool {
73        self.tables.is_empty()
74    }
75
76    /// Records a scalar semantic key for `table`. A value conflicting with one
77    /// already recorded collapses the key to `mixed`/`unknown`; once collapsed
78    /// it stays collapsed.
79    pub fn record_scalar(&mut self, table: &str, key: &'static str, value: &str) {
80        // Avoid allocating the table name (and an empty map) on the common path
81        // where the table is already present.
82        if let Some(scalars) = self.tables.get_mut(table) {
83            match scalars.get(key).map(String::as_str) {
84                Some(existing) if existing == value => {}
85                Some(SEMANTIC_VALUE_MIXED) | Some(SEMANTIC_VALUE_UNKNOWN) => {}
86                Some(_) => {
87                    scalars.insert(key, collapse_value(key));
88                }
89                None => {
90                    scalars.insert(key, value.to_string());
91                }
92            }
93        } else {
94            self.tables.insert(
95                table.to_string(),
96                BTreeMap::from([(key, value.to_string())]),
97            );
98        }
99    }
100
101    /// Serializes to the JSON `{schema -> {table -> {key -> value}}}` carried
102    /// on the context extension, with every table under `schema`. `None` when
103    /// nothing was recorded.
104    pub fn encode(&self, schema: &str) -> Option<String> {
105        if self.tables.is_empty() {
106            return None;
107        }
108        serde_json::to_string(&BTreeMap::from([(schema, &self.tables)])).ok()
109    }
110
111    pub(crate) fn merge_from(&mut self, other: &SemanticIndex) {
112        for (table, scalars) in &other.tables {
113            for (key, value) in scalars {
114                self.record_scalar(table, key, value);
115            }
116        }
117    }
118
119    /// Moves a metric's metadata on its first accepted output for this table.
120    pub(crate) fn merge_table_from(&mut self, other: &mut SemanticIndex, table: &str) {
121        if let Some(scalars) = other.tables.remove(table) {
122            for (key, value) in scalars {
123                self.record_scalar(table, key, &value);
124            }
125        }
126    }
127
128    #[cfg(test)]
129    fn options_of(&self, table: &str) -> Option<&BTreeMap<&'static str, String>> {
130        self.tables.get(table)
131    }
132}
133
134/// Schema-aware collection of [`SemanticIndex`]es: Prometheus remote write lets
135/// each series override the target schema with a special label, so one request
136/// may write the same metric name into several schemas — their metadata must
137/// not collapse into each other.
138#[derive(Debug, Default)]
139pub struct SemanticIndexes {
140    /// Tables written into the request's default schema (no override).
141    default: SemanticIndex,
142    /// Tables written under a per-series schema override.
143    overrides: HashMap<String, SemanticIndex>,
144}
145
146impl SemanticIndexes {
147    pub fn is_empty(&self) -> bool {
148        self.default.is_empty() && self.overrides.values().all(SemanticIndex::is_empty)
149    }
150
151    /// The index for `schema` (`None` = the request's default schema).
152    pub fn index_for(&mut self, schema: Option<&str>) -> &mut SemanticIndex {
153        match schema {
154            None => &mut self.default,
155            Some(schema) => {
156                if !self.overrides.contains_key(schema) {
157                    self.overrides
158                        .insert(schema.to_string(), SemanticIndex::default());
159                }
160                self.overrides.get_mut(schema).expect("just inserted")
161            }
162        }
163    }
164
165    /// Serializes to the JSON `{schema -> {table -> {key -> value}}}` carried on
166    /// the context extension, resolving the default index to `default_schema`.
167    /// An override explicitly naming `default_schema` merges into the default
168    /// with the usual conflict collapse.
169    pub fn encode(&self, default_schema: &str) -> Option<String> {
170        if self.is_empty() {
171            return None;
172        }
173        let mut by_schema: BTreeMap<&str, &HashMap<String, BTreeMap<&'static str, String>>> =
174            BTreeMap::new();
175        let mut merged_default;
176        if let Some(aliased) = self.overrides.get(default_schema) {
177            merged_default = SemanticIndex::default();
178            merged_default.merge_from(&self.default);
179            merged_default.merge_from(aliased);
180            by_schema.insert(default_schema, &merged_default.tables);
181        } else if !self.default.is_empty() {
182            by_schema.insert(default_schema, &self.default.tables);
183        }
184        for (schema, index) in &self.overrides {
185            if schema != default_schema && !index.is_empty() {
186                by_schema.insert(schema, &index.tables);
187            }
188        }
189        serde_json::to_string(&by_schema).ok()
190    }
191}
192
193/// The collapsed value for a conflicting scalar key: `mixed` when the key's
194/// domain accepts it, else `unknown`. Uses the vocabulary validator as the
195/// single source of truth for which keys allow `mixed`.
196fn collapse_value(key: &str) -> String {
197    if validate_semantic_option(key, SEMANTIC_VALUE_MIXED) {
198        SEMANTIC_VALUE_MIXED.to_string()
199    } else {
200        SEMANTIC_VALUE_UNKNOWN.to_string()
201    }
202}
203
204#[cfg(test)]
205mod tests {
206    use table::requests::{
207        SEMANTIC_METRIC_METADATA_QUALITY, SEMANTIC_METRIC_TYPE, SEMANTIC_METRIC_UNIT,
208    };
209
210    use super::*;
211
212    type Decoded = BTreeMap<String, BTreeMap<String, BTreeMap<String, String>>>;
213
214    #[test]
215    fn test_scalar_recording_keeps_first_then_collapses_on_conflict() {
216        let mut index = SemanticIndex::default();
217        index.record_scalar("t", SEMANTIC_METRIC_TYPE, "counter");
218        index.record_scalar("t", SEMANTIC_METRIC_TYPE, "counter");
219        assert_eq!(
220            index
221                .options_of("t")
222                .unwrap()
223                .get(SEMANTIC_METRIC_TYPE)
224                .map(String::as_str),
225            Some("counter")
226        );
227
228        // Conflict on a key whose domain has `mixed` collapses to `mixed`.
229        index.record_scalar("t", SEMANTIC_METRIC_TYPE, "gauge");
230        assert_eq!(
231            index
232                .options_of("t")
233                .unwrap()
234                .get(SEMANTIC_METRIC_TYPE)
235                .map(String::as_str),
236            Some("mixed")
237        );
238        // Further writes stay collapsed.
239        index.record_scalar("t", SEMANTIC_METRIC_TYPE, "histogram");
240        assert_eq!(
241            index
242                .options_of("t")
243                .unwrap()
244                .get(SEMANTIC_METRIC_TYPE)
245                .map(String::as_str),
246            Some("mixed")
247        );
248    }
249
250    #[test]
251    fn test_scalar_conflict_without_mixed_domain_collapses_to_unknown() {
252        let mut index = SemanticIndex::default();
253        index.record_scalar("t", SEMANTIC_METRIC_METADATA_QUALITY, "declared");
254        index.record_scalar("t", SEMANTIC_METRIC_METADATA_QUALITY, "inferred");
255        // metadata_quality accepts only declared/inferred/unknown, so a conflict
256        // is `unknown`.
257        assert_eq!(
258            index
259                .options_of("t")
260                .unwrap()
261                .get(SEMANTIC_METRIC_METADATA_QUALITY)
262                .map(String::as_str),
263            Some("unknown")
264        );
265    }
266
267    #[test]
268    fn test_encode_is_none_when_empty_and_round_trips() {
269        let index = SemanticIndex::default();
270        assert!(index.is_empty());
271        assert_eq!(index.encode("public"), None);
272
273        let mut index = SemanticIndex::default();
274        index.record_scalar("metric_a", SEMANTIC_METRIC_TYPE, "counter");
275        index.record_scalar("metric_a", SEMANTIC_METRIC_UNIT, "By");
276        let json = index.encode("public").unwrap();
277        let parsed: Decoded = serde_json::from_str(&json).unwrap();
278        let table = parsed.get("public").unwrap().get("metric_a").unwrap();
279        assert_eq!(
280            table.get(SEMANTIC_METRIC_TYPE).map(String::as_str),
281            Some("counter")
282        );
283        assert_eq!(
284            table.get(SEMANTIC_METRIC_UNIT).map(String::as_str),
285            Some("By")
286        );
287    }
288
289    #[test]
290    fn test_indexes_keep_schemas_apart_and_merge_default_alias() {
291        let mut indexes = SemanticIndexes::default();
292        assert!(indexes.is_empty());
293        assert_eq!(indexes.encode("public"), None);
294
295        // The same metric name in two schemas must not collapse to `mixed`.
296        indexes
297            .index_for(None)
298            .record_scalar("cpu_usage", SEMANTIC_METRIC_TYPE, "counter");
299        indexes.index_for(Some("tenant_b")).record_scalar(
300            "cpu_usage",
301            SEMANTIC_METRIC_TYPE,
302            "gauge",
303        );
304        let parsed: Decoded = serde_json::from_str(&indexes.encode("public").unwrap()).unwrap();
305        assert_eq!(
306            parsed["public"]["cpu_usage"][SEMANTIC_METRIC_TYPE],
307            "counter"
308        );
309        assert_eq!(
310            parsed["tenant_b"]["cpu_usage"][SEMANTIC_METRIC_TYPE],
311            "gauge"
312        );
313
314        // An override naming the default schema merges into it — and a real
315        // conflict then collapses.
316        indexes
317            .index_for(Some("public"))
318            .record_scalar("cpu_usage", SEMANTIC_METRIC_TYPE, "gauge");
319        let parsed: Decoded = serde_json::from_str(&indexes.encode("public").unwrap()).unwrap();
320        assert_eq!(parsed["public"]["cpu_usage"][SEMANTIC_METRIC_TYPE], "mixed");
321    }
322
323    #[test]
324    fn test_openmetrics_unit_mapping() {
325        assert_eq!(openmetrics_unit_to_ucum("seconds"), Some("s"));
326        assert_eq!(openmetrics_unit_to_ucum("bytes"), Some("By"));
327        assert_eq!(openmetrics_unit_to_ucum("ratios"), Some("1"));
328        // Outside the OpenMetrics base set: dropped, not passed through.
329        assert_eq!(openmetrics_unit_to_ucum("requests"), None);
330        assert_eq!(openmetrics_unit_to_ucum(""), None);
331        // No fuzzy matching: UCUM codes are not OpenMetrics names.
332        assert_eq!(openmetrics_unit_to_ucum("By"), None);
333    }
334}
servers/semantic.rs

servers/
semantic.rs
