Skip to main content

operator/statement/semantic_graph/
conventions.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! The built-in derivation conventions of the entity graph, embedded as
16//! [`conventions.yaml`](./conventions.yaml): the co-declared edge vocabulary,
17//! the virtual-destination candidates, and the implicit declarations of
18//! well-known Prometheus entity-descriptor metrics. The file is data shipped
19//! with the binary, not an operator-editable configuration surface; explicit
20//! `greptime.semantic.entity.*` declarations always override it.
21
22use std::collections::{BTreeMap, HashMap, HashSet};
23use std::sync::LazyLock;
24
25use serde::Deserialize;
26
27/// A same-row co-declaration rule: a source-table row carrying both entity
28/// identities witnesses `src -rel-> dst`.
29#[derive(Debug, Deserialize)]
30#[serde(deny_unknown_fields)]
31pub struct EdgeRule {
32    pub src: String,
33    pub dst: String,
34    pub rel: String,
35}
36
37/// A span-attribute column that may name an uninstrumented peer, and the
38/// `connection_type` its match implies.
39#[derive(Debug, Deserialize)]
40#[serde(deny_unknown_fields)]
41pub struct VirtualDstCandidate {
42    pub column: String,
43    pub connection_type: String,
44}
45
46/// One implicit entity declaration: of a whitelisted Prometheus or OTel info
47/// metric, or of a trace-v1 table's flattened resource attributes.
48#[derive(Debug, Deserialize)]
49#[serde(deny_unknown_fields)]
50pub struct ImplicitEntity {
51    pub entity: String,
52    /// Identifying label columns, ordered broad to narrow; every one must
53    /// exist on the table for the declaration to apply.
54    pub id: Vec<String>,
55    /// Column qualifying `id[0]` as `<qualifier>/<id[0]>`, so a source
56    /// carrying the parts separately matches one carrying them pre-composed.
57    /// Skipped when the column is absent or empty.
58    #[serde(default)]
59    pub qualified_by: Option<String>,
60    /// Descriptive label columns, filtered to those present (kube-state-metrics
61    /// label sets vary across versions).
62    #[serde(default)]
63    pub descriptive: Vec<String>,
64    /// Snapshot every tag column except the id columns as descriptive —
65    /// `target_info`-style enrichment over an open attribute set.
66    #[serde(default)]
67    pub descriptive_rest: bool,
68}
69
70/// The parsed, validated conventions file.
71#[derive(Debug, Deserialize)]
72#[serde(deny_unknown_fields)]
73pub struct Conventions {
74    pub co_declared_edges: Vec<EdgeRule>,
75    pub trace_co_declared_edges: Vec<EdgeRule>,
76    pub virtual_dst_candidates: Vec<VirtualDstCandidate>,
77    pub otlp_trace_entities: Vec<ImplicitEntity>,
78    /// Table name -> the entities that table declares, for Prometheus-sourced
79    /// descriptor metrics (`source = prometheus`).
80    pub prometheus_info_metrics: BTreeMap<String, Vec<ImplicitEntity>>,
81    /// Table name -> the entities that table declares, for OTLP-sourced
82    /// descriptor tables (`source = opentelemetry`).
83    pub otel_info_metrics: BTreeMap<String, Vec<ImplicitEntity>>,
84}
85
86/// The built-in entity-type vocabulary. User-declared types are open-ended;
87/// the embedded conventions file must stay inside this set.
88pub const ENTITY_TYPE_SERVICE: &str = "service";
89pub const ENTITY_TYPE_SERVICE_INSTANCE: &str = "service.instance";
90pub const ENTITY_TYPE_HOST: &str = "host";
91pub const ENTITY_TYPE_CONTAINER: &str = "container";
92pub const ENTITY_TYPE_PROCESS: &str = "process";
93pub const ENTITY_TYPE_K8S_POD: &str = "k8s.pod";
94pub const ENTITY_TYPE_K8S_NODE: &str = "k8s.node";
95pub const ENTITY_TYPE_K8S_CONTAINER: &str = "k8s.container";
96pub const ENTITY_TYPE_K8S_WORKLOAD: &str = "k8s.workload";
97pub const ENTITY_TYPE_K8S_SERVICE: &str = "k8s.service";
98pub const ENTITY_TYPE_GEN_AI_AGENT: &str = "gen_ai.agent";
99pub const ENTITY_TYPE_GEN_AI_MODEL: &str = "gen_ai.model";
100pub const ENTITY_TYPE_GEN_AI_TOOL: &str = "gen_ai.tool";
101
102const ENTITY_TYPES: [&str; 13] = [
103    ENTITY_TYPE_SERVICE,
104    ENTITY_TYPE_SERVICE_INSTANCE,
105    ENTITY_TYPE_HOST,
106    ENTITY_TYPE_CONTAINER,
107    ENTITY_TYPE_PROCESS,
108    ENTITY_TYPE_K8S_POD,
109    ENTITY_TYPE_K8S_NODE,
110    ENTITY_TYPE_K8S_CONTAINER,
111    ENTITY_TYPE_K8S_WORKLOAD,
112    ENTITY_TYPE_K8S_SERVICE,
113    ENTITY_TYPE_GEN_AI_AGENT,
114    ENTITY_TYPE_GEN_AI_MODEL,
115    ENTITY_TYPE_GEN_AI_TOOL,
116];
117
118/// The relationship vocabulary (the RFC's rel_type table) and the edge
119/// provenances.
120pub const REL_TYPE_CALLS: &str = "calls";
121pub const REL_TYPE_RUNS_ON: &str = "runs_on";
122pub const REL_TYPE_CONTAINS: &str = "contains";
123pub const REL_TYPE_PART_OF: &str = "part_of";
124pub const REL_TYPE_USES: &str = "uses";
125pub const REL_TYPE_INVOKES: &str = "invokes";
126pub const REL_TYPE_DEPENDS_ON: &str = "depends_on";
127pub const REL_TYPE_OWNS: &str = "owns";
128pub const PROVENANCE_TRACE: &str = "trace";
129pub const PROVENANCE_ATTRIBUTE: &str = "attribute";
130pub const PROVENANCE_DECLARED: &str = "declared";
131pub const PROVENANCE_AGENT: &str = "agent";
132
133const REL_TYPES: [&str; 8] = [
134    REL_TYPE_CALLS,
135    REL_TYPE_RUNS_ON,
136    REL_TYPE_CONTAINS,
137    REL_TYPE_PART_OF,
138    REL_TYPE_USES,
139    REL_TYPE_INVOKES,
140    REL_TYPE_DEPENDS_ON,
141    REL_TYPE_OWNS,
142];
143
144/// The `connection_type` values a virtual-destination candidate may imply;
145/// the calls derivation branches on them when building edge attributes.
146pub const CONNECTION_TYPE_DATABASE: &str = "database";
147pub const CONNECTION_TYPE_VIRTUAL_NODE: &str = "virtual_node";
148
149const CONNECTION_TYPES: [&str; 2] = [CONNECTION_TYPE_DATABASE, CONNECTION_TYPE_VIRTUAL_NODE];
150
151static CONVENTIONS: LazyLock<Result<Conventions, String>> =
152    LazyLock::new(|| parse(include_str!("conventions.yaml")));
153
154/// The embedded conventions. `Err` means the embedded file is broken — pinned
155/// by unit test, and propagated by the derivation paths rather than panicking.
156pub fn conventions() -> Result<&'static Conventions, String> {
157    CONVENTIONS.as_ref().map_err(Clone::clone)
158}
159
160fn parse(yaml: &str) -> Result<Conventions, String> {
161    let conventions: Conventions =
162        serde_yaml_ng::from_str(yaml).map_err(|e| format!("malformed conventions: {e}"))?;
163    validate(&conventions)?;
164    Ok(conventions)
165}
166
167fn validate(conventions: &Conventions) -> Result<(), String> {
168    let mut seen_edges = HashSet::new();
169    for rule in conventions
170        .co_declared_edges
171        .iter()
172        .chain(&conventions.trace_co_declared_edges)
173    {
174        for ty in [&rule.src, &rule.dst] {
175            if !ENTITY_TYPES.contains(&ty.as_str()) {
176                return Err(format!("unknown entity type `{ty}` in edge vocabulary"));
177            }
178        }
179        if !REL_TYPES.contains(&rule.rel.as_str()) {
180            return Err(format!(
181                "unknown rel_type `{}` in edge vocabulary",
182                rule.rel
183            ));
184        }
185        if !seen_edges.insert((&rule.src, &rule.dst, &rule.rel)) {
186            return Err(format!(
187                "duplicate edge rule `{} -{}-> {}`",
188                rule.src, rule.rel, rule.dst
189            ));
190        }
191    }
192
193    let mut seen_columns = HashSet::new();
194    for candidate in &conventions.virtual_dst_candidates {
195        if candidate.column.is_empty() {
196            return Err("empty virtual destination column".to_string());
197        }
198        if !CONNECTION_TYPES.contains(&candidate.connection_type.as_str()) {
199            return Err(format!(
200                "unknown connection_type `{}` for virtual destination `{}`",
201                candidate.connection_type, candidate.column
202            ));
203        }
204        if !seen_columns.insert(&candidate.column) {
205            return Err(format!(
206                "duplicate virtual destination column `{}`",
207                candidate.column
208            ));
209        }
210    }
211
212    let per_table = conventions
213        .prometheus_info_metrics
214        .iter()
215        .chain(&conventions.otel_info_metrics)
216        .map(|(table, entities)| (table.as_str(), entities))
217        .chain(std::iter::once((
218            "otlp traces",
219            &conventions.otlp_trace_entities,
220        )));
221    // An entity type declared with a different number of id columns by two
222    // sources yields ids that can never match, silently splitting one entity
223    // in two.
224    let mut arity = HashMap::new();
225    for (table, entities) in per_table {
226        let mut seen_types = HashSet::new();
227        for implicit in entities {
228            if !ENTITY_TYPES.contains(&implicit.entity.as_str()) {
229                return Err(format!(
230                    "unknown entity type `{}` for info metric `{table}`",
231                    implicit.entity
232                ));
233            }
234            if !seen_types.insert(&implicit.entity) {
235                return Err(format!(
236                    "duplicate entity type `{}` for info metric `{table}`",
237                    implicit.entity
238                ));
239            }
240            if implicit.id.is_empty() || implicit.id.iter().any(String::is_empty) {
241                return Err(format!(
242                    "entity `{}` of info metric `{table}` needs non-empty id columns",
243                    implicit.entity
244                ));
245            }
246            if implicit.qualified_by.as_ref().is_some_and(String::is_empty) {
247                return Err(format!(
248                    "entity `{}` of info metric `{table}` has an empty qualified_by",
249                    implicit.entity
250                ));
251            }
252            if implicit.descriptive_rest && !implicit.descriptive.is_empty() {
253                return Err(format!(
254                    "entity `{}` of info metric `{table}` sets both descriptive and \
255                     descriptive_rest",
256                    implicit.entity
257                ));
258            }
259            match arity.insert(implicit.entity.as_str(), implicit.id.len()) {
260                Some(previous) if previous != implicit.id.len() => {
261                    return Err(format!(
262                        "entity `{}` is declared with {previous} id columns elsewhere but \
263                         {} for `{table}`",
264                        implicit.entity,
265                        implicit.id.len()
266                    ));
267                }
268                _ => {}
269            }
270        }
271    }
272    Ok(())
273}
274
275#[cfg(test)]
276mod tests {
277    use super::*;
278
279    #[test]
280    fn embedded_conventions_parse_and_validate() {
281        conventions().unwrap();
282    }
283
284    /// Each case must fail on exactly the rule it names, so the shared
285    /// boilerplate is valid and the mutated part is inside the vocabulary.
286    fn broken(edges: &str, info_metrics: &str, otel_metrics: &str) -> String {
287        format!(
288            "co_declared_edges: [{edges}]\ntrace_co_declared_edges: []\n\
289             virtual_dst_candidates: []\notlp_trace_entities: []\n\
290             prometheus_info_metrics: {{{info_metrics}}}\n\
291             otel_info_metrics: {{{otel_metrics}}}"
292        )
293    }
294
295    #[test]
296    fn validation_rejects_broken_conventions() {
297        let err = |edges, info, otel| parse(&broken(edges, info, otel)).unwrap_err();
298
299        assert!(err("{src: host, dst: service, rel: pets}", "", "").contains("unknown rel_type"));
300        assert!(
301            err("{src: k8s.pods, dst: k8s.node, rel: runs_on}", "", "")
302                .contains("unknown entity type")
303        );
304        assert!(
305            err(
306                "{src: host, dst: service, rel: uses}, {src: host, dst: service, rel: uses}",
307                "",
308                ""
309            )
310            .contains("duplicate edge rule")
311        );
312        assert!(
313            err(
314                "",
315                "t: [{entity: host, id: [x], descriptive: [y], descriptive_rest: true}]",
316                ""
317            )
318            .contains("descriptive_rest")
319        );
320        // the otel map runs through the same per-table validation
321        assert!(err("", "", "t: [{entity: hosts, id: [x]}]").contains("unknown entity type"));
322        // ids of a different arity for one type can never match each other
323        assert!(
324            err(
325                "",
326                "a: [{entity: host, id: [x]}]",
327                "b: [{entity: host, id: [x, y]}]"
328            )
329            .contains("id columns")
330        );
331        // Unknown YAML keys are rejected, catching typos in the embedded file.
332        assert!(
333            parse(&broken(
334                "{src: host, dst: service, rel: uses, direction: down}",
335                "",
336                ""
337            ))
338            .is_err()
339        );
340    }
341}