operator/statement/semantic_graph/
conventions.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! The built-in derivation conventions of the entity graph, embedded as
16//! [`conventions.yaml`](./conventions.yaml): the co-declared edge vocabulary,
17//! the virtual-destination candidates, and the implicit declarations of
18//! well-known Prometheus entity-descriptor metrics. The file is data shipped
19//! with the binary, not an operator-editable configuration surface; explicit
20//! `greptime.semantic.entity.*` declarations always override it.
21
22use std::collections::{BTreeMap, HashSet};
23use std::sync::LazyLock;
24
25use serde::Deserialize;
26
27/// A same-row co-declaration rule: a source-table row carrying both entity
28/// identities witnesses `src -rel-> dst`.
29#[derive(Debug, Deserialize)]
30#[serde(deny_unknown_fields)]
31pub struct EdgeRule {
32    pub src: String,
33    pub dst: String,
34    pub rel: String,
35}
36
37/// A span-attribute column that may name an uninstrumented peer, and the
38/// `connection_type` its match implies.
39#[derive(Debug, Deserialize)]
40#[serde(deny_unknown_fields)]
41pub struct VirtualDstCandidate {
42    pub column: String,
43    pub connection_type: String,
44}
45
46/// One implicit entity declaration: of a whitelisted Prometheus info metric,
47/// or of a trace-v1 table's flattened resource attributes.
48#[derive(Debug, Deserialize)]
49#[serde(deny_unknown_fields)]
50pub struct ImplicitEntity {
51    pub entity: String,
52    /// Identifying label columns; every one must exist on the table for the
53    /// declaration to apply.
54    pub id: Vec<String>,
55    /// Descriptive label columns, filtered to those present (kube-state-metrics
56    /// label sets vary across versions).
57    #[serde(default)]
58    pub descriptive: Vec<String>,
59    /// Snapshot every tag column except the id columns as descriptive —
60    /// `target_info`-style enrichment over an open attribute set.
61    #[serde(default)]
62    pub descriptive_rest: bool,
63}
64
65/// The parsed, validated conventions file.
66#[derive(Debug, Deserialize)]
67#[serde(deny_unknown_fields)]
68pub struct Conventions {
69    pub co_declared_edges: Vec<EdgeRule>,
70    pub trace_co_declared_edges: Vec<EdgeRule>,
71    pub virtual_dst_candidates: Vec<VirtualDstCandidate>,
72    pub otlp_trace_entities: Vec<ImplicitEntity>,
73    pub prometheus_info_metrics: BTreeMap<String, Vec<ImplicitEntity>>,
74}
75
76/// The built-in entity-type vocabulary. User-declared types are open-ended;
77/// the embedded conventions file must stay inside this set.
78pub const ENTITY_TYPE_SERVICE: &str = "service";
79pub const ENTITY_TYPE_SERVICE_INSTANCE: &str = "service.instance";
80pub const ENTITY_TYPE_HOST: &str = "host";
81pub const ENTITY_TYPE_PROCESS: &str = "process";
82pub const ENTITY_TYPE_K8S_POD: &str = "k8s.pod";
83pub const ENTITY_TYPE_K8S_NODE: &str = "k8s.node";
84pub const ENTITY_TYPE_K8S_CONTAINER: &str = "k8s.container";
85pub const ENTITY_TYPE_K8S_WORKLOAD: &str = "k8s.workload";
86pub const ENTITY_TYPE_K8S_SERVICE: &str = "k8s.service";
87pub const ENTITY_TYPE_GEN_AI_AGENT: &str = "gen_ai.agent";
88pub const ENTITY_TYPE_GEN_AI_MODEL: &str = "gen_ai.model";
89pub const ENTITY_TYPE_GEN_AI_TOOL: &str = "gen_ai.tool";
90
91const ENTITY_TYPES: [&str; 12] = [
92    ENTITY_TYPE_SERVICE,
93    ENTITY_TYPE_SERVICE_INSTANCE,
94    ENTITY_TYPE_HOST,
95    ENTITY_TYPE_PROCESS,
96    ENTITY_TYPE_K8S_POD,
97    ENTITY_TYPE_K8S_NODE,
98    ENTITY_TYPE_K8S_CONTAINER,
99    ENTITY_TYPE_K8S_WORKLOAD,
100    ENTITY_TYPE_K8S_SERVICE,
101    ENTITY_TYPE_GEN_AI_AGENT,
102    ENTITY_TYPE_GEN_AI_MODEL,
103    ENTITY_TYPE_GEN_AI_TOOL,
104];
105
106/// The relationship vocabulary (the RFC's rel_type table) and the edge
107/// provenances.
108pub const REL_TYPE_CALLS: &str = "calls";
109pub const REL_TYPE_RUNS_ON: &str = "runs_on";
110pub const REL_TYPE_CONTAINS: &str = "contains";
111pub const REL_TYPE_PART_OF: &str = "part_of";
112pub const REL_TYPE_USES: &str = "uses";
113pub const REL_TYPE_INVOKES: &str = "invokes";
114pub const REL_TYPE_DEPENDS_ON: &str = "depends_on";
115pub const REL_TYPE_OWNS: &str = "owns";
116pub const PROVENANCE_TRACE: &str = "trace";
117pub const PROVENANCE_ATTRIBUTE: &str = "attribute";
118pub const PROVENANCE_DECLARED: &str = "declared";
119pub const PROVENANCE_AGENT: &str = "agent";
120
121const REL_TYPES: [&str; 8] = [
122    REL_TYPE_CALLS,
123    REL_TYPE_RUNS_ON,
124    REL_TYPE_CONTAINS,
125    REL_TYPE_PART_OF,
126    REL_TYPE_USES,
127    REL_TYPE_INVOKES,
128    REL_TYPE_DEPENDS_ON,
129    REL_TYPE_OWNS,
130];
131
132/// The `connection_type` values a virtual-destination candidate may imply;
133/// the calls derivation branches on them when building edge attributes.
134pub const CONNECTION_TYPE_DATABASE: &str = "database";
135pub const CONNECTION_TYPE_VIRTUAL_NODE: &str = "virtual_node";
136
137const CONNECTION_TYPES: [&str; 2] = [CONNECTION_TYPE_DATABASE, CONNECTION_TYPE_VIRTUAL_NODE];
138
139static CONVENTIONS: LazyLock<Result<Conventions, String>> =
140    LazyLock::new(|| parse(include_str!("conventions.yaml")));
141
142/// The embedded conventions. `Err` means the embedded file is broken — pinned
143/// by unit test, and propagated by the derivation paths rather than panicking.
144pub fn conventions() -> Result<&'static Conventions, String> {
145    CONVENTIONS.as_ref().map_err(Clone::clone)
146}
147
148fn parse(yaml: &str) -> Result<Conventions, String> {
149    let conventions: Conventions =
150        serde_yaml_ng::from_str(yaml).map_err(|e| format!("malformed conventions: {e}"))?;
151    validate(&conventions)?;
152    Ok(conventions)
153}
154
155fn validate(conventions: &Conventions) -> Result<(), String> {
156    let mut seen_edges = HashSet::new();
157    for rule in conventions
158        .co_declared_edges
159        .iter()
160        .chain(&conventions.trace_co_declared_edges)
161    {
162        for ty in [&rule.src, &rule.dst] {
163            if !ENTITY_TYPES.contains(&ty.as_str()) {
164                return Err(format!("unknown entity type `{ty}` in edge vocabulary"));
165            }
166        }
167        if !REL_TYPES.contains(&rule.rel.as_str()) {
168            return Err(format!(
169                "unknown rel_type `{}` in edge vocabulary",
170                rule.rel
171            ));
172        }
173        if !seen_edges.insert((&rule.src, &rule.dst, &rule.rel)) {
174            return Err(format!(
175                "duplicate edge rule `{} -{}-> {}`",
176                rule.src, rule.rel, rule.dst
177            ));
178        }
179    }
180
181    let mut seen_columns = HashSet::new();
182    for candidate in &conventions.virtual_dst_candidates {
183        if candidate.column.is_empty() {
184            return Err("empty virtual destination column".to_string());
185        }
186        if !CONNECTION_TYPES.contains(&candidate.connection_type.as_str()) {
187            return Err(format!(
188                "unknown connection_type `{}` for virtual destination `{}`",
189                candidate.connection_type, candidate.column
190            ));
191        }
192        if !seen_columns.insert(&candidate.column) {
193            return Err(format!(
194                "duplicate virtual destination column `{}`",
195                candidate.column
196            ));
197        }
198    }
199
200    let per_table = conventions
201        .prometheus_info_metrics
202        .iter()
203        .map(|(table, entities)| (table.as_str(), entities))
204        .chain(std::iter::once((
205            "otlp traces",
206            &conventions.otlp_trace_entities,
207        )));
208    for (table, entities) in per_table {
209        let mut seen_types = HashSet::new();
210        for implicit in entities {
211            if !ENTITY_TYPES.contains(&implicit.entity.as_str()) {
212                return Err(format!(
213                    "unknown entity type `{}` for info metric `{table}`",
214                    implicit.entity
215                ));
216            }
217            if !seen_types.insert(&implicit.entity) {
218                return Err(format!(
219                    "duplicate entity type `{}` for info metric `{table}`",
220                    implicit.entity
221                ));
222            }
223            if implicit.id.is_empty() || implicit.id.iter().any(String::is_empty) {
224                return Err(format!(
225                    "entity `{}` of info metric `{table}` needs non-empty id columns",
226                    implicit.entity
227                ));
228            }
229            if implicit.descriptive_rest && !implicit.descriptive.is_empty() {
230                return Err(format!(
231                    "entity `{}` of info metric `{table}` sets both descriptive and \
232                     descriptive_rest",
233                    implicit.entity
234                ));
235            }
236        }
237    }
238    Ok(())
239}
240
241#[cfg(test)]
242mod tests {
243    use super::*;
244
245    #[test]
246    fn embedded_conventions_parse_and_validate() {
247        conventions().unwrap();
248    }
249
250    /// Each case must fail on exactly the rule it names, so the shared
251    /// boilerplate is valid and the mutated part is inside the vocabulary.
252    fn broken(edges: &str, info_metrics: &str) -> String {
253        format!(
254            "co_declared_edges: [{edges}]\ntrace_co_declared_edges: []\n\
255             virtual_dst_candidates: []\notlp_trace_entities: []\n\
256             prometheus_info_metrics: {{{info_metrics}}}"
257        )
258    }
259
260    #[test]
261    fn validation_rejects_broken_conventions() {
262        let err = |edges, info| parse(&broken(edges, info)).unwrap_err();
263
264        assert!(err("{src: host, dst: service, rel: pets}", "").contains("unknown rel_type"));
265        assert!(
266            err("{src: k8s.pods, dst: k8s.node, rel: runs_on}", "").contains("unknown entity type")
267        );
268        assert!(
269            err(
270                "{src: host, dst: service, rel: uses}, {src: host, dst: service, rel: uses}",
271                ""
272            )
273            .contains("duplicate edge rule")
274        );
275        assert!(
276            err(
277                "",
278                "t: [{entity: host, id: [x], descriptive: [y], descriptive_rest: true}]"
279            )
280            .contains("descriptive_rest")
281        );
282        // Unknown YAML keys are rejected, catching typos in the embedded file.
283        assert!(
284            parse(&broken(
285                "{src: host, dst: service, rel: uses, direction: down}",
286                ""
287            ))
288            .is_err()
289        );
290    }
291}
operator/statement/semantic_graph/conventions.rs

operator/statement/semantic_graph/
conventions.rs
