Skip to main content

frontend/instance/
entity_graph.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Frontend implementation of the [`EntityGraphProvider`]: the live connector that
16//! makes the computed `greptime_private.semantic_entities` /
17//! `semantic_relationships` tables produce rows.
18//!
19//! It enumerates the entity-identity declarations by iterating the catalog's
20//! `TableInfo` options (`greptime.semantic.entity.*`), builds the read-time
21//! derivation plans as typed DataFusion `Expr`s over the declaring tables'
22//! DataFrames (`operator::statement::semantic_graph`), and executes them through
23//! the query engine. Injected into the catalog manager after the engine is built,
24//! breaking the `catalog -> query` cycle.
25
26use std::collections::HashMap;
27use std::sync::Weak;
28
29use async_trait::async_trait;
30use catalog::CatalogManager;
31use catalog::system_schema::semantic_graph::EntityGraphProvider;
32use common_catalog::consts::{
33    DEFAULT_PRIVATE_SCHEMA_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, PG_CATALOG_NAME,
34    SERVICE_NAME_COLUMN,
35};
36use common_error::ext::BoxedError;
37use common_query::OutputData;
38use common_recordbatch::SendableRecordBatchStream;
39use common_telemetry::warn;
40use datafusion::dataframe::DataFrame;
41use datafusion_expr::LogicalPlan;
42use futures::TryStreamExt;
43use operator::statement::semantic_graph::{
44    CallsSource, EntityDeclaration, GraphWindow, RegistrySource, build_calls_plan,
45    build_registry_plan,
46};
47use query::QueryEngineRef;
48use session::context::{QueryContextBuilder, QueryContextRef};
49use snafu::ResultExt;
50use store_api::storage::ScanRequest;
51use table::TableRef;
52use table::metadata::TableInfo;
53use table::requests::{
54    EntityRole, is_trace_v1_table, parse_entity_columns, parse_entity_option_key,
55};
56
57use crate::error;
58
59/// The live [`EntityGraphProvider`], backed by the query engine.
60pub struct EntityGraphProviderImpl {
61    query_engine: QueryEngineRef,
62    catalog_manager: Weak<dyn CatalogManager>,
63}
64
65struct EntitySource {
66    declarations: Vec<EntityDeclaration>,
67    table: TableRef,
68}
69
70impl EntityGraphProviderImpl {
71    pub fn new(query_engine: QueryEngineRef, catalog_manager: Weak<dyn CatalogManager>) -> Self {
72        Self {
73            query_engine,
74            catalog_manager,
75        }
76    }
77
78    /// Parses `greptime.semantic.entity.<type>.{id|descriptive|scope}` options of
79    /// one table into per-type declarations. A type with no `id` columns is skipped.
80    fn parse_declarations(table_info: &TableInfo) -> Vec<EntityDeclaration> {
81        let Some(time_index) = table_info
82            .meta
83            .schema
84            .timestamp_column()
85            .map(|c| c.name.clone())
86        else {
87            return vec![];
88        };
89
90        // entity_type -> (id_columns, descriptive_columns, scope_columns)
91        type RoleColumns = (Vec<String>, Vec<String>, Vec<String>);
92        let mut by_type: HashMap<String, RoleColumns> = HashMap::new();
93        for (key, value) in &table_info.meta.options.extra_options {
94            let Some((entity_type, role)) = parse_entity_option_key(key) else {
95                continue;
96            };
97            let cols = parse_entity_columns(value);
98            let entry = by_type.entry(entity_type.to_string()).or_default();
99            match role {
100                EntityRole::Id => entry.0 = cols,
101                EntityRole::Descriptive => entry.1 = cols,
102                EntityRole::Scope => entry.2 = cols,
103            }
104        }
105
106        by_type
107            .into_iter()
108            .filter(|(_, (id, _, _))| !id.is_empty())
109            .filter_map(
110                |(entity_type, (id_columns, descriptive_columns, scope_columns))| {
111                    // A stale declaration (e.g. its column was dropped later)
112                    // must not poison every graph scan; skip it.
113                    let schema = &table_info.meta.schema;
114                    if let Some(missing) = id_columns
115                        .iter()
116                        .chain(&descriptive_columns)
117                        .chain(&scope_columns)
118                        .find(|c| schema.column_schema_by_name(c).is_none())
119                    {
120                        warn!(
121                            "Skipping entity declaration `{}` of table `{}`: column `{}` not found",
122                            entity_type, table_info.name, missing
123                        );
124                        return None;
125                    }
126                    Some(EntityDeclaration {
127                        schema: table_info.schema_name.clone(),
128                        table: table_info.name.clone(),
129                        time_index: time_index.clone(),
130                        entity_type,
131                        id_columns,
132                        descriptive_columns,
133                        scope_columns,
134                    })
135                },
136            )
137            .collect()
138    }
139
140    /// All entity declarations of one table. Trace-v1 tables created before the
141    /// ingest-side auto-stamp carry no `entity.service.id` option; synthesize it
142    /// (their schema is fixed), so their `calls` edges have endpoint entities.
143    /// A table with an explicit service declaration never gets the synthesized
144    /// one — not even when the explicit declaration is invalid and skipped:
145    /// silently falling back would change entity identity behind the user's back.
146    fn declarations_for(table_info: &TableInfo) -> Vec<EntityDeclaration> {
147        let mut declarations = Self::parse_declarations(table_info);
148        let has_explicit_service = table_info.meta.options.extra_options.keys().any(|key| {
149            parse_entity_option_key(key)
150                .is_some_and(|(ty, role)| ty == "service" && role == EntityRole::Id)
151        });
152        if is_trace_v1_table(table_info)
153            && !has_explicit_service
154            && table_info
155                .meta
156                .schema
157                .column_schema_by_name(SERVICE_NAME_COLUMN)
158                .is_some()
159            && let Some(time_index) = table_info
160                .meta
161                .schema
162                .timestamp_column()
163                .map(|c| c.name.clone())
164        {
165            declarations.push(EntityDeclaration {
166                schema: table_info.schema_name.clone(),
167                table: table_info.name.clone(),
168                time_index,
169                entity_type: "service".to_string(),
170                id_columns: vec![SERVICE_NAME_COLUMN.to_string()],
171                descriptive_columns: vec![],
172                scope_columns: vec![],
173            });
174        }
175        declarations
176    }
177
178    /// Enumerates entity declarations and trace tables across a catalog. Trace
179    /// tables are keyed off the engine-native `table_data_model` option (same
180    /// check as the Jaeger query path) so pre-existing trace tables without the
181    /// newer `greptime.semantic.*` stamps are recognized too.
182    async fn enumerate(
183        &self,
184        catalog: &str,
185    ) -> Result<(Vec<EntitySource>, Vec<(EntityDeclaration, TableRef)>), BoxedError> {
186        let Some(catalog_manager) = self.catalog_manager.upgrade() else {
187            return Ok((vec![], vec![]));
188        };
189
190        let mut declarations = vec![];
191        let mut traces = vec![];
192        let schemas = catalog_manager
193            .schema_names(catalog, None)
194            .await
195            .map_err(BoxedError::new)?;
196        for schema in schemas {
197            // User telemetry never lives in the system schemas; skip them to avoid
198            // scanning information_schema (including the computed graph tables) etc.
199            if schema == INFORMATION_SCHEMA_NAME
200                || schema == PG_CATALOG_NAME
201                || schema == DEFAULT_PRIVATE_SCHEMA_NAME
202            {
203                continue;
204            }
205            let mut tables = catalog_manager.tables(catalog, &schema, None);
206            while let Some(table) = tables.try_next().await.map_err(BoxedError::new)? {
207                let table_info = table.table_info();
208                let table_declarations = Self::declarations_for(&table_info);
209                if is_trace_v1_table(&table_info) {
210                    // TODO(entity-graph): validate the complete fixed trace-v1
211                    // schema before adding the table to calls derivation, and
212                    // skip malformed/stale tables instead of letting one poison
213                    // the whole relationship scan.
214                    match table_declarations
215                        .iter()
216                        .find(|d| d.entity_type == "service")
217                    {
218                        Some(service) => traces.push((service.clone(), table.clone())),
219                        // No usable service identity: the table cannot
220                        // contribute calls edges (see declarations_for).
221                        None => warn!(
222                            "Trace table `{}` has no usable service declaration; skipping calls derivation",
223                            table_info.name
224                        ),
225                    }
226                }
227                if !table_declarations.is_empty() {
228                    declarations.push(EntitySource {
229                        declarations: table_declarations,
230                        table,
231                    });
232                }
233            }
234        }
235        Ok((declarations, traces))
236    }
237
238    /// The time window to derive over, taken from the scan's time predicate.
239    ///
240    /// TODO(entity-graph): read the `observed_at` bounds out of `request.filters`
241    /// and build the window from them. Until then this always returns the default
242    /// (last hour), so a query whose time filter falls entirely outside the last
243    /// hour derives nothing and returns empty — a known limitation tracked for the
244    /// scan-time-predicate-pushdown follow-up.
245    fn query_window(_request: &ScanRequest) -> GraphWindow {
246        GraphWindow::default_last_hour()
247    }
248
249    fn read_table(&self, table: TableRef) -> Result<DataFrame, BoxedError> {
250        self.query_engine.read_table(table).map_err(BoxedError::new)
251    }
252
253    /// Executes a derivation plan and returns its live result stream.
254    ///
255    /// TODO(entity-graph): the QueryContext must come from the outer query
256    /// instead of being built here, so the derivation inherits the caller's
257    /// permissions, cancellation and deadline. Requires threading the context
258    /// through the computed-table scan path (planned next PR).
259    async fn execute_plan(
260        &self,
261        catalog: &str,
262        plan: LogicalPlan,
263    ) -> Result<Option<SendableRecordBatchStream>, BoxedError> {
264        let query_ctx: QueryContextRef = QueryContextBuilder::default()
265            .current_catalog(catalog.to_string())
266            .current_schema(DEFAULT_SCHEMA_NAME.to_string())
267            .build()
268            .into();
269        let output = self
270            .query_engine
271            .execute(plan, query_ctx)
272            .await
273            .map_err(BoxedError::new)?;
274        let stream = match output.data {
275            OutputData::Stream(stream) => stream,
276            OutputData::RecordBatches(batches) => batches.as_stream(),
277            OutputData::AffectedRows(_) => return Ok(None),
278        };
279        Ok(Some(stream))
280    }
281}
282
283#[async_trait]
284impl EntityGraphProvider for EntityGraphProviderImpl {
285    async fn scan_entities(
286        &self,
287        catalog: &str,
288        request: ScanRequest,
289    ) -> Result<Option<SendableRecordBatchStream>, BoxedError> {
290        let (sources, _) = self.enumerate(catalog).await?;
291        let mut plans = Vec::with_capacity(sources.len());
292        for source in sources {
293            plans.push(RegistrySource {
294                declarations: source.declarations,
295                scan: self.read_table(source.table)?,
296            });
297        }
298        let window = Self::query_window(&request);
299        let Some(plan) = build_registry_plan(plans, &window)
300            .context(error::DataFusionSnafu)
301            .map_err(BoxedError::new)?
302        else {
303            return Ok(None);
304        };
305        self.execute_plan(catalog, plan).await
306    }
307
308    async fn scan_relationships(
309        &self,
310        catalog: &str,
311        request: ScanRequest,
312    ) -> Result<Option<SendableRecordBatchStream>, BoxedError> {
313        let (_, traces) = self.enumerate(catalog).await?;
314        let mut scans = Vec::with_capacity(traces.len());
315        for (service, trace) in traces {
316            scans.push(CallsSource {
317                service,
318                scan: self.read_table(trace)?,
319            });
320        }
321        let window = Self::query_window(&request);
322        let Some(plan) = build_calls_plan(scans, &window)
323            .context(error::DataFusionSnafu)
324            .map_err(BoxedError::new)?
325        else {
326            return Ok(None);
327        };
328        self.execute_plan(catalog, plan).await
329    }
330}
331
332#[cfg(test)]
333mod tests {
334    use std::collections::HashMap;
335    use std::sync::Arc;
336
337    use common_catalog::consts::{DEFAULT_CATALOG_NAME, MITO_ENGINE};
338    use datatypes::prelude::ConcreteDataType;
339    use datatypes::schema::{ColumnSchema, SchemaBuilder};
340    use table::metadata::{TableInfoBuilder, TableMeta, TableType};
341    use table::requests::{TABLE_DATA_MODEL, TABLE_DATA_MODEL_TRACE_V1, TableOptions};
342
343    use super::*;
344
345    fn table_info(columns: &[&str], extra: &[(&str, &str)]) -> TableInfo {
346        let mut column_schemas = vec![
347            ColumnSchema::new(
348                "ts",
349                ConcreteDataType::timestamp_millisecond_datatype(),
350                false,
351            )
352            .with_time_index(true),
353        ];
354        column_schemas.extend(
355            columns
356                .iter()
357                .map(|c| ColumnSchema::new(*c, ConcreteDataType::string_datatype(), true)),
358        );
359        let schema = Arc::new(
360            SchemaBuilder::try_from_columns(column_schemas)
361                .unwrap()
362                .build()
363                .unwrap(),
364        );
365        let options = TableOptions {
366            extra_options: extra
367                .iter()
368                .map(|(k, v)| (k.to_string(), v.to_string()))
369                .collect::<HashMap<_, _>>(),
370            ..Default::default()
371        };
372        let meta = TableMeta {
373            schema,
374            primary_key_indices: vec![],
375            value_indices: vec![],
376            engine: MITO_ENGINE.to_string(),
377            next_column_id: 1,
378            options,
379            created_on: Default::default(),
380            updated_on: Default::default(),
381            partition_key_indices: vec![],
382            column_ids: vec![],
383        };
384        TableInfoBuilder::default()
385            .table_id(1)
386            .name("t1")
387            .catalog_name(DEFAULT_CATALOG_NAME)
388            .schema_name(DEFAULT_SCHEMA_NAME)
389            .table_version(0)
390            .table_type(TableType::Base)
391            .meta(meta)
392            .build()
393            .unwrap()
394    }
395
396    #[test]
397    fn declaration_referencing_missing_column_is_skipped() {
398        let info = table_info(
399            &["service_name"],
400            &[
401                ("greptime.semantic.entity.service.id", "service_name"),
402                ("greptime.semantic.entity.host.id", "gone"),
403            ],
404        );
405        let declarations = EntityGraphProviderImpl::declarations_for(&info);
406        assert_eq!(declarations.len(), 1);
407        assert_eq!(declarations[0].entity_type, "service");
408
409        let info = table_info(
410            &["service_name"],
411            &[
412                ("greptime.semantic.entity.service.id", "service_name"),
413                ("greptime.semantic.entity.service.descriptive", "gone"),
414            ],
415        );
416        assert!(EntityGraphProviderImpl::declarations_for(&info).is_empty());
417    }
418
419    #[test]
420    fn trace_v1_table_gets_implicit_service_declaration() {
421        let info = table_info(
422            &["service_name"],
423            &[(TABLE_DATA_MODEL, TABLE_DATA_MODEL_TRACE_V1)],
424        );
425        let declarations = EntityGraphProviderImpl::declarations_for(&info);
426        assert_eq!(declarations.len(), 1);
427        let decl = &declarations[0];
428        assert_eq!(decl.entity_type, "service");
429        assert_eq!(decl.id_columns, vec![SERVICE_NAME_COLUMN.to_string()]);
430        assert_eq!(decl.time_index, "ts");
431
432        // An explicit service declaration wins; no duplicate is synthesized.
433        let info = table_info(
434            &["service_name"],
435            &[
436                (TABLE_DATA_MODEL, TABLE_DATA_MODEL_TRACE_V1),
437                ("greptime.semantic.entity.service.id", "service_name"),
438            ],
439        );
440        assert_eq!(EntityGraphProviderImpl::declarations_for(&info).len(), 1);
441
442        // A trace-model table without the fixed column synthesizes nothing.
443        let info = table_info(&[], &[(TABLE_DATA_MODEL, TABLE_DATA_MODEL_TRACE_V1)]);
444        assert!(EntityGraphProviderImpl::declarations_for(&info).is_empty());
445
446        // An explicit but invalid declaration must not silently fall back to
447        // the synthesized one: identity must not change behind the user's back.
448        let info = table_info(
449            &["service_name"],
450            &[
451                (TABLE_DATA_MODEL, TABLE_DATA_MODEL_TRACE_V1),
452                ("greptime.semantic.entity.service.id", "gone"),
453            ],
454        );
455        assert!(EntityGraphProviderImpl::declarations_for(&info).is_empty());
456    }
457}