1use std::collections::HashMap;
27use std::sync::Weak;
28
29use async_trait::async_trait;
30use catalog::CatalogManager;
31use catalog::system_schema::semantic_graph::EntityGraphProvider;
32use common_catalog::consts::{
33 DEFAULT_PRIVATE_SCHEMA_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, PG_CATALOG_NAME,
34 SERVICE_NAME_COLUMN,
35};
36use common_error::ext::BoxedError;
37use common_query::OutputData;
38use common_recordbatch::SendableRecordBatchStream;
39use common_telemetry::warn;
40use datafusion::dataframe::DataFrame;
41use datafusion_expr::LogicalPlan;
42use futures::TryStreamExt;
43use operator::statement::semantic_graph::{
44 CallsSource, EntityDeclaration, GraphWindow, RegistrySource, build_calls_plan,
45 build_registry_plan,
46};
47use query::QueryEngineRef;
48use session::context::{QueryContextBuilder, QueryContextRef};
49use snafu::ResultExt;
50use store_api::storage::ScanRequest;
51use table::TableRef;
52use table::metadata::TableInfo;
53use table::requests::{
54 EntityRole, is_trace_v1_table, parse_entity_columns, parse_entity_option_key,
55};
56
57use crate::error;
58
59pub struct EntityGraphProviderImpl {
61 query_engine: QueryEngineRef,
62 catalog_manager: Weak<dyn CatalogManager>,
63}
64
65struct EntitySource {
66 declarations: Vec<EntityDeclaration>,
67 table: TableRef,
68}
69
70impl EntityGraphProviderImpl {
71 pub fn new(query_engine: QueryEngineRef, catalog_manager: Weak<dyn CatalogManager>) -> Self {
72 Self {
73 query_engine,
74 catalog_manager,
75 }
76 }
77
78 fn parse_declarations(table_info: &TableInfo) -> Vec<EntityDeclaration> {
81 let Some(time_index) = table_info
82 .meta
83 .schema
84 .timestamp_column()
85 .map(|c| c.name.clone())
86 else {
87 return vec![];
88 };
89
90 type RoleColumns = (Vec<String>, Vec<String>, Vec<String>);
92 let mut by_type: HashMap<String, RoleColumns> = HashMap::new();
93 for (key, value) in &table_info.meta.options.extra_options {
94 let Some((entity_type, role)) = parse_entity_option_key(key) else {
95 continue;
96 };
97 let cols = parse_entity_columns(value);
98 let entry = by_type.entry(entity_type.to_string()).or_default();
99 match role {
100 EntityRole::Id => entry.0 = cols,
101 EntityRole::Descriptive => entry.1 = cols,
102 EntityRole::Scope => entry.2 = cols,
103 }
104 }
105
106 by_type
107 .into_iter()
108 .filter(|(_, (id, _, _))| !id.is_empty())
109 .filter_map(
110 |(entity_type, (id_columns, descriptive_columns, scope_columns))| {
111 let schema = &table_info.meta.schema;
114 if let Some(missing) = id_columns
115 .iter()
116 .chain(&descriptive_columns)
117 .chain(&scope_columns)
118 .find(|c| schema.column_schema_by_name(c).is_none())
119 {
120 warn!(
121 "Skipping entity declaration `{}` of table `{}`: column `{}` not found",
122 entity_type, table_info.name, missing
123 );
124 return None;
125 }
126 Some(EntityDeclaration {
127 schema: table_info.schema_name.clone(),
128 table: table_info.name.clone(),
129 time_index: time_index.clone(),
130 entity_type,
131 id_columns,
132 descriptive_columns,
133 scope_columns,
134 })
135 },
136 )
137 .collect()
138 }
139
140 fn declarations_for(table_info: &TableInfo) -> Vec<EntityDeclaration> {
147 let mut declarations = Self::parse_declarations(table_info);
148 let has_explicit_service = table_info.meta.options.extra_options.keys().any(|key| {
149 parse_entity_option_key(key)
150 .is_some_and(|(ty, role)| ty == "service" && role == EntityRole::Id)
151 });
152 if is_trace_v1_table(table_info)
153 && !has_explicit_service
154 && table_info
155 .meta
156 .schema
157 .column_schema_by_name(SERVICE_NAME_COLUMN)
158 .is_some()
159 && let Some(time_index) = table_info
160 .meta
161 .schema
162 .timestamp_column()
163 .map(|c| c.name.clone())
164 {
165 declarations.push(EntityDeclaration {
166 schema: table_info.schema_name.clone(),
167 table: table_info.name.clone(),
168 time_index,
169 entity_type: "service".to_string(),
170 id_columns: vec![SERVICE_NAME_COLUMN.to_string()],
171 descriptive_columns: vec![],
172 scope_columns: vec![],
173 });
174 }
175 declarations
176 }
177
178 async fn enumerate(
183 &self,
184 catalog: &str,
185 ) -> Result<(Vec<EntitySource>, Vec<(EntityDeclaration, TableRef)>), BoxedError> {
186 let Some(catalog_manager) = self.catalog_manager.upgrade() else {
187 return Ok((vec![], vec![]));
188 };
189
190 let mut declarations = vec![];
191 let mut traces = vec![];
192 let schemas = catalog_manager
193 .schema_names(catalog, None)
194 .await
195 .map_err(BoxedError::new)?;
196 for schema in schemas {
197 if schema == INFORMATION_SCHEMA_NAME
200 || schema == PG_CATALOG_NAME
201 || schema == DEFAULT_PRIVATE_SCHEMA_NAME
202 {
203 continue;
204 }
205 let mut tables = catalog_manager.tables(catalog, &schema, None);
206 while let Some(table) = tables.try_next().await.map_err(BoxedError::new)? {
207 let table_info = table.table_info();
208 let table_declarations = Self::declarations_for(&table_info);
209 if is_trace_v1_table(&table_info) {
210 match table_declarations
215 .iter()
216 .find(|d| d.entity_type == "service")
217 {
218 Some(service) => traces.push((service.clone(), table.clone())),
219 None => warn!(
222 "Trace table `{}` has no usable service declaration; skipping calls derivation",
223 table_info.name
224 ),
225 }
226 }
227 if !table_declarations.is_empty() {
228 declarations.push(EntitySource {
229 declarations: table_declarations,
230 table,
231 });
232 }
233 }
234 }
235 Ok((declarations, traces))
236 }
237
238 fn query_window(_request: &ScanRequest) -> GraphWindow {
246 GraphWindow::default_last_hour()
247 }
248
249 fn read_table(&self, table: TableRef) -> Result<DataFrame, BoxedError> {
250 self.query_engine.read_table(table).map_err(BoxedError::new)
251 }
252
253 async fn execute_plan(
260 &self,
261 catalog: &str,
262 plan: LogicalPlan,
263 ) -> Result<Option<SendableRecordBatchStream>, BoxedError> {
264 let query_ctx: QueryContextRef = QueryContextBuilder::default()
265 .current_catalog(catalog.to_string())
266 .current_schema(DEFAULT_SCHEMA_NAME.to_string())
267 .build()
268 .into();
269 let output = self
270 .query_engine
271 .execute(plan, query_ctx)
272 .await
273 .map_err(BoxedError::new)?;
274 let stream = match output.data {
275 OutputData::Stream(stream) => stream,
276 OutputData::RecordBatches(batches) => batches.as_stream(),
277 OutputData::AffectedRows(_) => return Ok(None),
278 };
279 Ok(Some(stream))
280 }
281}
282
283#[async_trait]
284impl EntityGraphProvider for EntityGraphProviderImpl {
285 async fn scan_entities(
286 &self,
287 catalog: &str,
288 request: ScanRequest,
289 ) -> Result<Option<SendableRecordBatchStream>, BoxedError> {
290 let (sources, _) = self.enumerate(catalog).await?;
291 let mut plans = Vec::with_capacity(sources.len());
292 for source in sources {
293 plans.push(RegistrySource {
294 declarations: source.declarations,
295 scan: self.read_table(source.table)?,
296 });
297 }
298 let window = Self::query_window(&request);
299 let Some(plan) = build_registry_plan(plans, &window)
300 .context(error::DataFusionSnafu)
301 .map_err(BoxedError::new)?
302 else {
303 return Ok(None);
304 };
305 self.execute_plan(catalog, plan).await
306 }
307
308 async fn scan_relationships(
309 &self,
310 catalog: &str,
311 request: ScanRequest,
312 ) -> Result<Option<SendableRecordBatchStream>, BoxedError> {
313 let (_, traces) = self.enumerate(catalog).await?;
314 let mut scans = Vec::with_capacity(traces.len());
315 for (service, trace) in traces {
316 scans.push(CallsSource {
317 service,
318 scan: self.read_table(trace)?,
319 });
320 }
321 let window = Self::query_window(&request);
322 let Some(plan) = build_calls_plan(scans, &window)
323 .context(error::DataFusionSnafu)
324 .map_err(BoxedError::new)?
325 else {
326 return Ok(None);
327 };
328 self.execute_plan(catalog, plan).await
329 }
330}
331
332#[cfg(test)]
333mod tests {
334 use std::collections::HashMap;
335 use std::sync::Arc;
336
337 use common_catalog::consts::{DEFAULT_CATALOG_NAME, MITO_ENGINE};
338 use datatypes::prelude::ConcreteDataType;
339 use datatypes::schema::{ColumnSchema, SchemaBuilder};
340 use table::metadata::{TableInfoBuilder, TableMeta, TableType};
341 use table::requests::{TABLE_DATA_MODEL, TABLE_DATA_MODEL_TRACE_V1, TableOptions};
342
343 use super::*;
344
345 fn table_info(columns: &[&str], extra: &[(&str, &str)]) -> TableInfo {
346 let mut column_schemas = vec![
347 ColumnSchema::new(
348 "ts",
349 ConcreteDataType::timestamp_millisecond_datatype(),
350 false,
351 )
352 .with_time_index(true),
353 ];
354 column_schemas.extend(
355 columns
356 .iter()
357 .map(|c| ColumnSchema::new(*c, ConcreteDataType::string_datatype(), true)),
358 );
359 let schema = Arc::new(
360 SchemaBuilder::try_from_columns(column_schemas)
361 .unwrap()
362 .build()
363 .unwrap(),
364 );
365 let options = TableOptions {
366 extra_options: extra
367 .iter()
368 .map(|(k, v)| (k.to_string(), v.to_string()))
369 .collect::<HashMap<_, _>>(),
370 ..Default::default()
371 };
372 let meta = TableMeta {
373 schema,
374 primary_key_indices: vec![],
375 value_indices: vec![],
376 engine: MITO_ENGINE.to_string(),
377 next_column_id: 1,
378 options,
379 created_on: Default::default(),
380 updated_on: Default::default(),
381 partition_key_indices: vec![],
382 column_ids: vec![],
383 };
384 TableInfoBuilder::default()
385 .table_id(1)
386 .name("t1")
387 .catalog_name(DEFAULT_CATALOG_NAME)
388 .schema_name(DEFAULT_SCHEMA_NAME)
389 .table_version(0)
390 .table_type(TableType::Base)
391 .meta(meta)
392 .build()
393 .unwrap()
394 }
395
396 #[test]
397 fn declaration_referencing_missing_column_is_skipped() {
398 let info = table_info(
399 &["service_name"],
400 &[
401 ("greptime.semantic.entity.service.id", "service_name"),
402 ("greptime.semantic.entity.host.id", "gone"),
403 ],
404 );
405 let declarations = EntityGraphProviderImpl::declarations_for(&info);
406 assert_eq!(declarations.len(), 1);
407 assert_eq!(declarations[0].entity_type, "service");
408
409 let info = table_info(
410 &["service_name"],
411 &[
412 ("greptime.semantic.entity.service.id", "service_name"),
413 ("greptime.semantic.entity.service.descriptive", "gone"),
414 ],
415 );
416 assert!(EntityGraphProviderImpl::declarations_for(&info).is_empty());
417 }
418
419 #[test]
420 fn trace_v1_table_gets_implicit_service_declaration() {
421 let info = table_info(
422 &["service_name"],
423 &[(TABLE_DATA_MODEL, TABLE_DATA_MODEL_TRACE_V1)],
424 );
425 let declarations = EntityGraphProviderImpl::declarations_for(&info);
426 assert_eq!(declarations.len(), 1);
427 let decl = &declarations[0];
428 assert_eq!(decl.entity_type, "service");
429 assert_eq!(decl.id_columns, vec![SERVICE_NAME_COLUMN.to_string()]);
430 assert_eq!(decl.time_index, "ts");
431
432 let info = table_info(
434 &["service_name"],
435 &[
436 (TABLE_DATA_MODEL, TABLE_DATA_MODEL_TRACE_V1),
437 ("greptime.semantic.entity.service.id", "service_name"),
438 ],
439 );
440 assert_eq!(EntityGraphProviderImpl::declarations_for(&info).len(), 1);
441
442 let info = table_info(&[], &[(TABLE_DATA_MODEL, TABLE_DATA_MODEL_TRACE_V1)]);
444 assert!(EntityGraphProviderImpl::declarations_for(&info).is_empty());
445
446 let info = table_info(
449 &["service_name"],
450 &[
451 (TABLE_DATA_MODEL, TABLE_DATA_MODEL_TRACE_V1),
452 ("greptime.semantic.entity.service.id", "gone"),
453 ],
454 );
455 assert!(EntityGraphProviderImpl::declarations_for(&info).is_empty());
456 }
457}