mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-10-03 10:35:35 +00:00
* feat(trace): support Semantic Graph for Trace V2 Signed-off-by: luofucong <luofc@foxmail.com> * fix: remove unused annotation context import Signed-off-by: luofucong <luofc@foxmail.com> --------- Signed-off-by: luofucong <luofc@foxmail.com>
This commit is contained in:
@@ -202,6 +202,12 @@ pub fn is_ddl_reserved_table(schema: &str, table: &str) -> bool {
|
||||
pub const TRACE_ID_COLUMN: &str = "trace_id";
|
||||
pub const SPAN_ID_COLUMN: &str = "span_id";
|
||||
pub const SPAN_NAME_COLUMN: &str = "span_name";
|
||||
/// Column containing span attributes as JSON/JSON2.
|
||||
pub const SPAN_ATTRIBUTES_COLUMN: &str = "span_attributes";
|
||||
/// Column containing instrumentation scope attributes as JSON2.
|
||||
pub const SCOPE_ATTRIBUTES_COLUMN: &str = "scope_attributes";
|
||||
/// Column containing resource attributes as JSON2.
|
||||
pub const RESOURCE_ATTRIBUTES_COLUMN: &str = "resource_attributes";
|
||||
pub const SERVICE_NAME_COLUMN: &str = "service_name";
|
||||
pub const PARENT_SPAN_ID_COLUMN: &str = "parent_span_id";
|
||||
// More fixed columns/values of the `greptime_trace_v1` data model, shared by
|
||||
|
||||
@@ -23,8 +23,8 @@
|
||||
//! the query engine. Injected into the catalog manager after the engine is built,
|
||||
//! breaking the `catalog -> query` cycle.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::sync::Weak;
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
use std::sync::{Arc, Weak};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use auth::{
|
||||
@@ -41,13 +41,16 @@ use common_catalog::consts::{
|
||||
};
|
||||
use common_error::ext::{BoxedError, ErrorExt};
|
||||
use common_error::status_code::StatusCode;
|
||||
use common_function::scalars::json::json_get::JsonGetWithType;
|
||||
use common_function::scalars::udf::create_udf;
|
||||
use common_query::OutputData;
|
||||
use common_query::prelude::OTLP_AGGREGATION_TEMPORALITY_LABEL;
|
||||
use common_recordbatch::SendableRecordBatchStream;
|
||||
use common_telemetry::{debug, warn};
|
||||
use common_time::timestamp::TimeUnit;
|
||||
use datafusion::common::{Column, ScalarValue};
|
||||
use datafusion::dataframe::DataFrame;
|
||||
use datafusion_expr::LogicalPlan;
|
||||
use datafusion_expr::{Expr, LogicalPlan, lit};
|
||||
use futures::TryStreamExt;
|
||||
use operator::statement::semantic_graph::{
|
||||
CallsSource, CoDeclaredSource, Conventions, DeclaredSource, ENTITY_TYPE_GEN_AI_AGENT,
|
||||
@@ -64,8 +67,8 @@ use table::metadata::TableInfo;
|
||||
use table::predicate::{TimeRangeExtraction, extract_time_range_strict};
|
||||
use table::requests::{
|
||||
EntityRole, SEMANTIC_METRIC_TYPE, SEMANTIC_SIGNAL_TYPE, SEMANTIC_SOURCE, SIGNAL_TYPE_METRIC,
|
||||
SOURCE_OPENTELEMETRY, SOURCE_PROMETHEUS, is_trace_v1_table, parse_entity_columns,
|
||||
parse_entity_option_key,
|
||||
SOURCE_OPENTELEMETRY, SOURCE_PROMETHEUS, TABLE_DATA_MODEL_TRACE_V2, is_trace_table,
|
||||
parse_entity_columns, parse_entity_option_key, trace_v2_attribute,
|
||||
};
|
||||
|
||||
use crate::error;
|
||||
@@ -164,12 +167,11 @@ impl EntityGraphProviderImpl {
|
||||
|(entity_type, (id_columns, descriptive_columns, scope_columns))| {
|
||||
// A stale declaration (e.g. its column was dropped later)
|
||||
// must not poison every graph scan; skip it.
|
||||
let schema = &table_info.meta.schema;
|
||||
if let Some(missing) = id_columns
|
||||
.iter()
|
||||
.chain(&descriptive_columns)
|
||||
.chain(&scope_columns)
|
||||
.find(|c| schema.column_schema_by_name(c).is_none())
|
||||
.find(|c| !Self::can_resolve_entity_column(table_info, c))
|
||||
{
|
||||
warn!(
|
||||
"Skipping entity declaration `{}` of table `{}`: column `{}` not found",
|
||||
@@ -194,7 +196,7 @@ impl EntityGraphProviderImpl {
|
||||
}
|
||||
|
||||
/// All entity declarations of one table: the explicit options plus the
|
||||
/// zero-configuration conventions (`otlp_trace_entities` for trace-v1
|
||||
/// zero-configuration conventions (`otlp_trace_entities` for trace-v1/v2
|
||||
/// tables — including the `service` identity of tables created before the
|
||||
/// ingest-side auto-stamp — and the Prometheus/OTel descriptor
|
||||
/// whitelists). An explicit declaration of a type always suppresses the
|
||||
@@ -207,7 +209,7 @@ impl EntityGraphProviderImpl {
|
||||
) -> Vec<EntityDeclaration> {
|
||||
let mut declarations = Self::parse_declarations(table_info);
|
||||
let mut supersessions = Vec::new();
|
||||
if is_trace_v1_table(table_info) {
|
||||
if is_trace_table(table_info) {
|
||||
Self::extend_with_implicit_entities(
|
||||
table_info,
|
||||
&conventions.otlp_trace_entities,
|
||||
@@ -303,8 +305,8 @@ impl EntityGraphProviderImpl {
|
||||
}
|
||||
|
||||
/// Synthesizes the applicable subset of `entities` on `table_info`:
|
||||
/// explicit declarations win, every id column must exist (no guessing),
|
||||
/// descriptive columns are filtered to those present.
|
||||
/// explicit declarations win, every id reference must be resolvable,
|
||||
/// and descriptive references are filtered by the same rule.
|
||||
fn extend_with_implicit_entities(
|
||||
table_info: &TableInfo,
|
||||
entities: &[ImplicitEntity],
|
||||
@@ -330,7 +332,7 @@ impl EntityGraphProviderImpl {
|
||||
if let Some(missing) = implicit
|
||||
.id
|
||||
.iter()
|
||||
.find(|c| schema.column_schema_by_name(c).is_none())
|
||||
.find(|c| !Self::can_resolve_entity_column(table_info, c))
|
||||
{
|
||||
debug!(
|
||||
"Table `{}` lacks the id column `{}`; skipping the implicit `{}` declaration",
|
||||
@@ -350,7 +352,7 @@ impl EntityGraphProviderImpl {
|
||||
implicit
|
||||
.descriptive
|
||||
.iter()
|
||||
.filter(|c| schema.column_schema_by_name(c).is_some())
|
||||
.filter(|c| Self::can_resolve_entity_column(table_info, c))
|
||||
.cloned()
|
||||
.collect()
|
||||
};
|
||||
@@ -359,7 +361,7 @@ impl EntityGraphProviderImpl {
|
||||
let id_qualifier = implicit
|
||||
.qualified_by
|
||||
.clone()
|
||||
.filter(|c| schema.column_schema_by_name(c).is_some());
|
||||
.filter(|c| Self::can_resolve_entity_column(table_info, c));
|
||||
if let Some(entity_type) = &implicit.superseded_by {
|
||||
supersessions.push((declarations.len(), entity_type.clone()));
|
||||
}
|
||||
@@ -434,7 +436,7 @@ impl EntityGraphProviderImpl {
|
||||
while let Some(table) = tables.try_next().await.map_err(BoxedError::new)? {
|
||||
let table_info = table.table_info();
|
||||
let table_declarations = Self::declarations_for(&table_info, conventions);
|
||||
let is_trace = is_trace_v1_table(&table_info);
|
||||
let is_trace = is_trace_table(&table_info);
|
||||
// Authorize only tables that would contribute rows.
|
||||
if per_table_auth
|
||||
&& (is_trace || !table_declarations.is_empty())
|
||||
@@ -532,8 +534,63 @@ impl EntityGraphProviderImpl {
|
||||
}
|
||||
}
|
||||
|
||||
fn read_table(&self, table: TableRef) -> Result<DataFrame, BoxedError> {
|
||||
self.query_engine.read_table(table).map_err(BoxedError::new)
|
||||
fn can_resolve_entity_column(table_info: &TableInfo, column: &str) -> bool {
|
||||
let table_schema = &table_info.meta.schema;
|
||||
if table_schema.column_schema_by_name(column).is_some() {
|
||||
return true;
|
||||
}
|
||||
let data_model = table_info.meta.options.data_model();
|
||||
trace_v2_attribute(table_schema, data_model, column).is_some()
|
||||
}
|
||||
|
||||
/// Projects referenced V2 attributes under their declaration names for
|
||||
/// the shared entity and relationship plans. Other models scan unchanged.
|
||||
fn read_table(
|
||||
&self,
|
||||
table: TableRef,
|
||||
declarations: &[EntityDeclaration],
|
||||
) -> Result<DataFrame, BoxedError> {
|
||||
let info = table.table_info();
|
||||
let mut scan = self
|
||||
.query_engine
|
||||
.read_table(table)
|
||||
.map_err(BoxedError::new)?;
|
||||
let data_model = info.meta.options.data_model();
|
||||
if data_model != Some(TABLE_DATA_MODEL_TRACE_V2) {
|
||||
return Ok(scan);
|
||||
}
|
||||
|
||||
// Read Trace V2 table:
|
||||
let mut columns = BTreeSet::new();
|
||||
for declaration in declarations {
|
||||
columns.extend(declaration.id_columns.iter());
|
||||
columns.extend(declaration.descriptive_columns.iter());
|
||||
columns.extend(declaration.scope_columns.iter());
|
||||
columns.extend(declaration.id_qualifier.iter());
|
||||
columns.extend(declaration.superseded_by_columns.iter());
|
||||
}
|
||||
let conventions = conventions()
|
||||
.map_err(datafusion::error::DataFusionError::Internal)
|
||||
.context(error::DataFusionSnafu)
|
||||
.map_err(BoxedError::new)?;
|
||||
let virtual_candidates = &conventions.virtual_dst_candidates;
|
||||
columns.extend(virtual_candidates.iter().map(|candidate| &candidate.column));
|
||||
let get = create_udf(Arc::new(JsonGetWithType::default()));
|
||||
for column in columns {
|
||||
if let Some((root, key)) = trace_v2_attribute(&info.meta.schema, data_model, column) {
|
||||
let path = format!("$.{}", serde_json::Value::String(key.to_string()));
|
||||
let value = get.call(vec![
|
||||
Expr::Column(Column::from_name(root)),
|
||||
lit(path),
|
||||
lit(ScalarValue::Utf8View(None)),
|
||||
]);
|
||||
scan = scan
|
||||
.with_column(column, value)
|
||||
.context(error::DataFusionSnafu)
|
||||
.map_err(BoxedError::new)?;
|
||||
}
|
||||
}
|
||||
Ok(scan)
|
||||
}
|
||||
|
||||
/// The declared-edge branch source, when the physical table exists, the
|
||||
@@ -587,7 +644,7 @@ impl EntityGraphProviderImpl {
|
||||
));
|
||||
}
|
||||
Ok(Some(DeclaredSource {
|
||||
scan: self.read_table(table)?,
|
||||
scan: self.read_table(table, &[])?,
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -633,8 +690,8 @@ impl EntityGraphProvider for EntityGraphProviderImpl {
|
||||
let mut plans = Vec::with_capacity(sources.len());
|
||||
for source in sources {
|
||||
plans.push(RegistrySource {
|
||||
scan: self.read_table(source.table, &source.declarations)?,
|
||||
declarations: source.declarations,
|
||||
scan: self.read_table(source.table)?,
|
||||
});
|
||||
}
|
||||
let Some(window) = Self::query_window(&request)? else {
|
||||
@@ -658,18 +715,24 @@ impl EntityGraphProvider for EntityGraphProviderImpl {
|
||||
let (sources, traces) = self.enumerate(catalog, query_ctx.as_deref()).await?;
|
||||
let mut calls = Vec::with_capacity(traces.len());
|
||||
for trace in traces {
|
||||
let declarations = trace
|
||||
.service
|
||||
.iter()
|
||||
.chain(trace.agent.iter())
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
calls.push(CallsSource {
|
||||
scan: self.read_table(trace.table, &declarations)?,
|
||||
service: trace.service,
|
||||
agent: trace.agent,
|
||||
scan: self.read_table(trace.table)?,
|
||||
});
|
||||
}
|
||||
let mut co_declared = Vec::with_capacity(sources.len());
|
||||
for source in sources {
|
||||
co_declared.push(CoDeclaredSource {
|
||||
scan: self.read_table(source.table, &source.declarations)?,
|
||||
declarations: source.declarations,
|
||||
is_trace: source.is_trace,
|
||||
scan: self.read_table(source.table)?,
|
||||
});
|
||||
}
|
||||
let declared = self.declared_source(catalog, query_ctx.as_deref()).await?;
|
||||
|
||||
@@ -370,12 +370,7 @@ async fn query_trace_table(
|
||||
})?;
|
||||
|
||||
let table_info = table.table_info();
|
||||
let data_model = table_info
|
||||
.meta
|
||||
.options
|
||||
.extra_options
|
||||
.get(table::requests::TABLE_DATA_MODEL)
|
||||
.map(String::as_str);
|
||||
let data_model = table_info.meta.options.data_model();
|
||||
|
||||
// collect to set
|
||||
let col_names = table_info
|
||||
|
||||
@@ -1509,7 +1509,7 @@ pub fn validate_trace_table_model(table_info: &TableInfo, ctx: &QueryContextRef)
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
if let Some(actual) = table_info.meta.options.extra_options.get(TABLE_DATA_MODEL) {
|
||||
if let Some(actual) = table_info.meta.options.data_model() {
|
||||
ensure!(
|
||||
actual == expected,
|
||||
InvalidInsertRequestSnafu {
|
||||
|
||||
@@ -100,8 +100,8 @@ use table::TableRef;
|
||||
use table::dist_table::DistTable;
|
||||
use table::metadata::{self, TableId, TableInfo, TableMeta, TableType};
|
||||
use table::requests::{
|
||||
AlterKind, AlterTableRequest, AnnotationContext, COMMENT_KEY, DDL_TIMEOUT, DDL_WAIT,
|
||||
INGEST_ROWS_RATE_LIMIT_KEY, TableOptions, validate_and_normalize_annotation_options,
|
||||
AlterKind, AlterTableRequest, COMMENT_KEY, DDL_TIMEOUT, DDL_WAIT, INGEST_ROWS_RATE_LIMIT_KEY,
|
||||
TableOptions, validate_and_normalize_annotation_options,
|
||||
};
|
||||
use table::table_name::TableName;
|
||||
use table::table_reference::TableReference;
|
||||
@@ -2657,11 +2657,7 @@ fn validate_and_normalize_annotations(
|
||||
partition_key_indices: &[usize],
|
||||
) -> Result<()> {
|
||||
use table::requests::AnnotationValidationError as CheckError;
|
||||
let cx = AnnotationContext {
|
||||
schema,
|
||||
partition_key_indices,
|
||||
};
|
||||
validate_and_normalize_annotation_options(options, &cx).map_err(|e| match e {
|
||||
let handle_error = |e: CheckError| match e {
|
||||
CheckError::ColumnNotFound { column } => ColumnNotFoundSnafu { msg: column }.build(),
|
||||
e @ (CheckError::UnknownKey { .. }
|
||||
| CheckError::InvalidValue { .. }
|
||||
@@ -2676,7 +2672,9 @@ fn validate_and_normalize_annotations(
|
||||
reason: e.to_string(),
|
||||
}
|
||||
.build(),
|
||||
})
|
||||
};
|
||||
validate_and_normalize_annotation_options(options, schema, partition_key_indices)
|
||||
.map_err(handle_error)
|
||||
}
|
||||
|
||||
fn find_partition_columns(partitions: &Option<Partitions>) -> Result<Vec<String>> {
|
||||
|
||||
@@ -44,7 +44,7 @@ pub struct VirtualDstCandidate {
|
||||
}
|
||||
|
||||
/// One implicit entity declaration: of a whitelisted Prometheus or OTel info
|
||||
/// metric, or of a trace-v1 table's flattened resource attributes.
|
||||
/// metric, or of a Trace V1/V2 table's resource attributes.
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ImplicitEntity {
|
||||
|
||||
@@ -37,7 +37,7 @@ virtual_dst_candidates:
|
||||
- { column: span_attributes.db.name, connection_type: database }
|
||||
- { column: span_attributes.server.address, connection_type: virtual_node }
|
||||
|
||||
# Implicit declarations for greptime_trace_v1 tables; the pod UID bridges
|
||||
# Implicit declarations for Trace V1/V2 tables; the pod UID bridges
|
||||
# trace-side pods onto the kube-state-metrics entities below. host.name and
|
||||
# container.name vary by SDK and resource detector, so identity is the id.
|
||||
# qualified_by folds the namespace into the service identity the same way the
|
||||
|
||||
@@ -28,7 +28,8 @@ pub use common_catalog::consts::{
|
||||
SPAN_STATUS_CODE_COLUMN as SPAN_STATUS_CODE, TRACE_TIMESTAMP_COLUMN as TIMESTAMP_COLUMN,
|
||||
};
|
||||
pub use common_catalog::consts::{
|
||||
PARENT_SPAN_ID_COLUMN, SPAN_ID_COLUMN, SPAN_NAME_COLUMN, TRACE_ID_COLUMN,
|
||||
PARENT_SPAN_ID_COLUMN, RESOURCE_ATTRIBUTES_COLUMN, SCOPE_ATTRIBUTES_COLUMN,
|
||||
SPAN_ATTRIBUTES_COLUMN, SPAN_ID_COLUMN, SPAN_NAME_COLUMN, TRACE_ID_COLUMN,
|
||||
};
|
||||
use pipeline::PipelineWay;
|
||||
|
||||
@@ -37,14 +38,10 @@ use crate::otlp::trace::span::TraceSpan;
|
||||
|
||||
pub const TIMESTAMP_END_COLUMN: &str = "timestamp_end";
|
||||
pub const SPAN_STATUS_MESSAGE_COLUMN: &str = "span_status_message";
|
||||
pub const SPAN_ATTRIBUTES_COLUMN: &str = "span_attributes";
|
||||
pub const SPAN_EVENTS_COLUMN: &str = "span_events";
|
||||
pub const SPAN_LINKS_COLUMN: &str = "span_links";
|
||||
pub const SCOPE_NAME_COLUMN: &str = "scope_name";
|
||||
pub const SCOPE_VERSION_COLUMN: &str = "scope_version";
|
||||
/// Column containing instrumentation scope attributes as a JSON2 object.
|
||||
pub const SCOPE_ATTRIBUTES_COLUMN: &str = "scope_attributes";
|
||||
pub const RESOURCE_ATTRIBUTES_COLUMN: &str = "resource_attributes";
|
||||
pub const TRACE_STATE_COLUMN: &str = "trace_state";
|
||||
|
||||
// const keys
|
||||
|
||||
@@ -470,6 +470,7 @@ impl TableMeta {
|
||||
.build()
|
||||
})?;
|
||||
let cx = AnnotationContext {
|
||||
data_model: self.options.data_model(),
|
||||
schema: &self.schema,
|
||||
partition_key_indices: &self.partition_key_indices,
|
||||
};
|
||||
|
||||
+25
-13
@@ -63,17 +63,14 @@ pub const TABLE_DATA_MODEL_TRACE_V1: &str = "greptime_trace_v1";
|
||||
/// Table data model used by the JSON2-based OTLP trace pipeline.
|
||||
pub const TABLE_DATA_MODEL_TRACE_V2: &str = "greptime_trace_v2";
|
||||
|
||||
/// Returns true if the table stores spans in the `greptime_trace_v1` data model
|
||||
/// (fixed span columns), the shape the Jaeger query path and the entity-graph
|
||||
/// derivation rely on.
|
||||
pub fn is_trace_v1_table(table_info: &crate::metadata::TableInfo) -> bool {
|
||||
table_info
|
||||
.meta
|
||||
.options
|
||||
.extra_options
|
||||
.get(TABLE_DATA_MODEL)
|
||||
.map(|v| v == TABLE_DATA_MODEL_TRACE_V1)
|
||||
.unwrap_or(false)
|
||||
/// Returns true for the Trace V1 and V2 data models supported
|
||||
/// by semantic graph derivation.
|
||||
pub fn is_trace_table(table_info: &crate::metadata::TableInfo) -> bool {
|
||||
let table_data_model = table_info.meta.options.data_model();
|
||||
matches!(
|
||||
table_data_model,
|
||||
Some(TABLE_DATA_MODEL_TRACE_V1 | TABLE_DATA_MODEL_TRACE_V2)
|
||||
)
|
||||
}
|
||||
|
||||
pub const OTLP_METRIC_COMPAT_KEY: &str = "otlp_metric_compat";
|
||||
@@ -226,6 +223,11 @@ pub const REPARTITION_COLUMN_HINT_KEY: &str = "repartition.column.hint";
|
||||
pub const REPARTITION_PARTITION_NUM_HINT_KEY: &str = "repartition.partition.num.hint";
|
||||
|
||||
impl TableOptions {
|
||||
/// Returns the table data model, if specified.
|
||||
pub fn data_model(&self) -> Option<&str> {
|
||||
self.extra_options.get(TABLE_DATA_MODEL).map(String::as_str)
|
||||
}
|
||||
|
||||
pub fn try_from_iter<T: ToString, U: IntoIterator<Item = (T, T)>>(
|
||||
iter: U,
|
||||
) -> Result<TableOptions> {
|
||||
@@ -480,6 +482,7 @@ pub fn validate_annotation_keys<'a>(
|
||||
|
||||
/// Table shape an annotation option is validated against.
|
||||
pub struct AnnotationContext<'a> {
|
||||
pub data_model: Option<&'a str>,
|
||||
pub schema: &'a Schema,
|
||||
pub partition_key_indices: &'a [usize],
|
||||
}
|
||||
@@ -570,6 +573,9 @@ pub(crate) fn validate_and_normalize_annotation(
|
||||
}
|
||||
if parse_entity_option_key(key).is_some() {
|
||||
for column in parse_entity_columns(value) {
|
||||
if trace_v2_attribute(cx.schema, cx.data_model, &column).is_some() {
|
||||
continue;
|
||||
}
|
||||
let schema = cx.schema.column_schema_by_name(&column).ok_or_else(|| {
|
||||
AnnotationValidationError::ColumnNotFound {
|
||||
column: column.clone(),
|
||||
@@ -620,14 +626,20 @@ pub(crate) fn validate_and_normalize_annotation(
|
||||
/// and writes normalized values back in place.
|
||||
pub fn validate_and_normalize_annotation_options(
|
||||
options: &mut TableOptions,
|
||||
cx: &AnnotationContext<'_>,
|
||||
schema: &Schema,
|
||||
partition_key_indices: &[usize],
|
||||
) -> std::result::Result<(), AnnotationValidationError> {
|
||||
let cx = AnnotationContext {
|
||||
data_model: options.data_model(),
|
||||
schema,
|
||||
partition_key_indices,
|
||||
};
|
||||
let mut normalized = Vec::new();
|
||||
for (key, value) in &options.extra_options {
|
||||
let Some(family) = AnnotationFamily::of_key(key) else {
|
||||
continue;
|
||||
};
|
||||
let checked = validate_and_normalize_annotation(family, cx, key, value)?;
|
||||
let checked = validate_and_normalize_annotation(family, &cx, key, value)?;
|
||||
if checked != *value {
|
||||
normalized.push((key.clone(), checked));
|
||||
}
|
||||
|
||||
@@ -30,7 +30,13 @@
|
||||
//! [`crate::requests::validate_table_option`], so they are accepted both on the
|
||||
//! ingestion auto-create path and on explicit `CREATE TABLE ... WITH (...)` DDL.
|
||||
|
||||
use common_catalog::consts::{
|
||||
RESOURCE_ATTRIBUTES_COLUMN, SCOPE_ATTRIBUTES_COLUMN, SPAN_ATTRIBUTES_COLUMN,
|
||||
};
|
||||
use datatypes::prelude::ConcreteDataType;
|
||||
use datatypes::schema::{ColumnSchema, Schema};
|
||||
|
||||
use crate::requests::TABLE_DATA_MODEL_TRACE_V2;
|
||||
|
||||
/// Reserved prefix for every public semantic table-option key.
|
||||
pub const SEMANTIC_PREFIX: &str = "greptime.semantic.";
|
||||
@@ -193,6 +199,34 @@ pub fn is_entity_option_key(key: &str) -> bool {
|
||||
parse_entity_option_key(key).is_some()
|
||||
}
|
||||
|
||||
/// Resolves a Trace V2 entity reference to a JSON2 root and a literal attribute key.
|
||||
/// Everything after the first dot belongs to the key, matching Trace V1's
|
||||
/// flattened column names. Existing physical columns take precedence.
|
||||
pub fn trace_v2_attribute<'a>(
|
||||
schema: &Schema,
|
||||
data_model: Option<&str>,
|
||||
column: &'a str,
|
||||
) -> Option<(&'a str, &'a str)> {
|
||||
if data_model != Some(TABLE_DATA_MODEL_TRACE_V2)
|
||||
|| schema.column_schema_by_name(column).is_some()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let (root, key) = column.split_once('.')?;
|
||||
(!key.is_empty()
|
||||
&& schema
|
||||
.column_schema_by_name(root)
|
||||
.is_some_and(is_v2_attribute_column))
|
||||
.then_some((root, key))
|
||||
}
|
||||
|
||||
fn is_v2_attribute_column(schema: &ColumnSchema) -> bool {
|
||||
matches!(
|
||||
schema.name.as_str(),
|
||||
RESOURCE_ATTRIBUTES_COLUMN | SCOPE_ATTRIBUTES_COLUMN | SPAN_ATTRIBUTES_COLUMN
|
||||
) && schema.data_type.is_json2()
|
||||
}
|
||||
|
||||
/// Returns true if a column of `data_type` renders as a stable string — the
|
||||
/// requirement for entity id/descriptive/scope columns. The read-time
|
||||
/// derivation casts them to strings, so a type without a stable string form
|
||||
@@ -217,8 +251,9 @@ pub fn has_stable_string_form(data_type: &ConcreteDataType) -> bool {
|
||||
pub fn parse_entity_columns(value: &str) -> Vec<String> {
|
||||
value
|
||||
.split(',')
|
||||
.map(|c| c.trim().to_string())
|
||||
.filter(|c| !c.is_empty())
|
||||
.map(str::trim)
|
||||
.filter(|column| !column.is_empty())
|
||||
.map(String::from)
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -244,7 +279,7 @@ pub fn is_semantic_option_key(key: &str) -> bool {
|
||||
/// are rejected.
|
||||
pub fn validate_semantic_option(key: &str, value: &str) -> bool {
|
||||
if is_entity_option_key(key) {
|
||||
return !value.is_empty() && value.split(',').all(|col| !col.trim().is_empty());
|
||||
return !value.is_empty() && value.split(',').all(|column| !column.trim().is_empty());
|
||||
}
|
||||
match key {
|
||||
SEMANTIC_PIPELINE
|
||||
@@ -293,8 +328,55 @@ pub fn validate_semantic_option(key: &str, value: &str) -> bool {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use datatypes::types::JsonType;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_trace_v2_attribute_references() {
|
||||
let schema = Schema::new(vec![
|
||||
ColumnSchema::new(
|
||||
RESOURCE_ATTRIBUTES_COLUMN,
|
||||
ConcreteDataType::Json(JsonType::null()),
|
||||
true,
|
||||
),
|
||||
ColumnSchema::new(
|
||||
SCOPE_ATTRIBUTES_COLUMN,
|
||||
ConcreteDataType::string_datatype(),
|
||||
true,
|
||||
),
|
||||
ColumnSchema::new(
|
||||
"resource_attributes.physical",
|
||||
ConcreteDataType::string_datatype(),
|
||||
true,
|
||||
),
|
||||
]);
|
||||
let model = Some(TABLE_DATA_MODEL_TRACE_V2);
|
||||
for key in ["host.name", "host..name", "host[0]", "host.*", r#"a"b\c.d"#] {
|
||||
let column = format!("resource_attributes.{key}");
|
||||
assert_eq!(
|
||||
trace_v2_attribute(&schema, model, &column),
|
||||
Some((RESOURCE_ATTRIBUTES_COLUMN, key))
|
||||
);
|
||||
}
|
||||
for column in [
|
||||
"resource_attributes",
|
||||
"resource_attributes.",
|
||||
"scope_attributes.name",
|
||||
"span_attributes.name",
|
||||
"resource_attributes.physical",
|
||||
"missing.key",
|
||||
] {
|
||||
assert_eq!(trace_v2_attribute(&schema, model, column), None, "{column}");
|
||||
}
|
||||
for model in [None, Some(crate::requests::TABLE_DATA_MODEL_TRACE_V1)] {
|
||||
assert_eq!(
|
||||
trace_v2_attribute(&schema, model, "resource_attributes.host.name"),
|
||||
None
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_semantic_option_key() {
|
||||
assert!(is_semantic_option_key(SEMANTIC_SIGNAL_TYPE));
|
||||
|
||||
@@ -0,0 +1,329 @@
|
||||
-- Trace V2 uses the same flattened attribute references as V1.
|
||||
-- Everything after the root column is one literal key, including dots.
|
||||
create table graph_trace_v2 (
|
||||
"timestamp" timestamp(9) time index,
|
||||
timestamp_end timestamp(9),
|
||||
duration_nano bigint,
|
||||
parent_span_id string,
|
||||
trace_id string,
|
||||
span_id string,
|
||||
span_kind string,
|
||||
span_name string,
|
||||
span_status_code string,
|
||||
span_status_message string,
|
||||
trace_state string,
|
||||
scope_name string,
|
||||
scope_version string,
|
||||
service_name string,
|
||||
span_attributes json2,
|
||||
scope_attributes json2,
|
||||
resource_attributes json2,
|
||||
span_events json,
|
||||
span_links json,
|
||||
primary key (service_name)
|
||||
) with (
|
||||
'table_data_model' = 'greptime_trace_v2',
|
||||
'append_mode' = 'true',
|
||||
'greptime.semantic.entity.gen_ai.agent.id' = 'span_attributes.gen_ai.agent.id',
|
||||
'greptime.semantic.entity.gen_ai.agent.scope' = 'scope_attributes.environment',
|
||||
'greptime.semantic.entity.gen_ai.model.id' = 'span_attributes.gen_ai.request.model',
|
||||
'greptime.semantic.entity.literal_peer.id' = 'span_attributes.db.namespace',
|
||||
'greptime.semantic.entity.literal_resource.id' = 'resource_attributes.host.name',
|
||||
'greptime.semantic.entity.literal_span.id' = 'span_attributes.gen_ai.agent.id',
|
||||
'greptime.semantic.entity.literal_scope.id' = 'scope_attributes.deployment.region.name'
|
||||
);
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
insert into graph_trace_v2
|
||||
("timestamp", duration_nano, trace_id, span_id, span_kind, span_status_code,
|
||||
service_name, span_attributes, scope_attributes, resource_attributes)
|
||||
values
|
||||
('2026-01-01 00:00:01', 100000000, 'paired', 'client', 'SPAN_KIND_CLIENT', 'STATUS_CODE_UNSET',
|
||||
'frontend', '{"gen_ai.agent.id":"agent-a","gen_ai.request.model":"model-a","gen_ai":{"agent":{"id":"nested-agent"}}}',
|
||||
'{"environment":"prod","deployment.region.name":"literal-region","deployment":{"region.name":"nested-region"},"a\"b\\c.d":7}', '{"service.namespace":"shop","service.instance.id":"front-1","host.id":"host-1","host.name":"node-1","host":{"name":"nested-node"},"service":{"namespace":"wrong-namespace"}}'),
|
||||
('2026-01-01 00:00:02', 200000000, 'virtual', 'client', 'SPAN_KIND_CLIENT', 'STATUS_CODE_ERROR',
|
||||
'frontend', '{"service.peer.name":"","db.namespace":"orders","db":{"namespace":"wrong-orders"}}', '{}',
|
||||
'{"service.namespace":"shop","service.instance.id":"front-1","host.id":"host-1","host.name":"node-1"}'),
|
||||
('2026-01-01 00:00:03', 100, 'missing', 'internal', 'SPAN_KIND_INTERNAL', 'STATUS_CODE_UNSET',
|
||||
'no-resource', '{"gen_ai.agent.id":null}', '{}', '{}');
|
||||
|
||||
Affected Rows: 3
|
||||
|
||||
-- Agent parent/child references are independent of span kind.
|
||||
insert into graph_trace_v2
|
||||
("timestamp", duration_nano, trace_id, span_id, parent_span_id, span_kind,
|
||||
span_status_code, service_name, span_attributes, scope_attributes, resource_attributes)
|
||||
values ('2026-01-01 00:00:01', 30000000, 'paired', 'agent-child', 'client',
|
||||
'SPAN_KIND_INTERNAL', 'STATUS_CODE_UNSET', 'frontend',
|
||||
'{"gen_ai.agent.id":"agent-b"}', '{"environment":"prod"}', '{"service.namespace":"shop"}');
|
||||
|
||||
Affected Rows: 1
|
||||
|
||||
-- Calls can pair a V2 client with a V1 server. The literal_resource declaration
|
||||
-- has exactly the same spelling in both models, including its output JSON keys.
|
||||
create table graph_trace_v1_peer (
|
||||
"timestamp" timestamp(9) time index,
|
||||
trace_id string, span_id string, parent_span_id string,
|
||||
span_kind string, span_status_code string, duration_nano bigint,
|
||||
service_name string,
|
||||
"resource_attributes.service.namespace" string,
|
||||
"resource_attributes.host.name" string,
|
||||
primary key (service_name)
|
||||
) with ('table_data_model' = 'greptime_trace_v1', 'append_mode' = 'true',
|
||||
'greptime.semantic.entity.literal_resource.id' = 'resource_attributes.host.name');
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
insert into graph_trace_v1_peer values
|
||||
('2026-01-01 00:00:01', 'paired', 'server', 'client', 'SPAN_KIND_SERVER', 'STATUS_CODE_UNSET', 50000000, 'backend', 'shop', 'node-v1');
|
||||
|
||||
Affected Rows: 1
|
||||
|
||||
select entity_type, entity_id, scope
|
||||
from greptime_private.semantic_entities
|
||||
where observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
|
||||
order by entity_type, entity_id;
|
||||
|
||||
+------------------+-----------------------+-------+
|
||||
| entity_type | entity_id | scope |
|
||||
+------------------+-----------------------+-------+
|
||||
| gen_ai.agent | agent-a | prod |
|
||||
| gen_ai.agent | agent-b | prod |
|
||||
| gen_ai.model | model-a | |
|
||||
| host | host-1 | |
|
||||
| literal_peer | orders | |
|
||||
| literal_resource | node-1 | |
|
||||
| literal_resource | node-v1 | |
|
||||
| literal_scope | literal-region | |
|
||||
| literal_span | agent-a | |
|
||||
| literal_span | agent-b | |
|
||||
| service | no-resource | |
|
||||
| service | shop/backend | |
|
||||
| service | shop/frontend | |
|
||||
| service.instance | shop/frontend,front-1 | |
|
||||
+------------------+-----------------------+-------+
|
||||
|
||||
select src_type, src_id, dst_type, dst_id, rel_type, request_count, error_count
|
||||
from greptime_private.semantic_relationships
|
||||
where observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
|
||||
order by rel_type, src_type, src_id, dst_id;
|
||||
|
||||
+------------------+-----------------------+--------------+---------------+----------+---------------+-------------+
|
||||
| src_type | src_id | dst_type | dst_id | rel_type | request_count | error_count |
|
||||
+------------------+-----------------------+--------------+---------------+----------+---------------+-------------+
|
||||
| gen_ai.agent | agent-a | gen_ai.agent | agent-b | calls | 1 | 0 |
|
||||
| service | shop/frontend | service | orders | calls | 1 | 1 |
|
||||
| service | shop/frontend | service | shop/backend | calls | 1 | 0 |
|
||||
| service.instance | shop/frontend,front-1 | service | shop/frontend | part_of | | |
|
||||
| service.instance | shop/frontend,front-1 | host | host-1 | runs_on | | |
|
||||
| gen_ai.agent | agent-a | gen_ai.model | model-a | uses | | |
|
||||
+------------------+-----------------------+--------------+---------------+----------+---------------+-------------+
|
||||
|
||||
-- Empty components in the V1-compatible comma-separated list remain invalid.
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'resource_attributes.host.id,';
|
||||
|
||||
Error: 1004(InvalidArguments), Invalid alter table(graph_trace_v2) request: invalid value `resource_attributes.host.id,` for semantic option `greptime.semantic.entity.invalid.id`
|
||||
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'resource_attributes.host.id,,service_name';
|
||||
|
||||
Error: 1004(InvalidArguments), Invalid alter table(graph_trace_v2) request: invalid value `resource_attributes.host.id,,service_name` for semantic option `greptime.semantic.entity.invalid.id`
|
||||
|
||||
-- ALTER accepts JSON2 references; a missing key simply yields no entity.
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.custom.id' = 'scope_attributes.missing';
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.host.descriptive' = 'resource_attributes.host.name';
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
-- Preserve implicit host identity by explicitly declaring it as well.
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.host.id' = 'resource_attributes.host.id';
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
admin flush_table('graph_trace_v2');
|
||||
|
||||
+-------------------------------------+
|
||||
| ADMIN flush_table('graph_trace_v2') |
|
||||
+-------------------------------------+
|
||||
| 0 |
|
||||
+-------------------------------------+
|
||||
|
||||
admin flush_table('graph_trace_v1_peer');
|
||||
|
||||
+------------------------------------------+
|
||||
| ADMIN flush_table('graph_trace_v1_peer') |
|
||||
+------------------------------------------+
|
||||
| 0 |
|
||||
+------------------------------------------+
|
||||
|
||||
-- SQLNESS PROTOCOL MYSQL
|
||||
select entity_type, entity_id, scope, entity_id_attrs, descriptive
|
||||
from greptime_private.semantic_entities
|
||||
where observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
|
||||
order by entity_type, entity_id;
|
||||
|
||||
+------------------+-----------------------+-------+--------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------+
|
||||
| entity_type | entity_id | scope | entity_id_attrs | descriptive |
|
||||
+------------------+-----------------------+-------+--------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------+
|
||||
| gen_ai.agent | agent-a | prod | {"span_attributes.gen_ai.agent.id":"agent-a"} | |
|
||||
| gen_ai.agent | agent-b | prod | {"span_attributes.gen_ai.agent.id":"agent-b"} | |
|
||||
| gen_ai.model | model-a | | {"span_attributes.gen_ai.request.model":"model-a"} | |
|
||||
| host | host-1 | | {"resource_attributes.host.id":"host-1"} | {"resource_attributes.host.name":"node-1"} |
|
||||
| literal_peer | orders | | {"span_attributes.db.namespace":"orders"} | |
|
||||
| literal_resource | node-1 | | {"resource_attributes.host.name":"node-1"} | |
|
||||
| literal_resource | node-v1 | | {"resource_attributes.host.name":"node-v1"} | |
|
||||
| literal_scope | literal-region | | {"scope_attributes.deployment.region.name":"literal-region"} | |
|
||||
| literal_span | agent-a | | {"span_attributes.gen_ai.agent.id":"agent-a"} | |
|
||||
| literal_span | agent-b | | {"span_attributes.gen_ai.agent.id":"agent-b"} | |
|
||||
| service | no-resource | | {"resource_attributes.service.namespace":"","service_name":"no-resource"} | |
|
||||
| service | shop/backend | | {"resource_attributes.service.namespace":"shop","service_name":"backend"} | |
|
||||
| service | shop/frontend | | {"resource_attributes.service.namespace":"shop","service_name":"frontend"} | |
|
||||
| service.instance | shop/frontend,front-1 | | {"resource_attributes.service.instance.id":"front-1","resource_attributes.service.namespace":"shop","service_name":"frontend"} | |
|
||||
+------------------+-----------------------+-------+--------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------+
|
||||
|
||||
select src_type, src_id, dst_type, dst_id, rel_type, request_count, error_count
|
||||
from greptime_private.semantic_relationships
|
||||
where observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
|
||||
order by rel_type, src_type, src_id, dst_id;
|
||||
|
||||
+------------------+-----------------------+--------------+---------------+----------+---------------+-------------+
|
||||
| src_type | src_id | dst_type | dst_id | rel_type | request_count | error_count |
|
||||
+------------------+-----------------------+--------------+---------------+----------+---------------+-------------+
|
||||
| gen_ai.agent | agent-a | gen_ai.agent | agent-b | calls | 1 | 0 |
|
||||
| service | shop/frontend | service | orders | calls | 1 | 1 |
|
||||
| service | shop/frontend | service | shop/backend | calls | 1 | 0 |
|
||||
| service.instance | shop/frontend,front-1 | service | shop/frontend | part_of | | |
|
||||
| service.instance | shop/frontend,front-1 | host | host-1 | runs_on | | |
|
||||
| gen_ai.agent | agent-a | gen_ai.model | model-a | uses | | |
|
||||
+------------------+-----------------------+--------------+---------------+----------+---------------+-------------+
|
||||
|
||||
-- Quotes, backslashes and dots remain a literal key; numbers render as strings.
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.custom.id' = 'scope_attributes.a"b\c.d';
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
select entity_type, entity_id
|
||||
from greptime_private.semantic_entities
|
||||
where entity_type = 'custom'
|
||||
and observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00';
|
||||
|
||||
+-------------+-----------+
|
||||
| entity_type | entity_id |
|
||||
+-------------+-----------+
|
||||
| custom | 7 |
|
||||
+-------------+-----------+
|
||||
|
||||
-- Path-looking suffixes are literal keys, not nested traversal or array access.
|
||||
-- These missing keys are valid declarations and produce no entities.
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.literal_missing.id' = 'resource_attributes.host..name';
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.literal_index.id' = 'resource_attributes.host[0]';
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
-- The root must exist and be one of the three JSON2 attribute columns.
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'span_events.name';
|
||||
|
||||
Error: 4002(TableColumnNotFound), Column span_events.name not exists in table graph_trace_v2
|
||||
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'resource_attributes.';
|
||||
|
||||
Error: 4002(TableColumnNotFound), Column resource_attributes. not exists in table graph_trace_v2
|
||||
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'missing.key';
|
||||
|
||||
Error: 4002(TableColumnNotFound), Column missing.key not exists in table graph_trace_v2
|
||||
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'resource_attributes';
|
||||
|
||||
Error: 1004(InvalidArguments), Invalid alter table(graph_trace_v2) request: entity column `resource_attributes` (option `greptime.semantic.entity.invalid.id`) has type `Json2"<Null>"`, which cannot render as a string
|
||||
|
||||
create table graph_bad_v2 (
|
||||
ts timestamp time index, resource_attributes string
|
||||
) with ('table_data_model' = 'greptime_trace_v2',
|
||||
'greptime.semantic.entity.host.id' = 'resource_attributes.host.id');
|
||||
|
||||
Error: 1004(InvalidArguments), Cannot find column by name: resource_attributes.host.id
|
||||
|
||||
-- Ordinary JSON2 tables do not acquire Trace V2 attribute-reference semantics.
|
||||
create table graph_not_trace (
|
||||
ts timestamp time index, resource_attributes json2
|
||||
) with ('append_mode' = 'true',
|
||||
'greptime.semantic.entity.host.id' = 'resource_attributes.host.id');
|
||||
|
||||
Error: 1004(InvalidArguments), Cannot find column by name: resource_attributes.host.id
|
||||
|
||||
-- Physical convention columns win over conflicting JSON2 keys.
|
||||
alter table graph_trace_v2 add column "resource_attributes.host.id" string;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
alter table graph_trace_v2 add column "resource_attributes.service.namespace" string;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
alter table graph_trace_v2 add column "span_attributes.db.namespace" string;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
-- Restore the implicit host declaration for the physical-column check.
|
||||
alter table graph_trace_v2 unset 'greptime.semantic.entity.host.id';
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
alter table graph_trace_v2 unset 'greptime.semantic.entity.host.descriptive';
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
insert into graph_trace_v2
|
||||
("timestamp", duration_nano, trace_id, span_id, span_kind, span_status_code,
|
||||
service_name, resource_attributes, span_attributes,
|
||||
"resource_attributes.host.id", "resource_attributes.service.namespace", "span_attributes.db.namespace")
|
||||
values ('2026-01-01 00:00:04', 100, 'physical', 'physical-client', 'SPAN_KIND_CLIENT', 'STATUS_CODE_UNSET',
|
||||
'physical-service', '{"host.id":"json-host","service.namespace":"json-namespace"}',
|
||||
'{"db.namespace":"json-database","db":{"namespace":"nested-database"}}',
|
||||
'physical-host', 'physical-namespace', 'physical-database');
|
||||
|
||||
Affected Rows: 1
|
||||
|
||||
select entity_type, entity_id
|
||||
from greptime_private.semantic_entities
|
||||
where entity_type in ('host', 'service')
|
||||
and observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
|
||||
order by entity_type, entity_id;
|
||||
|
||||
+-------------+-------------------------------------+
|
||||
| entity_type | entity_id |
|
||||
+-------------+-------------------------------------+
|
||||
| host | physical-host |
|
||||
| service | frontend |
|
||||
| service | no-resource |
|
||||
| service | physical-namespace/physical-service |
|
||||
| service | shop/backend |
|
||||
+-------------+-------------------------------------+
|
||||
|
||||
select src_id, dst_id, rel_type
|
||||
from greptime_private.semantic_relationships
|
||||
where src_id = 'physical-namespace/physical-service'
|
||||
and observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
|
||||
order by rel_type, dst_id;
|
||||
|
||||
+-------------------------------------+-------------------+----------+
|
||||
| src_id | dst_id | rel_type |
|
||||
+-------------------------------------+-------------------+----------+
|
||||
| physical-namespace/physical-service | physical-database | calls |
|
||||
+-------------------------------------+-------------------+----------+
|
||||
|
||||
drop table graph_trace_v2;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
drop table graph_trace_v1_peer;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
-- Trace V2 uses the same flattened attribute references as V1.
|
||||
-- Everything after the root column is one literal key, including dots.
|
||||
create table graph_trace_v2 (
|
||||
"timestamp" timestamp(9) time index,
|
||||
timestamp_end timestamp(9),
|
||||
duration_nano bigint,
|
||||
parent_span_id string,
|
||||
trace_id string,
|
||||
span_id string,
|
||||
span_kind string,
|
||||
span_name string,
|
||||
span_status_code string,
|
||||
span_status_message string,
|
||||
trace_state string,
|
||||
scope_name string,
|
||||
scope_version string,
|
||||
service_name string,
|
||||
span_attributes json2,
|
||||
scope_attributes json2,
|
||||
resource_attributes json2,
|
||||
span_events json,
|
||||
span_links json,
|
||||
primary key (service_name)
|
||||
) with (
|
||||
'table_data_model' = 'greptime_trace_v2',
|
||||
'append_mode' = 'true',
|
||||
'greptime.semantic.entity.gen_ai.agent.id' = 'span_attributes.gen_ai.agent.id',
|
||||
'greptime.semantic.entity.gen_ai.agent.scope' = 'scope_attributes.environment',
|
||||
'greptime.semantic.entity.gen_ai.model.id' = 'span_attributes.gen_ai.request.model',
|
||||
'greptime.semantic.entity.literal_peer.id' = 'span_attributes.db.namespace',
|
||||
'greptime.semantic.entity.literal_resource.id' = 'resource_attributes.host.name',
|
||||
'greptime.semantic.entity.literal_span.id' = 'span_attributes.gen_ai.agent.id',
|
||||
'greptime.semantic.entity.literal_scope.id' = 'scope_attributes.deployment.region.name'
|
||||
);
|
||||
|
||||
insert into graph_trace_v2
|
||||
("timestamp", duration_nano, trace_id, span_id, span_kind, span_status_code,
|
||||
service_name, span_attributes, scope_attributes, resource_attributes)
|
||||
values
|
||||
('2026-01-01 00:00:01', 100000000, 'paired', 'client', 'SPAN_KIND_CLIENT', 'STATUS_CODE_UNSET',
|
||||
'frontend', '{"gen_ai.agent.id":"agent-a","gen_ai.request.model":"model-a","gen_ai":{"agent":{"id":"nested-agent"}}}',
|
||||
'{"environment":"prod","deployment.region.name":"literal-region","deployment":{"region.name":"nested-region"},"a\"b\\c.d":7}', '{"service.namespace":"shop","service.instance.id":"front-1","host.id":"host-1","host.name":"node-1","host":{"name":"nested-node"},"service":{"namespace":"wrong-namespace"}}'),
|
||||
('2026-01-01 00:00:02', 200000000, 'virtual', 'client', 'SPAN_KIND_CLIENT', 'STATUS_CODE_ERROR',
|
||||
'frontend', '{"service.peer.name":"","db.namespace":"orders","db":{"namespace":"wrong-orders"}}', '{}',
|
||||
'{"service.namespace":"shop","service.instance.id":"front-1","host.id":"host-1","host.name":"node-1"}'),
|
||||
('2026-01-01 00:00:03', 100, 'missing', 'internal', 'SPAN_KIND_INTERNAL', 'STATUS_CODE_UNSET',
|
||||
'no-resource', '{"gen_ai.agent.id":null}', '{}', '{}');
|
||||
|
||||
-- Agent parent/child references are independent of span kind.
|
||||
insert into graph_trace_v2
|
||||
("timestamp", duration_nano, trace_id, span_id, parent_span_id, span_kind,
|
||||
span_status_code, service_name, span_attributes, scope_attributes, resource_attributes)
|
||||
values ('2026-01-01 00:00:01', 30000000, 'paired', 'agent-child', 'client',
|
||||
'SPAN_KIND_INTERNAL', 'STATUS_CODE_UNSET', 'frontend',
|
||||
'{"gen_ai.agent.id":"agent-b"}', '{"environment":"prod"}', '{"service.namespace":"shop"}');
|
||||
|
||||
-- Calls can pair a V2 client with a V1 server. The literal_resource declaration
|
||||
-- has exactly the same spelling in both models, including its output JSON keys.
|
||||
create table graph_trace_v1_peer (
|
||||
"timestamp" timestamp(9) time index,
|
||||
trace_id string, span_id string, parent_span_id string,
|
||||
span_kind string, span_status_code string, duration_nano bigint,
|
||||
service_name string,
|
||||
"resource_attributes.service.namespace" string,
|
||||
"resource_attributes.host.name" string,
|
||||
primary key (service_name)
|
||||
) with ('table_data_model' = 'greptime_trace_v1', 'append_mode' = 'true',
|
||||
'greptime.semantic.entity.literal_resource.id' = 'resource_attributes.host.name');
|
||||
|
||||
insert into graph_trace_v1_peer values
|
||||
('2026-01-01 00:00:01', 'paired', 'server', 'client', 'SPAN_KIND_SERVER', 'STATUS_CODE_UNSET', 50000000, 'backend', 'shop', 'node-v1');
|
||||
|
||||
select entity_type, entity_id, scope
|
||||
from greptime_private.semantic_entities
|
||||
where observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
|
||||
order by entity_type, entity_id;
|
||||
|
||||
select src_type, src_id, dst_type, dst_id, rel_type, request_count, error_count
|
||||
from greptime_private.semantic_relationships
|
||||
where observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
|
||||
order by rel_type, src_type, src_id, dst_id;
|
||||
|
||||
-- Empty components in the V1-compatible comma-separated list remain invalid.
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'resource_attributes.host.id,';
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'resource_attributes.host.id,,service_name';
|
||||
|
||||
-- ALTER accepts JSON2 references; a missing key simply yields no entity.
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.custom.id' = 'scope_attributes.missing';
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.host.descriptive' = 'resource_attributes.host.name';
|
||||
-- Preserve implicit host identity by explicitly declaring it as well.
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.host.id' = 'resource_attributes.host.id';
|
||||
|
||||
admin flush_table('graph_trace_v2');
|
||||
admin flush_table('graph_trace_v1_peer');
|
||||
|
||||
-- SQLNESS PROTOCOL MYSQL
|
||||
select entity_type, entity_id, scope, entity_id_attrs, descriptive
|
||||
from greptime_private.semantic_entities
|
||||
where observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
|
||||
order by entity_type, entity_id;
|
||||
|
||||
select src_type, src_id, dst_type, dst_id, rel_type, request_count, error_count
|
||||
from greptime_private.semantic_relationships
|
||||
where observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
|
||||
order by rel_type, src_type, src_id, dst_id;
|
||||
|
||||
-- Quotes, backslashes and dots remain a literal key; numbers render as strings.
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.custom.id' = 'scope_attributes.a"b\c.d';
|
||||
select entity_type, entity_id
|
||||
from greptime_private.semantic_entities
|
||||
where entity_type = 'custom'
|
||||
and observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00';
|
||||
|
||||
-- Path-looking suffixes are literal keys, not nested traversal or array access.
|
||||
-- These missing keys are valid declarations and produce no entities.
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.literal_missing.id' = 'resource_attributes.host..name';
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.literal_index.id' = 'resource_attributes.host[0]';
|
||||
|
||||
-- The root must exist and be one of the three JSON2 attribute columns.
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'span_events.name';
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'resource_attributes.';
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'missing.key';
|
||||
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'resource_attributes';
|
||||
|
||||
create table graph_bad_v2 (
|
||||
ts timestamp time index, resource_attributes string
|
||||
) with ('table_data_model' = 'greptime_trace_v2',
|
||||
'greptime.semantic.entity.host.id' = 'resource_attributes.host.id');
|
||||
|
||||
-- Ordinary JSON2 tables do not acquire Trace V2 attribute-reference semantics.
|
||||
create table graph_not_trace (
|
||||
ts timestamp time index, resource_attributes json2
|
||||
) with ('append_mode' = 'true',
|
||||
'greptime.semantic.entity.host.id' = 'resource_attributes.host.id');
|
||||
|
||||
-- Physical convention columns win over conflicting JSON2 keys.
|
||||
alter table graph_trace_v2 add column "resource_attributes.host.id" string;
|
||||
alter table graph_trace_v2 add column "resource_attributes.service.namespace" string;
|
||||
alter table graph_trace_v2 add column "span_attributes.db.namespace" string;
|
||||
-- Restore the implicit host declaration for the physical-column check.
|
||||
alter table graph_trace_v2 unset 'greptime.semantic.entity.host.id';
|
||||
alter table graph_trace_v2 unset 'greptime.semantic.entity.host.descriptive';
|
||||
insert into graph_trace_v2
|
||||
("timestamp", duration_nano, trace_id, span_id, span_kind, span_status_code,
|
||||
service_name, resource_attributes, span_attributes,
|
||||
"resource_attributes.host.id", "resource_attributes.service.namespace", "span_attributes.db.namespace")
|
||||
values ('2026-01-01 00:00:04', 100, 'physical', 'physical-client', 'SPAN_KIND_CLIENT', 'STATUS_CODE_UNSET',
|
||||
'physical-service', '{"host.id":"json-host","service.namespace":"json-namespace"}',
|
||||
'{"db.namespace":"json-database","db":{"namespace":"nested-database"}}',
|
||||
'physical-host', 'physical-namespace', 'physical-database');
|
||||
|
||||
select entity_type, entity_id
|
||||
from greptime_private.semantic_entities
|
||||
where entity_type in ('host', 'service')
|
||||
and observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
|
||||
order by entity_type, entity_id;
|
||||
|
||||
select src_id, dst_id, rel_type
|
||||
from greptime_private.semantic_relationships
|
||||
where src_id = 'physical-namespace/physical-service'
|
||||
and observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
|
||||
order by rel_type, dst_id;
|
||||
|
||||
drop table graph_trace_v2;
|
||||
drop table graph_trace_v1_peer;
|
||||
Reference in New Issue
Block a user