feat(trace): support Semantic Graph for Trace V2 (follow-up to #9192) (#9278)

* feat(trace): support Semantic Graph for Trace V2

Signed-off-by: luofucong <luofc@foxmail.com>

* fix: remove unused annotation context import

Signed-off-by: luofucong <luofc@foxmail.com>

---------

Signed-off-by: luofucong <luofc@foxmail.com>
This commit is contained in:
LFC
2026-09-22 02:39:36 +00:00
committed by GitHub
parent 4a980f350f
commit 16978cf6c2
13 changed files with 707 additions and 59 deletions
+6
View File
@@ -202,6 +202,12 @@ pub fn is_ddl_reserved_table(schema: &str, table: &str) -> bool {
pub const TRACE_ID_COLUMN: &str = "trace_id";
pub const SPAN_ID_COLUMN: &str = "span_id";
pub const SPAN_NAME_COLUMN: &str = "span_name";
/// Column containing span attributes as JSON/JSON2.
pub const SPAN_ATTRIBUTES_COLUMN: &str = "span_attributes";
/// Column containing instrumentation scope attributes as JSON2.
pub const SCOPE_ATTRIBUTES_COLUMN: &str = "scope_attributes";
/// Column containing resource attributes as JSON2.
pub const RESOURCE_ATTRIBUTES_COLUMN: &str = "resource_attributes";
pub const SERVICE_NAME_COLUMN: &str = "service_name";
pub const PARENT_SPAN_ID_COLUMN: &str = "parent_span_id";
// More fixed columns/values of the `greptime_trace_v1` data model, shared by
+84 -21
View File
@@ -23,8 +23,8 @@
//! the query engine. Injected into the catalog manager after the engine is built,
//! breaking the `catalog -> query` cycle.
use std::collections::{BTreeMap, HashMap};
use std::sync::Weak;
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::sync::{Arc, Weak};
use async_trait::async_trait;
use auth::{
@@ -41,13 +41,16 @@ use common_catalog::consts::{
};
use common_error::ext::{BoxedError, ErrorExt};
use common_error::status_code::StatusCode;
use common_function::scalars::json::json_get::JsonGetWithType;
use common_function::scalars::udf::create_udf;
use common_query::OutputData;
use common_query::prelude::OTLP_AGGREGATION_TEMPORALITY_LABEL;
use common_recordbatch::SendableRecordBatchStream;
use common_telemetry::{debug, warn};
use common_time::timestamp::TimeUnit;
use datafusion::common::{Column, ScalarValue};
use datafusion::dataframe::DataFrame;
use datafusion_expr::LogicalPlan;
use datafusion_expr::{Expr, LogicalPlan, lit};
use futures::TryStreamExt;
use operator::statement::semantic_graph::{
CallsSource, CoDeclaredSource, Conventions, DeclaredSource, ENTITY_TYPE_GEN_AI_AGENT,
@@ -64,8 +67,8 @@ use table::metadata::TableInfo;
use table::predicate::{TimeRangeExtraction, extract_time_range_strict};
use table::requests::{
EntityRole, SEMANTIC_METRIC_TYPE, SEMANTIC_SIGNAL_TYPE, SEMANTIC_SOURCE, SIGNAL_TYPE_METRIC,
SOURCE_OPENTELEMETRY, SOURCE_PROMETHEUS, is_trace_v1_table, parse_entity_columns,
parse_entity_option_key,
SOURCE_OPENTELEMETRY, SOURCE_PROMETHEUS, TABLE_DATA_MODEL_TRACE_V2, is_trace_table,
parse_entity_columns, parse_entity_option_key, trace_v2_attribute,
};
use crate::error;
@@ -164,12 +167,11 @@ impl EntityGraphProviderImpl {
|(entity_type, (id_columns, descriptive_columns, scope_columns))| {
// A stale declaration (e.g. its column was dropped later)
// must not poison every graph scan; skip it.
let schema = &table_info.meta.schema;
if let Some(missing) = id_columns
.iter()
.chain(&descriptive_columns)
.chain(&scope_columns)
.find(|c| schema.column_schema_by_name(c).is_none())
.find(|c| !Self::can_resolve_entity_column(table_info, c))
{
warn!(
"Skipping entity declaration `{}` of table `{}`: column `{}` not found",
@@ -194,7 +196,7 @@ impl EntityGraphProviderImpl {
}
/// All entity declarations of one table: the explicit options plus the
/// zero-configuration conventions (`otlp_trace_entities` for trace-v1
/// zero-configuration conventions (`otlp_trace_entities` for trace-v1/v2
/// tables — including the `service` identity of tables created before the
/// ingest-side auto-stamp — and the Prometheus/OTel descriptor
/// whitelists). An explicit declaration of a type always suppresses the
@@ -207,7 +209,7 @@ impl EntityGraphProviderImpl {
) -> Vec<EntityDeclaration> {
let mut declarations = Self::parse_declarations(table_info);
let mut supersessions = Vec::new();
if is_trace_v1_table(table_info) {
if is_trace_table(table_info) {
Self::extend_with_implicit_entities(
table_info,
&conventions.otlp_trace_entities,
@@ -303,8 +305,8 @@ impl EntityGraphProviderImpl {
}
/// Synthesizes the applicable subset of `entities` on `table_info`:
/// explicit declarations win, every id column must exist (no guessing),
/// descriptive columns are filtered to those present.
/// explicit declarations win, every id reference must be resolvable,
/// and descriptive references are filtered by the same rule.
fn extend_with_implicit_entities(
table_info: &TableInfo,
entities: &[ImplicitEntity],
@@ -330,7 +332,7 @@ impl EntityGraphProviderImpl {
if let Some(missing) = implicit
.id
.iter()
.find(|c| schema.column_schema_by_name(c).is_none())
.find(|c| !Self::can_resolve_entity_column(table_info, c))
{
debug!(
"Table `{}` lacks the id column `{}`; skipping the implicit `{}` declaration",
@@ -350,7 +352,7 @@ impl EntityGraphProviderImpl {
implicit
.descriptive
.iter()
.filter(|c| schema.column_schema_by_name(c).is_some())
.filter(|c| Self::can_resolve_entity_column(table_info, c))
.cloned()
.collect()
};
@@ -359,7 +361,7 @@ impl EntityGraphProviderImpl {
let id_qualifier = implicit
.qualified_by
.clone()
.filter(|c| schema.column_schema_by_name(c).is_some());
.filter(|c| Self::can_resolve_entity_column(table_info, c));
if let Some(entity_type) = &implicit.superseded_by {
supersessions.push((declarations.len(), entity_type.clone()));
}
@@ -434,7 +436,7 @@ impl EntityGraphProviderImpl {
while let Some(table) = tables.try_next().await.map_err(BoxedError::new)? {
let table_info = table.table_info();
let table_declarations = Self::declarations_for(&table_info, conventions);
let is_trace = is_trace_v1_table(&table_info);
let is_trace = is_trace_table(&table_info);
// Authorize only tables that would contribute rows.
if per_table_auth
&& (is_trace || !table_declarations.is_empty())
@@ -532,8 +534,63 @@ impl EntityGraphProviderImpl {
}
}
fn read_table(&self, table: TableRef) -> Result<DataFrame, BoxedError> {
self.query_engine.read_table(table).map_err(BoxedError::new)
fn can_resolve_entity_column(table_info: &TableInfo, column: &str) -> bool {
let table_schema = &table_info.meta.schema;
if table_schema.column_schema_by_name(column).is_some() {
return true;
}
let data_model = table_info.meta.options.data_model();
trace_v2_attribute(table_schema, data_model, column).is_some()
}
/// Projects referenced V2 attributes under their declaration names for
/// the shared entity and relationship plans. Other models scan unchanged.
fn read_table(
&self,
table: TableRef,
declarations: &[EntityDeclaration],
) -> Result<DataFrame, BoxedError> {
let info = table.table_info();
let mut scan = self
.query_engine
.read_table(table)
.map_err(BoxedError::new)?;
let data_model = info.meta.options.data_model();
if data_model != Some(TABLE_DATA_MODEL_TRACE_V2) {
return Ok(scan);
}
// Read Trace V2 table:
let mut columns = BTreeSet::new();
for declaration in declarations {
columns.extend(declaration.id_columns.iter());
columns.extend(declaration.descriptive_columns.iter());
columns.extend(declaration.scope_columns.iter());
columns.extend(declaration.id_qualifier.iter());
columns.extend(declaration.superseded_by_columns.iter());
}
let conventions = conventions()
.map_err(datafusion::error::DataFusionError::Internal)
.context(error::DataFusionSnafu)
.map_err(BoxedError::new)?;
let virtual_candidates = &conventions.virtual_dst_candidates;
columns.extend(virtual_candidates.iter().map(|candidate| &candidate.column));
let get = create_udf(Arc::new(JsonGetWithType::default()));
for column in columns {
if let Some((root, key)) = trace_v2_attribute(&info.meta.schema, data_model, column) {
let path = format!("$.{}", serde_json::Value::String(key.to_string()));
let value = get.call(vec![
Expr::Column(Column::from_name(root)),
lit(path),
lit(ScalarValue::Utf8View(None)),
]);
scan = scan
.with_column(column, value)
.context(error::DataFusionSnafu)
.map_err(BoxedError::new)?;
}
}
Ok(scan)
}
/// The declared-edge branch source, when the physical table exists, the
@@ -587,7 +644,7 @@ impl EntityGraphProviderImpl {
));
}
Ok(Some(DeclaredSource {
scan: self.read_table(table)?,
scan: self.read_table(table, &[])?,
}))
}
@@ -633,8 +690,8 @@ impl EntityGraphProvider for EntityGraphProviderImpl {
let mut plans = Vec::with_capacity(sources.len());
for source in sources {
plans.push(RegistrySource {
scan: self.read_table(source.table, &source.declarations)?,
declarations: source.declarations,
scan: self.read_table(source.table)?,
});
}
let Some(window) = Self::query_window(&request)? else {
@@ -658,18 +715,24 @@ impl EntityGraphProvider for EntityGraphProviderImpl {
let (sources, traces) = self.enumerate(catalog, query_ctx.as_deref()).await?;
let mut calls = Vec::with_capacity(traces.len());
for trace in traces {
let declarations = trace
.service
.iter()
.chain(trace.agent.iter())
.cloned()
.collect::<Vec<_>>();
calls.push(CallsSource {
scan: self.read_table(trace.table, &declarations)?,
service: trace.service,
agent: trace.agent,
scan: self.read_table(trace.table)?,
});
}
let mut co_declared = Vec::with_capacity(sources.len());
for source in sources {
co_declared.push(CoDeclaredSource {
scan: self.read_table(source.table, &source.declarations)?,
declarations: source.declarations,
is_trace: source.is_trace,
scan: self.read_table(source.table)?,
});
}
let declared = self.declared_source(catalog, query_ctx.as_deref()).await?;
+1 -6
View File
@@ -370,12 +370,7 @@ async fn query_trace_table(
})?;
let table_info = table.table_info();
let data_model = table_info
.meta
.options
.extra_options
.get(table::requests::TABLE_DATA_MODEL)
.map(String::as_str);
let data_model = table_info.meta.options.data_model();
// collect to set
let col_names = table_info
+1 -1
View File
@@ -1509,7 +1509,7 @@ pub fn validate_trace_table_model(table_info: &TableInfo, ctx: &QueryContextRef)
else {
return Ok(());
};
if let Some(actual) = table_info.meta.options.extra_options.get(TABLE_DATA_MODEL) {
if let Some(actual) = table_info.meta.options.data_model() {
ensure!(
actual == expected,
InvalidInsertRequestSnafu {
+6 -8
View File
@@ -100,8 +100,8 @@ use table::TableRef;
use table::dist_table::DistTable;
use table::metadata::{self, TableId, TableInfo, TableMeta, TableType};
use table::requests::{
AlterKind, AlterTableRequest, AnnotationContext, COMMENT_KEY, DDL_TIMEOUT, DDL_WAIT,
INGEST_ROWS_RATE_LIMIT_KEY, TableOptions, validate_and_normalize_annotation_options,
AlterKind, AlterTableRequest, COMMENT_KEY, DDL_TIMEOUT, DDL_WAIT, INGEST_ROWS_RATE_LIMIT_KEY,
TableOptions, validate_and_normalize_annotation_options,
};
use table::table_name::TableName;
use table::table_reference::TableReference;
@@ -2657,11 +2657,7 @@ fn validate_and_normalize_annotations(
partition_key_indices: &[usize],
) -> Result<()> {
use table::requests::AnnotationValidationError as CheckError;
let cx = AnnotationContext {
schema,
partition_key_indices,
};
validate_and_normalize_annotation_options(options, &cx).map_err(|e| match e {
let handle_error = |e: CheckError| match e {
CheckError::ColumnNotFound { column } => ColumnNotFoundSnafu { msg: column }.build(),
e @ (CheckError::UnknownKey { .. }
| CheckError::InvalidValue { .. }
@@ -2676,7 +2672,9 @@ fn validate_and_normalize_annotations(
reason: e.to_string(),
}
.build(),
})
};
validate_and_normalize_annotation_options(options, schema, partition_key_indices)
.map_err(handle_error)
}
fn find_partition_columns(partitions: &Option<Partitions>) -> Result<Vec<String>> {
@@ -44,7 +44,7 @@ pub struct VirtualDstCandidate {
}
/// One implicit entity declaration: of a whitelisted Prometheus or OTel info
/// metric, or of a trace-v1 table's flattened resource attributes.
/// metric, or of a Trace V1/V2 table's resource attributes.
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ImplicitEntity {
@@ -37,7 +37,7 @@ virtual_dst_candidates:
- { column: span_attributes.db.name, connection_type: database }
- { column: span_attributes.server.address, connection_type: virtual_node }
# Implicit declarations for greptime_trace_v1 tables; the pod UID bridges
# Implicit declarations for Trace V1/V2 tables; the pod UID bridges
# trace-side pods onto the kube-state-metrics entities below. host.name and
# container.name vary by SDK and resource detector, so identity is the id.
# qualified_by folds the namespace into the service identity the same way the
+2 -5
View File
@@ -28,7 +28,8 @@ pub use common_catalog::consts::{
SPAN_STATUS_CODE_COLUMN as SPAN_STATUS_CODE, TRACE_TIMESTAMP_COLUMN as TIMESTAMP_COLUMN,
};
pub use common_catalog::consts::{
PARENT_SPAN_ID_COLUMN, SPAN_ID_COLUMN, SPAN_NAME_COLUMN, TRACE_ID_COLUMN,
PARENT_SPAN_ID_COLUMN, RESOURCE_ATTRIBUTES_COLUMN, SCOPE_ATTRIBUTES_COLUMN,
SPAN_ATTRIBUTES_COLUMN, SPAN_ID_COLUMN, SPAN_NAME_COLUMN, TRACE_ID_COLUMN,
};
use pipeline::PipelineWay;
@@ -37,14 +38,10 @@ use crate::otlp::trace::span::TraceSpan;
pub const TIMESTAMP_END_COLUMN: &str = "timestamp_end";
pub const SPAN_STATUS_MESSAGE_COLUMN: &str = "span_status_message";
pub const SPAN_ATTRIBUTES_COLUMN: &str = "span_attributes";
pub const SPAN_EVENTS_COLUMN: &str = "span_events";
pub const SPAN_LINKS_COLUMN: &str = "span_links";
pub const SCOPE_NAME_COLUMN: &str = "scope_name";
pub const SCOPE_VERSION_COLUMN: &str = "scope_version";
/// Column containing instrumentation scope attributes as a JSON2 object.
pub const SCOPE_ATTRIBUTES_COLUMN: &str = "scope_attributes";
pub const RESOURCE_ATTRIBUTES_COLUMN: &str = "resource_attributes";
pub const TRACE_STATE_COLUMN: &str = "trace_state";
// const keys
+1
View File
@@ -470,6 +470,7 @@ impl TableMeta {
.build()
})?;
let cx = AnnotationContext {
data_model: self.options.data_model(),
schema: &self.schema,
partition_key_indices: &self.partition_key_indices,
};
+25 -13
View File
@@ -63,17 +63,14 @@ pub const TABLE_DATA_MODEL_TRACE_V1: &str = "greptime_trace_v1";
/// Table data model used by the JSON2-based OTLP trace pipeline.
pub const TABLE_DATA_MODEL_TRACE_V2: &str = "greptime_trace_v2";
/// Returns true if the table stores spans in the `greptime_trace_v1` data model
/// (fixed span columns), the shape the Jaeger query path and the entity-graph
/// derivation rely on.
pub fn is_trace_v1_table(table_info: &crate::metadata::TableInfo) -> bool {
table_info
.meta
.options
.extra_options
.get(TABLE_DATA_MODEL)
.map(|v| v == TABLE_DATA_MODEL_TRACE_V1)
.unwrap_or(false)
/// Returns true for the Trace V1 and V2 data models supported
/// by semantic graph derivation.
pub fn is_trace_table(table_info: &crate::metadata::TableInfo) -> bool {
let table_data_model = table_info.meta.options.data_model();
matches!(
table_data_model,
Some(TABLE_DATA_MODEL_TRACE_V1 | TABLE_DATA_MODEL_TRACE_V2)
)
}
pub const OTLP_METRIC_COMPAT_KEY: &str = "otlp_metric_compat";
@@ -226,6 +223,11 @@ pub const REPARTITION_COLUMN_HINT_KEY: &str = "repartition.column.hint";
pub const REPARTITION_PARTITION_NUM_HINT_KEY: &str = "repartition.partition.num.hint";
impl TableOptions {
/// Returns the table data model, if specified.
pub fn data_model(&self) -> Option<&str> {
self.extra_options.get(TABLE_DATA_MODEL).map(String::as_str)
}
pub fn try_from_iter<T: ToString, U: IntoIterator<Item = (T, T)>>(
iter: U,
) -> Result<TableOptions> {
@@ -480,6 +482,7 @@ pub fn validate_annotation_keys<'a>(
/// Table shape an annotation option is validated against.
pub struct AnnotationContext<'a> {
pub data_model: Option<&'a str>,
pub schema: &'a Schema,
pub partition_key_indices: &'a [usize],
}
@@ -570,6 +573,9 @@ pub(crate) fn validate_and_normalize_annotation(
}
if parse_entity_option_key(key).is_some() {
for column in parse_entity_columns(value) {
if trace_v2_attribute(cx.schema, cx.data_model, &column).is_some() {
continue;
}
let schema = cx.schema.column_schema_by_name(&column).ok_or_else(|| {
AnnotationValidationError::ColumnNotFound {
column: column.clone(),
@@ -620,14 +626,20 @@ pub(crate) fn validate_and_normalize_annotation(
/// and writes normalized values back in place.
pub fn validate_and_normalize_annotation_options(
options: &mut TableOptions,
cx: &AnnotationContext<'_>,
schema: &Schema,
partition_key_indices: &[usize],
) -> std::result::Result<(), AnnotationValidationError> {
let cx = AnnotationContext {
data_model: options.data_model(),
schema,
partition_key_indices,
};
let mut normalized = Vec::new();
for (key, value) in &options.extra_options {
let Some(family) = AnnotationFamily::of_key(key) else {
continue;
};
let checked = validate_and_normalize_annotation(family, cx, key, value)?;
let checked = validate_and_normalize_annotation(family, &cx, key, value)?;
if checked != *value {
normalized.push((key.clone(), checked));
}
+85 -3
View File
@@ -30,7 +30,13 @@
//! [`crate::requests::validate_table_option`], so they are accepted both on the
//! ingestion auto-create path and on explicit `CREATE TABLE ... WITH (...)` DDL.
use common_catalog::consts::{
RESOURCE_ATTRIBUTES_COLUMN, SCOPE_ATTRIBUTES_COLUMN, SPAN_ATTRIBUTES_COLUMN,
};
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::{ColumnSchema, Schema};
use crate::requests::TABLE_DATA_MODEL_TRACE_V2;
/// Reserved prefix for every public semantic table-option key.
pub const SEMANTIC_PREFIX: &str = "greptime.semantic.";
@@ -193,6 +199,34 @@ pub fn is_entity_option_key(key: &str) -> bool {
parse_entity_option_key(key).is_some()
}
/// Resolves a Trace V2 entity reference to a JSON2 root and a literal attribute key.
/// Everything after the first dot belongs to the key, matching Trace V1's
/// flattened column names. Existing physical columns take precedence.
pub fn trace_v2_attribute<'a>(
schema: &Schema,
data_model: Option<&str>,
column: &'a str,
) -> Option<(&'a str, &'a str)> {
if data_model != Some(TABLE_DATA_MODEL_TRACE_V2)
|| schema.column_schema_by_name(column).is_some()
{
return None;
}
let (root, key) = column.split_once('.')?;
(!key.is_empty()
&& schema
.column_schema_by_name(root)
.is_some_and(is_v2_attribute_column))
.then_some((root, key))
}
fn is_v2_attribute_column(schema: &ColumnSchema) -> bool {
matches!(
schema.name.as_str(),
RESOURCE_ATTRIBUTES_COLUMN | SCOPE_ATTRIBUTES_COLUMN | SPAN_ATTRIBUTES_COLUMN
) && schema.data_type.is_json2()
}
/// Returns true if a column of `data_type` renders as a stable string — the
/// requirement for entity id/descriptive/scope columns. The read-time
/// derivation casts them to strings, so a type without a stable string form
@@ -217,8 +251,9 @@ pub fn has_stable_string_form(data_type: &ConcreteDataType) -> bool {
pub fn parse_entity_columns(value: &str) -> Vec<String> {
value
.split(',')
.map(|c| c.trim().to_string())
.filter(|c| !c.is_empty())
.map(str::trim)
.filter(|column| !column.is_empty())
.map(String::from)
.collect()
}
@@ -244,7 +279,7 @@ pub fn is_semantic_option_key(key: &str) -> bool {
/// are rejected.
pub fn validate_semantic_option(key: &str, value: &str) -> bool {
if is_entity_option_key(key) {
return !value.is_empty() && value.split(',').all(|col| !col.trim().is_empty());
return !value.is_empty() && value.split(',').all(|column| !column.trim().is_empty());
}
match key {
SEMANTIC_PIPELINE
@@ -293,8 +328,55 @@ pub fn validate_semantic_option(key: &str, value: &str) -> bool {
#[cfg(test)]
mod tests {
use datatypes::types::JsonType;
use super::*;
#[test]
fn test_trace_v2_attribute_references() {
let schema = Schema::new(vec![
ColumnSchema::new(
RESOURCE_ATTRIBUTES_COLUMN,
ConcreteDataType::Json(JsonType::null()),
true,
),
ColumnSchema::new(
SCOPE_ATTRIBUTES_COLUMN,
ConcreteDataType::string_datatype(),
true,
),
ColumnSchema::new(
"resource_attributes.physical",
ConcreteDataType::string_datatype(),
true,
),
]);
let model = Some(TABLE_DATA_MODEL_TRACE_V2);
for key in ["host.name", "host..name", "host[0]", "host.*", r#"a"b\c.d"#] {
let column = format!("resource_attributes.{key}");
assert_eq!(
trace_v2_attribute(&schema, model, &column),
Some((RESOURCE_ATTRIBUTES_COLUMN, key))
);
}
for column in [
"resource_attributes",
"resource_attributes.",
"scope_attributes.name",
"span_attributes.name",
"resource_attributes.physical",
"missing.key",
] {
assert_eq!(trace_v2_attribute(&schema, model, column), None, "{column}");
}
for model in [None, Some(crate::requests::TABLE_DATA_MODEL_TRACE_V1)] {
assert_eq!(
trace_v2_attribute(&schema, model, "resource_attributes.host.name"),
None
);
}
}
#[test]
fn test_is_semantic_option_key() {
assert!(is_semantic_option_key(SEMANTIC_SIGNAL_TYPE));
@@ -0,0 +1,329 @@
-- Trace V2 uses the same flattened attribute references as V1.
-- Everything after the root column is one literal key, including dots.
create table graph_trace_v2 (
"timestamp" timestamp(9) time index,
timestamp_end timestamp(9),
duration_nano bigint,
parent_span_id string,
trace_id string,
span_id string,
span_kind string,
span_name string,
span_status_code string,
span_status_message string,
trace_state string,
scope_name string,
scope_version string,
service_name string,
span_attributes json2,
scope_attributes json2,
resource_attributes json2,
span_events json,
span_links json,
primary key (service_name)
) with (
'table_data_model' = 'greptime_trace_v2',
'append_mode' = 'true',
'greptime.semantic.entity.gen_ai.agent.id' = 'span_attributes.gen_ai.agent.id',
'greptime.semantic.entity.gen_ai.agent.scope' = 'scope_attributes.environment',
'greptime.semantic.entity.gen_ai.model.id' = 'span_attributes.gen_ai.request.model',
'greptime.semantic.entity.literal_peer.id' = 'span_attributes.db.namespace',
'greptime.semantic.entity.literal_resource.id' = 'resource_attributes.host.name',
'greptime.semantic.entity.literal_span.id' = 'span_attributes.gen_ai.agent.id',
'greptime.semantic.entity.literal_scope.id' = 'scope_attributes.deployment.region.name'
);
Affected Rows: 0
insert into graph_trace_v2
("timestamp", duration_nano, trace_id, span_id, span_kind, span_status_code,
service_name, span_attributes, scope_attributes, resource_attributes)
values
('2026-01-01 00:00:01', 100000000, 'paired', 'client', 'SPAN_KIND_CLIENT', 'STATUS_CODE_UNSET',
'frontend', '{"gen_ai.agent.id":"agent-a","gen_ai.request.model":"model-a","gen_ai":{"agent":{"id":"nested-agent"}}}',
'{"environment":"prod","deployment.region.name":"literal-region","deployment":{"region.name":"nested-region"},"a\"b\\c.d":7}', '{"service.namespace":"shop","service.instance.id":"front-1","host.id":"host-1","host.name":"node-1","host":{"name":"nested-node"},"service":{"namespace":"wrong-namespace"}}'),
('2026-01-01 00:00:02', 200000000, 'virtual', 'client', 'SPAN_KIND_CLIENT', 'STATUS_CODE_ERROR',
'frontend', '{"service.peer.name":"","db.namespace":"orders","db":{"namespace":"wrong-orders"}}', '{}',
'{"service.namespace":"shop","service.instance.id":"front-1","host.id":"host-1","host.name":"node-1"}'),
('2026-01-01 00:00:03', 100, 'missing', 'internal', 'SPAN_KIND_INTERNAL', 'STATUS_CODE_UNSET',
'no-resource', '{"gen_ai.agent.id":null}', '{}', '{}');
Affected Rows: 3
-- Agent parent/child references are independent of span kind.
insert into graph_trace_v2
("timestamp", duration_nano, trace_id, span_id, parent_span_id, span_kind,
span_status_code, service_name, span_attributes, scope_attributes, resource_attributes)
values ('2026-01-01 00:00:01', 30000000, 'paired', 'agent-child', 'client',
'SPAN_KIND_INTERNAL', 'STATUS_CODE_UNSET', 'frontend',
'{"gen_ai.agent.id":"agent-b"}', '{"environment":"prod"}', '{"service.namespace":"shop"}');
Affected Rows: 1
-- Calls can pair a V2 client with a V1 server. The literal_resource declaration
-- has exactly the same spelling in both models, including its output JSON keys.
create table graph_trace_v1_peer (
"timestamp" timestamp(9) time index,
trace_id string, span_id string, parent_span_id string,
span_kind string, span_status_code string, duration_nano bigint,
service_name string,
"resource_attributes.service.namespace" string,
"resource_attributes.host.name" string,
primary key (service_name)
) with ('table_data_model' = 'greptime_trace_v1', 'append_mode' = 'true',
'greptime.semantic.entity.literal_resource.id' = 'resource_attributes.host.name');
Affected Rows: 0
insert into graph_trace_v1_peer values
('2026-01-01 00:00:01', 'paired', 'server', 'client', 'SPAN_KIND_SERVER', 'STATUS_CODE_UNSET', 50000000, 'backend', 'shop', 'node-v1');
Affected Rows: 1
select entity_type, entity_id, scope
from greptime_private.semantic_entities
where observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
order by entity_type, entity_id;
+------------------+-----------------------+-------+
| entity_type | entity_id | scope |
+------------------+-----------------------+-------+
| gen_ai.agent | agent-a | prod |
| gen_ai.agent | agent-b | prod |
| gen_ai.model | model-a | |
| host | host-1 | |
| literal_peer | orders | |
| literal_resource | node-1 | |
| literal_resource | node-v1 | |
| literal_scope | literal-region | |
| literal_span | agent-a | |
| literal_span | agent-b | |
| service | no-resource | |
| service | shop/backend | |
| service | shop/frontend | |
| service.instance | shop/frontend,front-1 | |
+------------------+-----------------------+-------+
select src_type, src_id, dst_type, dst_id, rel_type, request_count, error_count
from greptime_private.semantic_relationships
where observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
order by rel_type, src_type, src_id, dst_id;
+------------------+-----------------------+--------------+---------------+----------+---------------+-------------+
| src_type | src_id | dst_type | dst_id | rel_type | request_count | error_count |
+------------------+-----------------------+--------------+---------------+----------+---------------+-------------+
| gen_ai.agent | agent-a | gen_ai.agent | agent-b | calls | 1 | 0 |
| service | shop/frontend | service | orders | calls | 1 | 1 |
| service | shop/frontend | service | shop/backend | calls | 1 | 0 |
| service.instance | shop/frontend,front-1 | service | shop/frontend | part_of | | |
| service.instance | shop/frontend,front-1 | host | host-1 | runs_on | | |
| gen_ai.agent | agent-a | gen_ai.model | model-a | uses | | |
+------------------+-----------------------+--------------+---------------+----------+---------------+-------------+
-- Empty components in the V1-compatible comma-separated list remain invalid.
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'resource_attributes.host.id,';
Error: 1004(InvalidArguments), Invalid alter table(graph_trace_v2) request: invalid value `resource_attributes.host.id,` for semantic option `greptime.semantic.entity.invalid.id`
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'resource_attributes.host.id,,service_name';
Error: 1004(InvalidArguments), Invalid alter table(graph_trace_v2) request: invalid value `resource_attributes.host.id,,service_name` for semantic option `greptime.semantic.entity.invalid.id`
-- ALTER accepts JSON2 references; a missing key simply yields no entity.
alter table graph_trace_v2 set 'greptime.semantic.entity.custom.id' = 'scope_attributes.missing';
Affected Rows: 0
alter table graph_trace_v2 set 'greptime.semantic.entity.host.descriptive' = 'resource_attributes.host.name';
Affected Rows: 0
-- Preserve implicit host identity by explicitly declaring it as well.
alter table graph_trace_v2 set 'greptime.semantic.entity.host.id' = 'resource_attributes.host.id';
Affected Rows: 0
admin flush_table('graph_trace_v2');
+-------------------------------------+
| ADMIN flush_table('graph_trace_v2') |
+-------------------------------------+
| 0 |
+-------------------------------------+
admin flush_table('graph_trace_v1_peer');
+------------------------------------------+
| ADMIN flush_table('graph_trace_v1_peer') |
+------------------------------------------+
| 0 |
+------------------------------------------+
-- SQLNESS PROTOCOL MYSQL
select entity_type, entity_id, scope, entity_id_attrs, descriptive
from greptime_private.semantic_entities
where observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
order by entity_type, entity_id;
+------------------+-----------------------+-------+--------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------+
| entity_type | entity_id | scope | entity_id_attrs | descriptive |
+------------------+-----------------------+-------+--------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------+
| gen_ai.agent | agent-a | prod | {"span_attributes.gen_ai.agent.id":"agent-a"} | |
| gen_ai.agent | agent-b | prod | {"span_attributes.gen_ai.agent.id":"agent-b"} | |
| gen_ai.model | model-a | | {"span_attributes.gen_ai.request.model":"model-a"} | |
| host | host-1 | | {"resource_attributes.host.id":"host-1"} | {"resource_attributes.host.name":"node-1"} |
| literal_peer | orders | | {"span_attributes.db.namespace":"orders"} | |
| literal_resource | node-1 | | {"resource_attributes.host.name":"node-1"} | |
| literal_resource | node-v1 | | {"resource_attributes.host.name":"node-v1"} | |
| literal_scope | literal-region | | {"scope_attributes.deployment.region.name":"literal-region"} | |
| literal_span | agent-a | | {"span_attributes.gen_ai.agent.id":"agent-a"} | |
| literal_span | agent-b | | {"span_attributes.gen_ai.agent.id":"agent-b"} | |
| service | no-resource | | {"resource_attributes.service.namespace":"","service_name":"no-resource"} | |
| service | shop/backend | | {"resource_attributes.service.namespace":"shop","service_name":"backend"} | |
| service | shop/frontend | | {"resource_attributes.service.namespace":"shop","service_name":"frontend"} | |
| service.instance | shop/frontend,front-1 | | {"resource_attributes.service.instance.id":"front-1","resource_attributes.service.namespace":"shop","service_name":"frontend"} | |
+------------------+-----------------------+-------+--------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------+
select src_type, src_id, dst_type, dst_id, rel_type, request_count, error_count
from greptime_private.semantic_relationships
where observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
order by rel_type, src_type, src_id, dst_id;
+------------------+-----------------------+--------------+---------------+----------+---------------+-------------+
| src_type | src_id | dst_type | dst_id | rel_type | request_count | error_count |
+------------------+-----------------------+--------------+---------------+----------+---------------+-------------+
| gen_ai.agent | agent-a | gen_ai.agent | agent-b | calls | 1 | 0 |
| service | shop/frontend | service | orders | calls | 1 | 1 |
| service | shop/frontend | service | shop/backend | calls | 1 | 0 |
| service.instance | shop/frontend,front-1 | service | shop/frontend | part_of | | |
| service.instance | shop/frontend,front-1 | host | host-1 | runs_on | | |
| gen_ai.agent | agent-a | gen_ai.model | model-a | uses | | |
+------------------+-----------------------+--------------+---------------+----------+---------------+-------------+
-- Quotes, backslashes and dots remain a literal key; numbers render as strings.
alter table graph_trace_v2 set 'greptime.semantic.entity.custom.id' = 'scope_attributes.a"b\c.d';
Affected Rows: 0
select entity_type, entity_id
from greptime_private.semantic_entities
where entity_type = 'custom'
and observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00';
+-------------+-----------+
| entity_type | entity_id |
+-------------+-----------+
| custom | 7 |
+-------------+-----------+
-- Path-looking suffixes are literal keys, not nested traversal or array access.
-- These missing keys are valid declarations and produce no entities.
alter table graph_trace_v2 set 'greptime.semantic.entity.literal_missing.id' = 'resource_attributes.host..name';
Affected Rows: 0
alter table graph_trace_v2 set 'greptime.semantic.entity.literal_index.id' = 'resource_attributes.host[0]';
Affected Rows: 0
-- The root must exist and be one of the three JSON2 attribute columns.
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'span_events.name';
Error: 4002(TableColumnNotFound), Column span_events.name not exists in table graph_trace_v2
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'resource_attributes.';
Error: 4002(TableColumnNotFound), Column resource_attributes. not exists in table graph_trace_v2
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'missing.key';
Error: 4002(TableColumnNotFound), Column missing.key not exists in table graph_trace_v2
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'resource_attributes';
Error: 1004(InvalidArguments), Invalid alter table(graph_trace_v2) request: entity column `resource_attributes` (option `greptime.semantic.entity.invalid.id`) has type `Json2"<Null>"`, which cannot render as a string
create table graph_bad_v2 (
ts timestamp time index, resource_attributes string
) with ('table_data_model' = 'greptime_trace_v2',
'greptime.semantic.entity.host.id' = 'resource_attributes.host.id');
Error: 1004(InvalidArguments), Cannot find column by name: resource_attributes.host.id
-- Ordinary JSON2 tables do not acquire Trace V2 attribute-reference semantics.
create table graph_not_trace (
ts timestamp time index, resource_attributes json2
) with ('append_mode' = 'true',
'greptime.semantic.entity.host.id' = 'resource_attributes.host.id');
Error: 1004(InvalidArguments), Cannot find column by name: resource_attributes.host.id
-- Physical convention columns win over conflicting JSON2 keys.
alter table graph_trace_v2 add column "resource_attributes.host.id" string;
Affected Rows: 0
alter table graph_trace_v2 add column "resource_attributes.service.namespace" string;
Affected Rows: 0
alter table graph_trace_v2 add column "span_attributes.db.namespace" string;
Affected Rows: 0
-- Restore the implicit host declaration for the physical-column check.
alter table graph_trace_v2 unset 'greptime.semantic.entity.host.id';
Affected Rows: 0
alter table graph_trace_v2 unset 'greptime.semantic.entity.host.descriptive';
Affected Rows: 0
insert into graph_trace_v2
("timestamp", duration_nano, trace_id, span_id, span_kind, span_status_code,
service_name, resource_attributes, span_attributes,
"resource_attributes.host.id", "resource_attributes.service.namespace", "span_attributes.db.namespace")
values ('2026-01-01 00:00:04', 100, 'physical', 'physical-client', 'SPAN_KIND_CLIENT', 'STATUS_CODE_UNSET',
'physical-service', '{"host.id":"json-host","service.namespace":"json-namespace"}',
'{"db.namespace":"json-database","db":{"namespace":"nested-database"}}',
'physical-host', 'physical-namespace', 'physical-database');
Affected Rows: 1
select entity_type, entity_id
from greptime_private.semantic_entities
where entity_type in ('host', 'service')
and observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
order by entity_type, entity_id;
+-------------+-------------------------------------+
| entity_type | entity_id |
+-------------+-------------------------------------+
| host | physical-host |
| service | frontend |
| service | no-resource |
| service | physical-namespace/physical-service |
| service | shop/backend |
+-------------+-------------------------------------+
select src_id, dst_id, rel_type
from greptime_private.semantic_relationships
where src_id = 'physical-namespace/physical-service'
and observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
order by rel_type, dst_id;
+-------------------------------------+-------------------+----------+
| src_id | dst_id | rel_type |
+-------------------------------------+-------------------+----------+
| physical-namespace/physical-service | physical-database | calls |
+-------------------------------------+-------------------+----------+
drop table graph_trace_v2;
Affected Rows: 0
drop table graph_trace_v1_peer;
Affected Rows: 0
@@ -0,0 +1,165 @@
-- Trace V2 uses the same flattened attribute references as V1.
-- Everything after the root column is one literal key, including dots.
create table graph_trace_v2 (
"timestamp" timestamp(9) time index,
timestamp_end timestamp(9),
duration_nano bigint,
parent_span_id string,
trace_id string,
span_id string,
span_kind string,
span_name string,
span_status_code string,
span_status_message string,
trace_state string,
scope_name string,
scope_version string,
service_name string,
span_attributes json2,
scope_attributes json2,
resource_attributes json2,
span_events json,
span_links json,
primary key (service_name)
) with (
'table_data_model' = 'greptime_trace_v2',
'append_mode' = 'true',
'greptime.semantic.entity.gen_ai.agent.id' = 'span_attributes.gen_ai.agent.id',
'greptime.semantic.entity.gen_ai.agent.scope' = 'scope_attributes.environment',
'greptime.semantic.entity.gen_ai.model.id' = 'span_attributes.gen_ai.request.model',
'greptime.semantic.entity.literal_peer.id' = 'span_attributes.db.namespace',
'greptime.semantic.entity.literal_resource.id' = 'resource_attributes.host.name',
'greptime.semantic.entity.literal_span.id' = 'span_attributes.gen_ai.agent.id',
'greptime.semantic.entity.literal_scope.id' = 'scope_attributes.deployment.region.name'
);
insert into graph_trace_v2
("timestamp", duration_nano, trace_id, span_id, span_kind, span_status_code,
service_name, span_attributes, scope_attributes, resource_attributes)
values
('2026-01-01 00:00:01', 100000000, 'paired', 'client', 'SPAN_KIND_CLIENT', 'STATUS_CODE_UNSET',
'frontend', '{"gen_ai.agent.id":"agent-a","gen_ai.request.model":"model-a","gen_ai":{"agent":{"id":"nested-agent"}}}',
'{"environment":"prod","deployment.region.name":"literal-region","deployment":{"region.name":"nested-region"},"a\"b\\c.d":7}', '{"service.namespace":"shop","service.instance.id":"front-1","host.id":"host-1","host.name":"node-1","host":{"name":"nested-node"},"service":{"namespace":"wrong-namespace"}}'),
('2026-01-01 00:00:02', 200000000, 'virtual', 'client', 'SPAN_KIND_CLIENT', 'STATUS_CODE_ERROR',
'frontend', '{"service.peer.name":"","db.namespace":"orders","db":{"namespace":"wrong-orders"}}', '{}',
'{"service.namespace":"shop","service.instance.id":"front-1","host.id":"host-1","host.name":"node-1"}'),
('2026-01-01 00:00:03', 100, 'missing', 'internal', 'SPAN_KIND_INTERNAL', 'STATUS_CODE_UNSET',
'no-resource', '{"gen_ai.agent.id":null}', '{}', '{}');
-- Agent parent/child references are independent of span kind.
insert into graph_trace_v2
("timestamp", duration_nano, trace_id, span_id, parent_span_id, span_kind,
span_status_code, service_name, span_attributes, scope_attributes, resource_attributes)
values ('2026-01-01 00:00:01', 30000000, 'paired', 'agent-child', 'client',
'SPAN_KIND_INTERNAL', 'STATUS_CODE_UNSET', 'frontend',
'{"gen_ai.agent.id":"agent-b"}', '{"environment":"prod"}', '{"service.namespace":"shop"}');
-- Calls can pair a V2 client with a V1 server. The literal_resource declaration
-- has exactly the same spelling in both models, including its output JSON keys.
create table graph_trace_v1_peer (
"timestamp" timestamp(9) time index,
trace_id string, span_id string, parent_span_id string,
span_kind string, span_status_code string, duration_nano bigint,
service_name string,
"resource_attributes.service.namespace" string,
"resource_attributes.host.name" string,
primary key (service_name)
) with ('table_data_model' = 'greptime_trace_v1', 'append_mode' = 'true',
'greptime.semantic.entity.literal_resource.id' = 'resource_attributes.host.name');
insert into graph_trace_v1_peer values
('2026-01-01 00:00:01', 'paired', 'server', 'client', 'SPAN_KIND_SERVER', 'STATUS_CODE_UNSET', 50000000, 'backend', 'shop', 'node-v1');
select entity_type, entity_id, scope
from greptime_private.semantic_entities
where observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
order by entity_type, entity_id;
select src_type, src_id, dst_type, dst_id, rel_type, request_count, error_count
from greptime_private.semantic_relationships
where observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
order by rel_type, src_type, src_id, dst_id;
-- Empty components in the V1-compatible comma-separated list remain invalid.
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'resource_attributes.host.id,';
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'resource_attributes.host.id,,service_name';
-- ALTER accepts JSON2 references; a missing key simply yields no entity.
alter table graph_trace_v2 set 'greptime.semantic.entity.custom.id' = 'scope_attributes.missing';
alter table graph_trace_v2 set 'greptime.semantic.entity.host.descriptive' = 'resource_attributes.host.name';
-- Preserve implicit host identity by explicitly declaring it as well.
alter table graph_trace_v2 set 'greptime.semantic.entity.host.id' = 'resource_attributes.host.id';
admin flush_table('graph_trace_v2');
admin flush_table('graph_trace_v1_peer');
-- SQLNESS PROTOCOL MYSQL
select entity_type, entity_id, scope, entity_id_attrs, descriptive
from greptime_private.semantic_entities
where observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
order by entity_type, entity_id;
select src_type, src_id, dst_type, dst_id, rel_type, request_count, error_count
from greptime_private.semantic_relationships
where observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
order by rel_type, src_type, src_id, dst_id;
-- Quotes, backslashes and dots remain a literal key; numbers render as strings.
alter table graph_trace_v2 set 'greptime.semantic.entity.custom.id' = 'scope_attributes.a"b\c.d';
select entity_type, entity_id
from greptime_private.semantic_entities
where entity_type = 'custom'
and observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00';
-- Path-looking suffixes are literal keys, not nested traversal or array access.
-- These missing keys are valid declarations and produce no entities.
alter table graph_trace_v2 set 'greptime.semantic.entity.literal_missing.id' = 'resource_attributes.host..name';
alter table graph_trace_v2 set 'greptime.semantic.entity.literal_index.id' = 'resource_attributes.host[0]';
-- The root must exist and be one of the three JSON2 attribute columns.
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'span_events.name';
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'resource_attributes.';
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'missing.key';
alter table graph_trace_v2 set 'greptime.semantic.entity.invalid.id' = 'resource_attributes';
create table graph_bad_v2 (
ts timestamp time index, resource_attributes string
) with ('table_data_model' = 'greptime_trace_v2',
'greptime.semantic.entity.host.id' = 'resource_attributes.host.id');
-- Ordinary JSON2 tables do not acquire Trace V2 attribute-reference semantics.
create table graph_not_trace (
ts timestamp time index, resource_attributes json2
) with ('append_mode' = 'true',
'greptime.semantic.entity.host.id' = 'resource_attributes.host.id');
-- Physical convention columns win over conflicting JSON2 keys.
alter table graph_trace_v2 add column "resource_attributes.host.id" string;
alter table graph_trace_v2 add column "resource_attributes.service.namespace" string;
alter table graph_trace_v2 add column "span_attributes.db.namespace" string;
-- Restore the implicit host declaration for the physical-column check.
alter table graph_trace_v2 unset 'greptime.semantic.entity.host.id';
alter table graph_trace_v2 unset 'greptime.semantic.entity.host.descriptive';
insert into graph_trace_v2
("timestamp", duration_nano, trace_id, span_id, span_kind, span_status_code,
service_name, resource_attributes, span_attributes,
"resource_attributes.host.id", "resource_attributes.service.namespace", "span_attributes.db.namespace")
values ('2026-01-01 00:00:04', 100, 'physical', 'physical-client', 'SPAN_KIND_CLIENT', 'STATUS_CODE_UNSET',
'physical-service', '{"host.id":"json-host","service.namespace":"json-namespace"}',
'{"db.namespace":"json-database","db":{"namespace":"nested-database"}}',
'physical-host', 'physical-namespace', 'physical-database');
select entity_type, entity_id
from greptime_private.semantic_entities
where entity_type in ('host', 'service')
and observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
order by entity_type, entity_id;
select src_id, dst_id, rel_type
from greptime_private.semantic_relationships
where src_id = 'physical-namespace/physical-service'
and observed_at >= '2026-01-01 00:00:00' and observed_at < '2026-01-01 00:01:00'
order by rel_type, dst_id;
drop table graph_trace_v2;
drop table graph_trace_v1_peer;