feat(operator): otel info-metric conventions with host/container entities

Whitelist the ingestion-synthesized otel_resource_info descriptor via a
new otel_info_metrics conventions map, gated on source=opentelemetry
(the existing gate hardcoded source=prometheus). Its declarations use
explicit descriptive lists instead of descriptive_rest so identifying
attributes of other entities do not leak into service.instance.

Conventions tightened per the Astronomy Shop findings: host identity is
host.id with host.name descriptive only (host.name is not stable across
SDKs and resource detectors), a generic container entity (new entity
type) is declared only when container.id is present, and trace-v1
tables now synthesize host/container from their flattened resource
attributes too. New co-declared edges: service.instance runs_on
container, container runs_on host.

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>
This commit is contained in:
Dennis Zhuang
2026-08-17 22:27:57 +08:00
parent 08a35d40b8
commit 0f789dd661
5 changed files with 395 additions and 31 deletions
+160 -21
View File
@@ -23,7 +23,7 @@
//! the query engine. Injected into the catalog manager after the engine is built,
//! breaking the `catalog -> query` cycle.
use std::collections::HashMap;
use std::collections::{BTreeMap, HashMap};
use std::sync::Weak;
use async_trait::async_trait;
@@ -60,8 +60,8 @@ use table::TableRef;
use table::metadata::TableInfo;
use table::predicate::{TimeRangeExtraction, extract_time_range_strict};
use table::requests::{
EntityRole, SEMANTIC_SIGNAL_TYPE, SEMANTIC_SOURCE, SIGNAL_TYPE_METRIC, SOURCE_PROMETHEUS,
is_trace_v1_table, parse_entity_columns, parse_entity_option_key,
EntityRole, SEMANTIC_SIGNAL_TYPE, SEMANTIC_SOURCE, SIGNAL_TYPE_METRIC, SOURCE_OPENTELEMETRY,
SOURCE_PROMETHEUS, is_trace_v1_table, parse_entity_columns, parse_entity_option_key,
};
use crate::error;
@@ -188,10 +188,11 @@ impl EntityGraphProviderImpl {
/// All entity declarations of one table: the explicit options plus the
/// zero-configuration conventions (`otlp_trace_entities` for trace-v1
/// tables — including the `service` identity of tables created before the
/// ingest-side auto-stamp — and the Prometheus descriptor whitelist). An
/// explicit declaration of a type always suppresses the implicit one, even
/// when the explicit declaration is invalid and skipped: silently falling
/// back would change entity identity behind the user's back.
/// ingest-side auto-stamp — and the Prometheus/OTel descriptor
/// whitelists). An explicit declaration of a type always suppresses the
/// implicit one, even when the explicit declaration is invalid and
/// skipped: silently falling back would change entity identity behind the
/// user's back.
fn declarations_for(
table_info: &TableInfo,
conventions: &Conventions,
@@ -204,7 +205,18 @@ impl EntityGraphProviderImpl {
&mut declarations,
);
}
Self::extend_with_prometheus_conventions(table_info, conventions, &mut declarations);
Self::extend_with_info_metric_conventions(
table_info,
&conventions.prometheus_info_metrics,
SOURCE_PROMETHEUS,
&mut declarations,
);
Self::extend_with_info_metric_conventions(
table_info,
&conventions.otel_info_metrics,
SOURCE_OPENTELEMETRY,
&mut declarations,
);
declarations
}
@@ -216,29 +228,29 @@ impl EntityGraphProviderImpl {
})
}
/// Implicit declarations of the well-known Prometheus entity-descriptor
/// metrics (the `prometheus_info_metrics` whitelist of `conventions.yaml`),
/// gated on the ingest-stamped `signal_type=metric` + `source=prometheus`
/// options and keyed by table name. The metric engine's physical table
/// aggregates every logical table's columns and must not contribute a
/// duplicate source.
fn extend_with_prometheus_conventions(
/// Implicit declarations of the well-known entity-descriptor metrics
/// (the `prometheus_info_metrics` / `otel_info_metrics` whitelists of
/// `conventions.yaml`), gated on the ingest-stamped `signal_type=metric`
/// option plus the whitelist's expected `source`, and keyed by table
/// name. The metric engine's physical table aggregates every logical
/// table's columns and must not contribute a duplicate source.
fn extend_with_info_metric_conventions(
table_info: &TableInfo,
conventions: &Conventions,
whitelist: &BTreeMap<String, Vec<ImplicitEntity>>,
expected_source: &str,
declarations: &mut Vec<EntityDeclaration>,
) {
let Some(implicit_entities) = conventions.prometheus_info_metrics.get(&table_info.name)
else {
let Some(implicit_entities) = whitelist.get(&table_info.name) else {
return;
};
let options = &table_info.meta.options.extra_options;
if options.get(SEMANTIC_SIGNAL_TYPE).map(String::as_str) != Some(SIGNAL_TYPE_METRIC)
|| options.get(SEMANTIC_SOURCE).map(String::as_str) != Some(SOURCE_PROMETHEUS)
|| options.get(SEMANTIC_SOURCE).map(String::as_str) != Some(expected_source)
|| table_info.is_physical_table()
{
debug!(
"Table `{}` matches the info-metric whitelist but is not a prometheus logical \
metric table; skipping its implicit declarations",
"Table `{}` matches the info-metric whitelist but is not a `{expected_source}` \
logical metric table; skipping its implicit declarations",
table_info.name
);
return;
@@ -856,6 +868,49 @@ mod tests {
assert!(sorted_declarations(&invalid_explicit).is_empty());
}
#[test]
fn trace_table_host_and_container_require_stable_ids() {
let with_ids = table_info(
&[
"service_name",
"resource_attributes.host.id",
"resource_attributes.host.name",
"resource_attributes.container.id",
],
&[(TABLE_DATA_MODEL, TABLE_DATA_MODEL_TRACE_V1)],
);
let declarations = sorted_declarations(&with_ids);
let types: Vec<&str> = declarations
.iter()
.map(|d| d.entity_type.as_str())
.collect();
assert_eq!(types, vec!["container", "host", "service"]);
assert_eq!(
declarations[1].id_columns,
vec!["resource_attributes.host.id"]
);
assert_eq!(
declarations[1].descriptive_columns,
vec!["resource_attributes.host.name"]
);
// Names alone synthesize nothing: host.name/container.name are not
// stable identities across SDKs and resource detectors.
let names_only = table_info(
&[
"service_name",
"resource_attributes.host.name",
"resource_attributes.container.name",
],
&[(TABLE_DATA_MODEL, TABLE_DATA_MODEL_TRACE_V1)],
);
let types: Vec<String> = sorted_declarations(&names_only)
.into_iter()
.map(|d| d.entity_type)
.collect();
assert_eq!(types, vec!["service"]);
}
#[test]
fn prometheus_implicit_declarations_are_gated() {
let labels: &[&str] = &["namespace", "pod", "node"];
@@ -900,6 +955,90 @@ mod tests {
assert_eq!(declarations[1].id_columns, vec!["pod"]);
}
const OTEL_STAMPS: &[(&str, &str)] = &[
(SEMANTIC_SIGNAL_TYPE, SIGNAL_TYPE_METRIC),
(SEMANTIC_SOURCE, SOURCE_OPENTELEMETRY),
];
#[test]
fn otel_resource_info_gets_implicit_declarations() {
let info = prom_table_info(
"otel_resource_info",
&[
"job",
"instance",
"service.name",
"service.namespace",
"host.id",
"host.name",
"container.id",
"container.name",
"k8s.pod.uid",
"k8s.pod.name",
"k8s.namespace.name",
],
OTEL_STAMPS,
);
let declarations = sorted_declarations(&info);
let types: Vec<&str> = declarations
.iter()
.map(|d| d.entity_type.as_str())
.collect();
assert_eq!(
types,
vec!["container", "host", "k8s.pod", "service", "service.instance"]
);
// host identity is host.id only; host.name stays descriptive
assert_eq!(declarations[1].id_columns, vec!["host.id"]);
assert_eq!(declarations[1].descriptive_columns, vec!["host.name"]);
assert_eq!(declarations[3].id_columns, vec!["job"]);
assert_eq!(
declarations[3].descriptive_columns,
vec!["service.name", "service.namespace"]
);
// no descriptive_rest: other entities' identifying attributes must
// not leak into service.instance
assert_eq!(declarations[4].id_columns, vec!["job", "instance"]);
assert!(declarations[4].descriptive_columns.is_empty());
}
#[test]
fn otel_resource_info_missing_id_columns_drop_entities() {
let info = prom_table_info(
"otel_resource_info",
&["job", "service.name", "host.id"],
OTEL_STAMPS,
);
let types: Vec<String> = sorted_declarations(&info)
.into_iter()
.map(|d| d.entity_type)
.collect();
assert_eq!(types, vec!["host", "service"]);
}
#[test]
fn otel_implicit_declarations_are_gated() {
let labels: &[&str] = &["job", "instance", "host.id"];
// a prometheus-stamped table under the otel-whitelisted name
assert!(
sorted_declarations(&prom_table_info("otel_resource_info", labels, PROM_STAMPS))
.is_empty()
);
let mut stamps = OTEL_STAMPS.to_vec();
stamps.push((PHYSICAL_TABLE_METADATA_KEY, "true"));
assert!(
sorted_declarations(&prom_table_info("otel_resource_info", labels, &stamps))
.is_empty()
);
// drift guard: the ingestion-side table name must stay whitelisted
assert!(
conventions()
.unwrap()
.otel_info_metrics
.contains_key(servers::otlp::metrics::OTEL_RESOURCE_INFO_TABLE_NAME)
);
}
#[test]
fn target_info_descriptive_rest_covers_remaining_tags() {
let info = prom_table_info(
@@ -43,8 +43,8 @@ pub struct VirtualDstCandidate {
pub connection_type: String,
}
/// One implicit entity declaration: of a whitelisted Prometheus info metric,
/// or of a trace-v1 table's flattened resource attributes.
/// One implicit entity declaration: of a whitelisted Prometheus or OTel info
/// metric, or of a trace-v1 table's flattened resource attributes.
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ImplicitEntity {
@@ -71,6 +71,10 @@ pub struct Conventions {
pub virtual_dst_candidates: Vec<VirtualDstCandidate>,
pub otlp_trace_entities: Vec<ImplicitEntity>,
pub prometheus_info_metrics: BTreeMap<String, Vec<ImplicitEntity>>,
/// Implicit declarations of OTLP-sourced descriptor tables (the
/// ingestion-synthesized `otel_resource_info`), gated on
/// `source = opentelemetry` instead of `prometheus`.
pub otel_info_metrics: BTreeMap<String, Vec<ImplicitEntity>>,
}
/// The built-in entity-type vocabulary. User-declared types are open-ended;
@@ -78,6 +82,7 @@ pub struct Conventions {
pub const ENTITY_TYPE_SERVICE: &str = "service";
pub const ENTITY_TYPE_SERVICE_INSTANCE: &str = "service.instance";
pub const ENTITY_TYPE_HOST: &str = "host";
pub const ENTITY_TYPE_CONTAINER: &str = "container";
pub const ENTITY_TYPE_PROCESS: &str = "process";
pub const ENTITY_TYPE_K8S_POD: &str = "k8s.pod";
pub const ENTITY_TYPE_K8S_NODE: &str = "k8s.node";
@@ -88,10 +93,11 @@ pub const ENTITY_TYPE_GEN_AI_AGENT: &str = "gen_ai.agent";
pub const ENTITY_TYPE_GEN_AI_MODEL: &str = "gen_ai.model";
pub const ENTITY_TYPE_GEN_AI_TOOL: &str = "gen_ai.tool";
const ENTITY_TYPES: [&str; 12] = [
const ENTITY_TYPES: [&str; 13] = [
ENTITY_TYPE_SERVICE,
ENTITY_TYPE_SERVICE_INSTANCE,
ENTITY_TYPE_HOST,
ENTITY_TYPE_CONTAINER,
ENTITY_TYPE_PROCESS,
ENTITY_TYPE_K8S_POD,
ENTITY_TYPE_K8S_NODE,
@@ -200,6 +206,7 @@ fn validate(conventions: &Conventions) -> Result<(), String> {
let per_table = conventions
.prometheus_info_metrics
.iter()
.chain(&conventions.otel_info_metrics)
.map(|(table, entities)| (table.as_str(), entities))
.chain(std::iter::once((
"otlp traces",
@@ -249,25 +256,28 @@ mod tests {
/// Each case must fail on exactly the rule it names, so the shared
/// boilerplate is valid and the mutated part is inside the vocabulary.
fn broken(edges: &str, info_metrics: &str) -> String {
fn broken(edges: &str, info_metrics: &str, otel_metrics: &str) -> String {
format!(
"co_declared_edges: [{edges}]\ntrace_co_declared_edges: []\n\
virtual_dst_candidates: []\notlp_trace_entities: []\n\
prometheus_info_metrics: {{{info_metrics}}}"
prometheus_info_metrics: {{{info_metrics}}}\n\
otel_info_metrics: {{{otel_metrics}}}"
)
}
#[test]
fn validation_rejects_broken_conventions() {
let err = |edges, info| parse(&broken(edges, info)).unwrap_err();
let err = |edges, info, otel| parse(&broken(edges, info, otel)).unwrap_err();
assert!(err("{src: host, dst: service, rel: pets}", "").contains("unknown rel_type"));
assert!(err("{src: host, dst: service, rel: pets}", "", "").contains("unknown rel_type"));
assert!(
err("{src: k8s.pods, dst: k8s.node, rel: runs_on}", "").contains("unknown entity type")
err("{src: k8s.pods, dst: k8s.node, rel: runs_on}", "", "")
.contains("unknown entity type")
);
assert!(
err(
"{src: host, dst: service, rel: uses}, {src: host, dst: service, rel: uses}",
"",
""
)
.contains("duplicate edge rule")
@@ -275,14 +285,20 @@ mod tests {
assert!(
err(
"",
"t: [{entity: host, id: [x], descriptive: [y], descriptive_rest: true}]"
"t: [{entity: host, id: [x], descriptive: [y], descriptive_rest: true}]",
""
)
.contains("descriptive_rest")
);
// the otel map runs through the same per-table validation
assert!(
err("", "", "t: [{entity: hosts, id: [x]}]").contains("unknown entity type")
);
// Unknown YAML keys are rejected, catching typos in the embedded file.
assert!(
parse(&broken(
"{src: host, dst: service, rel: uses, direction: down}",
"",
""
))
.is_err()
@@ -11,6 +11,8 @@ co_declared_edges:
- { src: service.instance, dst: service, rel: part_of }
- { src: k8s.pod, dst: k8s.workload, rel: part_of }
- { src: service.instance, dst: k8s.pod, rel: runs_on }
- { src: service.instance, dst: container, rel: runs_on }
- { src: container, dst: host, rel: runs_on }
# Applied only to trace sources; the endpoints are Greptime entity types
# derived from GenAI semantic-convention attributes.
@@ -30,12 +32,22 @@ virtual_dst_candidates:
- { column: span_attributes.server.address, connection_type: virtual_node }
# Implicit declarations for greptime_trace_v1 tables; the pod UID bridges
# trace-side pods onto the kube-state-metrics entities below.
# trace-side pods onto the kube-state-metrics entities below. Host and
# container identities are the stable ids only — host.name and container.name
# vary by SDK and resource detector, so they stay descriptive.
otlp_trace_entities:
- entity: service
id: [service_name]
- entity: service.instance
id: [service_name, resource_attributes.service.instance.id]
- entity: host
id: [resource_attributes.host.id]
descriptive:
- resource_attributes.host.name
- entity: container
id: [resource_attributes.container.id]
descriptive:
- resource_attributes.container.name
- entity: k8s.pod
id: [resource_attributes.k8s.pod.uid]
descriptive:
@@ -113,3 +125,33 @@ prometheus_info_metrics:
- entity: service.instance
id: [job, instance]
descriptive_rest: true
# Implicit declarations for the OTLP resource descriptor synthesized at
# metrics ingestion, gated on source=opentelemetry. Columns are the raw OTel
# attribute keys; job/instance are the derived Prometheus-compatible service
# identity, so services line up with target_info-sourced entities. Every
# descriptive column is explicit: the descriptor's attribute set is a known
# allowlist, and identifying attributes of other entities must not leak into
# service.instance the way target_info's descriptive_rest would.
otel_info_metrics:
otel_resource_info:
- entity: service
id: [job]
descriptive:
- service.name
- service.namespace
- entity: service.instance
id: [job, instance]
- entity: host
id: [host.id]
descriptive:
- host.name
- entity: container
id: [container.id]
descriptive:
- container.name
- entity: k8s.pod
id: [k8s.pod.uid]
descriptive:
- k8s.pod.name
- k8s.namespace.name
@@ -627,3 +627,104 @@ drop table http_requests_total;
Affected Rows: 0
-- OTel conventions: the ingestion-synthesized otel_resource_info descriptor
-- (stamped signal_type=metric + source=opentelemetry) gets implicit
-- declarations under the raw OTel column names. host/container identities are
-- the stable ids only, so rows with an empty host.id / container.id link no
-- infrastructure, and a row without an instance value declares no
-- service.instance. The same table name stamped source=prometheus is not
-- whitelisted.
create table otel_resource_info (
greptime_timestamp timestamp(3) time index,
job string,
instance string,
"service.name" string,
"service.namespace" string,
"host.id" string,
"host.name" string,
"container.id" string,
"container.name" string,
greptime_value double,
primary key (job, instance, "service.name", "service.namespace",
"host.id", "host.name", "container.id", "container.name")
) with (
'greptime.semantic.signal_type' = 'metric',
'greptime.semantic.source' = 'opentelemetry'
);
Affected Rows: 0
insert into otel_resource_info values
(now(), 'shop/api', 'inst-1', 'api', 'shop', 'h-1', 'node-a', 'c-1', 'api-ctr', 1),
(now(), 'shop/api', 'inst-2', 'api', 'shop', '', 'laptop', '', '', 1),
(now(), 'worker', '', 'worker', '', 'h-1', 'node-a', '', '', 1);
Affected Rows: 3
-- SQLNESS PROTOCOL MYSQL
select entity_type, entity_id, source_tables
from greptime_private.semantic_entities
order by entity_type, entity_id, source_tables;
+------------------+------------------------------+-------------------------------+
| entity_type | entity_id | source_tables |
+------------------+------------------------------+-------------------------------+
| container | c-1 | ["public.otel_resource_info"] |
| host | h-1 | ["public.otel_resource_info"] |
| service | shop/api | ["public.otel_resource_info"] |
| service | worker | ["public.otel_resource_info"] |
| service.instance | instance=inst-1,job=shop/api | ["public.otel_resource_info"] |
| service.instance | instance=inst-2,job=shop/api | ["public.otel_resource_info"] |
+------------------+------------------------------+-------------------------------+
-- SQLNESS PROTOCOL MYSQL
select src_type, src_id, dst_type, dst_id, rel_type, provenance
from greptime_private.semantic_relationships
order by rel_type, src_id, dst_id;
+------------------+------------------------------+-----------+----------+----------+------------+
| src_type | src_id | dst_type | dst_id | rel_type | provenance |
+------------------+------------------------------+-----------+----------+----------+------------+
| service.instance | instance=inst-1,job=shop/api | service | shop/api | part_of | attribute |
| service.instance | instance=inst-2,job=shop/api | service | shop/api | part_of | attribute |
| container | c-1 | host | h-1 | runs_on | attribute |
| service.instance | instance=inst-1,job=shop/api | container | c-1 | runs_on | attribute |
| service.instance | instance=inst-1,job=shop/api | host | h-1 | runs_on | attribute |
+------------------+------------------------------+-----------+----------+----------+------------+
drop table otel_resource_info;
Affected Rows: 0
-- The descriptor whitelist is gated on source=opentelemetry: the same table
-- shape stamped as a prometheus source contributes nothing.
create table otel_resource_info (
greptime_timestamp timestamp(3) time index,
job string,
instance string,
"host.id" string,
greptime_value double,
primary key (job, instance, "host.id")
) with (
'greptime.semantic.signal_type' = 'metric',
'greptime.semantic.source' = 'prometheus'
);
Affected Rows: 0
insert into otel_resource_info values
(now(), 'shop/api', 'inst-1', 'h-1', 1);
Affected Rows: 1
-- SQLNESS PROTOCOL MYSQL
select entity_type, entity_id
from greptime_private.semantic_entities
order by entity_type, entity_id;
affected_rows: 0
drop table otel_resource_info;
Affected Rows: 0
@@ -387,3 +387,69 @@ drop table kube_service_info;
drop table target_info;
drop table http_requests_total;
-- OTel conventions: the ingestion-synthesized otel_resource_info descriptor
-- (stamped signal_type=metric + source=opentelemetry) gets implicit
-- declarations under the raw OTel column names. host/container identities are
-- the stable ids only, so rows with an empty host.id / container.id link no
-- infrastructure, and a row without an instance value declares no
-- service.instance. The same table name stamped source=prometheus is not
-- whitelisted.
create table otel_resource_info (
greptime_timestamp timestamp(3) time index,
job string,
instance string,
"service.name" string,
"service.namespace" string,
"host.id" string,
"host.name" string,
"container.id" string,
"container.name" string,
greptime_value double,
primary key (job, instance, "service.name", "service.namespace",
"host.id", "host.name", "container.id", "container.name")
) with (
'greptime.semantic.signal_type' = 'metric',
'greptime.semantic.source' = 'opentelemetry'
);
insert into otel_resource_info values
(now(), 'shop/api', 'inst-1', 'api', 'shop', 'h-1', 'node-a', 'c-1', 'api-ctr', 1),
(now(), 'shop/api', 'inst-2', 'api', 'shop', '', 'laptop', '', '', 1),
(now(), 'worker', '', 'worker', '', 'h-1', 'node-a', '', '', 1);
-- SQLNESS PROTOCOL MYSQL
select entity_type, entity_id, source_tables
from greptime_private.semantic_entities
order by entity_type, entity_id, source_tables;
-- SQLNESS PROTOCOL MYSQL
select src_type, src_id, dst_type, dst_id, rel_type, provenance
from greptime_private.semantic_relationships
order by rel_type, src_id, dst_id;
drop table otel_resource_info;
-- The descriptor whitelist is gated on source=opentelemetry: the same table
-- shape stamped as a prometheus source contributes nothing.
create table otel_resource_info (
greptime_timestamp timestamp(3) time index,
job string,
instance string,
"host.id" string,
greptime_value double,
primary key (job, instance, "host.id")
) with (
'greptime.semantic.signal_type' = 'metric',
'greptime.semantic.source' = 'prometheus'
);
insert into otel_resource_info values
(now(), 'shop/api', 'inst-1', 'h-1', 1);
-- SQLNESS PROTOCOL MYSQL
select entity_type, entity_id
from greptime_private.semantic_entities
order by entity_type, entity_id;
drop table otel_resource_info;