Files
greptimedb/tests-integration/src/instance.rs
T
shuiyisong ee1fc7782f feat(prometheus): query persisted metric exemplars (#9924)
* feat(prometheus): add exemplar queries and lifecycle coverage

Expose /v1/prometheus/api/v1/query_exemplars with exact time bounds and series selectors. Resolve and authorize source selectors before scanning exemplar storage, enforce query limits and cancellation, preserve exact observation identity, and return warnings for malformed stored rows.

Includes HTTP and authorization tests, pre-created-table and concurrent-DDL coverage, OTLP Arrow ingestion checks, and cross-protocol lifecycle tests across memory, WAL, SST, compaction and reopen. The query-dependent ingestion tests live here so every lower layer builds independently. Add metric-exemplar usage and retention guidance.

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* fix(exemplar): preserve query labels with source identities

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* fix(exemplar): floor bounds and add audit hooks

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* • fix(exemplar): include aggregation parameter selectors

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* fix(prometheus): avoid v1 exemplar quota stalls

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* fix: docs

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* fix: align exemplar query timestamps with Prometheus

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: remove unnecessory doc

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

---------

Signed-off-by: shuiyisong <xixing.sys@gmail.com>
2026-11-10 08:11:49 +00:00

1115 lines
41 KiB
Rust

// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(test)]
mod tests {
use std::borrow::Cow;
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::AtomicU32;
use api::v1::region::QueryRequest;
use client::OutputData;
use common_base::Plugins;
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
use common_error::ext::ErrorExt;
use common_error::status_code::StatusCode;
use common_meta::key::table_name::TableNameKey;
use common_meta::rpc::router::region_distribution;
use common_query::Output;
use common_recordbatch::RecordBatches;
use common_telemetry::debug;
use datafusion_expr::LogicalPlan;
use frontend::error::{Error, Result};
use frontend::instance::Instance;
use query::parser::{QueryLanguageParser, QueryStatement};
use query::query_engine::DefaultSerializer;
use servers::interceptor::{SqlQueryInterceptor, SqlQueryInterceptorRef};
use servers::query_handler::sql::SqlQueryHandler;
use session::context::{QueryContext, QueryContextRef};
use sql::statements::statement::Statement;
use store_api::storage::RegionId;
use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
use crate::standalone::GreptimeDbStandaloneBuilder;
use crate::tests;
use crate::tests::MockDistributedInstance;
#[tokio::test(flavor = "multi_thread")]
async fn test_standalone_exec_sql() {
let standalone = GreptimeDbStandaloneBuilder::new("test_standalone_exec_sql")
.build()
.await;
let instance = standalone.fe_instance();
let sql = r#"
CREATE TABLE demo(
host STRING,
ts TIMESTAMP,
cpu DOUBLE NULL,
memory DOUBLE NULL,
disk_util DOUBLE DEFAULT 9.9,
TIME INDEX (ts),
PRIMARY KEY(host)
) engine=mito"#;
create_table(instance, sql).await;
insert_and_query(instance).await;
drop_table(instance).await;
}
#[tokio::test(flavor = "multi_thread")]
async fn test_distributed_exec_sql() {
common_telemetry::init_default_ut_logging();
let distributed = tests::create_distributed_instance("test_distributed_exec_sql").await;
let frontend = distributed.frontend();
let instance = frontend.as_ref();
let sql = r#"
CREATE TABLE demo(
host STRING,
ts TIMESTAMP,
cpu DOUBLE NULL,
memory DOUBLE NULL,
disk_util DOUBLE DEFAULT 9.9,
TIME INDEX (ts),
PRIMARY KEY(host)
)
PARTITION ON COLUMNS (host) (
host < '550-A',
host >= '550-A' AND host < '550-W',
host >= '550-W' AND host < 'MOSS',
host >= 'MOSS'
)
engine=mito"#;
create_table(instance, sql).await;
insert_and_query(instance).await;
verify_data_distribution(
&distributed,
HashMap::from([
(
0u32,
"\
+---------------------+------+
| ts | host |
+---------------------+------+
| 2013-12-31T16:00:00 | 490 |
+---------------------+------+",
),
(
1u32,
"\
+---------------------+-------+
| ts | host |
+---------------------+-------+
| 2022-12-31T16:00:00 | 550-A |
+---------------------+-------+",
),
(
2u32,
"\
+---------------------+-------+
| ts | host |
+---------------------+-------+
| 2023-12-31T16:00:00 | 550-W |
+---------------------+-------+",
),
(
3u32,
"\
+---------------------+------+
| ts | host |
+---------------------+------+
| 2043-12-31T16:00:00 | MOSS |
+---------------------+------+",
),
]),
)
.await;
drop_table(instance).await;
verify_table_is_dropped(&distributed).await;
}
async fn query(instance: &Instance, sql: &str) -> Output {
SqlQueryHandler::do_query(instance, sql, QueryContext::arc())
.await
.remove(0)
.unwrap()
}
async fn create_table(instance: &Instance, sql: &str) {
let output = query(instance, sql).await;
let OutputData::AffectedRows(x) = output.data else {
unreachable!()
};
assert_eq!(x, 0);
}
async fn insert_and_query(instance: &Instance) {
let sql = r#"INSERT INTO demo(host, cpu, memory, ts) VALUES
('490', 0.1, 1, 1388505600000),
('550-A', 1, 100, 1672502400000),
('550-W', 10000, 1000000, 1704038400000),
('MOSS', 100000000, 10000000000, 2335190400000)
"#;
let output = query(instance, sql).await;
let OutputData::AffectedRows(x) = output.data else {
unreachable!()
};
assert_eq!(x, 4);
let sql = "SELECT * FROM demo WHERE ts > cast(1000000000 as timestamp) ORDER BY host"; // use nanoseconds as where condition
let output = query(instance, sql).await;
let OutputData::Stream(s) = output.data else {
unreachable!()
};
let batches = common_recordbatch::util::collect_batches(s).await.unwrap();
let pretty_print = batches.pretty_print().unwrap();
let expected = "\
+-------+---------------------+-------------+---------------+-----------+
| host | ts | cpu | memory | disk_util |
+-------+---------------------+-------------+---------------+-----------+
| 490 | 2013-12-31T16:00:00 | 0.1 | 1.0 | 9.9 |
| 550-A | 2022-12-31T16:00:00 | 1.0 | 100.0 | 9.9 |
| 550-W | 2023-12-31T16:00:00 | 10000.0 | 1000000.0 | 9.9 |
| MOSS | 2043-12-31T16:00:00 | 100000000.0 | 10000000000.0 | 9.9 |
+-------+---------------------+-------------+---------------+-----------+";
assert_eq!(pretty_print, expected);
}
async fn verify_data_distribution(
instance: &MockDistributedInstance,
expected_distribution: HashMap<u32, &str>,
) {
let manager = instance.table_metadata_manager();
let table_id = manager
.table_name_manager()
.get(TableNameKey::new(
DEFAULT_CATALOG_NAME,
DEFAULT_SCHEMA_NAME,
"demo",
))
.await
.unwrap()
.unwrap()
.table_id();
debug!("Reading table {table_id}");
let table_route_value = manager
.table_route_manager()
.table_route_storage()
.get(table_id)
.await
.unwrap()
.unwrap();
let region_to_dn_map = region_distribution(
table_route_value
.region_routes()
.expect("region routes should be physical"),
)
.iter()
.map(|(k, v)| (v.leader_regions[0], *k))
.collect::<HashMap<u32, u64>>();
assert!(region_to_dn_map.len() <= instance.datanodes().len());
let stmt = QueryLanguageParser::parse_sql(
"SELECT ts, host FROM demo ORDER BY ts",
&QueryContext::arc(),
)
.unwrap();
let plan = instance
.frontend()
.statement_executor()
.plan(&stmt, QueryContext::arc())
.await
.unwrap();
let plan = DFLogicalSubstraitConvertor
.encode(&plan, DefaultSerializer)
.unwrap();
for (region, dn) in region_to_dn_map.iter() {
let region_server = instance.datanodes().get(dn).unwrap().region_server();
let region_id = RegionId::new(table_id, *region);
let stream = region_server
.handle_remote_read(
QueryRequest {
region_id: region_id.as_u64(),
plan: plan.to_vec(),
..Default::default()
},
QueryContext::arc(),
)
.await
.unwrap();
let recordbatches = RecordBatches::try_collect(stream).await.unwrap();
let actual = recordbatches.pretty_print().unwrap();
let expected = expected_distribution.get(region).unwrap();
assert_eq!(&actual, expected);
}
}
async fn drop_table(instance: &Instance) {
let sql = "DROP TABLE demo";
let output = query(instance, sql).await;
let OutputData::AffectedRows(x) = output.data else {
unreachable!()
};
assert_eq!(x, 0);
}
async fn verify_table_is_dropped(instance: &MockDistributedInstance) {
assert!(
instance
.frontend()
.catalog_manager()
.table("greptime", "public", "demo", None)
.await
.unwrap()
.is_none()
)
}
#[tokio::test(flavor = "multi_thread")]
async fn test_select_admin_function_requires_admin_privilege() {
use auth::{
DefaultPermissionChecker, Identity, Password, PermissionCheckerRef, UserProvider,
static_user_provider_from_option,
};
let plugins = Plugins::new();
plugins.insert::<PermissionCheckerRef>(DefaultPermissionChecker::arc());
let standalone =
GreptimeDbStandaloneBuilder::new("test_select_admin_function_requires_admin_privilege")
.with_plugin(plugins)
.build()
.await;
let instance = standalone.fe_instance();
create_table(
instance,
"CREATE TABLE admin_fn_demo(host STRING, ts TIMESTAMP TIME INDEX)",
)
.await;
async fn user_ctx(username: &str, mode: &str) -> QueryContextRef {
let provider = static_user_provider_from_option(&format!(
"static_user_provider:cmd:{username}:{mode}={username}_pwd"
))
.unwrap();
let user = provider
.authenticate(
Identity::UserId(username, None),
Password::PlainText(format!("{username}_pwd").into()),
)
.await
.unwrap();
let ctx = QueryContext::arc();
ctx.set_current_user(user);
ctx
}
async fn assert_select_admin_denied(instance: &Instance, ctx: &QueryContextRef) {
for sql in [
"SELECT flush_table('admin_fn_demo')",
"SELECT compact_table('admin_fn_demo')",
"SELECT flush_region(0)",
"SELECT compact_region(0)",
"SELECT cancel_region_compaction(0, 'x')",
"SELECT migrate_region(1, 2, 3)",
"SELECT gc_regions(0)",
"SELECT gc_table('admin_fn_demo')",
"SELECT build_index('admin_fn_demo')",
"SELECT build_series_index('admin_fn_demo')",
"SELECT flush_flow('admin_fn_demo')",
"SELECT reconcile_catalog()",
"SELECT reconcile_database('public')",
"SELECT reconcile_table('admin_fn_demo')",
] {
let mut results = SqlQueryHandler::do_query(instance, sql, ctx.clone()).await;
let output = results
.remove(0)
.unwrap_or_else(|e| panic!("unexpected error for {sql}: {e}"));
let OutputData::Stream(stream) = output.data else {
panic!("{sql} must return a stream, got {:?}", output.data)
};
let err = common_recordbatch::util::collect_batches(stream)
.await
.expect_err("read-only user must be denied");
assert_eq!(
StatusCode::PermissionDenied,
err.status_code(),
"unexpected error for {sql}: {err}"
);
}
}
let readonly_ctx = user_ctx("ro_user", "ro").await;
let readwrite_ctx = user_ctx("rw_user", "rw").await;
assert_select_admin_denied(instance, &readonly_ctx).await;
let mut results = SqlQueryHandler::do_query(
instance.as_ref(),
"ADMIN FLUSH_TABLE('admin_fn_demo')",
readwrite_ctx.clone(),
)
.await;
assert!(
results.remove(0).is_ok(),
"ADMIN FLUSH_TABLE must keep working for read-write users"
);
for sql in [
"SELECT flush_table('admin_fn_demo')",
"SELECT compact_table('admin_fn_demo')",
] {
let mut results =
SqlQueryHandler::do_query(instance.as_ref(), sql, readwrite_ctx.clone()).await;
let output = results
.remove(0)
.unwrap_or_else(|e| panic!("unexpected error for {sql}: {e}"));
let OutputData::Stream(stream) = output.data else {
panic!("{sql} must return a stream");
};
let batches = common_recordbatch::util::collect_batches(stream)
.await
.unwrap_or_else(|e| panic!("unexpected error for {sql}: {e}"));
assert_eq!(
1,
batches.iter().count(),
"{sql} must return one record batch"
);
}
let mut results = SqlQueryHandler::do_query(
instance.as_ref(),
"ADMIN FLUSH_TABLE('admin_fn_demo')",
readonly_ctx.clone(),
)
.await;
let err = results.remove(0).unwrap_err();
assert_eq!(
StatusCode::PermissionDenied,
err.status_code(),
"unexpected error: {err}"
);
let output = query(instance, "DROP TABLE admin_fn_demo").await;
let OutputData::AffectedRows(x) = output.data else {
unreachable!()
};
assert_eq!(x, 0);
}
#[tokio::test(flavor = "multi_thread")]
async fn test_sql_interceptor_plugin() {
#[derive(Default)]
struct AssertionHook {
pub(crate) c: AtomicU32,
}
impl SqlQueryInterceptor for AssertionHook {
type Error = Error;
fn pre_parsing<'a>(
&self,
query: &'a str,
_query_ctx: QueryContextRef,
) -> Result<Cow<'a, str>> {
let _ = self.c.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
assert!(query.starts_with("CREATE TABLE demo"));
Ok(Cow::Borrowed(query))
}
fn post_parsing(
&self,
statements: Vec<Statement>,
_query_ctx: QueryContextRef,
) -> Result<Vec<Statement>> {
let _ = self.c.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
assert!(matches!(statements[0], Statement::CreateTable(_)));
Ok(statements)
}
fn pre_execute(
&self,
_statement: Option<&Statement>,
_plan: Option<&LogicalPlan>,
_query_ctx: QueryContextRef,
) -> Result<()> {
let _ = self.c.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
Ok(())
}
fn post_execute(
&self,
mut output: Output,
_query_ctx: QueryContextRef,
) -> Result<Output> {
let _ = self.c.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
match &mut output.data {
OutputData::AffectedRows(rows) => {
assert_eq!(*rows, 0);
// update output result
*rows = 10;
}
_ => unreachable!(),
}
Ok(output)
}
}
let plugins = Plugins::new();
let counter_hook = Arc::new(AssertionHook::default());
plugins.insert::<SqlQueryInterceptorRef<Error>>(counter_hook.clone());
let standalone = GreptimeDbStandaloneBuilder::new("test_sql_interceptor_plugin")
.with_plugin(plugins)
.build()
.await;
let instance = standalone.fe_instance().clone();
let sql = r#"CREATE TABLE demo(
host STRING,
ts TIMESTAMP,
cpu DOUBLE NULL,
memory DOUBLE NULL,
disk_util DOUBLE DEFAULT 9.9,
TIME INDEX (ts),
PRIMARY KEY(host)
) engine=mito;"#;
let output = SqlQueryHandler::do_query(&*instance, sql, QueryContext::arc())
.await
.remove(0)
.unwrap();
// assert that the hook is called 3 times
assert_eq!(4, counter_hook.c.load(std::sync::atomic::Ordering::Relaxed));
match output.data {
OutputData::AffectedRows(rows) => assert_eq!(rows, 10),
_ => unreachable!(),
}
}
#[tokio::test(flavor = "multi_thread")]
async fn test_exec_plan_interceptor_plugin() {
use std::sync::atomic::{AtomicBool, Ordering};
#[derive(Default)]
struct ExecPlanHook {
pub(crate) pre_execute_called: AtomicBool,
pub(crate) post_execute_called: AtomicBool,
}
impl SqlQueryInterceptor for ExecPlanHook {
type Error = Error;
fn pre_execute(
&self,
_statement: Option<&Statement>,
_plan: Option<&LogicalPlan>,
_query_ctx: QueryContextRef,
) -> Result<()> {
self.pre_execute_called.store(true, Ordering::Relaxed);
Ok(())
}
fn post_execute(&self, output: Output, _query_ctx: QueryContextRef) -> Result<Output> {
self.post_execute_called.store(true, Ordering::Relaxed);
Ok(output)
}
}
let plugins = Plugins::new();
let hook = Arc::new(ExecPlanHook::default());
plugins.insert::<SqlQueryInterceptorRef<Error>>(hook.clone());
let standalone = GreptimeDbStandaloneBuilder::new("test_exec_plan_interceptor_plugin")
.with_plugin(plugins)
.build()
.await;
let instance = standalone.fe_instance().clone();
let sql = r#"CREATE TABLE demo(
host STRING,
ts TIMESTAMP,
cpu DOUBLE NULL,
TIME INDEX (ts),
PRIMARY KEY(host)
) engine=mito;"#;
SqlQueryHandler::do_query(&*instance, sql, QueryContext::arc())
.await
.remove(0)
.unwrap();
let query_ctx = QueryContext::arc();
let stmt = QueryLanguageParser::parse_sql("SELECT * FROM demo", &query_ctx).unwrap();
let plan = instance
.statement_executor()
.plan(&stmt, query_ctx.clone())
.await
.unwrap();
let QueryStatement::Sql(sql_stmt) = stmt else {
unreachable!()
};
SqlQueryHandler::do_exec_plan(&*instance, plan, Some(sql_stmt), query_ctx.clone())
.await
.unwrap();
assert!(
hook.pre_execute_called.load(Ordering::Relaxed),
"pre_execute should be called for do_exec_plan"
);
assert!(
hook.post_execute_called.load(Ordering::Relaxed),
"post_execute should be called for do_exec_plan"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn test_disable_db_operation_plugin() {
#[derive(Default)]
struct DisableDBOpHook;
impl SqlQueryInterceptor for DisableDBOpHook {
type Error = Error;
fn post_parsing(
&self,
statements: Vec<Statement>,
_query_ctx: QueryContextRef,
) -> Result<Vec<Statement>> {
for s in &statements {
match s {
Statement::CreateDatabase(_) | Statement::ShowDatabases(_) => {
return Err(Error::NotSupported {
feat: "Database operations".to_owned(),
});
}
_ => {}
}
}
Ok(statements)
}
}
let query_ctx = QueryContext::arc();
let plugins = Plugins::new();
let hook = Arc::new(DisableDBOpHook);
plugins.insert::<SqlQueryInterceptorRef<Error>>(hook.clone());
let standalone = GreptimeDbStandaloneBuilder::new("test_disable_db_operation_plugin")
.with_plugin(plugins)
.build()
.await;
let instance = standalone.fe_instance().clone();
let sql = r#"CREATE TABLE demo(
host STRING,
ts TIMESTAMP,
cpu DOUBLE NULL,
memory DOUBLE NULL,
disk_util DOUBLE DEFAULT 9.9,
TIME INDEX (ts),
PRIMARY KEY(host)
) engine=mito;"#;
let output = SqlQueryHandler::do_query(&*instance, sql, query_ctx.clone())
.await
.remove(0)
.unwrap();
match output.data {
OutputData::AffectedRows(rows) => assert_eq!(rows, 0),
_ => unreachable!(),
}
let sql = r#"CREATE DATABASE tomcat"#;
if let Err(e) = SqlQueryHandler::do_query(&*instance, sql, query_ctx.clone())
.await
.remove(0)
{
assert_eq!(e.status_code(), StatusCode::Unsupported);
} else {
unreachable!();
}
let sql = r#"SELECT 1; SHOW DATABASES"#;
if let Err(e) = SqlQueryHandler::do_query(&*instance, sql, query_ctx.clone())
.await
.remove(0)
{
assert_eq!(e.status_code(), StatusCode::Unsupported);
} else {
unreachable!();
}
}
#[tokio::test(flavor = "multi_thread")]
async fn test_exemplar_authorization_and_interception() {
use common_catalog::consts::METRICS_EXEMPLARS_TABLE;
use servers::exemplar::query::Query;
use servers::interceptor::{PromQueryInterceptor, PromQueryInterceptorRef};
use servers::prometheus_handler::PrometheusHandler;
use std::sync::Mutex;
use std::sync::atomic::{AtomicBool, Ordering};
use table::table_reference::TableReference;
#[derive(Default)]
struct Hook {
tables: Mutex<Vec<String>>,
reject: AtomicBool,
}
impl PromQueryInterceptor for Hook {
type Error = servers::error::Error;
fn pre_execute_exemplars(
&self,
query: &Query,
table: TableReference<'_>,
ctx: QueryContextRef,
) -> servers::error::Result<()> {
assert_eq!(query.bounds, (0, 1_000_999_999));
assert!(query.expression.starts_with("metric"));
assert_eq!(table.catalog, ctx.current_catalog());
assert_eq!(table.schema, ctx.current_schema());
assert_eq!(table.table, METRICS_EXEMPLARS_TABLE);
self.tables.lock().unwrap().push(table.to_string());
if self.reject.load(Ordering::Relaxed) {
return servers::error::NotSupportedSnafu {
feat: "intercepted exemplar query",
}
.fail();
}
Ok(())
}
}
struct Checker(Mutex<Vec<auth::PermissionTableTargets>>);
impl auth::PermissionChecker for Checker {
fn check_permission(
&self,
_: auth::UserInfoRef,
_: auth::PermissionReq,
) -> auth::error::Result<auth::PermissionResp> {
Ok(auth::PermissionResp::Allow)
}
fn check_permission_with_table_targets(
&self,
_: auth::UserInfoRef,
req: auth::PermissionReq,
targets: auth::PermissionTableTargets,
) -> auth::error::Result<auth::PermissionResp> {
if !matches!(req, auth::PermissionReq::Action(auth::PROMQL_QUERY)) {
return Ok(auth::PermissionResp::Allow);
}
self.0.lock().unwrap().push(targets.clone());
let denied = match targets {
auth::PermissionTableTargets::Unresolved => true,
auth::PermissionTableTargets::Resolved(targets) => targets.iter().any(|t| {
t.schema == "secret" && t.table == "metric"
|| t.schema == "shared_denied" && t.table == METRICS_EXEMPLARS_TABLE
}),
};
Ok(if denied {
auth::PermissionResp::Reject
} else {
auth::PermissionResp::Allow
})
}
}
let checker = Arc::new(Checker(Mutex::new(Vec::new())));
let hook = Arc::new(Hook::default());
let plugins = Plugins::new();
plugins.insert::<auth::PermissionCheckerRef>(checker.clone());
plugins.insert::<PromQueryInterceptorRef<servers::error::Error>>(hook.clone());
let standalone = GreptimeDbStandaloneBuilder::new("exemplar_query_auth")
.with_plugin(plugins)
.build()
.await;
let instance = standalone.fe_instance();
query(instance, "CREATE DATABASE other").await;
for schema in ["public", "other"] {
create_table(instance, &format!("CREATE TABLE {schema}.{METRICS_EXEMPLARS_TABLE} (__table_id INT UNSIGNED NOT NULL, __tsid BIGINT UNSIGNED NOT NULL, greptime_timestamp TIMESTAMP(9) TIME INDEX, greptime_metric STRING, greptime_value DOUBLE, exemplar_labels JSON2, PRIMARY KEY (__table_id, __tsid, greptime_metric)) WITH ('append_mode'='true', 'primary_key_encoding'='sparse')")).await;
}
for expression in [
"rate(metric{__schema__=\"public\"}[5m]) + metric{__database__=\"secret\"}",
"metric{__schema__=\"public\" or __schema__=\"secret\"}",
"metric{__database__=\"shared_denied\"}",
"{__name__=~\"metric.*\",__schema__=\"secret\"}",
"topk(scalar(metric{__schema__=\"secret\"}), metric)",
] {
checker.0.lock().unwrap().clear();
let query = Query::parse(expression.into(), "0", "1", "public").unwrap();
let error = instance
.query_exemplars(query, QueryContext::arc())
.await
.unwrap_err();
assert_eq!(
error.status_code(),
StatusCode::PermissionDenied,
"{expression}: {error:?}"
);
assert!(!checker.0.lock().unwrap().is_empty());
assert!(hook.tables.lock().unwrap().is_empty());
}
let query = Query::parse(
"metric{__schema__=\"public\" or __database__=\"other\"}".into(),
"0",
"1",
"public",
)
.unwrap();
checker.0.lock().unwrap().clear();
assert!(
instance
.query_exemplars(query, QueryContext::arc())
.await
.unwrap()
.is_empty()
);
let checks = checker.0.lock().unwrap().clone();
for schema in ["public", "other"] {
for table in ["metric", METRICS_EXEMPLARS_TABLE] {
assert!(checks.iter().any(|c| matches!(c, auth::PermissionTableTargets::Resolved(t) if t.iter().any(|t| t.schema == schema && t.table == table))));
}
}
assert_eq!(
*hook.tables.lock().unwrap(),
[
format!("greptime.other.{METRICS_EXEMPLARS_TABLE}"),
format!("greptime.public.{METRICS_EXEMPLARS_TABLE}"),
],
);
hook.tables.lock().unwrap().clear();
hook.reject.store(true, Ordering::Relaxed);
let error = instance
.query_exemplars(
Query::parse("metric".into(), "0", "1", "public").unwrap(),
QueryContext::arc(),
)
.await
.unwrap_err();
assert!(matches!(
error,
servers::error::Error::NotSupported { feat, .. }
if feat == "intercepted exemplar query"
));
assert_eq!(hook.tables.lock().unwrap().len(), 1);
hook.tables.lock().unwrap().clear();
assert!(
instance
.query_exemplars(
Query::parse("metric{__schema__=\"missing\"}".into(), "0", "1", "public")
.unwrap(),
QueryContext::arc(),
)
.await
.unwrap()
.is_empty()
);
assert!(hook.tables.lock().unwrap().is_empty());
}
/// The entity-graph derivation must run as the caller: a source table the
/// caller cannot read contributes neither entities nor edges.
#[tokio::test(flavor = "multi_thread")]
async fn test_entity_graph_derivation_runs_as_caller() {
common_telemetry::init_default_ut_logging();
/// Denies the entity-graph derivation on `secret_traces` only.
struct DenySecretTraces;
impl auth::PermissionChecker for DenySecretTraces {
fn check_permission(
&self,
_user_info: auth::UserInfoRef,
_req: auth::PermissionReq,
) -> auth::error::Result<auth::PermissionResp> {
Ok(auth::PermissionResp::Allow)
}
fn check_permission_with_table_targets(
&self,
_user_info: auth::UserInfoRef,
req: auth::PermissionReq,
targets: auth::PermissionTableTargets,
) -> auth::error::Result<auth::PermissionResp> {
if matches!(req, auth::PermissionReq::Action(auth::SEMANTIC_GRAPH_QUERY))
&& let auth::PermissionTableTargets::Resolved(targets) = targets
&& targets
.iter()
.any(|target| target.table.starts_with("secret_"))
{
return Ok(auth::PermissionResp::Reject);
}
Ok(auth::PermissionResp::Allow)
}
}
let plugins = Plugins::new();
plugins.insert::<auth::PermissionCheckerRef>(Arc::new(DenySecretTraces));
let standalone = GreptimeDbStandaloneBuilder::new("test_entity_graph_runs_as_caller")
.with_plugin(plugins)
.build()
.await;
let instance = standalone.fe_instance().clone();
for (table, trace_id) in [("open_traces", "t0"), ("secret_traces", "t1")] {
let create = format!(
r#"create table {table} (
"timestamp" timestamp(9) time index,
trace_id string,
span_id string,
parent_span_id string,
span_kind string,
span_status_code string,
service_name string,
duration_nano bigint unsigned,
primary key (service_name)
) with ('table_data_model' = 'greptime_trace_v1', 'append_mode' = 'true')"#
);
create_table(&instance, &create).await;
let insert = format!(
"insert into {table} values \
(now(), '{trace_id}', 'c1', NULL, 'SPAN_KIND_CLIENT', 'STATUS_CODE_UNSET', 'client-{table}', 0), \
(now(), '{trace_id}', 's1', 'c1', 'SPAN_KIND_SERVER', 'STATUS_CODE_UNSET', 'server-{table}', 100)"
);
let output = query(&instance, &insert).await;
let OutputData::AffectedRows(x) = output.data else {
unreachable!()
};
assert_eq!(x, 2);
}
// A pair split across tables (client here, server in a denied table):
// a join-derived edge needs read access to all of its input tables.
create_table(
&instance,
r#"create table cross_clients (
"timestamp" timestamp(9) time index,
trace_id string,
span_id string,
parent_span_id string,
span_kind string,
span_status_code string,
service_name string,
duration_nano bigint unsigned,
primary key (service_name)
) with ('table_data_model' = 'greptime_trace_v1', 'append_mode' = 'true')"#,
)
.await;
create_table(
&instance,
r#"create table secret_cross_servers (
"timestamp" timestamp(9) time index,
trace_id string,
span_id string,
parent_span_id string,
span_kind string,
span_status_code string,
service_name string,
duration_nano bigint unsigned,
primary key (service_name)
) with ('table_data_model' = 'greptime_trace_v1', 'append_mode' = 'true')"#,
)
.await;
for insert in [
"insert into cross_clients values \
(now(), 't9', 'c9', NULL, 'SPAN_KIND_CLIENT', 'STATUS_CODE_UNSET', 'client-cross', 0)",
"insert into secret_cross_servers values \
(now(), 't9', 's9', 'c9', 'SPAN_KIND_SERVER', 'STATUS_CODE_UNSET', 'server-cross', 100)",
] {
let output = query(&instance, insert).await;
let OutputData::AffectedRows(x) = output.data else {
unreachable!()
};
assert_eq!(x, 1);
}
let sql = "select src_id, dst_id from greptime_private.semantic_relationships \
order by src_id";
let output = query(&instance, sql).await;
let OutputData::Stream(s) = output.data else {
unreachable!()
};
let batches = common_recordbatch::util::collect_batches(s).await.unwrap();
let pretty_print = batches.pretty_print().unwrap();
assert!(
pretty_print.contains("client-open_traces"),
"allowed edge missing:\n{pretty_print}"
);
assert!(
!pretty_print.contains("secret_traces"),
"denied source leaked into edges:\n{pretty_print}"
);
assert!(
!pretty_print.contains("client-cross"),
"edge with a denied join side leaked:\n{pretty_print}"
);
let sql = "select entity_id from greptime_private.semantic_entities order by entity_id";
let output = query(&instance, sql).await;
let OutputData::Stream(s) = output.data else {
unreachable!()
};
let batches = common_recordbatch::util::collect_batches(s).await.unwrap();
let pretty_print = batches.pretty_print().unwrap();
assert!(
pretty_print.contains("client-open_traces"),
"allowed entity missing:\n{pretty_print}"
);
assert!(
!pretty_print.contains("secret_traces") && !pretty_print.contains("server-cross"),
"denied source leaked into entities:\n{pretty_print}"
);
}
/// Denying the descriptor's table must not fail the request: the metric
/// data is already committed, so the client would retry what the server
/// accepted.
#[tokio::test(flavor = "multi_thread")]
async fn test_otlp_descriptor_permission_denial_degrades() {
use otel_arrow_rust::proto::opentelemetry::collector::metrics::v1::ExportMetricsServiceRequest;
use otel_arrow_rust::proto::opentelemetry::common::v1::{AnyValue, KeyValue, any_value};
use otel_arrow_rust::proto::opentelemetry::metrics::v1::number_data_point::Value;
use otel_arrow_rust::proto::opentelemetry::metrics::v1::{
Gauge, Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics, metric,
};
use otel_arrow_rust::proto::opentelemetry::resource::v1::Resource;
use servers::query_handler::OpenTelemetryProtocolHandler;
common_telemetry::init_default_ut_logging();
/// Denies writes to the descriptor table only.
struct DenyResourceInfo;
impl auth::PermissionChecker for DenyResourceInfo {
fn check_permission(
&self,
_user_info: auth::UserInfoRef,
_req: auth::PermissionReq,
) -> auth::error::Result<auth::PermissionResp> {
Ok(auth::PermissionResp::Allow)
}
fn check_permission_with_table_targets(
&self,
_user_info: auth::UserInfoRef,
_req: auth::PermissionReq,
targets: auth::PermissionTableTargets,
) -> auth::error::Result<auth::PermissionResp> {
if let auth::PermissionTableTargets::Resolved(targets) = targets
&& targets
.iter()
.any(|target| target.table == "greptime_otel_resource_info")
{
return Ok(auth::PermissionResp::Reject);
}
Ok(auth::PermissionResp::Allow)
}
}
let plugins = Plugins::new();
plugins.insert::<auth::PermissionCheckerRef>(Arc::new(DenyResourceInfo));
let standalone = GreptimeDbStandaloneBuilder::new("test_otlp_descriptor_denied")
.with_plugin(plugins)
.build()
.await;
let instance = standalone.fe_instance().clone();
let attr = |key: &str, value: &str| KeyValue {
key: key.to_string(),
value: Some(AnyValue {
value: Some(any_value::Value::StringValue(value.to_string())),
}),
};
let request = ExportMetricsServiceRequest {
resource_metrics: vec![ResourceMetrics {
resource: Some(Resource {
attributes: vec![attr("service.name", "api"), attr("host.id", "h-1")],
..Default::default()
}),
scope_metrics: vec![ScopeMetrics {
metrics: vec![Metric {
name: "denied_descriptor_gauge".to_string(),
data: Some(metric::Data::Gauge(Gauge {
data_points: vec![NumberDataPoint {
time_unix_nano: 1_700_000_000_000_000_000,
value: Some(Value::AsDouble(1.0)),
..Default::default()
}],
})),
..Default::default()
}],
..Default::default()
}],
..Default::default()
}],
};
let errors = frontend::OTLP_RESOURCE_INFO_WRITE_ERRORS.get();
let outcome =
OpenTelemetryProtocolHandler::metrics(&*instance, request, QueryContext::arc(), None)
.await
.unwrap();
assert_eq!(
(outcome.accepted_data_points, outcome.rejected_data_points),
(1, 0)
);
assert_eq!(frontend::OTLP_RESOURCE_INFO_WRITE_ERRORS.get(), errors + 1);
let warning = outcome
.error_message
.expect("denial must surface as a warning");
assert!(
warning.contains("greptime_otel_resource_info"),
"unexpected warning: {warning}"
);
// the metric data is committed, the descriptor table was never created
let sql = "select table_name from information_schema.tables \
where table_schema = 'public' order by table_name";
let output = query(&instance, sql).await;
let OutputData::Stream(s) = output.data else {
unreachable!()
};
let batches = common_recordbatch::util::collect_batches(s).await.unwrap();
let pretty_print = batches.pretty_print().unwrap();
assert!(
pretty_print.contains("denied_descriptor_gauge"),
"metric data was not committed:\n{pretty_print}"
);
assert!(
!pretty_print.contains("greptime_otel_resource_info"),
"denied descriptor table was created:\n{pretty_print}"
);
}
}