diff --git a/config/config.md b/config/config.md
index 0151d505092..2ed75b08342 100644
--- a/config/config.md
+++ b/config/config.md
@@ -144,7 +144,7 @@
| `procedure.retry_delay` | String | `500ms` | Initial retry delay of procedures, increases exponentially |
| `procedure.max_running_procedures` | Integer | `128` | Max running procedures.
The maximum number of procedures that can be running at the same time.
If the number of running procedures exceeds this limit, the procedure will be rejected. |
| `query` | -- | -- | The query engine options. |
-| `query.experimental_native_promql` | Bool | `false` | Enable experimental native PromQL execution for compatible queries.
Requires the per-request query.experimental_native_promql=true extension.
Default: false |
+| `query.experimental_native_promql` | Bool | `false` | Enable experimental native PromQL execution for compatible queries.
This is the engine-wide default: a session
SET query.experimental_native_promql = true or a per-request
query.experimental_native_promql option overrides it, in that order.
Default: false |
| `query.parallelism` | Integer | `0` | Parallelism of the query engine.
Default to 0, which means the number of CPU cores. |
| `query.memory_pool_size` | String | 0 | Memory pool size for query execution operators (aggregation, sorting, join,
and internal series merges during table scans).
Supports absolute size (e.g., "2GB", "4GB") or percentage of system memory (e.g., "20%").
Setting it to 0 disables the limit. Defaults to 0 (unlimited); this example sets it to `50%`.
When this limit is reached, queries will fail with ResourceExhausted error.
Projected Parquet row groups and streamed scan batches use scan_memory_limit separately. |
| `query.experimental_memory_pool_policy` | String | `greedy` | Experimental memory pool allocation policy:
- "greedy" (default): first-come-first-served allocation; preserves current behavior.
- "fair": divides available memory among spillable operators and may spill earlier.
Only effective when `memory_pool_size` is bounded (>0). |
@@ -396,7 +396,7 @@
| `meta_client.metadata_cache_ttl` | String | `10m` | TTL of the metadata cache. |
| `meta_client.metadata_cache_tti` | String | `5m` | -- |
| `query` | -- | -- | The query engine options. |
-| `query.experimental_native_promql` | Bool | `false` | Enable experimental native PromQL execution for compatible standalone queries.
Requires in-process region transport and the per-request
query.experimental_native_promql=true extension. Default: false |
+| `query.experimental_native_promql` | Bool | `false` | Enable experimental native PromQL execution for compatible standalone queries.
Requires in-process region transport. This is the engine-wide default: a
session SET query.experimental_native_promql = true or a per-request
query.experimental_native_promql option overrides it, in that order.
Default: false |
| `query.parallelism` | Integer | `0` | Parallelism of the query engine.
Default to 0, which means the number of CPU cores. |
| `query.allow_query_fallback` | Bool | `false` | Whether to allow query fallback when push down optimize fails.
Default to false, meaning when push down optimize failed, return error msg |
| `query.memory_pool_size` | String | 0 | Memory pool size for query execution operators (aggregation, sorting, join).
Supports absolute size (e.g., "4GB", "8GB") or percentage of system memory (e.g., "30%").
Setting it to 0 disables the limit. Defaults to 0 (unlimited); this example sets it to `50%`.
When this limit is reached, queries will fail with ResourceExhausted error.
This pool applies to operators on this frontend. Datanodes use their own query
pools for local operators and internal series merges during table scans. |
diff --git a/config/frontend.example.toml b/config/frontend.example.toml
index f3be55fcaa0..37750e07c4d 100644
--- a/config/frontend.example.toml
+++ b/config/frontend.example.toml
@@ -372,8 +372,10 @@ metadata_cache_tti = "5m"
[query]
## Enable experimental native PromQL execution for compatible standalone queries.
-## Requires in-process region transport and the per-request
-## query.experimental_native_promql=true extension. Default: false
+## Requires in-process region transport. This is the engine-wide default: a
+## session SET query.experimental_native_promql = true or a per-request
+## query.experimental_native_promql option overrides it, in that order.
+## Default: false
#+ experimental_native_promql = false
## Parallelism of the query engine.
## Default to 0, which means the number of CPU cores.
diff --git a/config/standalone.example.toml b/config/standalone.example.toml
index f34a881af3d..ae0a1395bfd 100644
--- a/config/standalone.example.toml
+++ b/config/standalone.example.toml
@@ -565,7 +565,9 @@ max_running_procedures = 128
[query]
## Enable experimental native PromQL execution for compatible queries.
-## Requires the per-request query.experimental_native_promql=true extension.
+## This is the engine-wide default: a session
+## SET query.experimental_native_promql = true or a per-request
+## query.experimental_native_promql option overrides it, in that order.
## Default: false
#+ experimental_native_promql = false
## Parallelism of the query engine.
diff --git a/src/frontend/src/instance.rs b/src/frontend/src/instance.rs
index 973e82d62bf..3727cdec595 100644
--- a/src/frontend/src/instance.rs
+++ b/src/frontend/src/instance.rs
@@ -1444,12 +1444,22 @@ impl Instance {
expr: &promql_parser::parser::Expr,
query_ctx: &mut QueryContextRef,
) -> server_error::Result<()> {
- // Discovery scope is consumed by the native PromQL route only. The extension
- // is deliberately opt-in, matching `native_promql_requested` in query's planner.
- if !query_ctx
- .extension(query::promql::planner::NATIVE_PROMQL_EXTENSION_KEY)
- .is_some_and(|value| value.eq_ignore_ascii_case("true"))
- {
+ // Discovery scope is read by the native PromQL route only, so a query that does not ask
+ // for it pays nothing here beyond resolving the switch: the value is read through the
+ // query engine's own resolver, which takes what the query states (a request hint, or the
+ // session `SET` it inherits) over the engine default - the same two sources planning
+ // reads, so the two cannot disagree. A route that is off leaves discovery unrestricted,
+ // which is what it was before the switch existed; a route that is on is bounded below
+ // whether or not the fallback option later keeps the plan on the existing route.
+ let native_requested = query::promql::planner::native_promql_resolved(
+ query_ctx,
+ self.query_engine
+ .engine_state()
+ .experimental_native_promql(),
+ )
+ .map_err(BoxedError::new)
+ .context(ExecuteQuerySnafu)?;
+ if !native_requested {
Arc::make_mut(query_ctx).set_metric_discovery_scope(MetricDiscoveryScope::Unrestricted);
return Ok(());
}
@@ -2431,6 +2441,44 @@ mod tests {
}
}
+ /// A checker that authorizes per table and counts the grants it was asked for, so a test can
+ /// state that a path did (or did not) ask.
+ #[derive(Default)]
+ struct CountingPermissionChecker {
+ table_target_checks: atomic::AtomicUsize,
+ }
+
+ impl CountingPermissionChecker {
+ fn table_target_checks(&self) -> usize {
+ self.table_target_checks.load(atomic::Ordering::Relaxed)
+ }
+ }
+
+ impl PermissionChecker for CountingPermissionChecker {
+ fn check_permission(
+ &self,
+ _user_info: UserInfoRef,
+ _req: PermissionReq,
+ ) -> auth::error::Result {
+ Ok(PermissionResp::Allow)
+ }
+
+ fn check_permission_with_table_targets(
+ &self,
+ _user_info: UserInfoRef,
+ _req: PermissionReq,
+ _targets: PermissionTableTargets,
+ ) -> auth::error::Result {
+ self.table_target_checks
+ .fetch_add(1, atomic::Ordering::Relaxed);
+ Ok(PermissionResp::Allow)
+ }
+
+ fn uses_table_targets(&self) -> bool {
+ true
+ }
+ }
+
struct RejectUnresolvedPermissionChecker;
impl PermissionChecker for RejectUnresolvedPermissionChecker {
@@ -5745,6 +5793,181 @@ mod tests {
.unwrap();
assert!(found.is_some());
+ Ok(())
+ }
+ const NATIVE_PROMQL_OPTION: &str = "query.experimental_native_promql";
+
+ /// A frontend whose `public` schema holds two metric-engine logical tables, `cpu_usage` and
+ /// `target`, with `checker` as its permission checker.
+ async fn test_metric_discovery_instance(checker: PermissionCheckerRef) -> TestResult {
+ let plugins = Plugins::new();
+ plugins.insert::(checker);
+ test_instance_with_plugins_and_metric_names(
+ test_logical_table(1024, "cpu_usage")?,
+ test_logical_table(1025, "target")?,
+ plugins,
+ None,
+ )
+ .await
+ }
+
+ /// An expression that has to be answered by discovering the schema's metric tables: it names
+ /// no exact metric, only a name matcher.
+ fn discovery_metric_expr() -> promql_parser::parser::Expr {
+ promql_parser::parser::parse(r#"{__name__=~"cpu.*"}"#).unwrap()
+ }
+
+ /// The tables a bounded scope allows, sorted so a test can state them.
+ fn allowed_tables(scope: &MetricDiscoveryScope) -> Vec {
+ let MetricDiscoveryScope::Allowed(allowed) = scope else {
+ panic!("expected a table-bounded discovery scope, got {scope:?}");
+ };
+ let mut tables = allowed
+ .iter()
+ .map(|target| target.table.clone())
+ .collect::>();
+ tables.sort_unstable();
+ tables
+ }
+
+ /// A session `SET` turns the native route on without any request hint, so the scope has to be
+ /// bounded by the caller's grants in exactly that case too.
+ #[tokio::test]
+ async fn test_discovery_scope_bounds_a_session_enabled_native_route() -> TestResult<()> {
+ let instance =
+ test_metric_discovery_instance(Arc::new(RejectUnresolvedPermissionChecker)).await?;
+ let mut ctx = test_query_ctx(1);
+ ctx.configuration_parameter()
+ .set_query_option(NATIVE_PROMQL_OPTION, "true")
+ .unwrap();
+ assert!(
+ ctx.extension(NATIVE_PROMQL_OPTION).is_none(),
+ "a session SET is not a request hint"
+ );
+
+ instance
+ .prepare_metric_discovery_scope(&discovery_metric_expr(), &mut ctx)
+ .await
+ .unwrap();
+
+ assert_eq!(
+ vec!["cpu_usage".to_string(), "target".to_string()],
+ allowed_tables(ctx.metric_discovery_scope())
+ );
+
+ Ok(())
+ }
+
+ /// With the switch off and nothing stated, the route never reads discovery, so the scope stays
+ /// unrestricted and no per-table grant is asked for: a query that does not use the route pays
+ /// nothing here.
+ #[tokio::test]
+ async fn test_discovery_scope_is_not_computed_when_the_route_is_off() -> TestResult<()> {
+ let counter = Arc::new(CountingPermissionChecker::default());
+ let instance = test_metric_discovery_instance(counter.clone()).await?;
+ let mut ctx = test_query_ctx(1);
+ assert!(ctx.extension(NATIVE_PROMQL_OPTION).is_none());
+ assert_eq!(
+ None,
+ ctx.effective_query_options()
+ .unwrap()
+ .get(NATIVE_PROMQL_OPTION),
+ "nothing states the option here"
+ );
+
+ instance
+ .prepare_metric_discovery_scope(&discovery_metric_expr(), &mut ctx)
+ .await
+ .unwrap();
+
+ assert!(
+ matches!(
+ ctx.metric_discovery_scope(),
+ MetricDiscoveryScope::Unrestricted
+ ),
+ "a route that is off reads no scope: {:?}",
+ ctx.metric_discovery_scope()
+ );
+ assert_eq!(
+ 0,
+ counter.table_target_checks(),
+ "an off route asked for per-table grants"
+ );
+ assert!(
+ ctx.extension(NATIVE_PROMQL_OPTION).is_none(),
+ "resolving the switch must not state the option"
+ );
+
+ Ok(())
+ }
+
+ /// A checker that decides without table targets needs no bound: there is nothing to narrow.
+ #[tokio::test]
+ async fn test_discovery_scope_stays_unrestricted_without_table_targets() -> TestResult<()> {
+ let instance =
+ test_metric_discovery_instance(Arc::new(TargetIndependentPermissionChecker::default()))
+ .await?;
+ let mut ctx = test_query_ctx(1);
+
+ instance
+ .prepare_metric_discovery_scope(&discovery_metric_expr(), &mut ctx)
+ .await
+ .unwrap();
+
+ assert!(
+ matches!(
+ ctx.metric_discovery_scope(),
+ MetricDiscoveryScope::Unrestricted
+ ),
+ "got {:?}",
+ ctx.metric_discovery_scope()
+ );
+
+ Ok(())
+ }
+
+ /// Resolving the switch must not state it: the option a query inherits stays inherited, so
+ /// nothing this function does can change what planning later resolves. With nothing stated and
+ /// the engine default off the route is off, which leaves discovery unrestricted - the scope is
+ /// the only thing this function changes, and the route reads the option, not the scope.
+ ///
+ /// The routing itself is asserted where the plan is built
+ /// (`query::planner`'s `test_engine_option_is_the_default_a_request_or_session_overrides`
+ /// covers the "nothing stated, engine default" case): a catalog of plain in-memory tables
+ /// cannot plan a discovery expression natively, so a frontend-level plan assertion would only
+ /// restate the legacy planner's own rejections.
+ #[tokio::test]
+ async fn test_discovery_scope_does_not_state_the_native_option() -> TestResult<()> {
+ let instance =
+ test_metric_discovery_instance(Arc::new(RejectUnresolvedPermissionChecker)).await?;
+ let mut ctx = test_query_ctx(1);
+ assert!(ctx.extension(NATIVE_PROMQL_OPTION).is_none());
+
+ instance
+ .prepare_metric_discovery_scope(&discovery_metric_expr(), &mut ctx)
+ .await
+ .unwrap();
+
+ assert!(
+ matches!(
+ ctx.metric_discovery_scope(),
+ MetricDiscoveryScope::Unrestricted
+ ),
+ "an off route reads no scope: {:?}",
+ ctx.metric_discovery_scope()
+ );
+ assert!(
+ ctx.extension(NATIVE_PROMQL_OPTION).is_none(),
+ "computing the scope must not state the option"
+ );
+ assert_eq!(
+ None,
+ ctx.effective_query_options()
+ .unwrap()
+ .get(NATIVE_PROMQL_OPTION),
+ "the route stays on the engine default"
+ );
+
Ok(())
}
}
diff --git a/src/query/src/options.rs b/src/query/src/options.rs
index 39ef4233480..fb058d56bda 100644
--- a/src/query/src/options.rs
+++ b/src/query/src/options.rs
@@ -111,9 +111,12 @@ pub struct QueryOptions {
/// evenly among spillable reservations and may trigger earlier spills.
pub experimental_memory_pool_policy: QueryMemoryPoolPolicy,
/// Enables the experimental native PromQL execution path for compatible queries. This
- /// path currently applies only to standalone queries using in-process region transport,
- /// and also requires the per-request `query.experimental_native_promql=true` extension.
- /// Default: `false`.
+ /// path currently applies only to standalone queries using in-process region transport.
+ ///
+ /// This is the engine-wide default. A session `SET query.experimental_native_promql = true`
+ /// or a per-request `query.experimental_native_promql` option overrides it, in that order, so
+ /// a `true` default routes every query natively and a `false` one still lets one query or one
+ /// session ask for it. Default: `false`.
pub experimental_native_promql: bool,
}
@@ -453,6 +456,57 @@ impl ExtensionOptions for ScheduledTimeExtension {
}
}
+/// Carries the native PromQL route switch a query asked for through
+/// [`ConfigOptions::extensions`].
+///
+/// The value is resolved once per query, where the query options snapshot is applied to the
+/// planning state: the request extension
+/// [`NATIVE_PROMQL_EXTENSION_KEY`](crate::promql::planner::NATIVE_PROMQL_EXTENSION_KEY) over a
+/// session `SET` of the same option over the engine-wide
+/// [`QueryOptions::experimental_native_promql`] default. Planning and `SHOW` both read it back from
+/// there, so a query that states nothing routes as the engine default says.
+///
+/// What it records is the *request*, not the plan the query got: planning still applies the
+/// conditions that decide whether the route can be taken at all (a statement that allows query
+/// fallback or one planned inside a CTE keeps the existing plan whatever this says), so `SHOW
+/// query.experimental_native_promql` answers what the query asked for.
+#[derive(Debug, Clone, Copy)]
+pub(crate) struct NativePromqlExtension {
+ pub(crate) requested: bool,
+}
+
+impl ConfigExtension for NativePromqlExtension {
+ const PREFIX: &'static str = "native_promql";
+}
+
+impl ExtensionOptions for NativePromqlExtension {
+ fn as_any(&self) -> &dyn std::any::Any {
+ self
+ }
+
+ fn as_any_mut(&mut self) -> &mut dyn std::any::Any {
+ self
+ }
+
+ fn cloned(&self) -> Box {
+ Box::new(*self)
+ }
+
+ fn set(&mut self, key: &str, value: &str) -> datafusion::error::Result<()> {
+ Err(datafusion_common::DataFusionError::NotImplemented(format!(
+ "NativePromqlExtension does not support set key: {key} with value: {value}"
+ )))
+ }
+
+ fn entries(&self) -> Vec {
+ vec![ConfigEntry {
+ key: "requested".to_string(),
+ value: Some(self.requested.to_string()),
+ description: "Whether the query requests the native fused-series PromQL route",
+ }]
+ }
+}
+
fn invalid_query_context_extension(reason: String) -> Error {
InvalidQueryContextExtensionSnafu { reason }.build()
}
diff --git a/src/query/src/planner.rs b/src/query/src/planner.rs
index af68a9d70c9..2ae7525167c 100644
--- a/src/query/src/planner.rs
+++ b/src/query/src/planner.rs
@@ -59,7 +59,7 @@ use crate::error::{
};
use crate::log_query::planner::LogQueryPlanner;
use crate::optimizer::expr_depth;
-use crate::options::ScheduledTimeExtension;
+use crate::options::{NativePromqlExtension, ScheduledTimeExtension};
use crate::parser::{DEFAULT_LOOKBACK_STRING, PromQuery, QueryLanguageParser, QueryStatement};
use crate::promql::planner::PromPlanner;
use crate::query_engine::{DefaultPlanDecoder, QueryEngineState};
@@ -702,14 +702,13 @@ impl DfLogicalPlanner {
}
/// Plans a PromQL statement, offering it to the native fused-series route when the statement
- /// may take it and the state it is planned with asks for it.
+ /// may take it and the query options it is planned with ask for it.
///
/// `native_route_allowed` states whether the statement may take the route at all: a PromQL
/// statement embedded in a CTE may not (see [`Self::tql_to_logical_plan`]), every other one
- /// may. The remaining switches are read from the state the statement is planned with, after
- /// `apply_query_options` has resolved the per-query options into it: the fallback option,
- /// which decides whether a fallback rewriter may wrap the plan, and the engine-wide native
- /// option together with the per-request native extension.
+ /// may. Everything else the route depends on is read from the query options snapshot the
+ /// statement is planned with, where every option is already resolved as request > session
+ /// `SET` > engine default.
#[tracing::instrument(skip_all)]
async fn plan_pql(
&self,
@@ -724,22 +723,23 @@ impl DfLogicalPlanner {
// rewriter can wrap a native region scan in an unusable MergeScan, so the native route is
// only offered to a query that cannot fall back.
//
- // TODO(narthex-merge): this used to disjoin three sources - the engine option, the session
- // `allow_query_fallback` variable and the `query_fallback` hint - on the assumption that
- // each of them only ever turns the fallback on. `apply_query_options` above now resolves
- // all of them from the query option snapshot into this `DistPlannerOptions` extension,
- // which is why the read happens here instead. One semantics is kept from main rather than
- // narthex: a stated `false` overrides the engine default instead of leaving it on.
+ // `apply_query_options` above resolved the three sources this used to disjoin - the engine
+ // option, the session `allow_query_fallback` variable and the `query_fallback` hint - into
+ // this extension, so the read happens here instead. One mainline semantic is kept: a stated
+ // `false` overrides the engine default rather than leaving it on.
let allow_query_fallback = scheduled_state
.config_options()
.extensions
.get::()
.is_some_and(|options| options.allow_query_fallback);
// Whether the statement is offered to the native fused-series compiler. This is the second,
- // independent switch: the engine-wide option and the per-request extension it is requested
- // with, and no source of it is derived from the fallback option.
- let native_promql = self.engine_state.experimental_native_promql()
- && crate::promql::planner::native::native_promql_requested(&query_ctx);
+ // independent switch: it states what the query asks for, and no source of it is derived
+ // from the fallback option.
+ let native_promql = scheduled_state
+ .config_options()
+ .extensions
+ .get::()
+ .is_some_and(|options| options.requested);
let promql_annotations = query_ctx.remote_query_id().map(promql_annotation_collector);
if let Some(collector) = &promql_annotations {
scheduled_state
@@ -1465,10 +1465,7 @@ mod tests {
// The route a deployed frontend plans: a distributed planner over its regions.
true,
);
- let mut builder = QueryContextBuilder::default()
- // The native route request reaches the planner as a query context extension, the way
- // a client sends it.
- .set_extension(NATIVE_PROMQL_EXTENSION_KEY.to_string(), "true".to_string());
+ let mut builder = QueryContextBuilder::default();
if let Some((key, value)) = request {
builder = builder.set_extension(key.to_string(), value.to_string());
}
@@ -1501,6 +1498,45 @@ mod tests {
.map(|plan| plan.display_indent().to_string())
}
+ /// The native route follows the query options the snapshot resolves: the request, then the
+ /// session `SET`, then the engine-wide default. Only that one value decides, so a session that
+ /// says `true` routes natively with no request hint and the engine default off, and a request
+ /// that says `false` leaves the route however the other two are set.
+ #[tokio::test]
+ async fn test_plan_pql_native_route_follows_query_options() {
+ // (engine default, session `SET`, request hint, native route admitted)
+ for (engine_default, session, request, native) in [
+ // Nothing states the option: the engine default decides.
+ (false, None, None, false),
+ (true, None, None, true),
+ // A session `SET` alone enables the route while the engine default stays off.
+ (false, Some("true"), None, true),
+ // A request over the session value, in both directions.
+ (false, Some("true"), Some("false"), false),
+ (false, Some("false"), Some("true"), true),
+ // Over the engine default too, in both directions.
+ (true, Some("false"), None, false),
+ (true, Some("true"), Some("false"), false),
+ (false, None, Some("true"), true),
+ ] {
+ let plan = plan_native_promql_with_query_options(
+ crate::options::QueryOptions {
+ experimental_native_promql: engine_default,
+ ..Default::default()
+ },
+ session.map(|value| (NATIVE_PROMQL_EXTENSION_KEY, value)),
+ request.map(|value| (NATIVE_PROMQL_EXTENSION_KEY, value)),
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ plan.contains("FusedSeries"),
+ native,
+ "engine default={engine_default}, session={session:?}, request={request:?}: {plan}"
+ );
+ }
+ }
+
/// Plans `rate(some_metric[5m])` with the fallback option the caller states, while the native
/// route is on and requested, so only the fallback option decides.
async fn plan_native_promql_with_fallback(
@@ -1557,28 +1593,50 @@ mod tests {
}
}
- /// The engine-wide option is the outer gate: a request that asks for the native route is
- /// still planned through the legacy path while `experimental_native_promql` keeps its
- /// default. `plan_native_promql_with_query_options` always sets the request extension, so
- /// passing `QueryOptions::default()` here is exactly that case.
+ /// The engine-wide option is the default a request or a session overrides: it is not a gate
+ /// the request also has to pass. A default deployment stays on the legacy route when nothing
+ /// is stated, and a client that asks for the native route gets it - which is the case that
+ /// matters when this branch is merged back, because a client can always state the option.
#[tokio::test]
- async fn test_default_engine_option_keeps_a_native_request_on_the_legacy_route() {
- let plan = plan_native_promql_with_query_options(
- crate::options::QueryOptions::default(),
- None,
- None,
- )
- .await
- .unwrap();
+ async fn test_engine_option_is_the_default_a_request_or_session_overrides() {
+ const KEY: &str = "query.experimental_native_promql";
+ // (engine option, session `SET`, request hint, native route planned)
+ for (engine, session, request, native) in [
+ // Nothing states the option, so the engine default answers.
+ (false, None, None, false),
+ (true, None, None, true),
+ // A request that asks for the route gets it on a default deployment.
+ (false, None, Some((KEY, "true")), true),
+ // ... and a request that states `false` keeps the legacy route on an enabled one.
+ (true, None, Some((KEY, "false")), false),
+ // A session `SET` states it too, and the request outranks it in both directions.
+ (false, Some((KEY, "true")), None, true),
+ (true, Some((KEY, "false")), None, false),
+ (false, Some((KEY, "false")), Some((KEY, "true")), true),
+ (true, Some((KEY, "true")), Some((KEY, "false")), false),
+ ] {
+ let plan = plan_native_promql_with_query_options(
+ crate::options::QueryOptions {
+ experimental_native_promql: engine,
+ ..Default::default()
+ },
+ session,
+ request,
+ )
+ .await
+ .unwrap();
- assert!(
- !plan.contains("FusedSeries"),
- "a default deployment planned natively: {plan}"
- );
- assert!(
- !plan.contains("RawSeriesSource"),
- "a default deployment planned a raw-series source: {plan}"
- );
+ assert_eq!(
+ plan.contains("FusedSeries"),
+ native,
+ "engine={engine}, session={session:?}, request={request:?}: {plan}"
+ );
+ assert_eq!(
+ plan.contains("RawSeriesSource"),
+ native,
+ "engine={engine}, session={session:?}, request={request:?}: {plan}"
+ );
+ }
}
/// The fallback hint names a query option, so a value that is not a boolean is rejected instead
diff --git a/src/query/src/promql/planner.rs b/src/query/src/promql/planner.rs
index 7490db01fa6..25c3a5352e8 100644
--- a/src/query/src/promql/planner.rs
+++ b/src/query/src/promql/planner.rs
@@ -17,7 +17,7 @@ mod function_plans;
mod island;
mod matching_filters;
pub(crate) mod native;
-pub use native::{NATIVE_PROMQL_EXTENSION_KEY, discovers_metric_name};
+pub use native::{NATIVE_PROMQL_EXTENSION_KEY, discovers_metric_name, native_promql_resolved};
mod set_operator;
diff --git a/src/query/src/promql/planner/native.rs b/src/query/src/promql/planner/native.rs
index 070603f3a47..798a664bebb 100644
--- a/src/query/src/promql/planner/native.rs
+++ b/src/query/src/promql/planner/native.rs
@@ -87,27 +87,56 @@ use crate::promql::native_compile::{
use crate::promql::planner::{DEFAULT_FIELD_COLUMN, DEFAULT_TIME_INDEX_COLUMN, PromPlanner};
use crate::query_engine::QueryEngineState;
-/// The query-context key that turns the native fused-series route on for one query.
+/// The query option that turns the native fused-series route on for one query.
///
-/// Named after the engine-wide option it mirrors ([`QueryOptions::experimental_native_promql`]) and read
-/// the way the other per-query override in this crate is read: a query-context extension, which a client
-/// can set per request. The engine-wide option is applied while building query-engine state; this planner
-/// reads the per-query extension at the call site; see [`native_promql_requested`].
+/// Named after the engine-wide option it mirrors ([`QueryOptions::experimental_native_promql`]), it
+/// is a query option like the others in this crate: a client states it per request (the HTTP hints
+/// header, a gRPC context key) or per session (`SET query.experimental_native_promql = ...`), and
+/// the engine-wide option is the default. The three resolve in that order where the query options
+/// snapshot is applied, and the planner reads the resulting
+/// [`NativePromqlExtension`](crate::options::NativePromqlExtension) rather than this key.
///
/// [`QueryOptions::experimental_native_promql`]: crate::options::QueryOptions::experimental_native_promql
pub const NATIVE_PROMQL_EXTENSION_KEY: &str = "query.experimental_native_promql";
-/// Whether `query_ctx` asks for the native fused-series route.
+/// Whether the query options of one query ask for the native fused-series route.
///
-/// The route is off unless the query asks for it, so a query that says nothing keeps the plan the existing
-/// planner builds. The value is compared literally (`true` in any case), so a typo costs a slow query and
-/// never a wrong one.
+/// Read from the query options snapshot the query is planned with, so a per-request value and a
+/// session `SET` arrive here alike, already normalized: this compares literally (`true` in any
+/// case) rather than parsing again, because a value that is not a boolean never reaches the
+/// snapshot.
pub(crate) fn native_promql_requested(query_ctx: &QueryContextRef) -> bool {
query_ctx
.extension(NATIVE_PROMQL_EXTENSION_KEY)
.is_some_and(|value| value.eq_ignore_ascii_case("true"))
}
+/// Whether one query asks for the native fused-series route, resolved the way planning resolves
+/// it: a stated value wins (`request over session `SET``), otherwise the engine default answers.
+///
+/// This exists for the callers that have to act *before* a plan is built - the frontend bounds
+/// metric discovery by the caller's grants only when this route will read it - and it resolves
+/// from the same two sources planning reads: [`QueryContext::query_option_snapshot`] for what the
+/// query states, and the engine-wide option for what it inherits. Nothing here re-derives the
+/// option from a third place, so the two readers cannot disagree.
+pub fn native_promql_resolved(
+ query_ctx: &QueryContextRef,
+ engine_default: bool,
+) -> crate::error::Result {
+ let snapshot = query_ctx.query_option_snapshot().map_err(|error| {
+ crate::error::InvalidQueryContextExtensionSnafu {
+ reason: error.to_string(),
+ }
+ .build()
+ })?;
+ Ok(
+ match snapshot.extensions().get(NATIVE_PROMQL_EXTENSION_KEY) {
+ Some(value) => value.eq_ignore_ascii_case("true"),
+ None => engine_default,
+ },
+ )
+}
+
/// Whether this executable plan is the native raw PromQL topology, the one plan shape whose
/// region reads one query-wide body budget cannot deadlock.
///
@@ -2425,9 +2454,45 @@ mod tests {
ArrowDataType, FieldKind, METRIC_NAME, NATIVE_PROMQL_EXTENSION_KEY, NativeType,
PromPlanner, aggregate_reads_histogram, collect_selectors_with_set_operations,
fused_output_schema, independent_scan_pair, native_promql_requested,
- native_raw_plan_bounds_region_reads, scan_time_filter,
+ native_promql_resolved, native_raw_plan_bounds_region_reads, scan_time_filter,
};
+ /// The switch resolves from the two sources planning reads: what the query states (a request
+ /// value over the session `SET` it inherits), and the engine default when it states nothing.
+ /// The frontend reads the same resolver before a plan exists, so a query that never asks for
+ /// the route is not bounded for discovery and one that does is.
+ #[test]
+ fn the_route_switch_resolves_over_the_engine_default() {
+ const KEY: &str = "query.experimental_native_promql";
+ for (engine_default, session, request, resolved) in [
+ // Nothing is stated, so the engine default answers.
+ (false, None, None, false),
+ (true, None, None, true),
+ // A session `SET` states it, and a request value outranks it in both directions.
+ (false, Some("true"), None, true),
+ (true, Some("false"), None, false),
+ (false, Some("false"), Some("true"), true),
+ (true, Some("true"), Some("false"), false),
+ // The value is compared literally, as planning compares it.
+ (false, None, Some("TRUE"), true),
+ ] {
+ let mut ctx = QueryContextBuilder::default().build();
+ if let Some(value) = session {
+ ctx.configuration_parameter()
+ .set_query_option(KEY, value)
+ .unwrap();
+ }
+ if let Some(value) = request {
+ ctx.set_extension(KEY.to_string(), value.to_string());
+ }
+ assert_eq!(
+ resolved,
+ native_promql_resolved(&Arc::new(ctx), engine_default).unwrap(),
+ "engine={engine_default}, session={session:?}, request={request:?}"
+ );
+ }
+ }
+
#[test]
fn native_raw_region_read_budget_needs_the_native_topology() {
let schema = Arc::new(datafusion::arrow::datatypes::Schema::new(vec![
@@ -3344,8 +3409,8 @@ mod tests {
}
#[test]
- fn native_promql_requested_rejects_false_and_unrecognized_values() {
- for value in ["false", "FALSE", "yes", "1", " true ", ""] {
+ fn native_promql_requested_rejects_false_values() {
+ for value in ["false", "FALSE", "False"] {
let query_ctx = Arc::new(
QueryContextBuilder::default()
.set_extension(NATIVE_PROMQL_EXTENSION_KEY.to_string(), value.to_string())
diff --git a/src/query/src/query_engine/state.rs b/src/query/src/query_engine/state.rs
index 409ad798996..c9540682a13 100644
--- a/src/query/src/query_engine/state.rs
+++ b/src/query/src/query_engine/state.rs
@@ -86,7 +86,11 @@ use crate::optimizer::string_normalization::StringNormalizationRule;
use crate::optimizer::transcribe_atat::TranscribeAtatRule;
use crate::optimizer::type_conversion::TypeConversionRule;
use crate::optimizer::windowed_sort::WindowedSortPhysicalRule;
-use crate::options::{QueryMemoryPoolPolicy, QueryOptions as QueryOptionsNew};
+use crate::options::{
+ NativePromqlExtension, QueryMemoryPoolPolicy, QueryOptions as QueryOptionsNew,
+};
+use crate::promql::planner::NATIVE_PROMQL_EXTENSION_KEY;
+use crate::promql::planner::native::native_promql_requested;
use crate::query_engine::DefaultSerializer;
use crate::query_engine::options::QueryOptions;
use crate::query_engine::runtime::{
@@ -393,6 +397,23 @@ impl QueryEngineState {
allow_query_fallback: fallback == "true",
});
}
+ // The native PromQL route resolves like the other query options: a request value over a
+ // session `SET` over the engine-wide default. It is recorded here, once, so that planning
+ // and `SHOW query.experimental_native_promql` read the same value instead of each resolving
+ // the sources again.
+ let native_promql = match query_ctx.extension(NATIVE_PROMQL_EXTENSION_KEY) {
+ // The snapshot states the option: the request value, or the session `SET` it inherits.
+ Some(_) => native_promql_requested(&query_ctx),
+ // Neither source states it, so the engine-wide default answers.
+ None => self.experimental_native_promql(),
+ };
+ state
+ .config_mut()
+ .options_mut()
+ .extensions
+ .insert(NativePromqlExtension {
+ requested: native_promql,
+ });
Ok(query_ctx)
}
@@ -532,7 +553,8 @@ impl QueryEngineState {
self.df_context.register_udwf(func);
}
- pub(crate) fn experimental_native_promql(&self) -> bool {
+ /// The engine-wide default of the native PromQL route, which a query states over.
+ pub fn experimental_native_promql(&self) -> bool {
self.experimental_native_promql
}
diff --git a/src/query/src/sql.rs b/src/query/src/sql.rs
index 260b13c7541..a9dc822fe8a 100644
--- a/src/query/src/sql.rs
+++ b/src/query/src/sql.rs
@@ -886,6 +886,13 @@ pub fn show_query_option(
.to_string()
}
"query.parallelism" => context.state().config().target_partitions().to_string(),
+ "query.experimental_native_promql" => context
+ .state()
+ .config_options()
+ .extensions
+ .get::()
+ .is_some_and(|options| options.requested)
+ .to_string(),
"datafusion.optimizer.filter_null_join_keys"
if context
.state()
@@ -1655,13 +1662,14 @@ mod test {
use table::TableRef;
use table::test_util::MemTable;
- use super::{describe_column_type_name, show_variable};
+ use super::{describe_column_type_name, show_query_option, show_variable};
use crate::error;
use crate::error::Result;
use crate::sql::{
DESCRIBE_TABLE_OUTPUT_SCHEMA, NO_STR, SEMANTIC_TYPE_FIELD, SEMANTIC_TYPE_TIME_INDEX,
YES_STR, describe_table,
};
+ use crate::{QueryEngine, QueryEngineFactory};
#[test]
fn test_describe_table_multiple_columns() -> Result<()> {
@@ -1763,6 +1771,117 @@ mod test {
assert!(exec_show_variable("SYSTEM TIME ZONE", "Asia/Shanghai").is_err());
}
+ const NATIVE_PROMQL: &str = "query.experimental_native_promql";
+
+ /// `SHOW query.experimental_native_promql` reports the requested value: the request, over the
+ /// session `SET`, over the engine-wide default. It answers what the query asked for, not the
+ /// plan it got: a statement that allows query fallback still takes the existing plan.
+ #[test]
+ fn test_show_experimental_native_promql_reports_the_effective_value() {
+ let default_engine = native_promql_engine(false);
+ let enabled_engine = native_promql_engine(true);
+
+ // Nothing is stated per query, so the engine default answers.
+ for (engine, expected) in [(&default_engine, "false"), (&enabled_engine, "true")] {
+ let query_ctx = Arc::new(QueryContextBuilder::default().build());
+ assert_eq!(
+ expected,
+ exec_show_native_promql(engine.as_ref(), query_ctx)
+ );
+ }
+
+ // A session `SET`, read back before and after it changes.
+ let query_ctx = Arc::new(QueryContextBuilder::default().build());
+ query_ctx
+ .configuration_parameter()
+ .set_query_option(NATIVE_PROMQL, "true")
+ .unwrap();
+ assert_eq!(
+ "true",
+ exec_show_native_promql(default_engine.as_ref(), query_ctx.clone())
+ );
+ query_ctx
+ .configuration_parameter()
+ .set_query_option(NATIVE_PROMQL, "false")
+ .unwrap();
+ assert_eq!(
+ "false",
+ exec_show_native_promql(default_engine.as_ref(), query_ctx)
+ );
+
+ // A request value over a session value, which is what the view planner sets when it pins
+ // the route off for one statement.
+ let mut request = QueryContextBuilder::default().build();
+ request
+ .configuration_parameter()
+ .set_query_option(NATIVE_PROMQL, "true")
+ .unwrap();
+ request.set_extension(NATIVE_PROMQL, "false");
+ assert_eq!(
+ "false",
+ exec_show_native_promql(enabled_engine.as_ref(), Arc::new(request))
+ );
+ }
+
+ /// Sessions keep their own value.
+ #[test]
+ fn test_show_experimental_native_promql_is_isolated_between_sessions() {
+ let engine = native_promql_engine(false);
+ let configured = QueryContextBuilder::default().build();
+ configured
+ .configuration_parameter()
+ .set_query_option(NATIVE_PROMQL, "true")
+ .unwrap();
+ let plain = QueryContextBuilder::default().build();
+
+ assert_eq!(
+ "true",
+ exec_show_native_promql(engine.as_ref(), Arc::new(configured))
+ );
+ assert_eq!(
+ "false",
+ exec_show_native_promql(engine.as_ref(), Arc::new(plain))
+ );
+ }
+
+ /// A query engine that only states the native route option.
+ fn native_promql_engine(experimental_native_promql: bool) -> crate::QueryEngineRef {
+ QueryEngineFactory::new(
+ catalog::memory::MemoryCatalogManager::with_default_setup(),
+ None,
+ None,
+ None,
+ None,
+ false,
+ crate::options::QueryOptions {
+ experimental_native_promql,
+ ..Default::default()
+ },
+ )
+ .query_engine()
+ }
+
+ /// Runs `SHOW query.experimental_native_promql` and returns the one value it reports.
+ fn exec_show_native_promql(
+ engine: &dyn QueryEngine,
+ query_ctx: Arc,
+ ) -> String {
+ let stmt = ShowVariables {
+ variable: ObjectName::from(vec![
+ Ident::new("query"),
+ Ident::new("experimental_native_promql"),
+ ]),
+ };
+ let output = show_query_option(stmt, query_ctx, engine)
+ .unwrap()
+ .expect("the native route option is a supported query option");
+ let OutputData::RecordBatches(record) = output.data else {
+ unreachable!("SHOW answers record batches")
+ };
+ let record = record.take().first().cloned().unwrap();
+ record.iter_column_as_string(0).next().unwrap().unwrap()
+ }
+
fn exec_show_variable(variable: &str, tz: &str) -> Result {
let stmt = ShowVariables {
variable: ObjectName::from(vec![Ident::new(variable)]),
diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs
index af312b5ec5c..bacb67c597b 100644
--- a/tests-integration/tests/http.rs
+++ b/tests-integration/tests/http.rs
@@ -2039,7 +2039,7 @@ async fn assert_raw_scan_fused_formats(observer: &NativeFusedObserver, frontend:
);
observer.clear();
- let (_, row_schema, _) = promql_output(frontend, "native_demo", true, "0", "2").await;
+ let (_, row_schema, _) = promql_output(frontend, "native_demo", Some(true), "0", "2").await;
let origin = observer.take("native_demo");
let children = origin.children();
assert_eq!(children.len(), 1, "the fused origin has exactly one child");
@@ -2252,15 +2252,17 @@ async fn assert_raw_scan_fused_formats(observer: &NativeFusedObserver, frontend:
async fn promql_output(
frontend: &Arc,
query: &str,
- native: bool,
+ native: Option,
start: &str,
end: &str,
) -> (Arc, SchemaRef, Vec) {
let mut query_ctx = QueryContextBuilder::default();
- if native {
+ // `None` states nothing, so the session and the engine-wide default answer; `Some(false)` asks
+ // for the existing route explicitly, which the engine default cannot say on its own.
+ if let Some(native) = native {
query_ctx = query_ctx.set_extension(
"query.experimental_native_promql".to_string(),
- "true".to_string(),
+ native.to_string(),
);
}
let prom_query = PromQuery {
@@ -2451,9 +2453,16 @@ pub async fn test_prometheus_native_promql_route(store_type: StorageType) {
"/v1/prometheus/api/v1/query?query={}&time=1",
encode(query)
));
- if native {
- request = request.header(native_hint.0, native_hint.1);
- }
+ // The engine-wide default is on here, so the request states the route it wants in both
+ // directions: a missing value would leave the choice to the default, which is native.
+ request = request.header(
+ native_hint.0,
+ if native {
+ "query.experimental_native_promql=true"
+ } else {
+ "query.experimental_native_promql=false"
+ },
+ );
let res = request.send().await;
assert_eq!(res.status(), StatusCode::OK, "`{query}`, native={native}");
let body = serde_json::from_str::(&res.text().await).unwrap();
@@ -2508,9 +2517,16 @@ pub async fn test_prometheus_native_promql_route(store_type: StorageType) {
let mut request = client.get(&format!(
"/v1/prometheus/api/v1/query_range?query={query}&start=0&end=2&step=1s"
));
- if native {
- request = request.header(native_hint.0, native_hint.1);
- }
+ // Both routes are asked for explicitly: the engine default is on, so leaving the value out
+ // would ask for neither.
+ request = request.header(
+ native_hint.0,
+ if native {
+ "query.experimental_native_promql=true"
+ } else {
+ "query.experimental_native_promql=false"
+ },
+ );
let res = request.send().await;
assert_eq!(res.status(), StatusCode::OK, "range query, native={native}");
let body = serde_json::from_str::(&res.text().await).unwrap();
@@ -2746,7 +2762,7 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) {
} else {
&[]
};
- let (plan, schema, rows) = promql_output(frontend, query, true, start, end).await;
+ let (plan, schema, rows) = promql_output(frontend, query, Some(true), start, end).await;
assert_rows_eq(&rows, &expected, query);
assert_native_result_schema(&schema, tags, query, query);
// The body is drained, so every partition has finished and the node this statement ran is
@@ -2756,13 +2772,13 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) {
assert_native_plan(&plan, &exec, scan_partitions, query);
assert_native_partition_output(&exec, scan_partitions, expected.len(), query);
- // Without the per-query hint the same statement runs the legacy pipeline, which must answer
- // with the same rows.
+ // A request that turns the route off runs the legacy pipeline, which must answer with the
+ // same rows. The engine default is on here, so the request value is what selects the route.
let (legacy_plan, legacy_schema, legacy_rows) =
- promql_output(frontend, query, false, start, end).await;
+ promql_output(frontend, query, Some(false), start, end).await;
assert!(
!executes_fused_series(&legacy_plan),
- "`{query}` without the hint executed the fused route"
+ "`{query}` with the route turned off executed the fused plan"
);
assert_rows_eq(&legacy_rows, &expected, query);
assert_legacy_result_schema(&legacy_schema, tags, query);
@@ -2803,7 +2819,7 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) {
// different partition layout than the four non-empty queries above.
let query = "native_demo{host=\"absent\"} * 100";
observer.clear();
- let (plan, schema, rows) = promql_output(frontend, query, true, "0", "2").await;
+ let (plan, schema, rows) = promql_output(frontend, query, Some(true), "0", "2").await;
assert!(rows.is_empty(), "`{query}` reported {} rows", rows.len());
assert_native_result_schema(&schema, &["host"], query, query);
assert!(
@@ -2856,14 +2872,14 @@ pub async fn test_native_promql_route_plans_the_query(store_type: StorageType) {
// Without the per-query hint the same statement runs the legacy pipeline, which must answer
// with the same empty typed result.
let (legacy_plan, legacy_schema, legacy_rows) =
- promql_output(frontend, query, false, "0", "2").await;
+ promql_output(frontend, query, Some(false), "0", "2").await;
assert!(
!executes_fused_series(&legacy_plan),
- "`{query}` without the hint executed the fused route"
+ "`{query}` with the route turned off executed the fused plan"
);
assert!(
legacy_rows.is_empty(),
- "`{query}` without the hint reported {} rows",
+ "`{query}` with the route turned off reported {} rows",
legacy_rows.len()
);
assert_legacy_result_schema(&legacy_schema, &["host"], query);
@@ -5938,13 +5954,14 @@ async fn native_histogram_composition_query(
.to_string()
})
.collect::>();
+ // The tree a fused node prints names its own nodes, indented by depth inside the
+ // `native_tree=[ ... ]` block, so the binary is found by its node line rather than by the
+ // operator line (which prints the tree, not the statement).
assert!(
descriptions.iter().any(|description| {
- description.lines().any(|line| {
- let mut words = line.split_whitespace();
- matches!(words.next(), Some("root" | "input" | "left" | "right"))
- && words.next() == Some("Binary")
- })
+ description
+ .lines()
+ .any(|line| line.trim_start().starts_with("Binary:"))
}),
"`{query}` did not execute a native Binary: {descriptions:?}; response: {body}"
);
@@ -6340,10 +6357,13 @@ pub async fn test_remote_write_v2_histogram_native_composition(store_type: Stora
assert!(body.get("infos").is_none(), "{phase}: {query}: {body}");
assert!(body.get("warnings").is_none(), "{phase}: {query}: {body}");
- // The same two windows asked without a hint, so the legacy route answers them and the
- // observer proves it did. This is where the delegated kernel has to agree with the native
- // route: the raw sample's start timestamp is what lets `rate` measure the window, while
- // `* 1` gives that timestamp up, so the derived window has nothing to measure against.
+ // The same two windows asked with the route explicitly off, so the legacy route answers
+ // them and the observer proves it did. The engine option is a default here (this instance
+ // sets it to `true`), so an unstated request would take the native route; stating `false`
+ // is what selects the existing one. This is where the delegated kernel has to agree with
+ // the native route: the raw sample's start timestamp is what lets `rate` measure the
+ // window, while `* 1` gives that timestamp up, so the derived window has nothing to
+ // measure against.
for (query, expected_count) in [
("rate(native_comp_start[3s])", Some(8.0 / 3.0)),
("rate((native_comp_start * 1)[3s:1s])", None),
@@ -6353,6 +6373,7 @@ pub async fn test_remote_write_v2_histogram_native_composition(store_type: Stora
"/v1/prometheus/api/v1/query?query={}&time=3",
encode(query)
))
+ .header("x-greptime-hints", "query.experimental_native_promql=false")
.send()
.await;
let status = res.status();