Skip to main content

query/query_engine/
state.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::HashMap;
16use std::fmt;
17use std::num::NonZeroUsize;
18use std::sync::{Arc, RwLock};
19
20use async_trait::async_trait;
21use catalog::CatalogManagerRef;
22use common_base::Plugins;
23use common_function::aggrs::aggr_wrapper::fix_order::FixStateUdafOrderingAnalyzer;
24use common_function::function_factory::ScalarFunctionFactory;
25use common_function::function_registry::FUNCTION_REGISTRY;
26use common_function::handlers::{
27    FlowServiceHandlerRef, ProcedureServiceHandlerRef, TableMutationHandlerRef,
28};
29use common_function::state::FunctionState;
30use common_stat::get_total_memory_bytes;
31use common_telemetry::warn;
32use datafusion::catalog::{Session, TableFunction};
33use datafusion::dataframe::DataFrame;
34use datafusion::error::Result as DfResult;
35use datafusion::execution::SessionStateBuilder;
36use datafusion::execution::context::{QueryPlanner, SessionConfig, SessionContext, SessionState};
37use datafusion::execution::memory_pool::{
38    FairSpillPool, GreedyMemoryPool, MemoryConsumer, MemoryLimit, MemoryPool, MemoryReservation,
39    TrackConsumersPool,
40};
41use datafusion::physical_optimizer::PhysicalOptimizerRule;
42use datafusion::physical_optimizer::optimizer::PhysicalOptimizer;
43use datafusion::physical_optimizer::sanity_checker::SanityCheckPlan;
44use datafusion::physical_plan::ExecutionPlan;
45use datafusion::physical_planner::{DefaultPhysicalPlanner, ExtensionPlanner, PhysicalPlanner};
46use datafusion_expr::{AggregateUDF, LogicalPlan as DfLogicalPlan, WindowUDF};
47use datafusion_optimizer::Analyzer;
48use datafusion_optimizer::analyzer::function_rewrite::ApplyFunctionRewrites;
49use datafusion_optimizer::optimizer::Optimizer;
50use partition::manager::PartitionRuleManagerRef;
51use promql::extension_plan::PromExtensionPlanner;
52use session::context::QueryContextRef;
53use table::TableRef;
54use table::table::adapter::DfTableProviderAdapter;
55
56use crate::QueryEngineContext;
57use crate::dist_plan::{
58    DistExtensionPlanner, DistPlannerAnalyzer, DistPlannerOptions, DynFilterRegistryManager,
59    MergeSortExtensionPlanner, RemoteDynFilterReceiverExtensionPlanner,
60    RemoteDynFilterRegistryLease,
61};
62use crate::metrics::{QUERY_MEMORY_POOL_REJECTED_TOTAL, QUERY_MEMORY_POOL_USAGE_BYTES};
63use crate::optimizer::ExtensionAnalyzerRule;
64use crate::optimizer::const_normalization::ConstNormalizationRule;
65use crate::optimizer::constant_term::MatchesConstantTermOptimizer;
66use crate::optimizer::count_nest_aggr::CountNestAggrRule;
67use crate::optimizer::count_wildcard::CountWildcardToTimeIndexRule;
68use crate::optimizer::enforce_sorting::EnforceSorting;
69use crate::optimizer::global_limit::EnsureGlobalLimitForFetch;
70use crate::optimizer::json_schema_concretize::JsonSchemaConcretizeRule;
71use crate::optimizer::json_type_concretize::JsonTypeConcretizeRule;
72use crate::optimizer::parallelize_scan::ParallelizeScan;
73use crate::optimizer::pass_distribution::PassDistribution;
74use crate::optimizer::promql_tsid_narrow_join::PromqlTsidNarrowJoin;
75use crate::optimizer::remove_duplicate::RemoveDuplicate;
76use crate::optimizer::scan_hint::ScanHintRule;
77use crate::optimizer::string_normalization::StringNormalizationRule;
78use crate::optimizer::transcribe_atat::TranscribeAtatRule;
79use crate::optimizer::type_conversion::TypeConversionRule;
80use crate::optimizer::windowed_sort::WindowedSortPhysicalRule;
81use crate::options::{QueryMemoryPoolPolicy, QueryOptions as QueryOptionsNew};
82use crate::query_engine::DefaultSerializer;
83use crate::query_engine::options::QueryOptions;
84use crate::query_engine::runtime::{
85    DefaultQueryRuntimeProvider, QueryRuntimeContext, QueryRuntimeProvider, QueryRuntimeProviderRef,
86};
87use crate::range_select::planner::RangeSelectPlanner;
88use crate::region_query::RegionQueryHandlerRef;
89
90/// Query engine global state
91#[derive(Clone)]
92pub struct QueryEngineState {
93    df_context: SessionContext,
94    catalog_manager: CatalogManagerRef,
95    dyn_filter_registry_manager: Arc<DynFilterRegistryManager>,
96    function_state: Arc<FunctionState>,
97    scalar_functions: Arc<RwLock<HashMap<String, ScalarFunctionFactory>>>,
98    aggr_functions: Arc<RwLock<HashMap<String, AggregateUDF>>>,
99    table_functions: Arc<RwLock<HashMap<String, Arc<TableFunction>>>>,
100    extension_rules: Vec<Arc<dyn ExtensionAnalyzerRule + Send + Sync>>,
101    plugins: Plugins,
102}
103
104impl fmt::Debug for QueryEngineState {
105    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
106        f.debug_struct("QueryEngineState")
107            .field("state", &self.df_context.state())
108            .finish()
109    }
110}
111
112impl QueryEngineState {
113    #[allow(clippy::too_many_arguments)]
114    pub fn new(
115        catalog_list: CatalogManagerRef,
116        partition_rule_manager: Option<PartitionRuleManagerRef>,
117        region_query_handler: Option<RegionQueryHandlerRef>,
118        table_mutation_handler: Option<TableMutationHandlerRef>,
119        procedure_service_handler: Option<ProcedureServiceHandlerRef>,
120        flow_service_handler: Option<FlowServiceHandlerRef>,
121        with_dist_planner: bool,
122        plugins: Plugins,
123        options: QueryOptionsNew,
124    ) -> Self {
125        Self::try_new(
126            catalog_list,
127            partition_rule_manager,
128            region_query_handler,
129            table_mutation_handler,
130            procedure_service_handler,
131            flow_service_handler,
132            with_dist_planner,
133            plugins,
134            options,
135        )
136        .expect("Failed to build query engine state")
137    }
138
139    #[allow(clippy::too_many_arguments)]
140    pub fn try_new(
141        catalog_list: CatalogManagerRef,
142        partition_rule_manager: Option<PartitionRuleManagerRef>,
143        region_query_handler: Option<RegionQueryHandlerRef>,
144        table_mutation_handler: Option<TableMutationHandlerRef>,
145        procedure_service_handler: Option<ProcedureServiceHandlerRef>,
146        flow_service_handler: Option<FlowServiceHandlerRef>,
147        with_dist_planner: bool,
148        plugins: Plugins,
149        options: QueryOptionsNew,
150    ) -> DfResult<Self> {
151        let total_memory = get_total_memory_bytes().max(0) as u64;
152        let memory_pool_size = options.memory_pool_size.resolve(total_memory) as usize;
153        let runtime_provider = plugins.get::<QueryRuntimeProviderRef>();
154        let mut session_config = SessionConfig::new().with_create_default_catalog_and_schema(false);
155        if options.parallelism > 0 {
156            session_config = session_config.with_target_partitions(options.parallelism);
157        }
158        if options.allow_query_fallback {
159            session_config
160                .options_mut()
161                .extensions
162                .insert(DistPlannerOptions {
163                    allow_query_fallback: true,
164                });
165        }
166
167        // todo(hl): This serves as a workaround for https://github.com/GreptimeTeam/greptimedb/issues/5659
168        // and we can add that check back once we upgrade datafusion.
169        session_config
170            .options_mut()
171            .execution
172            .skip_physical_aggregate_schema_check = true;
173
174        let runtime_context = QueryRuntimeContext::new(&options, memory_pool_size);
175        let default_runtime_provider = DefaultQueryRuntimeProvider;
176        default_runtime_provider.configure_session_config(runtime_context, &mut session_config);
177        if let Some(provider) = runtime_provider.as_ref() {
178            provider.configure_session_config(runtime_context, &mut session_config);
179        }
180        let runtime_builder = DefaultQueryRuntimeProvider::runtime_env_builder(runtime_context);
181        let runtime_env = match runtime_provider {
182            Some(provider) => provider.build_runtime_env(runtime_context, runtime_builder)?,
183            None => default_runtime_provider.build_runtime_env(runtime_context, runtime_builder)?,
184        };
185
186        // Apply extension rules
187        let mut extension_rules = Vec::new();
188
189        // The [`TypeConversionRule`] must be at first
190        extension_rules.insert(0, Arc::new(TypeConversionRule) as _);
191        extension_rules.push(Arc::new(CountNestAggrRule) as _);
192
193        // Apply the datafusion rules
194        let mut analyzer = Analyzer::new();
195        analyzer.rules.insert(0, Arc::new(TranscribeAtatRule));
196        analyzer.rules.insert(0, Arc::new(StringNormalizationRule));
197        analyzer
198            .rules
199            .insert(0, Arc::new(CountWildcardToTimeIndexRule));
200        analyzer.rules.push(Arc::new(ConstNormalizationRule));
201
202        // Add ApplyFunctionRewrites rule,
203        // Note we cannot use `analyzer.add_function_rewrite`
204        // because only rules are copied into session_state
205        analyzer.rules.insert(
206            0,
207            Arc::new(ApplyFunctionRewrites::new(
208                FUNCTION_REGISTRY.function_rewrites(),
209            )),
210        );
211
212        if with_dist_planner {
213            analyzer.rules.push(Arc::new(DistPlannerAnalyzer));
214            analyzer.rules.push(Arc::new(JsonSchemaConcretizeRule));
215        }
216        analyzer.rules.push(Arc::new(FixStateUdafOrderingAnalyzer));
217
218        // Note: Postgres oid-alias string coercion (`'x'::regclass`,
219        // `'public'::regnamespace`, ...) is handled by the
220        // `PostgresCompatibilityParser`'s built-in `RewriteRegCastToSubquery`
221        // rule at SQL-parse time, so no analyzer rule is registered here. The
222        // implicit `oid_col = 'name'` form is not resolved (it needs schema
223        // awareness the parser lacks); the few client queries that use it are
224        // handled by the parser's blacklist.
225
226        let mut optimizer = Optimizer::new();
227        optimizer.rules.push(Arc::new(ScanHintRule));
228        optimizer.rules.push(Arc::new(JsonTypeConcretizeRule));
229
230        // add physical optimizer
231        let mut physical_optimizer = PhysicalOptimizer::new();
232        // Change TableScan's partition right before enforcing distribution
233        physical_optimizer
234            .rules
235            .insert(5, Arc::new(ParallelizeScan));
236        // Pass distribution requirement to MergeScanExec to avoid unnecessary shuffling
237        physical_optimizer
238            .rules
239            .insert(6, Arc::new(PassDistribution));
240        // Prefer collecting narrow PromQL build sides over repartitioning wide label streams.
241        physical_optimizer
242            .rules
243            .insert(7, Arc::new(PromqlTsidNarrowJoin));
244        // Re-enforce sorting after custom rules update scan partitioning and distribution.
245        // Keep it immediately before DataFusion's default EnsureRequirements.
246        physical_optimizer.rules.insert(8, Arc::new(EnforceSorting));
247        // Add rule for windowed sort
248        physical_optimizer
249            .rules
250            .push(Arc::new(WindowedSortPhysicalRule));
251        // explicitly not do filter pushdown for windowed sort&part sort
252        // (notice that `PartSortExec` create another new dyn filter that need to be pushdown if want to use dyn filter optimization)
253        // benchmark shows it can cause performance regression due to useless filtering and extra shuffle.
254        // We can add a rule to do filter pushdown for windowed sort in the future if we find a way to avoid the performance regression.
255        physical_optimizer
256            .rules
257            .push(Arc::new(MatchesConstantTermOptimizer));
258        physical_optimizer
259            .rules
260            .push(Arc::new(EnsureGlobalLimitForFetch));
261        // Add rule to remove duplicate nodes generated by other rules. Run this in the last.
262        physical_optimizer.rules.push(Arc::new(RemoveDuplicate));
263        // Place SanityCheckPlan at the end of the list to ensure that it runs after all other rules.
264        Self::remove_physical_optimizer_rule(
265            &mut physical_optimizer.rules,
266            SanityCheckPlan {}.name(),
267        );
268        physical_optimizer.rules.push(Arc::new(SanityCheckPlan {}));
269
270        let session_state = SessionStateBuilder::new()
271            .with_config(session_config)
272            .with_runtime_env(runtime_env)
273            .with_default_features()
274            .with_analyzer_rules(analyzer.rules)
275            .with_serializer_registry(Arc::new(DefaultSerializer))
276            .with_query_planner(Arc::new(DfQueryPlanner::new(
277                catalog_list.clone(),
278                partition_rule_manager,
279                region_query_handler.clone(),
280                options.enable_per_region_metrics,
281            )))
282            .with_optimizer_rules(optimizer.rules)
283            .with_physical_optimizer_rules(physical_optimizer.rules)
284            .build();
285        let df_context = SessionContext::new_with_state(session_state);
286        register_function_aliases(&df_context);
287        register_pg_catalog_compat(&df_context);
288
289        Ok(Self {
290            df_context,
291            catalog_manager: catalog_list,
292            dyn_filter_registry_manager: Arc::new(DynFilterRegistryManager::default()),
293            function_state: Arc::new(FunctionState {
294                plugins: plugins.clone(),
295                table_mutation_handler,
296                procedure_service_handler,
297                flow_service_handler,
298            }),
299            aggr_functions: Arc::new(RwLock::new(HashMap::new())),
300            table_functions: Arc::new(RwLock::new(HashMap::new())),
301            extension_rules,
302            plugins,
303            scalar_functions: Arc::new(RwLock::new(HashMap::new())),
304        })
305    }
306
307    fn remove_physical_optimizer_rule(
308        rules: &mut Vec<Arc<dyn PhysicalOptimizerRule + Send + Sync>>,
309        name: &str,
310    ) {
311        rules.retain(|rule| rule.name() != name);
312    }
313
314    /// Optimize the logical plan by the extension analyzer rules.
315    pub fn optimize_by_extension_rules(
316        &self,
317        plan: DfLogicalPlan,
318        context: &QueryEngineContext,
319    ) -> DfResult<DfLogicalPlan> {
320        self.extension_rules
321            .iter()
322            .try_fold(plan, |acc_plan, rule| {
323                rule.analyze(acc_plan, context, self.session_state().config_options())
324            })
325    }
326
327    /// Run the full logical plan optimize phase for the given plan.
328    pub fn optimize_logical_plan(&self, plan: DfLogicalPlan) -> DfResult<DfLogicalPlan> {
329        self.session_state().optimize(&plan)
330    }
331
332    /// Retrieve the scalar function by name
333    pub fn scalar_function(&self, function_name: &str) -> Option<ScalarFunctionFactory> {
334        self.scalar_functions
335            .read()
336            .unwrap()
337            .get(function_name)
338            .cloned()
339    }
340
341    /// Retrieve scalar function names.
342    pub fn scalar_names(&self) -> Vec<String> {
343        self.scalar_functions
344            .read()
345            .unwrap()
346            .keys()
347            .cloned()
348            .collect()
349    }
350
351    /// Retrieve the aggregate function by name
352    pub fn aggr_function(&self, function_name: &str) -> Option<AggregateUDF> {
353        self.aggr_functions
354            .read()
355            .unwrap()
356            .get(function_name)
357            .cloned()
358    }
359
360    /// Retrieve aggregate function names.
361    pub fn aggr_names(&self) -> Vec<String> {
362        self.aggr_functions
363            .read()
364            .unwrap()
365            .keys()
366            .cloned()
367            .collect()
368    }
369
370    /// Retrieve table function by name
371    pub fn table_function(&self, function_name: &str) -> Option<Arc<TableFunction>> {
372        self.table_functions
373            .read()
374            .unwrap()
375            .get(function_name)
376            .cloned()
377    }
378
379    /// Retrieve table function names.
380    pub fn table_function_names(&self) -> Vec<String> {
381        self.table_functions
382            .read()
383            .unwrap()
384            .keys()
385            .cloned()
386            .collect()
387    }
388
389    /// Register an scalar function.
390    /// Will override if the function with same name is already registered.
391    pub fn register_scalar_function(&self, func: ScalarFunctionFactory) {
392        let name = func.name().to_string();
393        let x = self
394            .scalar_functions
395            .write()
396            .unwrap()
397            .insert(name.clone(), func);
398
399        if x.is_some() {
400            warn!("Already registered scalar function '{name}'");
401        }
402    }
403
404    /// Register an aggregate function.
405    ///
406    /// # Panics
407    /// Will panic if the function with same name is already registered.
408    ///
409    /// Panicking consideration: currently the aggregated functions are all statically registered,
410    /// user cannot define their own aggregate functions on the fly. So we can panic here. If that
411    /// invariant is broken in the future, we should return an error instead of panicking.
412    pub fn register_aggr_function(&self, func: AggregateUDF) {
413        let name = func.name().to_string();
414        let x = self
415            .aggr_functions
416            .write()
417            .unwrap()
418            .insert(name.clone(), func);
419        assert!(
420            x.is_none(),
421            "Already registered aggregate function '{name}'"
422        );
423    }
424
425    pub fn register_table_function(&self, func: Arc<TableFunction>) {
426        let name = func.name();
427        let x = self
428            .table_functions
429            .write()
430            .unwrap()
431            .insert(name.to_string(), func.clone());
432
433        if x.is_some() {
434            warn!("Already registered table function '{name}'");
435        }
436    }
437
438    /// Register a window function (UDWF) directly on the DataFusion SessionContext.
439    ///
440    /// This makes the function visible via `session_state.window_functions()`,
441    /// which is used by `DfContextProviderAdapter::get_window_meta`.
442    pub fn register_window_function(&self, func: WindowUDF) {
443        self.df_context.register_udwf(func);
444    }
445
446    pub fn catalog_manager(&self) -> &CatalogManagerRef {
447        &self.catalog_manager
448    }
449
450    pub fn dyn_filter_registry_manager(&self) -> Arc<DynFilterRegistryManager> {
451        self.dyn_filter_registry_manager.clone()
452    }
453
454    pub fn acquire_remote_dyn_filter_registry_lease(
455        &self,
456        query_ctx: &QueryContextRef,
457    ) -> Option<RemoteDynFilterRegistryLease> {
458        let query_id = query_ctx.remote_query_id_value()?;
459        Some(
460            self.dyn_filter_registry_manager
461                .clone()
462                .acquire_lease(query_id),
463        )
464    }
465
466    pub fn function_state(&self) -> Arc<FunctionState> {
467        self.function_state.clone()
468    }
469
470    /// Returns the [`TableMutationHandlerRef`] in state.
471    pub fn table_mutation_handler(&self) -> Option<&TableMutationHandlerRef> {
472        self.function_state.table_mutation_handler.as_ref()
473    }
474
475    /// Returns the [`ProcedureServiceHandlerRef`] in state.
476    pub fn procedure_service_handler(&self) -> Option<&ProcedureServiceHandlerRef> {
477        self.function_state.procedure_service_handler.as_ref()
478    }
479
480    pub(crate) fn disallow_cross_catalog_query(&self) -> bool {
481        self.plugins
482            .map::<QueryOptions, _, _>(|x| x.disallow_cross_catalog_query)
483            .unwrap_or(false)
484    }
485
486    pub fn session_state(&self) -> SessionState {
487        self.df_context.state()
488    }
489
490    /// Create a DataFrame for a table
491    pub fn read_table(&self, table: TableRef) -> DfResult<DataFrame> {
492        self.df_context
493            .read_table(Arc::new(DfTableProviderAdapter::new(table)))
494    }
495}
496
497struct DfQueryPlanner {
498    physical_planner: DefaultPhysicalPlanner,
499}
500
501impl fmt::Debug for DfQueryPlanner {
502    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
503        f.debug_struct("DfQueryPlanner").finish()
504    }
505}
506
507#[async_trait]
508impl QueryPlanner for DfQueryPlanner {
509    async fn create_physical_plan(
510        &self,
511        logical_plan: &DfLogicalPlan,
512        session: &dyn Session,
513    ) -> DfResult<Arc<dyn ExecutionPlan>> {
514        self.physical_planner
515            .create_physical_plan(logical_plan, session)
516            .await
517    }
518}
519
520/// MySQL-compatible scalar function aliases: (target_name, alias)
521const SCALAR_FUNCTION_ALIASES: &[(&str, &str)] = &[
522    ("upper", "ucase"),
523    ("lower", "lcase"),
524    ("ceil", "ceiling"),
525    ("substr", "mid"),
526    ("random", "rand"),
527];
528
529/// MySQL-compatible aggregate function aliases: (target_name, alias)
530const AGGREGATE_FUNCTION_ALIASES: &[(&str, &str)] =
531    &[("stddev_pop", "std"), ("var_pop", "variance")];
532
533/// Register function aliases.
534///
535/// This function adds aliases like `ucase` -> `upper`, `lcase` -> `lower`, etc.
536/// to make GreptimeDB more compatible with MySQL syntax.
537fn register_function_aliases(ctx: &SessionContext) {
538    let state = ctx.state();
539
540    for (target, alias) in SCALAR_FUNCTION_ALIASES {
541        if let Some(func) = state.scalar_functions().get(*target) {
542            let aliased = func.as_ref().clone().with_aliases([*alias]);
543            ctx.register_udf(aliased);
544        }
545    }
546
547    for (target, alias) in AGGREGATE_FUNCTION_ALIASES {
548        if let Some(func) = state.aggregate_functions().get(*target) {
549            let aliased = func.as_ref().clone().with_aliases([*alias]);
550            ctx.register_udaf(aliased);
551        }
552    }
553}
554
555/// Register the session-level (not function-registry) Postgres-compatibility
556/// extension sourced from `datafusion-pg-catalog`: widen Postgres `int4`
557/// bounds to `int8` for `generate_series`/`range` (their stock implementations
558/// require `int8` literals, and DF invokes them at planning time, before any
559/// analyzer rule can run).
560///
561/// The oid-alias type planner is supplied per SQL-planner context by
562/// [`DfContextProviderAdapter::get_type_planner`](crate::datafusion::planner::DfContextProviderAdapter::get_type_planner).
563/// Forward oid-alias name->oid resolution is handled at SQL-parse time by the
564/// `PostgresCompatibilityParser`'s built-in rewrite rule, not here.
565fn register_pg_catalog_compat(ctx: &SessionContext) {
566    datafusion_pg_catalog::pg_catalog::generate_series_arg_coercion::CoerceIntArgsToBigInt::widen(
567        ctx,
568    );
569}
570
571impl DfQueryPlanner {
572    fn new(
573        catalog_manager: CatalogManagerRef,
574        partition_rule_manager: Option<PartitionRuleManagerRef>,
575        region_query_handler: Option<RegionQueryHandlerRef>,
576        enable_per_region_metrics: bool,
577    ) -> Self {
578        let mut planners: Vec<Arc<dyn ExtensionPlanner + Send + Sync>> = vec![
579            Arc::new(PromExtensionPlanner),
580            Arc::new(RangeSelectPlanner),
581            Arc::new(RemoteDynFilterReceiverExtensionPlanner),
582        ];
583        if let (Some(region_query_handler), Some(partition_rule_manager)) =
584            (region_query_handler, partition_rule_manager)
585        {
586            planners.push(Arc::new(DistExtensionPlanner::new(
587                catalog_manager,
588                partition_rule_manager,
589                region_query_handler,
590                enable_per_region_metrics,
591            )));
592            planners.push(Arc::new(MergeSortExtensionPlanner {}));
593        }
594        Self {
595            physical_planner: DefaultPhysicalPlanner::with_extension_planners(planners),
596        }
597    }
598}
599
600/// A wrapper around a memory pool that records metrics.
601///
602/// This wrapper intercepts all memory pool operations and updates
603/// Prometheus metrics for monitoring query memory usage and rejections.
604///
605/// The inner pool is wrapped with `TrackConsumersPool` to preserve
606/// top-consumer error context on rejection.
607#[derive(Debug)]
608pub(super) struct MetricsMemoryPool {
609    inner: Arc<dyn MemoryPool>,
610}
611
612impl MetricsMemoryPool {
613    // Number of top memory consumers to report in OOM error messages
614    const TOP_CONSUMERS_TO_REPORT: usize = 5;
615
616    /// Create a new metrics-wrapped memory pool with the given size limit and
617    /// allocation policy.
618    pub(super) fn new(limit: usize, policy: QueryMemoryPoolPolicy) -> Self {
619        let top_n = NonZeroUsize::new(Self::TOP_CONSUMERS_TO_REPORT).unwrap();
620        let inner: Arc<dyn MemoryPool> = match policy {
621            QueryMemoryPoolPolicy::Greedy => {
622                Arc::new(TrackConsumersPool::new(GreedyMemoryPool::new(limit), top_n))
623            }
624            QueryMemoryPoolPolicy::Fair => {
625                Arc::new(TrackConsumersPool::new(FairSpillPool::new(limit), top_n))
626            }
627        };
628        Self { inner }
629    }
630
631    #[inline]
632    fn update_metrics(&self) {
633        QUERY_MEMORY_POOL_USAGE_BYTES.set(self.inner.reserved() as i64);
634    }
635}
636
637impl fmt::Display for MetricsMemoryPool {
638    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
639        write!(f, "{}(inner_pool: {})", self.name(), self.inner)
640    }
641}
642
643impl MemoryPool for MetricsMemoryPool {
644    fn name(&self) -> &str {
645        "metrics"
646    }
647
648    fn register(&self, consumer: &MemoryConsumer) {
649        self.inner.register(consumer);
650    }
651
652    fn unregister(&self, consumer: &MemoryConsumer) {
653        self.inner.unregister(consumer);
654    }
655
656    fn grow(&self, reservation: &MemoryReservation, additional: usize) {
657        self.inner.grow(reservation, additional);
658        self.update_metrics();
659    }
660
661    fn shrink(&self, reservation: &MemoryReservation, shrink: usize) {
662        self.inner.shrink(reservation, shrink);
663        self.update_metrics();
664    }
665
666    fn try_grow(
667        &self,
668        reservation: &MemoryReservation,
669        additional: usize,
670    ) -> datafusion_common::Result<()> {
671        let result = self.inner.try_grow(reservation, additional);
672        if result.is_err() {
673            QUERY_MEMORY_POOL_REJECTED_TOTAL.inc();
674        }
675        self.update_metrics();
676        result
677    }
678
679    fn reserved(&self) -> usize {
680        self.inner.reserved()
681    }
682
683    fn memory_limit(&self) -> MemoryLimit {
684        self.inner.memory_limit()
685    }
686}
687
688#[cfg(test)]
689mod tests {
690    use std::sync::atomic::{AtomicBool, Ordering};
691
692    use common_base::Plugins;
693    use common_base::memory_limit::MemoryLimit;
694    use common_base::readable_size::ReadableSize;
695    use datafusion::error::DataFusionError;
696    use datafusion::execution::memory_pool::{GreedyMemoryPool, MemoryLimit as DfMemoryLimit};
697    use datafusion::execution::runtime_env::{RuntimeEnv, RuntimeEnvBuilder};
698    use datafusion_common::config::SpillCompression;
699    use session::context::QueryContext;
700
701    use super::*;
702    use crate::options::{QueryOptions, QuerySpillCompression, QuerySpillMode};
703    use crate::query_engine::runtime::{
704        DefaultQueryRuntimeProvider, QueryRuntimeContext, QueryRuntimeProvider,
705        QueryRuntimeProviderRef,
706    };
707
708    fn new_query_engine_state() -> QueryEngineState {
709        new_query_engine_state_with(Plugins::default(), QueryOptions::default())
710    }
711
712    fn new_query_engine_state_with(plugins: Plugins, options: QueryOptions) -> QueryEngineState {
713        QueryEngineState::new(
714            catalog::memory::new_memory_catalog_manager().unwrap(),
715            None,
716            None,
717            None,
718            None,
719            None,
720            false,
721            plugins,
722            options,
723        )
724    }
725
726    struct TestRuntimeProvider {
727        build_called: AtomicBool,
728        configure_called: AtomicBool,
729    }
730
731    impl TestRuntimeProvider {
732        fn new() -> Self {
733            Self {
734                build_called: AtomicBool::new(false),
735                configure_called: AtomicBool::new(false),
736            }
737        }
738    }
739
740    impl QueryRuntimeProvider for TestRuntimeProvider {
741        fn configure_session_config(
742            &self,
743            ctx: QueryRuntimeContext<'_>,
744            config: &mut SessionConfig,
745        ) {
746            assert_eq!(ctx.resolved_memory_pool_size, 1024);
747            self.configure_called.store(true, Ordering::SeqCst);
748            *config = config.clone().with_target_partitions(7);
749        }
750
751        fn build_runtime_env(
752            &self,
753            ctx: QueryRuntimeContext<'_>,
754            builder: RuntimeEnvBuilder,
755        ) -> DfResult<Arc<RuntimeEnv>> {
756            assert_eq!(ctx.resolved_memory_pool_size, 1024);
757            self.build_called.store(true, Ordering::SeqCst);
758            builder
759                .with_memory_pool(Arc::new(GreedyMemoryPool::new(2048)))
760                .build()
761                .map(Arc::new)
762        }
763    }
764
765    struct ErrorRuntimeProvider;
766
767    impl QueryRuntimeProvider for ErrorRuntimeProvider {
768        fn build_runtime_env(
769            &self,
770            _ctx: QueryRuntimeContext<'_>,
771            _builder: RuntimeEnvBuilder,
772        ) -> DfResult<Arc<RuntimeEnv>> {
773            Err(DataFusionError::Execution("runtime provider error".into()))
774        }
775    }
776
777    #[test]
778    fn query_runtime_default_provider_keeps_bounded_memory_pool() {
779        let state = new_query_engine_state_with(
780            Plugins::default(),
781            QueryOptions {
782                memory_pool_size: MemoryLimit::Size(ReadableSize(1024)),
783                ..Default::default()
784            },
785        );
786
787        assert!(matches!(
788            state
789                .session_state()
790                .runtime_env()
791                .memory_pool
792                .memory_limit(),
793            DfMemoryLimit::Finite(1024)
794        ));
795    }
796
797    #[test]
798    fn query_runtime_provider_from_plugins_builds_runtime_env() {
799        let plugins = Plugins::default();
800        let provider = Arc::new(TestRuntimeProvider::new());
801        plugins.insert::<QueryRuntimeProviderRef>(provider.clone());
802
803        let state = new_query_engine_state_with(
804            plugins,
805            QueryOptions {
806                memory_pool_size: MemoryLimit::Size(ReadableSize(1024)),
807                ..Default::default()
808            },
809        );
810
811        assert!(provider.build_called.load(Ordering::SeqCst));
812        assert!(matches!(
813            state
814                .session_state()
815                .runtime_env()
816                .memory_pool
817                .memory_limit(),
818            DfMemoryLimit::Finite(2048)
819        ));
820    }
821
822    #[test]
823    fn query_runtime_provider_from_plugins_configures_session_config() {
824        let plugins = Plugins::default();
825        let provider = Arc::new(TestRuntimeProvider::new());
826        plugins.insert::<QueryRuntimeProviderRef>(provider.clone());
827
828        let state = new_query_engine_state_with(
829            plugins,
830            QueryOptions {
831                memory_pool_size: MemoryLimit::Size(ReadableSize(1024)),
832                experimental_spill_mode: QuerySpillMode::Custom,
833                experimental_spill_compression: QuerySpillCompression::Zstd,
834                ..Default::default()
835            },
836        );
837
838        assert!(provider.configure_called.load(Ordering::SeqCst));
839        assert_eq!(7, state.session_state().config().target_partitions());
840        assert_eq!(
841            SpillCompression::Zstd,
842            state
843                .session_state()
844                .config()
845                .options()
846                .execution
847                .spill_compression
848        );
849    }
850
851    #[test]
852    fn query_runtime_provider_error_is_returned_by_try_new() {
853        let plugins = Plugins::default();
854        plugins.insert::<QueryRuntimeProviderRef>(Arc::new(ErrorRuntimeProvider));
855
856        let err = match QueryEngineState::try_new(
857            catalog::memory::new_memory_catalog_manager().unwrap(),
858            None,
859            None,
860            None,
861            None,
862            None,
863            false,
864            plugins,
865            QueryOptions::default(),
866        ) {
867            Err(err) => err,
868            Ok(_) => panic!("expected runtime provider error"),
869        };
870
871        assert!(
872            matches!(err, DataFusionError::Execution(message) if message == "runtime provider error")
873        );
874    }
875
876    #[test]
877    fn query_engine_state_reuses_query_scoped_dyn_filter_registry_lease() {
878        let state = new_query_engine_state();
879        let query_ctx = QueryContext::arc();
880
881        let first = state
882            .acquire_remote_dyn_filter_registry_lease(&query_ctx)
883            .unwrap();
884        let second = state
885            .acquire_remote_dyn_filter_registry_lease(&query_ctx)
886            .unwrap();
887
888        assert!(first.ptr_eq(&second));
889        assert_eq!(state.dyn_filter_registry_manager().registry_count(), 1);
890        assert_eq!(
891            first.registry().query_id(),
892            query_ctx.remote_query_id_value().unwrap()
893        );
894    }
895
896    #[test]
897    fn query_engine_state_relies_on_query_context_remote_query_id_contract() {
898        let state = new_query_engine_state();
899        let query_ctx = QueryContext::arc();
900
901        assert!(query_ctx.remote_query_id_value().is_some());
902
903        let lease = state
904            .acquire_remote_dyn_filter_registry_lease(&query_ctx)
905            .unwrap();
906
907        assert_eq!(
908            lease.registry().query_id(),
909            query_ctx.remote_query_id_value().unwrap()
910        );
911        assert_eq!(state.dyn_filter_registry_manager().registry_count(), 1);
912    }
913
914    #[test]
915    fn query_engine_state_separates_registries_for_different_query_contexts() {
916        let state = new_query_engine_state();
917        let first_query_ctx = QueryContext::arc();
918        let second_query_ctx = QueryContext::arc();
919
920        let first = state
921            .acquire_remote_dyn_filter_registry_lease(&first_query_ctx)
922            .unwrap();
923        let second = state
924            .acquire_remote_dyn_filter_registry_lease(&second_query_ctx)
925            .unwrap();
926
927        assert!(!first.ptr_eq(&second));
928        assert_eq!(state.dyn_filter_registry_manager().registry_count(), 2);
929        assert_eq!(
930            first.registry().query_id(),
931            first_query_ctx.remote_query_id_value().unwrap()
932        );
933        assert_eq!(
934            second.registry().query_id(),
935            second_query_ctx.remote_query_id_value().unwrap()
936        );
937    }
938
939    /// Builds a runtime env through the default provider seam, mirroring what
940    /// [`QueryEngineState::try_new`] does.
941    fn build_runtime_env(options: &QueryOptions, memory_pool_size: usize) -> Arc<RuntimeEnv> {
942        let ctx = QueryRuntimeContext::new(options, memory_pool_size);
943        let builder = DefaultQueryRuntimeProvider::runtime_env_builder(ctx);
944        DefaultQueryRuntimeProvider
945            .build_runtime_env(ctx, builder)
946            .expect("Failed to build RuntimeEnv")
947    }
948
949    #[test]
950    fn test_build_runtime_env_custom_mode_with_path() {
951        // Use a temp directory managed manually so we don't need the `tempfile` crate.
952        let spill_dir = std::env::temp_dir().join(format!("df_spill_test_{}", std::process::id()));
953        let _ = std::fs::remove_dir_all(&spill_dir);
954        let opts = QueryOptions {
955            experimental_spill_mode: QuerySpillMode::Custom,
956            experimental_spill_path: Some(spill_dir.clone()),
957            experimental_spill_max_temp_directory_size: ReadableSize::gb(1),
958            ..Default::default()
959        };
960        let env = build_runtime_env(&opts, 0);
961
962        assert!(env.disk_manager.tmp_files_enabled());
963        let tmp_file = env.disk_manager.create_tmp_file("test spill");
964        assert!(tmp_file.is_ok());
965        assert!(spill_dir.exists());
966
967        let _ = std::fs::remove_dir_all(&spill_dir);
968    }
969
970    #[test]
971    fn test_build_runtime_env_disabled_mode() {
972        let opts = QueryOptions {
973            experimental_spill_mode: QuerySpillMode::Disabled,
974            ..Default::default()
975        };
976        let env = build_runtime_env(&opts, 0);
977
978        assert!(!env.disk_manager.tmp_files_enabled());
979        let result = env.disk_manager.create_tmp_file("test spill");
980        assert!(result.is_err());
981        if let Err(error) = result {
982            assert!(format!("{error}").contains("DiskManager is disabled"));
983        }
984    }
985
986    #[test]
987    fn test_metrics_memory_pool_policy_differs_with_multiple_spillable_consumers() {
988        for (policy, greedy) in [
989            (QueryMemoryPoolPolicy::Greedy, true),
990            (QueryMemoryPoolPolicy::Fair, false),
991        ] {
992            let pool: Arc<dyn MemoryPool> = Arc::new(MetricsMemoryPool::new(100, policy));
993            let first = MemoryConsumer::new("first")
994                .with_can_spill(true)
995                .register(&pool);
996            let second = MemoryConsumer::new("second")
997                .with_can_spill(true)
998                .register(&pool);
999
1000            let first_result = first.try_grow(75);
1001            assert_eq!(first_result.is_ok(), greedy);
1002            if greedy {
1003                assert!(second.try_grow(26).is_err());
1004            } else {
1005                first.try_grow(50).unwrap();
1006                second.try_grow(50).unwrap();
1007            }
1008        }
1009    }
1010
1011    /// Builds a runtime with custom spill configuration and a bounded memory
1012    /// pool, then runs sort queries that probe spill-to-disk behaviour:
1013    ///
1014    /// - A pool large enough for merge chunks but still much smaller
1015    ///   than the total data.  Executes the physical plan directly so we can
1016    ///   walk the plan tree afterwards and sum `spill_count` / `spilled_rows` /
1017    ///   `spilled_bytes` on `SortExec` nodes.  All three must be > 0.
1018    #[tokio::test]
1019    async fn test_sort_spill_smoke_with_custom_runtime() {
1020        use arrow::array::Int32Array;
1021        use arrow::datatypes::{DataType, Field, Schema};
1022        use arrow::record_batch::RecordBatch;
1023        use datafusion::datasource::MemTable;
1024        use datafusion::physical_plan::collect as df_collect;
1025
1026        let spill_dir =
1027            std::env::temp_dir().join(format!("greptime_spill_smoke_{}", std::process::id()));
1028        let _ = std::fs::remove_dir_all(&spill_dir);
1029
1030        let opts = QueryOptions {
1031            experimental_spill_mode: QuerySpillMode::Custom,
1032            experimental_spill_path: Some(spill_dir.clone()),
1033            experimental_spill_max_temp_directory_size: ReadableSize::gb(1),
1034            experimental_memory_pool_policy: QueryMemoryPoolPolicy::Greedy,
1035            ..Default::default()
1036        };
1037
1038        let session_config = SessionConfig::new()
1039            .with_target_partitions(1)
1040            .with_sort_in_place_threshold_bytes(0)
1041            .with_sort_spill_reservation_bytes(64 * 1024);
1042
1043        let schema = Arc::new(Schema::new(vec![
1044            Field::new("id", DataType::Int32, false),
1045            Field::new("val", DataType::Int32, false),
1046        ]));
1047        let n_rows: i32 = 200_000;
1048        let batch_size: i32 = 5_000;
1049        let partitions = 1usize;
1050
1051        let mut table_partitions: Vec<Vec<RecordBatch>> = Vec::with_capacity(partitions);
1052        for _ in 0..partitions {
1053            let mut batches = Vec::new();
1054            let mut row_offset: i32 = 0;
1055            while row_offset < n_rows {
1056                let chunk_end = (row_offset + batch_size).min(n_rows);
1057                let chunk_len = (chunk_end - row_offset) as usize;
1058                let mut id_builder = Int32Array::builder(chunk_len);
1059                let mut val_builder = Int32Array::builder(chunk_len);
1060                for i in row_offset..chunk_end {
1061                    id_builder.append_value(i);
1062                    val_builder.append_value(n_rows - 1 - i);
1063                }
1064                batches.push(
1065                    RecordBatch::try_new(
1066                        Arc::clone(&schema),
1067                        vec![
1068                            Arc::new(id_builder.finish()),
1069                            Arc::new(val_builder.finish()),
1070                        ],
1071                    )
1072                    .unwrap(),
1073                );
1074                row_offset = chunk_end;
1075            }
1076            table_partitions.push(batches);
1077        }
1078
1079        fn build_ctx(
1080            session_config: &SessionConfig,
1081            runtime: &Arc<RuntimeEnv>,
1082            schema: &Arc<Schema>,
1083            partitions: &[Vec<RecordBatch>],
1084        ) -> SessionContext {
1085            let session_state = SessionStateBuilder::new()
1086                .with_config(session_config.clone())
1087                .with_runtime_env(Arc::clone(runtime))
1088                .with_default_features()
1089                .build();
1090            let ctx = SessionContext::new_with_state(session_state);
1091            let table = MemTable::try_new(Arc::clone(schema), partitions.to_vec()).unwrap();
1092            ctx.register_table("t", Arc::new(table)).unwrap();
1093            ctx
1094        }
1095
1096        {
1097            let mem_limit: usize = 512 * 1024; // 512 KB pool, 64 KB reserved for merge
1098
1099            let runtime = build_runtime_env(&opts, mem_limit);
1100            assert_eq!(runtime.memory_pool.reserved(), 0);
1101            let ctx = build_ctx(&session_config, &runtime, &schema, &table_partitions);
1102
1103            let df = ctx
1104                .sql("SELECT val FROM t ORDER BY val ASC")
1105                .await
1106                .expect("planning ORDER BY");
1107            let plan = df
1108                .create_physical_plan()
1109                .await
1110                .expect("creating physical plan");
1111            let task_ctx = ctx.task_ctx();
1112
1113            let batches = df_collect(Arc::clone(&plan), task_ctx)
1114                .await
1115                .expect("executing ORDER BY");
1116
1117            let (spill_count, spilled_rows, spilled_bytes) = sum_sort_spill_metrics(&plan);
1118            assert!(
1119                spill_count > 0,
1120                "expected SortExec spill_count > 0 (pool={} KB, reservation=64 KB), got 0",
1121                mem_limit / 1024,
1122            );
1123            assert!(
1124                spilled_rows > 0,
1125                "expected SortExec spilled_rows > 0, got 0 \
1126                 (spill_count={spill_count}, spilled_bytes={spilled_bytes})",
1127            );
1128            assert!(
1129                spilled_bytes > 0,
1130                "expected SortExec spilled_bytes > 0, got 0 \
1131                 (spill_count={spill_count}, spilled_rows={spilled_rows})",
1132            );
1133
1134            let vals: Vec<i32> = batches
1135                .iter()
1136                .flat_map(|b| {
1137                    let col = b.column(0).as_any().downcast_ref::<Int32Array>().unwrap();
1138                    (0..b.num_rows()).map(move |i| col.value(i))
1139                })
1140                .collect();
1141            assert_eq!(vals.len(), n_rows as usize);
1142            for w in vals.windows(2) {
1143                assert!(w[0] <= w[1], "sort order violation: {} > {}", w[0], w[1]);
1144            }
1145
1146            assert!(
1147                std::fs::read_dir(&spill_dir)
1148                    .ok()
1149                    .map(|mut entries| entries.any(|e| {
1150                        e.as_ref()
1151                            .map(|de| de.file_name().to_string_lossy().starts_with("datafusion-"))
1152                            .unwrap_or(false)
1153                    }))
1154                    .unwrap_or(false),
1155                "Expected 'datafusion-*' directory in spill path {:?} \
1156                 (DiskManager should create it on build).",
1157                spill_dir,
1158            );
1159
1160            assert_eq!(runtime.memory_pool.reserved(), 0);
1161        }
1162
1163        let _ = std::fs::remove_dir_all(&spill_dir);
1164    }
1165
1166    /// Walk a physical plan tree, summing spill metrics for every node whose
1167    /// name starts with `"SortExec"`.
1168    fn sum_sort_spill_metrics(plan: &Arc<dyn ExecutionPlan>) -> (usize, usize, usize) {
1169        let mut spill_count = 0usize;
1170        let mut spilled_rows = 0usize;
1171        let mut spilled_bytes = 0usize;
1172
1173        fn walk(plan: &Arc<dyn ExecutionPlan>, sc: &mut usize, sr: &mut usize, sb: &mut usize) {
1174            let name = plan.name();
1175            if name.starts_with("SortExec")
1176                && let Some(m) = plan.metrics()
1177            {
1178                *sc += m.spill_count().unwrap_or(0);
1179                *sr += m.spilled_rows().unwrap_or(0);
1180                *sb += m.spilled_bytes().unwrap_or(0);
1181            }
1182            for child in plan.children() {
1183                walk(child, sc, sr, sb);
1184            }
1185        }
1186
1187        walk(
1188            plan,
1189            &mut spill_count,
1190            &mut spilled_rows,
1191            &mut spilled_bytes,
1192        );
1193        (spill_count, spilled_rows, spilled_bytes)
1194    }
1195}