common_function/
function_registry.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! functions registry
16use std::collections::HashMap;
17use std::collections::hash_map::Entry;
18use std::sync::{Arc, LazyLock, RwLock};
19
20use datafusion::catalog::TableFunction;
21use datafusion_expr::expr_rewriter::FunctionRewrite;
22use datafusion_expr::{AggregateUDF, WindowUDF};
23
24use crate::admin::AdminFunction;
25use crate::aggrs::aggr_wrapper::StateMergeHelper;
26use crate::aggrs::approximate::ApproximateFunction;
27use crate::aggrs::count_hash::CountHash;
28use crate::aggrs::vector::VectorFunction as VectorAggrFunction;
29use crate::function::{Function, FunctionRef};
30use crate::function_factory::ScalarFunctionFactory;
31use crate::scalars::anomaly::AnomalyFunction;
32use crate::scalars::date::DateFunction;
33use crate::scalars::expression::ExpressionFunction;
34use crate::scalars::hll_count::HllCalcFunction;
35use crate::scalars::ip::IpFunctions;
36use crate::scalars::json::JsonFunction;
37use crate::scalars::matches::MatchesFunction;
38use crate::scalars::matches_term::MatchesTermFunction;
39use crate::scalars::math::MathFunction;
40use crate::scalars::primary_key::DecodePrimaryKeyFunction;
41use crate::scalars::string::register_string_functions;
42use crate::scalars::timestamp::TimestampFunction;
43use crate::scalars::uddsketch_calc::UddSketchCalcFunction;
44use crate::scalars::uddsketch_rank::UddSketchRankFunction;
45use crate::scalars::vector::VectorFunction as VectorScalarFunction;
46use crate::scalars::welford_stddev::WelfordStddevFunction;
47use crate::system::SystemFunction;
48
49#[derive(Default)]
50pub struct FunctionRegistry {
51    functions: RwLock<HashMap<String, ScalarFunctionFactory>>,
52    aggregate_functions: RwLock<HashMap<String, AggregateUDF>>,
53    table_functions: RwLock<HashMap<String, Arc<TableFunction>>>,
54    function_rewrites: RwLock<Vec<Arc<dyn FunctionRewrite + Send + Sync>>>,
55    window_functions: RwLock<HashMap<String, WindowUDF>>,
56}
57
58/// The result of registering a function.
59#[derive(Debug, Clone, Copy, PartialEq, Eq)]
60pub enum FunctionRegistrationResult {
61    /// The function was newly registered.
62    Registered,
63    /// A function with the same name was already registered and was kept.
64    AlreadyExists,
65}
66
67impl FunctionRegistry {
68    /// Register a function in the registry by converting it into a `ScalarFunctionFactory`.
69    ///
70    /// # Arguments
71    ///
72    /// * `func` - An object that can be converted into a `ScalarFunctionFactory`.
73    ///
74    /// The function is inserted into the internal function map, keyed by its name.
75    /// If a function with the same name already exists, it will be replaced.
76    pub fn register(&self, func: impl Into<ScalarFunctionFactory>) {
77        let func = func.into();
78        let _ = self
79            .functions
80            .write()
81            .unwrap()
82            .insert(func.name().to_string(), func);
83    }
84
85    /// Register a function only if no function with the same name exists.
86    ///
87    /// The duplicate check and the insert happen atomically under the same
88    /// write lock of the functions map. If a function with the same name
89    /// already exists, it is kept unchanged and
90    /// [`FunctionRegistrationResult::AlreadyExists`] is returned; otherwise the
91    /// function is registered and [`FunctionRegistrationResult::Registered`] is
92    /// returned.
93    pub fn register_if_absent(
94        &self,
95        func: impl Into<ScalarFunctionFactory>,
96    ) -> FunctionRegistrationResult {
97        let func = func.into();
98        let mut functions = self.functions.write().unwrap();
99        match functions.entry(func.name().to_string()) {
100            Entry::Occupied(_) => FunctionRegistrationResult::AlreadyExists,
101            Entry::Vacant(entry) => {
102                entry.insert(func);
103                FunctionRegistrationResult::Registered
104            }
105        }
106    }
107
108    /// Register a scalar function in the registry.
109    pub fn register_scalar(&self, func: impl Function + 'static) {
110        let func = Arc::new(func) as FunctionRef;
111
112        for alias in func.aliases() {
113            let func: ScalarFunctionFactory = func.clone().into();
114            let alias = ScalarFunctionFactory {
115                name: alias.clone(),
116                ..func
117            };
118            self.register(alias);
119        }
120
121        self.register(func)
122    }
123
124    /// Register an aggregate function in the registry.
125    pub fn register_aggr(&self, func: AggregateUDF) {
126        let _ = self
127            .aggregate_functions
128            .write()
129            .unwrap()
130            .insert(func.name().to_string(), func);
131    }
132
133    /// Register a table function
134    pub fn register_table_function(&self, func: TableFunction) {
135        let _ = self
136            .table_functions
137            .write()
138            .unwrap()
139            .insert(func.name().to_string(), Arc::new(func));
140    }
141
142    /// Register a function rewrite rule.
143    pub fn register_function_rewrite(&self, func: impl FunctionRewrite + Send + Sync + 'static) {
144        self.function_rewrites.write().unwrap().push(Arc::new(func));
145    }
146
147    /// Register a window function (UDWF).
148    pub fn register_window(&self, func: WindowUDF) {
149        let _ = self
150            .window_functions
151            .write()
152            .unwrap()
153            .insert(func.name().to_string(), func);
154    }
155
156    pub fn get_function(&self, name: &str) -> Option<ScalarFunctionFactory> {
157        self.functions.read().unwrap().get(name).cloned()
158    }
159
160    /// Returns a list of all scalar functions registered in the registry.
161    pub fn scalar_functions(&self) -> Vec<ScalarFunctionFactory> {
162        self.functions.read().unwrap().values().cloned().collect()
163    }
164
165    /// Returns a list of all aggregate functions registered in the registry.
166    pub fn aggregate_functions(&self) -> Vec<AggregateUDF> {
167        self.aggregate_functions
168            .read()
169            .unwrap()
170            .values()
171            .cloned()
172            .collect()
173    }
174
175    pub fn table_functions(&self) -> Vec<Arc<TableFunction>> {
176        self.table_functions
177            .read()
178            .unwrap()
179            .values()
180            .cloned()
181            .collect()
182    }
183
184    /// Returns a list of all window functions registered in the registry.
185    pub fn window_functions(&self) -> Vec<WindowUDF> {
186        self.window_functions
187            .read()
188            .unwrap()
189            .values()
190            .cloned()
191            .collect()
192    }
193
194    /// Returns true if an aggregate function with the given name exists in the registry.
195    pub fn is_aggr_func_exist(&self, name: &str) -> bool {
196        self.aggregate_functions.read().unwrap().contains_key(name)
197    }
198
199    /// Returns a list of all function rewrite rules registered in the registry.
200    pub fn function_rewrites(&self) -> Vec<Arc<dyn FunctionRewrite + Send + Sync>> {
201        self.function_rewrites.read().unwrap().clone()
202    }
203}
204
205pub static FUNCTION_REGISTRY: LazyLock<Arc<FunctionRegistry>> = LazyLock::new(|| {
206    let function_registry = FunctionRegistry::default();
207
208    // Utility functions
209    MathFunction::register(&function_registry);
210    TimestampFunction::register(&function_registry);
211    DateFunction::register(&function_registry);
212    ExpressionFunction::register(&function_registry);
213    UddSketchCalcFunction::register(&function_registry);
214    UddSketchRankFunction::register(&function_registry);
215    HllCalcFunction::register(&function_registry);
216    WelfordStddevFunction::register(&function_registry);
217    DecodePrimaryKeyFunction::register(&function_registry);
218
219    // Full text search function
220    MatchesFunction::register(&function_registry);
221    MatchesTermFunction::register(&function_registry);
222
223    // System and administration functions
224    SystemFunction::register(&function_registry);
225    AdminFunction::register(&function_registry);
226
227    // Json related functions
228    JsonFunction::register(&function_registry);
229
230    // String related functions
231    register_string_functions(&function_registry);
232
233    // Vector related functions
234    VectorScalarFunction::register(&function_registry);
235    VectorAggrFunction::register(&function_registry);
236
237    // Geo functions
238    #[cfg(feature = "geo")]
239    crate::scalars::geo::GeoFunctions::register(&function_registry);
240    #[cfg(feature = "geo")]
241    crate::aggrs::geo::GeoFunction::register(&function_registry);
242
243    // Ip functions
244    IpFunctions::register(&function_registry);
245
246    // Approximate functions
247    ApproximateFunction::register(&function_registry);
248
249    // CountHash function
250    CountHash::register(&function_registry);
251
252    // state function of supported aggregate functions
253    StateMergeHelper::register(&function_registry);
254
255    // Anomaly detection window functions
256    AnomalyFunction::register(&function_registry);
257
258    Arc::new(function_registry)
259});
260
261static ADMIN_FUNCTION_REGISTRY: LazyLock<FunctionRegistry> = LazyLock::new(|| {
262    let registry = FunctionRegistry::default();
263    AdminFunction::register_admin_only(&registry);
264    registry
265});
266
267/// Returns a function that is only available to the ADMIN statement executor.
268pub fn get_admin_function(name: &str) -> Option<ScalarFunctionFactory> {
269    ADMIN_FUNCTION_REGISTRY.get_function(name)
270}
271
272/// Register a function that is only available to the ADMIN statement executor.
273///
274/// If a function with the same name is already registered in the ADMIN
275/// registry, the existing one is kept and
276/// [`FunctionRegistrationResult::AlreadyExists`] is returned. A name that
277/// already exists in the normal [`FUNCTION_REGISTRY`] when this call
278/// linearizes is also rejected: the ADMIN executor resolves admin-only
279/// functions before falling back to the normal registry, so inserting such a
280/// name here would shadow the built-in. Otherwise the function is registered
281/// and [`FunctionRegistrationResult::Registered`] is returned.
282///
283/// The enforced contract is one-way: it only guards the ADMIN registration
284/// against names already present in the normal registry. A later ordinary
285/// [`FunctionRegistry::register`] may still install the same name in the
286/// normal registry because the normal registry keeps its legacy replace
287/// semantics.
288pub fn register_admin_function(
289    func: impl Into<ScalarFunctionFactory>,
290) -> FunctionRegistrationResult {
291    register_admin_function_in(&ADMIN_FUNCTION_REGISTRY, &FUNCTION_REGISTRY, func)
292}
293
294/// Core implementation of [`register_admin_function`] against a pair of
295/// registries, parameterized so tests can exercise it with local registries.
296///
297/// Locking: the ADMIN-registry write lock is acquired first, then a read lock
298/// on the normal registry, and the normal-registry guard (bound to
299/// `normal_functions`) is kept alive through both the normal-name check and
300/// the ADMIN insertion below. This is the only code path that holds both
301/// registries' locks, so the ADMIN -> FUNCTION acquisition order is
302/// consistent and a concurrent normal-registry registration cannot slip in
303/// between the check and the ADMIN insert and be shadowed.
304///
305/// The enforced contract is one-way: it only guards the ADMIN registration
306/// against names already present in the normal registry. A later ordinary
307/// [`FunctionRegistry::register`] may still install the same name in the
308/// normal registry because the normal registry keeps its legacy replace
309/// semantics.
310fn register_admin_function_in(
311    admin_registry: &FunctionRegistry,
312    normal_registry: &FunctionRegistry,
313    func: impl Into<ScalarFunctionFactory>,
314) -> FunctionRegistrationResult {
315    let func = func.into();
316    let mut admin_functions = admin_registry.functions.write().unwrap();
317    // The normal-registry guard is a read lock: it is held across the
318    // normal-name check and the ADMIN insertion below, and while it is alive
319    // no writer can acquire the normal-registry write lock, so a concurrent
320    // normal-registry registration cannot slip in between the check and the
321    // ADMIN insert and be shadowed.
322    let normal_functions = normal_registry.functions.read().unwrap();
323    if normal_functions.contains_key(func.name()) {
324        drop(normal_functions);
325        return FunctionRegistrationResult::AlreadyExists;
326    }
327    let result = match admin_functions.entry(func.name().to_string()) {
328        Entry::Occupied(_) => FunctionRegistrationResult::AlreadyExists,
329        Entry::Vacant(entry) => {
330            entry.insert(func);
331            FunctionRegistrationResult::Registered
332        }
333    };
334    // Drop the read guard only after the ADMIN insertion, so writers to the
335    // normal registry stay blocked until the check-and-insert is complete.
336    drop(normal_functions);
337    result
338}
339
340#[cfg(test)]
341mod tests {
342    use std::sync::{Arc, Barrier};
343    use std::thread;
344
345    use super::*;
346    use crate::scalars::test::TestAndFunction;
347    use crate::scalars::udf::create_udf;
348
349    /// Creates a [`ScalarFunctionFactory`] with the given name. Each call
350    /// allocates a distinct factory closure, so factories can be told apart by
351    /// [`Arc::ptr_eq`] on their `factory` field even when names are identical.
352    fn named_factory(name: &str) -> ScalarFunctionFactory {
353        ScalarFunctionFactory {
354            name: name.to_string(),
355            factory: Arc::new(|_ctx| create_udf(Arc::new(TestAndFunction::default()))),
356        }
357    }
358
359    #[test]
360    fn test_function_registry() {
361        let registry = FunctionRegistry::default();
362
363        assert!(registry.get_function("test_and").is_none());
364        assert!(registry.scalar_functions().is_empty());
365        registry.register_scalar(TestAndFunction::default());
366        let _ = registry.get_function("test_and").unwrap();
367        assert_eq!(1, registry.scalar_functions().len());
368    }
369
370    #[test]
371    fn test_uddsketch_rank_registered() {
372        assert!(FUNCTION_REGISTRY.get_function("uddsketch_rank").is_some());
373    }
374
375    #[test]
376    fn test_register_if_absent_registers_new_function() {
377        let registry = FunctionRegistry::default();
378        let name = "pr3_register_if_absent_new";
379        let factory = named_factory(name);
380
381        assert_eq!(
382            registry.register_if_absent(factory.clone()),
383            FunctionRegistrationResult::Registered
384        );
385        let registered = registry
386            .get_function(name)
387            .expect("function should be registered");
388        assert!(Arc::ptr_eq(&registered.factory, &factory.factory));
389    }
390
391    #[test]
392    fn test_register_if_absent_first_registration_wins() {
393        let registry = FunctionRegistry::default();
394        let name = "pr3_register_if_absent_duplicate";
395        let first = named_factory(name);
396        let second = named_factory(name);
397
398        assert_eq!(
399            registry.register_if_absent(first.clone()),
400            FunctionRegistrationResult::Registered
401        );
402        assert_eq!(
403            registry.register_if_absent(second.clone()),
404            FunctionRegistrationResult::AlreadyExists
405        );
406
407        let stored = registry
408            .get_function(name)
409            .expect("function should be registered");
410        assert!(Arc::ptr_eq(&stored.factory, &first.factory));
411        assert!(!Arc::ptr_eq(&stored.factory, &second.factory));
412    }
413
414    #[test]
415    fn test_register_replaces_existing_function() {
416        // Regression test: `register` keeps its replace semantics.
417        let registry = FunctionRegistry::default();
418        let name = "pr3_register_replaces";
419        let first = named_factory(name);
420        let second = named_factory(name);
421
422        registry.register(first.clone());
423        registry.register(second.clone());
424
425        let stored = registry
426            .get_function(name)
427            .expect("function should be registered");
428        assert!(Arc::ptr_eq(&stored.factory, &second.factory));
429        assert!(!Arc::ptr_eq(&stored.factory, &first.factory));
430    }
431
432    #[test]
433    fn test_concurrent_register_if_absent_same_name() {
434        const THREADS: usize = 8;
435        let registry = Arc::new(FunctionRegistry::default());
436        let name = "pr3_concurrent_same_name";
437        let barrier = Arc::new(Barrier::new(THREADS));
438
439        let handles: Vec<_> = (0..THREADS)
440            .map(|_| {
441                let registry = Arc::clone(&registry);
442                let barrier = Arc::clone(&barrier);
443                thread::spawn(move || {
444                    let factory = named_factory(name);
445                    // Synchronize so every thread attempts registration at the
446                    // same time; only one may win the write lock.
447                    barrier.wait();
448                    let result = registry.register_if_absent(factory.clone());
449                    (result, factory)
450                })
451            })
452            .collect();
453
454        let mut results: Vec<(FunctionRegistrationResult, ScalarFunctionFactory)> =
455            Vec::with_capacity(THREADS);
456        for handle in handles {
457            results.push(handle.join().expect("thread should not panic"));
458        }
459
460        let registered = results
461            .iter()
462            .filter(|(result, _)| *result == FunctionRegistrationResult::Registered)
463            .count();
464        let already_exists = results
465            .iter()
466            .filter(|(result, _)| *result == FunctionRegistrationResult::AlreadyExists)
467            .count();
468        assert_eq!(registered, 1);
469        assert_eq!(already_exists, THREADS - 1);
470
471        let winner = results
472            .iter()
473            .find(|(result, _)| *result == FunctionRegistrationResult::Registered)
474            .map(|(_, factory)| factory)
475            .expect("exactly one registration must win");
476
477        let stored = registry
478            .get_function(name)
479            .expect("function should be registered");
480        assert!(
481            Arc::ptr_eq(&stored.factory, &winner.factory),
482            "the stored factory must be the factory of the winning registration"
483        );
484    }
485
486    #[test]
487    fn test_register_admin_function_first_wins() {
488        // Tests touching the global registry must use unique names.
489        let name = "pr3_admin_runtime_register";
490        let first = named_factory(name);
491        let second = named_factory(name);
492
493        assert_eq!(
494            register_admin_function(first.clone()),
495            FunctionRegistrationResult::Registered
496        );
497        assert_eq!(
498            register_admin_function(second.clone()),
499            FunctionRegistrationResult::AlreadyExists
500        );
501
502        let stored = get_admin_function(name).expect("admin function should be queryable");
503        assert!(Arc::ptr_eq(&stored.factory, &first.factory));
504        assert!(!Arc::ptr_eq(&stored.factory, &second.factory));
505    }
506
507    #[test]
508    fn test_register_admin_function_duplicate_same_factory() {
509        // The exact same factory clone registered twice: the first registration
510        // wins, the duplicate is rejected, and the stored factory is
511        // pointer-equal to the original. Tests touching the global registry
512        // must use unique names.
513        let name = "pr3_admin_runtime_register_same_factory";
514        let factory = named_factory(name);
515
516        assert_eq!(
517            register_admin_function(factory.clone()),
518            FunctionRegistrationResult::Registered
519        );
520        assert_eq!(
521            register_admin_function(factory.clone()),
522            FunctionRegistrationResult::AlreadyExists
523        );
524
525        let stored = get_admin_function(name).expect("admin function should be queryable");
526        assert!(Arc::ptr_eq(&stored.factory, &factory.factory));
527    }
528
529    #[test]
530    fn test_register_admin_function_in_is_one_way_later_normal_register_allowed() {
531        // The guard is one-way: after the ADMIN registration completes, an
532        // ordinary normal-registry registration of the same name still
533        // succeeds because the normal registry keeps its legacy replace
534        // semantics.
535        let admin = FunctionRegistry::default();
536        let normal = FunctionRegistry::default();
537        let name = "pr3_admin_in_one_way";
538        let admin_factory = named_factory(name);
539        let normal_factory = named_factory(name);
540
541        assert_eq!(
542            register_admin_function_in(&admin, &normal, admin_factory.clone()),
543            FunctionRegistrationResult::Registered
544        );
545
546        normal.register(normal_factory.clone());
547
548        let stored_admin = admin
549            .get_function(name)
550            .expect("the ADMIN registration must be kept");
551        assert!(Arc::ptr_eq(&stored_admin.factory, &admin_factory.factory));
552        let stored_normal = normal
553            .get_function(name)
554            .expect("the later normal registration must succeed");
555        assert!(Arc::ptr_eq(&stored_normal.factory, &normal_factory.factory));
556    }
557
558    #[test]
559    fn test_register_admin_function_rejects_normal_registry_builtin_name() {
560        // Regression test: the ADMIN executor resolves `get_admin_function`
561        // before falling back to `FUNCTION_REGISTRY`, so registering a function
562        // whose name already exists in the normal registry would shadow the
563        // ADMIN-invocable built-in (e.g. `flush_table`). Such registrations
564        // must be rejected with
565        // [`FunctionRegistrationResult::AlreadyExists`] and must not be
566        // inserted into the ADMIN registry.
567        let factory = named_factory("flush_table");
568
569        assert_eq!(
570            register_admin_function(factory.clone()),
571            FunctionRegistrationResult::AlreadyExists
572        );
573        assert!(
574            get_admin_function("flush_table").is_none(),
575            "a normal-registry built-in must not be shadowed into the ADMIN registry"
576        );
577        assert!(
578            FUNCTION_REGISTRY.get_function("flush_table").is_some(),
579            "the normal-registry built-in must remain registered"
580        );
581    }
582
583    #[test]
584    fn test_builtin_admin_functions_remain_queryable() {
585        // Built-in admin-only functions registered at startup stay queryable
586        // through the same global registry used for runtime registrations.
587        #[cfg(feature = "enterprise")]
588        {
589            assert!(get_admin_function("purge_table").is_some());
590        }
591        #[cfg(not(feature = "enterprise"))]
592        {
593            assert!(get_admin_function("purge_table").is_none());
594        }
595    }
596}
common_function/function_registry.rs

common_function/
function_registry.rs
