Skip to main content

common_function/
function_registry.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! functions registry
16use std::collections::HashMap;
17use std::collections::hash_map::Entry;
18use std::sync::{Arc, LazyLock, RwLock};
19
20use datafusion::catalog::TableFunction;
21use datafusion_expr::expr_rewriter::FunctionRewrite;
22use datafusion_expr::{AggregateUDF, WindowUDF};
23
24use crate::admin::AdminFunction;
25use crate::aggrs::aggr_wrapper::StateMergeHelper;
26use crate::aggrs::approximate::ApproximateFunction;
27use crate::aggrs::count_hash::CountHash;
28use crate::aggrs::vector::VectorFunction as VectorAggrFunction;
29use crate::function::{Function, FunctionRef};
30use crate::function_factory::ScalarFunctionFactory;
31use crate::scalars::anomaly::AnomalyFunction;
32use crate::scalars::avg_calc::AvgCalcFunction;
33use crate::scalars::date::DateFunction;
34use crate::scalars::expression::ExpressionFunction;
35use crate::scalars::hll_count::HllCalcFunction;
36use crate::scalars::ip::IpFunctions;
37use crate::scalars::json::JsonFunction;
38use crate::scalars::matches::MatchesFunction;
39use crate::scalars::matches_term::MatchesTermFunction;
40use crate::scalars::math::MathFunction;
41use crate::scalars::primary_key::DecodePrimaryKeyFunction;
42use crate::scalars::string::register_string_functions;
43use crate::scalars::timestamp::TimestampFunction;
44use crate::scalars::uddsketch_calc::UddSketchCalcFunction;
45use crate::scalars::uddsketch_rank::UddSketchRankFunction;
46use crate::scalars::vector::VectorFunction as VectorScalarFunction;
47use crate::scalars::welford_stddev::WelfordStddevFunction;
48use crate::system::SystemFunction;
49
50#[derive(Default)]
51pub struct FunctionRegistry {
52    functions: RwLock<HashMap<String, ScalarFunctionFactory>>,
53    aggregate_functions: RwLock<HashMap<String, AggregateUDF>>,
54    table_functions: RwLock<HashMap<String, Arc<TableFunction>>>,
55    function_rewrites: RwLock<Vec<Arc<dyn FunctionRewrite + Send + Sync>>>,
56    window_functions: RwLock<HashMap<String, WindowUDF>>,
57}
58
59/// The result of registering a function.
60#[derive(Debug, Clone, Copy, PartialEq, Eq)]
61pub enum FunctionRegistrationResult {
62    /// The function was newly registered.
63    Registered,
64    /// A function with the same name was already registered and was kept.
65    AlreadyExists,
66}
67
68impl FunctionRegistry {
69    /// Register a function in the registry by converting it into a `ScalarFunctionFactory`.
70    ///
71    /// # Arguments
72    ///
73    /// * `func` - An object that can be converted into a `ScalarFunctionFactory`.
74    ///
75    /// The function is inserted into the internal function map, keyed by its name.
76    /// If a function with the same name already exists, it will be replaced.
77    pub fn register(&self, func: impl Into<ScalarFunctionFactory>) {
78        let func = func.into();
79        let _ = self
80            .functions
81            .write()
82            .unwrap()
83            .insert(func.name().to_string(), func);
84    }
85
86    /// Register a function only if no function with the same name exists.
87    ///
88    /// The duplicate check and the insert happen atomically under the same
89    /// write lock of the functions map. If a function with the same name
90    /// already exists, it is kept unchanged and
91    /// [`FunctionRegistrationResult::AlreadyExists`] is returned; otherwise the
92    /// function is registered and [`FunctionRegistrationResult::Registered`] is
93    /// returned.
94    pub fn register_if_absent(
95        &self,
96        func: impl Into<ScalarFunctionFactory>,
97    ) -> FunctionRegistrationResult {
98        let func = func.into();
99        let mut functions = self.functions.write().unwrap();
100        match functions.entry(func.name().to_string()) {
101            Entry::Occupied(_) => FunctionRegistrationResult::AlreadyExists,
102            Entry::Vacant(entry) => {
103                entry.insert(func);
104                FunctionRegistrationResult::Registered
105            }
106        }
107    }
108
109    /// Register a scalar function in the registry.
110    pub fn register_scalar(&self, func: impl Function + 'static) {
111        let func = Arc::new(func) as FunctionRef;
112
113        for alias in func.aliases() {
114            let func: ScalarFunctionFactory = func.clone().into();
115            let alias = ScalarFunctionFactory {
116                name: alias.clone(),
117                ..func
118            };
119            self.register(alias);
120        }
121
122        self.register(func)
123    }
124
125    /// Register an aggregate function in the registry.
126    pub fn register_aggr(&self, func: AggregateUDF) {
127        let _ = self
128            .aggregate_functions
129            .write()
130            .unwrap()
131            .insert(func.name().to_string(), func);
132    }
133
134    /// Register a table function
135    pub fn register_table_function(&self, func: TableFunction) {
136        let _ = self
137            .table_functions
138            .write()
139            .unwrap()
140            .insert(func.name().to_string(), Arc::new(func));
141    }
142
143    /// Register a function rewrite rule.
144    pub fn register_function_rewrite(&self, func: impl FunctionRewrite + Send + Sync + 'static) {
145        self.function_rewrites.write().unwrap().push(Arc::new(func));
146    }
147
148    /// Register a window function (UDWF).
149    pub fn register_window(&self, func: WindowUDF) {
150        let _ = self
151            .window_functions
152            .write()
153            .unwrap()
154            .insert(func.name().to_string(), func);
155    }
156
157    pub fn get_function(&self, name: &str) -> Option<ScalarFunctionFactory> {
158        self.functions.read().unwrap().get(name).cloned()
159    }
160
161    /// Returns a list of all scalar functions registered in the registry.
162    pub fn scalar_functions(&self) -> Vec<ScalarFunctionFactory> {
163        self.functions.read().unwrap().values().cloned().collect()
164    }
165
166    /// Returns a list of all aggregate functions registered in the registry.
167    pub fn aggregate_functions(&self) -> Vec<AggregateUDF> {
168        self.aggregate_functions
169            .read()
170            .unwrap()
171            .values()
172            .cloned()
173            .collect()
174    }
175
176    pub fn table_functions(&self) -> Vec<Arc<TableFunction>> {
177        self.table_functions
178            .read()
179            .unwrap()
180            .values()
181            .cloned()
182            .collect()
183    }
184
185    /// Returns a list of all window functions registered in the registry.
186    pub fn window_functions(&self) -> Vec<WindowUDF> {
187        self.window_functions
188            .read()
189            .unwrap()
190            .values()
191            .cloned()
192            .collect()
193    }
194
195    /// Returns a registered aggregate function by name.
196    pub fn get_aggr_func(&self, name: &str) -> Option<AggregateUDF> {
197        self.aggregate_functions.read().unwrap().get(name).cloned()
198    }
199
200    /// Returns true if an aggregate function with the given name exists in the registry.
201    pub fn is_aggr_func_exist(&self, name: &str) -> bool {
202        self.aggregate_functions.read().unwrap().contains_key(name)
203    }
204
205    /// Returns a list of all function rewrite rules registered in the registry.
206    pub fn function_rewrites(&self) -> Vec<Arc<dyn FunctionRewrite + Send + Sync>> {
207        self.function_rewrites.read().unwrap().clone()
208    }
209}
210
211pub static FUNCTION_REGISTRY: LazyLock<Arc<FunctionRegistry>> = LazyLock::new(|| {
212    let function_registry = FunctionRegistry::default();
213
214    // Utility functions
215    MathFunction::register(&function_registry);
216    TimestampFunction::register(&function_registry);
217    DateFunction::register(&function_registry);
218    ExpressionFunction::register(&function_registry);
219    AvgCalcFunction::register(&function_registry);
220    UddSketchCalcFunction::register(&function_registry);
221    UddSketchRankFunction::register(&function_registry);
222    HllCalcFunction::register(&function_registry);
223    WelfordStddevFunction::register(&function_registry);
224    DecodePrimaryKeyFunction::register(&function_registry);
225
226    // Full text search function
227    MatchesFunction::register(&function_registry);
228    MatchesTermFunction::register(&function_registry);
229
230    // System and administration functions
231    SystemFunction::register(&function_registry);
232    AdminFunction::register(&function_registry);
233
234    // Json related functions
235    JsonFunction::register(&function_registry);
236
237    // String related functions
238    register_string_functions(&function_registry);
239
240    // Vector related functions
241    VectorScalarFunction::register(&function_registry);
242    VectorAggrFunction::register(&function_registry);
243
244    // Geo functions
245    #[cfg(feature = "geo")]
246    crate::scalars::geo::GeoFunctions::register(&function_registry);
247    #[cfg(feature = "geo")]
248    crate::aggrs::geo::GeoFunction::register(&function_registry);
249
250    // Ip functions
251    IpFunctions::register(&function_registry);
252
253    // Approximate functions
254    ApproximateFunction::register(&function_registry);
255
256    // CountHash function
257    CountHash::register(&function_registry);
258
259    // state function of supported aggregate functions
260    StateMergeHelper::register(&function_registry);
261
262    // Anomaly detection window functions
263    AnomalyFunction::register(&function_registry);
264
265    Arc::new(function_registry)
266});
267
268static ADMIN_FUNCTION_REGISTRY: LazyLock<FunctionRegistry> = LazyLock::new(|| {
269    let registry = FunctionRegistry::default();
270    AdminFunction::register_admin_only(&registry);
271    registry
272});
273
274/// Returns a function that is only available to the ADMIN statement executor.
275pub fn get_admin_function(name: &str) -> Option<ScalarFunctionFactory> {
276    ADMIN_FUNCTION_REGISTRY.get_function(name)
277}
278
279/// Register a function that is only available to the ADMIN statement executor.
280///
281/// If a function with the same name is already registered in the ADMIN
282/// registry, the existing one is kept and
283/// [`FunctionRegistrationResult::AlreadyExists`] is returned. A name that
284/// already exists in the normal [`FUNCTION_REGISTRY`] when this call
285/// linearizes is also rejected: the ADMIN executor resolves admin-only
286/// functions before falling back to the normal registry, so inserting such a
287/// name here would shadow the built-in. Otherwise the function is registered
288/// and [`FunctionRegistrationResult::Registered`] is returned.
289///
290/// The enforced contract is one-way: it only guards the ADMIN registration
291/// against names already present in the normal registry. A later ordinary
292/// [`FunctionRegistry::register`] may still install the same name in the
293/// normal registry because the normal registry keeps its legacy replace
294/// semantics.
295pub fn register_admin_function(
296    func: impl Into<ScalarFunctionFactory>,
297) -> FunctionRegistrationResult {
298    register_admin_function_in(&ADMIN_FUNCTION_REGISTRY, &FUNCTION_REGISTRY, func)
299}
300
301/// Core implementation of [`register_admin_function`] against a pair of
302/// registries, parameterized so tests can exercise it with local registries.
303///
304/// Locking: the ADMIN-registry write lock is acquired first, then a read lock
305/// on the normal registry, and the normal-registry guard (bound to
306/// `normal_functions`) is kept alive through both the normal-name check and
307/// the ADMIN insertion below. This is the only code path that holds both
308/// registries' locks, so the ADMIN -> FUNCTION acquisition order is
309/// consistent and a concurrent normal-registry registration cannot slip in
310/// between the check and the ADMIN insert and be shadowed.
311///
312/// The enforced contract is one-way: it only guards the ADMIN registration
313/// against names already present in the normal registry. A later ordinary
314/// [`FunctionRegistry::register`] may still install the same name in the
315/// normal registry because the normal registry keeps its legacy replace
316/// semantics.
317fn register_admin_function_in(
318    admin_registry: &FunctionRegistry,
319    normal_registry: &FunctionRegistry,
320    func: impl Into<ScalarFunctionFactory>,
321) -> FunctionRegistrationResult {
322    let func = func.into();
323    let mut admin_functions = admin_registry.functions.write().unwrap();
324    // The normal-registry guard is a read lock: it is held across the
325    // normal-name check and the ADMIN insertion below, and while it is alive
326    // no writer can acquire the normal-registry write lock, so a concurrent
327    // normal-registry registration cannot slip in between the check and the
328    // ADMIN insert and be shadowed.
329    let normal_functions = normal_registry.functions.read().unwrap();
330    if normal_functions.contains_key(func.name()) {
331        drop(normal_functions);
332        return FunctionRegistrationResult::AlreadyExists;
333    }
334    let result = match admin_functions.entry(func.name().to_string()) {
335        Entry::Occupied(_) => FunctionRegistrationResult::AlreadyExists,
336        Entry::Vacant(entry) => {
337            entry.insert(func);
338            FunctionRegistrationResult::Registered
339        }
340    };
341    // Drop the read guard only after the ADMIN insertion, so writers to the
342    // normal registry stay blocked until the check-and-insert is complete.
343    drop(normal_functions);
344    result
345}
346
347#[cfg(test)]
348mod tests {
349    use std::sync::{Arc, Barrier};
350    use std::thread;
351
352    use super::*;
353    use crate::scalars::test::TestAndFunction;
354    use crate::scalars::udf::create_udf;
355
356    /// Creates a [`ScalarFunctionFactory`] with the given name. Each call
357    /// allocates a distinct factory closure, so factories can be told apart by
358    /// [`Arc::ptr_eq`] on their `factory` field even when names are identical.
359    fn named_factory(name: &str) -> ScalarFunctionFactory {
360        ScalarFunctionFactory {
361            name: name.to_string(),
362            factory: Arc::new(|_ctx| create_udf(Arc::new(TestAndFunction::default()))),
363        }
364    }
365
366    #[test]
367    fn test_function_registry() {
368        let registry = FunctionRegistry::default();
369
370        assert!(registry.get_function("test_and").is_none());
371        assert!(registry.scalar_functions().is_empty());
372        registry.register_scalar(TestAndFunction::default());
373        let _ = registry.get_function("test_and").unwrap();
374        assert_eq!(1, registry.scalar_functions().len());
375    }
376
377    #[test]
378    fn test_uddsketch_rank_registered() {
379        assert!(FUNCTION_REGISTRY.get_function("uddsketch_rank").is_some());
380    }
381
382    #[test]
383    fn test_register_if_absent_registers_new_function() {
384        let registry = FunctionRegistry::default();
385        let name = "pr3_register_if_absent_new";
386        let factory = named_factory(name);
387
388        assert_eq!(
389            registry.register_if_absent(factory.clone()),
390            FunctionRegistrationResult::Registered
391        );
392        let registered = registry
393            .get_function(name)
394            .expect("function should be registered");
395        assert!(Arc::ptr_eq(&registered.factory, &factory.factory));
396    }
397
398    #[test]
399    fn test_register_if_absent_first_registration_wins() {
400        let registry = FunctionRegistry::default();
401        let name = "pr3_register_if_absent_duplicate";
402        let first = named_factory(name);
403        let second = named_factory(name);
404
405        assert_eq!(
406            registry.register_if_absent(first.clone()),
407            FunctionRegistrationResult::Registered
408        );
409        assert_eq!(
410            registry.register_if_absent(second.clone()),
411            FunctionRegistrationResult::AlreadyExists
412        );
413
414        let stored = registry
415            .get_function(name)
416            .expect("function should be registered");
417        assert!(Arc::ptr_eq(&stored.factory, &first.factory));
418        assert!(!Arc::ptr_eq(&stored.factory, &second.factory));
419    }
420
421    #[test]
422    fn test_register_replaces_existing_function() {
423        // Regression test: `register` keeps its replace semantics.
424        let registry = FunctionRegistry::default();
425        let name = "pr3_register_replaces";
426        let first = named_factory(name);
427        let second = named_factory(name);
428
429        registry.register(first.clone());
430        registry.register(second.clone());
431
432        let stored = registry
433            .get_function(name)
434            .expect("function should be registered");
435        assert!(Arc::ptr_eq(&stored.factory, &second.factory));
436        assert!(!Arc::ptr_eq(&stored.factory, &first.factory));
437    }
438
439    #[test]
440    fn test_concurrent_register_if_absent_same_name() {
441        const THREADS: usize = 8;
442        let registry = Arc::new(FunctionRegistry::default());
443        let name = "pr3_concurrent_same_name";
444        let barrier = Arc::new(Barrier::new(THREADS));
445
446        let handles: Vec<_> = (0..THREADS)
447            .map(|_| {
448                let registry = Arc::clone(&registry);
449                let barrier = Arc::clone(&barrier);
450                thread::spawn(move || {
451                    let factory = named_factory(name);
452                    // Synchronize so every thread attempts registration at the
453                    // same time; only one may win the write lock.
454                    barrier.wait();
455                    let result = registry.register_if_absent(factory.clone());
456                    (result, factory)
457                })
458            })
459            .collect();
460
461        let mut results: Vec<(FunctionRegistrationResult, ScalarFunctionFactory)> =
462            Vec::with_capacity(THREADS);
463        for handle in handles {
464            results.push(handle.join().expect("thread should not panic"));
465        }
466
467        let registered = results
468            .iter()
469            .filter(|(result, _)| *result == FunctionRegistrationResult::Registered)
470            .count();
471        let already_exists = results
472            .iter()
473            .filter(|(result, _)| *result == FunctionRegistrationResult::AlreadyExists)
474            .count();
475        assert_eq!(registered, 1);
476        assert_eq!(already_exists, THREADS - 1);
477
478        let winner = results
479            .iter()
480            .find(|(result, _)| *result == FunctionRegistrationResult::Registered)
481            .map(|(_, factory)| factory)
482            .expect("exactly one registration must win");
483
484        let stored = registry
485            .get_function(name)
486            .expect("function should be registered");
487        assert!(
488            Arc::ptr_eq(&stored.factory, &winner.factory),
489            "the stored factory must be the factory of the winning registration"
490        );
491    }
492
493    #[test]
494    fn test_register_admin_function_first_wins() {
495        // Tests touching the global registry must use unique names.
496        let name = "pr3_admin_runtime_register";
497        let first = named_factory(name);
498        let second = named_factory(name);
499
500        assert_eq!(
501            register_admin_function(first.clone()),
502            FunctionRegistrationResult::Registered
503        );
504        assert_eq!(
505            register_admin_function(second.clone()),
506            FunctionRegistrationResult::AlreadyExists
507        );
508
509        let stored = get_admin_function(name).expect("admin function should be queryable");
510        assert!(Arc::ptr_eq(&stored.factory, &first.factory));
511        assert!(!Arc::ptr_eq(&stored.factory, &second.factory));
512    }
513
514    #[test]
515    fn test_register_admin_function_duplicate_same_factory() {
516        // The exact same factory clone registered twice: the first registration
517        // wins, the duplicate is rejected, and the stored factory is
518        // pointer-equal to the original. Tests touching the global registry
519        // must use unique names.
520        let name = "pr3_admin_runtime_register_same_factory";
521        let factory = named_factory(name);
522
523        assert_eq!(
524            register_admin_function(factory.clone()),
525            FunctionRegistrationResult::Registered
526        );
527        assert_eq!(
528            register_admin_function(factory.clone()),
529            FunctionRegistrationResult::AlreadyExists
530        );
531
532        let stored = get_admin_function(name).expect("admin function should be queryable");
533        assert!(Arc::ptr_eq(&stored.factory, &factory.factory));
534    }
535
536    #[test]
537    fn test_register_admin_function_in_is_one_way_later_normal_register_allowed() {
538        // The guard is one-way: after the ADMIN registration completes, an
539        // ordinary normal-registry registration of the same name still
540        // succeeds because the normal registry keeps its legacy replace
541        // semantics.
542        let admin = FunctionRegistry::default();
543        let normal = FunctionRegistry::default();
544        let name = "pr3_admin_in_one_way";
545        let admin_factory = named_factory(name);
546        let normal_factory = named_factory(name);
547
548        assert_eq!(
549            register_admin_function_in(&admin, &normal, admin_factory.clone()),
550            FunctionRegistrationResult::Registered
551        );
552
553        normal.register(normal_factory.clone());
554
555        let stored_admin = admin
556            .get_function(name)
557            .expect("the ADMIN registration must be kept");
558        assert!(Arc::ptr_eq(&stored_admin.factory, &admin_factory.factory));
559        let stored_normal = normal
560            .get_function(name)
561            .expect("the later normal registration must succeed");
562        assert!(Arc::ptr_eq(&stored_normal.factory, &normal_factory.factory));
563    }
564
565    #[test]
566    fn test_register_admin_function_rejects_normal_registry_builtin_name() {
567        // Regression test: the ADMIN executor resolves `get_admin_function`
568        // before falling back to `FUNCTION_REGISTRY`, so registering a function
569        // whose name already exists in the normal registry would shadow the
570        // ADMIN-invocable built-in (e.g. `flush_table`). Such registrations
571        // must be rejected with
572        // [`FunctionRegistrationResult::AlreadyExists`] and must not be
573        // inserted into the ADMIN registry.
574        let factory = named_factory("flush_table");
575
576        assert_eq!(
577            register_admin_function(factory.clone()),
578            FunctionRegistrationResult::AlreadyExists
579        );
580        assert!(
581            get_admin_function("flush_table").is_none(),
582            "a normal-registry built-in must not be shadowed into the ADMIN registry"
583        );
584        assert!(
585            FUNCTION_REGISTRY.get_function("flush_table").is_some(),
586            "the normal-registry built-in must remain registered"
587        );
588    }
589
590    #[test]
591    fn test_builtin_admin_functions_remain_queryable() {
592        // Built-in admin-only functions registered at startup stay queryable
593        // through the same global registry used for runtime registrations.
594        #[cfg(feature = "enterprise")]
595        {
596            assert!(get_admin_function("purge_table").is_some());
597        }
598        #[cfg(not(feature = "enterprise"))]
599        {
600            assert!(get_admin_function("purge_table").is_none());
601        }
602    }
603}