Skip to main content

frontend/instance/
builder.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::sync::Arc;
16use std::sync::atomic::AtomicBool;
17
18use auth::PermissionCheckerRef;
19use cache::{PARTITION_INFO_CACHE_NAME, TABLE_FLOWNODE_SET_CACHE_NAME, TABLE_ROUTE_CACHE_NAME};
20use catalog::CatalogManagerRef;
21use catalog::kvbackend::KvBackendCatalogManager;
22use catalog::process_manager::ProcessManagerRef;
23use catalog::system_schema::semantic_graph::EntityGraphProviderRef;
24use common_base::Plugins;
25use common_datasource::object_store::LocalFileAccess;
26use common_event_recorder::{EventRecorderImpl, EventRecorderRef};
27use common_meta::cache::{LayeredCacheRegistryRef, TableFlownodeSetCacheRef, TableRouteCacheRef};
28use common_meta::cache_invalidator::{CacheInvalidatorRef, DummyCacheInvalidator};
29use common_meta::key::TableMetadataManager;
30use common_meta::key::flow::FlowMetadataManager;
31use common_meta::kv_backend::KvBackendRef;
32use common_meta::node_manager::NodeManagerRef;
33use common_meta::procedure_executor::ProcedureExecutorRef;
34use dashmap::DashMap;
35use operator::batcher::PendingRowsBatcher;
36use operator::delete::Deleter;
37use operator::flow::FlowServiceOperator;
38use operator::insert::Inserter;
39use operator::procedure::ProcedureServiceOperator;
40use operator::request::Requester;
41#[cfg(feature = "enterprise")]
42use operator::statement::CreateDatabaseHandlerRef;
43use operator::statement::{
44    AdminEventRecorderHandle, AdminFunctionRecordingLayer, ExecutorConfigureContext,
45    StatementExecutor, StatementExecutorConfiguratorRef, StatementExecutorRef,
46};
47use operator::table::TableMutationOperator;
48use partition::cache::PartitionInfoCacheRef;
49use partition::manager::PartitionRuleManager;
50use pipeline::pipeline_operator::PipelineOperator;
51use query::QueryEngineFactory;
52use query::region_query::RegionQueryHandlerFactoryRef;
53use servers::batcher::table::TablePendingRowsBatcher;
54use servers::http::BatchingProtocol;
55use snafu::{OptionExt, ResultExt};
56
57use crate::error::{self, DataFusionSnafu, ExternalSnafu, Result};
58use crate::events::EventHandlerImpl;
59use crate::frontend::FrontendOptions;
60use crate::heartbeat::frontend_peer_addr;
61use crate::instance::Instance;
62use crate::instance::entity_graph::EntityGraphProviderImpl;
63use crate::instance::region_query::FrontendRegionQueryHandler;
64use crate::service_config::PendingRowsBatcherOptions;
65
66/// The frontend [`Instance`] builder.
67pub struct FrontendBuilder {
68    options: FrontendOptions,
69    kv_backend: KvBackendRef,
70    layered_cache_registry: LayeredCacheRegistryRef,
71    local_cache_invalidator: Option<CacheInvalidatorRef>,
72    catalog_manager: CatalogManagerRef,
73    node_manager: NodeManagerRef,
74    plugins: Option<Plugins>,
75    procedure_executor: ProcedureExecutorRef,
76    process_manager: ProcessManagerRef,
77    local_file_access: LocalFileAccess,
78}
79
80impl FrontendBuilder {
81    #[allow(clippy::too_many_arguments)]
82    pub fn new(
83        options: FrontendOptions,
84        kv_backend: KvBackendRef,
85        layered_cache_registry: LayeredCacheRegistryRef,
86        catalog_manager: CatalogManagerRef,
87        node_manager: NodeManagerRef,
88        procedure_executor: ProcedureExecutorRef,
89        process_manager: ProcessManagerRef,
90    ) -> Self {
91        Self {
92            options,
93            kv_backend,
94            layered_cache_registry,
95            local_cache_invalidator: None,
96            catalog_manager,
97            node_manager,
98            plugins: None,
99            procedure_executor,
100            process_manager,
101            local_file_access: LocalFileAccess::Disabled,
102        }
103    }
104
105    #[cfg(test)]
106    pub(crate) fn new_test(
107        options: &FrontendOptions,
108        meta_client: meta_client::MetaClientRef,
109    ) -> Self {
110        use cache::{build_fundamental_cache_registry, with_default_composite_cache_registry};
111        use common_meta::cache::LayeredCacheRegistryBuilder;
112        use common_meta::kv_backend::memory::MemoryKvBackend;
113
114        let kv_backend = Arc::new(MemoryKvBackend::new());
115
116        // Builds cache registry
117        let layered_cache_builder = LayeredCacheRegistryBuilder::default();
118        let fundamental_cache_registry = build_fundamental_cache_registry(kv_backend.clone());
119        let layered_cache_registry = Arc::new(
120            with_default_composite_cache_registry(
121                layered_cache_builder.add_cache_registry(fundamental_cache_registry),
122            )
123            .unwrap()
124            .build(),
125        );
126
127        Self::new(
128            options.clone(),
129            kv_backend,
130            layered_cache_registry,
131            catalog::memory::MemoryCatalogManager::with_default_setup(),
132            Arc::new(client::client_manager::NodeClients::default()),
133            meta_client,
134            Arc::new(catalog::process_manager::ProcessManager::new(
135                "".to_string(),
136                None,
137            )),
138        )
139    }
140
141    pub fn with_local_cache_invalidator(self, cache_invalidator: CacheInvalidatorRef) -> Self {
142        Self {
143            local_cache_invalidator: Some(cache_invalidator),
144            ..self
145        }
146    }
147
148    pub fn with_local_file_access(mut self, local_file_access: LocalFileAccess) -> Self {
149        self.local_file_access = local_file_access;
150        self
151    }
152
153    pub fn options(&self) -> &FrontendOptions {
154        &self.options
155    }
156
157    pub fn kv_backend(&self) -> &KvBackendRef {
158        &self.kv_backend
159    }
160
161    pub fn layered_cache_registry(&self) -> &LayeredCacheRegistryRef {
162        &self.layered_cache_registry
163    }
164
165    pub fn catalog_manager(&self) -> &CatalogManagerRef {
166        &self.catalog_manager
167    }
168
169    pub fn node_manager(&self) -> &NodeManagerRef {
170        &self.node_manager
171    }
172
173    pub fn procedure_executor(&self) -> &ProcedureExecutorRef {
174        &self.procedure_executor
175    }
176
177    pub fn process_manager(&self) -> &ProcessManagerRef {
178        &self.process_manager
179    }
180
181    pub fn with_plugin(self, plugins: Plugins) -> Self {
182        Self {
183            plugins: Some(plugins),
184            ..self
185        }
186    }
187
188    pub async fn try_build(self) -> Result<Instance> {
189        let kv_backend = self.kv_backend;
190        let node_manager = self.node_manager;
191        let plugins = self.plugins.unwrap_or_default();
192        let process_manager = self.process_manager;
193        let table_route_cache: TableRouteCacheRef =
194            self.layered_cache_registry
195                .get()
196                .context(error::CacheRequiredSnafu {
197                    name: TABLE_ROUTE_CACHE_NAME,
198                })?;
199        let partition_info_cache: PartitionInfoCacheRef = self
200            .layered_cache_registry
201            .get()
202            .context(error::CacheRequiredSnafu {
203                name: PARTITION_INFO_CACHE_NAME,
204            })?;
205        let partition_manager = Arc::new(PartitionRuleManager::new(
206            kv_backend.clone(),
207            table_route_cache.clone(),
208            partition_info_cache.clone(),
209        ));
210
211        let local_cache_invalidator = self
212            .local_cache_invalidator
213            .unwrap_or_else(|| Arc::new(DummyCacheInvalidator));
214
215        let region_query_handler =
216            if let Some(factory) = plugins.get::<RegionQueryHandlerFactoryRef>() {
217                factory.build(partition_manager.clone(), node_manager.clone())
218            } else {
219                FrontendRegionQueryHandler::arc(partition_manager.clone(), node_manager.clone())
220            };
221
222        let table_flownode_cache: TableFlownodeSetCacheRef = self
223            .layered_cache_registry
224            .get()
225            .context(error::CacheRequiredSnafu {
226                name: TABLE_FLOWNODE_SET_CACHE_NAME,
227            })?;
228
229        let create_inserter = || {
230            Inserter::new(
231                self.catalog_manager.clone(),
232                partition_manager.clone(),
233                node_manager.clone(),
234                table_flownode_cache.clone(),
235                self.options.auto_create_table,
236            )
237        };
238        // The execution-only inserter owns no batchers, avoiding an Arc cycle.
239        let bulk_inserter = Arc::new(create_inserter());
240        let build_batcher =
241            |options: &PendingRowsBatcherOptions| -> Option<Arc<dyn PendingRowsBatcher>> {
242                if !options.pending_rows_batching_enabled()
243                    || (self.options.prom_store.with_metric_engine
244                        && options
245                            .protocols
246                            .iter()
247                            .all(|protocol| *protocol == BatchingProtocol::Prom))
248                {
249                    return None;
250                }
251                TablePendingRowsBatcher::try_new(
252                    options.pending_rows_flush_interval,
253                    options.max_batch_rows,
254                    options.max_concurrent_flushes,
255                    options.worker_channel_capacity,
256                    options.max_inflight_requests,
257                    options.flow_notification_queue_capacity,
258                    bulk_inserter.clone(),
259                )
260                .map(|batcher| batcher as Arc<dyn PendingRowsBatcher>)
261            };
262        let inserter = Arc::new(
263            create_inserter()
264                .with_pending_rows_batcher(build_batcher(&self.options.pending_rows_batcher)),
265        );
266        let deleter = Arc::new(Deleter::new(
267            self.catalog_manager.clone(),
268            partition_manager.clone(),
269            node_manager.clone(),
270        ));
271        let requester = Arc::new(Requester::new(
272            self.catalog_manager.clone(),
273            partition_manager.clone(),
274            node_manager.clone(),
275        ));
276        let table_mutation_handler = Arc::new(TableMutationOperator::new(
277            inserter.clone(),
278            deleter.clone(),
279            requester,
280        ));
281
282        let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend.clone()));
283        let procedure_service_handler = Arc::new(ProcedureServiceOperator::new(
284            self.procedure_executor.clone(),
285            self.catalog_manager.clone(),
286            table_metadata_manager.clone(),
287        ));
288
289        let flow_metadata_manager: Arc<FlowMetadataManager> =
290            Arc::new(FlowMetadataManager::new(kv_backend.clone()));
291        let flow_service = FlowServiceOperator::new(flow_metadata_manager, node_manager.clone());
292
293        let mut query_options = self.options.query.clone();
294        query_options.enable_per_region_metrics = self.options.logging.enable_per_region_metrics;
295        let query_engine = QueryEngineFactory::try_new_with_plugins(
296            self.catalog_manager.clone(),
297            Some(partition_manager.clone()),
298            Some(region_query_handler.clone()),
299            Some(table_mutation_handler),
300            Some(procedure_service_handler),
301            Some(Arc::new(flow_service)),
302            true,
303            plugins.clone(),
304            query_options,
305        )
306        .context(DataFusionSnafu)?
307        .query_engine();
308
309        // Inject the entity-graph provider now that the query engine exists, so the
310        // computed `greptime_private.semantic_entities` / `semantic_relationships`
311        // tables can derive rows. Late binding here breaks the `catalog -> query`
312        // dependency cycle.
313        if let Some(kv_catalog) = self
314            .catalog_manager
315            .as_any()
316            .downcast_ref::<KvBackendCatalogManager>()
317        {
318            let provider: EntityGraphProviderRef = Arc::new(EntityGraphProviderImpl::new(
319                query_engine.clone(),
320                Arc::downgrade(&self.catalog_manager),
321                plugins.get::<PermissionCheckerRef>(),
322            ));
323            kv_catalog.set_entity_graph_provider(provider);
324        }
325
326        let frontend_peer_addr = frontend_peer_addr(&self.options);
327        let statement_executor = StatementExecutor::new(
328            self.catalog_manager.clone(),
329            query_engine.clone(),
330            self.procedure_executor,
331            kv_backend.clone(),
332            local_cache_invalidator,
333            inserter.clone(),
334            partition_manager,
335            Some(process_manager.clone()),
336            frontend_peer_addr.clone(),
337            self.local_file_access,
338        );
339
340        let statement_executor =
341            if let Some(configurator) = plugins.get::<StatementExecutorConfiguratorRef>() {
342                let ctx = ExecutorConfigureContext {
343                    kv_backend: kv_backend.clone(),
344                };
345                configurator
346                    .configure(statement_executor, ctx)
347                    .await
348                    .context(ExternalSnafu)?
349            } else {
350                statement_executor
351            };
352
353        #[cfg(feature = "enterprise")]
354        let statement_executor = if let Some(handler) = plugins.get::<CreateDatabaseHandlerRef>() {
355            statement_executor.with_create_database_handler(handler)
356        } else {
357            statement_executor
358        };
359
360        let admin_event_recorder = AdminEventRecorderHandle::default();
361        let statement_executor = statement_executor.with_admin_function_layer(Arc::new(
362            AdminFunctionRecordingLayer::new(admin_event_recorder.clone()),
363        ));
364        let statement_executor = Arc::new(statement_executor);
365
366        let pipeline_operator = Arc::new(PipelineOperator::new(
367            inserter.clone(),
368            statement_executor.clone(),
369            self.catalog_manager.clone(),
370            query_engine.clone(),
371            &self.options.pipeline,
372        ));
373
374        plugins.insert::<StatementExecutorRef>(statement_executor.clone());
375
376        let slow_query_recorder = Arc::new(EventRecorderImpl::new(Box::new(
377            EventHandlerImpl::new(statement_executor.clone(), self.options.slow_query.ttl),
378        )));
379        let event_recorder: EventRecorderRef = Arc::new(EventRecorderImpl::with_event_type_filter(
380            Box::new(EventHandlerImpl::new(
381                statement_executor.clone(),
382                self.options.event_recorder.ttl,
383            )),
384            self.options.event_recorder.event_types.clone(),
385        ));
386        admin_event_recorder.install(&event_recorder);
387
388        Ok(Instance {
389            frontend_peer_addr,
390            catalog_manager: self.catalog_manager,
391            pipeline_operator,
392            statement_executor,
393            query_engine,
394            plugins,
395            inserter,
396            deleter,
397            table_metadata_manager,
398            event_recorder,
399            slow_query_recorder,
400            process_manager,
401            otlp_metrics_table_legacy_cache: DashMap::new(),
402            slow_query_options: self.options.slow_query.clone(),
403            influxdb_default_merge_mode: self.options.influxdb.default_merge_mode,
404            trace_ingest_chunk_size: self.options.otlp.trace_ingest_chunk_size,
405            otlp_resource_info: self.options.otlp.experimental_enable_resource_info,
406            suspend: Arc::new(AtomicBool::new(false)),
407        })
408    }
409}