tests_integration/
standalone.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::sync::Arc;
16use std::time::Duration;
17
18use cache::{
19    build_datanode_cache_registry, build_fundamental_cache_registry,
20    with_default_composite_cache_registry,
21};
22use catalog::information_schema::NoopInformationExtension;
23use catalog::kvbackend::KvBackendCatalogManagerBuilder;
24use catalog::process_manager::ProcessManager;
25use cmd::error::StartFlownodeSnafu;
26use common_base::Plugins;
27use common_catalog::consts::{MIN_USER_FLOW_ID, MIN_USER_TABLE_ID};
28use common_config::KvBackendConfig;
29use common_meta::cache::LayeredCacheRegistryBuilder;
30use common_meta::ddl::flow_meta::FlowMetadataAllocator;
31use common_meta::ddl::table_meta::TableMetadataAllocator;
32use common_meta::ddl::{DdlContext, NoopRegionFailureDetectorControl};
33use common_meta::ddl_manager::DdlManager;
34use common_meta::key::TableMetadataManager;
35use common_meta::key::flow::FlowMetadataManager;
36use common_meta::kv_backend::KvBackendRef;
37use common_meta::procedure_executor::LocalProcedureExecutor;
38use common_meta::region_keeper::MemoryRegionKeeper;
39use common_meta::region_registry::LeaderRegionRegistry;
40use common_meta::sequence::SequenceBuilder;
41use common_meta::wal_provider::build_wal_provider;
42use common_procedure::ProcedureManagerRef;
43use common_procedure::options::ProcedureConfig;
44use common_telemetry::logging::SlowQueryOptions;
45use common_wal::config::{DatanodeWalConfig, MetasrvWalConfig};
46use datanode::datanode::DatanodeBuilder;
47use flow::{FlownodeBuilder, FrontendClient, GrpcQueryHandlerWithBoxedError};
48use frontend::frontend::Frontend;
49use frontend::instance::builder::FrontendBuilder;
50use frontend::instance::{Instance, StandaloneDatanodeManager};
51use frontend::server::Services;
52use meta_srv::metasrv::{FLOW_ID_SEQ, TABLE_ID_SEQ};
53use servers::grpc::GrpcOptions;
54use snafu::ResultExt;
55use standalone::StandaloneRepartitionProcedureFactory;
56use standalone::options::StandaloneOptions;
57
58use crate::test_util::{self, StorageType, TestGuard, create_tmp_dir_and_datanode_opts};
59
60pub struct GreptimeDbStandalone {
61    pub frontend: Arc<Frontend>,
62    pub opts: StandaloneOptions,
63    pub guard: TestGuard,
64    // Used in rebuild.
65    pub kv_backend: KvBackendRef,
66    pub procedure_manager: ProcedureManagerRef,
67}
68
69impl GreptimeDbStandalone {
70    pub fn fe_instance(&self) -> &Arc<Instance> {
71        &self.frontend.instance
72    }
73}
74
75pub struct GreptimeDbStandaloneBuilder {
76    instance_name: String,
77    datanode_wal_config: DatanodeWalConfig,
78    metasrv_wal_config: MetasrvWalConfig,
79    store_providers: Option<Vec<StorageType>>,
80    default_store: Option<StorageType>,
81    plugin: Option<Plugins>,
82    slow_query_options: SlowQueryOptions,
83    auto_create_table: bool,
84}
85
86impl GreptimeDbStandaloneBuilder {
87    pub fn new(instance_name: &str) -> Self {
88        Self {
89            instance_name: instance_name.to_string(),
90            store_providers: None,
91            plugin: None,
92            default_store: None,
93            datanode_wal_config: DatanodeWalConfig::default(),
94            metasrv_wal_config: MetasrvWalConfig::default(),
95            // Enable slow query log with 1s threshold by default for integration tests.
96            slow_query_options: SlowQueryOptions {
97                enable: true,
98                threshold: Duration::from_secs(1),
99                ..Default::default()
100            },
101            auto_create_table: true,
102        }
103    }
104
105    #[must_use]
106    pub fn with_auto_create_table(mut self, auto_create_table: bool) -> Self {
107        self.auto_create_table = auto_create_table;
108        self
109    }
110
111    #[must_use]
112    pub fn with_default_store_type(self, store_type: StorageType) -> Self {
113        Self {
114            default_store: Some(store_type),
115            ..self
116        }
117    }
118
119    #[cfg(test)]
120    #[must_use]
121    pub fn with_store_providers(self, store_providers: Vec<StorageType>) -> Self {
122        Self {
123            store_providers: Some(store_providers),
124            ..self
125        }
126    }
127
128    #[must_use]
129    pub fn with_plugin(self, plugin: Plugins) -> Self {
130        Self {
131            plugin: Some(plugin),
132            ..self
133        }
134    }
135
136    #[must_use]
137    pub fn with_slow_query_threshold(mut self, threshold: Duration) -> Self {
138        self.slow_query_options.threshold = threshold;
139        self
140    }
141
142    #[must_use]
143    pub fn with_datanode_wal_config(mut self, datanode_wal_config: DatanodeWalConfig) -> Self {
144        self.datanode_wal_config = datanode_wal_config;
145        self
146    }
147
148    #[must_use]
149    pub fn with_metasrv_wal_config(mut self, metasrv_wal_config: MetasrvWalConfig) -> Self {
150        self.metasrv_wal_config = metasrv_wal_config;
151        self
152    }
153
154    pub async fn build_with(
155        &self,
156        kv_backend: KvBackendRef,
157        guard: TestGuard,
158        opts: StandaloneOptions,
159        procedure_manager: ProcedureManagerRef,
160        register_procedure_loaders: bool,
161    ) -> GreptimeDbStandalone {
162        let plugins = self.plugin.clone().unwrap_or_default();
163
164        let layered_cache_registry = Arc::new(
165            LayeredCacheRegistryBuilder::default()
166                .add_cache_registry(build_datanode_cache_registry(kv_backend.clone()))
167                .build(),
168        );
169
170        let mut builder =
171            DatanodeBuilder::new(opts.datanode_options(), plugins.clone(), kv_backend.clone());
172        builder.with_cache_registry(layered_cache_registry);
173        let datanode = builder.build().await.unwrap();
174
175        let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend.clone()));
176        table_metadata_manager.init().await.unwrap();
177
178        let flow_metadata_manager = Arc::new(FlowMetadataManager::new(kv_backend.clone()));
179
180        let layered_cache_builder = LayeredCacheRegistryBuilder::default();
181        let fundamental_cache_registry = build_fundamental_cache_registry(kv_backend.clone());
182        let cache_registry = Arc::new(
183            with_default_composite_cache_registry(
184                layered_cache_builder.add_cache_registry(fundamental_cache_registry),
185            )
186            .unwrap()
187            .build(),
188        );
189
190        let catalog_manager = KvBackendCatalogManagerBuilder::new(
191            Arc::new(NoopInformationExtension),
192            kv_backend.clone(),
193            cache_registry.clone(),
194        )
195        .with_procedure_manager(procedure_manager.clone())
196        .build();
197
198        let (frontend_client, frontend_instance_handler) =
199            FrontendClient::from_empty_grpc_handler(opts.query.clone());
200        let flow_builder = FlownodeBuilder::new(
201            Default::default(),
202            plugins.clone(),
203            table_metadata_manager.clone(),
204            catalog_manager.clone(),
205            flow_metadata_manager.clone(),
206            Arc::new(frontend_client),
207        );
208        let flownode = Arc::new(flow_builder.build().await.unwrap());
209
210        let node_manager = Arc::new(StandaloneDatanodeManager {
211            region_server: datanode.region_server(),
212            flow_server: flownode.flow_engine(),
213        });
214
215        let table_id_allocator = Arc::new(
216            SequenceBuilder::new(TABLE_ID_SEQ, kv_backend.clone())
217                .initial(MIN_USER_TABLE_ID as u64)
218                .step(10)
219                .build(),
220        );
221        let flow_id_sequence = Arc::new(
222            SequenceBuilder::new(FLOW_ID_SEQ, kv_backend.clone())
223                .initial(MIN_USER_FLOW_ID as u64)
224                .step(10)
225                .build(),
226        );
227        let kafka_options = opts.wal.clone().try_into().unwrap();
228        let wal_provider = build_wal_provider(&kafka_options, kv_backend.clone())
229            .await
230            .unwrap();
231        let wal_provider = Arc::new(wal_provider);
232        let table_metadata_allocator = Arc::new(TableMetadataAllocator::new(
233            table_id_allocator,
234            wal_provider.clone(),
235        ));
236        let flow_metadata_allocator = Arc::new(FlowMetadataAllocator::with_noop_peer_allocator(
237            flow_id_sequence,
238        ));
239
240        let repartition_procedure_factory = Arc::new(StandaloneRepartitionProcedureFactory);
241        let ddl_manager = Arc::new(
242            DdlManager::try_new(
243                DdlContext {
244                    node_manager: node_manager.clone(),
245                    cache_invalidator: cache_registry.clone(),
246                    memory_region_keeper: Arc::new(MemoryRegionKeeper::default()),
247                    leader_region_registry: Arc::new(LeaderRegionRegistry::default()),
248                    table_metadata_manager,
249                    table_metadata_allocator,
250                    flow_metadata_manager,
251                    flow_metadata_allocator,
252                    region_failure_detector_controller: Arc::new(NoopRegionFailureDetectorControl),
253                },
254                procedure_manager.clone(),
255                repartition_procedure_factory,
256                register_procedure_loaders,
257            )
258            .unwrap(),
259        );
260        let procedure_executor = Arc::new(LocalProcedureExecutor::new(
261            ddl_manager,
262            procedure_manager.clone(),
263        ));
264
265        let server_addr = opts.frontend_options().grpc.server_addr.clone();
266
267        let instance = FrontendBuilder::new(
268            opts.frontend_options(),
269            kv_backend.clone(),
270            cache_registry.clone(),
271            catalog_manager.clone(),
272            node_manager.clone(),
273            procedure_executor.clone(),
274            Arc::new(ProcessManager::new(server_addr, None)),
275        )
276        .with_plugin(plugins.clone())
277        .try_build()
278        .await
279        .unwrap();
280        let instance = Arc::new(instance);
281
282        // set the frontend client for flownode
283        let grpc_handler = instance.clone() as Arc<dyn GrpcQueryHandlerWithBoxedError>;
284        let weak_grpc_handler = Arc::downgrade(&grpc_handler);
285        frontend_instance_handler
286            .set_handler(weak_grpc_handler)
287            .await;
288
289        let flow_streaming_engine = flownode.flow_engine().streaming_engine();
290        let invoker = flow::FrontendInvoker::build_from(
291            flow_streaming_engine.clone(),
292            catalog_manager.clone(),
293            kv_backend.clone(),
294            cache_registry.clone(),
295            procedure_executor.clone(),
296            node_manager.clone(),
297            instance.frontend_peer_addr().to_string(),
298        )
299        .await
300        .context(StartFlownodeSnafu)
301        .unwrap();
302
303        flow_streaming_engine.set_frontend_invoker(invoker).await;
304
305        procedure_manager.start().await.unwrap();
306        wal_provider.start().await.unwrap();
307
308        test_util::prepare_another_catalog_and_schema(&instance).await;
309
310        let servers = Services::new(opts.clone(), instance.clone(), plugins)
311            .build()
312            .unwrap();
313        let frontend = Frontend {
314            instance,
315            servers,
316            heartbeat_task: None,
317        };
318
319        GreptimeDbStandalone {
320            frontend: Arc::new(frontend),
321            opts,
322            guard,
323            kv_backend,
324            procedure_manager,
325        }
326    }
327
328    pub async fn build(&self) -> GreptimeDbStandalone {
329        let default_store_type = self.default_store.unwrap_or(StorageType::File);
330        let store_types = self.store_providers.clone().unwrap_or_default();
331
332        let (opts, guard) = create_tmp_dir_and_datanode_opts(
333            default_store_type,
334            store_types,
335            &self.instance_name,
336            self.datanode_wal_config.clone(),
337            Default::default(),
338        );
339
340        let kv_backend_config = KvBackendConfig::default();
341        let procedure_config = ProcedureConfig::default();
342
343        let kv_backend = standalone::build_metadata_kvbackend(
344            format!("{}/kv", &opts.storage.data_home),
345            kv_backend_config,
346        )
347        .unwrap();
348        let procedure_manager =
349            standalone::build_procedure_manager(kv_backend.clone(), procedure_config);
350
351        let standalone_opts = StandaloneOptions {
352            storage: opts.storage,
353            procedure: procedure_config,
354            metadata_store: kv_backend_config,
355            wal: self.metasrv_wal_config.clone().into(),
356            grpc: GrpcOptions::default().with_server_addr("127.0.0.1:4001"),
357            slow_query: self.slow_query_options.clone(),
358            auto_create_table: self.auto_create_table,
359            ..StandaloneOptions::default()
360        };
361
362        self.build_with(kv_backend, guard, standalone_opts, procedure_manager, true)
363            .await
364    }
365}
tests_integration/standalone.rs

tests_integration/
standalone.rs
