Skip to main content

flow/adapter/
flownode_impl.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! impl `FlowNode` trait for FlowNodeManager so standalone can call them
16use std::collections::{HashMap, HashSet};
17use std::sync::Arc;
18use std::sync::atomic::AtomicBool;
19
20use api::v1::flow::{
21    CreateRequest, DirtyWindowRequests, DropRequest, FlowRequest, FlowResponse, FlushFlow,
22    flow_request,
23};
24use api::v1::region::InsertRequests;
25use catalog::CatalogManager;
26use common_base::Plugins;
27use common_error::ext::BoxedError;
28use common_meta::ddl::create_flow::{
29    FlowType, INTERNAL_EVAL_SCHEDULE_KEY, effective_eval_schedule_from_flow_info,
30};
31use common_meta::error::Result as MetaResult;
32use common_meta::key::flow::FlowMetadataManager;
33use common_meta::key::flow::flow_info::FlowScheduleConfig;
34use common_meta::key::flow::flow_state::FlowStat;
35use common_runtime::JoinHandle;
36use common_telemetry::{error, info, trace, warn};
37use datatypes::value::Value;
38use futures::TryStreamExt;
39use itertools::Itertools;
40use operator::utils::try_to_session_query_context;
41use session::context::QueryContextBuilder;
42use snafu::{IntoError, OptionExt, ResultExt, ensure};
43use store_api::storage::{RegionId, TableId};
44use tokio::sync::{Mutex, RwLock};
45
46use crate::adapter::{CreateFlowArgs, StreamingEngine};
47use crate::batching_mode::engine::BatchingEngine;
48use crate::engine::{FlowEngine, FlowStatProvider};
49use crate::error::{
50    CreateFlowSnafu, ExternalSnafu, FlowNotFoundSnafu, FlowNotRecoveredSnafu,
51    IllegalCheckTaskStateSnafu, InsertIntoFlowSnafu, InternalSnafu, JoinTaskSnafu, ListFlowsSnafu,
52    NoAvailableFrontendSnafu, SyncCheckTaskSnafu, UnexpectedSnafu, UnsupportedSnafu,
53};
54use crate::metrics::{METRIC_FLOW_ROWS, METRIC_FLOW_TASK_COUNT};
55use crate::repr::{self, DiffRow};
56use crate::utils::StateReportHandler;
57use crate::{Error, FlowId};
58
59/// Ref to [`FlowDualEngine`]
60pub type FlowDualEngineRef = Arc<FlowDualEngine>;
61
62/// Manage both streaming and batching mode engine
63///
64/// including create/drop/flush flow
65/// and redirect insert requests to the appropriate engine
66pub struct FlowDualEngine {
67    streaming_engine: Arc<StreamingEngine>,
68    batching_engine: Arc<BatchingEngine>,
69    /// receive a oneshot sender to send state report
70    state_report_handler: RwLock<Option<StateReportHandler>>,
71    /// helper struct for faster query flow by table id or vice versa
72    src_table2flow: RwLock<SrcTableToFlow>,
73    flow_metadata_manager: Arc<FlowMetadataManager>,
74    catalog_manager: Arc<dyn CatalogManager>,
75    check_task: tokio::sync::Mutex<Option<ConsistentCheckTask>>,
76    plugins: Plugins,
77    done_recovering: AtomicBool,
78}
79
80impl FlowDualEngine {
81    pub fn new(
82        streaming_engine: Arc<StreamingEngine>,
83        batching_engine: Arc<BatchingEngine>,
84        flow_metadata_manager: Arc<FlowMetadataManager>,
85        catalog_manager: Arc<dyn CatalogManager>,
86        plugins: Plugins,
87    ) -> Self {
88        Self {
89            streaming_engine,
90            batching_engine,
91            state_report_handler: Default::default(),
92            src_table2flow: RwLock::new(SrcTableToFlow::default()),
93            flow_metadata_manager,
94            catalog_manager,
95            check_task: Mutex::new(None),
96            plugins,
97            done_recovering: AtomicBool::new(false),
98        }
99    }
100
101    /// Set `done_recovering` to true
102    /// indicate that we are ready to handle requests
103    pub fn set_done_recovering(&self) {
104        info!("FlowDualEngine done recovering");
105        self.done_recovering
106            .store(true, std::sync::atomic::Ordering::Release);
107    }
108
109    /// Check if `done_recovering` is true
110    pub fn is_recover_done(&self) -> bool {
111        self.done_recovering
112            .load(std::sync::atomic::Ordering::Acquire)
113    }
114
115    /// wait for recovering to be done, this will only happen when flownode just started
116    async fn wait_for_all_flow_recover(&self, waiting_req_cnt: usize) -> Result<(), Error> {
117        if self.is_recover_done() {
118            return Ok(());
119        }
120
121        warn!(
122            "FlowDualEngine is not done recovering, {} insert request waiting for recovery",
123            waiting_req_cnt
124        );
125        // wait 3 seconds, check every 1 second
126        // TODO(discord9): make this configurable
127        let mut retry = 0;
128        let max_retry = 3;
129        while retry < max_retry && !self.is_recover_done() {
130            warn!(
131                "FlowDualEngine is not done recovering, retry {} in 1s",
132                retry
133            );
134            tokio::time::sleep(std::time::Duration::from_secs(1)).await;
135            retry += 1;
136        }
137        if retry == max_retry {
138            return FlowNotRecoveredSnafu.fail();
139        } else {
140            info!("FlowDualEngine is done recovering");
141        }
142        // TODO(discord9): also put to centralized logging for flow once it implemented
143        Ok(())
144    }
145
146    pub fn plugins(&self) -> &Plugins {
147        &self.plugins
148    }
149
150    /// Determine if the engine is in distributed mode
151    pub fn is_distributed(&self) -> bool {
152        self.streaming_engine.node_id.is_some()
153    }
154
155    pub fn streaming_engine(&self) -> Arc<StreamingEngine> {
156        self.streaming_engine.clone()
157    }
158
159    pub fn batching_engine(&self) -> Arc<BatchingEngine> {
160        self.batching_engine.clone()
161    }
162
163    pub async fn set_state_report_handler(&self, handler: StateReportHandler) {
164        *self.state_report_handler.write().await = Some(handler);
165    }
166
167    pub async fn gen_state_report(&self) -> FlowStat {
168        let streaming = self.streaming_engine.flow_stat().await;
169        let batching = self.batching_engine.flow_stat().await;
170
171        let mut state_size = streaming.state_size;
172        state_size.extend(batching.state_size);
173
174        let mut last_exec_time_map = streaming.last_exec_time_map;
175        last_exec_time_map.extend(batching.last_exec_time_map);
176
177        let mut start_time_map = streaming.start_time_map;
178        start_time_map.extend(batching.start_time_map);
179
180        FlowStat {
181            state_size,
182            last_exec_time_map,
183            start_time_map,
184        }
185    }
186
187    /// Start state report task, which receives a sender from heartbeat task and sends report back.
188    ///
189    /// if heartbeat task is shutdown, this future exits too.
190    pub async fn start_state_report_task(self: Arc<Self>) -> Option<JoinHandle<()>> {
191        let state_report_handler = self.state_report_handler.write().await.take();
192        if let Some(mut handler) = state_report_handler {
193            let zelf = self.clone();
194            let handler = common_runtime::spawn_global(async move {
195                while let Some(ret_handler) = handler.recv().await {
196                    let state_report = zelf.gen_state_report().await;
197                    ret_handler.send(state_report).unwrap_or_else(|err| {
198                        common_telemetry::error!(err; "Send state report error");
199                    });
200                }
201            });
202            Some(handler)
203        } else {
204            None
205        }
206    }
207
208    /// In distributed mode, scan periodically(1s) until all advertised frontends
209    /// accept unauthenticated queries, or timeout. In standalone mode, return
210    /// immediately.
211    async fn wait_for_available_frontend(&self, timeout: std::time::Duration) -> Result<(), Error> {
212        if !self.is_distributed() {
213            return Ok(());
214        }
215        let frontend_client = self.batching_engine().frontend_client.clone();
216        let sleep_duration = std::time::Duration::from_millis(1_000);
217        let now = std::time::Instant::now();
218        loop {
219            let frontend_list = frontend_client.scan_for_frontend().await?;
220            if !frontend_list.is_empty() {
221                let fe_list = frontend_list
222                    .iter()
223                    .map(|peer| &peer.addr)
224                    .collect::<Vec<_>>();
225                let probe_failures = frontend_client
226                    .check_all_frontends_without_auth(&frontend_list)
227                    .await?;
228                if probe_failures.is_empty() {
229                    info!(
230                        "Available frontend found and unauthenticated probe succeeded: {:?}",
231                        fe_list
232                    );
233                    return Ok(());
234                }
235                warn!(
236                    "Unauthenticated frontend probe failed, will retry. frontends={:?}, failures={:?}",
237                    fe_list, probe_failures
238                );
239            }
240            let elapsed = now.elapsed();
241            tokio::time::sleep(sleep_duration).await;
242            info!("Waiting for available frontend, elapsed={:?}", elapsed);
243            if elapsed >= timeout {
244                return NoAvailableFrontendSnafu {
245                    timeout,
246                    context: "No frontend accepted unauthenticated flownode probe",
247                }
248                .fail();
249            }
250        }
251    }
252
253    /// Try to sync with check task, this is only used in drop flow&flush flow, so a flow id is required
254    ///
255    /// the need to sync is to make sure flush flow actually get called
256    async fn try_sync_with_check_task(
257        &self,
258        flow_id: FlowId,
259        allow_drop: bool,
260    ) -> Result<(), Error> {
261        // this function rarely get called so adding some log is helpful
262        info!("Try to sync with check task for flow {}", flow_id);
263        let mut retry = 0;
264        let max_retry = 10;
265        // keep trying to trigger consistent check
266        while retry < max_retry {
267            if let Some(task) = self.check_task.lock().await.as_ref() {
268                task.trigger(false, allow_drop).await?;
269                break;
270            }
271            retry += 1;
272            tokio::time::sleep(std::time::Duration::from_millis(500)).await;
273        }
274
275        if retry == max_retry {
276            error!(
277                "Can't sync with check task for flow {} with allow_drop={}",
278                flow_id, allow_drop
279            );
280            return SyncCheckTaskSnafu {
281                flow_id,
282                allow_drop,
283            }
284            .fail();
285        }
286        info!("Successfully sync with check task for flow {}", flow_id);
287
288        Ok(())
289    }
290
291    /// Spawn a task to consistently check if all flow tasks in metasrv is created on flownode,
292    /// so on startup, this will create all missing flow tasks, and constantly check at a interval
293    async fn check_flow_consistent(
294        &self,
295        allow_create: bool,
296        allow_drop: bool,
297    ) -> Result<(), Error> {
298        // use nodeid to determine if this is standalone/distributed mode, and retrieve all flows in this node(in distributed mode)/or all flows(in standalone mode)
299        let nodeid = self.streaming_engine.node_id;
300        let should_exists: Vec<_> = if let Some(nodeid) = nodeid {
301            // nodeid is available, so we only need to check flows on this node
302            // which also means we are in distributed mode
303            let to_be_recover = self
304                .flow_metadata_manager
305                .flownode_flow_manager()
306                .flows(nodeid.into())
307                .try_collect::<Vec<_>>()
308                .await
309                .context(ListFlowsSnafu {
310                    id: Some(nodeid.into()),
311                })?;
312            to_be_recover.into_iter().map(|(id, _)| id).collect()
313        } else {
314            // nodeid is not available, so we need to check all flows
315            // which also means we are in standalone mode
316            let all_catalogs = self
317                .catalog_manager
318                .catalog_names()
319                .await
320                .map_err(BoxedError::new)
321                .context(ExternalSnafu)?;
322            let mut all_flow_ids = vec![];
323            for catalog in all_catalogs {
324                let flows = self
325                    .flow_metadata_manager
326                    .flow_name_manager()
327                    .flow_names(&catalog)
328                    .await
329                    .try_collect::<Vec<_>>()
330                    .await
331                    .map_err(BoxedError::new)
332                    .context(ExternalSnafu)?;
333
334                all_flow_ids.extend(flows.into_iter().map(|(_, id)| id.flow_id()));
335            }
336            all_flow_ids
337        };
338        let should_exists = should_exists
339            .into_iter()
340            .map(|i| i as FlowId)
341            .collect::<HashSet<_>>();
342        let actual_exists = self.list_flows().await?.into_iter().collect::<HashSet<_>>();
343        let to_be_created = should_exists
344            .iter()
345            .filter(|id| !actual_exists.contains(id))
346            .collect::<Vec<_>>();
347        let to_be_dropped = actual_exists
348            .iter()
349            .filter(|id| !should_exists.contains(id))
350            .collect::<Vec<_>>();
351
352        if !to_be_created.is_empty() {
353            if allow_create {
354                info!(
355                    "Recovering {} flows: {:?}",
356                    to_be_created.len(),
357                    to_be_created
358                );
359                let mut errors = vec![];
360                for flow_id in to_be_created.clone() {
361                    let flow_id = *flow_id;
362                    let info = self
363                        .flow_metadata_manager
364                        .flow_info_manager()
365                        .get(flow_id as u32)
366                        .await
367                        .map_err(BoxedError::new)
368                        .context(ExternalSnafu)?
369                        .context(FlowNotFoundSnafu { id: flow_id })?;
370
371                    let sink_table_name = [
372                        info.sink_table_name().catalog_name.clone(),
373                        info.sink_table_name().schema_name.clone(),
374                        info.sink_table_name().table_name.clone(),
375                    ];
376                    let args = CreateFlowArgs {
377                        flow_id,
378                        sink_table_name,
379                        source_table_ids: info.source_table_ids().to_vec(),
380                        // because recover should only happen on restart the `create_if_not_exists` and `or_replace` can be arbitrary value(since flow doesn't exist)
381                        // but for the sake of consistency and to make sure recover of flow actually happen, we set both to true
382                        // (which is also fine since checks for not allow both to be true is on metasrv and we already pass that)
383                        create_if_not_exists: true,
384                        or_replace: true,
385                        expire_after: info.expire_after(),
386                        eval_interval: info.eval_interval(),
387                        comment: Some(info.comment().clone()),
388                        sql: info.raw_sql().clone(),
389                        flow_options: info.options().clone(),
390                        eval_schedule: effective_eval_schedule_from_flow_info(&info),
391                        query_ctx: info
392                            .query_context()
393                            .clone()
394                            .map(|ctx| {
395                                try_to_session_query_context(ctx)
396                                    .map_err(BoxedError::new)
397                                    .context(ExternalSnafu)
398                            })
399                            .transpose()?
400                            // or use default QueryContext with catalog_name from info
401                            // to keep compatibility with old version
402                            .or_else(|| {
403                                Some(
404                                    QueryContextBuilder::default()
405                                        .current_catalog(info.catalog_name().clone())
406                                        .build(),
407                                )
408                            }),
409                    };
410                    if let Err(err) = self
411                        .create_flow(args)
412                        .await
413                        .map_err(BoxedError::new)
414                        .with_context(|_| CreateFlowSnafu {
415                            sql: info.raw_sql().clone(),
416                        })
417                    {
418                        errors.push((flow_id, err));
419                    }
420                }
421                if errors.is_empty() {
422                    info!("Recover flows successfully, flows: {:?}", to_be_created);
423                }
424
425                for (flow_id, err) in errors {
426                    warn!("Failed to recreate flow {}, err={:#?}", flow_id, err);
427                }
428            } else {
429                warn!(
430                    "Flows do not exist in flownode for node {:?}, flow_ids={:?}",
431                    nodeid, to_be_created
432                );
433            }
434        }
435        if !to_be_dropped.is_empty() {
436            if allow_drop {
437                info!("Dropping flows: {:?}", to_be_dropped);
438                let mut errors = vec![];
439                for flow_id in to_be_dropped {
440                    let flow_id = *flow_id;
441                    if let Err(err) = self.remove_flow(flow_id).await {
442                        errors.push((flow_id, err));
443                    }
444                }
445                for (flow_id, err) in errors {
446                    warn!("Failed to drop flow {}, err={:#?}", flow_id, err);
447                }
448            } else {
449                warn!(
450                    "Flows do not exist in metadata for node {:?}, flow_ids={:?}",
451                    nodeid, to_be_dropped
452                );
453            }
454        }
455        Ok(())
456    }
457
458    // TODO(discord9): consider sync this with heartbeat(might become necessary in the future)
459    pub async fn start_flow_consistent_check_task(self: &Arc<Self>) -> Result<(), Error> {
460        let mut check_task = self.check_task.lock().await;
461        ensure!(
462            check_task.is_none(),
463            IllegalCheckTaskStateSnafu {
464                reason: "Flow consistent check task already exists",
465            }
466        );
467        let task = ConsistentCheckTask::start_check_task(self).await?;
468        *check_task = Some(task);
469        Ok(())
470    }
471
472    pub async fn stop_flow_consistent_check_task(&self) -> Result<(), Error> {
473        info!("Stopping flow consistent check task");
474        let mut check_task = self.check_task.lock().await;
475
476        ensure!(
477            check_task.is_some(),
478            IllegalCheckTaskStateSnafu {
479                reason: "Flow consistent check task does not exist",
480            }
481        );
482
483        check_task.take().unwrap().stop().await?;
484        info!("Stopped flow consistent check task");
485        Ok(())
486    }
487
488    /// Reconciles in-memory flow tasks from persisted metadata.
489    pub async fn reconcile_flows_from_metadata(&self) -> Result<(), Error> {
490        self.check_flow_consistent(true, true).await
491    }
492
493    /// TODO(discord9): also add a `exists` api using flow metadata manager's `exists` method
494    async fn flow_exist_in_metadata(&self, flow_id: FlowId) -> Result<bool, Error> {
495        self.flow_metadata_manager
496            .flow_info_manager()
497            .get(flow_id as u32)
498            .await
499            .map_err(BoxedError::new)
500            .context(ExternalSnafu)
501            .map(|info| info.is_some())
502    }
503}
504
505struct ConsistentCheckTask {
506    handle: JoinHandle<()>,
507    shutdown_tx: tokio::sync::mpsc::Sender<()>,
508    trigger_tx: tokio::sync::mpsc::Sender<(bool, bool, tokio::sync::oneshot::Sender<()>)>,
509}
510
511impl ConsistentCheckTask {
512    async fn start_check_task(engine: &Arc<FlowDualEngine>) -> Result<Self, Error> {
513        let engine = engine.clone();
514        let min_refresh_duration = engine
515            .batching_engine()
516            .batch_opts
517            .experimental_min_refresh_duration;
518        let frontend_scan_timeout = engine
519            .batching_engine()
520            .batch_opts
521            .experimental_frontend_scan_timeout;
522        engine
523            .wait_for_available_frontend(frontend_scan_timeout)
524            .await?;
525        let (tx, mut rx) = tokio::sync::mpsc::channel(1);
526        let (trigger_tx, mut trigger_rx) =
527            tokio::sync::mpsc::channel::<(bool, bool, tokio::sync::oneshot::Sender<()>)>(10);
528        let handle = common_runtime::spawn_global(async move {
529            // Recover flows after the startup frontend probe succeeds.
530            let mut recover_retry = 0;
531            while let Err(err) = engine.check_flow_consistent(true, false).await {
532                recover_retry += 1;
533                error!(
534                    "Failed to recover flows:\n {err:?}, retry {} in {}s",
535                    recover_retry,
536                    min_refresh_duration.as_secs()
537                );
538                tokio::time::sleep(min_refresh_duration).await;
539            }
540
541            engine.set_done_recovering();
542
543            // then do check flows, with configurable allow_create and allow_drop
544            let (mut allow_create, mut allow_drop) = (false, false);
545            let mut ret_signal: Option<tokio::sync::oneshot::Sender<()>> = None;
546            loop {
547                if let Err(err) = engine.check_flow_consistent(allow_create, allow_drop).await {
548                    error!(err; "Failed to check flow consistent");
549                }
550                if let Some(done) = ret_signal.take() {
551                    let _ = done.send(());
552                }
553                tokio::select! {
554                    _ = rx.recv() => break,
555                    incoming = trigger_rx.recv() => if let Some(incoming) = incoming {
556                        (allow_create, allow_drop) = (incoming.0, incoming.1);
557                        ret_signal = Some(incoming.2);
558                    },
559                    _ = tokio::time::sleep(std::time::Duration::from_secs(10)) => {
560                        (allow_create, allow_drop) = (false, false);
561                    },
562                }
563            }
564        });
565        Ok(ConsistentCheckTask {
566            handle,
567            shutdown_tx: tx,
568            trigger_tx,
569        })
570    }
571
572    async fn trigger(&self, allow_create: bool, allow_drop: bool) -> Result<(), Error> {
573        let (tx, rx) = tokio::sync::oneshot::channel();
574        self.trigger_tx
575            .send((allow_create, allow_drop, tx))
576            .await
577            .map_err(|_| {
578                IllegalCheckTaskStateSnafu {
579                    reason: "Failed to send trigger signal",
580                }
581                .build()
582            })?;
583        rx.await.map_err(|_| {
584            IllegalCheckTaskStateSnafu {
585                reason: "Failed to receive trigger signal",
586            }
587            .build()
588        })?;
589        Ok(())
590    }
591
592    async fn stop(self) -> Result<(), Error> {
593        self.shutdown_tx.send(()).await.map_err(|_| {
594            IllegalCheckTaskStateSnafu {
595                reason: "Failed to send shutdown signal",
596            }
597            .build()
598        })?;
599        // abort so no need to wait
600        self.handle.abort();
601        Ok(())
602    }
603}
604
605#[derive(Default)]
606struct SrcTableToFlow {
607    /// mapping of table ids to flow ids for streaming mode
608    stream: HashMap<TableId, HashSet<FlowId>>,
609    /// mapping of table ids to flow ids for batching mode
610    batch: HashMap<TableId, HashSet<FlowId>>,
611    /// mapping of flow ids to (flow type, source table ids)
612    flow_infos: HashMap<FlowId, (FlowType, Vec<TableId>)>,
613}
614
615impl SrcTableToFlow {
616    fn in_stream(&self, table_id: TableId) -> bool {
617        self.stream.contains_key(&table_id)
618    }
619    fn in_batch(&self, table_id: TableId) -> bool {
620        self.batch.contains_key(&table_id)
621    }
622    fn add_flow(&mut self, flow_id: FlowId, flow_type: FlowType, src_table_ids: Vec<TableId>) {
623        let mapping = match flow_type {
624            FlowType::Streaming => &mut self.stream,
625            FlowType::Batching => &mut self.batch,
626        };
627
628        for src_table in src_table_ids.clone() {
629            mapping
630                .entry(src_table)
631                .and_modify(|flows| {
632                    flows.insert(flow_id);
633                })
634                .or_insert_with(|| {
635                    let mut set = HashSet::new();
636                    set.insert(flow_id);
637                    set
638                });
639        }
640        self.flow_infos.insert(flow_id, (flow_type, src_table_ids));
641    }
642
643    fn remove_flow(&mut self, flow_id: FlowId) {
644        let mapping = match self.get_flow_type(flow_id) {
645            Some(FlowType::Streaming) => &mut self.stream,
646            Some(FlowType::Batching) => &mut self.batch,
647            None => return,
648        };
649        if let Some((_, src_table_ids)) = self.flow_infos.remove(&flow_id) {
650            for src_table in src_table_ids {
651                if let Some(flows) = mapping.get_mut(&src_table) {
652                    flows.remove(&flow_id);
653                }
654            }
655        }
656    }
657
658    fn get_flow_type(&self, flow_id: FlowId) -> Option<FlowType> {
659        self.flow_infos
660            .get(&flow_id)
661            .map(|(flow_type, _)| flow_type)
662            .cloned()
663    }
664}
665
666impl FlowEngine for FlowDualEngine {
667    async fn create_flow(&self, args: CreateFlowArgs) -> Result<Option<FlowId>, Error> {
668        let flow_type = args
669            .flow_options
670            .get(FlowType::FLOW_TYPE_KEY)
671            .map(|s| s.as_str());
672
673        let flow_type = match flow_type {
674            Some(FlowType::BATCHING) => FlowType::Batching,
675            Some(FlowType::STREAMING) => FlowType::Streaming,
676            None => FlowType::Batching,
677            Some(flow_type) => {
678                return InternalSnafu {
679                    reason: format!("Invalid flow type: {}", flow_type),
680                }
681                .fail();
682            }
683        };
684
685        let flow_id = args.flow_id;
686        let src_table_ids = args.source_table_ids.clone();
687
688        let res = match flow_type {
689            FlowType::Batching => self.batching_engine.create_flow(args).await,
690            FlowType::Streaming => self.streaming_engine.create_flow(args).await,
691        }?;
692
693        self.src_table2flow
694            .write()
695            .await
696            .add_flow(flow_id, flow_type, src_table_ids);
697
698        Ok(res)
699    }
700
701    async fn remove_flow(&self, flow_id: FlowId) -> Result<(), Error> {
702        let flow_type = self.src_table2flow.read().await.get_flow_type(flow_id);
703
704        match flow_type {
705            Some(FlowType::Batching) => self.batching_engine.remove_flow(flow_id).await,
706            Some(FlowType::Streaming) => self.streaming_engine.remove_flow(flow_id).await,
707            None => {
708                // this can happen if flownode just restart, and is stilling creating the flow
709                // since now that this flow should dropped, we need to trigger the consistent check and allow drop
710                // this rely on drop flow ddl delete metadata first, see src/common/meta/src/ddl/drop_flow.rs
711                warn!(
712                    "Flow {} is not exist in the underlying engine, but exist in metadata",
713                    flow_id
714                );
715                self.try_sync_with_check_task(flow_id, true).await?;
716
717                Ok(())
718            }
719        }?;
720        // remove mapping
721        self.src_table2flow.write().await.remove_flow(flow_id);
722        Ok(())
723    }
724
725    async fn flush_flow(&self, flow_id: FlowId) -> Result<usize, Error> {
726        // sync with check task
727        self.try_sync_with_check_task(flow_id, false).await?;
728        let flow_type = self.src_table2flow.read().await.get_flow_type(flow_id);
729        match flow_type {
730            Some(FlowType::Batching) => self.batching_engine.flush_flow(flow_id).await,
731            Some(FlowType::Streaming) => self.streaming_engine.flush_flow(flow_id).await,
732            None => {
733                warn!(
734                    "Currently flow={flow_id} doesn't exist in flownode, ignore flush_flow request"
735                );
736                Ok(0)
737            }
738        }
739    }
740
741    async fn flow_exist(&self, flow_id: FlowId) -> Result<bool, Error> {
742        let flow_type = self.src_table2flow.read().await.get_flow_type(flow_id);
743        // not using `flow_type.is_some()` to make sure the flow is actually exist in the underlying engine
744        match flow_type {
745            Some(FlowType::Batching) => self.batching_engine.flow_exist(flow_id).await,
746            Some(FlowType::Streaming) => self.streaming_engine.flow_exist(flow_id).await,
747            None => Ok(false),
748        }
749    }
750
751    async fn list_flows(&self) -> Result<impl IntoIterator<Item = FlowId>, Error> {
752        let stream_flows = self.streaming_engine.list_flows().await?;
753        let batch_flows = self.batching_engine.list_flows().await?;
754
755        Ok(stream_flows.into_iter().chain(batch_flows))
756    }
757
758    async fn handle_flow_inserts(
759        &self,
760        request: api::v1::region::InsertRequests,
761    ) -> Result<(), Error> {
762        self.wait_for_all_flow_recover(request.requests.len())
763            .await?;
764        // TODO(discord9): make as little clone as possible
765        let mut to_stream_engine = Vec::with_capacity(request.requests.len());
766        let mut to_batch_engine = request.requests;
767
768        let mut batching_row_cnt = 0;
769        let mut streaming_row_cnt = 0;
770
771        {
772            // not locking this, or recover flows will be starved when also handling flow inserts
773            let src_table2flow = self.src_table2flow.read().await;
774            to_batch_engine.retain(|req| {
775                let region_id = RegionId::from(req.region_id);
776                let table_id = region_id.table_id();
777                let is_in_stream = src_table2flow.in_stream(table_id);
778                let is_in_batch = src_table2flow.in_batch(table_id);
779                if is_in_stream {
780                    streaming_row_cnt += req.rows.as_ref().map(|rs| rs.rows.len()).unwrap_or(0);
781                    to_stream_engine.push(req.clone());
782                }
783                if is_in_batch {
784                    batching_row_cnt += req.rows.as_ref().map(|rs| rs.rows.len()).unwrap_or(0);
785                    return true;
786                }
787                if !is_in_batch && !is_in_stream {
788                    // TODO(discord9): also put to centralized logging for flow once it implemented
789                    warn!("Table {} is not any flow's source table", table_id)
790                }
791                false
792            });
793            // drop(src_table2flow);
794            // can't use drop due to https://github.com/rust-lang/rust/pull/128846
795        }
796
797        METRIC_FLOW_ROWS
798            .with_label_values(&["in-streaming"])
799            .inc_by(streaming_row_cnt as u64);
800
801        METRIC_FLOW_ROWS
802            .with_label_values(&["in-batching"])
803            .inc_by(batching_row_cnt as u64);
804
805        let streaming_engine = self.streaming_engine.clone();
806        let stream_handler: JoinHandle<Result<(), Error>> =
807            common_runtime::spawn_global(async move {
808                streaming_engine
809                    .handle_flow_inserts(api::v1::region::InsertRequests {
810                        requests: to_stream_engine,
811                    })
812                    .await?;
813                Ok(())
814            });
815        self.batching_engine
816            .handle_flow_inserts(api::v1::region::InsertRequests {
817                requests: to_batch_engine,
818            })
819            .await?;
820        stream_handler.await.context(JoinTaskSnafu)??;
821
822        Ok(())
823    }
824
825    async fn handle_mark_window_dirty(
826        &self,
827        req: api::v1::flow::DirtyWindowRequests,
828    ) -> Result<(), Error> {
829        self.batching_engine.handle_mark_window_dirty(req).await
830    }
831}
832
833#[async_trait::async_trait]
834impl common_meta::node_manager::Flownode for FlowDualEngine {
835    async fn handle(&self, request: FlowRequest) -> MetaResult<FlowResponse> {
836        let query_ctx = request
837            .header
838            .and_then(|h| h.query_context)
839            .map(|ctx| ctx.into());
840        match request.body {
841            Some(flow_request::Body::Create(CreateRequest {
842                flow_id: Some(task_id),
843                source_table_ids,
844                sink_table_name: Some(sink_table_name),
845                create_if_not_exists,
846                expire_after,
847                eval_interval,
848                comment,
849                sql,
850                mut flow_options,
851                or_replace,
852            })) => {
853                let source_table_ids = source_table_ids.into_iter().map(|id| id.id).collect_vec();
854                let sink_table_name = [
855                    sink_table_name.catalog_name,
856                    sink_table_name.schema_name,
857                    sink_table_name.table_name,
858                ];
859                let expire_after = expire_after.map(|e| e.value);
860
861                let eval_schedule = decode_internal_eval_schedule(&mut flow_options)
862                    .map_err(to_meta_err(snafu::location!()))?;
863
864                let args = CreateFlowArgs {
865                    flow_id: task_id.id as u64,
866                    sink_table_name,
867                    source_table_ids,
868                    create_if_not_exists,
869                    or_replace,
870                    expire_after,
871                    eval_interval: eval_interval.map(|e| e.seconds),
872                    comment: Some(comment),
873                    sql: sql.clone(),
874                    flow_options,
875                    query_ctx,
876                    eval_schedule,
877                };
878                let ret = self
879                    .create_flow(args)
880                    .await
881                    .map_err(BoxedError::new)
882                    .with_context(|_| CreateFlowSnafu { sql: sql.clone() })
883                    .map_err(to_meta_err(snafu::location!()))?;
884                METRIC_FLOW_TASK_COUNT.inc();
885                Ok(FlowResponse {
886                    affected_flows: ret
887                        .map(|id| greptime_proto::v1::FlowId { id: id as u32 })
888                        .into_iter()
889                        .collect_vec(),
890                    ..Default::default()
891                })
892            }
893            Some(flow_request::Body::Drop(DropRequest {
894                flow_id: Some(flow_id),
895            })) => {
896                self.remove_flow(flow_id.id as u64)
897                    .await
898                    .map_err(to_meta_err(snafu::location!()))?;
899                METRIC_FLOW_TASK_COUNT.dec();
900                Ok(Default::default())
901            }
902            Some(flow_request::Body::Flush(FlushFlow {
903                flow_id: Some(flow_id),
904            })) => {
905                let row = self
906                    .flush_flow(flow_id.id as u64)
907                    .await
908                    .map_err(to_meta_err(snafu::location!()))?;
909                Ok(FlowResponse {
910                    affected_flows: vec![flow_id],
911                    affected_rows: row as u64,
912                    ..Default::default()
913                })
914            }
915            other => common_meta::error::InvalidFlowRequestBodySnafu { body: other }.fail(),
916        }
917    }
918
919    async fn handle_inserts(&self, request: InsertRequests) -> MetaResult<FlowResponse> {
920        FlowEngine::handle_flow_inserts(self, request)
921            .await
922            .map(|_| Default::default())
923            .map_err(to_meta_err(snafu::location!()))
924    }
925
926    async fn handle_mark_window_dirty(&self, req: DirtyWindowRequests) -> MetaResult<FlowResponse> {
927        self.batching_engine()
928            .handle_mark_dirty_time_window(req)
929            .await
930            .map(|_| FlowResponse::default())
931            .map_err(to_meta_err(snafu::location!()))
932    }
933}
934
935/// Decode typed schedule config from the internal transient key emitted by metasrv.
936/// Malformed JSON is an internal error rather than a reason to silently fall back.
937fn decode_internal_eval_schedule(
938    flow_options: &mut HashMap<String, String>,
939) -> Result<Option<FlowScheduleConfig>, Error> {
940    match flow_options.remove(INTERNAL_EVAL_SCHEDULE_KEY) {
941        Some(json) => serde_json::from_str::<FlowScheduleConfig>(&json)
942            .map(Some)
943            .map_err(|err| {
944                InternalSnafu {
945                    reason: format!("Invalid internal eval schedule payload: {err}"),
946                }
947                .build()
948            }),
949        None => Ok(None),
950    }
951}
952
953/// return a function to convert `crate::error::Error` to `common_meta::error::Error`
954fn to_meta_err(
955    location: snafu::Location,
956) -> impl FnOnce(crate::error::Error) -> common_meta::error::Error {
957    move |err: crate::error::Error| -> common_meta::error::Error {
958        match err {
959            crate::error::Error::FlowNotFound { id, .. } => {
960                common_meta::error::Error::FlowNotFound {
961                    flow_name: format!("flow_id={id}"),
962                    location,
963                }
964            }
965            _ => common_meta::error::Error::External {
966                location,
967                source: BoxedError::new(err),
968            },
969        }
970    }
971}
972
973impl FlowEngine for StreamingEngine {
974    async fn create_flow(&self, args: CreateFlowArgs) -> Result<Option<FlowId>, Error> {
975        self.create_flow_inner(args).await
976    }
977
978    async fn remove_flow(&self, flow_id: FlowId) -> Result<(), Error> {
979        self.remove_flow_inner(flow_id).await
980    }
981
982    async fn flush_flow(&self, flow_id: FlowId) -> Result<usize, Error> {
983        self.flush_flow_inner(flow_id).await
984    }
985
986    async fn flow_exist(&self, flow_id: FlowId) -> Result<bool, Error> {
987        self.flow_exist_inner(flow_id).await
988    }
989
990    async fn list_flows(&self) -> Result<impl IntoIterator<Item = FlowId>, Error> {
991        Ok(self
992            .flow_err_collectors
993            .read()
994            .await
995            .keys()
996            .cloned()
997            .collect::<Vec<_>>())
998    }
999
1000    async fn handle_flow_inserts(
1001        &self,
1002        request: api::v1::region::InsertRequests,
1003    ) -> Result<(), Error> {
1004        self.handle_inserts_inner(request).await
1005    }
1006
1007    async fn handle_mark_window_dirty(
1008        &self,
1009        _req: api::v1::flow::DirtyWindowRequests,
1010    ) -> Result<(), Error> {
1011        UnsupportedSnafu {
1012            reason: "handle_mark_window_dirty in streaming engine",
1013        }
1014        .fail()
1015    }
1016}
1017
1018/// Simple helper enum for fetching value from row with default value
1019#[derive(Debug, Clone)]
1020enum FetchFromRow {
1021    Idx(usize),
1022    Default(Value),
1023}
1024
1025impl FetchFromRow {
1026    /// Panic if idx is out of bound
1027    fn fetch(&self, row: &repr::Row) -> Value {
1028        match self {
1029            FetchFromRow::Idx(idx) => row.get(*idx).unwrap().clone(),
1030            FetchFromRow::Default(v) => v.clone(),
1031        }
1032    }
1033}
1034
1035impl StreamingEngine {
1036    async fn handle_inserts_inner(
1037        &self,
1038        request: InsertRequests,
1039    ) -> std::result::Result<(), Error> {
1040        // using try_read to ensure two things:
1041        // 1. flush wouldn't happen until inserts before it is inserted
1042        // 2. inserts happening concurrently with flush wouldn't be block by flush
1043        let _flush_lock = self.flush_lock.try_read();
1044        for write_request in request.requests {
1045            let region_id = write_request.region_id;
1046            let table_id = RegionId::from(region_id).table_id();
1047
1048            let (insert_schema, rows_proto) = write_request
1049                .rows
1050                .map(|r| (r.schema, r.rows))
1051                .unwrap_or_default();
1052
1053            // TODO(discord9): reconsider time assignment mechanism
1054            let now = self.tick_manager.tick();
1055
1056            let (table_types, fetch_order) = {
1057                let ctx = self.node_context.read().await;
1058
1059                // TODO(discord9): also check schema version so that altered table can be reported
1060                let table_schema = ctx.table_source.table_from_id(&table_id).await?;
1061                let default_vals = table_schema
1062                    .default_values
1063                    .iter()
1064                    .zip(table_schema.relation_desc.typ().column_types.iter())
1065                    .map(|(v, ty)| {
1066                        v.as_ref().and_then(|v| {
1067                            match v.create_default(ty.scalar_type(), ty.nullable()) {
1068                                Ok(v) => Some(v),
1069                                Err(err) => {
1070                                    common_telemetry::error!(err; "Failed to create default value");
1071                                    None
1072                                }
1073                            }
1074                        })
1075                    })
1076                    .collect_vec();
1077
1078                let table_types = table_schema
1079                    .relation_desc
1080                    .typ()
1081                    .column_types
1082                    .clone()
1083                    .into_iter()
1084                    .map(|t| t.scalar_type)
1085                    .collect_vec();
1086                let table_col_names = table_schema.relation_desc.names;
1087                let table_col_names = table_col_names
1088                    .iter().enumerate()
1089                    .map(|(idx,name)| match name {
1090                        Some(name) => Ok(name.clone()),
1091                        None => InternalSnafu {
1092                            reason: format!("Expect column {idx} of table id={table_id} to have name in table schema, found None"),
1093                        }
1094                        .fail(),
1095                    })
1096                    .collect::<Result<Vec<_>, _>>()?;
1097                let name_to_col = HashMap::<_, _>::from_iter(
1098                    insert_schema
1099                        .iter()
1100                        .enumerate()
1101                        .map(|(i, name)| (&name.column_name, i)),
1102                );
1103
1104                let fetch_order: Vec<FetchFromRow> = table_col_names
1105                    .iter()
1106                    .zip(default_vals)
1107                    .map(|(col_name, col_default_val)| {
1108                        name_to_col
1109                            .get(col_name)
1110                            .copied()
1111                            .map(FetchFromRow::Idx)
1112                            .or_else(|| col_default_val.clone().map(FetchFromRow::Default))
1113                            .with_context(|| UnexpectedSnafu {
1114                                reason: format!(
1115                                    "Column not found: {}, default_value: {:?}",
1116                                    col_name, col_default_val
1117                                ),
1118                            })
1119                    })
1120                    .try_collect()?;
1121
1122                trace!("Reordering columns: {:?}", fetch_order);
1123                (table_types, fetch_order)
1124            };
1125
1126            // TODO(discord9): use column instead of row
1127            let rows: Vec<DiffRow> = rows_proto
1128                .into_iter()
1129                .map(|r| {
1130                    let r = repr::Row::from(r);
1131                    let reordered = fetch_order.iter().map(|i| i.fetch(&r)).collect_vec();
1132                    repr::Row::new(reordered)
1133                })
1134                .map(|r| (r, now, 1))
1135                .collect_vec();
1136            if let Err(err) = self
1137                .handle_write_request(region_id.into(), rows, &table_types)
1138                .await
1139            {
1140                let err = BoxedError::new(err);
1141                let flow_ids = self
1142                    .node_context
1143                    .read()
1144                    .await
1145                    .get_flow_ids(table_id)
1146                    .into_iter()
1147                    .flatten()
1148                    .cloned()
1149                    .collect_vec();
1150                let err = InsertIntoFlowSnafu {
1151                    region_id,
1152                    flow_ids,
1153                }
1154                .into_error(err);
1155                common_telemetry::error!(err; "Failed to handle write request");
1156                return Err(err);
1157            }
1158        }
1159        Ok(())
1160    }
1161}
1162
1163#[cfg(test)]
1164mod tests {
1165    use std::collections::HashMap;
1166
1167    use common_meta::ddl::create_flow::INTERNAL_EVAL_SCHEDULE_KEY;
1168
1169    use super::decode_internal_eval_schedule;
1170    use crate::error::Error;
1171
1172    #[test]
1173    fn test_malformed_internal_eval_schedule_json_is_error() {
1174        let mut flow_options = HashMap::new();
1175        flow_options.insert(
1176            INTERNAL_EVAL_SCHEDULE_KEY.to_string(),
1177            "not-json".to_string(),
1178        );
1179
1180        let err = decode_internal_eval_schedule(&mut flow_options).unwrap_err();
1181        assert!(matches!(
1182            err,
1183            Error::Internal { reason, .. }
1184                if reason.contains("Invalid internal eval schedule payload")
1185        ));
1186        assert!(!flow_options.contains_key(INTERNAL_EVAL_SCHEDULE_KEY));
1187    }
1188}