1use std::collections::{HashMap, HashSet};
17use std::sync::Arc;
18use std::sync::atomic::AtomicBool;
19
20use api::v1::flow::{
21 CreateRequest, DirtyWindowRequests, DropRequest, FlowRequest, FlowResponse, FlushFlow,
22 flow_request,
23};
24use api::v1::region::InsertRequests;
25use catalog::CatalogManager;
26use common_base::Plugins;
27use common_error::ext::BoxedError;
28use common_meta::ddl::create_flow::{
29 FlowType, INTERNAL_EVAL_SCHEDULE_KEY, effective_eval_schedule_from_flow_info,
30};
31use common_meta::error::Result as MetaResult;
32use common_meta::key::flow::FlowMetadataManager;
33use common_meta::key::flow::flow_info::FlowScheduleConfig;
34use common_meta::key::flow::flow_state::FlowStat;
35use common_runtime::JoinHandle;
36use common_telemetry::{error, info, trace, warn};
37use datatypes::value::Value;
38use futures::TryStreamExt;
39use itertools::Itertools;
40use operator::utils::try_to_session_query_context;
41use session::context::QueryContextBuilder;
42use snafu::{IntoError, OptionExt, ResultExt, ensure};
43use store_api::storage::{RegionId, TableId};
44use tokio::sync::{Mutex, RwLock};
45
46use crate::adapter::{CreateFlowArgs, StreamingEngine};
47use crate::batching_mode::engine::BatchingEngine;
48use crate::engine::{FlowEngine, FlowStatProvider};
49use crate::error::{
50 CreateFlowSnafu, ExternalSnafu, FlowNotFoundSnafu, FlowNotRecoveredSnafu,
51 IllegalCheckTaskStateSnafu, InsertIntoFlowSnafu, InternalSnafu, JoinTaskSnafu, ListFlowsSnafu,
52 NoAvailableFrontendSnafu, SyncCheckTaskSnafu, UnexpectedSnafu, UnsupportedSnafu,
53};
54use crate::metrics::{METRIC_FLOW_ROWS, METRIC_FLOW_TASK_COUNT};
55use crate::repr::{self, DiffRow};
56use crate::utils::StateReportHandler;
57use crate::{Error, FlowId};
58
59pub type FlowDualEngineRef = Arc<FlowDualEngine>;
61
62pub struct FlowDualEngine {
67 streaming_engine: Arc<StreamingEngine>,
68 batching_engine: Arc<BatchingEngine>,
69 state_report_handler: RwLock<Option<StateReportHandler>>,
71 src_table2flow: RwLock<SrcTableToFlow>,
73 flow_metadata_manager: Arc<FlowMetadataManager>,
74 catalog_manager: Arc<dyn CatalogManager>,
75 check_task: tokio::sync::Mutex<Option<ConsistentCheckTask>>,
76 plugins: Plugins,
77 done_recovering: AtomicBool,
78}
79
80impl FlowDualEngine {
81 pub fn new(
82 streaming_engine: Arc<StreamingEngine>,
83 batching_engine: Arc<BatchingEngine>,
84 flow_metadata_manager: Arc<FlowMetadataManager>,
85 catalog_manager: Arc<dyn CatalogManager>,
86 plugins: Plugins,
87 ) -> Self {
88 Self {
89 streaming_engine,
90 batching_engine,
91 state_report_handler: Default::default(),
92 src_table2flow: RwLock::new(SrcTableToFlow::default()),
93 flow_metadata_manager,
94 catalog_manager,
95 check_task: Mutex::new(None),
96 plugins,
97 done_recovering: AtomicBool::new(false),
98 }
99 }
100
101 pub fn set_done_recovering(&self) {
104 info!("FlowDualEngine done recovering");
105 self.done_recovering
106 .store(true, std::sync::atomic::Ordering::Release);
107 }
108
109 pub fn is_recover_done(&self) -> bool {
111 self.done_recovering
112 .load(std::sync::atomic::Ordering::Acquire)
113 }
114
115 async fn wait_for_all_flow_recover(&self, waiting_req_cnt: usize) -> Result<(), Error> {
117 if self.is_recover_done() {
118 return Ok(());
119 }
120
121 warn!(
122 "FlowDualEngine is not done recovering, {} insert request waiting for recovery",
123 waiting_req_cnt
124 );
125 let mut retry = 0;
128 let max_retry = 3;
129 while retry < max_retry && !self.is_recover_done() {
130 warn!(
131 "FlowDualEngine is not done recovering, retry {} in 1s",
132 retry
133 );
134 tokio::time::sleep(std::time::Duration::from_secs(1)).await;
135 retry += 1;
136 }
137 if retry == max_retry {
138 return FlowNotRecoveredSnafu.fail();
139 } else {
140 info!("FlowDualEngine is done recovering");
141 }
142 Ok(())
144 }
145
146 pub fn plugins(&self) -> &Plugins {
147 &self.plugins
148 }
149
150 pub fn is_distributed(&self) -> bool {
152 self.streaming_engine.node_id.is_some()
153 }
154
155 pub fn streaming_engine(&self) -> Arc<StreamingEngine> {
156 self.streaming_engine.clone()
157 }
158
159 pub fn batching_engine(&self) -> Arc<BatchingEngine> {
160 self.batching_engine.clone()
161 }
162
163 pub async fn set_state_report_handler(&self, handler: StateReportHandler) {
164 *self.state_report_handler.write().await = Some(handler);
165 }
166
167 pub async fn gen_state_report(&self) -> FlowStat {
168 let streaming = self.streaming_engine.flow_stat().await;
169 let batching = self.batching_engine.flow_stat().await;
170
171 let mut state_size = streaming.state_size;
172 state_size.extend(batching.state_size);
173
174 let mut last_exec_time_map = streaming.last_exec_time_map;
175 last_exec_time_map.extend(batching.last_exec_time_map);
176
177 let mut start_time_map = streaming.start_time_map;
178 start_time_map.extend(batching.start_time_map);
179
180 FlowStat {
181 state_size,
182 last_exec_time_map,
183 start_time_map,
184 }
185 }
186
187 pub async fn start_state_report_task(self: Arc<Self>) -> Option<JoinHandle<()>> {
191 let state_report_handler = self.state_report_handler.write().await.take();
192 if let Some(mut handler) = state_report_handler {
193 let zelf = self.clone();
194 let handler = common_runtime::spawn_global(async move {
195 while let Some(ret_handler) = handler.recv().await {
196 let state_report = zelf.gen_state_report().await;
197 ret_handler.send(state_report).unwrap_or_else(|err| {
198 common_telemetry::error!(err; "Send state report error");
199 });
200 }
201 });
202 Some(handler)
203 } else {
204 None
205 }
206 }
207
208 async fn wait_for_available_frontend(&self, timeout: std::time::Duration) -> Result<(), Error> {
212 if !self.is_distributed() {
213 return Ok(());
214 }
215 let frontend_client = self.batching_engine().frontend_client.clone();
216 let sleep_duration = std::time::Duration::from_millis(1_000);
217 let now = std::time::Instant::now();
218 loop {
219 let frontend_list = frontend_client.scan_for_frontend().await?;
220 if !frontend_list.is_empty() {
221 let fe_list = frontend_list
222 .iter()
223 .map(|peer| &peer.addr)
224 .collect::<Vec<_>>();
225 let probe_failures = frontend_client
226 .check_all_frontends_without_auth(&frontend_list)
227 .await?;
228 if probe_failures.is_empty() {
229 info!(
230 "Available frontend found and unauthenticated probe succeeded: {:?}",
231 fe_list
232 );
233 return Ok(());
234 }
235 warn!(
236 "Unauthenticated frontend probe failed, will retry. frontends={:?}, failures={:?}",
237 fe_list, probe_failures
238 );
239 }
240 let elapsed = now.elapsed();
241 tokio::time::sleep(sleep_duration).await;
242 info!("Waiting for available frontend, elapsed={:?}", elapsed);
243 if elapsed >= timeout {
244 return NoAvailableFrontendSnafu {
245 timeout,
246 context: "No frontend accepted unauthenticated flownode probe",
247 }
248 .fail();
249 }
250 }
251 }
252
253 async fn try_sync_with_check_task(
257 &self,
258 flow_id: FlowId,
259 allow_drop: bool,
260 ) -> Result<(), Error> {
261 info!("Try to sync with check task for flow {}", flow_id);
263 let mut retry = 0;
264 let max_retry = 10;
265 while retry < max_retry {
267 if let Some(task) = self.check_task.lock().await.as_ref() {
268 task.trigger(false, allow_drop).await?;
269 break;
270 }
271 retry += 1;
272 tokio::time::sleep(std::time::Duration::from_millis(500)).await;
273 }
274
275 if retry == max_retry {
276 error!(
277 "Can't sync with check task for flow {} with allow_drop={}",
278 flow_id, allow_drop
279 );
280 return SyncCheckTaskSnafu {
281 flow_id,
282 allow_drop,
283 }
284 .fail();
285 }
286 info!("Successfully sync with check task for flow {}", flow_id);
287
288 Ok(())
289 }
290
291 async fn check_flow_consistent(
294 &self,
295 allow_create: bool,
296 allow_drop: bool,
297 ) -> Result<(), Error> {
298 let nodeid = self.streaming_engine.node_id;
300 let should_exists: Vec<_> = if let Some(nodeid) = nodeid {
301 let to_be_recover = self
304 .flow_metadata_manager
305 .flownode_flow_manager()
306 .flows(nodeid.into())
307 .try_collect::<Vec<_>>()
308 .await
309 .context(ListFlowsSnafu {
310 id: Some(nodeid.into()),
311 })?;
312 to_be_recover.into_iter().map(|(id, _)| id).collect()
313 } else {
314 let all_catalogs = self
317 .catalog_manager
318 .catalog_names()
319 .await
320 .map_err(BoxedError::new)
321 .context(ExternalSnafu)?;
322 let mut all_flow_ids = vec![];
323 for catalog in all_catalogs {
324 let flows = self
325 .flow_metadata_manager
326 .flow_name_manager()
327 .flow_names(&catalog)
328 .await
329 .try_collect::<Vec<_>>()
330 .await
331 .map_err(BoxedError::new)
332 .context(ExternalSnafu)?;
333
334 all_flow_ids.extend(flows.into_iter().map(|(_, id)| id.flow_id()));
335 }
336 all_flow_ids
337 };
338 let should_exists = should_exists
339 .into_iter()
340 .map(|i| i as FlowId)
341 .collect::<HashSet<_>>();
342 let actual_exists = self.list_flows().await?.into_iter().collect::<HashSet<_>>();
343 let to_be_created = should_exists
344 .iter()
345 .filter(|id| !actual_exists.contains(id))
346 .collect::<Vec<_>>();
347 let to_be_dropped = actual_exists
348 .iter()
349 .filter(|id| !should_exists.contains(id))
350 .collect::<Vec<_>>();
351
352 if !to_be_created.is_empty() {
353 if allow_create {
354 info!(
355 "Recovering {} flows: {:?}",
356 to_be_created.len(),
357 to_be_created
358 );
359 let mut errors = vec![];
360 for flow_id in to_be_created.clone() {
361 let flow_id = *flow_id;
362 let info = self
363 .flow_metadata_manager
364 .flow_info_manager()
365 .get(flow_id as u32)
366 .await
367 .map_err(BoxedError::new)
368 .context(ExternalSnafu)?
369 .context(FlowNotFoundSnafu { id: flow_id })?;
370
371 let sink_table_name = [
372 info.sink_table_name().catalog_name.clone(),
373 info.sink_table_name().schema_name.clone(),
374 info.sink_table_name().table_name.clone(),
375 ];
376 let args = CreateFlowArgs {
377 flow_id,
378 sink_table_name,
379 source_table_ids: info.source_table_ids().to_vec(),
380 create_if_not_exists: true,
384 or_replace: true,
385 expire_after: info.expire_after(),
386 eval_interval: info.eval_interval(),
387 comment: Some(info.comment().clone()),
388 sql: info.raw_sql().clone(),
389 flow_options: info.options().clone(),
390 eval_schedule: effective_eval_schedule_from_flow_info(&info),
391 query_ctx: info
392 .query_context()
393 .clone()
394 .map(|ctx| {
395 try_to_session_query_context(ctx)
396 .map_err(BoxedError::new)
397 .context(ExternalSnafu)
398 })
399 .transpose()?
400 .or_else(|| {
403 Some(
404 QueryContextBuilder::default()
405 .current_catalog(info.catalog_name().clone())
406 .build(),
407 )
408 }),
409 };
410 if let Err(err) = self
411 .create_flow(args)
412 .await
413 .map_err(BoxedError::new)
414 .with_context(|_| CreateFlowSnafu {
415 sql: info.raw_sql().clone(),
416 })
417 {
418 errors.push((flow_id, err));
419 }
420 }
421 if errors.is_empty() {
422 info!("Recover flows successfully, flows: {:?}", to_be_created);
423 }
424
425 for (flow_id, err) in errors {
426 warn!("Failed to recreate flow {}, err={:#?}", flow_id, err);
427 }
428 } else {
429 warn!(
430 "Flows do not exist in flownode for node {:?}, flow_ids={:?}",
431 nodeid, to_be_created
432 );
433 }
434 }
435 if !to_be_dropped.is_empty() {
436 if allow_drop {
437 info!("Dropping flows: {:?}", to_be_dropped);
438 let mut errors = vec![];
439 for flow_id in to_be_dropped {
440 let flow_id = *flow_id;
441 if let Err(err) = self.remove_flow(flow_id).await {
442 errors.push((flow_id, err));
443 }
444 }
445 for (flow_id, err) in errors {
446 warn!("Failed to drop flow {}, err={:#?}", flow_id, err);
447 }
448 } else {
449 warn!(
450 "Flows do not exist in metadata for node {:?}, flow_ids={:?}",
451 nodeid, to_be_dropped
452 );
453 }
454 }
455 Ok(())
456 }
457
458 pub async fn start_flow_consistent_check_task(self: &Arc<Self>) -> Result<(), Error> {
460 let mut check_task = self.check_task.lock().await;
461 ensure!(
462 check_task.is_none(),
463 IllegalCheckTaskStateSnafu {
464 reason: "Flow consistent check task already exists",
465 }
466 );
467 let task = ConsistentCheckTask::start_check_task(self).await?;
468 *check_task = Some(task);
469 Ok(())
470 }
471
472 pub async fn stop_flow_consistent_check_task(&self) -> Result<(), Error> {
473 info!("Stopping flow consistent check task");
474 let mut check_task = self.check_task.lock().await;
475
476 ensure!(
477 check_task.is_some(),
478 IllegalCheckTaskStateSnafu {
479 reason: "Flow consistent check task does not exist",
480 }
481 );
482
483 check_task.take().unwrap().stop().await?;
484 info!("Stopped flow consistent check task");
485 Ok(())
486 }
487
488 pub async fn reconcile_flows_from_metadata(&self) -> Result<(), Error> {
490 self.check_flow_consistent(true, true).await
491 }
492
493 async fn flow_exist_in_metadata(&self, flow_id: FlowId) -> Result<bool, Error> {
495 self.flow_metadata_manager
496 .flow_info_manager()
497 .get(flow_id as u32)
498 .await
499 .map_err(BoxedError::new)
500 .context(ExternalSnafu)
501 .map(|info| info.is_some())
502 }
503}
504
505struct ConsistentCheckTask {
506 handle: JoinHandle<()>,
507 shutdown_tx: tokio::sync::mpsc::Sender<()>,
508 trigger_tx: tokio::sync::mpsc::Sender<(bool, bool, tokio::sync::oneshot::Sender<()>)>,
509}
510
511impl ConsistentCheckTask {
512 async fn start_check_task(engine: &Arc<FlowDualEngine>) -> Result<Self, Error> {
513 let engine = engine.clone();
514 let min_refresh_duration = engine
515 .batching_engine()
516 .batch_opts
517 .experimental_min_refresh_duration;
518 let frontend_scan_timeout = engine
519 .batching_engine()
520 .batch_opts
521 .experimental_frontend_scan_timeout;
522 engine
523 .wait_for_available_frontend(frontend_scan_timeout)
524 .await?;
525 let (tx, mut rx) = tokio::sync::mpsc::channel(1);
526 let (trigger_tx, mut trigger_rx) =
527 tokio::sync::mpsc::channel::<(bool, bool, tokio::sync::oneshot::Sender<()>)>(10);
528 let handle = common_runtime::spawn_global(async move {
529 let mut recover_retry = 0;
531 while let Err(err) = engine.check_flow_consistent(true, false).await {
532 recover_retry += 1;
533 error!(
534 "Failed to recover flows:\n {err:?}, retry {} in {}s",
535 recover_retry,
536 min_refresh_duration.as_secs()
537 );
538 tokio::time::sleep(min_refresh_duration).await;
539 }
540
541 engine.set_done_recovering();
542
543 let (mut allow_create, mut allow_drop) = (false, false);
545 let mut ret_signal: Option<tokio::sync::oneshot::Sender<()>> = None;
546 loop {
547 if let Err(err) = engine.check_flow_consistent(allow_create, allow_drop).await {
548 error!(err; "Failed to check flow consistent");
549 }
550 if let Some(done) = ret_signal.take() {
551 let _ = done.send(());
552 }
553 tokio::select! {
554 _ = rx.recv() => break,
555 incoming = trigger_rx.recv() => if let Some(incoming) = incoming {
556 (allow_create, allow_drop) = (incoming.0, incoming.1);
557 ret_signal = Some(incoming.2);
558 },
559 _ = tokio::time::sleep(std::time::Duration::from_secs(10)) => {
560 (allow_create, allow_drop) = (false, false);
561 },
562 }
563 }
564 });
565 Ok(ConsistentCheckTask {
566 handle,
567 shutdown_tx: tx,
568 trigger_tx,
569 })
570 }
571
572 async fn trigger(&self, allow_create: bool, allow_drop: bool) -> Result<(), Error> {
573 let (tx, rx) = tokio::sync::oneshot::channel();
574 self.trigger_tx
575 .send((allow_create, allow_drop, tx))
576 .await
577 .map_err(|_| {
578 IllegalCheckTaskStateSnafu {
579 reason: "Failed to send trigger signal",
580 }
581 .build()
582 })?;
583 rx.await.map_err(|_| {
584 IllegalCheckTaskStateSnafu {
585 reason: "Failed to receive trigger signal",
586 }
587 .build()
588 })?;
589 Ok(())
590 }
591
592 async fn stop(self) -> Result<(), Error> {
593 self.shutdown_tx.send(()).await.map_err(|_| {
594 IllegalCheckTaskStateSnafu {
595 reason: "Failed to send shutdown signal",
596 }
597 .build()
598 })?;
599 self.handle.abort();
601 Ok(())
602 }
603}
604
605#[derive(Default)]
606struct SrcTableToFlow {
607 stream: HashMap<TableId, HashSet<FlowId>>,
609 batch: HashMap<TableId, HashSet<FlowId>>,
611 flow_infos: HashMap<FlowId, (FlowType, Vec<TableId>)>,
613}
614
615impl SrcTableToFlow {
616 fn in_stream(&self, table_id: TableId) -> bool {
617 self.stream.contains_key(&table_id)
618 }
619 fn in_batch(&self, table_id: TableId) -> bool {
620 self.batch.contains_key(&table_id)
621 }
622 fn add_flow(&mut self, flow_id: FlowId, flow_type: FlowType, src_table_ids: Vec<TableId>) {
623 let mapping = match flow_type {
624 FlowType::Streaming => &mut self.stream,
625 FlowType::Batching => &mut self.batch,
626 };
627
628 for src_table in src_table_ids.clone() {
629 mapping
630 .entry(src_table)
631 .and_modify(|flows| {
632 flows.insert(flow_id);
633 })
634 .or_insert_with(|| {
635 let mut set = HashSet::new();
636 set.insert(flow_id);
637 set
638 });
639 }
640 self.flow_infos.insert(flow_id, (flow_type, src_table_ids));
641 }
642
643 fn remove_flow(&mut self, flow_id: FlowId) {
644 let mapping = match self.get_flow_type(flow_id) {
645 Some(FlowType::Streaming) => &mut self.stream,
646 Some(FlowType::Batching) => &mut self.batch,
647 None => return,
648 };
649 if let Some((_, src_table_ids)) = self.flow_infos.remove(&flow_id) {
650 for src_table in src_table_ids {
651 if let Some(flows) = mapping.get_mut(&src_table) {
652 flows.remove(&flow_id);
653 }
654 }
655 }
656 }
657
658 fn get_flow_type(&self, flow_id: FlowId) -> Option<FlowType> {
659 self.flow_infos
660 .get(&flow_id)
661 .map(|(flow_type, _)| flow_type)
662 .cloned()
663 }
664}
665
666impl FlowEngine for FlowDualEngine {
667 async fn create_flow(&self, args: CreateFlowArgs) -> Result<Option<FlowId>, Error> {
668 let flow_type = args
669 .flow_options
670 .get(FlowType::FLOW_TYPE_KEY)
671 .map(|s| s.as_str());
672
673 let flow_type = match flow_type {
674 Some(FlowType::BATCHING) => FlowType::Batching,
675 Some(FlowType::STREAMING) => FlowType::Streaming,
676 None => FlowType::Batching,
677 Some(flow_type) => {
678 return InternalSnafu {
679 reason: format!("Invalid flow type: {}", flow_type),
680 }
681 .fail();
682 }
683 };
684
685 let flow_id = args.flow_id;
686 let src_table_ids = args.source_table_ids.clone();
687
688 let res = match flow_type {
689 FlowType::Batching => self.batching_engine.create_flow(args).await,
690 FlowType::Streaming => self.streaming_engine.create_flow(args).await,
691 }?;
692
693 self.src_table2flow
694 .write()
695 .await
696 .add_flow(flow_id, flow_type, src_table_ids);
697
698 Ok(res)
699 }
700
701 async fn remove_flow(&self, flow_id: FlowId) -> Result<(), Error> {
702 let flow_type = self.src_table2flow.read().await.get_flow_type(flow_id);
703
704 match flow_type {
705 Some(FlowType::Batching) => self.batching_engine.remove_flow(flow_id).await,
706 Some(FlowType::Streaming) => self.streaming_engine.remove_flow(flow_id).await,
707 None => {
708 warn!(
712 "Flow {} is not exist in the underlying engine, but exist in metadata",
713 flow_id
714 );
715 self.try_sync_with_check_task(flow_id, true).await?;
716
717 Ok(())
718 }
719 }?;
720 self.src_table2flow.write().await.remove_flow(flow_id);
722 Ok(())
723 }
724
725 async fn flush_flow(&self, flow_id: FlowId) -> Result<usize, Error> {
726 self.try_sync_with_check_task(flow_id, false).await?;
728 let flow_type = self.src_table2flow.read().await.get_flow_type(flow_id);
729 match flow_type {
730 Some(FlowType::Batching) => self.batching_engine.flush_flow(flow_id).await,
731 Some(FlowType::Streaming) => self.streaming_engine.flush_flow(flow_id).await,
732 None => {
733 warn!(
734 "Currently flow={flow_id} doesn't exist in flownode, ignore flush_flow request"
735 );
736 Ok(0)
737 }
738 }
739 }
740
741 async fn flow_exist(&self, flow_id: FlowId) -> Result<bool, Error> {
742 let flow_type = self.src_table2flow.read().await.get_flow_type(flow_id);
743 match flow_type {
745 Some(FlowType::Batching) => self.batching_engine.flow_exist(flow_id).await,
746 Some(FlowType::Streaming) => self.streaming_engine.flow_exist(flow_id).await,
747 None => Ok(false),
748 }
749 }
750
751 async fn list_flows(&self) -> Result<impl IntoIterator<Item = FlowId>, Error> {
752 let stream_flows = self.streaming_engine.list_flows().await?;
753 let batch_flows = self.batching_engine.list_flows().await?;
754
755 Ok(stream_flows.into_iter().chain(batch_flows))
756 }
757
758 async fn handle_flow_inserts(
759 &self,
760 request: api::v1::region::InsertRequests,
761 ) -> Result<(), Error> {
762 self.wait_for_all_flow_recover(request.requests.len())
763 .await?;
764 let mut to_stream_engine = Vec::with_capacity(request.requests.len());
766 let mut to_batch_engine = request.requests;
767
768 let mut batching_row_cnt = 0;
769 let mut streaming_row_cnt = 0;
770
771 {
772 let src_table2flow = self.src_table2flow.read().await;
774 to_batch_engine.retain(|req| {
775 let region_id = RegionId::from(req.region_id);
776 let table_id = region_id.table_id();
777 let is_in_stream = src_table2flow.in_stream(table_id);
778 let is_in_batch = src_table2flow.in_batch(table_id);
779 if is_in_stream {
780 streaming_row_cnt += req.rows.as_ref().map(|rs| rs.rows.len()).unwrap_or(0);
781 to_stream_engine.push(req.clone());
782 }
783 if is_in_batch {
784 batching_row_cnt += req.rows.as_ref().map(|rs| rs.rows.len()).unwrap_or(0);
785 return true;
786 }
787 if !is_in_batch && !is_in_stream {
788 warn!("Table {} is not any flow's source table", table_id)
790 }
791 false
792 });
793 }
796
797 METRIC_FLOW_ROWS
798 .with_label_values(&["in-streaming"])
799 .inc_by(streaming_row_cnt as u64);
800
801 METRIC_FLOW_ROWS
802 .with_label_values(&["in-batching"])
803 .inc_by(batching_row_cnt as u64);
804
805 let streaming_engine = self.streaming_engine.clone();
806 let stream_handler: JoinHandle<Result<(), Error>> =
807 common_runtime::spawn_global(async move {
808 streaming_engine
809 .handle_flow_inserts(api::v1::region::InsertRequests {
810 requests: to_stream_engine,
811 })
812 .await?;
813 Ok(())
814 });
815 self.batching_engine
816 .handle_flow_inserts(api::v1::region::InsertRequests {
817 requests: to_batch_engine,
818 })
819 .await?;
820 stream_handler.await.context(JoinTaskSnafu)??;
821
822 Ok(())
823 }
824
825 async fn handle_mark_window_dirty(
826 &self,
827 req: api::v1::flow::DirtyWindowRequests,
828 ) -> Result<(), Error> {
829 self.batching_engine.handle_mark_window_dirty(req).await
830 }
831}
832
833#[async_trait::async_trait]
834impl common_meta::node_manager::Flownode for FlowDualEngine {
835 async fn handle(&self, request: FlowRequest) -> MetaResult<FlowResponse> {
836 let query_ctx = request
837 .header
838 .and_then(|h| h.query_context)
839 .map(|ctx| ctx.into());
840 match request.body {
841 Some(flow_request::Body::Create(CreateRequest {
842 flow_id: Some(task_id),
843 source_table_ids,
844 sink_table_name: Some(sink_table_name),
845 create_if_not_exists,
846 expire_after,
847 eval_interval,
848 comment,
849 sql,
850 mut flow_options,
851 or_replace,
852 })) => {
853 let source_table_ids = source_table_ids.into_iter().map(|id| id.id).collect_vec();
854 let sink_table_name = [
855 sink_table_name.catalog_name,
856 sink_table_name.schema_name,
857 sink_table_name.table_name,
858 ];
859 let expire_after = expire_after.map(|e| e.value);
860
861 let eval_schedule = decode_internal_eval_schedule(&mut flow_options)
862 .map_err(to_meta_err(snafu::location!()))?;
863
864 let args = CreateFlowArgs {
865 flow_id: task_id.id as u64,
866 sink_table_name,
867 source_table_ids,
868 create_if_not_exists,
869 or_replace,
870 expire_after,
871 eval_interval: eval_interval.map(|e| e.seconds),
872 comment: Some(comment),
873 sql: sql.clone(),
874 flow_options,
875 query_ctx,
876 eval_schedule,
877 };
878 let ret = self
879 .create_flow(args)
880 .await
881 .map_err(BoxedError::new)
882 .with_context(|_| CreateFlowSnafu { sql: sql.clone() })
883 .map_err(to_meta_err(snafu::location!()))?;
884 METRIC_FLOW_TASK_COUNT.inc();
885 Ok(FlowResponse {
886 affected_flows: ret
887 .map(|id| greptime_proto::v1::FlowId { id: id as u32 })
888 .into_iter()
889 .collect_vec(),
890 ..Default::default()
891 })
892 }
893 Some(flow_request::Body::Drop(DropRequest {
894 flow_id: Some(flow_id),
895 })) => {
896 self.remove_flow(flow_id.id as u64)
897 .await
898 .map_err(to_meta_err(snafu::location!()))?;
899 METRIC_FLOW_TASK_COUNT.dec();
900 Ok(Default::default())
901 }
902 Some(flow_request::Body::Flush(FlushFlow {
903 flow_id: Some(flow_id),
904 })) => {
905 let row = self
906 .flush_flow(flow_id.id as u64)
907 .await
908 .map_err(to_meta_err(snafu::location!()))?;
909 Ok(FlowResponse {
910 affected_flows: vec![flow_id],
911 affected_rows: row as u64,
912 ..Default::default()
913 })
914 }
915 other => common_meta::error::InvalidFlowRequestBodySnafu { body: other }.fail(),
916 }
917 }
918
919 async fn handle_inserts(&self, request: InsertRequests) -> MetaResult<FlowResponse> {
920 FlowEngine::handle_flow_inserts(self, request)
921 .await
922 .map(|_| Default::default())
923 .map_err(to_meta_err(snafu::location!()))
924 }
925
926 async fn handle_mark_window_dirty(&self, req: DirtyWindowRequests) -> MetaResult<FlowResponse> {
927 self.batching_engine()
928 .handle_mark_dirty_time_window(req)
929 .await
930 .map(|_| FlowResponse::default())
931 .map_err(to_meta_err(snafu::location!()))
932 }
933}
934
935fn decode_internal_eval_schedule(
938 flow_options: &mut HashMap<String, String>,
939) -> Result<Option<FlowScheduleConfig>, Error> {
940 match flow_options.remove(INTERNAL_EVAL_SCHEDULE_KEY) {
941 Some(json) => serde_json::from_str::<FlowScheduleConfig>(&json)
942 .map(Some)
943 .map_err(|err| {
944 InternalSnafu {
945 reason: format!("Invalid internal eval schedule payload: {err}"),
946 }
947 .build()
948 }),
949 None => Ok(None),
950 }
951}
952
953fn to_meta_err(
955 location: snafu::Location,
956) -> impl FnOnce(crate::error::Error) -> common_meta::error::Error {
957 move |err: crate::error::Error| -> common_meta::error::Error {
958 match err {
959 crate::error::Error::FlowNotFound { id, .. } => {
960 common_meta::error::Error::FlowNotFound {
961 flow_name: format!("flow_id={id}"),
962 location,
963 }
964 }
965 _ => common_meta::error::Error::External {
966 location,
967 source: BoxedError::new(err),
968 },
969 }
970 }
971}
972
973impl FlowEngine for StreamingEngine {
974 async fn create_flow(&self, args: CreateFlowArgs) -> Result<Option<FlowId>, Error> {
975 self.create_flow_inner(args).await
976 }
977
978 async fn remove_flow(&self, flow_id: FlowId) -> Result<(), Error> {
979 self.remove_flow_inner(flow_id).await
980 }
981
982 async fn flush_flow(&self, flow_id: FlowId) -> Result<usize, Error> {
983 self.flush_flow_inner(flow_id).await
984 }
985
986 async fn flow_exist(&self, flow_id: FlowId) -> Result<bool, Error> {
987 self.flow_exist_inner(flow_id).await
988 }
989
990 async fn list_flows(&self) -> Result<impl IntoIterator<Item = FlowId>, Error> {
991 Ok(self
992 .flow_err_collectors
993 .read()
994 .await
995 .keys()
996 .cloned()
997 .collect::<Vec<_>>())
998 }
999
1000 async fn handle_flow_inserts(
1001 &self,
1002 request: api::v1::region::InsertRequests,
1003 ) -> Result<(), Error> {
1004 self.handle_inserts_inner(request).await
1005 }
1006
1007 async fn handle_mark_window_dirty(
1008 &self,
1009 _req: api::v1::flow::DirtyWindowRequests,
1010 ) -> Result<(), Error> {
1011 UnsupportedSnafu {
1012 reason: "handle_mark_window_dirty in streaming engine",
1013 }
1014 .fail()
1015 }
1016}
1017
1018#[derive(Debug, Clone)]
1020enum FetchFromRow {
1021 Idx(usize),
1022 Default(Value),
1023}
1024
1025impl FetchFromRow {
1026 fn fetch(&self, row: &repr::Row) -> Value {
1028 match self {
1029 FetchFromRow::Idx(idx) => row.get(*idx).unwrap().clone(),
1030 FetchFromRow::Default(v) => v.clone(),
1031 }
1032 }
1033}
1034
1035impl StreamingEngine {
1036 async fn handle_inserts_inner(
1037 &self,
1038 request: InsertRequests,
1039 ) -> std::result::Result<(), Error> {
1040 let _flush_lock = self.flush_lock.try_read();
1044 for write_request in request.requests {
1045 let region_id = write_request.region_id;
1046 let table_id = RegionId::from(region_id).table_id();
1047
1048 let (insert_schema, rows_proto) = write_request
1049 .rows
1050 .map(|r| (r.schema, r.rows))
1051 .unwrap_or_default();
1052
1053 let now = self.tick_manager.tick();
1055
1056 let (table_types, fetch_order) = {
1057 let ctx = self.node_context.read().await;
1058
1059 let table_schema = ctx.table_source.table_from_id(&table_id).await?;
1061 let default_vals = table_schema
1062 .default_values
1063 .iter()
1064 .zip(table_schema.relation_desc.typ().column_types.iter())
1065 .map(|(v, ty)| {
1066 v.as_ref().and_then(|v| {
1067 match v.create_default(ty.scalar_type(), ty.nullable()) {
1068 Ok(v) => Some(v),
1069 Err(err) => {
1070 common_telemetry::error!(err; "Failed to create default value");
1071 None
1072 }
1073 }
1074 })
1075 })
1076 .collect_vec();
1077
1078 let table_types = table_schema
1079 .relation_desc
1080 .typ()
1081 .column_types
1082 .clone()
1083 .into_iter()
1084 .map(|t| t.scalar_type)
1085 .collect_vec();
1086 let table_col_names = table_schema.relation_desc.names;
1087 let table_col_names = table_col_names
1088 .iter().enumerate()
1089 .map(|(idx,name)| match name {
1090 Some(name) => Ok(name.clone()),
1091 None => InternalSnafu {
1092 reason: format!("Expect column {idx} of table id={table_id} to have name in table schema, found None"),
1093 }
1094 .fail(),
1095 })
1096 .collect::<Result<Vec<_>, _>>()?;
1097 let name_to_col = HashMap::<_, _>::from_iter(
1098 insert_schema
1099 .iter()
1100 .enumerate()
1101 .map(|(i, name)| (&name.column_name, i)),
1102 );
1103
1104 let fetch_order: Vec<FetchFromRow> = table_col_names
1105 .iter()
1106 .zip(default_vals)
1107 .map(|(col_name, col_default_val)| {
1108 name_to_col
1109 .get(col_name)
1110 .copied()
1111 .map(FetchFromRow::Idx)
1112 .or_else(|| col_default_val.clone().map(FetchFromRow::Default))
1113 .with_context(|| UnexpectedSnafu {
1114 reason: format!(
1115 "Column not found: {}, default_value: {:?}",
1116 col_name, col_default_val
1117 ),
1118 })
1119 })
1120 .try_collect()?;
1121
1122 trace!("Reordering columns: {:?}", fetch_order);
1123 (table_types, fetch_order)
1124 };
1125
1126 let rows: Vec<DiffRow> = rows_proto
1128 .into_iter()
1129 .map(|r| {
1130 let r = repr::Row::from(r);
1131 let reordered = fetch_order.iter().map(|i| i.fetch(&r)).collect_vec();
1132 repr::Row::new(reordered)
1133 })
1134 .map(|r| (r, now, 1))
1135 .collect_vec();
1136 if let Err(err) = self
1137 .handle_write_request(region_id.into(), rows, &table_types)
1138 .await
1139 {
1140 let err = BoxedError::new(err);
1141 let flow_ids = self
1142 .node_context
1143 .read()
1144 .await
1145 .get_flow_ids(table_id)
1146 .into_iter()
1147 .flatten()
1148 .cloned()
1149 .collect_vec();
1150 let err = InsertIntoFlowSnafu {
1151 region_id,
1152 flow_ids,
1153 }
1154 .into_error(err);
1155 common_telemetry::error!(err; "Failed to handle write request");
1156 return Err(err);
1157 }
1158 }
1159 Ok(())
1160 }
1161}
1162
1163#[cfg(test)]
1164mod tests {
1165 use std::collections::HashMap;
1166
1167 use common_meta::ddl::create_flow::INTERNAL_EVAL_SCHEDULE_KEY;
1168
1169 use super::decode_internal_eval_schedule;
1170 use crate::error::Error;
1171
1172 #[test]
1173 fn test_malformed_internal_eval_schedule_json_is_error() {
1174 let mut flow_options = HashMap::new();
1175 flow_options.insert(
1176 INTERNAL_EVAL_SCHEDULE_KEY.to_string(),
1177 "not-json".to_string(),
1178 );
1179
1180 let err = decode_internal_eval_schedule(&mut flow_options).unwrap_err();
1181 assert!(matches!(
1182 err,
1183 Error::Internal { reason, .. }
1184 if reason.contains("Invalid internal eval schedule payload")
1185 ));
1186 assert!(!flow_options.contains_key(INTERNAL_EVAL_SCHEDULE_KEY));
1187 }
1188}