feat(flow): mirror insert req to flow node (#3858)

* feat: mirror insert req to flow node

* refactor: group_requests_by_peer

* chore: rename `nodes` to `flows` to be more apt

* docs: add TODO

* refactor: split flow&data node grouping to two func

* refactor: mirror_flow_node_request

* chore: add some TODOs

* refactor: use Option in value

* feat: skip non-src table quickly

* docs: add TODO for  `Peer.address`

* fix: dedup

src/common/meta/src/key/flow.rs

@@ -26,10 +26,10 @@ use snafu::{ensure, OptionExt};
 use self::flow_info::FlowInfoValue;
 use crate::ensure_values;
 use crate::error::{self, Result};
-use crate::key::flow::flow_info::FlowInfoManager;
-use crate::key::flow::flow_name::FlowNameManager;
-use crate::key::flow::flownode_flow::FlownodeFlowManager;
-use crate::key::flow::table_flow::TableFlowManager;
+use crate::key::flow::flow_info::{FlowInfoManager, FlowInfoManagerRef};
+use crate::key::flow::flow_name::{FlowNameManager, FlowNameManagerRef};
+use crate::key::flow::flownode_flow::{FlownodeFlowManager, FlownodeFlowManagerRef};
+pub use crate::key::flow::table_flow::{TableFlowManager, TableFlowManagerRef};
 use crate::key::txn_helper::TxnOpGetResponseSet;
 use crate::key::{FlowId, MetaKey};
 use crate::kv_backend::txn::Txn;
@@ -306,7 +306,7 @@ mod tests {
         for table_id in [1024, 1025, 1026] {
             let nodes = flow_metadata_manager
                 .table_flow_manager()
-                .nodes(table_id)
+                .flows(table_id)
                 .try_collect::<Vec<_>>()
                 .await
                 .unwrap();

src/common/meta/src/key/flow/flow_info.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use std::collections::{BTreeMap, HashMap};
+use std::sync::Arc;
 
 use lazy_static::lazy_static;
 use regex::Regex;
@@ -143,6 +144,8 @@ impl FlowInfoValue {
     }
 }
 
+pub type FlowInfoManagerRef = Arc<FlowInfoManager>;
+
 /// The manager of [FlowInfoKey].
 pub struct FlowInfoManager {
     kv_backend: KvBackendRef,

src/common/meta/src/key/flow/flow_name.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::Arc;
+
 use api::v1::flow::flow_server::Flow;
 use lazy_static::lazy_static;
 use regex::Regex;
@@ -141,6 +143,8 @@ impl FlowNameValue {
     }
 }
 
+pub type FlowNameManagerRef = Arc<FlowNameManager>;
+
 /// The manager of [FlowNameKey].
 pub struct FlowNameManager {
     kv_backend: KvBackendRef,

src/common/meta/src/key/flow/flownode_flow.rs

@@ -156,6 +156,8 @@ impl<'a> MetaKey<'a, FlownodeFlowKeyInner> for FlownodeFlowKeyInner {
     }
 }
 
+pub type FlownodeFlowManagerRef = Arc<FlownodeFlowManager>;
+
 /// The manager of [FlownodeFlowKey].
 pub struct FlownodeFlowManager {
     kv_backend: KvBackendRef,

src/common/meta/src/key/flow/table_flow.rs

@@ -176,6 +176,8 @@ pub fn table_flow_decoder(kv: KeyValue) -> Result<TableFlowKey> {
     TableFlowKey::from_bytes(&kv.key)
 }
 
+pub type TableFlowManagerRef = Arc<TableFlowManager>;
+
 /// The manager of [TableFlowKey].
 pub struct TableFlowManager {
     kv_backend: KvBackendRef,
@@ -188,7 +190,9 @@ impl TableFlowManager {
     }
 
     /// Retrieves all [TableFlowKey]s of the specified `table_id`.
-    pub fn nodes(&self, table_id: TableId) -> BoxStream<'static, Result<TableFlowKey>> {
+    ///
+    /// TODO(discord9): add cache for it since range request does not support cache.
+    pub fn flows(&self, table_id: TableId) -> BoxStream<'static, Result<TableFlowKey>> {
         let start_key = TableFlowKey::range_start_key(table_id);
         let req = RangeRequest::new().with_prefix(start_key);
         let stream = PaginationStream::new(

src/common/meta/src/node_manager.rs

@@ -15,8 +15,8 @@
 use std::sync::Arc;
 
 use api::region::RegionResponse;
-use api::v1::flow::{FlowRequest, FlowResponse, InsertRequest};
-use api::v1::region::{QueryRequest, RegionRequest};
+use api::v1::flow::{FlowRequest, FlowResponse};
+use api::v1::region::{InsertRequests, QueryRequest, RegionRequest};
 pub use common_base::AffectedRows;
 use common_recordbatch::SendableRecordBatchStream;
 
@@ -40,7 +40,7 @@ pub type DatanodeRef = Arc<dyn Datanode>;
 pub trait Flownode: Send + Sync {
     async fn handle(&self, request: FlowRequest) -> Result<FlowResponse>;
 
-    async fn handle_insert(&self, request: InsertRequest) -> Result<FlowResponse>;
+    async fn handle_inserts(&self, request: InsertRequests) -> Result<FlowResponse>;
 }
 
 pub type FlownodeRef = Arc<dyn Flownode>;