feat: pull meta config in frontend during startup (#7727)

* chore: init impl for meta config

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: use ser string for config

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: add metasrv config ser trait

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: use trait

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: minor fix

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: change to vec options

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: rename mod name

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: add comment

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: rename and add comment

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: tmp save

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: add log and error in server

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: introduce deserializer

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: add blank line

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: impl config service

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: fmt

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: update proto commit

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

---------

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

src/plugins/Cargo.toml

@@ -15,11 +15,13 @@ cli.workspace = true
 common-base.workspace = true
 common-error.workspace = true
 common-meta.workspace = true
+common-options.workspace = true
 datanode.workspace = true
 flow.workspace = true
 frontend.workspace = true
 meta-client.workspace = true
 meta-srv.workspace = true
 serde.workspace = true
+serde_json.workspace = true
 snafu.workspace = true
 standalone.workspace = true

src/plugins/src/frontend.rs

@@ -37,6 +37,19 @@ pub async fn setup_frontend_plugins(
     Ok(())
 }
 
+/// Setup dynamic plugins based on the meta config in frontend.
+/// This is called after the `setup_frontend_plugins` because the meta client needs to be created first.
+///
+/// For those configs/plugins which are corresponding with the metasrv's config,
+/// we pull from metasrv first, then create/override the current config/plugin.
+/// Note: make sure the override works as expected.
+pub async fn setup_frontend_dynamic_plugins(
+    _meta_config: Vec<PluginOptions>,
+    _plugins: &mut Plugins,
+) -> Result<()> {
+    Ok(())
+}
+
 pub async fn start_frontend_plugins(_plugins: Plugins) -> Result<()> {
     Ok(())
 }

src/plugins/src/lib.rs

@@ -17,7 +17,7 @@ pub mod datanode;
 pub mod flownode;
 pub mod frontend;
 mod meta_srv;
-mod options;
+pub mod options;
 pub mod standalone;
 
 pub use cli::SubCommand;

src/plugins/src/options.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use common_options::plugin_options::{PluginOptionsDeserializer, PluginOptionsSerializer};
 use serde::{Deserialize, Serialize};
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
@@ -21,3 +22,26 @@ pub struct DummyOptions;
 pub enum PluginOptions {
     Dummy(DummyOptions),
 }
+
+pub struct PluginOptionsList(pub Vec<PluginOptions>);
+
+impl PluginOptionsSerializer for PluginOptionsList {
+    fn serialize(&self) -> Result<String, serde_json::Error> {
+        if self.0.is_empty() {
+            Ok(String::new())
+        } else {
+            serde_json::to_string(&self.0)
+        }
+    }
+}
+pub struct PluginOptionsDeserializerImpl;
+
+impl PluginOptionsDeserializer<Vec<PluginOptions>> for PluginOptionsDeserializerImpl {
+    fn deserialize(&self, payload: &str) -> Result<Vec<PluginOptions>, serde_json::Error> {
+        if payload.is_empty() {
+            Ok(vec![])
+        } else {
+            serde_json::from_str(payload)
+        }
+    }
+}