rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-05-24 17:00:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: execute python script in distributed mode (#1264)

Browse Source 
* feat: execute python script in distributed mode

* fix: rebase develop
This commit is contained in:
LFC
2023-04-02 20:36:48 +08:00
committed by GitHub
parent d2542552d3
commit 48c2841e4d
27 changed files with 343 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/datanode/Cargo.toml
View File

@@ -4,10 +4,6 @@ version.workspace = true
edition.workspace = true
license.workspace = true
[features]
default = ["python"]
python = ["dep:script"]
[dependencies]
async-compat = "0.2"
async-stream.workspace = true
@@ -49,7 +45,6 @@ pin-project = "1.0"
prost.workspace = true
query = { path = "../query" }
regex = "1.6"
script = { path = "../script", features = ["python"], optional = true }
serde = "1.0"
serde_json = "1.0"
servers = { path = "../servers" }

7
src/datanode/src/error.rs
View File

@@ -314,12 +314,6 @@ pub enum Error {
source: sql::error::Error,
},
#[snafu(display("Failed to start script manager, source: {}", source))]
StartScriptManager {
#[snafu(backtrace)]
source: script::error::Error,
},
#[snafu(display(
"Failed to parse string to timestamp, string: {}, source: {}",
raw,
@@ -601,7 +595,6 @@ impl ErrorExt for Error {
| WriteObject { .. }
| ListObjects { .. } => StatusCode::StorageUnavailable,
OpenLogStore { source } => source.status_code(),
StartScriptManager { source } => source.status_code(),
OpenStorageEngine { source } => source.status_code(),
RuntimeResource { .. } => StatusCode::RuntimeResourcesExhausted,
MetaClientInit { source, .. } => source.status_code(),

6
src/datanode/src/instance.rs
View File

@@ -58,11 +58,9 @@ use crate::error::{
NewCatalogSnafu, OpenLogStoreSnafu, RecoverProcedureSnafu, Result, ShutdownInstanceSnafu,
};
use crate::heartbeat::HeartbeatTask;
use crate::script::ScriptExecutor;
use crate::sql::{SqlHandler, SqlRequest};
mod grpc;
mod script;
pub mod sql;
pub(crate) type DefaultEngine = MitoEngine<EngineImpl<RaftEngineLogStore>>;
@@ -72,7 +70,6 @@ pub struct Instance {
pub(crate) query_engine: QueryEngineRef,
pub(crate) sql_handler: SqlHandler,
pub(crate) catalog_manager: CatalogManagerRef,
pub(crate) script_executor: ScriptExecutor,
pub(crate) table_id_provider: Option<TableIdProviderRef>,
pub(crate) heartbeat_task: Option<HeartbeatTask>,
procedure_manager: Option<ProcedureManagerRef>,
@@ -170,8 +167,6 @@ impl Instance {
let factory = QueryEngineFactory::new(catalog_manager.clone());
let query_engine = factory.query_engine();
let script_executor =
ScriptExecutor::new(catalog_manager.clone(), query_engine.clone()).await?;
let heartbeat_task = match opts.mode {
Mode::Standalone => None,
@@ -205,7 +200,6 @@ impl Instance {
procedure_manager.clone(),
),
catalog_manager,
script_executor,
heartbeat_task,
table_id_provider,
procedure_manager,

50
src/datanode/src/instance/script.rs
View File

@@ -1,50 +0,0 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::HashMap;
use async_trait::async_trait;
use common_query::Output;
use common_telemetry::timer;
use servers::query_handler::ScriptHandler;
use crate::instance::Instance;
use crate::metrics;
#[async_trait]
impl ScriptHandler for Instance {
async fn insert_script(
&self,
schema: &str,
name: &str,
script: &str,
) -> servers::error::Result<()> {
let _timer = timer!(metrics::METRIC_HANDLE_SCRIPTS_ELAPSED);
self.script_executor
.insert_script(schema, name, script)
.await
}
async fn execute_script(
&self,
schema: &str,
name: &str,
params: HashMap<String, String>,
) -> servers::error::Result<Output> {
let _timer = timer!(metrics::METRIC_RUN_SCRIPT_ELAPSED);
self.script_executor
.execute_script(schema, name, params)
.await
}
}

1
src/datanode/src/lib.rs
View File

@@ -21,7 +21,6 @@ mod heartbeat;
pub mod instance;
pub mod metrics;
mod mock;
mod script;
pub mod server;
pub mod sql;
#[cfg(test)]

2
src/datanode/src/metrics.rs
View File

@@ -15,6 +15,4 @@
//! datanode metrics
pub const METRIC_HANDLE_SQL_ELAPSED: &str = "datanode.handle_sql_elapsed";
pub const METRIC_HANDLE_SCRIPTS_ELAPSED: &str = "datanode.handle_scripts_elapsed";
pub const METRIC_RUN_SCRIPT_ELAPSED: &str = "datanode.run_script_elapsed";
pub const METRIC_HANDLE_PROMQL_ELAPSED: &str = "datanode.handle_promql_elapsed";

122
src/datanode/src/script.rs
View File

@@ -1,122 +0,0 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::HashMap;
use catalog::CatalogManagerRef;
use common_query::Output;
use query::QueryEngineRef;
use crate::error::Result;
#[cfg(not(feature = "python"))]
mod dummy {
use super::*;
pub struct ScriptExecutor;
impl ScriptExecutor {
pub async fn new(
_catalog_manager: CatalogManagerRef,
_query_engine: QueryEngineRef,
) -> Result<Self> {
Ok(Self {})
}
pub async fn insert_script(
&self,
_schema: &str,
_name: &str,
_script: &str,
) -> servers::error::Result<()> {
servers::error::NotSupportedSnafu { feat: "script" }.fail()
}
pub async fn execute_script(
&self,
_schema: &str,
_name: &str,
_params: HashMap<String, String>,
) -> servers::error::Result<Output> {
servers::error::NotSupportedSnafu { feat: "script" }.fail()
}
}
}
#[cfg(feature = "python")]
mod python {
use common_error::prelude::BoxedError;
use common_telemetry::logging::error;
use script::manager::ScriptManager;
use snafu::ResultExt;
use super::*;
pub struct ScriptExecutor {
script_manager: ScriptManager,
}
impl ScriptExecutor {
pub async fn new(
catalog_manager: CatalogManagerRef,
query_engine: QueryEngineRef,
) -> Result<Self> {
Ok(Self {
script_manager: ScriptManager::new(catalog_manager, query_engine)
.await
.context(crate::error::StartScriptManagerSnafu)?,
})
}
pub async fn insert_script(
&self,
schema: &str,
name: &str,
script: &str,
) -> servers::error::Result<()> {
let _s = self
.script_manager
.insert_and_compile(schema, name, script)
.await
.map_err(|e| {
error!(e; "Instance failed to insert script");
BoxedError::new(e)
})
.context(servers::error::InsertScriptSnafu { name })?;
Ok(())
}
pub async fn execute_script(
&self,
schema: &str,
name: &str,
params: HashMap<String, String>,
) -> servers::error::Result<Output> {
self.script_manager
.execute(schema, name, params)
.await
.map_err(|e| {
error!(e; "Instance failed to execute script");
BoxedError::new(e)
})
.context(servers::error::ExecuteScriptSnafu { name })
}
}
}
#[cfg(not(feature = "python"))]
pub use self::dummy::*;
#[cfg(feature = "python")]
pub use self::python::*;