rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-05-31 12:20:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat!: Remove script crate and python feature (#5321)

Browse Source 
* feat: exclude script crate

* chore: simplify feature

* feat: remove the script crate

* chore: remove python feature and some comments

* chore: fix warning
This commit is contained in:
Yingwen
2025-01-08 20:11:53 +08:00
committed by GitHub
parent 7f307a4cac
commit c19a56c79f
73 changed files with 85 additions and 14961 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/frontend/Cargo.toml
View File

@@ -5,8 +5,6 @@ edition.workspace = true
license.workspace = true
[features]
default = ["python"]
python = ["dep:script"]
testing = []
[lints]
@@ -52,7 +50,6 @@ prometheus.workspace = true
prost.workspace = true
query.workspace = true
raft-engine.workspace = true
script = { workspace = true, features = ["python"], optional = true }
serde.workspace = true
servers.workspace = true
session.workspace = true

11
src/frontend/src/error.rs
View File

@@ -238,14 +238,6 @@ pub enum Error {
location: Location,
},
#[cfg(feature = "python")]
#[snafu(display("Failed to start script manager"))]
StartScriptManager {
#[snafu(implicit)]
location: Location,
source: script::error::Error,
},
#[snafu(display("Failed to insert value into table: {}", table_name))]
Insert {
table_name: String,
@@ -394,9 +386,6 @@ impl ErrorExt for Error {
}
Error::FindTableRoute { source, .. } => source.status_code(),
#[cfg(feature = "python")]
Error::StartScriptManager { source, .. } => source.status_code(),
Error::TableOperation { source, .. } => source.status_code(),
Error::InFlightWriteBytesExceeded { .. } => StatusCode::RateLimited,

8
src/frontend/src/instance.rs
View File

@@ -21,7 +21,6 @@ mod opentsdb;
mod otlp;
mod prom_store;
mod region_query;
mod script;
pub mod standalone;
use std::sync::Arc;
@@ -66,7 +65,7 @@ use servers::query_handler::grpc::GrpcQueryHandler;
use servers::query_handler::sql::SqlQueryHandler;
use servers::query_handler::{
InfluxdbLineProtocolHandler, LogQueryHandler, OpenTelemetryProtocolHandler,
OpentsdbProtocolHandler, PipelineHandler, PromStoreProtocolHandler, ScriptHandler,
OpentsdbProtocolHandler, PipelineHandler, PromStoreProtocolHandler,
};
use servers::server::ServerHandlers;
use session::context::QueryContextRef;
@@ -88,7 +87,6 @@ use crate::error::{
use crate::frontend::FrontendOptions;
use crate::heartbeat::HeartbeatTask;
use crate::limiter::LimiterRef;
use crate::script::ScriptExecutor;
#[async_trait]
pub trait FrontendInstance:
@@ -98,7 +96,6 @@ pub trait FrontendInstance:
+ InfluxdbLineProtocolHandler
+ PromStoreProtocolHandler
+ OpenTelemetryProtocolHandler
+ ScriptHandler
+ PrometheusHandler
+ PipelineHandler
+ LogQueryHandler
@@ -115,7 +112,6 @@ pub type FrontendInstanceRef = Arc<dyn FrontendInstance>;
pub struct Instance {
options: FrontendOptions,
catalog_manager: CatalogManagerRef,
script_executor: Arc<ScriptExecutor>,
pipeline_operator: Arc<PipelineOperator>,
statement_executor: Arc<StatementExecutor>,
query_engine: QueryEngineRef,
@@ -205,8 +201,6 @@ impl FrontendInstance for Instance {
heartbeat_task.start().await?;
}
self.script_executor.start(self)?;
if let Some(t) = self.export_metrics_task.as_ref() {
if t.send_by_handler {
let handler = ExportMetricHandler::new_handler(

6
src/frontend/src/instance/builder.rs
View File

@@ -44,7 +44,6 @@ use crate::heartbeat::HeartbeatTask;
use crate::instance::region_query::FrontendRegionQueryHandler;
use crate::instance::Instance;
use crate::limiter::Limiter;
use crate::script::ScriptExecutor;
/// The frontend [`Instance`] builder.
pub struct FrontendBuilder {
@@ -174,10 +173,6 @@ impl FrontendBuilder {
)
.query_engine();
let script_executor = Arc::new(
ScriptExecutor::new(self.catalog_manager.clone(), query_engine.clone()).await?,
);
let statement_executor = Arc::new(StatementExecutor::new(
self.catalog_manager.clone(),
query_engine.clone(),
@@ -208,7 +203,6 @@ impl FrontendBuilder {
Ok(Instance {
options: self.options,
catalog_manager: self.catalog_manager,
script_executor,
pipeline_operator,
statement_executor,
query_engine,

58
src/frontend/src/instance/script.rs
View File

@@ -1,58 +0,0 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::HashMap;
use async_trait::async_trait;
use common_query::Output;
use servers::error::Error;
use servers::interceptor::{ScriptInterceptor, ScriptInterceptorRef};
use servers::query_handler::ScriptHandler;
use session::context::QueryContextRef;
use crate::instance::Instance;
use crate::metrics;
#[async_trait]
impl ScriptHandler for Instance {
async fn insert_script(
&self,
query_ctx: QueryContextRef,
name: &str,
script: &str,
) -> servers::error::Result<()> {
let interceptor_ref = self.plugins.get::<ScriptInterceptorRef<Error>>();
interceptor_ref.pre_execute(name, query_ctx.clone())?;
let _timer = metrics::INSERT_SCRIPTS_ELAPSED.start_timer();
self.script_executor
.insert_script(query_ctx, name, script)
.await
}
async fn execute_script(
&self,
query_ctx: QueryContextRef,
name: &str,
params: HashMap<String, String>,
) -> servers::error::Result<Output> {
let interceptor_ref = self.plugins.get::<ScriptInterceptorRef<Error>>();
interceptor_ref.pre_execute(name, query_ctx.clone())?;
let _timer = metrics::EXECUTE_SCRIPT_ELAPSED.start_timer();
self.script_executor
.execute_script(query_ctx, name, params)
.await
}
}

1
src/frontend/src/lib.rs
View File

@@ -20,6 +20,5 @@ pub mod heartbeat;
pub mod instance;
pub(crate) mod limiter;
pub(crate) mod metrics;
mod script;
pub mod server;
pub mod service_config;

13
src/frontend/src/metrics.rs
View File

@@ -29,19 +29,6 @@ lazy_static! {
pub static ref GRPC_HANDLE_PROMQL_ELAPSED: Histogram = GRPC_HANDLE_QUERY_ELAPSED
.with_label_values(&["promql"]);
/// Timer of handling scripts in the script handler.
pub static ref HANDLE_SCRIPT_ELAPSED: HistogramVec = register_histogram_vec!(
"greptime_frontend_handle_script_elapsed",
"Elapsed time of handling scripts in the script handler",
&["type"],
vec![0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0, 60.0, 300.0]
)
.unwrap();
pub static ref INSERT_SCRIPTS_ELAPSED: Histogram = HANDLE_SCRIPT_ELAPSED
.with_label_values(&["insert"]);
pub static ref EXECUTE_SCRIPT_ELAPSED: Histogram = HANDLE_SCRIPT_ELAPSED
.with_label_values(&["execute"]);
/// The number of OpenTelemetry metrics send by frontend node.
pub static ref OTLP_METRICS_ROWS: IntCounter = register_int_counter!(
"greptime_frontend_otlp_metrics_rows",

294
src/frontend/src/script.rs
View File

@@ -1,294 +0,0 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::HashMap;
use catalog::CatalogManagerRef;
use common_query::Output;
use query::QueryEngineRef;
use session::context::QueryContextRef;
use crate::error::Result;
use crate::instance::Instance;
#[cfg(not(feature = "python"))]
mod dummy {
use super::*;
pub struct ScriptExecutor;
impl ScriptExecutor {
pub async fn new(
_catalog_manager: CatalogManagerRef,
_query_engine: QueryEngineRef,
) -> Result<Self> {
Ok(Self {})
}
pub fn start(&self, _instance: &Instance) -> Result<()> {
Ok(())
}
pub async fn insert_script(
&self,
_query_ctx: QueryContextRef,
_name: &str,
_script: &str,
) -> servers::error::Result<()> {
servers::error::NotSupportedSnafu { feat: "script" }.fail()
}
pub async fn execute_script(
&self,
_query_ctx: QueryContextRef,
_name: &str,
_params: HashMap<String, String>,
) -> servers::error::Result<Output> {
servers::error::NotSupportedSnafu { feat: "script" }.fail()
}
}
}
#[cfg(feature = "python")]
mod python {
use std::sync::Arc;
use api::v1::ddl_request::Expr;
use api::v1::greptime_request::Request;
use api::v1::DdlRequest;
use arc_swap::ArcSwap;
use catalog::RegisterSystemTableRequest;
use common_error::ext::{BoxedError, ErrorExt};
use common_telemetry::{error, info};
use script::manager::ScriptManager;
use servers::query_handler::grpc::GrpcQueryHandler;
use session::context::QueryContext;
use snafu::{OptionExt, ResultExt};
use table::table_name::TableName;
use super::*;
use crate::error::{CatalogSnafu, Error, TableNotFoundSnafu};
type FrontendGrpcQueryHandlerRef = Arc<dyn GrpcQueryHandler<Error = Error> + Send + Sync>;
/// A placeholder for the real gRPC handler.
/// It is temporary and will be replaced soon.
struct DummyHandler;
impl DummyHandler {
fn arc() -> Arc<Self> {
Arc::new(Self {})
}
}
#[async_trait::async_trait]
impl GrpcQueryHandler for DummyHandler {
type Error = Error;
async fn do_query(
&self,
_query: Request,
_ctx: QueryContextRef,
) -> std::result::Result<Output, Self::Error> {
unreachable!();
}
}
pub struct ScriptExecutor {
script_manager: ScriptManager<Error>,
grpc_handler: ArcSwap<FrontendGrpcQueryHandlerRef>,
catalog_manager: CatalogManagerRef,
}
impl ScriptExecutor {
pub async fn new(
catalog_manager: CatalogManagerRef,
query_engine: QueryEngineRef,
) -> Result<Self> {
let grpc_handler = DummyHandler::arc();
Ok(Self {
grpc_handler: ArcSwap::new(Arc::new(grpc_handler.clone() as _)),
script_manager: ScriptManager::new(grpc_handler as _, query_engine)
.await
.context(crate::error::StartScriptManagerSnafu)?,
catalog_manager,
})
}
pub fn start(&self, instance: &Instance) -> Result<()> {
let handler = Arc::new(instance.clone());
self.grpc_handler.store(Arc::new(handler.clone() as _));
self.script_manager
.start(handler)
.context(crate::error::StartScriptManagerSnafu)?;
Ok(())
}
/// Create scripts table for the specific catalog if it's not exists.
/// The function is idempotent and safe to be called more than once for the same catalog
async fn create_scripts_table_if_need(&self, catalog: &str) -> Result<()> {
let scripts_table = self.script_manager.get_scripts_table(catalog);
if scripts_table.is_some() {
return Ok(());
}
let RegisterSystemTableRequest {
create_table_expr: expr,
open_hook,
} = self.script_manager.create_table_request(catalog);
if let Some(table) = self
.catalog_manager
.table(
&expr.catalog_name,
&expr.schema_name,
&expr.table_name,
None,
)
.await
.context(CatalogSnafu)?
{
if let Some(open_hook) = open_hook {
(open_hook)(table.clone()).await.context(CatalogSnafu)?;
}
self.script_manager.insert_scripts_table(catalog, table);
return Ok(());
}
let table_name =
TableName::new(&expr.catalog_name, &expr.schema_name, &expr.table_name);
let _ = self
.grpc_handler
.load()
.do_query(
Request::Ddl(DdlRequest {
expr: Some(Expr::CreateTable(expr)),
}),
QueryContext::arc(),
)
.await?;
let table = self
.catalog_manager
.table(
&table_name.catalog_name,
&table_name.schema_name,
&table_name.table_name,
None,
)
.await
.context(CatalogSnafu)?
.with_context(|| TableNotFoundSnafu {
table_name: table_name.to_string(),
})?;
if let Some(open_hook) = open_hook {
(open_hook)(table.clone()).await.context(CatalogSnafu)?;
}
info!(
"Created scripts table {}.",
table.table_info().full_table_name()
);
self.script_manager.insert_scripts_table(catalog, table);
Ok(())
}
pub async fn insert_script(
&self,
query_ctx: QueryContextRef,
name: &str,
script: &str,
) -> servers::error::Result<()> {
self.create_scripts_table_if_need(query_ctx.current_catalog())
.await
.map_err(|e| {
if e.status_code().should_log_error() {
error!(e; "Failed to create scripts table");
}
servers::error::InternalSnafu {
err_msg: e.to_string(),
}
.build()
})?;
let _s = self
.script_manager
.insert_and_compile(
query_ctx.current_catalog(),
&query_ctx.current_schema(),
name,
script,
)
.await
.map_err(|e| {
if e.status_code().should_log_error() {
error!(e; "Failed to insert script");
}
BoxedError::new(e)
})
.context(servers::error::InsertScriptSnafu { name })?;
Ok(())
}
pub async fn execute_script(
&self,
query_ctx: QueryContextRef,
name: &str,
params: HashMap<String, String>,
) -> servers::error::Result<Output> {
self.create_scripts_table_if_need(query_ctx.current_catalog())
.await
.map_err(|e| {
error!(e; "Failed to create scripts table");
servers::error::InternalSnafu {
err_msg: e.to_string(),
}
.build()
})?;
self.script_manager
.execute(
query_ctx.current_catalog(),
&query_ctx.current_schema(),
name,
params,
)
.await
.map_err(|e| {
if e.status_code().should_log_error() {
error!(e; "Failed to execute script");
}
BoxedError::new(e)
})
.context(servers::error::ExecuteScriptSnafu { name })
}
}
}
#[cfg(not(feature = "python"))]
pub use self::dummy::*;
#[cfg(feature = "python")]
pub use self::python::*;

6
src/frontend/src/server.rs
View File

@@ -78,10 +78,8 @@ where
}
pub fn http_server_builder(&self, opts: &FrontendOptions) -> HttpServerBuilder {
let mut builder = HttpServerBuilder::new(opts.http.clone()).with_sql_handler(
ServerSqlQueryHandlerAdapter::arc(self.instance.clone()),
Some(self.instance.clone()),
);
let mut builder = HttpServerBuilder::new(opts.http.clone())
.with_sql_handler(ServerSqlQueryHandlerAdapter::arc(self.instance.clone()));
let validator = self.plugins.get::<LogValidatorRef>();
let ingest_interceptor = self.plugins.get::<LogIngestInterceptorRef<ServerError>>();