mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-08-18 12:08:22 +00:00
perf: reduce cold workspace compile time (#8801)
* refactor: remove datanode and meta-srv dep from frontend Signed-off-by: shuiyisong <xixing.sys@gmail.com> * refactor: use on-device protoc if possible Signed-off-by: shuiyisong <xixing.sys@gmail.com> * refactor: remove unused dep Signed-off-by: shuiyisong <xixing.sys@gmail.com> * fix: CR issue Signed-off-by: shuiyisong <xixing.sys@gmail.com> * fix: version and docs Signed-off-by: shuiyisong <xixing.sys@gmail.com> * chore: update logs Co-authored-by: fys <40801205+fengys1996@users.noreply.github.com> --------- Signed-off-by: shuiyisong <xixing.sys@gmail.com> Co-authored-by: fys <40801205+fengys1996@users.noreply.github.com>
This commit is contained in:
shuiyisong
GitHub
@@ -46,8 +46,8 @@ The workspace is layered; dependencies point downward only.
|
||||
`file-engine`) implement it; `datanode` drives engines **through the trait**,
|
||||
not through engine internals.
|
||||
- `frontend` reaches storage through `operator` / `query` / `catalog`, not by
|
||||
depending on `datanode` internals. Standalone mode is the one bridge, via a
|
||||
`RegionServer` wrapper (`src/frontend/src/instance/standalone.rs`).
|
||||
depending on `datanode` internals. Standalone mode is the one bridge, via the
|
||||
`RegionServer` adapter in `src/standalone/src/datanode_manager.rs`.
|
||||
- Do not introduce circular dependencies. New deps go through
|
||||
`[workspace.dependencies]` in the root `Cargo.toml`, not per-crate version
|
||||
literals.
|
||||
|
||||
Generated
+102
-36
@@ -989,15 +989,6 @@ version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
|
||||
|
||||
[[package]]
|
||||
name = "autotools"
|
||||
version = "0.2.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef941527c41b0fc0dd48511a8154cd5fc7e29200a0ff8b7203c5d777dbc795cf"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-lc-rs"
|
||||
version = "1.17.0"
|
||||
@@ -2917,7 +2908,6 @@ dependencies = [
|
||||
name = "common-test-util"
|
||||
version = "1.3.0"
|
||||
dependencies = [
|
||||
"client",
|
||||
"common-grpc",
|
||||
"common-query",
|
||||
"common-recordbatch",
|
||||
@@ -5274,7 +5264,6 @@ dependencies = [
|
||||
"common-datasource",
|
||||
"common-error",
|
||||
"common-macro",
|
||||
"common-procedure",
|
||||
"common-query",
|
||||
"common-recordbatch",
|
||||
"common-telemetry",
|
||||
@@ -5587,7 +5576,6 @@ dependencies = [
|
||||
"dashmap",
|
||||
"datafusion",
|
||||
"datafusion-expr",
|
||||
"datanode",
|
||||
"datatypes",
|
||||
"futures",
|
||||
"hostname 0.4.1",
|
||||
@@ -5597,7 +5585,6 @@ dependencies = [
|
||||
"lazy_static",
|
||||
"log-query",
|
||||
"meta-client",
|
||||
"meta-srv",
|
||||
"num_cpus",
|
||||
"opentelemetry-proto",
|
||||
"opentelemetry-semantic-conventions 0.31.0",
|
||||
@@ -8013,7 +8000,8 @@ dependencies = [
|
||||
"pin-project",
|
||||
"prometheus 0.14.0",
|
||||
"protobuf 2.28.0",
|
||||
"protobuf-build",
|
||||
"protoc-bin-vendored",
|
||||
"protoc-rust",
|
||||
"raft-engine",
|
||||
"rand 0.9.4",
|
||||
"rskafka",
|
||||
@@ -11379,19 +11367,6 @@ dependencies = [
|
||||
"thiserror 1.0.69",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "protobuf-build"
|
||||
version = "0.15.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c852d9625b912c3e50480cdc701f60f49890b5d7ad46198dd583600f15e7c6ec"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
"protobuf 2.28.0",
|
||||
"protobuf-codegen",
|
||||
"protobuf-src",
|
||||
"regex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "protobuf-codegen"
|
||||
version = "2.28.0"
|
||||
@@ -11401,15 +11376,6 @@ dependencies = [
|
||||
"protobuf 2.28.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "protobuf-src"
|
||||
version = "1.1.0+21.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c7ac8852baeb3cc6fb83b93646fb93c0ffe5d14bf138c945ceb4b9948ee0e3c1"
|
||||
dependencies = [
|
||||
"autotools",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "protobuf-support"
|
||||
version = "3.7.2"
|
||||
@@ -11419,6 +11385,92 @@ dependencies = [
|
||||
"thiserror 1.0.69",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "protoc"
|
||||
version = "2.28.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a0218039c514f9e14a5060742ecd50427f8ac4f85a6dc58f2ddb806e318c55ee"
|
||||
dependencies = [
|
||||
"log",
|
||||
"which",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "protoc-bin-vendored"
|
||||
version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d1c381df33c98266b5f08186583660090a4ffa0889e76c7e9a5e175f645a67fa"
|
||||
dependencies = [
|
||||
"protoc-bin-vendored-linux-aarch_64",
|
||||
"protoc-bin-vendored-linux-ppcle_64",
|
||||
"protoc-bin-vendored-linux-s390_64",
|
||||
"protoc-bin-vendored-linux-x86_32",
|
||||
"protoc-bin-vendored-linux-x86_64",
|
||||
"protoc-bin-vendored-macos-aarch_64",
|
||||
"protoc-bin-vendored-macos-x86_64",
|
||||
"protoc-bin-vendored-win32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "protoc-bin-vendored-linux-aarch_64"
|
||||
version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c350df4d49b5b9e3ca79f7e646fde2377b199e13cfa87320308397e1f37e1a4c"
|
||||
|
||||
[[package]]
|
||||
name = "protoc-bin-vendored-linux-ppcle_64"
|
||||
version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a55a63e6c7244f19b5c6393f025017eb5d793fd5467823a099740a7a4222440c"
|
||||
|
||||
[[package]]
|
||||
name = "protoc-bin-vendored-linux-s390_64"
|
||||
version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1dba5565db4288e935d5330a07c264a4ee8e4a5b4a4e6f4e83fad824cc32f3b0"
|
||||
|
||||
[[package]]
|
||||
name = "protoc-bin-vendored-linux-x86_32"
|
||||
version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8854774b24ee28b7868cd71dccaae8e02a2365e67a4a87a6cd11ee6cdbdf9cf5"
|
||||
|
||||
[[package]]
|
||||
name = "protoc-bin-vendored-linux-x86_64"
|
||||
version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b38b07546580df720fa464ce124c4b03630a6fb83e05c336fea2a241df7e5d78"
|
||||
|
||||
[[package]]
|
||||
name = "protoc-bin-vendored-macos-aarch_64"
|
||||
version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "89278a9926ce312e51f1d999fee8825d324d603213344a9a706daa009f1d8092"
|
||||
|
||||
[[package]]
|
||||
name = "protoc-bin-vendored-macos-x86_64"
|
||||
version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81745feda7ccfb9471d7a4de888f0652e806d5795b61480605d4943176299756"
|
||||
|
||||
[[package]]
|
||||
name = "protoc-bin-vendored-win32"
|
||||
version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3"
|
||||
|
||||
[[package]]
|
||||
name = "protoc-rust"
|
||||
version = "2.28.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22f8a182bb17c485f20bdc4274a8c39000a61024cfe461c799b50fec77267838"
|
||||
dependencies = [
|
||||
"protobuf 2.28.0",
|
||||
"protobuf-codegen",
|
||||
"protoc",
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "psl"
|
||||
version = "2.1.123"
|
||||
@@ -14066,6 +14118,7 @@ dependencies = [
|
||||
name = "standalone"
|
||||
version = "1.3.0"
|
||||
dependencies = [
|
||||
"api",
|
||||
"async-trait",
|
||||
"catalog",
|
||||
"client",
|
||||
@@ -14079,6 +14132,7 @@ dependencies = [
|
||||
"common-options",
|
||||
"common-procedure",
|
||||
"common-query",
|
||||
"common-recordbatch",
|
||||
"common-stat",
|
||||
"common-telemetry",
|
||||
"common-time",
|
||||
@@ -16360,6 +16414,18 @@ dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "which"
|
||||
version = "4.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
|
||||
dependencies = [
|
||||
"either",
|
||||
"home",
|
||||
"once_cell",
|
||||
"rustix 0.38.44",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "whoami"
|
||||
version = "1.6.0"
|
||||
|
||||
@@ -202,6 +202,8 @@ prometheus = { version = "0.14", features = ["process"] }
|
||||
promql-parser = { version = "0.10.0", features = ["ser"] }
|
||||
prost = { version = "0.14", features = ["no-recursion-limit"] }
|
||||
prost-types = "0.14"
|
||||
protoc-bin-vendored = "3.2"
|
||||
protoc-rust = "2.28"
|
||||
raft-engine = { version = "0.4.1", default-features = false }
|
||||
rand = "0.9"
|
||||
ratelimit = "0.10"
|
||||
|
||||
@@ -60,7 +60,6 @@ use flow::{
|
||||
FrontendInvoker, GrpcQueryHandlerWithBoxedError,
|
||||
};
|
||||
use frontend::frontend::Frontend;
|
||||
use frontend::instance::StandaloneDatanodeManager;
|
||||
use frontend::instance::builder::FrontendBuilder;
|
||||
use frontend::server::Services;
|
||||
use meta_srv::metasrv::{FLOW_ID_SEQ, TABLE_ID_SEQ};
|
||||
@@ -72,7 +71,10 @@ use plugins::standalone::context::DdlManagerConfigureContext;
|
||||
use servers::tls::{TlsMode, TlsOption, merge_tls_option};
|
||||
use snafu::{OptionExt, ResultExt};
|
||||
use standalone::options::StandaloneOptions;
|
||||
use standalone::{StandaloneInformationExtension, StandaloneRepartitionProcedureFactory};
|
||||
use standalone::{
|
||||
StandaloneDatanodeManager, StandaloneInformationExtension,
|
||||
StandaloneRepartitionProcedureFactory,
|
||||
};
|
||||
use tracing_appender::non_blocking::WorkerGuard;
|
||||
|
||||
use crate::error::{OtherSnafu, Result, StartFlownodeSnafu};
|
||||
|
||||
@@ -9,11 +9,13 @@ common-base.workspace = true
|
||||
common-runtime.workspace = true
|
||||
common-telemetry.workspace = true
|
||||
lazy_static.workspace = true
|
||||
nix.workspace = true
|
||||
num_cpus.workspace = true
|
||||
prometheus.workspace = true
|
||||
sysinfo.workspace = true
|
||||
tokio.workspace = true
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
nix.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -8,7 +8,6 @@ license.workspace = true
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
client = { workspace = true, features = ["testing"] }
|
||||
common-grpc.workspace = true
|
||||
common-query.workspace = true
|
||||
common-recordbatch.workspace = true
|
||||
|
||||
@@ -12,15 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use client::Database;
|
||||
use common_query::OutputData;
|
||||
use common_recordbatch::util;
|
||||
|
||||
pub enum ExpectedOutput<'a> {
|
||||
AffectedRows(usize),
|
||||
QueryResult(&'a str),
|
||||
}
|
||||
|
||||
pub async fn check_output_stream(output: OutputData, expected: &str) {
|
||||
let recordbatches = match output {
|
||||
OutputData::Stream(stream) => util::collect_batches(stream).await.unwrap(),
|
||||
@@ -30,25 +24,3 @@ pub async fn check_output_stream(output: OutputData, expected: &str) {
|
||||
let pretty_print = recordbatches.pretty_print().unwrap();
|
||||
assert_eq!(pretty_print, expected.trim(), "actual: \n{}", pretty_print);
|
||||
}
|
||||
|
||||
pub async fn execute_and_check_output(db: &Database, sql: &str, expected: ExpectedOutput<'_>) {
|
||||
let output = db.sql(sql).await.unwrap();
|
||||
let output = output.data;
|
||||
|
||||
match (&output, expected) {
|
||||
(OutputData::AffectedRows(x), ExpectedOutput::AffectedRows(y)) => {
|
||||
assert_eq!(
|
||||
*x, y,
|
||||
r#"
|
||||
expected: {y}
|
||||
actual: {x}
|
||||
"#
|
||||
)
|
||||
}
|
||||
(OutputData::RecordBatches(_), ExpectedOutput::QueryResult(x))
|
||||
| (OutputData::Stream(_), ExpectedOutput::QueryResult(x)) => {
|
||||
check_output_stream(output, x).await
|
||||
}
|
||||
_ => panic!(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,6 @@ common-catalog.workspace = true
|
||||
common-datasource.workspace = true
|
||||
common-error.workspace = true
|
||||
common-macro.workspace = true
|
||||
common-procedure.workspace = true
|
||||
common-query.workspace = true
|
||||
common-recordbatch.workspace = true
|
||||
common-telemetry.workspace = true
|
||||
|
||||
@@ -16,8 +16,9 @@ reads go to the query engine (`query` crate), writes go to the inserter/deleter
|
||||
Boundary with `servers`: the `servers` crate implements the wire protocols and
|
||||
network I/O; `frontend` provides the business logic by implementing handler
|
||||
traits (`SqlQueryHandler`, `GrpcQueryHandler`, `InfluxdbLineProtocolHandler`,
|
||||
...). In standalone mode the frontend embeds a datanode `RegionServer`; in
|
||||
distributed mode it talks to remote datanodes via `operator`/`client`.
|
||||
...). In standalone mode the `standalone` crate bridges the frontend to an
|
||||
embedded datanode `RegionServer`; in distributed mode the frontend talks to
|
||||
remote datanodes via `operator`/`client`.
|
||||
|
||||
## Module map
|
||||
|
||||
@@ -26,7 +27,6 @@ distributed mode it talks to remote datanodes via `operator`/`client`.
|
||||
| `instance` | `src/frontend/src/instance.rs` | `Instance`: the core handler; implements `SqlQueryHandler`, `PrometheusHandler`, etc. |
|
||||
| `instance/builder` | `src/frontend/src/instance/builder.rs` | `FrontendBuilder` assembles `Instance` from its dependencies |
|
||||
| `instance/grpc` | `src/frontend/src/instance/grpc.rs` | `GrpcQueryHandler`: insert/delete/query/promql over gRPC |
|
||||
| `instance/standalone` | `src/frontend/src/instance/standalone.rs` | Calls the local `RegionServer` instead of RPC |
|
||||
| `instance/region_query` | `src/frontend/src/instance/region_query.rs` | Routes distributed region reads to datanodes |
|
||||
| `instance/*` | `src/frontend/src/instance/` | Per-protocol handlers (`influxdb.rs`, `promql.rs`, `otlp/`, `jaeger.rs`, `logs.rs`, `prom_store.rs`, ...) |
|
||||
| `frontend` | `src/frontend/src/frontend.rs` | `Frontend` lifecycle wrapper (`FrontendOptions`, start/shutdown) |
|
||||
@@ -70,9 +70,9 @@ cargo nextest run -p frontend
|
||||
|
||||
- Keep the frontend/servers split straight: wire format and network live in
|
||||
`servers`; permissions, planning, and routing live here.
|
||||
- Standalone vs distributed diverge in datanode access (local `RegionServer` vs
|
||||
`NodeClients` RPC), MetaClient usage, and whether heartbeat matters. In
|
||||
standalone, the cache invalidator is a no-op.
|
||||
- Standalone vs distributed diverge in datanode access (the `standalone` crate's
|
||||
local `RegionServer` adapter vs `NodeClients` RPC), MetaClient usage, and
|
||||
whether heartbeat matters. In standalone, the cache invalidator is a no-op.
|
||||
|
||||
## Maintenance contract
|
||||
|
||||
|
||||
@@ -52,7 +52,6 @@ common-version.workspace = true
|
||||
dashmap.workspace = true
|
||||
datafusion.workspace = true
|
||||
datafusion-expr.workspace = true
|
||||
datanode.workspace = true
|
||||
datatypes.workspace = true
|
||||
futures.workspace = true
|
||||
hostname.workspace = true
|
||||
@@ -91,11 +90,9 @@ tracing.workspace = true
|
||||
[dev-dependencies]
|
||||
catalog = { workspace = true, features = ["testing"] }
|
||||
common-test-util.workspace = true
|
||||
datanode.workspace = true
|
||||
datatypes.workspace = true
|
||||
futures.workspace = true
|
||||
hyper-util = { workspace = true, features = ["tokio"] }
|
||||
meta-srv.workspace = true
|
||||
reqwest.workspace = true
|
||||
serde_json.workspace = true
|
||||
strfmt = "0.2"
|
||||
|
||||
@@ -203,13 +203,6 @@ pub enum Error {
|
||||
source: query::error::Error,
|
||||
},
|
||||
|
||||
#[snafu(display("Operation to region server failed"))]
|
||||
InvokeRegionServer {
|
||||
#[snafu(implicit)]
|
||||
location: Location,
|
||||
source: servers::error::Error,
|
||||
},
|
||||
|
||||
#[snafu(display("Not supported: {}", feat))]
|
||||
NotSupported { feat: String },
|
||||
|
||||
@@ -296,9 +289,6 @@ pub enum Error {
|
||||
location: Location,
|
||||
},
|
||||
|
||||
#[snafu(display("Invalid region request, reason: {}", reason))]
|
||||
InvalidRegionRequest { reason: String },
|
||||
|
||||
#[snafu(display("Table operation error"))]
|
||||
TableOperation {
|
||||
source: operator::error::Error,
|
||||
@@ -426,8 +416,6 @@ impl ErrorExt for Error {
|
||||
|
||||
Error::CacheRequired { .. } => StatusCode::Internal,
|
||||
|
||||
Error::InvalidRegionRequest { .. } => StatusCode::IllegalState,
|
||||
|
||||
Error::TableNotFound { .. } => StatusCode::TableNotFound,
|
||||
|
||||
Error::Catalog { source, .. } => source.status_code(),
|
||||
@@ -438,7 +426,6 @@ impl ErrorExt for Error {
|
||||
| Error::ReadTable { source, .. }
|
||||
| Error::ExecLogicalPlan { source, .. } => source.status_code(),
|
||||
|
||||
Error::InvokeRegionServer { source, .. } => source.status_code(),
|
||||
Error::External { source, .. } | Error::InitPlugin { source, .. } => {
|
||||
source.status_code()
|
||||
}
|
||||
@@ -472,7 +459,6 @@ impl ErrorExt for Error {
|
||||
|
||||
Error::StartServer { source, .. }
|
||||
| Error::ShutdownServer { source, .. }
|
||||
| Error::InvokeRegionServer { source, .. }
|
||||
| Error::ExecutePromql { source, .. }
|
||||
| Error::PromStoreRemoteQueryPlan { source, .. }
|
||||
| Error::PrometheusMetricNamesQueryPlan { source, .. } => source.retry_hint(),
|
||||
|
||||
@@ -165,6 +165,7 @@ impl Frontend {
|
||||
mod tests {
|
||||
use std::any::Any;
|
||||
use std::net::SocketAddr;
|
||||
use std::pin::Pin;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -186,9 +187,9 @@ mod tests {
|
||||
use common_meta::heartbeat::handler::suspend::SuspendHandler;
|
||||
use common_meta::instruction::Instruction;
|
||||
use common_stat::ResourceStatImpl;
|
||||
use futures::Stream;
|
||||
use meta_client::MetaClientRef;
|
||||
use meta_client::client::MetaClientBuilder;
|
||||
use meta_srv::service::GrpcStream;
|
||||
use servers::grpc::{FlightCompression, GRPC_SERVER};
|
||||
use servers::http::HTTP_SERVER;
|
||||
use servers::http::result::greptime_result_v1::GreptimedbV1Response;
|
||||
@@ -206,6 +207,9 @@ mod tests {
|
||||
use crate::instance::builder::FrontendBuilder;
|
||||
use crate::server::Services;
|
||||
|
||||
type GrpcStream<T> =
|
||||
Pin<Box<dyn Stream<Item = std::result::Result<T, Status>> + Send + Sync + 'static>>;
|
||||
|
||||
#[test]
|
||||
fn test_toml() {
|
||||
let opts = FrontendOptions::default();
|
||||
|
||||
@@ -25,7 +25,6 @@ mod otlp;
|
||||
pub mod prom_store;
|
||||
mod promql;
|
||||
mod region_query;
|
||||
pub mod standalone;
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::pin::Pin;
|
||||
@@ -100,7 +99,6 @@ use sql::statements::statement::Statement;
|
||||
use sql::statements::tql::Tql;
|
||||
use sql::util::{extract_tables_from_prom_expr_checked, extract_tables_from_statement_checked};
|
||||
use sqlparser::ast::{AnalyzeFormat, ObjectName};
|
||||
pub use standalone::StandaloneDatanodeManager;
|
||||
use table::requests::{OTLP_METRIC_COMPAT_KEY, OTLP_METRIC_COMPAT_PROM};
|
||||
use tracing::Span;
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ license.workspace = true
|
||||
workspace = true
|
||||
|
||||
[features]
|
||||
vector_index = ["dep:usearch"]
|
||||
vector_index = ["dep:nalgebra", "dep:usearch"]
|
||||
|
||||
[dependencies]
|
||||
async-trait.workspace = true
|
||||
@@ -29,7 +29,7 @@ itertools.workspace = true
|
||||
jieba-rs = "0.10"
|
||||
lazy_static.workspace = true
|
||||
mockall.workspace = true
|
||||
nalgebra.workspace = true
|
||||
nalgebra = { workspace = true, optional = true }
|
||||
pin-project.workspace = true
|
||||
prost.workspace = true
|
||||
puffin.workspace = true
|
||||
|
||||
@@ -5,9 +5,8 @@ edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[build-dependencies]
|
||||
protobuf-build = { version = "0.15", default-features = false, features = [
|
||||
"protobuf-codec",
|
||||
] }
|
||||
protoc-bin-vendored.workspace = true
|
||||
protoc-rust.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
+88
-7
@@ -12,13 +12,94 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use protobuf_build::Builder;
|
||||
//! Generates Rust protobuf bindings for `proto/logstore.proto`.
|
||||
//!
|
||||
//! The protoc compiler is selected in the following order:
|
||||
//!
|
||||
//! 1. Use the compiler specified by the `PROTOC` environment variable.
|
||||
//! If the configured compiler is unusable or does not meet the minimum
|
||||
//! supported version requirement, the build will panic instead of falling
|
||||
//! back to another compiler.
|
||||
//!
|
||||
//! 2. Use a compatible `protoc` compiler available from `PATH`.
|
||||
//! If it is unavailable or does not meet the minimum supported version
|
||||
//! requirement, continue with the vendored compiler fallback.
|
||||
//!
|
||||
//! 3. Use a prebuilt vendored `protoc` compiler.
|
||||
//! This provides a portable fallback when no supported system compiler is
|
||||
//! available.
|
||||
|
||||
use std::env;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
use protoc_rust::{Codegen, Customize};
|
||||
|
||||
/// Returns whether `protoc` runs and meets GreptimeDB's minimum supported version.
|
||||
fn usable_protoc(protoc: &Path) -> bool {
|
||||
let Ok(output) = Command::new(protoc).arg("--version").output() else {
|
||||
return false;
|
||||
};
|
||||
if !output.status.success() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let Some(version) = std::str::from_utf8(&output.stdout)
|
||||
.ok()
|
||||
.and_then(|output| output.trim().strip_prefix("libprotoc "))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
let mut components = version.split('.');
|
||||
matches!(
|
||||
(
|
||||
components.next().and_then(|part| part.parse::<u32>().ok()),
|
||||
components.next().and_then(|part| part.parse::<u32>().ok()),
|
||||
),
|
||||
(Some(major), Some(minor)) if (major, minor) >= (3, 15)
|
||||
)
|
||||
}
|
||||
|
||||
/// Selects a compiler from `PROTOC`, `PATH`, or the vendored fallback, in that order.
|
||||
///
|
||||
/// An invalid explicit `PROTOC` is rejected so configuration errors are not
|
||||
/// silently masked.
|
||||
fn protoc_path() -> PathBuf {
|
||||
if let Some(protoc) = env::var_os("PROTOC") {
|
||||
let protoc = PathBuf::from(protoc);
|
||||
assert!(usable_protoc(&protoc), "PROTOC version not usable");
|
||||
return protoc;
|
||||
}
|
||||
|
||||
let protoc = PathBuf::from("protoc");
|
||||
if usable_protoc(&protoc) {
|
||||
return protoc;
|
||||
}
|
||||
|
||||
protoc_bin_vendored::protoc_bin_path().expect("no bundled protoc for this platform")
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let base = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".to_string());
|
||||
Builder::new()
|
||||
.search_dir_for_protos(&format!("{base}/proto"))
|
||||
.includes(&[format!("{base}/include"), format!("{base}/proto")])
|
||||
.include_google_protos()
|
||||
.generate()
|
||||
let base =
|
||||
PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set"));
|
||||
let proto_dir = base.join("proto");
|
||||
let proto = proto_dir.join("logstore.proto");
|
||||
let out_dir = PathBuf::from(env::var_os("OUT_DIR").expect("OUT_DIR not set")).join("protos");
|
||||
|
||||
println!("cargo:rerun-if-env-changed=PROTOC");
|
||||
println!("cargo:rerun-if-env-changed=PATH");
|
||||
println!("cargo:rerun-if-changed={}", proto.display());
|
||||
std::fs::create_dir_all(&out_dir).expect("failed to create protobuf output directory");
|
||||
|
||||
Codegen::new()
|
||||
.protoc_path(protoc_path())
|
||||
.out_dir(out_dir)
|
||||
.input(proto)
|
||||
.include(proto_dir)
|
||||
.customize(Customize {
|
||||
gen_mod_rs: Some(true),
|
||||
..Default::default()
|
||||
})
|
||||
.run()
|
||||
.expect("failed to generate log-store protobuf bindings")
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ license.workspace = true
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
api.workspace = true
|
||||
async-trait.workspace = true
|
||||
catalog.workspace = true
|
||||
client.workspace = true
|
||||
@@ -21,6 +22,7 @@ common-meta.workspace = true
|
||||
common-options.workspace = true
|
||||
common-procedure.workspace = true
|
||||
common-query.workspace = true
|
||||
common-recordbatch.workspace = true
|
||||
common-stat.workspace = true
|
||||
common-telemetry.workspace = true
|
||||
common-time.workspace = true
|
||||
|
||||
@@ -54,12 +54,12 @@ impl FlownodeManager for StandaloneDatanodeManager {
|
||||
}
|
||||
|
||||
/// Relative to [client::region::RegionRequester]
|
||||
pub struct RegionInvoker {
|
||||
struct RegionInvoker {
|
||||
region_server: RegionServer,
|
||||
}
|
||||
|
||||
impl RegionInvoker {
|
||||
pub fn arc(region_server: RegionServer) -> Arc<Self> {
|
||||
fn arc(region_server: RegionServer) -> Arc<Self> {
|
||||
Arc::new(Self { region_server })
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use std::any::Any;
|
||||
|
||||
use common_error::ext::{BoxedError, ErrorExt};
|
||||
use common_error::ext::{BoxedError, ErrorExt, RetryHint};
|
||||
use common_error::status_code::StatusCode;
|
||||
use common_macro::stack_trace_debug;
|
||||
use snafu::{Location, Snafu};
|
||||
@@ -30,6 +30,13 @@ pub enum Error {
|
||||
source: BoxedError,
|
||||
},
|
||||
|
||||
#[snafu(display("Operation to region server failed"))]
|
||||
InvokeRegionServer {
|
||||
#[snafu(implicit)]
|
||||
location: Location,
|
||||
source: servers::error::Error,
|
||||
},
|
||||
|
||||
#[snafu(display("External error"))]
|
||||
External {
|
||||
source: BoxedError,
|
||||
@@ -49,6 +56,9 @@ pub enum Error {
|
||||
#[snafu(display("Invalid url scheme: {}", scheme))]
|
||||
InvalidUrlScheme { scheme: String },
|
||||
|
||||
#[snafu(display("Invalid region request, reason: {}", reason))]
|
||||
InvalidRegionRequest { reason: String },
|
||||
|
||||
#[snafu(display("Repartition procedure is not supported in standalone mode"))]
|
||||
NoSupportRepartitionProcedure {
|
||||
#[snafu(implicit)]
|
||||
@@ -62,9 +72,11 @@ impl ErrorExt for Error {
|
||||
fn status_code(&self) -> StatusCode {
|
||||
match self {
|
||||
Error::OpenMetadataKvBackend { source, .. } => source.status_code(),
|
||||
Error::InvokeRegionServer { source, .. } => source.status_code(),
|
||||
Error::External { source, .. } => source.status_code(),
|
||||
Error::ParseUrl { .. } => StatusCode::InvalidArguments,
|
||||
Error::InvalidUrlScheme { .. } => StatusCode::InvalidArguments,
|
||||
Error::InvalidRegionRequest { .. } => StatusCode::IllegalState,
|
||||
Error::NoSupportRepartitionProcedure { .. } => StatusCode::Unsupported,
|
||||
}
|
||||
}
|
||||
@@ -72,4 +84,11 @@ impl ErrorExt for Error {
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
|
||||
fn retry_hint(&self) -> RetryHint {
|
||||
match self {
|
||||
Error::InvokeRegionServer { source, .. } => source.retry_hint(),
|
||||
_ => RetryHint::NonRetryable,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,12 +12,14 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
mod datanode_manager;
|
||||
pub mod error;
|
||||
pub mod information_extension;
|
||||
pub mod metadata;
|
||||
pub mod options;
|
||||
pub mod procedure;
|
||||
|
||||
pub use datanode_manager::StandaloneDatanodeManager;
|
||||
pub use information_extension::StandaloneInformationExtension;
|
||||
pub use metadata::{build_metadata_kv_from_url, build_metadata_kvbackend};
|
||||
pub use procedure::{StandaloneRepartitionProcedureFactory, build_procedure_manager};
|
||||
|
||||
@@ -49,14 +49,14 @@ use common_wal::config::{DatanodeWalConfig, MetasrvWalConfig};
|
||||
use datanode::datanode::DatanodeBuilder;
|
||||
use flow::{FlownodeBuilder, FrontendClient, GrpcQueryHandlerWithBoxedError};
|
||||
use frontend::frontend::Frontend;
|
||||
use frontend::instance::Instance;
|
||||
use frontend::instance::builder::FrontendBuilder;
|
||||
use frontend::instance::{Instance, StandaloneDatanodeManager};
|
||||
use frontend::server::Services;
|
||||
use meta_srv::metasrv::{FLOW_ID_SEQ, TABLE_ID_SEQ};
|
||||
use servers::grpc::GrpcOptions;
|
||||
use snafu::ResultExt;
|
||||
use standalone::StandaloneRepartitionProcedureFactory;
|
||||
use standalone::options::StandaloneOptions;
|
||||
use standalone::{StandaloneDatanodeManager, StandaloneRepartitionProcedureFactory};
|
||||
|
||||
use crate::test_util::{self, StorageType, TestGuard, create_tmp_dir_and_datanode_opts};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user