rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2025-12-26 16:10:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: update substrait and prost version (#1080)

Browse Source
This commit is contained in:
Ning Sun
2023-02-27 15:18:12 +08:00
committed by GitHub
parent bd377ef329
commit 4b58a8a18d
9 changed files with 193 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

243
Cargo.lock generated
View File

@@ -143,7 +143,7 @@ dependencies = [
"common-time",
"datatypes",
"greptime-proto",
"prost 0.11.6",
"prost",
"snafu",
"tonic",
"tonic-build",
@@ -301,9 +301,9 @@ dependencies = [
"bytes",
"futures",
"proc-macro2",
"prost 0.11.6",
"prost-build 0.11.3",
"prost-derive 0.11.6",
"prost",
"prost-build",
"prost-derive",
"tokio",
"tonic",
"tonic-build",
@@ -1311,12 +1311,11 @@ dependencies = [
"enum_dispatch",
"futures-util",
"parking_lot",
"prost 0.11.6",
"prost 0.9.0",
"prost",
"rand 0.8.5",
"snafu",
"substrait 0.1.0",
"substrait 0.2.0",
"substrait 0.4.0",
"tokio",
"tonic",
"tracing",
@@ -1494,7 +1493,7 @@ dependencies = [
"datatypes",
"flatbuffers",
"futures",
"prost 0.11.6",
"prost",
"rand 0.8.5",
"snafu",
"tokio",
@@ -1650,8 +1649,8 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e57ff02e8ad8e06ab9731d5dc72dc23bef9200778eae1a89d555d8c42e5d4a86"
dependencies = [
"prost 0.11.6",
"prost-types 0.11.6",
"prost",
"prost-types",
"tonic",
"tracing-core",
]
@@ -1668,7 +1667,7 @@ dependencies = [
"futures",
"hdrhistogram",
"humantime",
"prost-types 0.11.6",
"prost-types",
"serde",
"serde_json",
"thread_local",
@@ -2206,7 +2205,7 @@ dependencies = [
"mito",
"object-store",
"pin-project",
"prost 0.11.6",
"prost",
"query",
"script",
"serde",
@@ -2550,7 +2549,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1259da3b15ec7e54bd7203adb2c4335adb9ca1d47b56220d650e52c247e824a"
dependencies = [
"http",
"prost 0.11.6",
"prost",
"tokio",
"tokio-stream",
"tonic",
@@ -2709,7 +2708,7 @@ dependencies = [
"moka",
"openmetrics-parser",
"partition",
"prost 0.11.6",
"prost",
"query",
"rustls",
"serde",
@@ -2990,7 +2989,7 @@ name = "greptime-proto"
version = "0.1.0"
source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=1599ae2a0d1d8f42ee23ed26e4ad7a7b34134c60#1599ae2a0d1d8f42ee23ed26e4ad7a7b34134c60"
dependencies = [
"prost 0.11.6",
"prost",
"tonic",
"tonic-build",
]
@@ -3124,6 +3123,15 @@ dependencies = [
"digest",
]
[[package]]
name = "home"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "747309b4b440c06d57b0b25f2aee03ee9b5e5397d288c60e21fc709bb98a7408"
dependencies = [
"winapi",
]
[[package]]
name = "http"
version = "0.2.8"
@@ -3920,7 +3928,7 @@ dependencies = [
"http-body",
"lazy_static",
"parking_lot",
"prost 0.11.6",
"prost",
"regex",
"serde",
"serde_json",
@@ -5338,16 +5346,6 @@ dependencies = [
"regex",
]
[[package]]
name = "prost"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001"
dependencies = [
"bytes",
"prost-derive 0.9.0",
]
[[package]]
name = "prost"
version = "0.11.6"
@@ -5355,27 +5353,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21dc42e00223fc37204bd4aa177e69420c604ca4a183209a8f9de30c6d934698"
dependencies = [
"bytes",
"prost-derive 0.11.6",
]
[[package]]
name = "prost-build"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"
dependencies = [
"bytes",
"heck 0.3.3",
"itertools",
"lazy_static",
"log",
"multimap",
"petgraph",
"prost 0.9.0",
"prost-types 0.9.0",
"regex",
"tempfile",
"which",
"prost-derive",
]
[[package]]
@@ -5392,27 +5370,14 @@ dependencies = [
"multimap",
"petgraph",
"prettyplease",
"prost 0.11.6",
"prost-types 0.11.6",
"prost",
"prost-types",
"regex",
"syn",
"tempfile",
"which",
]
[[package]]
name = "prost-derive"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe"
dependencies = [
"anyhow",
"itertools",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "prost-derive"
version = "0.11.6"
@@ -5426,16 +5391,6 @@ dependencies = [
"syn",
]
[[package]]
name = "prost-types"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a"
dependencies = [
"bytes",
"prost 0.9.0",
]
[[package]]
name = "prost-types"
version = "0.11.6"
@@ -5443,7 +5398,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5e0526209433e96d83d750dd81a99118edbc55739e7e61a46764fd2ad537788"
dependencies = [
"bytes",
"prost 0.11.6",
"prost",
]
[[package]]
@@ -5806,6 +5761,15 @@ version = "0.6.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
[[package]]
name = "regress"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a92ff21fe8026ce3f2627faaf43606f0b67b014dbc9ccf027181a804f75d92e"
dependencies = [
"memchr",
]
[[package]]
name = "remove_dir_all"
version = "0.5.3"
@@ -6062,6 +6026,19 @@ dependencies = [
"semver 1.0.16",
]
[[package]]
name = "rustfmt-wrapper"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed729e3bee08ec2befd593c27e90ca9fdd25efdc83c94c3b82eaef16e4f7406e"
dependencies = [
"serde",
"tempfile",
"thiserror",
"toml",
"toolchain_find",
]
[[package]]
name = "rustix"
version = "0.36.7"
@@ -6757,6 +6734,17 @@ dependencies = [
"syn",
]
[[package]]
name = "serde_tokenstream"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "274f512d6748a01e67cbcde5b4307ab2c9d52a98a2b870a980ef0793a351deff"
dependencies = [
"proc-macro2",
"serde",
"syn",
]
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
@@ -6769,6 +6757,19 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_yaml"
version = "0.9.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fb06d4b6cdaef0e0c51fa881acb721bed3c924cfaa71d9c94a3b771dfdf6567"
dependencies = [
"indexmap",
"itoa 1.0.5",
"ryu",
"serde",
"unsafe-libyaml",
]
[[package]]
name = "servers"
version = "0.1.0"
@@ -6814,7 +6815,7 @@ dependencies = [
"pin-project",
"postgres-types",
"promql-parser",
"prost 0.11.6",
"prost",
"query",
"rand 0.8.5",
"regex",
@@ -7206,7 +7207,7 @@ dependencies = [
"parquet",
"paste",
"planus",
"prost 0.11.6",
"prost",
"rand 0.8.5",
"regex",
"serde",
@@ -7368,23 +7369,29 @@ dependencies = [
"datafusion-expr",
"datatypes",
"futures",
"prost 0.9.0",
"prost",
"snafu",
"substrait 0.2.0",
"substrait 0.4.0",
"table",
"tokio",
]
[[package]]
name = "substrait"
version = "0.2.0"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46079e9004f5e069eae2976d4e23ea29c4e215b1096d3d53b76b19879f346100"
checksum = "e2feb96a6a106e21161551af32dc4e0fdab3aceb926b940d7e92a086b640fc7c"
dependencies = [
"glob",
"prost 0.9.0",
"prost-build 0.9.0",
"prost-types 0.9.0",
"heck 0.4.0",
"prost",
"prost-build",
"prost-types",
"schemars",
"serde",
"serde_json",
"serde_yaml",
"typify",
"walkdir",
]
[[package]]
@@ -7945,8 +7952,8 @@ dependencies = [
"hyper-timeout",
"percent-encoding",
"pin-project",
"prost 0.11.6",
"prost-derive 0.11.6",
"prost",
"prost-derive",
"rustls-pemfile",
"tokio",
"tokio-rustls",
@@ -7967,11 +7974,24 @@ checksum = "5bf5e9b9c0f7e0a7c027dcfaba7b2c60816c7049171f679d99ee2ff65d0de8c4"
dependencies = [
"prettyplease",
"proc-macro2",
"prost-build 0.11.3",
"prost-build",
"quote",
"syn",
]
[[package]]
name = "toolchain_find"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e85654a10e7a07a47c6f19d93818f3f343e22927f2fa280c84f7c8042743413"
dependencies = [
"home",
"lazy_static",
"regex",
"semver 0.11.0",
"walkdir",
]
[[package]]
name = "tower"
version = "0.4.13"
@@ -8191,6 +8211,51 @@ version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
[[package]]
name = "typify"
version = "0.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e8486352f3c946e69f983558cfc09b295250b01e01b381ec67a05a812d01d63"
dependencies = [
"typify-impl",
"typify-macro",
]
[[package]]
name = "typify-impl"
version = "0.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7624d0b911df6e2bbf34a236f76281f93b294cdde1d4df1dbdb748e5a7fefa5"
dependencies = [
"heck 0.4.0",
"log",
"proc-macro2",
"quote",
"regress",
"rustfmt-wrapper",
"schemars",
"serde_json",
"syn",
"thiserror",
"unicode-ident",
]
[[package]]
name = "typify-macro"
version = "0.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c42802aa033cee7650a4e1509ba7d5848a56f84be7c4b31e4385ee12445e942"
dependencies = [
"proc-macro2",
"quote",
"schemars",
"serde",
"serde_json",
"serde_tokenstream",
"syn",
"typify-impl",
]
[[package]]
name = "ucd-trie"
version = "0.1.5"
@@ -8391,6 +8456,12 @@ version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "029df4cc8238cefc911704ff8fa210853a0f3bce2694d8f51181dd41ee0f3301"
[[package]]
name = "unsafe-libyaml"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc7ed8ba44ca06be78ea1ad2c3682a43349126c8818054231ee6f4748012aed2"
[[package]]
name = "untrusted"
version = "0.7.1"

8
src/client/Cargo.toml
View File

@@ -32,12 +32,8 @@ substrait = { path = "../common/substrait" }
tokio.workspace = true
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
# TODO(ruihang): upgrade to 0.11 once substrait-rs supports it.
[dev-dependencies.prost_09]
package = "prost"
version = "0.9"
prost.workspace = true
[dev-dependencies.substrait_proto]
package = "substrait"
version = "0.2"
version = "0.4"

16
src/client/examples/logical.rs
View File

@@ -15,11 +15,11 @@
use api::v1::{ColumnDataType, ColumnDef, CreateTableExpr, TableId};
use client::{Client, Database};
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
use prost_09::Message;
use substrait_proto::protobuf::plan_rel::RelType as PlanRelType;
use substrait_proto::protobuf::read_rel::{NamedTable, ReadType};
use substrait_proto::protobuf::rel::RelType;
use substrait_proto::protobuf::{PlanRel, ReadRel, Rel};
use prost::Message;
use substrait_proto::proto::plan_rel::RelType as PlanRelType;
use substrait_proto::proto::read_rel::{NamedTable, ReadType};
use substrait_proto::proto::rel::RelType;
use substrait_proto::proto::{PlanRel, ReadRel, Rel};
use tracing::{event, Level};
fn main() {
@@ -89,12 +89,8 @@ fn mock_logical_plan() -> Vec<u8> {
let read_type = ReadType::NamedTable(named_table);
let read_rel = ReadRel {
common: None,
base_schema: None,
filter: None,
projection: None,
advanced_extension: None,
read_type: Some(read_type),
..Default::default()
};
let mut buf = vec![];

4
src/common/substrait/Cargo.toml
View File

@@ -14,13 +14,13 @@ datafusion.workspace = true
datafusion-expr.workspace = true
datatypes = { path = "../../datatypes" }
futures = "0.3"
prost = "0.9"
prost.workspace = true
snafu.workspace = true
table = { path = "../../table" }
[dependencies.substrait_proto]
package = "substrait"
version = "0.2"
version = "0.4"
[dev-dependencies]
datatypes = { path = "../../datatypes" }

4
src/common/substrait/src/context.rs
View File

@@ -15,10 +15,10 @@
use std::collections::HashMap;
use datafusion::common::DFSchemaRef;
use substrait_proto::protobuf::extensions::simple_extension_declaration::{
use substrait_proto::proto::extensions::simple_extension_declaration::{
ExtensionFunction, MappingType,
};
use substrait_proto::protobuf::extensions::SimpleExtensionDeclaration;
use substrait_proto::proto::extensions::SimpleExtensionDeclaration;
#[derive(Default)]
pub struct ConvertorContext {

17
src/common/substrait/src/df_expr.rs
View File

@@ -20,15 +20,15 @@ use datafusion_expr::expr::Sort;
use datafusion_expr::{expr_fn, lit, Between, BinaryExpr, BuiltinScalarFunction, Expr, Operator};
use datatypes::schema::Schema;
use snafu::{ensure, OptionExt};
use substrait_proto::protobuf::expression::field_reference::ReferenceType as FieldReferenceType;
use substrait_proto::protobuf::expression::reference_segment::{
use substrait_proto::proto::expression::field_reference::ReferenceType as FieldReferenceType;
use substrait_proto::proto::expression::reference_segment::{
ReferenceType as SegReferenceType, StructField,
};
use substrait_proto::protobuf::expression::{
use substrait_proto::proto::expression::{
FieldReference, Literal, ReferenceSegment, RexType, ScalarFunction,
};
use substrait_proto::protobuf::function_argument::ArgType;
use substrait_proto::protobuf::Expression;
use substrait_proto::proto::function_argument::ArgType;
use substrait_proto::proto::Expression;
use crate::context::ConvertorContext;
use crate::error::{
@@ -61,6 +61,7 @@ pub(crate) fn to_df_expr(
| RexType::MultiOrList(_)
| RexType::Cast(_)
| RexType::Subquery(_)
| RexType::Nested(_)
| RexType::Enum(_) => UnsupportedExprSnafu {
name: format!("substrait expression {expr_rex_type:?}"),
}
@@ -615,9 +616,9 @@ pub fn convert_column(column: &Column, schema: &Schema) -> Result<FieldReference
/// Some utils special for this `DataFusion::Expr` and `Substrait::Expression` conversion.
mod utils {
use datafusion_expr::{BuiltinScalarFunction, Operator};
use substrait_proto::protobuf::expression::{RexType, ScalarFunction};
use substrait_proto::protobuf::function_argument::ArgType;
use substrait_proto::protobuf::{Expression, FunctionArgument};
use substrait_proto::proto::expression::{RexType, ScalarFunction};
use substrait_proto::proto::function_argument::ArgType;
use substrait_proto::proto::{Expression, FunctionArgument};
pub(crate) fn name_df_operator(op: &Operator) -> &str {
match op {

16
src/common/substrait/src/df_logical.rs
View File

@@ -25,13 +25,13 @@ use datafusion::physical_plan::project_schema;
use datafusion_expr::{Filter, LogicalPlan, TableScan, TableSource};
use prost::Message;
use snafu::{ensure, OptionExt, ResultExt};
use substrait_proto::protobuf::expression::mask_expression::{StructItem, StructSelect};
use substrait_proto::protobuf::expression::MaskExpression;
use substrait_proto::protobuf::extensions::simple_extension_declaration::MappingType;
use substrait_proto::protobuf::plan_rel::RelType as PlanRelType;
use substrait_proto::protobuf::read_rel::{NamedTable, ReadType};
use substrait_proto::protobuf::rel::RelType;
use substrait_proto::protobuf::{FilterRel, Plan, PlanRel, ReadRel, Rel};
use substrait_proto::proto::expression::mask_expression::{StructItem, StructSelect};
use substrait_proto::proto::expression::MaskExpression;
use substrait_proto::proto::extensions::simple_extension_declaration::MappingType;
use substrait_proto::proto::plan_rel::RelType as PlanRelType;
use substrait_proto::proto::read_rel::{NamedTable, ReadType};
use substrait_proto::proto::rel::RelType;
use substrait_proto::proto::{FilterRel, Plan, PlanRel, ReadRel, Rel};
use table::table::adapter::DfTableProviderAdapter;
use crate::context::ConvertorContext;
@@ -424,6 +424,7 @@ impl DFLogicalSubstraitConvertor {
relations: vec![plan_rel],
advanced_extensions: None,
expected_type_urls: vec![],
..Default::default()
})
}
@@ -485,6 +486,7 @@ impl DFLogicalSubstraitConvertor {
projection,
advanced_extension: None,
read_type: Some(read_type),
..Default::default()
};
Ok(read_rel)

4
src/common/substrait/src/schema.rs
View File

@@ -13,8 +13,8 @@
// limitations under the License.
use datatypes::schema::{ColumnSchema, Schema};
use substrait_proto::protobuf::r#type::{Nullability, Struct as SubstraitStruct};
use substrait_proto::protobuf::NamedStruct;
use substrait_proto::proto::r#type::{Nullability, Struct as SubstraitStruct};
use substrait_proto::proto::NamedStruct;
use crate::error::Result;
use crate::types::{from_concrete_type, to_concrete_type};

7
src/common/substrait/src/types.rs
View File

@@ -20,9 +20,9 @@
use datafusion::scalar::ScalarValue;
use datatypes::prelude::ConcreteDataType;
use substrait_proto::protobuf::expression::literal::LiteralType;
use substrait_proto::protobuf::r#type::{self as s_type, Kind, Nullability};
use substrait_proto::protobuf::{Type as SType, Type};
use substrait_proto::proto::expression::literal::LiteralType;
use substrait_proto::proto::r#type::{self as s_type, Kind, Nullability};
use substrait_proto::proto::{Type as SType, Type};
use crate::error::{self, Result, UnsupportedConcreteTypeSnafu, UnsupportedSubstraitTypeSnafu};
@@ -86,6 +86,7 @@ pub fn to_concrete_type(ty: &SType) -> Result<(ConcreteDataType, bool)> {
| Kind::Struct(_)
| Kind::List(_)
| Kind::Map(_)
| Kind::UserDefined(_)
| Kind::UserDefinedTypeReference(_) => UnsupportedSubstraitTypeSnafu {
ty: format!("{kind:?}"),
}