From 4b58a8a18d6e51a518c3ec9bd6d93d85a0cdbf12 Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Mon, 27 Feb 2023 15:18:12 +0800 Subject: [PATCH] feat: update substrait and prost version (#1080) --- Cargo.lock | 243 ++++++++++++++++--------- src/client/Cargo.toml | 8 +- src/client/examples/logical.rs | 16 +- src/common/substrait/Cargo.toml | 4 +- src/common/substrait/src/context.rs | 4 +- src/common/substrait/src/df_expr.rs | 17 +- src/common/substrait/src/df_logical.rs | 16 +- src/common/substrait/src/schema.rs | 4 +- src/common/substrait/src/types.rs | 7 +- 9 files changed, 193 insertions(+), 126 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a54d3b0ef7..1b68d3e446 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,7 +143,7 @@ dependencies = [ "common-time", "datatypes", "greptime-proto", - "prost 0.11.6", + "prost", "snafu", "tonic", "tonic-build", @@ -301,9 +301,9 @@ dependencies = [ "bytes", "futures", "proc-macro2", - "prost 0.11.6", - "prost-build 0.11.3", - "prost-derive 0.11.6", + "prost", + "prost-build", + "prost-derive", "tokio", "tonic", "tonic-build", @@ -1311,12 +1311,11 @@ dependencies = [ "enum_dispatch", "futures-util", "parking_lot", - "prost 0.11.6", - "prost 0.9.0", + "prost", "rand 0.8.5", "snafu", "substrait 0.1.0", - "substrait 0.2.0", + "substrait 0.4.0", "tokio", "tonic", "tracing", @@ -1494,7 +1493,7 @@ dependencies = [ "datatypes", "flatbuffers", "futures", - "prost 0.11.6", + "prost", "rand 0.8.5", "snafu", "tokio", @@ -1650,8 +1649,8 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e57ff02e8ad8e06ab9731d5dc72dc23bef9200778eae1a89d555d8c42e5d4a86" dependencies = [ - "prost 0.11.6", - "prost-types 0.11.6", + "prost", + "prost-types", "tonic", "tracing-core", ] @@ -1668,7 +1667,7 @@ dependencies = [ "futures", "hdrhistogram", "humantime", - "prost-types 0.11.6", + "prost-types", "serde", "serde_json", "thread_local", @@ -2206,7 +2205,7 @@ dependencies = [ "mito", "object-store", "pin-project", - "prost 0.11.6", + "prost", "query", "script", "serde", @@ -2550,7 +2549,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1259da3b15ec7e54bd7203adb2c4335adb9ca1d47b56220d650e52c247e824a" dependencies = [ "http", - "prost 0.11.6", + "prost", "tokio", "tokio-stream", "tonic", @@ -2709,7 +2708,7 @@ dependencies = [ "moka", "openmetrics-parser", "partition", - "prost 0.11.6", + "prost", "query", "rustls", "serde", @@ -2990,7 +2989,7 @@ name = "greptime-proto" version = "0.1.0" source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=1599ae2a0d1d8f42ee23ed26e4ad7a7b34134c60#1599ae2a0d1d8f42ee23ed26e4ad7a7b34134c60" dependencies = [ - "prost 0.11.6", + "prost", "tonic", "tonic-build", ] @@ -3124,6 +3123,15 @@ dependencies = [ "digest", ] +[[package]] +name = "home" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "747309b4b440c06d57b0b25f2aee03ee9b5e5397d288c60e21fc709bb98a7408" +dependencies = [ + "winapi", +] + [[package]] name = "http" version = "0.2.8" @@ -3920,7 +3928,7 @@ dependencies = [ "http-body", "lazy_static", "parking_lot", - "prost 0.11.6", + "prost", "regex", "serde", "serde_json", @@ -5338,16 +5346,6 @@ dependencies = [ "regex", ] -[[package]] -name = "prost" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" -dependencies = [ - "bytes", - "prost-derive 0.9.0", -] - [[package]] name = "prost" version = "0.11.6" @@ -5355,27 +5353,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21dc42e00223fc37204bd4aa177e69420c604ca4a183209a8f9de30c6d934698" dependencies = [ "bytes", - "prost-derive 0.11.6", -] - -[[package]] -name = "prost-build" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5" -dependencies = [ - "bytes", - "heck 0.3.3", - "itertools", - "lazy_static", - "log", - "multimap", - "petgraph", - "prost 0.9.0", - "prost-types 0.9.0", - "regex", - "tempfile", - "which", + "prost-derive", ] [[package]] @@ -5392,27 +5370,14 @@ dependencies = [ "multimap", "petgraph", "prettyplease", - "prost 0.11.6", - "prost-types 0.11.6", + "prost", + "prost-types", "regex", "syn", "tempfile", "which", ] -[[package]] -name = "prost-derive" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "prost-derive" version = "0.11.6" @@ -5426,16 +5391,6 @@ dependencies = [ "syn", ] -[[package]] -name = "prost-types" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" -dependencies = [ - "bytes", - "prost 0.9.0", -] - [[package]] name = "prost-types" version = "0.11.6" @@ -5443,7 +5398,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5e0526209433e96d83d750dd81a99118edbc55739e7e61a46764fd2ad537788" dependencies = [ "bytes", - "prost 0.11.6", + "prost", ] [[package]] @@ -5806,6 +5761,15 @@ version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +[[package]] +name = "regress" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a92ff21fe8026ce3f2627faaf43606f0b67b014dbc9ccf027181a804f75d92e" +dependencies = [ + "memchr", +] + [[package]] name = "remove_dir_all" version = "0.5.3" @@ -6062,6 +6026,19 @@ dependencies = [ "semver 1.0.16", ] +[[package]] +name = "rustfmt-wrapper" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed729e3bee08ec2befd593c27e90ca9fdd25efdc83c94c3b82eaef16e4f7406e" +dependencies = [ + "serde", + "tempfile", + "thiserror", + "toml", + "toolchain_find", +] + [[package]] name = "rustix" version = "0.36.7" @@ -6757,6 +6734,17 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_tokenstream" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "274f512d6748a01e67cbcde5b4307ab2c9d52a98a2b870a980ef0793a351deff" +dependencies = [ + "proc-macro2", + "serde", + "syn", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -6769,6 +6757,19 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fb06d4b6cdaef0e0c51fa881acb721bed3c924cfaa71d9c94a3b771dfdf6567" +dependencies = [ + "indexmap", + "itoa 1.0.5", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "servers" version = "0.1.0" @@ -6814,7 +6815,7 @@ dependencies = [ "pin-project", "postgres-types", "promql-parser", - "prost 0.11.6", + "prost", "query", "rand 0.8.5", "regex", @@ -7206,7 +7207,7 @@ dependencies = [ "parquet", "paste", "planus", - "prost 0.11.6", + "prost", "rand 0.8.5", "regex", "serde", @@ -7368,23 +7369,29 @@ dependencies = [ "datafusion-expr", "datatypes", "futures", - "prost 0.9.0", + "prost", "snafu", - "substrait 0.2.0", + "substrait 0.4.0", "table", "tokio", ] [[package]] name = "substrait" -version = "0.2.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46079e9004f5e069eae2976d4e23ea29c4e215b1096d3d53b76b19879f346100" +checksum = "e2feb96a6a106e21161551af32dc4e0fdab3aceb926b940d7e92a086b640fc7c" dependencies = [ - "glob", - "prost 0.9.0", - "prost-build 0.9.0", - "prost-types 0.9.0", + "heck 0.4.0", + "prost", + "prost-build", + "prost-types", + "schemars", + "serde", + "serde_json", + "serde_yaml", + "typify", + "walkdir", ] [[package]] @@ -7945,8 +7952,8 @@ dependencies = [ "hyper-timeout", "percent-encoding", "pin-project", - "prost 0.11.6", - "prost-derive 0.11.6", + "prost", + "prost-derive", "rustls-pemfile", "tokio", "tokio-rustls", @@ -7967,11 +7974,24 @@ checksum = "5bf5e9b9c0f7e0a7c027dcfaba7b2c60816c7049171f679d99ee2ff65d0de8c4" dependencies = [ "prettyplease", "proc-macro2", - "prost-build 0.11.3", + "prost-build", "quote", "syn", ] +[[package]] +name = "toolchain_find" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e85654a10e7a07a47c6f19d93818f3f343e22927f2fa280c84f7c8042743413" +dependencies = [ + "home", + "lazy_static", + "regex", + "semver 0.11.0", + "walkdir", +] + [[package]] name = "tower" version = "0.4.13" @@ -8191,6 +8211,51 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +[[package]] +name = "typify" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e8486352f3c946e69f983558cfc09b295250b01e01b381ec67a05a812d01d63" +dependencies = [ + "typify-impl", + "typify-macro", +] + +[[package]] +name = "typify-impl" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7624d0b911df6e2bbf34a236f76281f93b294cdde1d4df1dbdb748e5a7fefa5" +dependencies = [ + "heck 0.4.0", + "log", + "proc-macro2", + "quote", + "regress", + "rustfmt-wrapper", + "schemars", + "serde_json", + "syn", + "thiserror", + "unicode-ident", +] + +[[package]] +name = "typify-macro" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c42802aa033cee7650a4e1509ba7d5848a56f84be7c4b31e4385ee12445e942" +dependencies = [ + "proc-macro2", + "quote", + "schemars", + "serde", + "serde_json", + "serde_tokenstream", + "syn", + "typify-impl", +] + [[package]] name = "ucd-trie" version = "0.1.5" @@ -8391,6 +8456,12 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "029df4cc8238cefc911704ff8fa210853a0f3bce2694d8f51181dd41ee0f3301" +[[package]] +name = "unsafe-libyaml" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7ed8ba44ca06be78ea1ad2c3682a43349126c8818054231ee6f4748012aed2" + [[package]] name = "untrusted" version = "0.7.1" diff --git a/src/client/Cargo.toml b/src/client/Cargo.toml index bf56b1ead3..9f25e299ef 100644 --- a/src/client/Cargo.toml +++ b/src/client/Cargo.toml @@ -32,12 +32,8 @@ substrait = { path = "../common/substrait" } tokio.workspace = true tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } - -# TODO(ruihang): upgrade to 0.11 once substrait-rs supports it. -[dev-dependencies.prost_09] -package = "prost" -version = "0.9" +prost.workspace = true [dev-dependencies.substrait_proto] package = "substrait" -version = "0.2" +version = "0.4" diff --git a/src/client/examples/logical.rs b/src/client/examples/logical.rs index e3cd1f7747..291097055d 100644 --- a/src/client/examples/logical.rs +++ b/src/client/examples/logical.rs @@ -15,11 +15,11 @@ use api::v1::{ColumnDataType, ColumnDef, CreateTableExpr, TableId}; use client::{Client, Database}; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; -use prost_09::Message; -use substrait_proto::protobuf::plan_rel::RelType as PlanRelType; -use substrait_proto::protobuf::read_rel::{NamedTable, ReadType}; -use substrait_proto::protobuf::rel::RelType; -use substrait_proto::protobuf::{PlanRel, ReadRel, Rel}; +use prost::Message; +use substrait_proto::proto::plan_rel::RelType as PlanRelType; +use substrait_proto::proto::read_rel::{NamedTable, ReadType}; +use substrait_proto::proto::rel::RelType; +use substrait_proto::proto::{PlanRel, ReadRel, Rel}; use tracing::{event, Level}; fn main() { @@ -89,12 +89,8 @@ fn mock_logical_plan() -> Vec { let read_type = ReadType::NamedTable(named_table); let read_rel = ReadRel { - common: None, - base_schema: None, - filter: None, - projection: None, - advanced_extension: None, read_type: Some(read_type), + ..Default::default() }; let mut buf = vec![]; diff --git a/src/common/substrait/Cargo.toml b/src/common/substrait/Cargo.toml index 5681022d3e..4b019591d5 100644 --- a/src/common/substrait/Cargo.toml +++ b/src/common/substrait/Cargo.toml @@ -14,13 +14,13 @@ datafusion.workspace = true datafusion-expr.workspace = true datatypes = { path = "../../datatypes" } futures = "0.3" -prost = "0.9" +prost.workspace = true snafu.workspace = true table = { path = "../../table" } [dependencies.substrait_proto] package = "substrait" -version = "0.2" +version = "0.4" [dev-dependencies] datatypes = { path = "../../datatypes" } diff --git a/src/common/substrait/src/context.rs b/src/common/substrait/src/context.rs index fb7e90cfff..9258d4088e 100644 --- a/src/common/substrait/src/context.rs +++ b/src/common/substrait/src/context.rs @@ -15,10 +15,10 @@ use std::collections::HashMap; use datafusion::common::DFSchemaRef; -use substrait_proto::protobuf::extensions::simple_extension_declaration::{ +use substrait_proto::proto::extensions::simple_extension_declaration::{ ExtensionFunction, MappingType, }; -use substrait_proto::protobuf::extensions::SimpleExtensionDeclaration; +use substrait_proto::proto::extensions::SimpleExtensionDeclaration; #[derive(Default)] pub struct ConvertorContext { diff --git a/src/common/substrait/src/df_expr.rs b/src/common/substrait/src/df_expr.rs index 148bafc3e1..bca8b974b4 100644 --- a/src/common/substrait/src/df_expr.rs +++ b/src/common/substrait/src/df_expr.rs @@ -20,15 +20,15 @@ use datafusion_expr::expr::Sort; use datafusion_expr::{expr_fn, lit, Between, BinaryExpr, BuiltinScalarFunction, Expr, Operator}; use datatypes::schema::Schema; use snafu::{ensure, OptionExt}; -use substrait_proto::protobuf::expression::field_reference::ReferenceType as FieldReferenceType; -use substrait_proto::protobuf::expression::reference_segment::{ +use substrait_proto::proto::expression::field_reference::ReferenceType as FieldReferenceType; +use substrait_proto::proto::expression::reference_segment::{ ReferenceType as SegReferenceType, StructField, }; -use substrait_proto::protobuf::expression::{ +use substrait_proto::proto::expression::{ FieldReference, Literal, ReferenceSegment, RexType, ScalarFunction, }; -use substrait_proto::protobuf::function_argument::ArgType; -use substrait_proto::protobuf::Expression; +use substrait_proto::proto::function_argument::ArgType; +use substrait_proto::proto::Expression; use crate::context::ConvertorContext; use crate::error::{ @@ -61,6 +61,7 @@ pub(crate) fn to_df_expr( | RexType::MultiOrList(_) | RexType::Cast(_) | RexType::Subquery(_) + | RexType::Nested(_) | RexType::Enum(_) => UnsupportedExprSnafu { name: format!("substrait expression {expr_rex_type:?}"), } @@ -615,9 +616,9 @@ pub fn convert_column(column: &Column, schema: &Schema) -> Result &str { match op { diff --git a/src/common/substrait/src/df_logical.rs b/src/common/substrait/src/df_logical.rs index 3357bc2075..0687959c8b 100644 --- a/src/common/substrait/src/df_logical.rs +++ b/src/common/substrait/src/df_logical.rs @@ -25,13 +25,13 @@ use datafusion::physical_plan::project_schema; use datafusion_expr::{Filter, LogicalPlan, TableScan, TableSource}; use prost::Message; use snafu::{ensure, OptionExt, ResultExt}; -use substrait_proto::protobuf::expression::mask_expression::{StructItem, StructSelect}; -use substrait_proto::protobuf::expression::MaskExpression; -use substrait_proto::protobuf::extensions::simple_extension_declaration::MappingType; -use substrait_proto::protobuf::plan_rel::RelType as PlanRelType; -use substrait_proto::protobuf::read_rel::{NamedTable, ReadType}; -use substrait_proto::protobuf::rel::RelType; -use substrait_proto::protobuf::{FilterRel, Plan, PlanRel, ReadRel, Rel}; +use substrait_proto::proto::expression::mask_expression::{StructItem, StructSelect}; +use substrait_proto::proto::expression::MaskExpression; +use substrait_proto::proto::extensions::simple_extension_declaration::MappingType; +use substrait_proto::proto::plan_rel::RelType as PlanRelType; +use substrait_proto::proto::read_rel::{NamedTable, ReadType}; +use substrait_proto::proto::rel::RelType; +use substrait_proto::proto::{FilterRel, Plan, PlanRel, ReadRel, Rel}; use table::table::adapter::DfTableProviderAdapter; use crate::context::ConvertorContext; @@ -424,6 +424,7 @@ impl DFLogicalSubstraitConvertor { relations: vec![plan_rel], advanced_extensions: None, expected_type_urls: vec![], + ..Default::default() }) } @@ -485,6 +486,7 @@ impl DFLogicalSubstraitConvertor { projection, advanced_extension: None, read_type: Some(read_type), + ..Default::default() }; Ok(read_rel) diff --git a/src/common/substrait/src/schema.rs b/src/common/substrait/src/schema.rs index 0235263507..a9611baec8 100644 --- a/src/common/substrait/src/schema.rs +++ b/src/common/substrait/src/schema.rs @@ -13,8 +13,8 @@ // limitations under the License. use datatypes::schema::{ColumnSchema, Schema}; -use substrait_proto::protobuf::r#type::{Nullability, Struct as SubstraitStruct}; -use substrait_proto::protobuf::NamedStruct; +use substrait_proto::proto::r#type::{Nullability, Struct as SubstraitStruct}; +use substrait_proto::proto::NamedStruct; use crate::error::Result; use crate::types::{from_concrete_type, to_concrete_type}; diff --git a/src/common/substrait/src/types.rs b/src/common/substrait/src/types.rs index 35d7a5e39b..0652e30e8e 100644 --- a/src/common/substrait/src/types.rs +++ b/src/common/substrait/src/types.rs @@ -20,9 +20,9 @@ use datafusion::scalar::ScalarValue; use datatypes::prelude::ConcreteDataType; -use substrait_proto::protobuf::expression::literal::LiteralType; -use substrait_proto::protobuf::r#type::{self as s_type, Kind, Nullability}; -use substrait_proto::protobuf::{Type as SType, Type}; +use substrait_proto::proto::expression::literal::LiteralType; +use substrait_proto::proto::r#type::{self as s_type, Kind, Nullability}; +use substrait_proto::proto::{Type as SType, Type}; use crate::error::{self, Result, UnsupportedConcreteTypeSnafu, UnsupportedSubstraitTypeSnafu}; @@ -86,6 +86,7 @@ pub fn to_concrete_type(ty: &SType) -> Result<(ConcreteDataType, bool)> { | Kind::Struct(_) | Kind::List(_) | Kind::Map(_) + | Kind::UserDefined(_) | Kind::UserDefinedTypeReference(_) => UnsupportedSubstraitTypeSnafu { ty: format!("{kind:?}"), }