chore: upgrade to Arrow 29.0 and use workspace package and dependencies (#782)

* chore: upgrade to Arrow 29.0 and use workspace package and dependencies

* fix: resolve PR comments

Co-authored-by: luofucong <luofucong@greptime.com>
This commit is contained in:
LFC
2022-12-23 14:28:37 +08:00
committed by GitHub
parent 26af9e6214
commit dc52a51576
48 changed files with 296 additions and 296 deletions

143
Cargo.lock generated
View File

@@ -194,9 +194,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
[[package]]
name = "arrow"
version = "28.0.0"
version = "29.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aed9849f86164fad5cb66ce4732782b15f1bc97f8febab04e782c20cce9d4b6c"
checksum = "2fe17dc0113da7e2eaeaedbd304d347aa8ea64916d225b79a5c3f3b6b5d8da4c"
dependencies = [
"ahash 0.8.2",
"arrow-array",
@@ -206,8 +206,10 @@ dependencies = [
"arrow-data",
"arrow-ipc",
"arrow-json",
"arrow-ord",
"arrow-schema",
"arrow-select",
"arrow-string",
"chrono",
"comfy-table",
"half 2.1.0",
@@ -215,14 +217,13 @@ dependencies = [
"multiversion",
"num",
"regex",
"regex-syntax",
]
[[package]]
name = "arrow-array"
version = "28.0.0"
version = "29.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b8504cf0a6797e908eecf221a865e7d339892720587f87c8b90262863015b08"
checksum = "b9452131e027aec3276e43449162af084db611c42ef875e54d231e6580bc6254"
dependencies = [
"ahash 0.8.2",
"arrow-buffer",
@@ -236,9 +237,9 @@ dependencies = [
[[package]]
name = "arrow-buffer"
version = "28.0.0"
version = "29.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6de64a27cea684b24784647d9608314bc80f7c4d55acb44a425e05fab39d916"
checksum = "4a301001e8ed7da638a12fa579ac5f3f154c44c0655f2ca6ed0f8586b418a779"
dependencies = [
"half 2.1.0",
"num",
@@ -246,9 +247,9 @@ dependencies = [
[[package]]
name = "arrow-cast"
version = "28.0.0"
version = "29.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bec4a54502eefe05923c385c90a005d69474fa06ca7aa2a2b123c9f9532f6178"
checksum = "048c91d067f2eb8cc327f086773e5b0f0d7714780807fc4db09366584e23bac8"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -262,9 +263,9 @@ dependencies = [
[[package]]
name = "arrow-csv"
version = "28.0.0"
version = "29.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7902bbf8127eac48554fe902775303377047ad49a9fd473c2b8cb399d092080"
checksum = "ed914cd0006a3bb9cac8136b3098ac7796ad26b82362f00d4f2e7c1a54684b86"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -280,9 +281,9 @@ dependencies = [
[[package]]
name = "arrow-data"
version = "28.0.0"
version = "29.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e4882efe617002449d5c6b5de9ddb632339074b36df8a96ea7147072f1faa8a"
checksum = "e59619d9d102e4e6b22087b2bd60c07df76fcb68683620841718f6bc8e8f02cb"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -292,9 +293,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
version = "28.0.0"
version = "29.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa0703a6de2785828561b03a4d7793ecd333233e1b166316b4bfc7cfce55a4a7"
checksum = "fb7ad6d2fa06a1cebdaa213c59fc953b9230e560d8374aba133b572b864ec55e"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -306,9 +307,9 @@ dependencies = [
[[package]]
name = "arrow-json"
version = "28.0.0"
version = "29.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3bd23fc8c6d251f96cd63b96fece56bbb9710ce5874a627cb786e2600673595a"
checksum = "1e22efab3ad70336057660c5e5f2b72e2417e3444c27cb42dc477d678ddd6979"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -323,19 +324,33 @@ dependencies = [
]
[[package]]
name = "arrow-schema"
version = "28.0.0"
name = "arrow-ord"
version = "29.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da9f143882a80be168538a60e298546314f50f11f2a288c8d73e11108da39d26"
checksum = "e23b623332804a65ad11e7732c351896dcb132c19f8e25d99fdb13b00aae5206"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-select",
"num",
]
[[package]]
name = "arrow-schema"
version = "29.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69ef17c144f1253b9864f5a3e8f4c6f1e436bdd52394855d5942f132f776b64e"
dependencies = [
"serde",
]
[[package]]
name = "arrow-select"
version = "28.0.0"
version = "29.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "520406331d4ad60075359524947ebd804e479816439af82bcb17f8d280d9b38c"
checksum = "e2accaf218ff107e3df0ee8f1e09b092249a1cc741c4377858a1470fd27d7096"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -344,6 +359,21 @@ dependencies = [
"num",
]
[[package]]
name = "arrow-string"
version = "29.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4a0954f9e1f45b04815ddacbde72899bf3c03a08fa6c0375f42178c4a01a510"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-select",
"regex",
"regex-syntax",
]
[[package]]
name = "ascii"
version = "1.1.0"
@@ -1384,7 +1414,7 @@ dependencies = [
"common-function-macro",
"common-query",
"common-time",
"datafusion-common",
"datafusion",
"datatypes",
"libc",
"num",
@@ -1969,8 +1999,7 @@ dependencies = [
[[package]]
name = "datafusion"
version = "15.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b75a088adf79515b04fd3895c1a14dc249c8f7a7f27b59870a05546fe9a55542"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=4917235a398ae20145c87d20984e6367dc1a0c1e#4917235a398ae20145c87d20984e6367dc1a0c1e"
dependencies = [
"ahash 0.8.2",
"arrow",
@@ -2002,7 +2031,6 @@ dependencies = [
"pin-project-lite",
"rand 0.8.5",
"smallvec",
"sqllogictest",
"sqlparser",
"tempfile",
"tokio",
@@ -2016,8 +2044,7 @@ dependencies = [
[[package]]
name = "datafusion-common"
version = "15.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b17262b899f79afdf502846d1138a8b48441afe24dc6e07c922105289248137"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=4917235a398ae20145c87d20984e6367dc1a0c1e#4917235a398ae20145c87d20984e6367dc1a0c1e"
dependencies = [
"arrow",
"chrono",
@@ -2029,8 +2056,7 @@ dependencies = [
[[package]]
name = "datafusion-expr"
version = "15.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "533d2226b4636a1306d1f6f4ac02e436947c5d6e8bfc85f6d8f91a425c10a407"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=4917235a398ae20145c87d20984e6367dc1a0c1e#4917235a398ae20145c87d20984e6367dc1a0c1e"
dependencies = [
"ahash 0.8.2",
"arrow",
@@ -2042,8 +2068,7 @@ dependencies = [
[[package]]
name = "datafusion-optimizer"
version = "15.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce7ba274267b6baf1714a67727249aa56d648c8814b0f4c43387fbe6d147e619"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=4917235a398ae20145c87d20984e6367dc1a0c1e#4917235a398ae20145c87d20984e6367dc1a0c1e"
dependencies = [
"arrow",
"async-trait",
@@ -2058,8 +2083,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
version = "15.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f35cb53e6c2f9c40accdf45aef2be7fde030ea3051b1145a059d96109e65b0bf"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=4917235a398ae20145c87d20984e6367dc1a0c1e#4917235a398ae20145c87d20984e6367dc1a0c1e"
dependencies = [
"ahash 0.8.2",
"arrow",
@@ -2088,8 +2112,7 @@ dependencies = [
[[package]]
name = "datafusion-row"
version = "15.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27c77b1229ae5cf6a6e0e2ba43ed4e98131dbf1cc4a97fad17c94230b32e0812"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=4917235a398ae20145c87d20984e6367dc1a0c1e#4917235a398ae20145c87d20984e6367dc1a0c1e"
dependencies = [
"arrow",
"datafusion-common",
@@ -2100,12 +2123,12 @@ dependencies = [
[[package]]
name = "datafusion-sql"
version = "15.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "569423fa8a50db39717080949e3b4f8763582b87baf393cc3fcf27cc21467ba7"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=4917235a398ae20145c87d20984e6367dc1a0c1e#4917235a398ae20145c87d20984e6367dc1a0c1e"
dependencies = [
"arrow-schema",
"datafusion-common",
"datafusion-expr",
"log",
"sqlparser",
]
@@ -2230,12 +2253,6 @@ version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8"
[[package]]
name = "difference"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198"
[[package]]
name = "digest"
version = "0.10.6"
@@ -3452,17 +3469,6 @@ version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb"
[[package]]
name = "libtest-mimic"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7b603516767d1ab23d0de09d023e62966c3322f7148297c35cf3d97aa8b37fa"
dependencies = [
"clap 4.0.29",
"termcolor",
"threadpool",
]
[[package]]
name = "libz-sys"
version = "1.1.8"
@@ -4540,9 +4546,9 @@ dependencies = [
[[package]]
name = "parquet"
version = "28.0.0"
version = "29.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21433e9209111bb3720b747f2f137e0d115af1af0420a7a1c26b6e88227fa353"
checksum = "d906343fd18ace6b998d5074697743e8e9358efa8c3c796a1381b98cba813338"
dependencies = [
"ahash 0.8.2",
"arrow-array",
@@ -6689,25 +6695,6 @@ dependencies = [
"sqlparser",
]
[[package]]
name = "sqllogictest"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba41e01d229d7725401de371e323851f82d839d68732a06162405362b60852fe"
dependencies = [
"async-trait",
"difference",
"futures",
"glob",
"humantime",
"itertools",
"libtest-mimic",
"regex",
"tempfile",
"thiserror",
"tracing",
]
[[package]]
name = "sqlness"
version = "0.1.0"
@@ -6737,9 +6724,9 @@ dependencies = [
[[package]]
name = "sqlparser"
version = "0.27.0"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aba319938d4bfe250a769ac88278b629701024fe16f34257f9563bc628081970"
checksum = "249ae674b9f636b8ff64d8bfe218774cf05a26de40fd9f358669dccc4c0a9d7d"
dependencies = [
"log",
]

View File

@@ -39,5 +39,23 @@ members = [
"tests/runner",
]
[workspace.package]
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
[workspace.dependencies]
arrow = "29.0"
arrow-schema = { version = "29.0", features = ["serde"] }
# TODO(LFC): Use released Datafusion when it officially dpendent on Arrow 29.0
datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "4917235a398ae20145c87d20984e6367dc1a0c1e" }
datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "4917235a398ae20145c87d20984e6367dc1a0c1e" }
datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "4917235a398ae20145c87d20984e6367dc1a0c1e" }
datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "4917235a398ae20145c87d20984e6367dc1a0c1e" }
datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "4917235a398ae20145c87d20984e6367dc1a0c1e" }
datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "4917235a398ae20145c87d20984e6367dc1a0c1e" }
parquet = "29.0"
sqlparser = "0.28"
[profile.release]
debug = true

View File

@@ -1,14 +1,14 @@
[package]
name = "benchmarks"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
arrow = "28.0"
arrow.workspace = true
clap = { version = "4.0", features = ["derive"] }
client = { path = "../src/client" }
indicatif = "0.17.1"
itertools = "0.10.5"
parquet = "28.0"
parquet.workspace = true
tokio = { version = "1.21", features = ["full"] }

View File

@@ -1,9 +1,8 @@
[package]
name = "api"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
common-base = { path = "../common/base" }

View File

@@ -1,9 +1,8 @@
[package]
name = "catalog"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
api = { path = "../api" }
@@ -19,7 +18,7 @@ common-recordbatch = { path = "../common/recordbatch" }
common-runtime = { path = "../common/runtime" }
common-telemetry = { path = "../common/telemetry" }
common-time = { path = "../common/time" }
datafusion = "15.0"
datafusion.workspace = true
datatypes = { path = "../datatypes" }
futures = "0.3"
futures-util = "0.3"

View File

@@ -1,9 +1,8 @@
[package]
name = "client"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
api = { path = "../api" }
@@ -15,7 +14,7 @@ common-grpc-expr = { path = "../common/grpc-expr" }
common-query = { path = "../common/query" }
common-recordbatch = { path = "../common/recordbatch" }
common-time = { path = "../common/time" }
datafusion = "15.0"
datafusion.workspace = true
datatypes = { path = "../datatypes" }
enum_dispatch = "0.3"
parking_lot = "0.12"

View File

@@ -1,9 +1,9 @@
[package]
name = "cmd"
version = "0.1.0"
edition = "2021"
version.workspace = true
edition.workspace = true
license.workspace = true
default-run = "greptime"
license = "Apache-2.0"
[[bin]]
name = "greptime"

View File

@@ -1,8 +1,8 @@
[package]
name = "common-base"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
bitvec = "1.0"

View File

@@ -1,8 +1,8 @@
[package]
name = "common-catalog"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
async-trait = "0.1"

View File

@@ -1,8 +1,8 @@
[package]
name = "common-error"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
snafu = { version = "0.7", features = ["backtraces"] }

View File

@@ -1,8 +1,8 @@
[package]
name = "common-function-macro"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[lib]
proc-macro = true

View File

@@ -1,8 +1,8 @@
[package]
edition = "2021"
name = "common-function"
version = "0.1.0"
license = "Apache-2.0"
edition.workspace = true
version.workspace = true
license.workspace = true
[dependencies]
arc-swap = "1.0"
@@ -11,7 +11,7 @@ common-error = { path = "../error" }
common-function-macro = { path = "../function-macro" }
common-query = { path = "../query" }
common-time = { path = "../time" }
datafusion-common = "15.0"
datafusion.workspace = true
datatypes = { path = "../../datatypes" }
libc = "0.2"
num = "0.4"

View File

@@ -1,8 +1,8 @@
[package]
name = "common-grpc-expr"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
api = { path = "../../api" }

View File

@@ -1,8 +1,8 @@
[package]
name = "common-grpc"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
api = { path = "../../api" }
@@ -13,7 +13,7 @@ common-query = { path = "../query" }
common-recordbatch = { path = "../recordbatch" }
common-runtime = { path = "../runtime" }
dashmap = "5.4"
datafusion = "15.0"
datafusion.workspace = true
datatypes = { path = "../../datatypes" }
snafu = { version = "0.7", features = ["backtraces"] }
tokio = { version = "1.0", features = ["full"] }

View File

@@ -1,17 +1,17 @@
[package]
name = "common-query"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
async-trait = "0.1"
common-error = { path = "../error" }
common-recordbatch = { path = "../recordbatch" }
common-time = { path = "../time" }
datafusion = "15.0"
datafusion-common = "15.0"
datafusion-expr = "15.0"
datafusion.workspace = true
datafusion-common.workspace = true
datafusion-expr.workspace = true
datatypes = { path = "../../datatypes" }
snafu = { version = "0.7", features = ["backtraces"] }
statrs = "0.15"

View File

@@ -18,7 +18,7 @@ use std::fmt::Debug;
use std::sync::Arc;
use datafusion_common::Result as DfResult;
use datafusion_expr::{Accumulator as DfAccumulator, AggregateState};
use datafusion_expr::Accumulator as DfAccumulator;
use datatypes::arrow::array::ArrayRef;
use datatypes::prelude::*;
use datatypes::vectors::{Helper as VectorHelper, VectorRef};
@@ -126,7 +126,7 @@ impl DfAccumulatorAdaptor {
}
impl DfAccumulator for DfAccumulatorAdaptor {
fn state(&self) -> DfResult<Vec<AggregateState>> {
fn state(&self) -> DfResult<Vec<ScalarValue>> {
let state_values = self.accumulator.state()?;
let state_types = self.creator.state_types()?;
if state_values.len() != state_types.len() {
@@ -138,12 +138,7 @@ impl DfAccumulator for DfAccumulatorAdaptor {
Ok(state_values
.into_iter()
.zip(state_types.iter())
.map(|(v, t)| {
let scalar = v
.try_to_scalar_value(t)
.context(error::ToScalarValueSnafu)?;
Ok(AggregateState::Scalar(scalar))
})
.map(|(v, t)| v.try_to_scalar_value(t).context(error::ToScalarValueSnafu))
.collect::<Result<Vec<_>>>()?)
}

View File

@@ -1,13 +1,13 @@
[package]
name = "common-recordbatch"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
common-error = { path = "../error" }
datafusion = "15.0"
datafusion-common = "15.0"
datafusion.workspace = true
datafusion-common.workspace = true
datatypes = { path = "../../datatypes" }
futures = "0.3"
paste = "1.0"

View File

@@ -1,8 +1,8 @@
[package]
name = "common-runtime"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
common-error = { path = "../error" }

View File

@@ -1,8 +1,8 @@
[package]
name = "substrait"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
bytes = "1.1"
@@ -10,8 +10,8 @@ catalog = { path = "../../catalog" }
common-catalog = { path = "../catalog" }
common-error = { path = "../error" }
common-telemetry = { path = "../telemetry" }
datafusion = "15.0"
datafusion-expr = "15.0"
datafusion.workspace = true
datafusion-expr.workspace = true
datatypes = { path = "../../datatypes" }
futures = "0.3"
prost = "0.9"

View File

@@ -16,6 +16,7 @@ use std::collections::VecDeque;
use std::str::FromStr;
use datafusion::common::Column;
use datafusion_expr::expr::Sort;
use datafusion_expr::{expr_fn, lit, Between, BinaryExpr, BuiltinScalarFunction, Expr, Operator};
use datatypes::schema::Schema;
use snafu::{ensure, OptionExt};
@@ -331,19 +332,19 @@ pub fn convert_scalar_function(
// skip Cast and TryCast, is covered in substrait::Cast.
"sort" | "sort_des" => {
ensure_arg_len(1)?;
Expr::Sort {
Expr::Sort(Sort {
expr: Box::new(inputs.pop_front().unwrap()),
asc: false,
nulls_first: false,
}
})
}
"sort_asc" => {
ensure_arg_len(1)?;
Expr::Sort {
Expr::Sort(Sort {
expr: Box::new(inputs.pop_front().unwrap()),
asc: true,
nulls_first: false,
}
})
}
// those are datafusion built-in "scalar functions".
"abs"
@@ -537,11 +538,11 @@ pub fn expression_from_df_expr(
name: expr.to_string(),
}
.fail()?,
Expr::Sort {
Expr::Sort(Sort {
expr,
asc,
nulls_first: _,
} => {
}) => {
let expr = expression_from_df_expr(ctx, expr, schema)?;
let arguments = utils::expression_to_argument(vec![expr]);
let op_name = if *asc { "sort_asc" } else { "sort_des" };
@@ -577,6 +578,7 @@ pub fn expression_from_df_expr(
| Expr::Exists { .. }
| Expr::InSubquery { .. }
| Expr::ScalarSubquery(..)
| Expr::Placeholder { .. }
| Expr::QualifiedWildcard { .. } => todo!(),
Expr::GroupingSet(_) => UnsupportedExprSnafu {
name: expr.to_string(),
@@ -646,6 +648,8 @@ mod utils {
Operator::BitwiseShiftRight => "bitwise_shift_right",
Operator::BitwiseShiftLeft => "bitwise_shift_left",
Operator::StringConcat => "string_concat",
Operator::ILike => "i_like",
Operator::NotILike => "not_i_like",
}
}

View File

@@ -395,7 +395,8 @@ impl DFLogicalSubstraitConvertor {
| LogicalPlan::Values(_)
| LogicalPlan::Explain(_)
| LogicalPlan::Analyze(_)
| LogicalPlan::Extension(_) => InvalidParametersSnafu {
| LogicalPlan::Extension(_)
| LogicalPlan::Prepare(_) => InvalidParametersSnafu {
reason: format!(
"Trying to convert DDL/DML plan to substrait proto, plan: {plan:?}",
),

View File

@@ -1,8 +1,8 @@
[package]
name = "common-telemetry"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[features]
console = ["console-subscriber"]

View File

@@ -1,8 +1,8 @@
[package]
name = "common-time"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
chrono = "0.4"

View File

@@ -1,8 +1,8 @@
[package]
name = "datanode"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[features]
default = ["python"]
@@ -25,7 +25,7 @@ common-recordbatch = { path = "../common/recordbatch" }
common-runtime = { path = "../common/runtime" }
common-telemetry = { path = "../common/telemetry" }
common-time = { path = "../common/time" }
datafusion = "15.0"
datafusion.workspace = true
datatypes = { path = "../datatypes" }
futures = "0.3"
hyper = { version = "0.14", features = ["full"] }
@@ -57,5 +57,5 @@ tower-http = { version = "0.3", features = ["full"] }
axum-test-helper = { git = "https://github.com/sunng87/axum-test-helper.git", branch = "patch-1" }
client = { path = "../client" }
common-query = { path = "../common/query" }
datafusion-common = "15.0"
datafusion-common.workspace = true
tempdir = "0.3"

View File

@@ -1,20 +1,20 @@
[package]
name = "datatypes"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[features]
default = []
test = []
[dependencies]
arrow = "28.0"
arrow-schema = { version = "28.0", features = ["serde"] }
arrow.workspace = true
arrow-schema.workspace = true
common-base = { path = "../common/base" }
common-error = { path = "../common/error" }
common-time = { path = "../common/time" }
datafusion-common = "15.0"
datafusion-common.workspace = true
enum_dispatch = "0.3"
num = "0.4"
num-traits = "0.2"

View File

@@ -1,8 +1,8 @@
[package]
name = "frontend"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
anymap = "1.0.0-beta.2"
@@ -22,9 +22,9 @@ common-recordbatch = { path = "../common/recordbatch" }
common-runtime = { path = "../common/runtime" }
common-telemetry = { path = "../common/telemetry" }
common-time = { path = "../common/time" }
datafusion = "15.0"
datafusion-common = "15.0"
datafusion-expr = "15.0"
datafusion.workspace = true
datafusion-common.workspace = true
datafusion-expr.workspace = true
datanode = { path = "../datanode" }
datatypes = { path = "../datatypes" }
futures = "0.3"

View File

@@ -105,9 +105,11 @@ impl DatanodeInstance {
.context(error::BuildDfLogicalPlanSnafu)?;
}
builder
.limit(0, table_scan.limit)
.context(error::BuildDfLogicalPlanSnafu)?;
if table_scan.limit.is_some() {
builder = builder
.limit(0, table_scan.limit)
.context(error::BuildDfLogicalPlanSnafu)?;
}
builder.build().context(error::BuildDfLogicalPlanSnafu)
}

View File

@@ -1,8 +1,8 @@
[package]
name = "log-store"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
arc-swap = "1.5"

View File

@@ -1,8 +1,8 @@
[package]
name = "meta-client"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
api = { path = "../api" }

View File

@@ -1,8 +1,8 @@
[package]
name = "meta-srv"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[features]
mock = []

View File

@@ -1,8 +1,8 @@
[package]
name = "mito"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[features]
default = []
@@ -19,8 +19,8 @@ common-query = { path = "../common/query" }
common-recordbatch = { path = "../common/recordbatch" }
common-telemetry = { path = "../common/telemetry" }
common-time = { path = "../common/time" }
datafusion = "15.0"
datafusion-common = "15.0"
datafusion.workspace = true
datafusion-common.workspace = true
datatypes = { path = "../datatypes" }
futures = "0.3"
log-store = { path = "../log-store" }

View File

@@ -1,8 +1,8 @@
[package]
name = "object-store"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
futures = { version = "0.3" }

View File

@@ -1,7 +1,8 @@
[package]
name = "promql"
version = "0.1.0"
edition = "2021"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
common-error = { path = "../common/error" }

View File

@@ -1,8 +1,8 @@
[package]
name = "query"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
arc-swap = "1.0"
@@ -15,12 +15,12 @@ common-query = { path = "../common/query" }
common-recordbatch = { path = "../common/recordbatch" }
common-telemetry = { path = "../common/telemetry" }
common-time = { path = "../common/time" }
datafusion = "15.0"
datafusion-common = "15.0"
datafusion-expr = "15.0"
datafusion-optimizer = "15.0"
datafusion-physical-expr = "15.0"
datafusion-sql = "15.0"
datafusion.workspace = true
datafusion-common.workspace = true
datafusion-expr.workspace = true
datafusion-optimizer.workspace = true
datafusion-physical-expr.workspace = true
datafusion-sql.workspace = true
datatypes = { path = "../datatypes" }
futures = "0.3"
futures-util = "0.3"

View File

@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::HashMap;
use std::sync::Arc;
use common_query::logical_plan::create_aggregate_function;
@@ -20,7 +19,7 @@ use datafusion::catalog::TableReference;
use datafusion::error::Result as DfResult;
use datafusion::physical_plan::udaf::AggregateUDF;
use datafusion::physical_plan::udf::ScalarUDF;
use datafusion::sql::planner::{ContextProvider, SqlToRel};
use datafusion::sql::planner::{ContextProvider, PlannerContext, SqlToRel};
use datafusion_common::ScalarValue;
use datafusion_expr::TableSource;
use datatypes::arrow::datatypes::DataType;
@@ -53,7 +52,7 @@ impl<'a, S: ContextProvider + Send + Sync> DfPlanner<'a, S> {
let sql = query.inner.to_string();
let result = self
.sql_to_rel
.query_to_plan(query.inner, &mut HashMap::new())
.query_to_plan(query.inner, &mut PlannerContext::default())
.context(error::PlanSqlSnafu { sql })?;
Ok(LogicalPlan::DfPlan(result))

View File

@@ -34,16 +34,24 @@ use datatypes::arrow::datatypes::DataType;
pub struct TypeConversionRule;
impl OptimizerRule for TypeConversionRule {
fn optimize(&self, plan: &LogicalPlan, _config: &mut OptimizerConfig) -> Result<LogicalPlan> {
fn try_optimize(
&self,
plan: &LogicalPlan,
_config: &dyn OptimizerConfig,
) -> Result<Option<LogicalPlan>> {
let mut converter = TypeConverter {
schemas: plan.all_schemas(),
};
match plan {
LogicalPlan::Filter(filter) => Ok(LogicalPlan::Filter(Filter::try_new(
filter.predicate().clone().rewrite(&mut converter)?,
Arc::new(self.optimize(filter.input(), _config)?),
)?)),
LogicalPlan::Filter(filter) => {
let rewritten = filter.predicate().clone().rewrite(&mut converter)?;
let Some(plan) = self.try_optimize(filter.input(), _config)? else { return Ok(None) };
Ok(Some(LogicalPlan::Filter(Filter::try_new(
rewritten,
Arc::new(plan),
)?)))
}
LogicalPlan::TableScan(TableScan {
table_name,
source,
@@ -57,14 +65,14 @@ impl OptimizerRule for TypeConversionRule {
.into_iter()
.map(|e| e.rewrite(&mut converter))
.collect::<Result<Vec<_>>>()?;
Ok(LogicalPlan::TableScan(TableScan {
Ok(Some(LogicalPlan::TableScan(TableScan {
table_name: table_name.clone(),
source: source.clone(),
projection: projection.clone(),
projected_schema: projected_schema.clone(),
filters: rewrite_filters,
fetch: *fetch,
}))
})))
}
LogicalPlan::Projection { .. }
| LogicalPlan::Window { .. }
@@ -86,10 +94,11 @@ impl OptimizerRule for TypeConversionRule {
| LogicalPlan::SetVariable { .. }
| LogicalPlan::Analyze { .. } => {
let inputs = plan.inputs();
let new_inputs = inputs
.iter()
.map(|plan| self.optimize(plan, _config))
.collect::<Result<Vec<_>>>()?;
let mut new_inputs = Vec::with_capacity(inputs.len());
for input in inputs {
let Some(plan) = self.try_optimize(input, _config)? else { return Ok(None) };
new_inputs.push(plan);
}
let expr = plan
.expressions()
@@ -97,7 +106,7 @@ impl OptimizerRule for TypeConversionRule {
.map(|e| e.rewrite(&mut converter))
.collect::<Result<Vec<_>>>()?;
datafusion_expr::utils::from_plan(plan, &expr, &new_inputs)
datafusion_expr::utils::from_plan(plan, &expr, &new_inputs).map(Some)
}
LogicalPlan::Subquery { .. }
@@ -105,7 +114,8 @@ impl OptimizerRule for TypeConversionRule {
| LogicalPlan::CreateView { .. }
| LogicalPlan::CreateCatalogSchema { .. }
| LogicalPlan::CreateCatalog { .. }
| LogicalPlan::EmptyRelation { .. } => Ok(plan.clone()),
| LogicalPlan::EmptyRelation(_)
| LogicalPlan::Prepare(_) => Ok(Some(plan.clone())),
}
}

View File

@@ -29,7 +29,7 @@ use datafusion::physical_plan::udf::ScalarUDF;
use datafusion::physical_plan::ExecutionPlan;
use datafusion_common::ScalarValue;
use datafusion_expr::{LogicalPlan as DfLogicalPlan, TableSource};
use datafusion_optimizer::optimizer::{Optimizer, OptimizerConfig};
use datafusion_optimizer::optimizer::Optimizer;
use datafusion_sql::planner::ContextProvider;
use datatypes::arrow::datatypes::DataType;
@@ -59,7 +59,7 @@ impl QueryEngineState {
let runtime_env = Arc::new(RuntimeEnv::default());
let session_config = SessionConfig::new()
.with_default_catalog_and_schema(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
let mut optimizer = Optimizer::new(&OptimizerConfig::new());
let mut optimizer = Optimizer::new();
// Apply the type conversion rule first.
optimizer.rules.insert(0, Arc::new(TypeConversionRule {}));
@@ -79,15 +79,7 @@ impl QueryEngineState {
/// Register a udf function
// TODO(dennis): manage UDFs by ourself.
pub fn register_udf(&self, udf: ScalarUdf) {
// `SessionContext` has a `register_udf()` method, which requires `&mut self`, this is
// a workaround.
// TODO(yingwen): Use `SessionContext::register_udf()` once it taks `&self`.
// It's implemented in https://github.com/apache/arrow-datafusion/pull/4612
self.df_context
.state
.write()
.scalar_functions
.insert(udf.name.clone(), Arc::new(udf.into_df_udf()));
self.df_context.register_udf(udf.into_df_udf());
}
pub fn aggregate_function(&self, function_name: &str) -> Option<AggregateFunctionMetaRef> {
@@ -122,32 +114,24 @@ impl QueryEngineState {
schema: Option<&str>,
name: TableReference,
) -> DfResult<Arc<dyn TableSource>> {
let state = self.df_context.state.read();
match name {
TableReference::Bare { table } if schema.is_some() => {
state.get_table_provider(TableReference::Partial {
// unwrap safety: checked in this match's arm
schema: schema.unwrap(),
table,
})
}
_ => state.get_table_provider(name),
}
let name = if let (Some(schema), TableReference::Bare { table }) = (schema, name) {
TableReference::Partial { schema, table }
} else {
name
};
self.df_context.state().get_table_provider(name)
}
pub(crate) fn get_function_meta(&self, name: &str) -> Option<Arc<ScalarUDF>> {
let state = self.df_context.state.read();
state.get_function_meta(name)
self.df_context.state().get_function_meta(name)
}
pub(crate) fn get_variable_type(&self, variable_names: &[String]) -> Option<DataType> {
let state = self.df_context.state.read();
state.get_variable_type(variable_names)
self.df_context.state().get_variable_type(variable_names)
}
pub(crate) fn get_config_option(&self, variable: &str) -> Option<ScalarValue> {
let state = self.df_context.state.read();
state.get_config_option(variable)
self.df_context.state().get_config_option(variable)
}
pub(crate) fn optimize(&self, plan: &DfLogicalPlan) -> DfResult<DfLogicalPlan> {
@@ -165,7 +149,7 @@ impl QueryEngineState {
&self,
mut plan: Arc<dyn ExecutionPlan>,
) -> DfResult<Arc<dyn ExecutionPlan>> {
let state = self.df_context.state.read();
let state = self.df_context.state();
let config = &state.config;
for optimizer in &state.physical_optimizers {
plan = optimizer.optimize(plan, config)?;

View File

@@ -1,8 +1,8 @@
[package]
edition = "2021"
name = "script"
version = "0.1.0"
license = "Apache-2.0"
edition.workspace = true
version.workspace = true
license.workspace = true
[features]
default = ["python"]
@@ -33,10 +33,10 @@ common-recordbatch = { path = "../common/recordbatch" }
common-telemetry = { path = "../common/telemetry" }
common-time = { path = "../common/time" }
console = "0.15"
datafusion = { version = "15.0", optional = true }
datafusion-common = { version = "15.0", optional = true }
datafusion-expr = { version = "15.0", optional = true }
datafusion-physical-expr = { version = "15.0", optional = true }
datafusion = { workspace = true, optional = true }
datafusion-common = { workspace = true, optional = true }
datafusion-expr = { workspace = true, optional = true }
datafusion-physical-expr = { workspace = true, optional = true }
datatypes = { path = "../datatypes" }
futures = "0.3"
futures-util = "0.3"

View File

@@ -1,8 +1,8 @@
[package]
name = "servers"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
aide = { version = "0.9", features = ["axum"] }

View File

@@ -1,8 +1,8 @@
[package]
name = "session"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
arc-swap = "1.5"

View File

@@ -1,8 +1,8 @@
[package]
name = "sql"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
api = { path = "../api" }
@@ -15,4 +15,4 @@ itertools = "0.10"
mito = { path = "../mito" }
once_cell = "1.10"
snafu = { version = "0.7", features = ["backtraces"] }
sqlparser = "0.27"
sqlparser.workspace = true

View File

@@ -386,7 +386,7 @@ mod tests {
use std::assert_matches::assert_matches;
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
use sqlparser::ast::{Query as SpQuery, Statement as SpStatement};
use sqlparser::ast::{Query as SpQuery, Statement as SpStatement, WildcardAdditionalOptions};
use sqlparser::dialect::GenericDialect;
use super::*;
@@ -533,7 +533,9 @@ mod tests {
let select = sqlparser::ast::Select {
distinct: false,
top: None,
projection: vec![sqlparser::ast::SelectItem::Wildcard],
projection: vec![sqlparser::ast::SelectItem::Wildcard(
WildcardAdditionalOptions::default(),
)],
into: None,
from: vec![sqlparser::ast::TableWithJoins {
relation: sqlparser::ast::TableFactor::Table {

View File

@@ -50,7 +50,7 @@ impl Insert {
pub fn values(&self) -> Result<Vec<Vec<Value>>> {
let values = match &self.inner {
Statement::Insert { source, .. } => match &*source.body {
SetExpr::Values(Values(exprs)) => sql_exprs_to_values(exprs)?,
SetExpr::Values(Values { rows, .. }) => sql_exprs_to_values(rows)?,
_ => unreachable!(),
},
_ => unreachable!(),

View File

@@ -1,8 +1,8 @@
[package]
name = "storage"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
arc-swap = "1.0"
@@ -21,7 +21,7 @@ futures = "0.3"
futures-util = "0.3"
lazy_static = "1.4"
object-store = { path = "../object-store" }
parquet = { version = "28.0", features = ["async"] }
parquet = { workspace = true, features = ["async"] }
paste = "1.0"
planus = "0.2"
prost = "0.11"

View File

@@ -1,8 +1,8 @@
[package]
name = "store-api"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
async-trait = "0.1"

View File

@@ -1,8 +1,8 @@
[package]
name = "table"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
async-trait = "0.1"
@@ -12,9 +12,9 @@ common-error = { path = "../common/error" }
common-query = { path = "../common/query" }
common-recordbatch = { path = "../common/recordbatch" }
common-telemetry = { path = "../common/telemetry" }
datafusion = "15.0"
datafusion-common = "15.0"
datafusion-expr = "15.0"
datafusion.workspace = true
datafusion-common.workspace = true
datafusion-expr.workspace = true
datatypes = { path = "../datatypes" }
derive_builder = "0.11"
futures = "0.3"
@@ -26,7 +26,6 @@ store-api = { path = "../store-api" }
tokio = { version = "1.18", features = ["full"] }
[dev-dependencies]
datafusion-expr = "15.0"
parquet = { version = "28.0", features = ["async"] }
parquet = { workspace = true, features = ["async"] }
tempdir = "0.3"
tokio-util = { version = "0.7", features = ["compat"] }

View File

@@ -1,8 +1,8 @@
[package]
name = "tests-integration"
version = "0.1.0"
edition = "2021"
license = "Apache-2.0"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
api = { path = "../src/api" }

View File

@@ -1,7 +1,8 @@
[package]
name = "sqlness-runner"
version = "0.1.0"
edition = "2021"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
async-trait = "0.1"