From 03c01893a495cba0fc2def8e213cd8c898e3521e Mon Sep 17 00:00:00 2001 From: Hengfei Yang Date: Tue, 21 Mar 2023 18:40:15 +0800 Subject: [PATCH] fix: full sql_mode allow without order by (#337) --- .cargo/config.toml | 4 +- .gitignore | 1 + Cargo.lock | 147 +++++++++++++++++--------- Cargo.toml | 18 ++-- src/job/files/disk.rs | 10 +- src/job/files/memory.rs | 10 +- src/service/search/datafusion/exec.rs | 16 ++- src/service/search/sql.rs | 2 +- 8 files changed, 131 insertions(+), 77 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 5d47cd1f77..04d005b1b4 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -2,7 +2,7 @@ rustflags = ["-C", "target-feature=+aes,+avx,+avx2,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2"] [target.x86_64-unknown-linux-musl] -rustflags = ["-C", "target-feature=+aes,+avx,+avx2,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2"] +rustflags = ["-C", "target-feature=+sse2,+sse3,+ssse3,+sse4.1,+sse4.2"] [target.aarch64-unknown-linux-gnu] linker = "aarch64-linux-gnu-gcc" @@ -15,4 +15,4 @@ rustflags = ["-C", "target-feature=+aes,+avx,+avx2,+sse2,+sse3,+ssse3,+sse4.1,+s rustflags = ["-C", "target-feature=+aes,+crc,+lse,+neon"] [target.x86_64-pc-windows-msvc] -rustflags = ["-C", "target-feature=+aes,+avx,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2"] +rustflags = ["-C", "target-feature=+sse2,+sse3,+ssse3,+sse4.1,+sse4.2"] diff --git a/.gitignore b/.gitignore index 34f16131ec..46de77ce8b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Commonly files .DS_Store +.history report.json # Generated by Cargo diff --git a/Cargo.lock b/Cargo.lock index 258008b793..87a5fff795 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,7 +70,7 @@ dependencies = [ "tokio", "tokio-util", "tracing", - "zstd", + "zstd 0.12.3+zstd.1.5.2", ] [[package]] @@ -371,9 +371,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "arrow" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3724c874f1517cf898cd1c3ad18ab5071edf893c48e73139ab1e16cf0f2affe" +checksum = "f410d3907b6b3647b9e7bca4551274b2e3d716aa940afb67b7287257401da921" dependencies = [ "ahash 0.8.3", "arrow-arith", @@ -394,9 +394,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e958823b8383ca14d0a2e973de478dd7674cd9f72837f8c41c132a0fda6a4e5e" +checksum = "f87391cf46473c9bc53dab68cb8872c3a81d4dfd1703f1c8aa397dba9880a043" dependencies = [ "arrow-array", "arrow-buffer", @@ -409,9 +409,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db670eab50e76654065b5aed930f4367101fcddcb2223802007d1e0b4d5a2579" +checksum = "d35d5475e65c57cffba06d0022e3006b677515f99b54af33a7cd54f6cdd4a5b5" dependencies = [ "ahash 0.8.3", "arrow-buffer", @@ -426,9 +426,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f0e01c931882448c0407bd32311a624b9f099739e94e786af68adc97016b5f2" +checksum = "68b4ec72eda7c0207727df96cf200f539749d736b21f3e782ece113e18c1a0a7" dependencies = [ "half", "num", @@ -436,9 +436,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bf35d78836c93f80d9362f3ccb47ff5e2c5ecfc270ff42cdf1ef80334961d44" +checksum = "0a7285272c9897321dfdba59de29f5b05aeafd3cdedf104a941256d155f6d304" dependencies = [ "arrow-array", "arrow-buffer", @@ -452,9 +452,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6aa7c2531d89d01fed8c469a9b1bf97132a0bdf70b4724fe4bbb4537a50880" +checksum = "981ee4e7f6a120da04e00d0b39182e1eeacccb59c8da74511de753c56b7fddf7" dependencies = [ "arrow-array", "arrow-buffer", @@ -471,9 +471,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea50db4d1e1e4c2da2bfdea7b6d2722eef64267d5ab680d815f7ae42428057f5" +checksum = "27cc673ee6989ea6e4b4e8c7d461f7e06026a096c8f0b1a7288885ff71ae1e56" dependencies = [ "arrow-buffer", "arrow-schema", @@ -483,9 +483,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4042fe6585155d1ec28a8e4937ec901a3ca7a19a22b9f6cd3f551b935cd84f5" +checksum = "e37b8b69d9e59116b6b538e8514e0ec63a30f08b617ce800d31cb44e3ef64c1a" dependencies = [ "arrow-array", "arrow-buffer", @@ -494,14 +494,14 @@ dependencies = [ "arrow-schema", "flatbuffers", "lz4", - "zstd", + "zstd 0.12.3+zstd.1.5.2", ] [[package]] name = "arrow-json" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c907c4ab4f26970a3719dc06e78e8054a01d0c96da3664d23b941e201b33d2b" +checksum = "80c3fa0bed7cfebf6d18e46b733f9cb8a1cb43ce8e6539055ca3e1e48a426266" dependencies = [ "arrow-array", "arrow-buffer", @@ -518,9 +518,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e131b447242a32129efc7932f58ed8931b42f35d8701c1a08f9f524da13b1d3c" +checksum = "d247dce7bed6a8d6a3c6debfa707a3a2f694383f0c692a39d736a593eae5ef94" dependencies = [ "arrow-array", "arrow-buffer", @@ -532,9 +532,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b591ef70d76f4ac28dd7666093295fece0e5f9298f49af51ea49c001e1635bb6" +checksum = "8d609c0181f963cea5c70fddf9a388595b5be441f3aa1d1cdbf728ca834bbd3a" dependencies = [ "ahash 0.8.3", "arrow-array", @@ -547,18 +547,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb327717d87eb94be5eff3b0cb8987f54059d343ee5235abf7f143c85f54cfc8" +checksum = "64951898473bfb8e22293e83a44f02874d2257514d49cd95f9aa4afcff183fbc" dependencies = [ "serde", ] [[package]] name = "arrow-select" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79d3c389d1cea86793934f31594f914c8547d82e91e3411d4833ad0aac3266a7" +checksum = "2a513d89c2e1ac22b28380900036cf1f3992c6443efc5e079de631dcf83c6888" dependencies = [ "arrow-array", "arrow-buffer", @@ -569,9 +569,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ee67790496dd310ddbf5096870324431e89aa76453e010020ac29b1184d356" +checksum = "5288979b2705dae1114c864d73150629add9153b9b8f1d7ee3963db94c372ba5" dependencies = [ "arrow-array", "arrow-buffer", @@ -655,6 +655,8 @@ dependencies = [ "pin-project-lite", "tokio", "xz2", + "zstd 0.11.2+zstd.1.5.2", + "zstd-safe 5.0.2+zstd.1.5.2", ] [[package]] @@ -1643,9 +1645,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "19.0.0" +version = "20.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12d462c103bd1cfd24f8e8a199986d89582af6280528e085c393c4be2ff25da7" +checksum = "2c467c5802cb75ecb0acffa2121d8361a8903fef05b21fd1ca12a55797df8a75" dependencies = [ "ahash 0.8.3", "arrow", @@ -1656,6 +1658,7 @@ dependencies = [ "chrono", "dashmap", "datafusion-common", + "datafusion-execution", "datafusion-expr", "datafusion-optimizer", "datafusion-physical-expr", @@ -1686,13 +1689,14 @@ dependencies = [ "url", "uuid", "xz2", + "zstd 0.12.3+zstd.1.5.2", ] [[package]] name = "datafusion-common" -version = "19.0.0" +version = "20.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5babdbcf102862b1f1828c1ab41094e39ba881d5ece4cee2d481d528148f592" +checksum = "c5c60e0a92bae06c6a55c4618947ae0837f833929b8d52ade1d50cf8b5f3b381" dependencies = [ "arrow", "chrono", @@ -1703,10 +1707,28 @@ dependencies = [ ] [[package]] -name = "datafusion-expr" -version = "19.0.0" +name = "datafusion-execution" +version = "20.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f0c34e87fa541a59d378dc7ee7c9c3dd1fcfa793eab09561b8b4cb35e1827a" +checksum = "96eab4ef369c3972a4f99418ba8673aa0cf818aa4b892d44c35b473edce8e021" +dependencies = [ + "dashmap", + "datafusion-common", + "datafusion-expr", + "hashbrown 0.13.2", + "log", + "object_store", + "parking_lot 0.12.1", + "rand 0.8.5", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "20.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0533db37a619a045a4fd37dabc2d99a85f10127d7f100d75914c194e9e36ed7" dependencies = [ "ahash 0.8.3", "arrow", @@ -1717,9 +1739,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "19.0.0" +version = "20.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d0c6d912b7b7e4637d85947222455cd948ea193ca454ebf649e7265fd10b048" +checksum = "3dbb8f467d31c3efebd372854e35cf1ca1572f163d0f031e2c6f8d78b317a43b" dependencies = [ "arrow", "async-trait", @@ -1728,15 +1750,16 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr", "hashbrown 0.13.2", + "itertools", "log", "regex-syntax", ] [[package]] name = "datafusion-physical-expr" -version = "19.0.0" +version = "20.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8000e8f8efafb810ff2943323bb48bd722ac5bb919fe302a66b832ed9c25245f" +checksum = "ceb003f6c0cdc25d2e9e92f8ee0322faca2a8f94f03d070b5ca15224c5503a3c" dependencies = [ "ahash 0.8.3", "arrow", @@ -1756,6 +1779,7 @@ dependencies = [ "md-5", "num-traits", "paste", + "petgraph", "rand 0.8.5", "regex", "sha2", @@ -1765,9 +1789,9 @@ dependencies = [ [[package]] name = "datafusion-row" -version = "19.0.0" +version = "20.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e900f05d7e5666e8ab714a96a28cb6f143e62aa1d501ba1199024f8635c726c" +checksum = "ba0b4478a205b767f6d98e26730a604562155a8b3cf64fe3600367e741b0f129" dependencies = [ "arrow", "datafusion-common", @@ -1777,9 +1801,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "19.0.0" +version = "20.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "096f293799e8ae883e0f79f8ebaa51e4292e690ba45e0269b48ca9bd79f57094" +checksum = "a07af4200a9353ebe7bf0d6024894275cdebb303bb534b3a38eb0f532583083d" dependencies = [ "arrow-schema", "datafusion-common", @@ -3281,9 +3305,9 @@ dependencies = [ [[package]] name = "parquet" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1b076829801167d889795cd1957989055543430fa1469cb1f6e32b789bfc764" +checksum = "7ac135ecf63ebb5f53dda0921b0b76d6048b3ef631a5f4760b9e8f863ff00cfa" dependencies = [ "ahash 0.8.3", "arrow-array", @@ -3309,7 +3333,7 @@ dependencies = [ "thrift", "tokio", "twox-hash", - "zstd", + "zstd 0.12.3+zstd.1.5.2", ] [[package]] @@ -4219,9 +4243,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.30.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db67dc6ef36edb658196c3fef0464a80b53dbbc194a904e81f9bd4190f9ecc5b" +checksum = "0366f270dbabb5cc2e4c88427dc4c08bba144f81e32fbd459a013f26a4d16aa0" dependencies = [ "log", "serde", @@ -5280,7 +5304,7 @@ dependencies = [ "utoipa-swagger-ui", "uuid", "walkdir", - "zstd", + "zstd 0.12.3+zstd.1.5.2", ] [[package]] @@ -5295,13 +5319,32 @@ dependencies = [ "flate2", ] +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe 5.0.2+zstd.1.5.2", +] + [[package]] name = "zstd" version = "0.12.3+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" dependencies = [ - "zstd-safe", + "zstd-safe 6.0.4+zstd.1.5.4", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index aa25cb1a2d..f000fc67ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,11 +40,11 @@ actix-web-rust-embed-responder = { version = "2.2.0", default-features = false, "support-rust-embed-for-web", "base64", ] } -ahash = { version = "0.8.3", features = ["serde"] } +ahash = { version = "0.8", features = ["serde"] } anyhow = "1.0" argon2 = { version = "0.4.1", features = ["alloc", "password-hash"] } -arrow = { version = "33.0", features = ["simd", "ipc_compression"] } -arrow-schema = { version = "33.0", features = ["serde"] } +arrow = { version = "34.0", features = ["simd", "ipc_compression"] } +arrow-schema = { version = "34.0", features = ["serde"] } async-trait = "0.1.57" async_once = "0.2.6" awc = "3.1.0" @@ -52,7 +52,7 @@ aws-config = "0.54" aws-smithy-async = "0.54" aws-sdk-s3 = "0.24" base64 = "0.21.0" -bytes = "1.1" +bytes = "1.4" chrono = "0.4" clap = { version = "4.1.6", default-features = false, features = [ "std", @@ -62,8 +62,8 @@ clap = { version = "4.1.6", default-features = false, features = [ "cargo", ] } dashmap = { version = "5.4.0", features = ["serde"] } -datafusion = { version = "19.0", features = ["simd"] } -datafusion-common = "19.0" +datafusion = { version = "20.0", features = ["simd"] } +datafusion-common = "20.0" dotenv_config = "0.1.5" dotenvy = "0.15.6" env_logger = "0.10" @@ -84,7 +84,7 @@ mlua = { version = "0.8.4", features = [ "vendored", "serialize", ], optional = true } -object_store = { version = "0.5.3", features = ["aws", "gcp"] } +object_store = { version = "0.5.4", features = ["aws", "gcp"] } once_cell = "1.15.0" opentelemetry = { version = "0.18", features = ["rt-tokio"] } opentelemetry-otlp = { version = "0.11", features = [ @@ -99,7 +99,7 @@ opentelemetry-proto = { version = "0.1", features = [ "build-server", ] } parking_lot = "0.12" -parquet = { version = "33.0", features = ["arrow", "async"] } +parquet = { version = "34.0", features = ["arrow", "async"] } prometheus = "0.13.3" prost = "0.11.2" rand = "0.8.5" @@ -117,7 +117,7 @@ serde_json = "1" simd-json = "0.7.0" sled = "0.34.7" snap = "1" -sqlparser = { version = "0.30", features = ["serde"] } +sqlparser = { version = "0.32", features = ["serde"] } sys-info = "0.9.1" thiserror = "1.0" time = "0.3.17" diff --git a/src/job/files/disk.rs b/src/job/files/disk.rs index b254f37ae0..4be494c3df 100644 --- a/src/job/files/disk.rs +++ b/src/job/files/disk.rs @@ -219,12 +219,10 @@ async fn upload_file( drop(schema_reader); file.seek(SeekFrom::Start(0)).unwrap(); - let mut json_reader = BufReader::new(&file); - let json = arrow::json::Reader::new( - &mut json_reader, - arrow_schema.clone(), - arrow::json::reader::DecoderOptions::new(), - ); + let json_reader = BufReader::new(&file); + let json = arrow::json::RawReaderBuilder::new(arrow_schema.clone()) + .build(json_reader) + .unwrap(); let mut buf_parquet = Vec::new(); let props = WriterProperties::builder() diff --git a/src/job/files/memory.rs b/src/job/files/memory.rs index 0b0eea5fd0..73f96a6276 100644 --- a/src/job/files/memory.rs +++ b/src/job/files/memory.rs @@ -175,12 +175,10 @@ async fn upload_file( let arrow_schema = Arc::new(inferred_schema.clone()); drop(schema_reader); - let mut json_reader = BufReader::new(buf.as_ref()); - let json = arrow::json::Reader::new( - &mut json_reader, - arrow_schema.clone(), - arrow::json::reader::DecoderOptions::new(), - ); + let json_reader = BufReader::new(buf.as_ref()); + let json = arrow::json::RawReaderBuilder::new(arrow_schema.clone()) + .build(json_reader) + .unwrap(); let mut buf_parquet = Vec::new(); let props = WriterProperties::builder() diff --git a/src/service/search/datafusion/exec.rs b/src/service/search/datafusion/exec.rs index a2e13d63bb..3c1de5db23 100644 --- a/src/service/search/datafusion/exec.rs +++ b/src/service/search/datafusion/exec.rs @@ -342,7 +342,21 @@ pub async fn merge( return Err(e); } }; - let batches = df.collect().await?; + let schema: Schema = df.schema().into(); + let mut batches = df.collect().await?; + if batches.len() > 1 { + // try to merge recordbatch + let schema = Arc::new(schema); + let mut merged_batch = batches[0].clone(); + for item in batches.iter().skip(1) { + if item.num_rows() > 0 { + merged_batch = + arrow::compute::concat_batches(&schema.clone(), &[merged_batch, item.clone()]) + .unwrap(); + } + } + batches = vec![merged_batch]; + } ctx.deregister_table("tbl")?; log::info!("Merge took {:.3} seconds.", start.elapsed().as_secs_f64()); diff --git a/src/service/search/sql.rs b/src/service/search/sql.rs index 93f5715444..e46c760f43 100644 --- a/src/service/search/sql.rs +++ b/src/service/search/sql.rs @@ -272,7 +272,7 @@ impl Sql { // sql mode full, disallow without limit, default limit 1000 meta.limit = SQL_DEFAULT_FULL_MODE_LIMIT; } - origin_sql = if meta.order_by.is_empty() { + origin_sql = if meta.order_by.is_empty() && !sql_mode.eq(&SqlMode::Full) { format!( "{} ORDER BY {} DESC LIMIT {}", origin_sql,