From ae8203fafa825944a246a426902f2b793d1b67d9 Mon Sep 17 00:00:00 2001 From: dennis zhuang Date: Wed, 31 May 2023 20:14:58 +0800 Subject: [PATCH] fix: prepare statement doesn't support insert clause (#1680) * fix: insert clause doesn't support prepare statement * fix: manifeste dir * fix: format * fix: temp path --- Cargo.lock | 445 +++++++++++++++--- config/datanode.example.toml | 10 +- config/frontend.example.toml | 6 +- config/metasrv.example.toml | 6 +- config/standalone.example.toml | 6 +- src/datanode/src/store.rs | 7 +- src/datanode/src/store/fs.rs | 2 +- src/file-table-engine/src/manifest.rs | 2 +- .../src/manifest/immutable.rs | 12 +- src/mito/src/table.rs | 7 +- src/servers/src/mysql/handler.rs | 7 - tests-integration/Cargo.toml | 5 + tests-integration/src/test_util.rs | 104 ++++ tests-integration/tests/main.rs | 4 +- tests-integration/tests/sql.rs | 99 ++++ 15 files changed, 623 insertions(+), 99 deletions(-) create mode 100644 tests-integration/tests/sql.rs diff --git a/Cargo.lock b/Cargo.lock index 9574a05311..3ffce3fa2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -636,6 +636,15 @@ dependencies = [ "syn 2.0.18", ] +[[package]] +name = "atoi" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c57d12312ff59c811c0643f4d80830505833c9ffaebd193d819392b265be8e" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic" version = "0.5.3" @@ -1247,7 +1256,7 @@ dependencies = [ "mito", "moka 0.11.1", "object-store", - "parking_lot", + "parking_lot 0.12.1", "regex", "serde", "serde_json", @@ -1518,7 +1527,7 @@ dependencies = [ "datatypes", "enum_dispatch", "futures-util", - "parking_lot", + "parking_lot 0.12.1", "prost", "rand", "snafu", @@ -1894,7 +1903,7 @@ dependencies = [ "once_cell", "opentelemetry", "opentelemetry-jaeger", - "parking_lot", + "parking_lot 0.12.1", "serde", "tokio", "tracing", @@ -2005,6 +2014,12 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "const-oid" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4c78c047431fee22c1a7bb92e00ad095a02a983affe4d8a72e2a2c62c1b94f3" + [[package]] name = "const-oid" version = "0.9.2" @@ -2235,6 +2250,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +[[package]] +name = "crypto-bigint" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c6a1d5fa1de37e071642dfa44ec552ca5b299adb128fab16138e24b548fd21" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -2346,7 +2371,7 @@ dependencies = [ "hashbrown 0.12.3", "lock_api", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.7", ] [[package]] @@ -2381,7 +2406,7 @@ dependencies = [ "log", "num_cpus", "object_store", - "parking_lot", + "parking_lot 0.12.1", "parquet", "percent-encoding", "pin-project-lite", @@ -2423,7 +2448,7 @@ dependencies = [ "hashbrown 0.13.2", "log", "object_store", - "parking_lot", + "parking_lot 0.12.1", "rand", "tempfile", "url", @@ -2622,14 +2647,25 @@ dependencies = [ "snafu", ] +[[package]] +name = "der" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6919815d73839e7ad218de758883aae3a257ba6759ce7a9992501efbb53d705c" +dependencies = [ + "const-oid 0.7.1", + "crypto-bigint", + "pem-rfc7468 0.3.1", +] + [[package]] name = "der" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56acb310e15652100da43d130af8d97b509e95af61aab1c5a7939ef24337ee17" dependencies = [ - "const-oid", - "pem-rfc7468", + "const-oid 0.9.2", + "pem-rfc7468 0.7.0", "zeroize", ] @@ -2719,7 +2755,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", - "const-oid", + "const-oid 0.9.2", "crypto-common", "subtle", ] @@ -2804,6 +2840,12 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dunce" version = "1.0.4" @@ -3293,6 +3335,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a604f7a68fbf8103337523b1fadc8ade7361ee3f112f7c680ad179651616aed5" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot 0.11.2", +] + [[package]] name = "futures-io" version = "0.3.28" @@ -3707,7 +3760,7 @@ checksum = "e4e55e40dfd694884f0eb78796c5bddcf2f8b295dace47039099dd7e76534973" dependencies = [ "gix-hash 0.10.4", "hashbrown 0.13.2", - "parking_lot", + "parking_lot 0.12.1", ] [[package]] @@ -3786,7 +3839,7 @@ dependencies = [ "gix-pack", "gix-path", "gix-quote", - "parking_lot", + "parking_lot 0.12.1", "tempfile", "thiserror", ] @@ -3808,7 +3861,7 @@ dependencies = [ "gix-tempfile", "gix-traverse", "memmap2", - "parking_lot", + "parking_lot 0.12.1", "smallvec", "thiserror", ] @@ -3832,7 +3885,7 @@ dependencies = [ "gix-command", "gix-config-value", "nix 0.26.2", - "parking_lot", + "parking_lot 0.12.1", "thiserror", ] @@ -3916,7 +3969,7 @@ dependencies = [ "gix-fs", "libc", "once_cell", - "parking_lot", + "parking_lot 0.12.1", "signal-hook", "signal-hook-registry", "tempfile", @@ -4056,6 +4109,15 @@ dependencies = [ "ahash 0.8.3", ] +[[package]] +name = "hashlink" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0761a1b9491c4f2e3d66aa0f62d0fba0af9a0e2852e4d48ea506632a4b56e6aa" +dependencies = [ + "hashbrown 0.13.2", +] + [[package]] name = "hdrhistogram" version = "7.5.2" @@ -4074,6 +4136,9 @@ name = "heck" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] [[package]] name = "hermit-abi" @@ -4111,6 +4176,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" +[[package]] +name = "hkdf" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" +dependencies = [ + "hmac", +] + [[package]] name = "hmac" version = "0.12.1" @@ -4217,9 +4291,9 @@ checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7" dependencies = [ "http", "hyper", - "rustls", + "rustls 0.21.1", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.0", ] [[package]] @@ -5036,7 +5110,7 @@ dependencies = [ "lazy_static", "metrics", "once_cell", - "parking_lot", + "parking_lot 0.12.1", "prost", "rand", "regex", @@ -5062,7 +5136,7 @@ source = "git+https://github.com/GreptimeTeam/greptime-meter.git?rev=f0798c4c648 dependencies = [ "anymap", "once_cell", - "parking_lot", + "parking_lot 0.12.1", ] [[package]] @@ -5093,7 +5167,7 @@ dependencies = [ "indexmap", "metrics", "metrics-util", - "parking_lot", + "parking_lot 0.12.1", "portable-atomic 0.3.20", "quanta 0.10.1", "thiserror", @@ -5139,7 +5213,7 @@ dependencies = [ "metrics", "num_cpus", "ordered-float 2.10.0", - "parking_lot", + "parking_lot 0.12.1", "portable-atomic 0.3.20", "quanta 0.10.1", "radix_trie", @@ -5248,7 +5322,7 @@ dependencies = [ "futures-util", "num_cpus", "once_cell", - "parking_lot", + "parking_lot 0.12.1", "quanta 0.10.1", "rustc_version 0.4.0", "scheduled-thread-pool", @@ -5274,7 +5348,7 @@ dependencies = [ "futures-util", "num_cpus", "once_cell", - "parking_lot", + "parking_lot 0.12.1", "quanta 0.11.1", "rustc_version 0.4.0", "scheduled-thread-pool", @@ -5339,19 +5413,19 @@ dependencies = [ "percent-encoding", "pin-project", "priority-queue", - "rustls", + "rustls 0.21.1", "rustls-pemfile", "serde", "serde_json", "socket2 0.5.3", "thiserror", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.0", "tokio-util", "twox-hash", "url", "webpki", - "webpki-roots", + "webpki-roots 0.23.1", ] [[package]] @@ -5688,7 +5762,7 @@ dependencies = [ "chrono", "futures", "itertools", - "parking_lot", + "parking_lot 0.12.1", "percent-encoding", "snafu", "tokio", @@ -5730,7 +5804,7 @@ dependencies = [ "md-5", "metrics", "once_cell", - "parking_lot", + "parking_lot 0.12.1", "percent-encoding", "pin-project", "quick-xml 0.27.1", @@ -5768,7 +5842,7 @@ dependencies = [ "nom", "pin-project-lite", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.0", ] [[package]] @@ -5924,6 +5998,17 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -5931,7 +6016,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.7", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -6066,6 +6165,15 @@ dependencies = [ "serde", ] +[[package]] +name = "pem-rfc7468" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01de5d978f34aa4b2296576379fcc416034702fd94117c56ffd8a1a767cefb30" +dependencies = [ + "base64ct", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -6158,7 +6266,7 @@ dependencies = [ "thiserror", "time 0.3.21", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.0", "tokio-util", "x509-certificate", ] @@ -6243,15 +6351,37 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a78f66c04ccc83dd4486fd46c33896f4e17b24a7a3a6400dedc48ed0ddd72320" +dependencies = [ + "der 0.5.1", + "pkcs8 0.8.0", + "zeroize", +] + [[package]] name = "pkcs1" version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der", - "pkcs8", - "spki", + "der 0.7.6", + "pkcs8 0.10.2", + "spki 0.7.2", +] + +[[package]] +name = "pkcs8" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cabda3fb821068a9a4fab19a683eac3af12edf0f34b94a8be53c4972b8149d0" +dependencies = [ + "der 0.5.1", + "spki 0.5.4", + "zeroize", ] [[package]] @@ -6260,8 +6390,8 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.6", + "spki 0.7.2", ] [[package]] @@ -6518,7 +6648,7 @@ dependencies = [ "fnv", "lazy_static", "memchr", - "parking_lot", + "parking_lot 0.12.1", "protobuf", "thiserror", ] @@ -6701,7 +6831,7 @@ dependencies = [ "indoc", "libc", "memoffset 0.8.0", - "parking_lot", + "parking_lot 0.12.1", "pyo3-build-config", "pyo3-ffi", "pyo3-macros", @@ -6905,7 +7035,7 @@ dependencies = [ "nix 0.25.1", "num-derive", "num-traits", - "parking_lot", + "parking_lot 0.12.1", "prometheus", "prometheus-static-metric", "protobuf", @@ -7108,7 +7238,7 @@ dependencies = [ "quick-xml 0.28.2", "rand", "reqwest", - "rsa", + "rsa 0.9.2", "rust-ini 0.19.0", "serde", "serde_json", @@ -7141,14 +7271,14 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls", + "rustls 0.21.1", "rustls-native-certs", "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.0", "tokio-util", "tower-service", "url", @@ -7259,22 +7389,42 @@ dependencies = [ [[package]] name = "rsa" -version = "0.9.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ab43bb47d23c1a631b4b680199a45255dce26fa9ab2fa902581f624ff13e6a8" +checksum = "4cf22754c49613d2b3b119f0e5d46e34a2c628a937e3024b8762de4e7d8c710b" dependencies = [ "byteorder", - "const-oid", "digest", "num-bigint-dig", "num-integer", "num-iter", "num-traits", - "pkcs1", - "pkcs8", + "pkcs1 0.3.3", + "pkcs8 0.8.0", + "rand_core", + "smallvec", + "subtle", + "zeroize", +] + +[[package]] +name = "rsa" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ab43bb47d23c1a631b4b680199a45255dce26fa9ab2fa902581f624ff13e6a8" +dependencies = [ + "byteorder", + "const-oid 0.9.2", + "digest", + "num-bigint-dig", + "num-integer", + "num-iter", + "num-traits", + "pkcs1 0.7.5", + "pkcs8 0.10.2", "rand_core", "signature", - "spki", + "spki 0.7.2", "subtle", "zeroize", ] @@ -7447,6 +7597,18 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "rustls" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + [[package]] name = "rustls" version = "0.21.1" @@ -7672,7 +7834,7 @@ dependencies = [ "num_enum", "once_cell", "page_size", - "parking_lot", + "parking_lot 0.12.1", "paste", "puruspe", "rand", @@ -7738,7 +7900,7 @@ dependencies = [ "num_enum", "once_cell", "optional", - "parking_lot", + "parking_lot 0.12.1", "paste", "rand", "result-like", @@ -7923,7 +8085,7 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" dependencies = [ - "parking_lot", + "parking_lot 0.12.1", ] [[package]] @@ -8257,7 +8419,7 @@ dependencies = [ "once_cell", "openmetrics-parser", "opensrv-mysql", - "parking_lot", + "parking_lot 0.12.1", "pgwire", "pin-project", "postgres-types", @@ -8267,7 +8429,7 @@ dependencies = [ "rand", "regex", "rust-embed", - "rustls", + "rustls 0.21.1", "rustls-pemfile", "schemars", "script", @@ -8284,7 +8446,7 @@ dependencies = [ "tokio", "tokio-postgres", "tokio-postgres-rustls", - "tokio-rustls", + "tokio-rustls 0.24.0", "tokio-stream", "tokio-test", "tonic 0.9.2", @@ -8548,6 +8710,16 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spki" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d01ac02a6ccf3e07db148d2be087da624fea0221a16152ed01f0496a6b0a27" +dependencies = [ + "base64ct", + "der 0.5.1", +] + [[package]] name = "spki" version = "0.7.2" @@ -8555,7 +8727,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" dependencies = [ "base64ct", - "der", + "der 0.7.6", ] [[package]] @@ -8578,6 +8750,17 @@ dependencies = [ "sqlparser", ] +[[package]] +name = "sqlformat" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c12bc9199d1db8234678b7051747c07f517cdcf019262d1847b94ec8b1aee3e" +dependencies = [ + "itertools", + "nom", + "unicode_categories", +] + [[package]] name = "sqlness" version = "0.4.3" @@ -8630,6 +8813,103 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "sqlx" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8de3b03a925878ed54a954f621e64bf55a3c1bd29652d0d1a17830405350188" +dependencies = [ + "sqlx-core", + "sqlx-macros", +] + +[[package]] +name = "sqlx-core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8241483a83a3f33aa5fff7e7d9def398ff9990b2752b6c6112b83c6d246029" +dependencies = [ + "ahash 0.7.6", + "atoi", + "base64 0.13.1", + "bitflags 1.3.2", + "byteorder", + "bytes", + "crc", + "crossbeam-queue", + "digest", + "dirs", + "dotenvy", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-util", + "generic-array", + "hashlink", + "hex", + "hkdf", + "hmac", + "indexmap", + "itoa", + "libc", + "log", + "md-5", + "memchr", + "num-bigint", + "once_cell", + "paste", + "percent-encoding", + "rand", + "rsa 0.6.1", + "rustls 0.20.8", + "rustls-pemfile", + "serde", + "serde_json", + "sha1", + "sha2", + "smallvec", + "sqlformat", + "sqlx-rt", + "stringprep", + "thiserror", + "tokio-stream", + "url", + "webpki-roots 0.22.6", + "whoami", +] + +[[package]] +name = "sqlx-macros" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9966e64ae989e7e575b19d7265cb79d7fc3cbbdf179835cb0d716f294c2049c9" +dependencies = [ + "dotenvy", + "either", + "heck", + "once_cell", + "proc-macro2", + "quote", + "sha2", + "sqlx-core", + "sqlx-rt", + "syn 1.0.109", + "url", +] + +[[package]] +name = "sqlx-rt" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804d3f245f894e61b1e6263c84b23ca675d96753b5abfd5cc8597d86806e8024" +dependencies = [ + "once_cell", + "tokio", + "tokio-rustls 0.23.4", +] + [[package]] name = "sre-engine" version = "0.4.1" @@ -8771,7 +9051,7 @@ checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" dependencies = [ "new_debug_unreachable", "once_cell", - "parking_lot", + "parking_lot 0.12.1", "phf_shared 0.10.0", "precomputed-hash", ] @@ -9047,7 +9327,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9547444bfe52cbd79515c6c8087d8ae6ca8d64d2d31a27746320f5cb81d1a15c" dependencies = [ - "parking_lot", + "parking_lot 0.12.1", ] [[package]] @@ -9140,6 +9420,7 @@ dependencies = [ "session", "snafu", "sql", + "sqlx", "store-api", "table", "tempfile", @@ -9365,7 +9646,7 @@ dependencies = [ "libc", "mio", "num_cpus", - "parking_lot", + "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", "socket2 0.4.9", @@ -9408,7 +9689,7 @@ dependencies = [ "futures-channel", "futures-util", "log", - "parking_lot", + "parking_lot 0.12.1", "percent-encoding", "phf", "pin-project-lite", @@ -9427,10 +9708,21 @@ checksum = "dd5831152cb0d3f79ef5523b357319ba154795d64c7078b2daa95a803b54057f" dependencies = [ "futures", "ring", - "rustls", + "rustls 0.21.1", "tokio", "tokio-postgres", - "tokio-rustls", + "tokio-rustls 0.24.0", +] + +[[package]] +name = "tokio-rustls" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +dependencies = [ + "rustls 0.20.8", + "tokio", + "webpki", ] [[package]] @@ -9439,7 +9731,7 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0d409377ff5b1e3ca6437aa86c1eb7d40c134bfec254e44c830defa92669db5" dependencies = [ - "rustls", + "rustls 0.21.1", "tokio", ] @@ -9563,7 +9855,7 @@ dependencies = [ "prost", "rustls-pemfile", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.0", "tokio-stream", "tower", "tower-layer", @@ -10147,6 +10439,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "unicode_names2" version = "0.6.0" @@ -10407,6 +10705,15 @@ dependencies = [ "untrusted", ] +[[package]] +name = "webpki-roots" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +dependencies = [ + "webpki", +] + [[package]] name = "webpki-roots" version = "0.23.1" @@ -10427,6 +10734,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "whoami" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c70234412ca409cc04e864e89523cb0fc37f5e1344ebed5a3ebf4192b6b9f68" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wide" version = "0.7.9" @@ -10724,12 +11041,12 @@ dependencies = [ "bcder", "bytes", "chrono", - "der", + "der 0.7.6", "hex", "pem 1.1.1", "ring", "signature", - "spki", + "spki 0.7.2", "thiserror", ] diff --git a/config/datanode.example.toml b/config/datanode.example.toml index 4e610dc866..247b81a5d5 100644 --- a/config/datanode.example.toml +++ b/config/datanode.example.toml @@ -73,7 +73,9 @@ global_write_buffer_size = "1GB" max_retry_times = 3 retry_delay = "500ms" -# Log options, see `standalone.example.toml` -[logging] -dir = "/tmp/greptimedb/logs" -level = "info" +# Log options +# [logging] +# Specify logs directory. +# dir = "/tmp/greptimedb/logs" +# Specify the log level [info | debug | error | warn] +# level = "info" diff --git a/config/frontend.example.toml b/config/frontend.example.toml index f0296426bf..e8020a2b69 100644 --- a/config/frontend.example.toml +++ b/config/frontend.example.toml @@ -58,6 +58,6 @@ connect_timeout_millis = 5000 tcp_nodelay = true # Log options, see `standalone.example.toml` -[logging] -dir = "/tmp/greptimedb/logs" -level = "info" +# [logging] +# dir = "/tmp/greptimedb/logs" +# level = "info" diff --git a/config/metasrv.example.toml b/config/metasrv.example.toml index 47bd8addea..bd572fcb7f 100644 --- a/config/metasrv.example.toml +++ b/config/metasrv.example.toml @@ -15,6 +15,6 @@ selector = "LeaseBased" use_memory_store = false # Log options, see `standalone.example.toml` -[logging] -dir = "/tmp/greptimedb/logs" -level = "info" +# [logging] +# dir = "/tmp/greptimedb/logs" +# level = "info" diff --git a/config/standalone.example.toml b/config/standalone.example.toml index 1f39c6b4bc..c4b6800630 100644 --- a/config/standalone.example.toml +++ b/config/standalone.example.toml @@ -140,8 +140,8 @@ max_retry_times = 3 retry_delay = "500ms" # Log options -[logging] +# [logging] # Specify logs directory. -dir = "/tmp/greptimedb/logs" +# dir = "/tmp/greptimedb/logs" # Specify the log level [info | debug | error | warn] -level = "debug" +# level = "info" diff --git a/src/datanode/src/store.rs b/src/datanode/src/store.rs index a245b70e54..c3315ed6df 100644 --- a/src/datanode/src/store.rs +++ b/src/datanode/src/store.rs @@ -26,7 +26,7 @@ use common_base::readable_size::ReadableSize; use common_telemetry::logging::info; use object_store::layers::{LoggingLayer, LruCacheLayer, MetricsLayer, RetryLayer, TracingLayer}; use object_store::services::Fs as FsBuilder; -use object_store::{ObjectStore, ObjectStoreBuilder}; +use object_store::{util, ObjectStore, ObjectStoreBuilder}; use snafu::prelude::*; use crate::datanode::{ObjectStoreConfig, DEFAULT_OBJECT_STORE_CACHE_SIZE}; @@ -92,10 +92,11 @@ async fn create_object_store_with_cache( }; if let Some(path) = cache_path { - let atomic_temp_dir = format!("{path}/.tmp/"); + let path = util::normalize_dir(path); + let atomic_temp_dir = format!("{path}.tmp/"); clean_temp_dir(&atomic_temp_dir)?; let cache_store = FsBuilder::default() - .root(path) + .root(&path) .atomic_write_dir(&atomic_temp_dir) .build() .context(error::InitBackendSnafu)?; diff --git a/src/datanode/src/store/fs.rs b/src/datanode/src/store/fs.rs index df29ac0b27..34af23c9f7 100644 --- a/src/datanode/src/store/fs.rs +++ b/src/datanode/src/store/fs.rs @@ -29,7 +29,7 @@ pub(crate) async fn new_fs_object_store(file_config: &FileConfig) -> Result String { - format!("{table_dir}/manifest/") + format!("{table_dir}manifest/") } diff --git a/src/file-table-engine/src/manifest/immutable.rs b/src/file-table-engine/src/manifest/immutable.rs index 00b91f7e17..e4c1a30d48 100644 --- a/src/file-table-engine/src/manifest/immutable.rs +++ b/src/file-table-engine/src/manifest/immutable.rs @@ -108,7 +108,7 @@ mod tests { write_table_manifest( TEST_TABLE_NAME, - &table_manifest_dir(TEST_TABLE_NAME), + &table_manifest_dir(&format!("{TEST_TABLE_NAME}/")), &store, &metadata, ) @@ -118,7 +118,7 @@ mod tests { // try to overwrite immutable manifest let write_immutable = write_table_manifest( TEST_TABLE_NAME, - &table_manifest_dir(TEST_TABLE_NAME), + &table_manifest_dir(&format!("{TEST_TABLE_NAME}/")), &store, &metadata, ) @@ -135,7 +135,7 @@ mod tests { write_table_manifest( TEST_TABLE_NAME, - &table_manifest_dir(TEST_TABLE_NAME), + &table_manifest_dir(&format!("{TEST_TABLE_NAME}/")), &store, &metadata, ) @@ -144,7 +144,7 @@ mod tests { let read = read_table_manifest( TEST_TABLE_NAME, - &table_manifest_dir(TEST_TABLE_NAME), + &table_manifest_dir(&format!("{TEST_TABLE_NAME}/")), &store, ) .await @@ -158,7 +158,7 @@ mod tests { let (_dir, store) = new_test_object_store("test_read_non_exist_table_manifest"); let not_fount = read_table_manifest( TEST_TABLE_NAME, - &table_manifest_dir(TEST_TABLE_NAME), + &table_manifest_dir(&format!("{TEST_TABLE_NAME}/")), &store, ) .await @@ -172,7 +172,7 @@ mod tests { let (_dir, store) = new_test_object_store("test_delete_table_manifest"); let metadata = build_test_table_metadata(); - let table_dir = &table_manifest_dir(TEST_TABLE_NAME); + let table_dir = &table_manifest_dir(&format!("{TEST_TABLE_NAME}/")); write_table_manifest(TEST_TABLE_NAME, table_dir, &store, &metadata) .await .unwrap(); diff --git a/src/mito/src/table.rs b/src/mito/src/table.rs index 1f057b848d..a5ecf8fa92 100644 --- a/src/mito/src/table.rs +++ b/src/mito/src/table.rs @@ -59,7 +59,8 @@ use crate::manifest::action::*; use crate::manifest::TableManifest; #[inline] fn table_manifest_dir(table_dir: &str) -> String { - format!("{table_dir}/manifest/") + assert!(table_dir.ends_with('/')); + format!("{table_dir}manifest/") } /// [Table] implementation. @@ -770,7 +771,7 @@ mod tests { #[test] fn test_table_manifest_dir() { - assert_eq!("demo/manifest/", table_manifest_dir("demo")); - assert_eq!("numbers/manifest/", table_manifest_dir("numbers")); + assert_eq!("demo/manifest/", table_manifest_dir("demo/")); + assert_eq!("numbers/manifest/", table_manifest_dir("numbers/")); } } diff --git a/src/servers/src/mysql/handler.rs b/src/servers/src/mysql/handler.rs index 6e40f976b9..5b0e1a024d 100644 --- a/src/servers/src/mysql/handler.rs +++ b/src/servers/src/mysql/handler.rs @@ -352,13 +352,6 @@ async fn validate_query(query: &str) -> Result { let statement = statement.remove(0); - ensure!( - matches!(statement, Statement::Query(_)), - InvalidPrepareStatementSnafu { - err_msg: "prepare statement only support SELECT for now".to_string(), - } - ); - Ok(statement) } diff --git a/tests-integration/Cargo.toml b/tests-integration/Cargo.toml index 1d6a4739f1..1ceaca7be7 100644 --- a/tests-integration/Cargo.toml +++ b/tests-integration/Cargo.toml @@ -44,6 +44,11 @@ servers = { path = "../src/servers" } session = { path = "../src/session" } snafu.workspace = true sql = { path = "../src/sql" } +sqlx = { version = "0.6", features = [ + "runtime-tokio-rustls", + "mysql", + "postgres", +] } table = { path = "../src/table" } tempfile.workspace = true tokio.workspace = true diff --git a/tests-integration/src/test_util.rs b/tests-integration/src/test_util.rs index 41b826a9b0..0dae640074 100644 --- a/tests-integration/src/test_util.rs +++ b/tests-integration/src/test_util.rs @@ -37,6 +37,7 @@ use datanode::sql::SqlHandler; use datatypes::data_type::ConcreteDataType; use datatypes::schema::{ColumnSchema, RawSchema}; use frontend::instance::Instance as FeInstance; +use frontend::service_config::{MysqlOptions, PostgresOptions}; use object_store::services::{Azblob, Oss, S3}; use object_store::test_util::TempFolder; use object_store::ObjectStore; @@ -44,6 +45,8 @@ use secrecy::ExposeSecret; use servers::grpc::GrpcServer; use servers::http::{HttpOptions, HttpServerBuilder}; use servers::metrics_handler::MetricsHandler; +use servers::mysql::server::{MysqlServer, MysqlSpawnConfig, MysqlSpawnRef}; +use servers::postgres::PostgresServer; use servers::prom::PromServer; use servers::query_handler::grpc::ServerGrpcQueryHandlerAdaptor; use servers::query_handler::sql::ServerSqlQueryHandlerAdaptor; @@ -454,3 +457,104 @@ pub async fn check_output_stream(output: Output, expected: &str) { let pretty_print = recordbatches.pretty_print().unwrap(); assert_eq!(pretty_print, expected, "actual: \n{}", pretty_print); } + +pub async fn setup_mysql_server( + store_type: StorageType, + name: &str, +) -> (String, TestGuard, Arc>) { + common_telemetry::init_default_ut_logging(); + + let (opts, guard) = create_tmp_dir_and_datanode_opts(store_type, name); + let instance = Arc::new(Instance::with_mock_meta_client(&opts).await.unwrap()); + + let runtime = Arc::new( + RuntimeBuilder::default() + .worker_threads(2) + .thread_name("mysql-runtime") + .build() + .unwrap(), + ); + + let fe_mysql_addr = format!("127.0.0.1:{}", ports::get_port()); + + let fe_instance = FeInstance::try_new_standalone(instance.clone()) + .await + .unwrap(); + instance.start().await.unwrap(); + let fe_instance_ref = Arc::new(fe_instance); + let opts = MysqlOptions { + addr: fe_mysql_addr.clone(), + ..Default::default() + }; + let fe_mysql_server = Arc::new(MysqlServer::create_server( + runtime, + Arc::new(MysqlSpawnRef::new( + ServerSqlQueryHandlerAdaptor::arc(fe_instance_ref), + None, + )), + Arc::new(MysqlSpawnConfig::new( + false, + opts.tls.setup().unwrap().map(Arc::new), + opts.reject_no_database.unwrap_or(false), + )), + )); + + let fe_mysql_addr_clone = fe_mysql_addr.clone(); + let fe_mysql_server_clone = fe_mysql_server.clone(); + tokio::spawn(async move { + let addr = fe_mysql_addr_clone.parse::().unwrap(); + fe_mysql_server_clone.start(addr).await.unwrap() + }); + + tokio::time::sleep(Duration::from_secs(1)).await; + + (fe_mysql_addr, guard, fe_mysql_server) +} + +#[allow(dead_code)] +pub async fn setup_pg_server( + store_type: StorageType, + name: &str, +) -> (String, TestGuard, Arc>) { + common_telemetry::init_default_ut_logging(); + + let (opts, guard) = create_tmp_dir_and_datanode_opts(store_type, name); + let instance = Arc::new(Instance::with_mock_meta_client(&opts).await.unwrap()); + + let runtime = Arc::new( + RuntimeBuilder::default() + .worker_threads(2) + .thread_name("pg-runtime") + .build() + .unwrap(), + ); + + let fe_pg_addr = format!("127.0.0.1:{}", ports::get_port()); + + let fe_instance = FeInstance::try_new_standalone(instance.clone()) + .await + .unwrap(); + instance.start().await.unwrap(); + let fe_instance_ref = Arc::new(fe_instance); + let opts = PostgresOptions { + addr: fe_pg_addr.clone(), + ..Default::default() + }; + let fe_pg_server = Arc::new(Box::new(PostgresServer::new( + ServerSqlQueryHandlerAdaptor::arc(fe_instance_ref), + opts.tls.clone(), + runtime, + None, + )) as Box); + + let fe_pg_addr_clone = fe_pg_addr.clone(); + let fe_pg_server_clone = fe_pg_server.clone(); + tokio::spawn(async move { + let addr = fe_pg_addr_clone.parse::().unwrap(); + fe_pg_server_clone.start(addr).await.unwrap() + }); + + tokio::time::sleep(Duration::from_secs(1)).await; + + (fe_pg_addr, guard, fe_pg_server) +} diff --git a/tests-integration/tests/main.rs b/tests-integration/tests/main.rs index 50af7c523e..5537e8d4b9 100644 --- a/tests-integration/tests/main.rs +++ b/tests-integration/tests/main.rs @@ -16,10 +16,12 @@ mod grpc; #[macro_use] mod http; - +#[macro_use] +mod sql; #[macro_use] mod region_failover; grpc_tests!(File, S3, S3WithCache, Oss); http_tests!(File, S3, S3WithCache, Oss); region_failover_tests!(File, S3, S3WithCache, Oss); +sql_tests!(File); diff --git a/tests-integration/tests/sql.rs b/tests-integration/tests/sql.rs new file mode 100644 index 0000000000..1d8c9d5fbc --- /dev/null +++ b/tests-integration/tests/sql.rs @@ -0,0 +1,99 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +use sqlx::mysql::MySqlPoolOptions; +use sqlx::Row; +use tests_integration::test_util::{setup_mysql_server, StorageType}; + +#[macro_export] +macro_rules! sql_test { + ($service:ident, $($(#[$meta:meta])* $test:ident),*,) => { + paste::item! { + mod [] { + $( + #[tokio::test(flavor = "multi_thread")] + $( + #[$meta] + )* + async fn [< $test >]() { + let store_type = tests_integration::test_util::StorageType::$service; + if store_type.test_on() { + let _ = $crate::sql::$test(store_type).await; + } + + } + )* + } + } + }; +} + +#[macro_export] +macro_rules! sql_tests { + ($($service:ident),*) => { + $( + sql_test!( + $service, + + test_mysql_crud, + ); + )* + }; +} + +pub async fn test_mysql_crud(store_type: StorageType) { + let (addr, mut guard, fe_mysql_server) = setup_mysql_server(store_type, "sql_crud").await; + + let pool = MySqlPoolOptions::new() + .max_connections(2) + .connect(&format!("mysql://{addr}/public")) + .await + .unwrap(); + + sqlx::query("create table demo(i bigint, ts timestamp time index)") + .execute(&pool) + .await + .unwrap(); + for i in 0..10 { + sqlx::query("insert table demo values(?, ?)") + .bind(i) + .bind(i) + .execute(&pool) + .await + .unwrap(); + } + + let rows = sqlx::query("select i from demo") + .fetch_all(&pool) + .await + .unwrap(); + assert_eq!(rows.len(), 10); + + for (i, row) in rows.iter().enumerate() { + let ret: i64 = row.get(0); + assert_eq!(ret, i as i64); + } + + sqlx::query("delete from demo") + .execute(&pool) + .await + .unwrap(); + let rows = sqlx::query("select i from demo") + .fetch_all(&pool) + .await + .unwrap(); + assert_eq!(rows.len(), 0); + + let _ = fe_mysql_server.shutdown().await; + guard.remove_all().await; +}