From 6166f2072ee8029d3a38a8923e5da79918a6f8d1 Mon Sep 17 00:00:00 2001 From: discord9 <55937128+discord9@users.noreply.github.com> Date: Wed, 30 Apr 2025 05:30:06 +0800 Subject: [PATCH] chore: upgrade hydroflow depend (#6011) * chore: `hydroflow` -> `dfir` * chore: refine log msg --- Cargo.lock | 633 +++++++++++------------- src/flow/Cargo.toml | 6 +- src/flow/src/adapter/worker.rs | 10 +- src/flow/src/batching_mode/engine.rs | 2 +- src/flow/src/batching_mode/task.rs | 2 +- src/flow/src/batching_mode/utils.rs | 12 +- src/flow/src/compute/render.rs | 26 +- src/flow/src/compute/render/map.rs | 12 +- src/flow/src/compute/render/reduce.rs | 20 +- src/flow/src/compute/render/src_sink.rs | 2 +- src/flow/src/compute/state.rs | 10 +- src/flow/src/compute/types.rs | 12 +- src/flow/src/expr/scalar.rs | 2 +- 13 files changed, 351 insertions(+), 398 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1603528b22..aaeced3556 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -713,12 +713,6 @@ dependencies = [ "regex-syntax 0.8.5", ] -[[package]] -name = "ascii" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" - [[package]] name = "async-broadcast" version = "0.7.1" @@ -1742,12 +1736,6 @@ dependencies = [ "phf_codegen", ] -[[package]] -name = "chunked_transfer" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" - [[package]] name = "ciborium" version = "0.2.2" @@ -1793,22 +1781,7 @@ checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", - "libloading 0.8.5", -] - -[[package]] -name = "clap" -version = "2.34.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" -dependencies = [ - "ansi_term", - "atty", - "bitflags 1.3.2", - "strsim 0.8.0", - "textwrap 0.11.0", - "unicode-width", - "vec_map", + "libloading", ] [[package]] @@ -1820,7 +1793,7 @@ dependencies = [ "bitflags 1.3.2", "clap_lex 0.2.4", "indexmap 1.9.3", - "textwrap 0.16.1", + "textwrap", ] [[package]] @@ -2793,6 +2766,15 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -2803,6 +2785,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -3860,18 +3852,110 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "dfir_datalog" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71628d2d9704d55ddce51376fcc566be2d6a46b24f006c0dea0581185233b67d" +dependencies = [ + "dfir_datalog_core", + "proc-macro-crate 1.3.1", + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "dfir_datalog_core" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da8df0f4ee446c3c3f44bca2758bbf47625f6af150f16a2a5013a88ddf42e21a" +dependencies = [ + "dfir_lang", + "proc-macro2", + "quote", + "rust-sitter", + "rust-sitter-tool", + "slotmap", + "syn 2.0.100", +] + +[[package]] +name = "dfir_lang" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e77f7d1dc8053ee0bbb907ed98ca1f56ebfba0b933b23debe8af16fdd0908f" +dependencies = [ + "auto_impl", + "clap 4.5.19", + "data-encoding", + "documented", + "itertools 0.13.0", + "prettyplease", + "proc-macro2", + "quote", + "rustc_version", + "serde", + "serde_json", + "slotmap", + "syn 2.0.100", + "webbrowser", +] + +[[package]] +name = "dfir_macro" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed88e18d2c40026d19c9ddde53fee650a9e8accf62ddf94851f41a08fe3df3e" +dependencies = [ + "dfir_lang", + "itertools 0.13.0", + "proc-macro-crate 1.3.1", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.100", +] + +[[package]] +name = "dfir_rs" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3b08cdfdba4b482d762906a16ac2cfd45d9949ae62e03ed3c690cfd7dae5dc" +dependencies = [ + "bincode", + "byteorder", + "bytes", + "dfir_datalog", + "dfir_lang", + "dfir_macro", + "futures", + "getrandom 0.2.15", + "itertools 0.13.0", + "lattices", + "pusherator", + "ref-cast", + "regex", + "rustc-hash 1.1.0", + "sealed", + "serde", + "serde_json", + "slotmap", + "smallvec", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", + "variadics", + "web-time 1.1.0", +] + [[package]] name = "diff" version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" -[[package]] -name = "difference" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" - [[package]] name = "digest" version = "0.10.7" @@ -3884,15 +3968,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "dirs" -version = "3.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309" -dependencies = [ - "dirs-sys", -] - [[package]] name = "dirs-next" version = "2.0.0" @@ -3903,17 +3978,6 @@ dependencies = [ "dirs-sys-next", ] -[[package]] -name = "dirs-sys" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -3966,6 +4030,32 @@ dependencies = [ "litrs", ] +[[package]] +name = "documented" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6db32f0995bc4553d2de888999075acd0dbeef75ba923503f6a724263dc6f3" +dependencies = [ + "documented-macros", + "phf", + "thiserror 1.0.64", +] + +[[package]] +name = "documented-macros" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a394bb35929b58f9a5fd418f7c6b17a4b616efcc1e53e6995ca123948f87e5fa" +dependencies = [ + "convert_case", + "itertools 0.13.0", + "optfield", + "proc-macro2", + "quote", + "strum 0.26.3", + "syn 2.0.100", +] + [[package]] name = "dotenv" version = "0.15.0" @@ -4417,13 +4507,13 @@ dependencies = [ "datafusion-physical-expr", "datafusion-substrait", "datatypes", + "dfir_rs", "enum-as-inner", "enum_dispatch", "futures", "get-size2", "greptime-proto", "http 1.1.0", - "hydroflow", "itertools 0.14.0", "lazy_static", "meta-client", @@ -5205,15 +5295,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "html-escape" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476" -dependencies = [ - "utf8-width", -] - [[package]] name = "htmlescape" version = "0.3.1" @@ -5326,98 +5407,6 @@ dependencies = [ "serde", ] -[[package]] -name = "hydroflow" -version = "0.6.2" -source = "git+https://github.com/GreptimeTeam/hydroflow.git?branch=main#b072ee026f97f8537165e1fb247101e0ab2fb320" -dependencies = [ - "bincode", - "byteorder", - "bytes", - "futures", - "getrandom 0.2.15", - "hydroflow_datalog", - "hydroflow_lang", - "hydroflow_macro", - "instant", - "itertools 0.10.5", - "lattices", - "pusherator", - "ref-cast", - "regex", - "rustc-hash 1.1.0", - "sealed", - "serde", - "serde_json", - "slotmap", - "smallvec", - "tokio", - "tokio-stream", - "tokio-util", - "tracing", - "variadics", -] - -[[package]] -name = "hydroflow_datalog" -version = "0.6.0" -source = "git+https://github.com/GreptimeTeam/hydroflow.git?branch=main#b072ee026f97f8537165e1fb247101e0ab2fb320" -dependencies = [ - "hydroflow_datalog_core", - "proc-macro-crate 1.3.1", - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "hydroflow_datalog_core" -version = "0.6.1" -source = "git+https://github.com/GreptimeTeam/hydroflow.git?branch=main#b072ee026f97f8537165e1fb247101e0ab2fb320" -dependencies = [ - "hydroflow_lang", - "proc-macro-crate 1.3.1", - "proc-macro2", - "quote", - "rust-sitter", - "rust-sitter-tool", - "slotmap", - "syn 2.0.100", -] - -[[package]] -name = "hydroflow_lang" -version = "0.6.2" -source = "git+https://github.com/GreptimeTeam/hydroflow.git?branch=main#b072ee026f97f8537165e1fb247101e0ab2fb320" -dependencies = [ - "auto_impl", - "clap 4.5.19", - "data-encoding", - "itertools 0.10.5", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "serde", - "serde_json", - "slotmap", - "syn 2.0.100", - "webbrowser", -] - -[[package]] -name = "hydroflow_macro" -version = "0.6.0" -source = "git+https://github.com/GreptimeTeam/hydroflow.git?branch=main#b072ee026f97f8537165e1fb247101e0ab2fb320" -dependencies = [ - "hydroflow_lang", - "itertools 0.10.5", - "proc-macro-crate 1.3.1", - "proc-macro2", - "quote", - "syn 2.0.100", -] - [[package]] name = "hyper" version = "0.14.30" @@ -5855,6 +5844,12 @@ dependencies = [ "serde", ] +[[package]] +name = "indoc" +version = "2.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" + [[package]] name = "inferno" version = "0.11.21" @@ -6353,12 +6348,29 @@ dependencies = [ [[package]] name = "lattices" -version = "0.5.4" -source = "git+https://github.com/GreptimeTeam/hydroflow.git?branch=main#b072ee026f97f8537165e1fb247101e0ab2fb320" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51222529a85db7b6e228b279b2b9bb7bc1e47d9ee7b680404bdf5440b255913c" dependencies = [ "cc-traits", + "lattices_macro", + "ref-cast", "sealed", "serde", + "variadics", + "variadics_macro", +] + +[[package]] +name = "lattices_macro" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eee67863077d060b8a25c754241a6e6bf234b82207a62df7b3d5387b6d523c3e" +dependencies = [ + "proc-macro-crate 1.3.1", + "proc-macro2", + "quote", + "syn 2.0.100", ] [[package]] @@ -6492,16 +6504,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "libloading" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" -dependencies = [ - "cfg-if", - "winapi", -] - [[package]] name = "libloading" version = "0.8.5" @@ -6793,15 +6795,6 @@ dependencies = [ "thiserror 1.0.64", ] -[[package]] -name = "malloc_buf" -version = "0.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb" -dependencies = [ - "libc", -] - [[package]] name = "match_cfg" version = "0.1.0" @@ -7796,12 +7789,28 @@ dependencies = [ ] [[package]] -name = "objc" -version = "0.2.7" +name = "objc2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" +checksum = "88c6597e14493ab2e44ce58f2fdecf095a51f12ca57bec060a11c57332520551" dependencies = [ - "malloc_buf", + "objc2-encode", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900831247d2fe1a09a683278e5384cfb8c80c79fe6b166f9d14bfdde0ea1b03c" +dependencies = [ + "bitflags 2.9.0", + "objc2", ] [[package]] @@ -8175,6 +8184,17 @@ dependencies = [ "tonic 0.12.3", ] +[[package]] +name = "optfield" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa59f025cde9c698fcb4fcb3533db4621795374065bee908215263488f2d2a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "orc-rust" version = "0.6.0" @@ -8639,6 +8659,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ + "phf_macros", "phf_shared", ] @@ -8662,6 +8683,19 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "phf_shared" version = "0.11.2" @@ -9405,8 +9439,9 @@ dependencies = [ [[package]] name = "pusherator" -version = "0.0.5" -source = "git+https://github.com/GreptimeTeam/hydroflow.git?branch=main#b072ee026f97f8537165e1fb247101e0ab2fb320" +version = "0.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3378af79ad42241f1075683daa2a0c06c6654404c3d096a493348395e620f668" dependencies = [ "either", "variadics", @@ -9718,12 +9753,6 @@ dependencies = [ "thiserror 1.0.64", ] -[[package]] -name = "raw-window-handle" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ff9a1f06a88b01621b7ae906ef0211290d1c8a168a15542486a8f61c0833b9" - [[package]] name = "rawpointer" version = "0.2.1" @@ -10245,9 +10274,9 @@ dependencies = [ [[package]] name = "rust-sitter" -version = "0.3.4" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a0f365b4eb9591dd3e685791389a932041b0dc6ccf5db1ec3d8913f67279365" +checksum = "45edd9ca3d6ee6aee5fcef90dc38ed59763d470ebbe21c469de25b98e2b229e8" dependencies = [ "rust-sitter-macro", "tree-sitter-c2rust", @@ -10255,41 +10284,41 @@ dependencies = [ [[package]] name = "rust-sitter-common" -version = "0.3.4" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c0a0b1da7317031274502b7c52cbb7cf529e7d1e1f3e23876519372b173a94" +checksum = "a50be02780352f27c15e8579a6fd1b519b97a4ddd4363b1eda3187ec228e39e9" dependencies = [ "quote", - "syn 1.0.109", + "syn 2.0.100", ] [[package]] name = "rust-sitter-macro" -version = "0.3.4" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d25e213e40efa00713547cc0f3529694aca547cfceb0839bbc9406632e14d410" +checksum = "401c78a344235363683778b211b17d6c45762baf5144adff4f57ae2fc55a4563" dependencies = [ "proc-macro2", "quote", "rust-sitter-common", - "syn 1.0.109", + "syn 2.0.100", ] [[package]] name = "rust-sitter-tool" -version = "0.3.4" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "803c6596476a188a4dd18106eb927a926a202e00077cdaa5648dd620262af158" +checksum = "964aeedc0934679d59ea2a2c758394a01b1677def26391aac6f24d06afad3b06" dependencies = [ "cc", "rust-sitter-common", "serde", "serde_json", - "syn 1.0.109", + "syn 2.0.100", "syn-inline-mod", "tempfile", "tree-sitter", - "tree-sitter-cli", + "tree-sitter-generate", ] [[package]] @@ -10646,7 +10675,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags 2.9.0", - "core-foundation", + "core-foundation 0.9.4", "core-foundation-sys", "libc", "security-framework-sys", @@ -11688,6 +11717,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + [[package]] name = "strfmt" version = "0.2.4" @@ -11705,12 +11740,6 @@ dependencies = [ "unicode-properties", ] -[[package]] -name = "strsim" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" - [[package]] name = "strsim" version = "0.10.0" @@ -11737,6 +11766,9 @@ name = "strum" version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros 0.26.4", +] [[package]] name = "strum" @@ -11916,12 +11948,12 @@ dependencies = [ [[package]] name = "syn-inline-mod" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b670f535364c67358ecffb60b9f2579f9b45d3c71e8cca6d45d22ee0fadaa7eb" +checksum = "2fa6dca1fdb7b2ed46dd534a326725419d4fb10f23d8c85a8b2860e5eb25d0f9" dependencies = [ "proc-macro2", - "syn 1.0.109", + "syn 2.0.100", ] [[package]] @@ -12355,15 +12387,6 @@ dependencies = [ "zstd 0.13.2", ] -[[package]] -name = "textwrap" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "unicode-width", -] - [[package]] name = "textwrap" version = "0.16.1" @@ -12514,18 +12537,6 @@ dependencies = [ "crunchy", ] -[[package]] -name = "tiny_http" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82" -dependencies = [ - "ascii", - "chunked_transfer", - "httpdate", - "log", -] - [[package]] name = "tinystr" version = "0.7.6" @@ -13092,114 +13103,60 @@ dependencies = [ [[package]] name = "tree-sitter" -version = "0.20.10" +version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e747b1f9b7b931ed39a548c1fae149101497de3c1fc8d9e18c62c1a66c683d3d" +checksum = "b9ac5ea5e7f2f1700842ec071401010b9c59bf735295f6e9fa079c3dc035b167" dependencies = [ "cc", "regex", + "regex-syntax 0.8.5", + "serde_json", + "streaming-iterator", + "tree-sitter-language", ] [[package]] name = "tree-sitter-c2rust" -version = "0.20.10" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee40a4d9cf5a30c199935f346887588239daceae4d1418d81b789276fffb8d91" +checksum = "6e1e4909668d7a5e0eb8d66bd9fe2e789106066626056d3bead80dc6b9d5aeee" dependencies = [ "c2rust-bitfields", "once_cell", "regex", + "regex-syntax 0.8.5", + "streaming-iterator", + "tree-sitter-language", ] [[package]] -name = "tree-sitter-cli" -version = "0.20.8" +name = "tree-sitter-generate" +version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae7e9d844d4d38e511a7b93fe8ced79f2a364c32fdea10d04546f1c8317d5a0c" +checksum = "0b41c38754ea1aea0467fe50e1fb4dcb657d69d601b310643b3317e2eb272701" dependencies = [ - "ansi_term", "anyhow", - "atty", - "clap 2.34.0", - "difference", - "dirs", - "glob", - "html-escape", - "indexmap 1.9.3", - "lazy_static", + "heck 0.5.0", + "indexmap 2.9.0", + "indoc", "log", "regex", - "regex-syntax 0.6.29", - "rustc-hash 1.1.0", + "regex-syntax 0.8.5", + "rustc-hash 2.1.1", "semver", "serde", "serde_json", "smallbitvec", - "tiny_http", - "toml 0.5.11", + "thiserror 2.0.12", "tree-sitter", - "tree-sitter-config", - "tree-sitter-highlight", - "tree-sitter-loader", - "tree-sitter-tags", - "walkdir", - "webbrowser", - "which", + "url", ] [[package]] -name = "tree-sitter-config" -version = "0.19.0" +name = "tree-sitter-language" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5fec4cb27f052ead2246631b332dba0cb6af9a54ce012badee59c4b0ded5e03" -dependencies = [ - "anyhow", - "dirs", - "serde", - "serde_json", -] - -[[package]] -name = "tree-sitter-highlight" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "042342584c5a7a0b833d9fc4e2bdab3f9868ddc6c4b339a1e01451c6720868bc" -dependencies = [ - "regex", - "thiserror 1.0.64", - "tree-sitter", -] - -[[package]] -name = "tree-sitter-loader" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0b17eef4833c7c139abed66d562dfa23228e97e647597baf246fd56c21bbfaf" -dependencies = [ - "anyhow", - "cc", - "dirs", - "libloading 0.7.4", - "once_cell", - "regex", - "serde", - "serde_json", - "tree-sitter", - "tree-sitter-highlight", - "tree-sitter-tags", -] - -[[package]] -name = "tree-sitter-tags" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccb3f1376219530a37a809751ecf65aa35fd8b9c1c4ab6d4faf5f6a9eeda2c05" -dependencies = [ - "memchr", - "regex", - "thiserror 1.0.64", - "tree-sitter", -] +checksum = "c4013970217383f67b18aef68f6fb2e8d409bc5755227092d32efb0422ba24b8" [[package]] name = "trim-in-place" @@ -13483,6 +13440,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -13503,12 +13461,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" -[[package]] -name = "utf8-width" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -13553,24 +13505,33 @@ dependencies = [ [[package]] name = "variadics" -version = "0.0.4" -source = "git+https://github.com/GreptimeTeam/hydroflow.git?branch=main#b072ee026f97f8537165e1fb247101e0ab2fb320" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d49933ddcb1a4ec2ebb9a2931bc526ed269a9276a456c0595ddcc7cb188896" dependencies = [ + "hashbrown 0.14.5", "sealed", ] +[[package]] +name = "variadics_macro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55476accb4f25086be8a1d18bed73aef9452cefb5ccd133e122e4350a1543ff3" +dependencies = [ + "proc-macro-crate 1.3.1", + "proc-macro2", + "quote", + "syn 2.0.100", + "variadics", +] + [[package]] name = "vcpkg" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "vec_map" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" - [[package]] name = "vergen" version = "8.3.2" @@ -13754,17 +13715,17 @@ dependencies = [ [[package]] name = "webbrowser" -version = "0.8.15" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db67ae75a9405634f5882791678772c94ff5f16a66535aae186e26aa0841fc8b" +checksum = "d5df295f8451142f1856b1bd86a606dfe9587d439bc036e319c827700dbd555e" dependencies = [ - "core-foundation", + "core-foundation 0.10.0", "home", "jni", "log", "ndk-context", - "objc", - "raw-window-handle", + "objc2", + "objc2-foundation", "url", "web-sys", ] @@ -13788,18 +13749,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix", -] - [[package]] name = "whoami" version = "1.5.2" diff --git a/src/flow/Cargo.toml b/src/flow/Cargo.toml index c416910440..ec16ca3c9f 100644 --- a/src/flow/Cargo.toml +++ b/src/flow/Cargo.toml @@ -16,6 +16,7 @@ async-trait.workspace = true bytes.workspace = true cache.workspace = true catalog.workspace = true +chrono.workspace = true client.workspace = true common-base.workspace = true common-config.workspace = true @@ -39,16 +40,13 @@ datafusion-expr.workspace = true datafusion-physical-expr.workspace = true datafusion-substrait.workspace = true datatypes.workspace = true +dfir_rs = "0.13.0" enum-as-inner = "0.6.0" enum_dispatch = "0.3" futures.workspace = true get-size2 = "0.1.2" greptime-proto.workspace = true -# This fork of hydroflow is simply for keeping our dependency in our org, and pin the version -# otherwise it is the same with upstream repo -chrono.workspace = true http.workspace = true -hydroflow = { git = "https://github.com/GreptimeTeam/hydroflow.git", branch = "main" } itertools.workspace = true lazy_static.workspace = true meta-client.workspace = true diff --git a/src/flow/src/adapter/worker.rs b/src/flow/src/adapter/worker.rs index 6713e232db..3be60cc7e6 100644 --- a/src/flow/src/adapter/worker.rs +++ b/src/flow/src/adapter/worker.rs @@ -19,8 +19,8 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use common_telemetry::info; +use dfir_rs::scheduled::graph::Dfir; use enum_as_inner::EnumAsInner; -use hydroflow::scheduled::graph::Hydroflow; use snafu::ensure; use tokio::sync::{broadcast, mpsc, oneshot, Mutex}; @@ -49,9 +49,9 @@ pub fn create_worker<'a>() -> (WorkerHandle, Worker<'a>) { (worker_handle, worker) } -/// ActiveDataflowState is a wrapper around `Hydroflow` and `DataflowState` +/// ActiveDataflowState is a wrapper around `Dfir` and `DataflowState` pub(crate) struct ActiveDataflowState<'subgraph> { - df: Hydroflow<'subgraph>, + df: Dfir<'subgraph>, state: DataflowState, err_collector: ErrCollector, } @@ -59,7 +59,7 @@ pub(crate) struct ActiveDataflowState<'subgraph> { impl std::fmt::Debug for ActiveDataflowState<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("ActiveDataflowState") - .field("df", &"") + .field("df", &"") .field("state", &self.state) .field("err_collector", &self.err_collector) .finish() @@ -69,7 +69,7 @@ impl std::fmt::Debug for ActiveDataflowState<'_> { impl Default for ActiveDataflowState<'_> { fn default() -> Self { ActiveDataflowState { - df: Hydroflow::new(), + df: Dfir::new(), state: DataflowState::default(), err_collector: ErrCollector::default(), } diff --git a/src/flow/src/batching_mode/engine.rs b/src/flow/src/batching_mode/engine.rs index fbfea7715c..a29d87ca0b 100644 --- a/src/flow/src/batching_mode/engine.rs +++ b/src/flow/src/batching_mode/engine.rs @@ -303,7 +303,7 @@ impl BatchingEngine { }) .transpose()?; - info!( + debug!( "Flow id={}, found time window expr={}", flow_id, phy_expr diff --git a/src/flow/src/batching_mode/task.rs b/src/flow/src/batching_mode/task.rs index bb1f296c90..af35152fec 100644 --- a/src/flow/src/batching_mode/task.rs +++ b/src/flow/src/batching_mode/task.rs @@ -192,7 +192,7 @@ impl BatchingTask { frontend_client: &Arc, ) -> Result, Error> { if let Some(new_query) = self.gen_insert_plan(engine).await? { - debug!("Generate new query: {:#?}", new_query); + debug!("Generate new query: {}", new_query); self.execute_logical_plan(frontend_client, &new_query).await } else { debug!("Generate no query"); diff --git a/src/flow/src/batching_mode/utils.rs b/src/flow/src/batching_mode/utils.rs index 117db03665..76793f85f4 100644 --- a/src/flow/src/batching_mode/utils.rs +++ b/src/flow/src/batching_mode/utils.rs @@ -138,9 +138,12 @@ impl TreeNodeVisitor<'_> for FindGroupByFinalName { fn f_down(&mut self, node: &Self::Node) -> datafusion_common::Result { if let LogicalPlan::Aggregate(aggregate) = node { self.group_exprs = Some(aggregate.group_expr.iter().cloned().collect()); - debug!("Group by exprs: {:?}", self.group_exprs); + debug!( + "FindGroupByFinalName: Get Group by exprs from Aggregate: {:?}", + self.group_exprs + ); } else if let LogicalPlan::Distinct(distinct) = node { - debug!("Distinct: {:#?}", distinct); + debug!("FindGroupByFinalName: Distinct: {}", node); match distinct { Distinct::All(input) => { if let LogicalPlan::TableScan(table_scan) = &**input { @@ -162,7 +165,10 @@ impl TreeNodeVisitor<'_> for FindGroupByFinalName { self.group_exprs = Some(distinct_on.on_expr.iter().cloned().collect()) } } - debug!("Group by exprs: {:?}", self.group_exprs); + debug!( + "FindGroupByFinalName: Get Group by exprs from Distinct: {:?}", + self.group_exprs + ); } Ok(TreeNodeRecursion::Continue) diff --git a/src/flow/src/compute/render.rs b/src/flow/src/compute/render.rs index b2d0c0c12d..5be923aeac 100644 --- a/src/flow/src/compute/render.rs +++ b/src/flow/src/compute/render.rs @@ -18,9 +18,9 @@ use std::collections::BTreeMap; -use hydroflow::scheduled::graph::Hydroflow; -use hydroflow::scheduled::graph_ext::GraphExt; -use hydroflow::scheduled::port::{PortCtx, SEND}; +use dfir_rs::scheduled::graph::Dfir; +use dfir_rs::scheduled::graph_ext::GraphExt; +use dfir_rs::scheduled::port::{PortCtx, SEND}; use itertools::Itertools; use snafu::OptionExt; @@ -38,7 +38,7 @@ mod src_sink; /// The Context for build a Operator with id of `GlobalId` pub struct Context<'referred, 'df> { pub id: GlobalId, - pub df: &'referred mut Hydroflow<'df>, + pub df: &'referred mut Dfir<'df>, pub compute_state: &'referred mut DataflowState, /// a list of all collections being used in the operator /// @@ -361,16 +361,16 @@ mod test { use std::cell::RefCell; use std::rc::Rc; - use hydroflow::scheduled::graph::Hydroflow; - use hydroflow::scheduled::graph_ext::GraphExt; - use hydroflow::scheduled::handoff::VecHandoff; + use dfir_rs::scheduled::graph::Dfir; + use dfir_rs::scheduled::graph_ext::GraphExt; + use dfir_rs::scheduled::handoff::VecHandoff; use pretty_assertions::assert_eq; use super::*; use crate::repr::Row; pub fn run_and_check( state: &mut DataflowState, - df: &mut Hydroflow, + df: &mut Dfir, time_range: std::ops::Range, expected: BTreeMap>, output: Rc>>, @@ -416,7 +416,7 @@ mod test { } pub fn harness_test_ctx<'r, 'h>( - df: &'r mut Hydroflow<'h>, + df: &'r mut Dfir<'h>, state: &'r mut DataflowState, ) -> Context<'r, 'h> { let err_collector = state.get_err_collector(); @@ -436,7 +436,7 @@ mod test { /// that is it only emit once, not multiple times #[test] fn test_render_constant() { - let mut df = Hydroflow::new(); + let mut df = Dfir::new(); let mut state = DataflowState::default(); let mut ctx = harness_test_ctx(&mut df, &mut state); @@ -473,7 +473,7 @@ mod test { /// a simple example to show how to use source and sink #[test] fn example_source_sink() { - let mut df = Hydroflow::new(); + let mut df = Dfir::new(); let (send_port, recv_port) = df.make_edge::<_, VecHandoff>("test_handoff"); df.add_subgraph_source("test_handoff_source", send_port, move |_ctx, send| { for i in 0..10 { @@ -498,8 +498,8 @@ mod test { #[test] fn test_tee_auto_schedule() { - use hydroflow::scheduled::handoff::TeeingHandoff as Toff; - let mut df = Hydroflow::new(); + use dfir_rs::scheduled::handoff::TeeingHandoff as Toff; + let mut df = Dfir::new(); let (send_port, recv_port) = df.make_edge::<_, Toff>("test_handoff"); let source = df.add_subgraph_source("test_handoff_source", send_port, move |_ctx, send| { for i in 0..10 { diff --git a/src/flow/src/compute/render/map.rs b/src/flow/src/compute/render/map.rs index eb0f8b6933..77636c5b3a 100644 --- a/src/flow/src/compute/render/map.rs +++ b/src/flow/src/compute/render/map.rs @@ -14,8 +14,8 @@ use std::collections::BTreeMap; -use hydroflow::scheduled::graph_ext::GraphExt; -use hydroflow::scheduled::port::{PortCtx, SEND}; +use dfir_rs::scheduled::graph_ext::GraphExt; +use dfir_rs::scheduled::port::{PortCtx, SEND}; use itertools::Itertools; use snafu::OptionExt; @@ -256,7 +256,7 @@ fn eval_mfp_core( mod test { use datatypes::data_type::ConcreteDataType; - use hydroflow::scheduled::graph::Hydroflow; + use dfir_rs::scheduled::graph::Dfir; use super::*; use crate::compute::render::test::{get_output_handle, harness_test_ctx, run_and_check}; @@ -269,7 +269,7 @@ mod test { /// namely: if mfp operator can schedule a delete at the correct time #[test] fn test_render_mfp_with_temporal() { - let mut df = Hydroflow::new(); + let mut df = Dfir::new(); let mut state = DataflowState::default(); let mut ctx = harness_test_ctx(&mut df, &mut state); @@ -348,7 +348,7 @@ mod test { /// that is it filter the rows correctly #[test] fn test_render_mfp() { - let mut df = Hydroflow::new(); + let mut df = Dfir::new(); let mut state = DataflowState::default(); let mut ctx = harness_test_ctx(&mut df, &mut state); @@ -388,7 +388,7 @@ mod test { /// test if mfp operator can run multiple times within same tick #[test] fn test_render_mfp_multiple_times() { - let mut df = Hydroflow::new(); + let mut df = Dfir::new(); let mut state = DataflowState::default(); let mut ctx = harness_test_ctx(&mut df, &mut state); diff --git a/src/flow/src/compute/render/reduce.rs b/src/flow/src/compute/render/reduce.rs index 0bbc613260..78bc027ee4 100644 --- a/src/flow/src/compute/render/reduce.rs +++ b/src/flow/src/compute/render/reduce.rs @@ -22,7 +22,7 @@ use datatypes::data_type::ConcreteDataType; use datatypes::prelude::DataType; use datatypes::value::{ListValue, Value}; use datatypes::vectors::{BooleanVector, NullVector}; -use hydroflow::scheduled::graph_ext::GraphExt; +use dfir_rs::scheduled::graph_ext::GraphExt; use itertools::Itertools; use snafu::{ensure, OptionExt, ResultExt}; @@ -1212,7 +1212,7 @@ mod test { use common_time::Timestamp; use datatypes::data_type::{ConcreteDataType, ConcreteDataType as CDT}; - use hydroflow::scheduled::graph::Hydroflow; + use dfir_rs::scheduled::graph::Dfir; use super::*; use crate::compute::render::test::{get_output_handle, harness_test_ctx, run_and_check}; @@ -1228,7 +1228,7 @@ mod test { /// expected: sum(number), window_start, window_end #[test] fn test_tumble_group_by() { - let mut df = Hydroflow::new(); + let mut df = Dfir::new(); let mut state = DataflowState::default(); let mut ctx = harness_test_ctx(&mut df, &mut state); const START: i64 = 1625097600000; @@ -1389,7 +1389,7 @@ mod test { /// select avg(number) from number; #[test] fn test_avg_eval() { - let mut df = Hydroflow::new(); + let mut df = Dfir::new(); let mut state = DataflowState::default(); let mut ctx = harness_test_ctx(&mut df, &mut state); @@ -1500,7 +1500,7 @@ mod test { /// | col | Int64 | #[test] fn test_basic_distinct() { - let mut df = Hydroflow::new(); + let mut df = Dfir::new(); let mut state = DataflowState::default(); let mut ctx = harness_test_ctx(&mut df, &mut state); @@ -1556,7 +1556,7 @@ mod test { /// | col | Int64 | #[test] fn test_basic_batch_reduce_accum() { - let mut df = Hydroflow::new(); + let mut df = Dfir::new(); let mut state = DataflowState::default(); let now = state.current_time_ref(); let mut ctx = harness_test_ctx(&mut df, &mut state); @@ -1662,7 +1662,7 @@ mod test { /// | col | Int64 | #[test] fn test_basic_reduce_accum() { - let mut df = Hydroflow::new(); + let mut df = Dfir::new(); let mut state = DataflowState::default(); let mut ctx = harness_test_ctx(&mut df, &mut state); @@ -1739,7 +1739,7 @@ mod test { /// this test include even more insert/delete case to cover all case for eval_distinct_core #[test] fn test_delete_reduce_distinct_accum() { - let mut df = Hydroflow::new(); + let mut df = Dfir::new(); let mut state = DataflowState::default(); let mut ctx = harness_test_ctx(&mut df, &mut state); @@ -1818,7 +1818,7 @@ mod test { /// this test include insert and delete which should cover all case for eval_distinct_core #[test] fn test_basic_reduce_distinct_accum() { - let mut df = Hydroflow::new(); + let mut df = Dfir::new(); let mut state = DataflowState::default(); let mut ctx = harness_test_ctx(&mut df, &mut state); @@ -1896,7 +1896,7 @@ mod test { /// | col | Int64 | #[test] fn test_composite_reduce_distinct_accum() { - let mut df = Hydroflow::new(); + let mut df = Dfir::new(); let mut state = DataflowState::default(); let mut ctx = harness_test_ctx(&mut df, &mut state); diff --git a/src/flow/src/compute/render/src_sink.rs b/src/flow/src/compute/render/src_sink.rs index cc8cf01ff7..1bf3699ceb 100644 --- a/src/flow/src/compute/render/src_sink.rs +++ b/src/flow/src/compute/render/src_sink.rs @@ -17,7 +17,7 @@ use std::collections::BTreeMap; use common_telemetry::{debug, trace}; -use hydroflow::scheduled::graph_ext::GraphExt; +use dfir_rs::scheduled::graph_ext::GraphExt; use itertools::Itertools; use snafu::OptionExt; use tokio::sync::broadcast::error::TryRecvError; diff --git a/src/flow/src/compute/state.rs b/src/flow/src/compute/state.rs index ecd3415f37..4728503e4e 100644 --- a/src/flow/src/compute/state.rs +++ b/src/flow/src/compute/state.rs @@ -16,16 +16,16 @@ use std::cell::RefCell; use std::collections::{BTreeMap, VecDeque}; use std::rc::Rc; +use dfir_rs::scheduled::graph::Dfir; +use dfir_rs::scheduled::SubgraphId; use get_size2::GetSize; -use hydroflow::scheduled::graph::Hydroflow; -use hydroflow::scheduled::SubgraphId; use crate::compute::types::ErrCollector; use crate::repr::{self, Timestamp}; use crate::utils::{ArrangeHandler, Arrangement}; /// input/output of a dataflow -/// One `ComputeState` manage the input/output/schedule of one `Hydroflow` +/// One `ComputeState` manage the input/output/schedule of one `Dfir` #[derive(Debug, Default)] pub struct DataflowState { /// it is important to use a deque to maintain the order of subgraph here @@ -38,7 +38,7 @@ pub struct DataflowState { /// Which means it's also the current time in temporal filter to get current correct result as_of: Rc>, /// error collector local to this `ComputeState`, - /// useful for distinguishing errors from different `Hydroflow` + /// useful for distinguishing errors from different `Dfir` err_collector: ErrCollector, /// save all used arrange in this dataflow, since usually there is no delete operation /// we can just keep track of all used arrange and schedule subgraph when they need to be updated @@ -65,7 +65,7 @@ impl DataflowState { /// schedule all subgraph that need to run with time <= `as_of` and run_available() /// /// return true if any subgraph actually executed - pub fn run_available_with_schedule(&mut self, df: &mut Hydroflow) -> bool { + pub fn run_available_with_schedule(&mut self, df: &mut Dfir) -> bool { // first split keys <= as_of into another map let mut before = self .schedule_subgraph diff --git a/src/flow/src/compute/types.rs b/src/flow/src/compute/types.rs index b92ec30f9f..63ad143840 100644 --- a/src/flow/src/compute/types.rs +++ b/src/flow/src/compute/types.rs @@ -18,10 +18,10 @@ use std::rc::Rc; use std::sync::Arc; use common_error::ext::ErrorExt; -use hydroflow::scheduled::graph::Hydroflow; -use hydroflow::scheduled::handoff::TeeingHandoff; -use hydroflow::scheduled::port::RecvPort; -use hydroflow::scheduled::SubgraphId; +use dfir_rs::scheduled::graph::Dfir; +use dfir_rs::scheduled::handoff::TeeingHandoff; +use dfir_rs::scheduled::port::RecvPort; +use dfir_rs::scheduled::SubgraphId; use itertools::Itertools; use tokio::sync::Mutex; @@ -46,7 +46,7 @@ impl Collection { /// clone a collection, require a mutable reference to the hydroflow instance /// /// Note: need to be the same hydroflow instance that this collection is created from - pub fn clone(&self, df: &mut Hydroflow) -> Self { + pub fn clone(&self, df: &mut Dfir) -> Self { Collection { stream: self.stream.tee(df), } @@ -151,7 +151,7 @@ impl CollectionBundle { } impl CollectionBundle { - pub fn clone(&self, df: &mut Hydroflow) -> Self { + pub fn clone(&self, df: &mut Dfir) -> Self { Self { collection: self.collection.clone(df), arranged: self diff --git a/src/flow/src/expr/scalar.rs b/src/flow/src/expr/scalar.rs index f0abe20f6c..7b31e5a5df 100644 --- a/src/flow/src/expr/scalar.rs +++ b/src/flow/src/expr/scalar.rs @@ -21,7 +21,7 @@ use common_error::ext::BoxedError; use datatypes::prelude::{ConcreteDataType, DataType}; use datatypes::value::Value; use datatypes::vectors::{BooleanVector, Helper, VectorRef}; -use hydroflow::lattices::cc_traits::Iter; +use dfir_rs::lattices::cc_traits::Iter; use itertools::Itertools; use snafu::{ensure, OptionExt, ResultExt};