From 0831badd024bb7b8bdfb3c99676c3c2b1725dd06 Mon Sep 17 00:00:00 2001 From: Huaijin Date: Tue, 16 Jun 2026 21:27:26 +0800 Subject: [PATCH] perf: upgrade datafusion to 54 (#12665) - [x] ~add sort when to many groups, we can do this, but for now do not do this~ - [x] remove the unsafe for memory pool - [x] add approx_distinct improve, already fork, will add in next pr - [x] fork vortex to add cast fix, and export `.exclude_schemes([IntDictScheme.id()])` - [x] ~enable_physical_uncorrelated_scalar_subquery test~ --- Cargo.lock | 563 ++++++++++-------- Cargo.toml | 10 +- src/config/Cargo.toml | 2 +- src/config/src/utils/file.rs | 4 +- src/config/src/utils/parquet.rs | 20 +- src/handler/grpc/flight/visitor.rs | 44 +- .../codec/aggregate_topk_exec.rs | 6 +- .../codec/deduplication_exec.rs | 13 +- .../distributed_plan/codec/empty_exec.rs | 6 +- .../distributed_plan/codec/enrichment_exec.rs | 6 +- .../codec/physical_plan_node.rs | 20 +- .../codec/streaming_aggs_exec.rs | 19 +- .../distributed_plan/codec/tmp_exec.rs | 6 +- .../distribute_analyze_exec.rs | 7 +- .../datafusion/distributed_plan/empty_exec.rs | 13 +- .../distributed_plan/enrich_exec.rs | 13 +- .../distributed_plan/enrichment_exec.rs | 6 +- .../search/datafusion/distributed_plan/mod.rs | 6 +- .../distributed_plan/remote_scan_exec.rs | 13 +- .../datafusion/distributed_plan/rewrite.rs | 2 +- src/service/search/datafusion/exec.rs | 42 +- .../limit_join_right_side.rs | 17 +- .../physical_optimizer/distribute_analyze.rs | 7 +- .../physical_optimizer/enrichment.rs | 8 +- .../optimizer/physical_optimizer/index.rs | 10 +- .../index_optimizer/count.rs | 2 +- .../index_optimizer/distinct.rs | 10 +- .../index_optimizer/histogram.rs | 18 +- .../physical_optimizer/index_optimizer/mod.rs | 24 +- .../index_optimizer/select.rs | 2 +- .../index_optimizer/topn.rs | 6 +- .../index_optimizer/utils.rs | 6 +- .../physical_optimizer/join_reorder.rs | 2 +- .../physical_optimizer/remote_scan.rs | 19 +- .../physical_optimizer/rewrite_match.rs | 21 +- .../optimizer/physical_optimizer/utils.rs | 22 +- .../search/datafusion/optimizer/utils.rs | 11 +- .../search/datafusion/peak_memory_pool.rs | 19 +- .../datafusion/plan/deduplication_exec.rs | 10 +- .../datafusion/plan/tantivy_optimize_exec.rs | 6 +- .../storage/file_statistics_cache.rs | 289 +++++---- .../datafusion/table_provider/catalog.rs | 8 - .../datafusion/table_provider/empty_table.rs | 6 +- .../datafusion/table_provider/enrich_table.rs | 6 +- .../table_provider/listing_adapter.rs | 11 +- .../datafusion/table_provider/memtable.rs | 6 +- .../datafusion/table_provider/uniontable.rs | 6 +- .../datafusion/udaf/summary_percentile.rs | 4 - .../datafusion/udf/cast_to_timestamp_udf.rs | 6 +- .../search/datafusion/udf/histogram_udf.rs | 6 +- .../search/datafusion/udf/match_all_udf.rs | 10 +- .../datafusion/udf/regexp_matches_udf.rs | 9 +- .../search/datafusion/udf/regexp_udf.rs | 4 - .../search/datafusion/udf/str_match_udf.rs | 13 +- src/service/search/index.rs | 20 +- .../search/sql/rewriter/track_total_hits.rs | 4 +- src/service/search/sql/visitor/group_by.rs | 2 +- src/service/search/utils.rs | 2 +- src/service/tantivy/reader/vortex.rs | 5 +- .../tests/vortex/test_schema_evolution.py | 10 +- 60 files changed, 715 insertions(+), 753 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d61a3d6a1..84e1fce02e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,7 +23,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] @@ -50,7 +50,7 @@ dependencies = [ "cmac", "ctr", "dbl", - "digest", + "digest 0.10.7", "zeroize", ] @@ -225,9 +225,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.8.2" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" dependencies = [ "rustversion", ] @@ -1062,7 +1062,7 @@ dependencies = [ "http 0.2.12", "http 1.4.0", "percent-encoding", - "sha2", + "sha2 0.10.9", "time", "tracing", ] @@ -1571,7 +1571,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -1598,6 +1598,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block-padding" version = "0.3.3" @@ -2054,7 +2063,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", "zeroize", ] @@ -2142,7 +2151,7 @@ checksum = "8543454e3c3f5126effff9cd44d562af4e31fb8ce1cc0d3dcd8f084515dbc1aa" dependencies = [ "cipher", "dbl", - "digest", + "digest 0.10.7", ] [[package]] @@ -2162,7 +2171,7 @@ checksum = "fe6d2e5af09e8c8ad56c969f2157a3d4238cebc7c55f0a517728c38f7b200f81" dependencies = [ "serde", "termcolor", - "unicode-width 0.2.2", + "unicode-width 0.1.14", ] [[package]] @@ -2385,6 +2394,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -2494,6 +2509,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + [[package]] name = "cpp_demangle" version = "0.5.1" @@ -2651,6 +2672,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "crypto_secretbox" version = "0.1.1" @@ -2705,7 +2735,7 @@ dependencies = [ "cfg-if", "cpufeatures 0.2.17", "curve25519-dalek-derive", - "digest", + "digest 0.10.7", "fiat-crypto", "rustc_version", "subtle", @@ -2826,14 +2856,13 @@ checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" [[package]] name = "datafusion" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b" +checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040" dependencies = [ "arrow", "arrow-schema", "async-trait", - "bytes", "bzip2", "chrono", "datafusion-catalog", @@ -2863,14 +2892,13 @@ dependencies = [ "datafusion-sql", "flate2", "futures", + "indexmap", "itertools 0.14.0", "liblzma", "log", "object_store", "parking_lot 0.12.5", "parquet", - "rand 0.9.2", - "regex", "sqlparser", "tempfile", "tokio", @@ -2881,9 +2909,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66" +checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139" dependencies = [ "arrow", "async-trait", @@ -2906,9 +2934,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" +checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891" dependencies = [ "arrow", "async-trait", @@ -2929,34 +2957,35 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" +checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65" dependencies = [ - "ahash 0.8.12", "arrow", "arrow-ipc", + "arrow-schema", "chrono", + "foldhash 0.2.0", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap", "itertools 0.14.0", "libc", "log", "object_store", "parquet", - "paste", "recursive", "sqlparser", "tokio", + "uuid", "web-time", ] [[package]] name = "datafusion-common-runtime" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def" +checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1" dependencies = [ "futures", "log", @@ -2965,9 +2994,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" +checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe" dependencies = [ "arrow", "async-compression", @@ -2991,6 +3020,7 @@ dependencies = [ "liblzma", "log", "object_store", + "parking_lot 0.12.5", "rand 0.9.2", "tokio", "tokio-util", @@ -3000,9 +3030,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096" +checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f" dependencies = [ "arrow", "arrow-ipc", @@ -3024,9 +3054,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca" +checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada" dependencies = [ "arrow", "async-trait", @@ -3047,9 +3077,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2" +checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f" dependencies = [ "arrow", "async-trait", @@ -3064,16 +3094,15 @@ dependencies = [ "datafusion-session", "futures", "object_store", - "serde_json", "tokio", "tokio-stream", ] [[package]] name = "datafusion-datasource-parquet" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997" +checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450" dependencies = [ "arrow", "async-trait", @@ -3083,6 +3112,7 @@ dependencies = [ "datafusion-datasource", "datafusion-execution", "datafusion-expr", + "datafusion-functions", "datafusion-functions-aggregate-common", "datafusion-physical-expr", "datafusion-physical-expr-adapter", @@ -3101,20 +3131,19 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" +checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e" [[package]] name = "datafusion-execution" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709" +checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0" dependencies = [ "arrow", "arrow-buffer", "async-trait", - "chrono", "dashmap", "datafusion-common", "datafusion-expr", @@ -3130,11 +3159,12 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" +checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0" dependencies = [ "arrow", + "arrow-schema", "async-trait", "chrono", "datafusion-common", @@ -3145,7 +3175,6 @@ dependencies = [ "datafusion-physical-expr-common", "indexmap", "itertools 0.14.0", - "paste", "recursive", "serde_json", "sqlparser", @@ -3153,22 +3182,21 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" +checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2" dependencies = [ "arrow", "datafusion-common", "indexmap", "itertools 0.14.0", - "paste", ] [[package]] name = "datafusion-functions" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" +checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3" dependencies = [ "arrow", "arrow-buffer", @@ -3183,26 +3211,25 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-macros", + "datafusion-physical-expr-common", "hex", "itertools 0.14.0", "log", - "md-5", + "md-5 0.11.0", "memchr", "num-traits", "rand 0.9.2", "regex", - "sha2", - "unicode-segmentation", + "sha2 0.11.0", "uuid", ] [[package]] name = "datafusion-functions-aggregate" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" +checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09" dependencies = [ - "ahash 0.8.12", "arrow", "datafusion-common", "datafusion-doc", @@ -3212,19 +3239,18 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", + "foldhash 0.2.0", "half", "log", "num-traits", - "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" +checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c" dependencies = [ - "ahash 0.8.12", "arrow", "datafusion-common", "datafusion-expr-common", @@ -3233,21 +3259,22 @@ dependencies = [ [[package]] name = "datafusion-functions-json" -version = "0.53.0" +version = "0.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75184ea8f2a291183525f40f8b86e2e596655fa052b7d9ab23af97b19e332c31" +checksum = "fc833cd1d6d8d84516f53d702f2986d09ef58a2053ad63a3a3324aa73a915f3b" dependencies = [ "datafusion", "jiter", "log", "paste", + "serde_json", ] [[package]] name = "datafusion-functions-nested" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" +checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5" dependencies = [ "arrow", "arrow-ord", @@ -3261,34 +3288,34 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "itertools 0.14.0", "itoa", "log", - "paste", + "memchr", ] [[package]] name = "datafusion-functions-table" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" +checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26" dependencies = [ "arrow", "async-trait", "datafusion-catalog", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-plan", "parking_lot 0.12.5", - "paste", ] [[package]] name = "datafusion-functions-window" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" +checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78" dependencies = [ "arrow", "datafusion-common", @@ -3299,14 +3326,13 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "log", - "paste", ] [[package]] name = "datafusion-functions-window-common" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" +checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -3314,9 +3340,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" +checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5" dependencies = [ "datafusion-doc", "quote", @@ -3325,9 +3351,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" +checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281" dependencies = [ "arrow", "chrono", @@ -3345,11 +3371,10 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" +checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c" dependencies = [ - "ahash 0.8.12", "arrow", "datafusion-common", "datafusion-expr", @@ -3357,11 +3382,10 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap", "itertools 0.14.0", "parking_lot 0.12.5", - "paste", "petgraph 0.8.3", "recursive", "tokio", @@ -3369,9 +3393,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" +checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235" dependencies = [ "arrow", "datafusion-common", @@ -3384,26 +3408,26 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362" +checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8" dependencies = [ - "ahash 0.8.12", "arrow", "chrono", "datafusion-common", "datafusion-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap", "itertools 0.14.0", "parking_lot 0.12.5", + "pin-project", ] [[package]] name = "datafusion-physical-optimizer" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" +checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36" dependencies = [ "arrow", "datafusion-common", @@ -3420,12 +3444,13 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79" +checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc" dependencies = [ - "ahash 0.8.12", "arrow", + "arrow-data", + "arrow-ipc", "arrow-ord", "arrow-schema", "async-trait", @@ -3440,7 +3465,7 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap", "itertools 0.14.0", "log", @@ -3452,9 +3477,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677ee4448a010ed5faeff8d73ff78972c2ace59eff3cd7bd15833a1dafa00492" +checksum = "9dd15a1ba5d3af93808241065c6c44dbca8296a189845e8a587c45c07bf0ffae" dependencies = [ "arrow", "chrono", @@ -3475,14 +3500,13 @@ dependencies = [ "datafusion-proto-common", "object_store", "prost 0.14.3", - "rand 0.9.2", ] [[package]] name = "datafusion-proto-common" -version = "53.0.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "965eca01edc8259edbbd95883a00b6d81e329fd44a019cfac3a03b026a83eade" +checksum = "90042982cf9462eb06a0b81f92efa4188dae871e7ea3ab8dc61aa9c9349b2530" dependencies = [ "arrow", "datafusion-common", @@ -3491,9 +3515,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" +checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3" dependencies = [ "arrow", "datafusion-common", @@ -3502,15 +3526,14 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "itertools 0.14.0", "log", ] [[package]] name = "datafusion-session" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" +checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417" dependencies = [ "async-trait", "datafusion-common", @@ -3522,9 +3545,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "53.1.0" +version = "54.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1" +checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099" dependencies = [ "arrow", "bigdecimal", @@ -3576,7 +3599,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "pem-rfc7468", "zeroize", ] @@ -3689,12 +3712,23 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", +] + [[package]] name = "directories" version = "5.0.1" @@ -3884,7 +3918,7 @@ checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" dependencies = [ "curve25519-dalek", "ed25519", - "sha2", + "sha2 0.10.9", "signature", "subtle", ] @@ -4159,12 +4193,13 @@ checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" [[package]] name = "fastlanes" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414cb755aee48ff7b0907995d2949c68c8c17900970076dff6a808e18e592d71" +checksum = "20c597e23b8ec8506f589d18bc701ca83a3def6086748f628ad23092e1dfe577" dependencies = [ "arrayref", "const_for", + "core_detect", "num-traits", "paste", "seq-macro", @@ -4366,9 +4401,12 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst-rs" -version = "0.5.6" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561f2458a3407836ab8f1acc9113b8cda91b9d6378ba8dad13b2fe1a1d3af5ce" +checksum = "9b13ac798afc0d9194eb4efefef8b9332efbd80b43f302a968cb8cb23b9d5360" +dependencies = [ + "rustc-hash 2.1.1", +] [[package]] name = "funty" @@ -4644,12 +4682,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "handle" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ed72152641d513a6084a3907bfcba3f35ae2d3335c22ce2242969c25ff8e46" - [[package]] name = "hashbrown" version = "0.12.3" @@ -4665,6 +4697,7 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ + "ahash 0.8.12", "allocator-api2", ] @@ -4697,6 +4730,11 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "hashlink" @@ -4792,7 +4830,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -4912,6 +4950,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.8.1" @@ -5135,12 +5182,12 @@ checksum = "cd62e6b5e86ea8eeeb8db1de02880a6abc01a397b2ebb64b5d74ac255318f5cb" [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -5467,9 +5514,9 @@ dependencies = [ [[package]] name = "jiter" -version = "0.13.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020ba671987d7444d251d3ee5340be1bf4606cd6c0b53e6f4066b5a1ee376b22" +checksum = "7866f284df68dbc242796251ed9768bde2b58ebc4d7d32cc07cc7fd993e3ed6c" dependencies = [ "ahash 0.8.12", "bitvec", @@ -5616,6 +5663,16 @@ dependencies = [ "rustversion", ] +[[package]] +name = "lasso" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e14eda50a3494b3bf7b9ce51c52434a761e383d7238ce1dd5dcec2fbc13e9fb" +dependencies = [ + "dashmap", + "hashbrown 0.14.5", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -5755,9 +5812,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.182" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libloading" @@ -5976,7 +6033,7 @@ version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47bb1e988e6fb779cf720ad431242d3f03167c1b3f2b1aae7f1a94b2495b36ae" dependencies = [ - "sha2", + "sha2 0.10.9", ] [[package]] @@ -6054,7 +6111,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", ] [[package]] @@ -6139,9 +6206,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.1" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", @@ -6191,28 +6258,6 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" -[[package]] -name = "multiversion" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edb7f0ff51249dfda9ab96b5823695e15a052dc15074c9dbf3d118afaf2c201" -dependencies = [ - "multiversion-macros", - "target-features", -] - -[[package]] -name = "multiversion-macros" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b093064383341eb3271f42e381cb8f10a01459478446953953c75d24bd339fc0" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", - "target-features", -] - [[package]] name = "munge" version = "0.4.7" @@ -6659,7 +6704,7 @@ dependencies = [ "humantime", "hyper", "itertools 0.14.0", - "md-5", + "md-5 0.10.6", "parking_lot 0.12.5", "percent-encoding", "quick-xml 0.39.2", @@ -7222,7 +7267,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ - "digest", + "digest 0.10.7", "hmac", ] @@ -7309,7 +7354,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220" dependencies = [ "pest", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -7780,7 +7825,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.117", @@ -7793,7 +7838,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.117", @@ -8828,8 +8873,8 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "const-oid", - "digest", + "const-oid 0.9.6", + "digest 0.10.7", "num-bigint-dig", "num-integer", "num-traits", @@ -8891,7 +8936,7 @@ dependencies = [ "chrono", "globset", "new_mime_guess", - "sha2", + "sha2 0.10.9", "walkdir", ] @@ -8914,7 +8959,7 @@ version = "8.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5bcdef0be6fe7f6fa333b1073c949729274b05f123a0ad7efcb8efd878e5c3b1" dependencies = [ - "sha2", + "sha2 0.10.9", "walkdir", ] @@ -9500,7 +9545,7 @@ checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", ] [[package]] @@ -9511,7 +9556,7 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", ] [[package]] @@ -9522,7 +9567,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -9534,7 +9590,7 @@ dependencies = [ "async-trait", "bytes", "hex", - "sha2", + "sha2 0.10.9", "tokio", ] @@ -9544,7 +9600,7 @@ version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" dependencies = [ - "digest", + "digest 0.10.7", "keccak", ] @@ -9600,7 +9656,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest", + "digest 0.10.7", "rand_core 0.6.4", ] @@ -9749,12 +9805,12 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" -version = "0.6.2" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -9817,9 +9873,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.61.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ "log", "recursive", @@ -9880,7 +9936,7 @@ dependencies = [ "rustls", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", "thiserror 2.0.18", "time", @@ -9920,7 +9976,7 @@ dependencies = [ "quote", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "sqlx-core", "sqlx-mysql", "sqlx-postgres", @@ -9944,7 +10000,7 @@ dependencies = [ "bytes", "chrono", "crc", - "digest", + "digest 0.10.7", "dotenvy", "either", "futures-channel", @@ -9957,7 +10013,7 @@ dependencies = [ "hmac", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "once_cell", "percent-encoding", @@ -9966,7 +10022,7 @@ dependencies = [ "rust_decimal", "serde", "sha1", - "sha2", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", @@ -10001,7 +10057,7 @@ dependencies = [ "home", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "num-bigint", "once_cell", @@ -10009,7 +10065,7 @@ dependencies = [ "rust_decimal", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", @@ -10464,12 +10520,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" -[[package]] -name = "target-features" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1bbb9f3c5c463a01705937a24fdabc5047929ac764b2d5b9cf681c1f5041ed5" - [[package]] name = "target-lexicon" version = "0.13.5" @@ -10710,9 +10760,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.50.0" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -10728,9 +10778,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", @@ -11121,9 +11171,9 @@ checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "typetag" @@ -11270,7 +11320,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] @@ -11404,9 +11454,9 @@ checksum = "e2eebbbfe4093922c2b6734d7c679ebfebd704a0d7e56dfcb0d05818ce28977d" [[package]] name = "uuid" -version = "1.21.0" +version = "1.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -11480,7 +11530,7 @@ dependencies = [ [[package]] name = "vortex" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "vortex-alp", "vortex-array", @@ -11514,7 +11564,7 @@ dependencies = [ [[package]] name = "vortex-alp" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "itertools 0.14.0", "num-traits", @@ -11532,8 +11582,9 @@ dependencies = [ [[package]] name = "vortex-array" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ + "arc-swap", "arcref", "arrow-arith", "arrow-array", @@ -11550,13 +11601,11 @@ dependencies = [ "enum-iterator", "flatbuffers", "futures", - "getrandom 0.4.2", "half", "humansize", "inventory", "itertools 0.14.0", "jiff", - "multiversion", "num-traits", "num_enum", "parking_lot 0.12.5", @@ -11566,10 +11615,12 @@ dependencies = [ "rand 0.10.1", "rustc-hash 2.1.1", "simdutf8", + "smallvec", "static_assertions", "termtree", "tracing", "uuid", + "vortex-array-macros", "vortex-buffer", "vortex-error", "vortex-flatbuffers", @@ -11579,28 +11630,34 @@ dependencies = [ "vortex-utils", ] +[[package]] +name = "vortex-array-macros" +version = "0.1.0" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "vortex-btrblocks" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ - "enum-iterator", - "getrandom 0.4.2", "itertools 0.14.0", "num-traits", "pco", "rand 0.10.1", - "rustc-hash 2.1.1", - "tracing", "vortex-alp", "vortex-array", "vortex-buffer", + "vortex-compressor", "vortex-datetime-parts", "vortex-decimal-byte-parts", "vortex-error", "vortex-fastlanes", "vortex-fsst", - "vortex-mask", "vortex-pco", "vortex-runend", "vortex-sequence", @@ -11613,7 +11670,7 @@ dependencies = [ [[package]] name = "vortex-buffer" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "arrow-buffer", "bitvec", @@ -11626,19 +11683,38 @@ dependencies = [ [[package]] name = "vortex-bytebool" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "num-traits", "vortex-array", "vortex-buffer", "vortex-error", + "vortex-mask", "vortex-session", ] +[[package]] +name = "vortex-compressor" +version = "0.1.0" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" +dependencies = [ + "itertools 0.14.0", + "num-traits", + "parking_lot 0.12.5", + "rand 0.10.1", + "rustc-hash 2.1.1", + "tracing", + "vortex-array", + "vortex-buffer", + "vortex-error", + "vortex-mask", + "vortex-utils", +] + [[package]] name = "vortex-datetime-parts" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "num-traits", "prost 0.14.3", @@ -11652,7 +11728,7 @@ dependencies = [ [[package]] name = "vortex-decimal-byte-parts" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "num-traits", "prost 0.14.3", @@ -11666,7 +11742,7 @@ dependencies = [ [[package]] name = "vortex-error" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "arrow-schema", "flatbuffers", @@ -11678,9 +11754,8 @@ dependencies = [ [[package]] name = "vortex-fastlanes" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ - "arrayref", "fastlanes", "itertools 0.14.0", "lending-iterator", @@ -11696,13 +11771,12 @@ dependencies = [ [[package]] name = "vortex-file" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "async-trait", "bytes", "flatbuffers", "futures", - "getrandom 0.4.2", "itertools 0.14.0", "kanal", "moka", @@ -11711,7 +11785,6 @@ dependencies = [ "pin-project-lite", "tokio", "tracing", - "uuid", "vortex-alp", "vortex-array", "vortex-btrblocks", @@ -11741,7 +11814,7 @@ dependencies = [ [[package]] name = "vortex-flatbuffers" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "flatbuffers", "vortex-buffer", @@ -11751,7 +11824,7 @@ dependencies = [ [[package]] name = "vortex-fsst" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "fsst-rs", "prost 0.14.3", @@ -11765,7 +11838,7 @@ dependencies = [ [[package]] name = "vortex-io" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "async-fs", "async-stream", @@ -11773,9 +11846,7 @@ dependencies = [ "bytes", "custom-labels", "futures", - "getrandom 0.4.2", "glob", - "handle", "kanal", "oneshot 0.2.1", "parking_lot 0.12.5", @@ -11788,13 +11859,14 @@ dependencies = [ "vortex-error", "vortex-metrics", "vortex-session", + "vortex-utils", "wasm-bindgen-futures", ] [[package]] name = "vortex-ipc" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "bytes", "flatbuffers", @@ -11811,7 +11883,7 @@ dependencies = [ [[package]] name = "vortex-layout" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "arcref", "arrow-array", @@ -11843,6 +11915,7 @@ dependencies = [ "vortex-io", "vortex-mask", "vortex-metrics", + "vortex-runend", "vortex-scan", "vortex-sequence", "vortex-session", @@ -11852,7 +11925,7 @@ dependencies = [ [[package]] name = "vortex-mask" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "itertools 0.14.0", "vortex-buffer", @@ -11862,9 +11935,8 @@ dependencies = [ [[package]] name = "vortex-metrics" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ - "getrandom 0.4.2", "parking_lot 0.12.5", "sketches-ddsketch", ] @@ -11872,7 +11944,7 @@ dependencies = [ [[package]] name = "vortex-pco" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "pco", "prost 0.14.3", @@ -11886,7 +11958,7 @@ dependencies = [ [[package]] name = "vortex-proto" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "prost 0.14.3", "prost-types 0.14.3", @@ -11895,7 +11967,7 @@ dependencies = [ [[package]] name = "vortex-runend" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "arrow-array", "itertools 0.14.0", @@ -11911,7 +11983,7 @@ dependencies = [ [[package]] name = "vortex-scan" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "async-trait", "futures", @@ -11927,10 +11999,11 @@ dependencies = [ [[package]] name = "vortex-sequence" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "num-traits", "prost 0.14.3", + "smallvec", "vortex-array", "vortex-buffer", "vortex-error", @@ -11942,10 +12015,10 @@ dependencies = [ [[package]] name = "vortex-session" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ - "arcref", "dashmap", + "lasso", "parking_lot 0.12.5", "vortex-error", "vortex-utils", @@ -11954,7 +12027,7 @@ dependencies = [ [[package]] name = "vortex-sparse" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "itertools 0.14.0", "num-traits", @@ -11969,17 +12042,17 @@ dependencies = [ [[package]] name = "vortex-utils" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "dashmap", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "vortex-error", ] [[package]] name = "vortex-zigzag" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "vortex-array", "vortex-buffer", @@ -11992,7 +12065,7 @@ dependencies = [ [[package]] name = "vortex-zstd" version = "0.1.0" -source = "git+https://github.com/openobserve/vortex?rev=13423ca204f81dc1b20ca040da2aeb386220a312#13423ca204f81dc1b20ca040da2aeb386220a312" +source = "git+https://github.com/openobserve/vortex?branch=0.75.0-fix#120c122e211b4dfd0bfc97231d3a27825a6d25d2" dependencies = [ "itertools 0.14.0", "prost 0.14.3", @@ -12035,7 +12108,7 @@ dependencies = [ "crypto_secretbox", "csv", "ctr", - "digest", + "digest 0.10.7", "dns-lookup", "domain", "dyn-clone", @@ -12058,7 +12131,7 @@ dependencies = [ "lalrpop", "lalrpop-util", "lz4_flex 0.11.6", - "md-5", + "md-5 0.10.6", "nom 8.0.0", "nom-language", "ofb", @@ -12093,7 +12166,7 @@ dependencies = [ "serde_yaml", "serde_yaml_ng", "sha-1", - "sha2", + "sha2 0.10.9", "sha3", "simdutf8", "snafu 0.8.9", @@ -12450,7 +12523,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 5da5154434..9969a939e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -319,10 +319,10 @@ cityhasher = { version = "0.1", default-features = false } collapse = "0.1.2" cron = "0.15" dashmap = { version = "6.1", features = ["serde"] } -datafusion = "53" -datafusion-proto = "53" -datafusion-functions-aggregate-common = "53" -datafusion-functions-json = "0.53" +datafusion = "54" +datafusion-proto = "54" +datafusion-functions-aggregate-common = "54" +datafusion-functions-json = "0.54" expect-test = "1.4" arrow = { version = "58", features = ["ipc_compression", "prettyprint"] } arrow-flight = "58" @@ -330,7 +330,7 @@ arrow-json = "58" arrow-schema = { version = "58", features = ["serde"] } parquet = { version = "58", features = ["arrow", "async", "object_store"] } object_store = { version = "0.13.2", features = ["aws", "azure", "gcp"] } -sqlparser = { version = "0.61", features = ["serde", "visitor"] } +sqlparser = { version = "0.62", features = ["serde", "visitor"] } dotenv_config = "0.2" dotenvy = "0.15" env_logger = "0.11" diff --git a/src/config/Cargo.toml b/src/config/Cargo.toml index 354cdc675e..6b28336bee 100644 --- a/src/config/Cargo.toml +++ b/src/config/Cargo.toml @@ -90,7 +90,7 @@ vrl.workspace = true tonic.workspace = true tonic-prost.workspace = true tracing-opentelemetry.workspace = true -vortex = { git = "https://github.com/openobserve/vortex", rev = "13423ca204f81dc1b20ca040da2aeb386220a312", features = [ +vortex = { git = "https://github.com/openobserve/vortex", branch = "0.75.0-fix", features = [ "tokio", ], optional = true } diff --git a/src/config/src/utils/file.rs b/src/config/src/utils/file.rs index 588584b6ee..9fbd720586 100644 --- a/src/config/src/utils/file.rs +++ b/src/config/src/utils/file.rs @@ -431,7 +431,9 @@ mod tests { assert!(result.is_err()); // Test range where start > end (this should not cause overflow) - let result = get_file_contents(file_path.to_str().unwrap(), Some(5..3)); + let start = 5; + let end = 3; + let result = get_file_contents(file_path.to_str().unwrap(), Some(start..end)); assert!(result.is_err()); } diff --git a/src/config/src/utils/parquet.rs b/src/config/src/utils/parquet.rs index b58d79baae..c4ab7ae4f5 100644 --- a/src/config/src/utils/parquet.rs +++ b/src/config/src/utils/parquet.rs @@ -22,7 +22,10 @@ use std::{ }; #[cfg(feature = "vortex")] -use arrow::{array::StructArray, datatypes::DataType}; +use arrow::{ + array::StructArray, + datatypes::{DataType, Field}, +}; use arrow::{error::ArrowError, record_batch::RecordBatch}; use arrow_schema::Schema; #[cfg(feature = "vortex")] @@ -36,7 +39,7 @@ use parquet::{ #[cfg(feature = "vortex")] use vortex::{ VortexSessionDefault, - array::{ArrayRef, arrow::IntoArrowArray}, + array::{ArrayRef, VortexSessionExecute, arrow::ArrowSessionExt}, buffer::Buffer, file::OpenOptionsSessionExt, io::session::RuntimeSessionExt, @@ -144,11 +147,15 @@ pub type RecordBatchStream = Pin Result { - let array = array - .into_arrow(data_type) + let mut ctx = session.create_execution_ctx(); + let target = Field::new("", data_type.clone(), array.dtype().is_nullable()); + let array = session + .arrow() + .execute_arrow(array, Some(&target), &mut ctx) .map_err(|e| ArrowError::ExternalError(Box::new(e)))?; let struct_array = array .as_any() @@ -183,9 +190,12 @@ pub async fn get_recordbatch_reader_from_bytes( let stream = vortex_stream.then(move |result| { let arrow_data_type = arrow_data_type.clone(); + let session = session.clone(); async move { match result { - Ok(array) => vortex_array_to_record_batch(array, &arrow_data_type), + Ok(array) => { + vortex_array_to_record_batch(&session, array, &arrow_data_type) + } Err(e) => Err(ArrowError::ExternalError(Box::new(e))), } } diff --git a/src/handler/grpc/flight/visitor.rs b/src/handler/grpc/flight/visitor.rs index a1ea08a82e..3c75a7559f 100644 --- a/src/handler/grpc/flight/visitor.rs +++ b/src/handler/grpc/flight/visitor.rs @@ -53,7 +53,7 @@ impl<'n> TreeNodeVisitor<'n> for RemoteScanVisitor { fn f_up(&mut self, node: &'n Self::Node) -> Result { let name = node.name(); if name == "RemoteScanExec" { - let remote_scan_exec = node.as_any().downcast_ref::().unwrap(); + let remote_scan_exec = node.downcast_ref::().unwrap(); self.scan_stats = Some(remote_scan_exec.scan_stats()); Ok(TreeNodeRecursion::Stop) } else { @@ -86,7 +86,7 @@ impl<'n> TreeNodeVisitor<'n> for MetricsVisitor { fn f_up(&mut self, node: &'n Self::Node) -> Result { let name = node.name(); if name == "RemoteScanExec" { - let remote_scan_exec = node.as_any().downcast_ref::().unwrap(); + let remote_scan_exec = node.downcast_ref::().unwrap(); self.metrics.push(remote_scan_exec.cluster_metrics()); } Ok(TreeNodeRecursion::Continue) @@ -114,7 +114,7 @@ impl<'n> TreeNodeVisitor<'n> for PartialErrVisitor { fn f_up(&mut self, node: &'n Self::Node) -> Result { if node.name() == "RemoteScanExec" { - let remote_scan_exec = node.as_any().downcast_ref::().unwrap(); + let remote_scan_exec = node.downcast_ref::().unwrap(); self.partial_err = Some(remote_scan_exec.partial_err()); Ok(TreeNodeRecursion::Stop) } else { @@ -145,7 +145,7 @@ impl<'n> TreeNodeVisitor<'n> for PeakMemoryVisitor { fn f_up(&mut self, node: &'n Self::Node) -> Result { let name = node.name(); if name == "RemoteScanExec" { - let remote_scan_exec = node.as_any().downcast_ref::().unwrap(); + let remote_scan_exec = node.downcast_ref::().unwrap(); self.peak_memory = Some(remote_scan_exec.peak_memory()); } Ok(TreeNodeRecursion::Continue) @@ -154,40 +154,14 @@ impl<'n> TreeNodeVisitor<'n> for PeakMemoryVisitor { /// Get the peak memory from the SessionContext's memory pool. /// This should be called after the query execution to get the peak memory usage. -/// -/// # Safety -/// This function uses unsafe code to downcast Arc to Arc. -/// This is sound because: -/// 1. The memory pool is always created as PeakMemoryPool in datafusion context creation (see -/// exec.rs) -/// 2. Arc::into_raw/from_raw maintain proper Arc semantics -/// 3. The type name is checked at runtime to verify the cast is valid pub fn get_peak_memory_from_ctx(ctx: &SessionContext) -> Arc { let memory_pool = ctx.runtime_env().memory_pool.clone(); - // Runtime type check: verify this is actually a PeakMemoryPool - let type_name = format!("{:?}", memory_pool); - if !type_name.contains("PeakMemoryPool") { - log::warn!( - "Memory pool is not PeakMemoryPool (type: {}), returning 0", - type_name - ); - return Arc::new(AtomicUsize::new(0)); - } - - // SAFETY: We've verified the concrete type is PeakMemoryPool via runtime check above. - // The memory pool is created as PeakMemoryPool in datafusion context creation. - // We use Arc::into_raw and Arc::from_raw to maintain proper reference counting. - unsafe { - let raw_ptr = Arc::into_raw(memory_pool); - let peak_pool_ptr = raw_ptr as *const PeakMemoryPool; - let peak_memory = (*peak_pool_ptr).peak_memory.clone(); - - // Reconstruct the Arc to prevent memory leak - let _ = Arc::from_raw(raw_ptr); - - peak_memory - } + memory_pool + .downcast_ref::() + .unwrap() + .peak_memory + .clone() } #[cfg(test)] diff --git a/src/service/search/datafusion/distributed_plan/codec/aggregate_topk_exec.rs b/src/service/search/datafusion/distributed_plan/codec/aggregate_topk_exec.rs index 84b815d4a3..f7382cb4dd 100644 --- a/src/service/search/datafusion/distributed_plan/codec/aggregate_topk_exec.rs +++ b/src/service/search/datafusion/distributed_plan/codec/aggregate_topk_exec.rs @@ -40,7 +40,7 @@ pub(crate) fn try_decode( } pub(crate) fn try_encode(node: Arc, buf: &mut Vec) -> Result<()> { - let Some(node) = node.as_any().downcast_ref::() else { + let Some(node) = node.downcast_ref::() else { return internal_err!("Not supported"); }; let plan_node = cluster_rpc::AggregateTopkExecNode { @@ -120,8 +120,8 @@ mod tests { let ctx = datafusion::prelude::SessionContext::new(); let plan2 = physical_plan_from_bytes_with_extension_codec(&plan_bytes, &ctx.task_ctx(), &proto)?; - let plan2 = plan2.as_any().downcast_ref::().unwrap(); - let plan = plan.as_any().downcast_ref::().unwrap(); + let plan2 = plan2.downcast_ref::().unwrap(); + let plan = plan.downcast_ref::().unwrap(); // check assert_eq!(plan.limit(), plan2.limit()); diff --git a/src/service/search/datafusion/distributed_plan/codec/deduplication_exec.rs b/src/service/search/datafusion/distributed_plan/codec/deduplication_exec.rs index 309b2a8dc3..9dc0c07be5 100644 --- a/src/service/search/datafusion/distributed_plan/codec/deduplication_exec.rs +++ b/src/service/search/datafusion/distributed_plan/codec/deduplication_exec.rs @@ -57,14 +57,11 @@ pub(super) fn try_decode( } pub(super) fn try_encode(node: Arc, buf: &mut Vec) -> Result<()> { - let exec = node - .as_any() - .downcast_ref::() - .ok_or_else(|| { - datafusion::error::DataFusionError::Internal( - "Failed to downcast to DeduplicationExec".to_string(), - ) - })?; + let exec = node.downcast_ref::().ok_or_else(|| { + datafusion::error::DataFusionError::Internal( + "Failed to downcast to DeduplicationExec".to_string(), + ) + })?; // Encode deduplication columns as PhysicalExprNodes let mut deduplication_columns_proto = Vec::new(); diff --git a/src/service/search/datafusion/distributed_plan/codec/empty_exec.rs b/src/service/search/datafusion/distributed_plan/codec/empty_exec.rs index 0481904927..1747178927 100644 --- a/src/service/search/datafusion/distributed_plan/codec/empty_exec.rs +++ b/src/service/search/datafusion/distributed_plan/codec/empty_exec.rs @@ -62,7 +62,7 @@ pub(crate) fn try_decode( } pub(crate) fn try_encode(node: Arc, buf: &mut Vec) -> Result<()> { - let Some(node) = node.as_any().downcast_ref::() else { + let Some(node) = node.downcast_ref::() else { return internal_err!("Not supported"); }; let extension_codec = DefaultLogicalExtensionCodec {}; @@ -119,8 +119,8 @@ mod tests { let ctx = datafusion::prelude::SessionContext::new(); let plan2 = physical_plan_from_bytes_with_extension_codec(&plan_bytes, &ctx.task_ctx(), &proto)?; - let plan2 = plan2.as_any().downcast_ref::().unwrap(); - let plan = plan.as_any().downcast_ref::().unwrap(); + let plan2 = plan2.downcast_ref::().unwrap(); + let plan = plan.downcast_ref::().unwrap(); // check assert_eq!(plan.name(), plan2.name()); diff --git a/src/service/search/datafusion/distributed_plan/codec/enrichment_exec.rs b/src/service/search/datafusion/distributed_plan/codec/enrichment_exec.rs index 11fe84958d..35a6e8335a 100644 --- a/src/service/search/datafusion/distributed_plan/codec/enrichment_exec.rs +++ b/src/service/search/datafusion/distributed_plan/codec/enrichment_exec.rs @@ -42,7 +42,7 @@ pub(crate) fn try_decode( } pub(crate) fn try_encode(node: Arc, buf: &mut Vec) -> Result<()> { - let Some(node) = node.as_any().downcast_ref::() else { + let Some(node) = node.downcast_ref::() else { return internal_err!("Not supported"); }; let plan_node = cluster_rpc::EnrichmentExecNode { @@ -92,8 +92,8 @@ mod tests { let ctx = datafusion::prelude::SessionContext::new(); let plan2 = physical_plan_from_bytes_with_extension_codec(&plan_bytes, &ctx.task_ctx(), &proto)?; - let plan2 = plan2.as_any().downcast_ref::().unwrap(); - let plan = plan.as_any().downcast_ref::().unwrap(); + let plan2 = plan2.downcast_ref::().unwrap(); + let plan = plan.downcast_ref::().unwrap(); // check assert_eq!(plan.trace_id(), plan2.trace_id()); diff --git a/src/service/search/datafusion/distributed_plan/codec/physical_plan_node.rs b/src/service/search/datafusion/distributed_plan/codec/physical_plan_node.rs index 0dd3716c66..1690b03bb1 100644 --- a/src/service/search/datafusion/distributed_plan/codec/physical_plan_node.rs +++ b/src/service/search/datafusion/distributed_plan/codec/physical_plan_node.rs @@ -88,25 +88,25 @@ impl PhysicalExtensionCodec for PhysicalPlanNodePhysicalExtensionCodec { fn try_encode(&self, node: Arc, buf: &mut Vec) -> Result<()> { #[cfg(feature = "enterprise")] - if node.as_any().downcast_ref::().is_some() { + if node.downcast_ref::().is_some() { super::empty_exec::try_encode(node, buf) - } else if node.as_any().downcast_ref::().is_some() { + } else if node.downcast_ref::().is_some() { super::deduplication_exec::try_encode(node, buf) - } else if node.as_any().downcast_ref::().is_some() { + } else if node.downcast_ref::().is_some() { super::aggregate_topk_exec::try_encode(node, buf) - } else if node.as_any().downcast_ref::().is_some() { + } else if node.downcast_ref::().is_some() { super::streaming_aggs_exec::try_encode(node, buf) - } else if node.as_any().downcast_ref::().is_some() { + } else if node.downcast_ref::().is_some() { super::tmp_exec::try_encode(node, buf) - } else if node.as_any().downcast_ref::().is_some() { + } else if node.downcast_ref::().is_some() { super::enrichment_exec::try_encode(node, buf) } else { internal_err!("Not supported") } #[cfg(not(feature = "enterprise"))] - if node.as_any().downcast_ref::().is_some() { + if node.downcast_ref::().is_some() { super::empty_exec::try_encode(node, buf) - } else if node.as_any().downcast_ref::().is_some() { + } else if node.downcast_ref::().is_some() { super::deduplication_exec::try_encode(node, buf) } else { internal_err!("Not supported") @@ -144,8 +144,8 @@ mod tests { let ctx = datafusion::prelude::SessionContext::new(); let plan2 = physical_plan_from_bytes_with_extension_codec(&plan_bytes, &ctx.task_ctx(), &proto)?; - let plan2 = plan2.as_any().downcast_ref::().unwrap(); - let plan = plan.as_any().downcast_ref::().unwrap(); + let plan2 = plan2.downcast_ref::().unwrap(); + let plan = plan.downcast_ref::().unwrap(); // check assert_eq!(plan.name(), plan2.name()); diff --git a/src/service/search/datafusion/distributed_plan/codec/streaming_aggs_exec.rs b/src/service/search/datafusion/distributed_plan/codec/streaming_aggs_exec.rs index ca31dcf0ed..124d26cb4f 100644 --- a/src/service/search/datafusion/distributed_plan/codec/streaming_aggs_exec.rs +++ b/src/service/search/datafusion/distributed_plan/codec/streaming_aggs_exec.rs @@ -36,7 +36,7 @@ pub(crate) fn try_decode( }; let extension_codec = super::get_physical_extension_codec(); let aggregate_plan = aggregate_plan.try_into_physical_plan(ctx, &extension_codec)?; - let Some(aggregate_plan) = aggregate_plan.as_any().downcast_ref::() else { + let Some(aggregate_plan) = aggregate_plan.downcast_ref::() else { return internal_err!("aggregate_plan is not an AggregateExec"); }; let aggregate_plan = Arc::new(aggregate_plan.clone()); @@ -61,7 +61,7 @@ pub(crate) fn try_decode( } pub(crate) fn try_encode(node: Arc, buf: &mut Vec) -> Result<()> { - let Some(node) = node.as_any().downcast_ref::() else { + let Some(node) = node.downcast_ref::() else { return internal_err!("Not supported"); }; @@ -169,8 +169,8 @@ mod tests { let ctx = datafusion::prelude::SessionContext::new(); let plan2 = physical_plan_from_bytes_with_extension_codec(&plan_bytes, &ctx.task_ctx(), &proto)?; - let plan2 = plan2.as_any().downcast_ref::().unwrap(); - let plan = plan.as_any().downcast_ref::().unwrap(); + let plan2 = plan2.downcast_ref::().unwrap(); + let plan = plan.downcast_ref::().unwrap(); // check assert_eq!(plan.id(), plan2.id()); @@ -255,8 +255,8 @@ mod tests { ctx.register_udf(STR_MATCH_UDF.clone()); let plan2 = physical_plan_from_bytes_with_extension_codec(&plan_bytes, &ctx.task_ctx(), &proto)?; - let plan2 = plan2.as_any().downcast_ref::().unwrap(); - let plan = plan.as_any().downcast_ref::().unwrap(); + let plan2 = plan2.downcast_ref::().unwrap(); + let plan = plan.downcast_ref::().unwrap(); // check assert_eq!(plan.id(), plan2.id()); @@ -268,14 +268,11 @@ mod tests { // Verify that the aggregate plan contains a FilterExec let agg_plan_decoded = plan2.aggregate_plan(); - let agg_exec = agg_plan_decoded - .as_any() - .downcast_ref::() - .unwrap(); + let agg_exec = agg_plan_decoded.as_ref(); let input_plan = agg_exec.input(); // The input should be a FilterExec - assert!(input_plan.as_any().downcast_ref::().is_some()); + assert!(input_plan.downcast_ref::().is_some()); Ok(()) } diff --git a/src/service/search/datafusion/distributed_plan/codec/tmp_exec.rs b/src/service/search/datafusion/distributed_plan/codec/tmp_exec.rs index ee8bb20b28..573d8b99af 100644 --- a/src/service/search/datafusion/distributed_plan/codec/tmp_exec.rs +++ b/src/service/search/datafusion/distributed_plan/codec/tmp_exec.rs @@ -42,7 +42,7 @@ pub(crate) fn try_decode( } pub(crate) fn try_encode(node: Arc, buf: &mut Vec) -> Result<()> { - let Some(node) = node.as_any().downcast_ref::() else { + let Some(node) = node.downcast_ref::() else { return internal_err!("Not supported"); }; let plan_node = cluster_rpc::TmpExecNode { @@ -111,8 +111,8 @@ mod tests { let ctx = datafusion::prelude::SessionContext::new(); let plan2 = physical_plan_from_bytes_with_extension_codec(&plan_bytes, &ctx.task_ctx(), &proto)?; - let plan2 = plan2.as_any().downcast_ref::().unwrap(); - let plan = plan.as_any().downcast_ref::().unwrap(); + let plan2 = plan2.downcast_ref::().unwrap(); + let plan = plan.downcast_ref::().unwrap(); // check assert_eq!(plan.trace_id(), plan2.trace_id()); diff --git a/src/service/search/datafusion/distributed_plan/distribute_analyze_exec.rs b/src/service/search/datafusion/distributed_plan/distribute_analyze_exec.rs index 90a4e1e6d3..291a16072c 100644 --- a/src/service/search/datafusion/distributed_plan/distribute_analyze_exec.rs +++ b/src/service/search/datafusion/distributed_plan/distribute_analyze_exec.rs @@ -13,7 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{any::Any, sync::Arc}; +use std::sync::Arc; use arrow::{ array::{Int32Builder, StringBuilder}, @@ -104,11 +104,6 @@ impl ExecutionPlan for DistributeAnalyzeExec { "DistributeAnalyzeExec" } - /// Return a reference to Any that can be used for downcasting - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { &self.cache } diff --git a/src/service/search/datafusion/distributed_plan/empty_exec.rs b/src/service/search/datafusion/distributed_plan/empty_exec.rs index a22a50c784..5e35ec0281 100644 --- a/src/service/search/datafusion/distributed_plan/empty_exec.rs +++ b/src/service/search/datafusion/distributed_plan/empty_exec.rs @@ -13,7 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{any::Any, sync::Arc}; +use std::sync::Arc; use arrow_schema::SortOptions; use config::TIMESTAMP_COL_NAME; @@ -196,11 +196,6 @@ impl ExecutionPlan for NewEmptyExec { "NewEmptyExec" } - /// Return a reference to Any that can be used for downcasting - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { &self.cache } @@ -314,12 +309,10 @@ mod tests { #[test] fn test_as_any_downcasts() { - use datafusion::physical_plan::ExecutionPlan; - let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); let exec = NewEmptyExec::new("t", schema.clone(), None, &[], None, false, schema); - let any = exec.as_any(); - assert!(any.downcast_ref::().is_some()); + let exec_plan: &dyn ExecutionPlan = &exec; + assert!(exec_plan.downcast_ref::().is_some()); } #[test] diff --git a/src/service/search/datafusion/distributed_plan/enrich_exec.rs b/src/service/search/datafusion/distributed_plan/enrich_exec.rs index b3de8a7cf1..f2d608dd14 100644 --- a/src/service/search/datafusion/distributed_plan/enrich_exec.rs +++ b/src/service/search/datafusion/distributed_plan/enrich_exec.rs @@ -13,7 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{any::Any, sync::Arc}; +use std::sync::Arc; use config::utils::record_batch_ext::convert_json_to_record_batch; use datafusion::{ @@ -93,11 +93,6 @@ impl ExecutionPlan for EnrichExec { "EnrichExec" } - /// Return a reference to Any that can be used for downcasting - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { &self.cache } @@ -222,12 +217,10 @@ mod tests { #[test] fn test_as_any_downcasts() { - use datafusion::physical_plan::ExecutionPlan; - let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); let exec = EnrichExec::new("org", "stream", schema); - let any = exec.as_any(); - assert!(any.downcast_ref::().is_some()); + let exec_plan: &dyn ExecutionPlan = &exec; + assert!(exec_plan.downcast_ref::().is_some()); } #[test] diff --git a/src/service/search/datafusion/distributed_plan/enrichment_exec.rs b/src/service/search/datafusion/distributed_plan/enrichment_exec.rs index 2f39f10634..c6c900278e 100644 --- a/src/service/search/datafusion/distributed_plan/enrichment_exec.rs +++ b/src/service/search/datafusion/distributed_plan/enrichment_exec.rs @@ -13,7 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{any::Any, sync::Arc}; +use std::sync::Arc; use arrow::{ array::{ArrayRef, RecordBatch}, @@ -146,10 +146,6 @@ impl ExecutionPlan for EnrichmentExec { "EnrichmentExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { &self.cache } diff --git a/src/service/search/datafusion/distributed_plan/mod.rs b/src/service/search/datafusion/distributed_plan/mod.rs index 3fa4484c04..bf3bc8a3bc 100644 --- a/src/service/search/datafusion/distributed_plan/mod.rs +++ b/src/service/search/datafusion/distributed_plan/mod.rs @@ -54,7 +54,6 @@ impl NewEmptyExecVisitor { self.plan .as_ref() .unwrap() - .as_any() .downcast_ref::() .unwrap() } @@ -107,7 +106,7 @@ impl TreeNodeRewriter for ReplaceTableScanExec { #[cfg(test)] mod tests { - use std::{any::Any, sync::Arc}; + use std::sync::Arc; use datafusion::physical_plan::{ExecutionPlan, PlanProperties}; @@ -128,9 +127,6 @@ mod tests { fn name(&self) -> &'static str { "NewEmptyExec" } - fn as_any(&self) -> &dyn Any { - self - } fn properties(&self) -> &Arc { panic!("not needed") } diff --git a/src/service/search/datafusion/distributed_plan/remote_scan_exec.rs b/src/service/search/datafusion/distributed_plan/remote_scan_exec.rs index 1c9f935abb..f50dc8cbdf 100644 --- a/src/service/search/datafusion/distributed_plan/remote_scan_exec.rs +++ b/src/service/search/datafusion/distributed_plan/remote_scan_exec.rs @@ -13,10 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{ - any::Any, - sync::{Arc, atomic::AtomicUsize}, -}; +use std::sync::{Arc, atomic::AtomicUsize}; use arrow_schema::SchemaRef; use config::{ @@ -207,11 +204,6 @@ impl ExecutionPlan for RemoteScanExec { "RemoteScanExec" } - /// Return a reference to Any that can be used for downcasting - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { Arc::clone(&self.input.schema()) } @@ -493,7 +485,8 @@ mod tests { #[test] fn test_remote_scan_exec_as_any() { let exec = make_remote_exec(); - assert!(exec.as_any().downcast_ref::().is_some()); + let exec_plan: &dyn ExecutionPlan = &exec; + assert!(exec_plan.downcast_ref::().is_some()); } #[test] diff --git a/src/service/search/datafusion/distributed_plan/rewrite.rs b/src/service/search/datafusion/distributed_plan/rewrite.rs index d03053ae2f..590207d297 100644 --- a/src/service/search/datafusion/distributed_plan/rewrite.rs +++ b/src/service/search/datafusion/distributed_plan/rewrite.rs @@ -75,7 +75,7 @@ impl TreeNodeRewriter for TantivyOptimizeRewriter { fn f_up(&mut self, node: Arc) -> Result> { if node.name() == "AggregateExec" { - let aggregate = node.as_any().downcast_ref::().unwrap(); + let aggregate = node.downcast_ref::().unwrap(); if *aggregate.mode() == AggregateMode::Partial { let tantivy_exec = Arc::new(TantivyOptimizeExec::new( self.query.clone(), diff --git a/src/service/search/datafusion/exec.rs b/src/service/search/datafusion/exec.rs index 6e84d80451..5796b2f035 100644 --- a/src/service/search/datafusion/exec.rs +++ b/src/service/search/datafusion/exec.rs @@ -60,7 +60,10 @@ use super::{ storage::file_list, udf::transform_udf::get_all_transform, }; use crate::service::search::{ - datafusion::table_provider::{listing_adapter::ListingTableAdapter, uniontable::NewUnionTable}, + datafusion::{ + storage::file_statistics_cache, + table_provider::{listing_adapter::ListingTableAdapter, uniontable::NewUnionTable}, + }, index::IndexCondition, }; @@ -107,6 +110,29 @@ pub fn create_session_config( .execution .skip_physical_aggregate_schema_check = true; + // DataFusion 54 executes uncorrelated scalar subqueries physically via + // `ScalarSubqueryExec`/`ScalarSubqueryExpr` instead of rewriting them into joins. + // `ScalarSubqueryExpr` can only be (de)serialized inside its surrounding + // `ScalarSubqueryExec`, which breaks our distributed plan splitting across the Flight + // boundary. Disable the physical path so `ScalarSubqueryToJoin` decorrelates them into + // joins again, keeping the serialized follower plans valid. + config + .options_mut() + .optimizer + .enable_physical_uncorrelated_scalar_subquery = false; + + // DataFusion 54 builds a runtime `DynamicFilterPhysicalExpr` from a `HashJoinExec`'s + // build-side join keys and pushes it into the probe-side scan. That runtime state can't + // cross our distributed RemoteScan/Flight boundary, and after our custom join rewrites + // (`swap_inputs` + broadcast/enrichment join) the filter ends up referencing build-side + // columns by index against the projected probe-side batch, producing + // "PhysicalExpr Column references column ... but input schema only has N columns" at + // execution time. Disable join dynamic filter pushdown to keep the split plans valid. + config + .options_mut() + .optimizer + .enable_join_dynamic_filter_pushdown = false; + Ok(config) } @@ -126,9 +152,9 @@ pub async fn create_runtime_env(trace_id: &str, memory_limit: usize) -> Result 0 { let cache_config = CacheManagerConfig::default(); - let cache_config = cache_config.with_files_statistics_cache(Some( - super::storage::file_statistics_cache::GLOBAL_CACHE.clone(), - )); + let cache_config = cache_config + .with_file_statistics_cache(Some(file_statistics_cache::GLOBAL_CACHE.clone())) + .with_file_statistics_cache_limit(cfg.limit.datafusion_file_stat_cache_max_size); builder = builder.with_cache_manager(cache_config); } @@ -614,6 +640,14 @@ mod tests { assert_eq!(config.options().sql_parser.dialect, Dialect::PostgreSQL); assert!(!config.options().execution.listing_table_ignore_subdirectory); assert!(config.information_schema()); + // Join dynamic filter pushdown must stay disabled: its runtime filter can't cross our + // distributed RemoteScan/Flight boundary and breaks our custom join rewrites. + assert!( + !config + .options() + .optimizer + .enable_join_dynamic_filter_pushdown + ); Ok(()) } diff --git a/src/service/search/datafusion/optimizer/logical_optimizer/limit_join_right_side.rs b/src/service/search/datafusion/optimizer/logical_optimizer/limit_join_right_side.rs index 089cb5ad5b..e3907e33aa 100644 --- a/src/service/search/datafusion/optimizer/logical_optimizer/limit_join_right_side.rs +++ b/src/service/search/datafusion/optimizer/logical_optimizer/limit_join_right_side.rs @@ -375,15 +375,14 @@ mod tests { let expected = vec![ "HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(prod_id@1, prod_id@0)], projection=[usr_id@0, prod_id@1]", - " ProjectionExec: expr=[usr_id@1 as usr_id, prod_id@0 as prod_id]", - " HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(usr_id@0, usr_id@0)], projection=[prod_id@1, usr_id@2]", - " ProjectionExec: expr=[usr_id@0 as usr_id, prod_id@1 as prod_id]", - " DeduplicationExec: columns: [usr_id@0]", - " SortExec: expr=[usr_id@0 DESC NULLS LAST, _timestamp@2 DESC NULLS LAST], preserve_partitioning=[false]", - " SortPreservingMergeExec: [_timestamp@2 DESC NULLS LAST], fetch=50000", - " SortExec: TopK(fetch=50000), expr=[_timestamp@2 DESC NULLS LAST], preserve_partitioning=[true]", - " DataSourceExec: partitions=2, partition_sizes=[1, 1]", - " DataSourceExec: partitions=1, partition_sizes=[1]", + " HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(usr_id@0, usr_id@0)], projection=[usr_id@2, prod_id@1]", + " ProjectionExec: expr=[usr_id@0 as usr_id, prod_id@1 as prod_id]", + " DeduplicationExec: columns: [usr_id@0]", + " SortExec: expr=[usr_id@0 DESC NULLS LAST, _timestamp@2 DESC NULLS LAST], preserve_partitioning=[false]", + " SortPreservingMergeExec: [_timestamp@2 DESC NULLS LAST], fetch=50000", + " SortExec: TopK(fetch=50000), expr=[_timestamp@2 DESC NULLS LAST], preserve_partitioning=[true]", + " DataSourceExec: partitions=2, partition_sizes=[1, 1]", + " DataSourceExec: partitions=1, partition_sizes=[1]", " ProjectionExec: expr=[prod_id@0 as prod_id]", " DeduplicationExec: columns: [prod_id@0]", " SortExec: expr=[prod_id@0 DESC NULLS LAST, _timestamp@1 DESC NULLS LAST], preserve_partitioning=[false]", diff --git a/src/service/search/datafusion/optimizer/physical_optimizer/distribute_analyze.rs b/src/service/search/datafusion/optimizer/physical_optimizer/distribute_analyze.rs index fb178b9b17..61dbecb60d 100644 --- a/src/service/search/datafusion/optimizer/physical_optimizer/distribute_analyze.rs +++ b/src/service/search/datafusion/optimizer/physical_optimizer/distribute_analyze.rs @@ -29,7 +29,7 @@ use crate::service::search::datafusion::distributed_plan::{ // replace the AnalyzeExec to DistributeAnalyzeExec pub fn optimize_distribute_analyze(plan: Arc) -> Result> { - if let Some(analyze) = plan.as_any().downcast_ref::() { + if let Some(analyze) = plan.downcast_ref::() { let distribute_analyze = Arc::new(DistributeAnalyzeExec::new( analyze.verbose(), analyze.show_statistics(), @@ -55,7 +55,7 @@ impl TreeNodeRewriter for AnalyzeRewrite { type Node = Arc; fn f_up(&mut self, node: Arc) -> Result> { - if let Some(remote) = node.as_any().downcast_ref::() { + if let Some(remote) = node.downcast_ref::() { let remote = Arc::new(remote.clone().set_analyze()) as Arc; return Ok(Transformed::yes(remote)); } @@ -84,6 +84,7 @@ mod tests { false, false, vec![], + None, Arc::new(EmptyExec::new(get_schema())), get_schema(), )); @@ -102,7 +103,7 @@ mod tests { ) as Arc; let mut rewriter = AnalyzeRewrite::new(); let plan = plan.rewrite(&mut rewriter).unwrap().data; - let remote_scan = plan.as_any().downcast_ref::().unwrap(); + let remote_scan = plan.downcast_ref::().unwrap(); assert!(remote_scan.analyze()); } } diff --git a/src/service/search/datafusion/optimizer/physical_optimizer/enrichment.rs b/src/service/search/datafusion/optimizer/physical_optimizer/enrichment.rs index 5ccf2d8638..aa3920ff68 100644 --- a/src/service/search/datafusion/optimizer/physical_optimizer/enrichment.rs +++ b/src/service/search/datafusion/optimizer/physical_optimizer/enrichment.rs @@ -88,7 +88,7 @@ impl TreeNodeRewriter for EnrichmentExecRewriter { type Node = Arc; fn f_up(&mut self, node: Arc) -> Result> { - if let Some(new_empty) = node.as_any().downcast_ref::() { + if let Some(new_empty) = node.downcast_ref::() { let table_name = TableReference::parse_str(new_empty.name()); if matches!( table_name.schema(), @@ -140,7 +140,7 @@ pub fn should_use_enrichment_broadcast_join(plan: &Arc) -> bo let mut result = false; let _ = plan.apply(|node| { Ok(if node.name() == "HashJoinExec" { - if let Some(hash_join) = node.as_any().downcast_ref::() { + if let Some(hash_join) = node.downcast_ref::() { result = is_enrichment_table(hash_join.left()) && is_valid_enrich_broadcast_right(hash_join.right()); } @@ -157,7 +157,7 @@ pub fn is_enrichment_table(plan: &Arc) -> bool { let mut found_enrichment = false; let _ = plan.apply(|node| { Ok( - if let Some(new_empty) = node.as_any().downcast_ref::() { + if let Some(new_empty) = node.downcast_ref::() { let table_name = TableReference::parse_str(new_empty.name()); if matches!( table_name.schema(), @@ -180,7 +180,7 @@ fn is_valid_enrich_broadcast_right(plan: &Arc) -> bool { let mut is_valid = true; let _ = plan.apply(|node| { Ok( - if let Some(new_empty) = node.as_any().downcast_ref::() { + if let Some(new_empty) = node.downcast_ref::() { let table_name = TableReference::parse_str(new_empty.name()); if matches!( table_name.schema(), diff --git a/src/service/search/datafusion/optimizer/physical_optimizer/index.rs b/src/service/search/datafusion/optimizer/physical_optimizer/index.rs index 56baf8e56f..821a6606c7 100644 --- a/src/service/search/datafusion/optimizer/physical_optimizer/index.rs +++ b/src/service/search/datafusion/optimizer/physical_optimizer/index.rs @@ -177,7 +177,7 @@ impl TreeNodeRewriter for IndexOptimizer { type Node = Arc; fn f_up(&mut self, node: Self::Node) -> Result> { - if let Some(filter) = node.as_any().downcast_ref::() { + if let Some(filter) = node.downcast_ref::() { self.has_filter = true; let mut index_conditions = IndexCondition::new(); let mut other_conditions = Vec::new(); @@ -254,7 +254,7 @@ fn construct_filter_exec( // Check if the expression is valid for the index. fn is_expr_valid_for_index(expr: &Arc, index_fields: &HashSet) -> bool { - if let Some(expr) = expr.as_any().downcast_ref::() { + if let Some(expr) = expr.downcast_ref::() { match expr.op() { Operator::Eq | Operator::NotEq => { let column = if is_value(expr.left()) && is_column(expr.right()) { @@ -275,7 +275,7 @@ fn is_expr_valid_for_index(expr: &Arc, index_fields: &HashSet< } _ => return false, } - } else if let Some(expr) = expr.as_any().downcast_ref::() { + } else if let Some(expr) = expr.downcast_ref::() { if !is_column(expr.expr()) || !index_fields.contains(get_column_name(expr.expr())) { return false; } @@ -285,7 +285,7 @@ fn is_expr_valid_for_index(expr: &Arc, index_fields: &HashSet< return false; } } - } else if let Some(expr) = expr.as_any().downcast_ref::() { + } else if let Some(expr) = expr.downcast_ref::() { let name = expr.name(); return match name { MATCH_ALL_UDF_NAME => { @@ -303,7 +303,7 @@ fn is_expr_valid_for_index(expr: &Arc, index_fields: &HashSet< } _ => false, }; - } else if let Some(expr) = expr.as_any().downcast_ref::() { + } else if let Some(expr) = expr.downcast_ref::() { return is_expr_valid_for_index(expr.arg(), index_fields); } else { return false; diff --git a/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/count.rs b/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/count.rs index 15fb1922e0..ce09b6c758 100644 --- a/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/count.rs +++ b/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/count.rs @@ -69,7 +69,7 @@ impl<'n> TreeNodeVisitor<'n> for SimpleCountVisitor { type Node = Arc; fn f_down(&mut self, node: &'n Self::Node) -> Result { - if let Some(aggregate) = node.as_any().downcast_ref::() { + if let Some(aggregate) = node.downcast_ref::() { if aggregate.group_expr().is_empty() && aggregate.aggr_expr().len() == 1 && is_count_rows_aggregate(&aggregate.aggr_expr()[0]) diff --git a/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/distinct.rs b/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/distinct.rs index ff5d5546f0..904b5080eb 100644 --- a/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/distinct.rs +++ b/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/distinct.rs @@ -82,7 +82,7 @@ impl<'n> TreeNodeVisitor<'n> for SimpleDistinctVisitor { type Node = Arc; fn f_down(&mut self, node: &'n Self::Node) -> Result { - if let Some(sort_merge) = node.as_any().downcast_ref::() { + if let Some(sort_merge) = node.downcast_ref::() { if let Some(fetch) = sort_merge.fetch() && fetch > 0 && sort_merge.expr().len() == 1 @@ -99,7 +99,7 @@ impl<'n> TreeNodeVisitor<'n> for SimpleDistinctVisitor { // if not simple distinct, stop visiting self.simple_distinct = None; return Ok(TreeNodeRecursion::Stop); - } else if let Some(aggregate) = node.as_any().downcast_ref::() { + } else if let Some(aggregate) = node.downcast_ref::() { // check if the group by is simple distinct, // only one group by field, no aggregate function if aggregate.group_expr().expr().len() == 1 @@ -122,7 +122,7 @@ impl<'n> TreeNodeVisitor<'n> for SimpleDistinctVisitor { // if not simple distinct, stop visiting self.simple_distinct = None; return Ok(TreeNodeRecursion::Stop); - } else if let Some(projection) = node.as_any().downcast_ref::() { + } else if let Some(projection) = node.downcast_ref::() { // Check ProjectionExec for the structure: [index_field] let exprs = projection.expr(); if exprs.len() == 1 { @@ -133,7 +133,7 @@ impl<'n> TreeNodeVisitor<'n> for SimpleDistinctVisitor { // If projection doesn't have exactly 2 expressions, stop visiting self.simple_distinct = None; return Ok(TreeNodeRecursion::Stop); - } else if let Some(filter) = node.as_any().downcast_ref::() { + } else if let Some(filter) = node.downcast_ref::() { let predicate = filter.predicate(); let exprs = split_conjunction(predicate); if exprs.len() == 2 && is_only_timestamp_filter(&exprs) { @@ -159,7 +159,7 @@ impl<'n> TreeNodeVisitor<'n> for SimpleDistinctVisitor { } fn is_simple_str_match(expr: &Arc) -> Option { - if let Some(func) = expr.as_any().downcast_ref::() + if let Some(func) = expr.downcast_ref::() && func.fun().name().to_lowercase() == "str_match" && func.args().len() == 2 { diff --git a/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/histogram.rs b/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/histogram.rs index f43c2b86bb..a74c3742bb 100644 --- a/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/histogram.rs +++ b/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/histogram.rs @@ -86,7 +86,7 @@ impl<'n> TreeNodeVisitor<'n> for SimpleHistogramVisitor { type Node = Arc; fn f_down(&mut self, node: &'n Self::Node) -> Result { - if let Some(aggregate) = node.as_any().downcast_ref::() { + if let Some(aggregate) = node.downcast_ref::() { // Check if the AggregateExec matches SimpleHistogram pattern if aggregate.group_expr().expr().len() == 1 && aggregate.aggr_expr().len() == 1 @@ -118,7 +118,7 @@ impl<'n> TreeNodeVisitor<'n> for SimpleHistogramVisitor { // If AggregateExec doesn't match SimpleHistogram pattern, stop visiting self.simple_histogram = None; return Ok(TreeNodeRecursion::Stop); - } else if let Some(projection) = node.as_any().downcast_ref::() { + } else if let Some(projection) = node.downcast_ref::() { // Check ProjectionExec for the structure: [histogram(_timestamp), count(*)] let exprs = projection.expr(); if exprs.len() == 2 { @@ -139,7 +139,7 @@ impl<'n> TreeNodeVisitor<'n> for SimpleHistogramVisitor { } fn get_data_bin(expr: &Arc) -> Option<&ScalarFunctionExpr> { - if let Some(func) = expr.as_any().downcast_ref::() + if let Some(func) = expr.downcast_ref::() && func.fun().name().to_lowercase() == "date_bin" { Some(func) @@ -150,7 +150,7 @@ fn get_data_bin(expr: &Arc) -> Option<&ScalarFunctionExpr> { // unit: microseconds fn get_histogram_interval(expr: &Arc) -> Option { - let interval = expr.as_any().downcast_ref::()?.value(); + let interval = expr.downcast_ref::()?.value(); match interval { ScalarValue::IntervalMonthDayNano(Some(interval)) => { // convert interval to nanoseconds @@ -168,16 +168,16 @@ fn get_histogram_interval(expr: &Arc) -> Option { /// `to_timestamp_micros(_timestamp + offset)` — the shape histogram() with a timezone /// rewrites to — yields the offset. None when the source is not the timestamp column. fn get_timestamp_offset(expr: &Arc) -> Option { - let func = expr.as_any().downcast_ref::()?; + let func = expr.downcast_ref::()?; let arg = func.args().first()?; if get_column_name(arg) == TIMESTAMP_COL_NAME { return Some(0); } - let bin = arg.as_any().downcast_ref::()?; + let bin = arg.downcast_ref::()?; if *bin.op() != Operator::Plus || get_column_name(bin.left()) != TIMESTAMP_COL_NAME { return None; } - match bin.right().as_any().downcast_ref::()?.value() { + match bin.right().downcast_ref::()?.value() { ScalarValue::Int64(Some(ts_offset)) => Some(*ts_offset), _ => None, } @@ -239,7 +239,7 @@ impl<'n> TreeNodeVisitor<'n> for SimpleMultiHistogramVisitor { type Node = Arc; fn f_down(&mut self, node: &'n Self::Node) -> Result { - if let Some(aggregate) = node.as_any().downcast_ref::() { + if let Some(aggregate) = node.downcast_ref::() { // Exactly 2 group-by expressions (histogram + breakdown) and 1 aggregate (count(*)) if aggregate.group_expr().expr().len() == 2 && aggregate.aggr_expr().len() == 1 @@ -285,7 +285,7 @@ impl<'n> TreeNodeVisitor<'n> for SimpleMultiHistogramVisitor { // If AggregateExec doesn't match, stop visiting self.simple_multi_histogram = None; return Ok(TreeNodeRecursion::Stop); - } else if let Some(projection) = node.as_any().downcast_ref::() { + } else if let Some(projection) = node.downcast_ref::() { // Projection should have 3 expressions: timestamp, breakdown, count let exprs = projection.expr(); if exprs.len() == 3 { diff --git a/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/mod.rs b/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/mod.rs index 0745106102..5815af0601 100644 --- a/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/mod.rs +++ b/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/mod.rs @@ -132,11 +132,7 @@ impl TreeNodeRewriter for FollowerIndexOptimizer { return Ok(Transformed::new(plan, false, TreeNodeRecursion::Stop)); } - if plan - .as_any() - .downcast_ref::() - .is_some() - { + if plan.downcast_ref::().is_some() { // Check for SimpleSelect if let Some(index_optimize_mode) = is_simple_select(Arc::clone(&plan)) { *self.index_optimizer_mode.lock() = Some(index_optimize_mode); @@ -158,7 +154,7 @@ impl TreeNodeRewriter for FollowerIndexOptimizer { } } return Ok(Transformed::new(plan, false, TreeNodeRecursion::Continue)); - } else if plan.as_any().downcast_ref::().is_some() { + } else if plan.downcast_ref::().is_some() { // Check for SimpleCount if let Some(index_optimize_mode) = is_simple_count(Arc::clone(&plan)) { *self.index_optimizer_mode.lock() = Some(index_optimize_mode); @@ -242,11 +238,7 @@ impl TreeNodeRewriter for LeaderIndexOptimizer { return Ok(Transformed::new(plan, false, TreeNodeRecursion::Stop)); } - if plan - .as_any() - .downcast_ref::() - .is_some() - { + if plan.downcast_ref::().is_some() { // Get the index fields of the underlying table let mut visitor = TableNameVisitor::new(); plan.visit(&mut visitor)?; @@ -298,7 +290,7 @@ impl TreeNodeRewriter for IndexOptimizerRewrite { type Node = Arc; fn f_up(&mut self, node: Arc) -> Result> { - if let Some(remote) = node.as_any().downcast_ref::() { + if let Some(remote) = node.downcast_ref::() { let remote = Arc::new( remote .clone() @@ -327,7 +319,7 @@ impl<'n> TreeNodeVisitor<'n> for TableNameVisitor { fn f_up(&mut self, node: &'n Self::Node) -> Result { let name = node.name(); if name == "NewEmptyExec" { - let table = node.as_any().downcast_ref::().unwrap(); + let table = node.downcast_ref::().unwrap(); self.table_name = Some(TableReference::from(table.name())); Ok(TreeNodeRecursion::Stop) } else { @@ -402,11 +394,7 @@ mod tests { let result = plan.rewrite(&mut rewriter).unwrap(); assert!(result.transformed, "plan should be marked as transformed"); assert_eq!(result.data.name(), "RemoteScanExec"); - let remote_scan = result - .data - .as_any() - .downcast_ref::() - .unwrap(); + let remote_scan = result.data.downcast_ref::().unwrap(); assert_eq!(remote_scan.index_optimize_mode(), Some(mode)); } diff --git a/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/select.rs b/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/select.rs index fdd53845c1..965b7dbd6a 100644 --- a/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/select.rs +++ b/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/select.rs @@ -64,7 +64,7 @@ impl<'n> TreeNodeVisitor<'n> for SimpleSelectVisitor { type Node = Arc; fn f_down(&mut self, node: &'n Self::Node) -> Result { - if let Some(sort_merge) = node.as_any().downcast_ref::() { + if let Some(sort_merge) = node.downcast_ref::() { if let Some(fetch) = sort_merge.fetch() && fetch > 0 && sort_merge.expr().len() == 1 diff --git a/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/topn.rs b/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/topn.rs index 8902028882..9a545d3a6f 100644 --- a/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/topn.rs +++ b/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/topn.rs @@ -89,7 +89,7 @@ impl<'n> TreeNodeVisitor<'n> for SimpleTopnVisitor { type Node = Arc; fn f_down(&mut self, node: &'n Self::Node) -> Result { - if let Some(sort_merge) = node.as_any().downcast_ref::() { + if let Some(sort_merge) = node.downcast_ref::() { // need a fetch limit, a primary sort plus up to one secondary sort per group field let fetch = sort_merge.fetch().unwrap_or(0); let sort_exprs = sort_merge.expr(); @@ -120,7 +120,7 @@ impl<'n> TreeNodeVisitor<'n> for SimpleTopnVisitor { // fields are filled in at the AggregateExec self.simple_topn = Some((vec![], fetch, !first_sort.options.descending)); - } else if let Some(projection) = node.as_any().downcast_ref::() { + } else if let Some(projection) = node.downcast_ref::() { // expect [field1, .., fieldN, count(*)]; the exact length is validated against // the group by at the AggregateExec let exprs = projection.expr(); @@ -141,7 +141,7 @@ impl<'n> TreeNodeVisitor<'n> for SimpleTopnVisitor { { return self.reject(); } - } else if let Some(aggregate) = node.as_any().downcast_ref::() { + } else if let Some(aggregate) = node.downcast_ref::() { let group_len = aggregate.group_expr().expr().len(); if !(1..=MAX_SIMPLE_TOPN_FIELDS).contains(&group_len) || aggregate.aggr_expr().len() != 1 diff --git a/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/utils.rs b/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/utils.rs index cc993eb6e4..d44827d92b 100644 --- a/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/utils.rs +++ b/src/service/search/datafusion/optimizer/physical_optimizer/index_optimizer/utils.rs @@ -83,7 +83,7 @@ pub(crate) mod tests { let mut visitor = AggregateVisitor::new(); let _ = plan.visit(&mut visitor); let data = visitor.get_data(); - data.map(|v| v.as_any().downcast_ref::().unwrap().clone()) + data.map(|v| v.downcast_ref::().unwrap().clone()) } struct AggregateVisitor { @@ -111,7 +111,7 @@ pub(crate) mod tests { fn f_up(&mut self, node: &'n Self::Node) -> Result { if node.name() == "AggregateExec" { - let agg = node.as_any().downcast_ref::().unwrap(); + let agg = node.downcast_ref::().unwrap(); if *agg.mode() == AggregateMode::Partial { self.data = Some(node.clone()); Ok(TreeNodeRecursion::Stop) @@ -155,7 +155,7 @@ pub(crate) mod tests { fn f_up(&mut self, node: &'n Self::Node) -> Result { if node.name() == "RemoteScanExec" { - let remote_scan = node.as_any().downcast_ref::().unwrap(); + let remote_scan = node.downcast_ref::().unwrap(); self.data.push(Arc::new(remote_scan.clone())); } Ok(TreeNodeRecursion::Continue) diff --git a/src/service/search/datafusion/optimizer/physical_optimizer/join_reorder.rs b/src/service/search/datafusion/optimizer/physical_optimizer/join_reorder.rs index a12cb685de..f93b54d7d5 100644 --- a/src/service/search/datafusion/optimizer/physical_optimizer/join_reorder.rs +++ b/src/service/search/datafusion/optimizer/physical_optimizer/join_reorder.rs @@ -62,7 +62,7 @@ impl PhysicalOptimizerRule for JoinReorderRule { } fn swap_join_order(plan: Arc) -> Result>> { - if let Some(hash_join) = plan.as_any().downcast_ref::() { + if let Some(hash_join) = plan.downcast_ref::() { let left = hash_join.left(); let right = hash_join.right(); diff --git a/src/service/search/datafusion/optimizer/physical_optimizer/remote_scan.rs b/src/service/search/datafusion/optimizer/physical_optimizer/remote_scan.rs index cc30bede6e..494d2cbcbe 100644 --- a/src/service/search/datafusion/optimizer/physical_optimizer/remote_scan.rs +++ b/src/service/search/datafusion/optimizer/physical_optimizer/remote_scan.rs @@ -273,7 +273,7 @@ impl TreeNodeRewriter for RemoteScanRewriter { // For sort, we should add a SortPreservingMergeExec if child.name() == "SortExec" { let table_name = visitor.table_name.clone().unwrap(); - let sort = child.as_any().downcast_ref::().unwrap(); + let sort = child.downcast_ref::().unwrap(); let sort_merge = Arc::new( SortPreservingMergeExec::new( LexOrdering::new(sort.expr().to_vec()).unwrap(), @@ -338,7 +338,7 @@ fn wrap_partial_reduce( if !partial_reduce_enabled { return Ok(input); } - let Some(agg) = input.as_any().downcast_ref::() else { + let Some(agg) = input.downcast_ref::() else { return Ok(input); }; if *agg.mode() != AggregateMode::Partial { @@ -429,7 +429,7 @@ impl<'n> TreeNodeVisitor<'n> for TableNameVisitor { self.has_remote_scan = true; Ok(TreeNodeRecursion::Stop) } else if name == "NewEmptyExec" { - let table = node.as_any().downcast_ref::().unwrap(); + let table = node.downcast_ref::().unwrap(); self.table_name = Some(TableReference::from(table.name())); Ok(TreeNodeRecursion::Continue) } else { @@ -546,7 +546,7 @@ mod tests { )?; let input: Arc = Arc::new(agg); let result = wrap_partial_reduce(true, input)?; - let result_agg = result.as_any().downcast_ref::().unwrap(); + let result_agg = result.downcast_ref::().unwrap(); assert_eq!(*result_agg.mode(), AggregateMode::Single); Ok(()) } @@ -566,7 +566,7 @@ mod tests { )?; let input: Arc = Arc::new(agg); let result = wrap_partial_reduce(true, input)?; - let result_agg = result.as_any().downcast_ref::().unwrap(); + let result_agg = result.downcast_ref::().unwrap(); assert_eq!(*result_agg.mode(), AggregateMode::PartialReduce); assert!(result_agg.group_expr().expr().is_empty()); Ok(()) @@ -592,13 +592,12 @@ mod tests { )?; let input: Arc = Arc::new(agg); let result = wrap_partial_reduce(true, input)?; - let result_agg = result.as_any().downcast_ref::().unwrap(); + let result_agg = result.downcast_ref::().unwrap(); assert_eq!(*result_agg.mode(), AggregateMode::PartialReduce); assert_eq!(result_agg.group_expr().expr().len(), 1); let (expr, name) = &result_agg.group_expr().expr()[0]; assert_eq!(name, "a"); let col_expr = expr - .as_any() .downcast_ref::() .expect("group expr should be a Column reference into partial output schema"); assert_eq!(col_expr.name(), "a"); @@ -638,7 +637,7 @@ mod tests { )?; let input: Arc = Arc::new(agg); let result = wrap_partial_reduce(true, input)?; - let result_agg = result.as_any().downcast_ref::().unwrap(); + let result_agg = result.downcast_ref::().unwrap(); assert_eq!(*result_agg.mode(), AggregateMode::PartialReduce); // The critical invariant: input_schema must be the original scan schema so that // aggregate expression argument types (e.g. _timestamp:Int64) are reconstructed @@ -694,7 +693,7 @@ mod tests { )?; let input: Arc = Arc::new(agg); let result = wrap_partial_reduce(true, input)?; - let result_agg = result.as_any().downcast_ref::().unwrap(); + let result_agg = result.downcast_ref::().unwrap(); assert_eq!(*result_agg.mode(), AggregateMode::PartialReduce); let null_exprs = result_agg.group_expr().null_expr(); @@ -702,7 +701,7 @@ mod tests { // Both must still be Literal(NULL), NOT remapped to Column refs. for (expr, _) in null_exprs { assert!( - expr.as_any().is::(), + expr.is::(), "null_expr must remain a Literal, not be replaced with a Column ref" ); } diff --git a/src/service/search/datafusion/optimizer/physical_optimizer/rewrite_match.rs b/src/service/search/datafusion/optimizer/physical_optimizer/rewrite_match.rs index 3d4ddf4e14..514a947272 100644 --- a/src/service/search/datafusion/optimizer/physical_optimizer/rewrite_match.rs +++ b/src/service/search/datafusion/optimizer/physical_optimizer/rewrite_match.rs @@ -99,7 +99,7 @@ impl TreeNodeRewriter for PlanRewriter { type Node = Arc; fn f_up(&mut self, plan: Self::Node) -> Result> { - if let Some(filter) = plan.as_any().downcast_ref::() { + if let Some(filter) = plan.downcast_ref::() { let predicate = filter.predicate(); // 1. Check if the predicate contains any match_all functions @@ -160,7 +160,7 @@ impl TreeNodeRewriter for ColumnIndexRewriter { fn f_up(&mut self, expr: Self::Node) -> Result> { // Check if this is a Column expression - if let Some(column) = expr.as_any().downcast_ref::() { + if let Some(column) = expr.downcast_ref::() { let field_name = column.name(); // Find the new index in the new schema if let Ok(new_index) = self.new_schema.index_of(field_name) { @@ -210,12 +210,9 @@ fn rewrite_match_all_physical( schema: SchemaRef, fields: &[(String, DataType)], ) -> Result> { - let scalar_fn = expr - .as_any() - .downcast_ref::() - .ok_or_else(|| { - DataFusionError::Internal("Expected ScalarFunctionExpr for match_all".to_string()) - })?; + let scalar_fn = expr.downcast_ref::().ok_or_else(|| { + DataFusionError::Internal("Expected ScalarFunctionExpr for match_all".to_string()) + })?; let name = scalar_fn.name(); let args = scalar_fn.args(); @@ -398,7 +395,7 @@ fn has_match_all_function(expr: &Arc) -> bool { // check if the expr is match_all function fn is_match_all_physical(expr: &Arc) -> bool { - if let Some(scalar_fn) = expr.as_any().downcast_ref::() { + if let Some(scalar_fn) = expr.downcast_ref::() { let name = scalar_fn.name(); name.to_lowercase() == MATCH_ALL_UDF_NAME || name == FUZZY_MATCH_ALL_UDF_NAME @@ -428,7 +425,7 @@ impl TreeNodeRewriter for AddFstFieldsToProjection { type Node = Arc; fn f_up(&mut self, plan: Self::Node) -> Result> { - if let Some(empty_exec) = plan.as_any().downcast_ref::() { + if let Some(empty_exec) = plan.downcast_ref::() { let schema = empty_exec.full_schema(); // used for read field from parquet file, because match_all function do not include // the field in argument, so when we rewrite match_all function, we need to add @@ -658,13 +655,13 @@ mod tests { // Verify that all column indices in FilterExec match the input schema let _ = physical_plan.apply(&mut |node: &Arc| -> Result { - if let Some(filter) = node.as_any().downcast_ref::() { + if let Some(filter) = node.downcast_ref::() { let input_schema = filter.input().schema(); let predicate = filter.predicate(); // Traverse the predicate expression tree to find all Column references let _ = predicate.apply(&mut |expr: &Arc| -> Result { - if let Some(column) = expr.as_any().downcast_ref::() { + if let Some(column) = expr.downcast_ref::() { let column_name = column.name(); let column_index = column.index(); diff --git a/src/service/search/datafusion/optimizer/physical_optimizer/utils.rs b/src/service/search/datafusion/optimizer/physical_optimizer/utils.rs index 0707f4aa33..7ef5a094fc 100644 --- a/src/service/search/datafusion/optimizer/physical_optimizer/utils.rs +++ b/src/service/search/datafusion/optimizer/physical_optimizer/utils.rs @@ -38,7 +38,7 @@ pub fn is_aggregate_exec(plan: &Arc) -> bool { } pub fn extract_string_literal(expr: &Arc) -> Result { - if let Some(literal) = expr.as_any().downcast_ref::() { + if let Some(literal) = expr.downcast_ref::() { match literal.value() { ScalarValue::Utf8(Some(s)) => Ok(s.clone()), ScalarValue::Utf8View(Some(s)) => Ok(s.to_string()), @@ -56,7 +56,7 @@ pub fn extract_string_literal(expr: &Arc) -> Result { } pub fn extract_column(expr: &Arc) -> Result { - if let Some(column) = expr.as_any().downcast_ref::() { + if let Some(column) = expr.downcast_ref::() { Ok(column.clone()) } else { Err(DataFusionError::Internal( @@ -66,7 +66,7 @@ pub fn extract_column(expr: &Arc) -> Result { } pub fn extract_int64_literal(expr: &Arc) -> Result { - if let Some(literal) = expr.as_any().downcast_ref::() { + if let Some(literal) = expr.downcast_ref::() { match literal.value() { ScalarValue::Int64(Some(s)) => Ok(*s), _ => Err(DataFusionError::Internal(format!( @@ -100,9 +100,9 @@ fn disjunction_opt( } pub fn is_column(expr: &Arc) -> bool { - if expr.as_any().downcast_ref::().is_some() { + if expr.downcast_ref::().is_some() { true - } else if let Some(expr) = expr.as_any().downcast_ref::() { + } else if let Some(expr) = expr.downcast_ref::() { is_column(expr.expr()) } else { false @@ -110,9 +110,9 @@ pub fn is_column(expr: &Arc) -> bool { } pub fn get_column_name(expr: &Arc) -> &str { - if let Some(expr) = expr.as_any().downcast_ref::() { + if let Some(expr) = expr.downcast_ref::() { expr.name() - } else if let Some(expr) = expr.as_any().downcast_ref::() { + } else if let Some(expr) = expr.downcast_ref::() { get_column_name(expr.expr()) } else { UNKNOWN_NAME @@ -133,7 +133,7 @@ pub fn is_count_rows_aggregate(expr: &AggregateFunctionExpr) -> bool { } pub fn is_value(expr: &Arc) -> bool { - expr.as_any().downcast_ref::().is_some() + expr.downcast_ref::().is_some() } pub fn is_only_timestamp_filter(expr: &[&Arc]) -> bool { @@ -141,7 +141,7 @@ pub fn is_only_timestamp_filter(expr: &[&Arc]) -> bool { } fn is_timestamp_filter(expr: &Arc) -> bool { - if let Some(expr) = expr.as_any().downcast_ref::() { + if let Some(expr) = expr.downcast_ref::() { match expr.op() { Operator::Gt | Operator::GtEq | Operator::Lt | Operator::LtEq => { let column = if is_value(expr.left()) && is_column(expr.right()) { @@ -286,9 +286,7 @@ mod tests { fn test_disjunction_empty_returns_true_literal() { let result = disjunction(vec![]); // empty → lit(true) - let lit = result - .as_any() - .downcast_ref::(); + let lit = result.downcast_ref::(); assert!(lit.is_some()); } diff --git a/src/service/search/datafusion/optimizer/utils.rs b/src/service/search/datafusion/optimizer/utils.rs index 9dbd685844..c1fff29411 100644 --- a/src/service/search/datafusion/optimizer/utils.rs +++ b/src/service/search/datafusion/optimizer/utils.rs @@ -287,15 +287,8 @@ impl TreeNodeRewriter for SortByTime { fn generate_table_source_with_sorted_by_time( table_source: Arc, ) -> Arc { - let source: &DefaultTableSource = table_source - .as_any() - .downcast_ref::() - .unwrap(); - if let Some(table_provider) = source - .table_provider - .as_any() - .downcast_ref::() - { + let source: &DefaultTableSource = table_source.downcast_ref::().unwrap(); + if let Some(table_provider) = source.table_provider.downcast_ref::() { let mut new_table_provider = (*table_provider).clone(); new_table_provider.sorted_by_time = true; let new_source = DefaultTableSource::new(Arc::new(new_table_provider)); diff --git a/src/service/search/datafusion/peak_memory_pool.rs b/src/service/search/datafusion/peak_memory_pool.rs index 4be7fa13a3..1d30486dea 100644 --- a/src/service/search/datafusion/peak_memory_pool.rs +++ b/src/service/search/datafusion/peak_memory_pool.rs @@ -13,9 +13,12 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::sync::{ - Arc, - atomic::{AtomicUsize, Ordering}, +use std::{ + fmt, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, }; use datafusion::execution::memory_pool::{ @@ -51,6 +54,10 @@ impl PeakMemoryPool { } impl MemoryPool for PeakMemoryPool { + fn name(&self) -> &str { + "PeakMemoryPool" + } + fn register(&self, consumer: &MemoryConsumer) { self.inner.register(consumer); } @@ -91,6 +98,12 @@ impl MemoryPool for PeakMemoryPool { } } +impl fmt::Display for PeakMemoryPool { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}({})", self.name(), self.inner) + } +} + impl Drop for PeakMemoryPool { fn drop(&mut self) { let peak = self.peak_memory.load(Ordering::Relaxed); diff --git a/src/service/search/datafusion/plan/deduplication_exec.rs b/src/service/search/datafusion/plan/deduplication_exec.rs index 3e37152a04..0f3206542c 100644 --- a/src/service/search/datafusion/plan/deduplication_exec.rs +++ b/src/service/search/datafusion/plan/deduplication_exec.rs @@ -14,7 +14,6 @@ // along with this program. If not, see . use std::{ - any::Any, sync::Arc, task::{Context, Poll}, }; @@ -111,10 +110,6 @@ impl ExecutionPlan for DeduplicationExec { "DeduplicationExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { &self.cache } @@ -161,7 +156,7 @@ impl ExecutionPlan for DeduplicationExec { ))) } - fn partition_statistics(&self, partition: Option) -> Result { + fn partition_statistics(&self, partition: Option) -> Result> { self.input.partition_statistics(partition) } @@ -455,7 +450,8 @@ mod tests { #[test] fn test_deduplication_exec_as_any() { let exec = make_exec(); - assert!(exec.as_any().downcast_ref::().is_some()); + let exec_plan: &dyn ExecutionPlan = &exec; + assert!(exec_plan.downcast_ref::().is_some()); } #[test] diff --git a/src/service/search/datafusion/plan/tantivy_optimize_exec.rs b/src/service/search/datafusion/plan/tantivy_optimize_exec.rs index 4019b4d0dc..fb6f765aaf 100644 --- a/src/service/search/datafusion/plan/tantivy_optimize_exec.rs +++ b/src/service/search/datafusion/plan/tantivy_optimize_exec.rs @@ -13,7 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{any::Any, collections::HashSet, sync::Arc}; +use std::{collections::HashSet, sync::Arc}; use arrow::array::{ Array, Int64Array, TimestampMicrosecondArray, TimestampNanosecondArray, UInt64Array, @@ -114,10 +114,6 @@ impl ExecutionPlan for TantivyOptimizeExec { "TantivyOptimizeExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { &self.cache } diff --git a/src/service/search/datafusion/storage/file_statistics_cache.rs b/src/service/search/datafusion/storage/file_statistics_cache.rs index 0bda971f30..30642cd1f1 100644 --- a/src/service/search/datafusion/storage/file_statistics_cache.rs +++ b/src/service/search/datafusion/storage/file_statistics_cache.rs @@ -25,10 +25,10 @@ use std::{ use config::metrics; use dashmap::DashMap; use datafusion::{ - common::Statistics, + common::{Statistics, TableReference}, execution::cache::{ - CacheAccessor, - cache_manager::{CachedFileMetadata, FileStatisticsCacheEntry}, + CacheAccessor, TableScopedPath, + cache_manager::{self, CachedFileMetadata, FileStatisticsCacheEntry}, }, }; use object_store::{ObjectMeta, path::Path}; @@ -145,10 +145,10 @@ impl Default for FileStatisticsCache { } } -impl CacheAccessor for FileStatisticsCache { +impl CacheAccessor for FileStatisticsCache { /// Get cached metadata for file location. - fn get(&self, k: &Path) -> Option { - let k = self.format_key(k); + fn get(&self, k: &TableScopedPath) -> Option { + let k = self.format_key(&k.path); match self.statistics.get(&k) { Some(s) => { metrics::QUERY_PARQUET_METADATA_CACHE_HITS_TOTAL @@ -171,8 +171,8 @@ impl CacheAccessor for FileStatisticsCache { } /// Save collected file statistics - fn put(&self, k: &Path, value: CachedFileMetadata) -> Option { - let k = self.format_key(k); + fn put(&self, k: &TableScopedPath, value: CachedFileMetadata) -> Option { + let k = self.format_key(&k.path); let entry_size = Self::estimate_entry_size(&k, &value.meta, &value.statistics); let old = self.statistics.insert( @@ -201,8 +201,8 @@ impl CacheAccessor for FileStatisticsCache { old.map(|(meta, stats, _)| CachedFileMetadata::new(meta, stats, None)) } - fn remove(&self, k: &Path) -> Option { - let k = self.format_key(k); + fn remove(&self, k: &TableScopedPath) -> Option { + let k = self.format_key(&k.path); self.statistics.remove(&k).map(|(_, (meta, stats, size))| { self.current_memory .fetch_sub(size as i64, Ordering::Relaxed); @@ -210,8 +210,8 @@ impl CacheAccessor for FileStatisticsCache { }) } - fn contains_key(&self, k: &Path) -> bool { - let k = self.format_key(k); + fn contains_key(&self, k: &TableScopedPath) -> bool { + let k = self.format_key(&k.path); self.statistics.contains_key(&k) } @@ -224,17 +224,34 @@ impl CacheAccessor for FileStatisticsCache { self.cacher.lock().clear(); self.current_memory.store(0, Ordering::Relaxed); } + fn name(&self) -> String { "FileStatisticsCache".to_string() } } -impl datafusion::execution::cache::cache_manager::FileStatisticsCache for FileStatisticsCache { - fn list_entries(&self) -> HashMap { - let mut entries = HashMap::::new(); +impl cache_manager::FileStatisticsCache for FileStatisticsCache { + fn cache_limit(&self) -> usize { + config::get_config() + .limit + .datafusion_file_stat_cache_max_size + } + + fn update_cache_limit(&self, _limit: usize) { + // No-op: this is a process-wide, long-lived global cache whose eviction + // budget is owned solely by `put()` via `datafusion_file_stat_cache_max_size`. + // DataFusion calls this once per `CacheManager::try_new` (i.e. per session / + // per query) with its own per-session limit; + } + + fn list_entries(&self) -> HashMap { + let mut entries = HashMap::::new(); for entry in &self.statistics { - let path = Path::from(entry.key().as_str()); + let path = TableScopedPath { + table: None, + path: Path::from(entry.key().as_str()), + }; let (object_meta, stats, _) = entry.value(); entries.insert( path, @@ -251,86 +268,102 @@ impl datafusion::execution::cache::cache_manager::FileStatisticsCache for FileSt entries } + + fn drop_table_entries( + &self, + _table_ref: &Option, + ) -> datafusion::error::Result<()> { + Ok(()) + } } #[cfg(test)] mod tests { - use chrono::DateTime; + use chrono::{DateTime, Utc}; use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; use super::*; - #[test] - fn test_file_statistics_cache() { - let meta = ObjectMeta { - location: Path::from("test"), - last_modified: DateTime::parse_from_rfc3339("2022-09-27T22:36:00+02:00") - .unwrap() - .into(), - size: 1024, + /// Wrap a [`Path`] into the table-scoped key the cache API expects. + fn key(location: &Path) -> TableScopedPath { + TableScopedPath { + table: None, + path: location.clone(), + } + } + + /// Parse an RFC3339 timestamp, panicking on malformed input (test-only). + fn ts(s: &str) -> DateTime { + DateTime::parse_from_rfc3339(s).unwrap().into() + } + + /// Minimal [`ObjectMeta`] for a location, fixed `last_modified`, no etag/version. + fn object_meta(location: &str, size: u64) -> ObjectMeta { + ObjectMeta { + location: Path::from(location), + last_modified: ts("2024-01-15T00:00:00+00:00"), + size, e_tag: None, version: None, - }; - let cache = FileStatisticsCache::default(); - assert!(cache.get(&meta.location).is_none()); + } + } - let stats: Arc = Statistics::new_unknown(&Schema::new(vec![Field::new( + /// Unknown statistics for the given fields, ready to cache. + fn unknown_stats(fields: Vec) -> Arc { + Statistics::new_unknown(&Schema::new(fields)).into() + } + + /// Insert an entry into the cache under its own location. + fn put(cache: &FileStatisticsCache, meta: ObjectMeta, stats: Arc) { + cache.put( + &key(&meta.location), + CachedFileMetadata::new(meta, stats, None), + ); + } + + #[test] + fn test_file_statistics_cache() { + let meta = object_meta("test", 1024); + let cache = FileStatisticsCache::default(); + assert!(cache.get(&key(&meta.location)).is_none()); + + let stats = unknown_stats(vec![Field::new( "test_column", DataType::Timestamp(TimeUnit::Second, None), false, - )])) - .into(); - cache.put( - &meta.location, - CachedFileMetadata::new(meta.clone(), stats, None), - ); - let cached = cache.get(&meta.location); - assert!(cached.is_some()); - assert!(cached.unwrap().is_valid_for(&meta)); + )]); + put(&cache, meta.clone(), stats); - // file size changed - let mut meta2 = meta.clone(); - meta2.size = 2048; - let cached = cache.get(&meta2.location); - assert!(cached.is_some()); - assert!(!cached.unwrap().is_valid_for(&meta2)); + // exact match is valid + let cached = cache.get(&key(&meta.location)).expect("entry present"); + assert!(cached.is_valid_for(&meta)); - // file last_modified changed - let mut meta2 = meta.clone(); - meta2.last_modified = DateTime::parse_from_rfc3339("2022-09-27T22:40:00+02:00") - .unwrap() - .into(); - let cached = cache.get(&meta2.location); - assert!(cached.is_some()); - assert!(!cached.unwrap().is_valid_for(&meta2)); + // same location but file size changed -> cached but stale + let mut changed = meta.clone(); + changed.size = 2048; + let cached = cache.get(&key(&changed.location)).expect("entry present"); + assert!(!cached.is_valid_for(&changed)); - // different file - let mut meta2 = meta; - meta2.location = Path::from("test2"); - assert!(cache.get(&meta2.location).is_none()); + // same location but last_modified changed -> cached but stale + let mut changed = meta.clone(); + changed.last_modified = ts("2024-01-15T01:00:00+00:00"); + let cached = cache.get(&key(&changed.location)).expect("entry present"); + assert!(!cached.is_valid_for(&changed)); + + // different location -> miss + assert!(cache.get(&key(&Path::from("test2"))).is_none()); } #[test] fn test_memory_size_calculation() { let cache = FileStatisticsCache::new(); + assert_eq!(cache.memory_size(), 0, "empty cache tracks zero memory"); - // Empty cache tracks zero memory - let empty_size = cache.memory_size(); - assert_eq!(empty_size, 0); - - // Add some entries - for i in 0..10 { - let meta = ObjectMeta { - location: Path::from(format!("test_file_{i}")), - last_modified: DateTime::parse_from_rfc3339("2022-09-27T22:36:00+02:00") - .unwrap() - .into(), - size: 1024 * (i + 1), - e_tag: Some(format!("etag_{i}")), - version: Some(format!("v{i}")), - }; - - let schema = Schema::new(vec![ + for i in 0..10u64 { + let mut meta = object_meta(&format!("test_file_{i}"), 1024 * (i + 1)); + meta.e_tag = Some(format!("etag_{i}")); + meta.version = Some(format!("v{i}")); + let stats = unknown_stats(vec![ Field::new("column1", DataType::Utf8, false), Field::new("column2", DataType::Int64, true), Field::new( @@ -339,84 +372,55 @@ mod tests { false, ), ]); - - cache.put( - &meta.location, - CachedFileMetadata::new( - meta.clone(), - Statistics::new_unknown(&schema).into(), - None, - ), - ); + put(&cache, meta, stats); } - // Size should increase after adding entries - let filled_size = cache.memory_size(); - assert!(filled_size > empty_size); - - println!( - "Cache with {} entries uses {} bytes", - cache.len(), - filled_size, + assert!( + cache.memory_size() > 0, + "memory tracked after inserting entries" ); + assert_eq!(cache.len(), 10); - // After clearing, tracked memory must return to zero cache.clear(); - assert_eq!(cache.memory_size(), 0); + assert_eq!(cache.memory_size(), 0, "cleared cache returns to zero"); } #[test] fn test_cache_name() { - let cache = FileStatisticsCache::new(); - assert_eq!(cache.name(), "FileStatisticsCache"); + assert_eq!(FileStatisticsCache::new().name(), "FileStatisticsCache"); } #[test] fn test_cache_contains_key_and_remove() { let cache = FileStatisticsCache::new(); - let path = Path::from("test_file"); - let meta = ObjectMeta { - location: path.clone(), - last_modified: DateTime::parse_from_rfc3339("2024-01-15T00:00:00+00:00") - .unwrap() - .into(), - size: 512, - e_tag: None, - version: None, - }; - let schema = Schema::new(vec![Field::new("col", DataType::Utf8, false)]); - let stats: Arc = Statistics::new_unknown(&schema).into(); + let meta = object_meta("test_file", 512); + let k = key(&meta.location); - assert!(!cache.contains_key(&path)); + assert!(!cache.contains_key(&k)); - cache.put(&path, CachedFileMetadata::new(meta.clone(), stats, None)); - assert!(cache.contains_key(&path)); + put( + &cache, + meta, + unknown_stats(vec![Field::new("col", DataType::Utf8, false)]), + ); + assert!(cache.contains_key(&k)); assert_eq!(cache.len(), 1); - let removed = cache.remove(&path); - assert!(removed.is_some()); - assert!(!cache.contains_key(&path)); + assert!(cache.remove(&k).is_some()); + assert!(!cache.contains_key(&k)); assert_eq!(cache.len(), 0); } #[test] fn test_cache_clear() { let cache = FileStatisticsCache::new(); - let schema = Schema::new(vec![Field::new("col", DataType::Int64, false)]); - - for i in 0..3 { - let path = Path::from(format!("file_{i}")); - let meta = ObjectMeta { - location: path.clone(), - last_modified: DateTime::parse_from_rfc3339("2024-01-15T00:00:00+00:00") - .unwrap() - .into(), - size: 100 * (i + 1), - e_tag: None, - version: None, - }; - let stats: Arc = Statistics::new_unknown(&schema).into(); - cache.put(&path, CachedFileMetadata::new(meta, stats, None)); + for i in 0..3u64 { + let meta = object_meta(&format!("file_{i}"), 100 * (i + 1)); + put( + &cache, + meta, + unknown_stats(vec![Field::new("col", DataType::Int64, false)]), + ); } assert_eq!(cache.len(), 3); @@ -428,34 +432,25 @@ mod tests { fn test_list_entries() { use datafusion::execution::cache::cache_manager::FileStatisticsCache as FscTrait; - let cache = super::FileStatisticsCache::new(); - let path = Path::from("list_test"); - let meta = ObjectMeta { - location: path.clone(), - last_modified: DateTime::parse_from_rfc3339("2024-01-15T00:00:00+00:00") - .unwrap() - .into(), - size: 256, - e_tag: None, - version: None, - }; - let schema = Schema::new(vec![Field::new("col", DataType::Utf8, false)]); - let stats: Arc = Statistics::new_unknown(&schema).into(); + let cache = FileStatisticsCache::new(); + let meta = object_meta("list_test", 256); + let k = key(&meta.location); - let entries_before = FscTrait::list_entries(&cache); - assert!(entries_before.is_empty()); + assert!(FscTrait::list_entries(&cache).is_empty()); - cache.put(&path, CachedFileMetadata::new(meta, stats, None)); + put( + &cache, + meta, + unknown_stats(vec![Field::new("col", DataType::Utf8, false)]), + ); let entries = FscTrait::list_entries(&cache); assert_eq!(entries.len(), 1); - assert!(entries.contains_key(&path)); + assert!(entries.contains_key(&k)); } #[test] fn test_cache_remove_nonexistent_returns_none() { let cache = FileStatisticsCache::new(); - let path = Path::from("does_not_exist"); - let removed = cache.remove(&path); - assert!(removed.is_none()); + assert!(cache.remove(&key(&Path::from("does_not_exist"))).is_none()); } } diff --git a/src/service/search/datafusion/table_provider/catalog.rs b/src/service/search/datafusion/table_provider/catalog.rs index 6aaef8e6a9..3399757ca7 100644 --- a/src/service/search/datafusion/table_provider/catalog.rs +++ b/src/service/search/datafusion/table_provider/catalog.rs @@ -14,7 +14,6 @@ // along with this program. If not, see . use std::{ - any::Any, collections::HashMap, sync::{Arc, RwLock}, }; @@ -40,10 +39,6 @@ impl StreamTypeProvider { #[async_trait] impl SchemaProvider for StreamTypeProvider { - fn as_any(&self) -> &dyn Any { - self - } - fn table_names(&self) -> Vec { let tables = self.tables.read().unwrap(); tables.keys().cloned().collect::>() @@ -104,9 +99,6 @@ mod tests { #[async_trait] impl TableProvider for DummyTable { - fn as_any(&self) -> &dyn std::any::Any { - self - } fn schema(&self) -> SchemaRef { self.0.clone() } diff --git a/src/service/search/datafusion/table_provider/empty_table.rs b/src/service/search/datafusion/table_provider/empty_table.rs index d212d287c4..e8c1e0c1e8 100644 --- a/src/service/search/datafusion/table_provider/empty_table.rs +++ b/src/service/search/datafusion/table_provider/empty_table.rs @@ -13,7 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{any::Any, sync::Arc}; +use std::sync::Arc; use async_trait::async_trait; use datafusion::{ @@ -64,10 +64,6 @@ impl NewEmptyTable { #[async_trait] impl TableProvider for NewEmptyTable { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.schema.clone() } diff --git a/src/service/search/datafusion/table_provider/enrich_table.rs b/src/service/search/datafusion/table_provider/enrich_table.rs index e6e71583b3..7fcaeb8197 100644 --- a/src/service/search/datafusion/table_provider/enrich_table.rs +++ b/src/service/search/datafusion/table_provider/enrich_table.rs @@ -13,7 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{any::Any, sync::Arc}; +use std::sync::Arc; use async_trait::async_trait; use config::TIMESTAMP_COL_NAME; @@ -67,10 +67,6 @@ impl EnrichTable { #[async_trait] impl TableProvider for EnrichTable { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.schema.clone() } diff --git a/src/service/search/datafusion/table_provider/listing_adapter.rs b/src/service/search/datafusion/table_provider/listing_adapter.rs index a8aa88785c..a131b031c0 100644 --- a/src/service/search/datafusion/table_provider/listing_adapter.rs +++ b/src/service/search/datafusion/table_provider/listing_adapter.rs @@ -13,7 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{any::Any, sync::Arc}; +use std::sync::Arc; use arrow_schema::{SchemaRef, SortOptions}; use config::{TIMESTAMP_COL_NAME, get_config}; @@ -74,10 +74,6 @@ impl ListingTableAdapter { #[async_trait] impl TableProvider for ListingTableAdapter { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { Arc::clone(&self.listing_table.schema()) } @@ -188,10 +184,9 @@ fn handler_tantivy_index( order_by_time_desc: bool, target_partitions: usize, ) -> Arc { - if let Some(data_source_exec) = plan.as_any().downcast_ref::() + if let Some(data_source_exec) = plan.downcast_ref::() && let Some(config) = data_source_exec .data_source() - .as_any() .downcast_ref::() { let mut file_groups = config.file_groups.clone(); @@ -250,7 +245,7 @@ fn handler_tantivy_index( .into_iter() .map(|mut file| { if let Some(access_plan) = generate_access_plan(&file) { - file = file.with_extensions(access_plan); + file = file.with_extension(access_plan); } file }) diff --git a/src/service/search/datafusion/table_provider/memtable.rs b/src/service/search/datafusion/table_provider/memtable.rs index 3e00d55e8b..02395b86c0 100644 --- a/src/service/search/datafusion/table_provider/memtable.rs +++ b/src/service/search/datafusion/table_provider/memtable.rs @@ -13,7 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{any::Any, sync::Arc}; +use std::sync::Arc; use arrow::array::RecordBatch; use arrow_schema::{DataType, SchemaRef, SortOptions}; @@ -120,10 +120,6 @@ mod tests { #[async_trait] impl TableProvider for NewMemTable { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.mem_table.schema() } diff --git a/src/service/search/datafusion/table_provider/uniontable.rs b/src/service/search/datafusion/table_provider/uniontable.rs index 1f49d0623c..938b0b6fd9 100644 --- a/src/service/search/datafusion/table_provider/uniontable.rs +++ b/src/service/search/datafusion/table_provider/uniontable.rs @@ -13,7 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{any::Any, sync::Arc}; +use std::sync::Arc; use arrow_schema::SchemaRef; use async_trait::async_trait; @@ -40,10 +40,6 @@ impl NewUnionTable { #[async_trait] impl TableProvider for NewUnionTable { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { Arc::clone(&self.schema) } diff --git a/src/service/search/datafusion/udaf/summary_percentile.rs b/src/service/search/datafusion/udaf/summary_percentile.rs index 85eb5a6f59..4ee3b042b6 100644 --- a/src/service/search/datafusion/udaf/summary_percentile.rs +++ b/src/service/search/datafusion/udaf/summary_percentile.rs @@ -82,10 +82,6 @@ impl Default for SummaryPercentile { } impl AggregateUDFImpl for SummaryPercentile { - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn name(&self) -> &str { SUMMARY_PERCENTILE } diff --git a/src/service/search/datafusion/udf/cast_to_timestamp_udf.rs b/src/service/search/datafusion/udf/cast_to_timestamp_udf.rs index 652f3f97e6..f1ce52d733 100644 --- a/src/service/search/datafusion/udf/cast_to_timestamp_udf.rs +++ b/src/service/search/datafusion/udf/cast_to_timestamp_udf.rs @@ -13,7 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{any::Any, sync::LazyLock as Lazy}; +use std::sync::LazyLock as Lazy; use arrow_schema::TimeUnit; use datafusion::{ @@ -47,10 +47,6 @@ impl CastToTimestampUdf { } impl ScalarUDFImpl for CastToTimestampUdf { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { CAST_TO_TIMESTAMP_UDF_NAME } diff --git a/src/service/search/datafusion/udf/histogram_udf.rs b/src/service/search/datafusion/udf/histogram_udf.rs index 2e14759a53..8d878d6cc3 100644 --- a/src/service/search/datafusion/udf/histogram_udf.rs +++ b/src/service/search/datafusion/udf/histogram_udf.rs @@ -13,7 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{any::Any, sync::LazyLock as Lazy}; +use std::sync::LazyLock as Lazy; use arrow::datatypes::{DataType, DataType::Timestamp, TimeUnit::Microsecond}; use datafusion::{ @@ -40,10 +40,6 @@ impl HistogramUdf { } impl ScalarUDFImpl for HistogramUdf { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { HISTOGRAM_UDF_NAME } diff --git a/src/service/search/datafusion/udf/match_all_udf.rs b/src/service/search/datafusion/udf/match_all_udf.rs index 4c0b34e7bf..d53680751b 100644 --- a/src/service/search/datafusion/udf/match_all_udf.rs +++ b/src/service/search/datafusion/udf/match_all_udf.rs @@ -13,7 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{any::Any, sync::LazyLock as Lazy}; +use std::sync::LazyLock as Lazy; use arrow::datatypes::DataType; use datafusion::{ @@ -44,10 +44,6 @@ impl MatchAllUdf { } impl ScalarUDFImpl for MatchAllUdf { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { MATCH_ALL_UDF_NAME } @@ -87,10 +83,6 @@ impl FuzzyMatchAllUdf { } impl ScalarUDFImpl for FuzzyMatchAllUdf { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { FUZZY_MATCH_ALL_UDF_NAME } diff --git a/src/service/search/datafusion/udf/regexp_matches_udf.rs b/src/service/search/datafusion/udf/regexp_matches_udf.rs index dc1bba5978..d0ba9371fc 100644 --- a/src/service/search/datafusion/udf/regexp_matches_udf.rs +++ b/src/service/search/datafusion/udf/regexp_matches_udf.rs @@ -13,10 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{ - any::Any, - sync::{Arc, LazyLock as Lazy}, -}; +use std::sync::{Arc, LazyLock as Lazy}; use arrow::array::{Array, ArrayRef, GenericStringBuilder, ListBuilder, OffsetSizeTrait}; use arrow_schema::Field; @@ -87,10 +84,6 @@ impl RegexpMatchesFunc { } impl ScalarUDFImpl for RegexpMatchesFunc { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { REGEX_MATCHES_UDF_NAME } diff --git a/src/service/search/datafusion/udf/regexp_udf.rs b/src/service/search/datafusion/udf/regexp_udf.rs index 83289b83a3..0e41116095 100644 --- a/src/service/search/datafusion/udf/regexp_udf.rs +++ b/src/service/search/datafusion/udf/regexp_udf.rs @@ -247,10 +247,6 @@ impl RegxpMatchToFields { } impl ScalarUDFImpl for RegxpMatchToFields { - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn name(&self) -> &str { "regexp_match_to_fields" } diff --git a/src/service/search/datafusion/udf/str_match_udf.rs b/src/service/search/datafusion/udf/str_match_udf.rs index f7a5223375..71cbfd8be8 100644 --- a/src/service/search/datafusion/udf/str_match_udf.rs +++ b/src/service/search/datafusion/udf/str_match_udf.rs @@ -13,10 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{ - any::Any, - sync::{Arc, LazyLock as Lazy}, -}; +use std::sync::{Arc, LazyLock as Lazy}; use datafusion::{ arrow::{ @@ -54,10 +51,6 @@ impl StrMatchUdf { } impl ScalarUDFImpl for StrMatchUdf { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { super::STR_MATCH_UDF_NAME } @@ -95,10 +88,6 @@ impl StrMatchIgnoreCaseUdf { } impl ScalarUDFImpl for StrMatchIgnoreCaseUdf { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { super::STR_MATCH_UDF_IGNORE_CASE_NAME } diff --git a/src/service/search/index.rs b/src/service/search/index.rs index 02a62414ea..7497145baa 100644 --- a/src/service/search/index.rs +++ b/src/service/search/index.rs @@ -267,7 +267,7 @@ impl Condition { } pub fn from_physical_expr(expr: &Arc) -> Self { - if let Some(expr) = expr.as_any().downcast_ref::() { + if let Some(expr) = expr.downcast_ref::() { match expr.op() { Operator::Eq | Operator::NotEq => { let (field, value) = if is_physical_value(expr.left()) @@ -302,11 +302,11 @@ impl Condition { ), _ => unreachable!(), } - } else if let Some(expr) = expr.as_any().downcast_ref::() { + } else if let Some(expr) = expr.downcast_ref::() { let field = get_physical_column_name(expr.expr()).to_string(); let values = expr.list().iter().map(get_physical_value).collect(); Condition::In(field, values, expr.negated()) - } else if let Some(expr) = expr.as_any().downcast_ref::() { + } else if let Some(expr) = expr.downcast_ref::() { let name = expr.name(); match name { MATCH_ALL_UDF_NAME => Condition::MatchAll(get_physical_value(&expr.args()[0])), @@ -327,7 +327,7 @@ impl Condition { } _ => unreachable!(), } - } else if let Some(expr) = expr.as_any().downcast_ref::() { + } else if let Some(expr) = expr.downcast_ref::() { Condition::Not(Box::new(Condition::from_physical_expr(expr.arg()))) } else { unreachable!() @@ -705,9 +705,9 @@ impl Condition { // TODO: duplication with datafusion/optimizer/physical_optimizer/utils.rs fn is_physical_column(expr: &Arc) -> bool { - if expr.as_any().downcast_ref::().is_some() { + if expr.downcast_ref::().is_some() { true - } else if let Some(expr) = expr.as_any().downcast_ref::() { + } else if let Some(expr) = expr.downcast_ref::() { is_physical_column(expr.expr()) } else { false @@ -716,9 +716,9 @@ fn is_physical_column(expr: &Arc) -> bool { // TODO: duplication with datafusion/optimizer/physical_optimizer/utils.rs fn get_physical_column_name(expr: &Arc) -> &str { - if let Some(expr) = expr.as_any().downcast_ref::() { + if let Some(expr) = expr.downcast_ref::() { expr.name() - } else if let Some(expr) = expr.as_any().downcast_ref::() { + } else if let Some(expr) = expr.downcast_ref::() { get_physical_column_name(expr.expr()) } else { UNKNOWN_NAME @@ -726,11 +726,11 @@ fn get_physical_column_name(expr: &Arc) -> &str { } fn is_physical_value(expr: &Arc) -> bool { - expr.as_any().downcast_ref::().is_some() + expr.downcast_ref::().is_some() } fn get_physical_value(expr: &Arc) -> String { - if let Some(literal) = expr.as_any().downcast_ref::() { + if let Some(literal) = expr.downcast_ref::() { match literal.value() { ScalarValue::Boolean(Some(b)) => b.to_string(), ScalarValue::Int64(Some(i)) => i.to_string(), diff --git a/src/service/search/sql/rewriter/track_total_hits.rs b/src/service/search/sql/rewriter/track_total_hits.rs index 8a5498c748..c1df4b5dae 100644 --- a/src/service/search/sql/rewriter/track_total_hits.rs +++ b/src/service/search/sql/rewriter/track_total_hits.rs @@ -89,7 +89,7 @@ impl VisitorMut for TrackTotalHitsVisitor { value_table_mode: None, exclude: None, flavor: SelectFlavor::Standard, - optimizer_hint: None, + optimizer_hints: vec![], select_modifiers: None, }))); *query = Query { @@ -177,7 +177,7 @@ impl VisitorMut for TrackTotalHitsVisitor { value_table_mode: None, exclude: None, flavor: SelectFlavor::Standard, - optimizer_hint: None, + optimizer_hints: vec![], select_modifiers: None, }))); *query = Query { diff --git a/src/service/search/sql/visitor/group_by.rs b/src/service/search/sql/visitor/group_by.rs index 5940384e98..ada0740204 100644 --- a/src/service/search/sql/visitor/group_by.rs +++ b/src/service/search/sql/visitor/group_by.rs @@ -73,7 +73,7 @@ impl<'n> TreeNodeVisitor<'n> for GroupByFieldVisitor { fn f_up(&mut self, node: &'n Self::Node) -> Result { let name = node.name(); if name == "AggregateExec" { - let aggregate = node.as_any().downcast_ref::().unwrap(); + let aggregate = node.downcast_ref::().unwrap(); let group_by = aggregate.group_expr(); let fields = group_by .expr() diff --git a/src/service/search/utils.rs b/src/service/search/utils.rs index 45bdba7476..f06ae0b34e 100644 --- a/src/service/search/utils.rs +++ b/src/service/search/utils.rs @@ -78,7 +78,7 @@ impl ExecutionPlanVisitor for ScanStatsVisitor { type Error = datafusion::common::DataFusionError; fn pre_visit(&mut self, plan: &dyn ExecutionPlan) -> Result { - let mayby_remote_scan_exec = plan.as_any().downcast_ref::(); + let mayby_remote_scan_exec = plan.downcast_ref::(); if let Some(remote_scan_exec) = mayby_remote_scan_exec { { let guard = remote_scan_exec.scan_stats.lock(); diff --git a/src/service/tantivy/reader/vortex.rs b/src/service/tantivy/reader/vortex.rs index eb699dc689..76b17a4ab6 100644 --- a/src/service/tantivy/reader/vortex.rs +++ b/src/service/tantivy/reader/vortex.rs @@ -49,9 +49,10 @@ pub(super) async fn scan_vortex_async( let stream = scan.into_array_stream()?.then(move |result| { let data_type = data_type.clone(); + let session = session.clone(); async move { match result { - Ok(array) => vortex_array_to_record_batch(array, &data_type), + Ok(array) => vortex_array_to_record_batch(&session, array, &data_type), Err(e) => Err(ArrowError::ExternalError(Box::new(e))), } } @@ -78,7 +79,7 @@ pub(super) fn scan_vortex_row_range( let iter: Box> + 'static> = Box::new( scan.into_array_iter(&runtime)? .map(move |result| match result { - Ok(array) => vortex_array_to_record_batch(array, &data_type), + Ok(array) => vortex_array_to_record_batch(&session, array, &data_type), Err(e) => Err(ArrowError::ExternalError(Box::new(e))), }), ); diff --git a/tests/api-testing/tests/vortex/test_schema_evolution.py b/tests/api-testing/tests/vortex/test_schema_evolution.py index d038adf97b..3392b385af 100644 --- a/tests/api-testing/tests/vortex/test_schema_evolution.py +++ b/tests/api-testing/tests/vortex/test_schema_evolution.py @@ -61,9 +61,13 @@ class TestNewFieldAddition: ingest(client, s, batch2) flush_and_wait(client, s, expected=10) - for field in ("field_a", "field_b"): - n = count_records(client, s, where=f"{field} IS NOT NULL AND {field} != ''") - assert n == 10, f"{field} should appear in all 10 rows, got {n}" + # field_a is a string — IS NOT NULL AND != '' is a valid emptiness check. + n = count_records(client, s, where="field_a IS NOT NULL AND field_a != ''") + assert n == 10, f"field_a should appear in all 10 rows, got {n}" + # field_b is int — comparing to '' is a numeric/string type mismatch; use + # the actual value (1 in every row) instead. + n = count_records(client, s, where="field_b = 1") + assert n == 10, f"field_b should appear in all 10 rows, got {n}" # Use value-specific checks: IS NOT NULL AND != '' is unreliable for # numeric/boolean fields (e.g. field_d=0, field_e=false may look empty).