From 9a57e1195d95b77ddfa03cfe73a360b3fb896ec3 Mon Sep 17 00:00:00 2001 From: fys <40801205+fengys1996@users.noreply.github.com> Date: Sat, 10 Oct 2026 08:31:18 +0000 Subject: [PATCH] feat(json2): json_get respect type hints (#9349) Signed-off-by: fys --- Cargo.lock | 70 ++-- Cargo.toml | 30 +- src/query/src/datafusion/json_expr_planner.rs | 236 ++++++++++++-- .../json/json2_query_respect_type_hint.result | 307 +++++++++++++++--- .../json/json2_query_respect_type_hint.sql | 106 +++++- 5 files changed, 620 insertions(+), 129 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 055bf712394..3b66b13e949 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3722,7 +3722,7 @@ checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "datafusion" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "arrow-schema 59.2.0", @@ -3774,7 +3774,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "async-trait", @@ -3798,7 +3798,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "async-trait", @@ -3821,7 +3821,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "arrow-ipc 59.2.0", @@ -3847,7 +3847,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "futures", "log", @@ -3857,7 +3857,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "async-compression", @@ -3893,7 +3893,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "arrow-ipc 59.2.0", @@ -3917,7 +3917,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "async-trait", @@ -3940,7 +3940,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "async-trait", @@ -3963,7 +3963,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "arrow-schema 59.2.0", @@ -3995,12 +3995,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" [[package]] name = "datafusion-execution" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "arrow-buffer 59.2.0", @@ -4025,7 +4025,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "arrow-schema 59.2.0", @@ -4049,7 +4049,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4060,7 +4060,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "arrow-buffer 59.2.0", @@ -4091,7 +4091,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4111,7 +4111,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4122,7 +4122,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "arrow-ord 59.2.0", @@ -4146,7 +4146,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "async-trait", @@ -4161,7 +4161,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4177,7 +4177,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -4186,7 +4186,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "datafusion-doc", "quote", @@ -4196,7 +4196,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "chrono", @@ -4244,7 +4244,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "arrow-schema 59.2.0", @@ -4267,7 +4267,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4281,7 +4281,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "chrono", @@ -4298,7 +4298,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4317,7 +4317,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "arrow-data 59.2.0", @@ -4354,7 +4354,7 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "datafusion-catalog", @@ -4380,7 +4380,7 @@ dependencies = [ [[package]] name = "datafusion-proto-common" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4390,7 +4390,7 @@ dependencies = [ [[package]] name = "datafusion-proto-models" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "datafusion-common", "datafusion-proto-common", @@ -4400,7 +4400,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "datafusion-common", @@ -4415,7 +4415,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow-schema 59.2.0", "async-trait", @@ -4429,7 +4429,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "arrow 59.2.0", "bigdecimal 0.4.8", @@ -4448,7 +4448,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "55.1.0" -source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=4bd567a884dcbfd89d0f0a7859ef8367e38fff97#4bd567a884dcbfd89d0f0a7859ef8367e38fff97" +source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=e3221b8f74b15c7192621576e5d1e354a6cf097a#e3221b8f74b15c7192621576e5d1e354a6cf097a" dependencies = [ "async-recursion", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index cce9b502928..549412f369a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -363,21 +363,21 @@ rev = "f743c3a5fe8c1f57363eac5866c356b7d91f9b39" [patch.crates-io] substrait = { git = "https://github.com/GreptimeTeam/substrait-rs.git", rev = "91ec978b0649417ad3da8390e7a515baec723b1b" } -datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } -datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "4bd567a884dcbfd89d0f0a7859ef8367e38fff97" } +datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } +datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "e3221b8f74b15c7192621576e5d1e354a6cf097a" } datafusion-pg-catalog = { git = "https://github.com/datafusion-contrib/datafusion-postgres.git", rev = "ffb14a52b3d7c7489559812e1ec38096f225ce93" } arrow-pg = { git = "https://github.com/datafusion-contrib/datafusion-postgres.git", rev = "ffb14a52b3d7c7489559812e1ec38096f225ce93" } sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "9e9019bb1c7040ed956f654e39378dd42ab17884" } diff --git a/src/query/src/datafusion/json_expr_planner.rs b/src/query/src/datafusion/json_expr_planner.rs index 4ec1b729007..4837403c73f 100644 --- a/src/query/src/datafusion/json_expr_planner.rs +++ b/src/query/src/datafusion/json_expr_planner.rs @@ -16,11 +16,12 @@ use std::sync::{Arc, LazyLock}; use arrow_schema::Field; use arrow_schema::extension::ExtensionType; -use common_function::scalars::json::json_get::JsonGetWithType; +use common_function::scalars::json::json_get::{JsonGetWithType, parse_json_get_path}; use common_function::scalars::udf::create_udf; use datafusion_common::arrow::datatypes::DataType; use datafusion_common::{ - Column, DFSchema, DataFusionError, Result, ScalarValue, TableReference, plan_datafusion_err, + Column, DFSchema, DataFusionError, ExprSchema, Result, ScalarValue, TableReference, + plan_datafusion_err, }; use datafusion_expr::expr::{BinaryExpr, ScalarFunction}; use datafusion_expr::planner::{ @@ -35,6 +36,7 @@ use datatypes::extension::json::{ Json2ExtensionType, is_json2_extension_type, parse_legacy_json2_settings, }; use datatypes::types::json_type::JsonNativeType; +use jsonb::jsonpath::Path; use sqlparser::ast::BinaryOperator; /// Rewrites JSON-aware SQL expressions into DataFusion expressions. @@ -44,6 +46,8 @@ use sqlparser::ast::BinaryOperator; /// For example, `select a.b.c` => `select json_get(a, '$.b.c')`. /// - Extends a JSON path with list indexes and fields following an index. /// For example, `select a.b[0].c` => `select json_get(a, '$.b[0].c')`. +/// - Applies JSON2 path type hints to both compound identifiers and explicit `json_get` calls +/// before inferring types from their enclosing expressions. /// - Pushes an "expected type" argument into the `json_get` function when it participates in a /// binary operator. So that `json_get` knows the wanted data type when dealing with variant /// JSON values. @@ -152,12 +156,7 @@ impl ExprPlanner for JsonExprPlanner { Expr::Column(Column::from((qualifier, field))), Expr::Literal(ScalarValue::Utf8(Some(path)), None), ]; - if let Some(json_type) = json_type_hint(field, nested_names)? { - args.push(Expr::Literal( - ScalarValue::try_new_null(&json_type.as_arrow_type())?, - None, - )); - } + apply_json_type_hint(field, nested_names, &mut args)?; Ok(PlannerResult::Planned(Expr::ScalarFunction( ScalarFunction::new_udf(json_get, args), @@ -168,7 +167,27 @@ impl ExprPlanner for JsonExprPlanner { /// /// `Original` carries the possibly modified raw expression to subsequent planners and then /// DataFusion's default function construction. Returning `Planned` would short-circuit both. - fn plan_scalar(&self, mut expr: RawScalarExpr) -> Result> { + fn plan_scalar( + &self, + mut expr: RawScalarExpr, + schema: &DFSchema, + ) -> Result> { + if expr.func.name().eq_ignore_ascii_case(JsonGetWithType::NAME) && expr.args.len() == 2 { + let field = match &expr.args[0] { + // Resolving an unqualified column may require JOIN USING information unavailable + // in this schema. Skip hint injection if lookup fails and let DataFusion resolve + // the column or report the error later. + Expr::Column(column) => schema.field_from_column(column).ok().cloned(), + // Outer references carry their field metadata from the outer query's schema. + Expr::OuterReferenceColumn(field, _) => Some(Arc::clone(field)), + _ => None, + }; + if let Some(field) = field + && let Some(path) = json_get_object_path(&expr.args[1]) + { + apply_json_type_hint(&field, &path, &mut expr.args)?; + } + } push_function_arg_types(expr.func.as_ref(), &mut expr.args)?; Ok(PlannerResult::Original(expr)) } @@ -196,6 +215,46 @@ impl ExprPlanner for JsonExprPlanner { } } +/// Sets the native read type for an exact hinted object path. Both SQL syntaxes use this before +/// an enclosing expression can infer a type or push down a cast. +fn apply_json_type_hint(field: &Field, path: &[String], args: &mut Vec) -> Result<()> { + if !is_json2_extension_type(field) || args.len() != 2 { + return Ok(()); + } + if let Some(json_type) = json_type_hint(field, path)? { + args.push(Expr::Literal( + ScalarValue::try_new_null(&json_type.as_arrow_type())?, + None, + )); + } + Ok(()) +} + +/// Resolves `json_get`'s second argument expression (the path) to object field names for type hint +/// lookup. For example, the string literal `'$.user.age'` resolves to `["user", "age"]`. +/// +/// Returns `None` for non-string literals, non-literal expressions, invalid JSONPaths, or paths +/// containing array indexes, wildcards, or filters. +/// Returning `None` only skips hint injection; the original `json_get` expression is preserved. +fn json_get_object_path(expr: &Expr) -> Option> { + let path = expr.as_literal()?.try_as_str().flatten()?; + // Invalid paths skip hint lookup here. Later planning still validates the path and reports + // the error, preserving the existing JSONPath error behavior. + let json_path = parse_json_get_path(path).ok()?; + let mut names = Vec::new(); + for segment in json_path.paths { + match segment { + Path::Root => {} + Path::DotField(name) | Path::ColonField(name) | Path::ObjectField(name) => { + names.push(name.into_owned()); + } + // Array indexes, wildcards and filters do not identify a hinted object path. + _ => return None, + } + } + Some(names) +} + /// Returns the configured native type for an exact JSON2 object path. fn json_type_hint(field: &Field, path: &[String]) -> Result> { let settings = if field.extension_type_name() == Some(Json2ExtensionType::NAME) { @@ -705,15 +764,138 @@ mod tests { Ok(()) } + #[test] + fn test_plan_json_get_applies_type_hint() -> Result<()> { + let settings = JsonSettings::try_new( + vec![ + JsonTypeHint { + path: vec!["payload".to_string(), "cpu".to_string()], + data_type: ConcreteDataType::int64_datatype(), + inverted_index: false, + }, + JsonTypeHint { + path: vec!["payload.cpu".to_string()], + data_type: ConcreteDataType::string_datatype(), + inverted_index: false, + }, + ], + None, + ) + .unwrap(); + let field = Field::new("j", DataType::Struct(Fields::empty()), true).with_extension_type( + Json2ExtensionType::new(Arc::new(JsonMetadata::new(settings))), + ); + let relation = TableReference::bare("events"); + let schema = DFSchema::new_with_metadata( + vec![ + (Some(relation.clone()), Arc::new(field.clone())), + ( + Some(TableReference::bare("other")), + Arc::new(Field::new("j", DataType::Binary, true)), + ), + ], + Default::default(), + )?; + let base = Expr::Column(Column::new(Some(relation.clone()), "j")); + + for (path, names, expected) in [ + ("payload.cpu", vec!["payload", "cpu"], Some(DataType::Int64)), + ( + "$.payload.cpu", + vec!["payload", "cpu"], + Some(DataType::Int64), + ), + ( + r#"$."payload.cpu""#, + vec!["payload.cpu"], + Some(DataType::Utf8View), + ), + ("missing", vec!["missing"], None), + ] { + let Expr::ScalarFunction(function) = json_get_expr(base.clone(), path) else { + unreachable!(); + }; + let PlannerResult::Original(scalar) = JsonExprPlanner.plan_scalar( + RawScalarExpr { + func: function.func, + args: function.args, + }, + &schema, + )? + else { + unreachable!(); + }; + let direct = Expr::ScalarFunction(ScalarFunction::new_udf(scalar.func, scalar.args)); + let names = names.into_iter().map(String::from).collect::>(); + let PlannerResult::Planned(dotted) = + JsonExprPlanner.plan_compound_identifier(&field, Some(&relation), &names)? + else { + unreachable!(); + }; + assert_eq!(expected, extract_json_get_type(&direct), "{path}"); + assert_eq!( + extract_json_get_type(&dotted), + extract_json_get_type(&direct) + ); + + // Outer function inference must not replace a declared hint. + if expected.is_some() { + let PlannerResult::Original(outer) = JsonExprPlanner.plan_scalar( + RawScalarExpr { + func: coalesce(), + args: vec![ + direct.clone(), + Expr::Literal(ScalarValue::Float64(Some(1.0)), None), + ], + }, + &schema, + )? + else { + unreachable!(); + }; + assert_eq!(direct, outer.args[0]); + } + } + + // Non-object paths cannot match an object type hint. A different table's unhinted + // column must not inherit the same-named JSON2 column's hints either. + for (base, path) in [ + (base.clone(), "$"), + (base.clone(), "$.payload.*"), + (base.clone(), "$.payload.cpu[0]"), + (base.clone(), "$.payload["), + (Expr::Column(Column::new(Some("other"), "j")), "payload.cpu"), + ] { + let Expr::ScalarFunction(function) = json_get_expr(base, path) else { + unreachable!(); + }; + let PlannerResult::Original(scalar) = JsonExprPlanner.plan_scalar( + RawScalarExpr { + func: function.func, + args: function.args, + }, + &schema, + )? + else { + unreachable!(); + }; + assert_eq!(2, scalar.args.len(), "{path}"); + } + Ok(()) + } + #[test] fn test_plan_functions() -> Result<()> { let planner = JsonExprPlanner; let json_get = || json_get_expr(Expr::Column(Column::new_unqualified("j")), "a.b"); - let PlannerResult::Original(scalar) = planner.plan_scalar(RawScalarExpr { - func: abs(), - args: vec![json_get()], - })? + let PlannerResult::Original(scalar) = planner.plan_scalar( + RawScalarExpr { + func: abs(), + args: vec![json_get()], + }, + &DFSchema::empty(), + )? else { unreachable!(); }; @@ -722,13 +904,16 @@ mod tests { extract_json_get_type(&scalar.args[0]) ); - let PlannerResult::Original(scalar) = planner.plan_scalar(RawScalarExpr { - func: power(), - args: vec![ - json_get(), - Expr::Column(Column::new_unqualified("exponent")), - ], - })? + let PlannerResult::Original(scalar) = planner.plan_scalar( + RawScalarExpr { + func: power(), + args: vec![ + json_get(), + Expr::Column(Column::new_unqualified("exponent")), + ], + }, + &DFSchema::empty(), + )? else { unreachable!(); }; @@ -793,10 +978,13 @@ mod tests { let mut typed = json_get(); push_json_get_type_arg(&mut typed, &DataType::Float64)?; - let PlannerResult::Original(scalar) = planner.plan_scalar(RawScalarExpr { - func: coalesce(), - args: vec![json_get(), typed], - })? + let PlannerResult::Original(scalar) = planner.plan_scalar( + RawScalarExpr { + func: coalesce(), + args: vec![json_get(), typed], + }, + &DFSchema::empty(), + )? else { unreachable!(); }; diff --git a/tests/cases/standalone/common/types/json/json2_query_respect_type_hint.result b/tests/cases/standalone/common/types/json/json2_query_respect_type_hint.result index dcbffbf6058..d022976cb22 100644 --- a/tests/cases/standalone/common/types/json/json2_query_respect_type_hint.result +++ b/tests/cases/standalone/common/types/json/json2_query_respect_type_hint.result @@ -13,9 +13,43 @@ Affected Rows: 0 INSERT INTO json2_query_respect_type_hint VALUES (1, '{"a":9007199254740993,"b":"three"}', 2), - (2, '{"a":7,"b":"7"}', 2); + (2, '{"a":7,"b":"7"}', 2), + (3, '{}', 2), + (4, '{"a":null,"b":null}', 2); -Affected Rows: 2 +Affected Rows: 4 + +-- Both syntaxes use the hint when reading the memtable, including missing/null paths. +SELECT j.a AS dotted, json_get(j, 'a') AS direct, + arrow_typeof(j.a) AS dotted_type, arrow_typeof(json_get(j, 'a')) AS direct_type +FROM json2_query_respect_type_hint +ORDER BY ts; + ++------------------+------------------+-------------+-------------+ +| dotted | direct | dotted_type | direct_type | ++------------------+------------------+-------------+-------------+ +| 9007199254740993 | 9007199254740993 | Int64 | Int64 | +| 7 | 7 | Int64 | Int64 | +| | | Int64 | Int64 | +| | | Int64 | Int64 | ++------------------+------------------+-------------+-------------+ + +-- An explicit third argument takes precedence over the BIGINT path hint in memtable reads. +SELECT json_get(j, 'a') AS hinted_bigint, + json_get(j, 'a', NULL::DOUBLE) AS explicit_double, + arrow_typeof(json_get(j, 'a')) AS hinted_type, + arrow_typeof(json_get(j, 'a', NULL::DOUBLE)) AS explicit_type +FROM json2_query_respect_type_hint +ORDER BY ts; + ++------------------+--------------------+-------------+---------------+ +| hinted_bigint | explicit_double | hinted_type | explicit_type | ++------------------+--------------------+-------------+---------------+ +| 9007199254740993 | 9007199254740992.0 | Int64 | Float64 | +| 7 | 7.0 | Int64 | Float64 | +| | | Int64 | Float64 | +| | | Int64 | Float64 | ++------------------+--------------------+-------------+---------------+ ADMIN FLUSH_TABLE('json2_query_respect_type_hint'); @@ -25,83 +59,172 @@ ADMIN FLUSH_TABLE('json2_query_respect_type_hint'); | 0 | +----------------------------------------------------+ +-- The explicit read type also overrides the hint in the plan and SST reads. -- SQLNESS REPLACE (peers.*) REDACTED -EXPLAIN SELECT j.a AS hinted_bigint +EXPLAIN SELECT json_get(j, 'a', NULL::DOUBLE) AS explicit_double FROM json2_query_respect_type_hint; -+---------------+--------------------------------------------------------------------------------------------------+ -| plan_type | plan | -+---------------+--------------------------------------------------------------------------------------------------+ -| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | -| | Projection: json_get(json2_query_respect_type_hint.j, Utf8("$.a"), Int64(NULL)) AS hinted_bigint | -| | TableScan: json2_query_respect_type_hint | -| | ]] | -| physical_plan | CooperativeExec | ++---------------+----------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+----------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Projection: json_get(json2_query_respect_type_hint.j, Utf8("a"), Float64(NULL)) AS explicit_double | +| | TableScan: json2_query_respect_type_hint | +| | ]] | +| physical_plan | CooperativeExec | | | MergeScanExec: REDACTED -| | | -+---------------+--------------------------------------------------------------------------------------------------+ +| | | ++---------------+----------------------------------------------------------------------------------------------------+ + +SELECT json_get(j, 'a') AS hinted_bigint, + json_get(j, 'a', NULL::DOUBLE) AS explicit_double, + arrow_typeof(json_get(j, 'a')) AS hinted_type, + arrow_typeof(json_get(j, 'a', NULL::DOUBLE)) AS explicit_type +FROM json2_query_respect_type_hint +ORDER BY ts; + ++------------------+--------------------+-------------+---------------+ +| hinted_bigint | explicit_double | hinted_type | explicit_type | ++------------------+--------------------+-------------+---------------+ +| 9007199254740993 | 9007199254740992.0 | Int64 | Float64 | +| 7 | 7.0 | Int64 | Float64 | +| | | Int64 | Float64 | +| | | Int64 | Float64 | ++------------------+--------------------+-------------+---------------+ + +-- SQLNESS REPLACE (peers.*) REDACTED +EXPLAIN SELECT j.a AS hinted_bigint, json_get(j, 'a') AS direct_bigint +FROM json2_query_respect_type_hint; + ++---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Projection: json_get(json2_query_respect_type_hint.j, Utf8("$.a"), Int64(NULL)) AS hinted_bigint, json_get(json2_query_respect_type_hint.j, Utf8("a"), Int64(NULL)) AS direct_bigint | +| | TableScan: json2_query_respect_type_hint | +| | ]] | +| physical_plan | CooperativeExec | +| | MergeScanExec: REDACTED +| | | ++---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -- Uncast paths use their JSON2 type hints. SELECT j.a AS hinted_bigint, - j.b AS hinted_string + json_get(j, 'a') AS direct_bigint, + json_get(j, '$.a') AS rooted_bigint, + j.b AS hinted_string, + json_get(j, 'b') AS direct_string FROM json2_query_respect_type_hint ORDER BY ts; -+------------------+---------------+ -| hinted_bigint | hinted_string | -+------------------+---------------+ -| 9007199254740993 | three | -| 7 | 7 | -+------------------+---------------+ ++------------------+------------------+------------------+---------------+---------------+ +| hinted_bigint | direct_bigint | rooted_bigint | hinted_string | direct_string | ++------------------+------------------+------------------+---------------+---------------+ +| 9007199254740993 | 9007199254740993 | 9007199254740993 | three | three | +| 7 | 7 | 7 | 7 | 7 | +| | | | | | +| | | | | | ++------------------+------------------+------------------+---------------+---------------+ + +-- The SST read keeps the same result types as the memtable read. +SELECT arrow_typeof(j.a) AS dotted_type, arrow_typeof(json_get(j, 'a')) AS direct_type, + arrow_typeof(j.b) AS dotted_string_type, arrow_typeof(json_get(j, 'b')) AS direct_string_type +FROM json2_query_respect_type_hint LIMIT 1; + ++-------------+-------------+--------------------+--------------------+ +| dotted_type | direct_type | dotted_string_type | direct_string_type | ++-------------+-------------+--------------------+--------------------+ +| Int64 | Int64 | Utf8View | Utf8View | ++-------------+-------------+--------------------+--------------------+ -- A JSON2 type hint takes precedence over an explicit SQL cast for the native -- read type. -- SQLNESS REPLACE (peers.*) REDACTED -EXPLAIN SELECT j.a::STRING +EXPLAIN SELECT j.a::STRING, json_get(j, 'a')::STRING FROM json2_query_respect_type_hint; -+---------------+-----------------------------------------------------------------------------------------------+ -| plan_type | plan | -+---------------+-----------------------------------------------------------------------------------------------+ -| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | -| | Projection: CAST(json_get(json2_query_respect_type_hint.j, Utf8("$.a"), Int64(NULL)) AS Utf8) | -| | TableScan: json2_query_respect_type_hint | -| | ]] | -| physical_plan | CooperativeExec | ++---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Projection: CAST(json_get(json2_query_respect_type_hint.j, Utf8("$.a"), Int64(NULL)) AS Utf8), CAST(json_get(json2_query_respect_type_hint.j, Utf8("a"), Int64(NULL)) AS Utf8) | +| | TableScan: json2_query_respect_type_hint | +| | ]] | +| physical_plan | CooperativeExec | | | MergeScanExec: REDACTED -| | | -+---------------+-----------------------------------------------------------------------------------------------+ +| | | ++---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -- An explicit CAST changes the expression result type. SELECT - j.a::STRING AS cast_string + j.a::STRING AS cast_string, + json_get(j, 'a')::STRING AS direct_cast_string, + arrow_typeof(json_get(j, 'a')::STRING) AS cast_type FROM json2_query_respect_type_hint ORDER BY ts; -+------------------+ -| cast_string | -+------------------+ -| 9007199254740993 | -| 7 | -+------------------+ ++------------------+--------------------+-----------+ +| cast_string | direct_cast_string | cast_type | ++------------------+--------------------+-----------+ +| 9007199254740993 | 9007199254740993 | Utf8 | +| 7 | 7 | Utf8 | +| | | Utf8 | +| | | Utf8 | ++------------------+--------------------+-----------+ -- A JSON2 type hint takes precedence over expression-context inference. -- SQLNESS REPLACE (peers.*) REDACTED -EXPLAIN SELECT coalesce(j.b, exponent) +EXPLAIN SELECT coalesce(j.b, exponent), coalesce(json_get(j, 'b'), exponent) FROM json2_query_respect_type_hint; -+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| plan_type | plan | -+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | -| | Projection: CASE WHEN CAST(json_get(json2_query_respect_type_hint.j, Utf8("$.b"), Utf8View(NULL)) AS Float64) IS NOT NULL THEN CAST(json_get(json2_query_respect_type_hint.j, Utf8("$.b"), Utf8View(NULL)) AS Float64) ELSE json2_query_respect_type_hint.exponent END AS coalesce(json_get(json2_query_respect_type_hint.j,Utf8("$.b"),Utf8View(NULL)),json2_query_respect_type_hint.exponent) | -| | TableScan: json2_query_respect_type_hint | -| | ]] | -| physical_plan | CooperativeExec | ++---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeScan [is_placeholder=false, remote_input=[ | +| | Projection: CASE WHEN CAST(json_get(json2_query_respect_type_hint.j, Utf8("$.b"), Utf8View(NULL)) AS Float64) IS NOT NULL THEN CAST(json_get(json2_query_respect_type_hint.j, Utf8("$.b"), Utf8View(NULL)) AS Float64) ELSE json2_query_respect_type_hint.exponent END AS coalesce(json_get(json2_query_respect_type_hint.j,Utf8("$.b"),Utf8View(NULL)),json2_query_respect_type_hint.exponent), CASE WHEN CAST(json_get(json2_query_respect_type_hint.j, Utf8("b"), Utf8View(NULL)) AS Float64) IS NOT NULL THEN CAST(json_get(json2_query_respect_type_hint.j, Utf8("b"), Utf8View(NULL)) AS Float64) ELSE json2_query_respect_type_hint.exponent END AS coalesce(json_get(json2_query_respect_type_hint.j,Utf8("b"),Utf8View(NULL)),json2_query_respect_type_hint.exponent) | +| | TableScan: json2_query_respect_type_hint | +| | ]] | +| physical_plan | CooperativeExec | | | MergeScanExec: REDACTED -| | | -+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| | | ++---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +-- Hint injection precedes scalar, aggregate, window and binary-expression inference. +SELECT abs(j.a) AS dotted_abs, abs(json_get(j, 'a')) AS direct_abs, + j.a + 1 AS dotted_plus, json_get(j, 'a') + 1 AS direct_plus +FROM json2_query_respect_type_hint ORDER BY ts; + ++------------------+------------------+------------------+------------------+ +| dotted_abs | direct_abs | dotted_plus | direct_plus | ++------------------+------------------+------------------+------------------+ +| 9007199254740993 | 9007199254740993 | 9007199254740994 | 9007199254740994 | +| 7 | 7 | 8 | 8 | +| | | | | +| | | | | ++------------------+------------------+------------------+------------------+ + +SELECT sum(j.a) AS dotted_sum, sum(json_get(j, 'a')) AS direct_sum +FROM json2_query_respect_type_hint; + ++------------------+------------------+ +| dotted_sum | direct_sum | ++------------------+------------------+ +| 9007199254741000 | 9007199254741000 | ++------------------+------------------+ + +SELECT sum(j.a) OVER (ORDER BY ts) AS dotted_sum, + sum(json_get(j, 'a')) OVER (ORDER BY ts) AS direct_sum +FROM json2_query_respect_type_hint ORDER BY ts; + ++------------------+------------------+ +| dotted_sum | direct_sum | ++------------------+------------------+ +| 9007199254740993 | 9007199254740993 | +| 9007199254741000 | 9007199254741000 | +| 9007199254741000 | 9007199254741000 | +| 9007199254741000 | 9007199254741000 | ++------------------+------------------+ -- A STRING type hint is respected, so numeric arithmetic requires an explicit cast. SELECT j.b + exponent @@ -109,6 +232,11 @@ FROM json2_query_respect_type_hint; Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Cannot coerce arithmetic expression Utf8View + Float64 to valid types +SELECT json_get(j, 'b') + exponent +FROM json2_query_respect_type_hint; + +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Cannot coerce arithmetic expression Utf8View + Float64 to valid types + -- An explicit cast changes the expression type while preserving the STRING native read type. -- The invalid string value still reports a normal SQL cast error. SELECT j.b::DOUBLE + exponent @@ -116,7 +244,88 @@ FROM json2_query_respect_type_hint; Error: 3001(EngineExecuteQuery), Cast error: Cannot cast string 'three' to value of Float64 type +SELECT json_get(j, 'b')::DOUBLE + exponent +FROM json2_query_respect_type_hint; + +Error: 3001(EngineExecuteQuery), Cast error: Cannot cast string 'three' to value of Float64 type + +-- Resolve the input field through table aliases and a renamed derived-table column. +SELECT t.j.a AS dotted, json_get(t.j, 'a') AS direct +FROM json2_query_respect_type_hint AS t ORDER BY t.ts; + ++------------------+------------------+ +| dotted | direct | ++------------------+------------------+ +| 9007199254740993 | 9007199254740993 | +| 7 | 7 | +| | | +| | | ++------------------+------------------+ + +SELECT q.payload.a AS dotted, json_get(q.payload, 'a') AS direct +FROM (SELECT ts, j AS payload FROM json2_query_respect_type_hint) AS q ORDER BY q.ts; + ++------------------+------------------+ +| dotted | direct | ++------------------+------------------+ +| 9007199254740993 | 9007199254740993 | +| 7 | 7 | +| | | +| | | ++------------------+------------------+ + DROP TABLE json2_query_respect_type_hint; Affected Rows: 0 +CREATE TABLE json2_hint_paths ( + ts TIMESTAMP TIME INDEX, + j JSON2 (nested.a BIGINT, "nested.a" BOOLEAN) +) WITH ( + 'append_mode' = 'true' +); + +Affected Rows: 0 + +INSERT INTO json2_hint_paths VALUES + (1, '{"nested":{"a":42},"nested.a":true,"unhinted":7}'); + +Affected Rows: 1 + +-- Quoted keys are literal names, not nested paths; unhinted paths retain their default type. +SELECT j.nested.a AS dotted_nested, json_get(j, '$.nested.a') AS direct_nested, + j."nested.a" AS dotted_key, json_get(j, '$."nested.a"') AS direct_key, + arrow_typeof(json_get(j, 'nested.a')) AS nested_type, + arrow_typeof(json_get(j, '$."nested.a"')) AS key_type, + arrow_typeof(j.unhinted) AS dotted_unhinted_type, + arrow_typeof(json_get(j, 'unhinted')) AS direct_unhinted_type +FROM json2_hint_paths; + ++---------------+---------------+------------+------------+-------------+----------+----------------------+----------------------+ +| dotted_nested | direct_nested | dotted_key | direct_key | nested_type | key_type | dotted_unhinted_type | direct_unhinted_type | ++---------------+---------------+------------+------------+-------------+----------+----------------------+----------------------+ +| 42 | 42 | true | true | Int64 | Boolean | Utf8View | Utf8View | ++---------------+---------------+------------+------------+-------------+----------+----------------------+----------------------+ + +DROP TABLE json2_hint_paths; + +Affected Rows: 0 + +-- Hint lookup must defer unqualified JOIN USING columns to DataFusion, even for legacy JSON. +SELECT json_get(j, 'a') AS value +FROM (SELECT parse_json('{"a":"x"}') AS j) AS l +JOIN (SELECT parse_json('{"a":"x"}') AS j) AS r USING (j); + ++-------+ +| value | ++-------+ +| x | ++-------+ + +-- Without USING, the unqualified column is genuinely ambiguous and must still fail. +SELECT json_get(j, 'a') AS value +FROM (SELECT parse_json('{"a":"x"}') AS j) AS l +JOIN (SELECT parse_json('{"a":"x"}') AS j) AS r ON l.j = r.j; + +Error: 3000(PlanQuery), Failed to plan SQL: Ambiguous reference to unqualified field j + diff --git a/tests/cases/standalone/common/types/json/json2_query_respect_type_hint.sql b/tests/cases/standalone/common/types/json/json2_query_respect_type_hint.sql index 604f7d1f31b..eb91536b40b 100644 --- a/tests/cases/standalone/common/types/json/json2_query_respect_type_hint.sql +++ b/tests/cases/standalone/common/types/json/json2_query_respect_type_hint.sql @@ -11,45 +11,139 @@ CREATE TABLE json2_query_respect_type_hint ( INSERT INTO json2_query_respect_type_hint VALUES (1, '{"a":9007199254740993,"b":"three"}', 2), - (2, '{"a":7,"b":"7"}', 2); + (2, '{"a":7,"b":"7"}', 2), + (3, '{}', 2), + (4, '{"a":null,"b":null}', 2); + +-- Both syntaxes use the hint when reading the memtable, including missing/null paths. +SELECT j.a AS dotted, json_get(j, 'a') AS direct, + arrow_typeof(j.a) AS dotted_type, arrow_typeof(json_get(j, 'a')) AS direct_type +FROM json2_query_respect_type_hint +ORDER BY ts; + +-- An explicit third argument takes precedence over the BIGINT path hint in memtable reads. +SELECT json_get(j, 'a') AS hinted_bigint, + json_get(j, 'a', NULL::DOUBLE) AS explicit_double, + arrow_typeof(json_get(j, 'a')) AS hinted_type, + arrow_typeof(json_get(j, 'a', NULL::DOUBLE)) AS explicit_type +FROM json2_query_respect_type_hint +ORDER BY ts; ADMIN FLUSH_TABLE('json2_query_respect_type_hint'); +-- The explicit read type also overrides the hint in the plan and SST reads. -- SQLNESS REPLACE (peers.*) REDACTED -EXPLAIN SELECT j.a AS hinted_bigint +EXPLAIN SELECT json_get(j, 'a', NULL::DOUBLE) AS explicit_double +FROM json2_query_respect_type_hint; + +SELECT json_get(j, 'a') AS hinted_bigint, + json_get(j, 'a', NULL::DOUBLE) AS explicit_double, + arrow_typeof(json_get(j, 'a')) AS hinted_type, + arrow_typeof(json_get(j, 'a', NULL::DOUBLE)) AS explicit_type +FROM json2_query_respect_type_hint +ORDER BY ts; + +-- SQLNESS REPLACE (peers.*) REDACTED +EXPLAIN SELECT j.a AS hinted_bigint, json_get(j, 'a') AS direct_bigint FROM json2_query_respect_type_hint; -- Uncast paths use their JSON2 type hints. SELECT j.a AS hinted_bigint, - j.b AS hinted_string + json_get(j, 'a') AS direct_bigint, + json_get(j, '$.a') AS rooted_bigint, + j.b AS hinted_string, + json_get(j, 'b') AS direct_string FROM json2_query_respect_type_hint ORDER BY ts; +-- The SST read keeps the same result types as the memtable read. +SELECT arrow_typeof(j.a) AS dotted_type, arrow_typeof(json_get(j, 'a')) AS direct_type, + arrow_typeof(j.b) AS dotted_string_type, arrow_typeof(json_get(j, 'b')) AS direct_string_type +FROM json2_query_respect_type_hint LIMIT 1; + -- A JSON2 type hint takes precedence over an explicit SQL cast for the native -- read type. -- SQLNESS REPLACE (peers.*) REDACTED -EXPLAIN SELECT j.a::STRING +EXPLAIN SELECT j.a::STRING, json_get(j, 'a')::STRING FROM json2_query_respect_type_hint; -- An explicit CAST changes the expression result type. SELECT - j.a::STRING AS cast_string + j.a::STRING AS cast_string, + json_get(j, 'a')::STRING AS direct_cast_string, + arrow_typeof(json_get(j, 'a')::STRING) AS cast_type FROM json2_query_respect_type_hint ORDER BY ts; -- A JSON2 type hint takes precedence over expression-context inference. -- SQLNESS REPLACE (peers.*) REDACTED -EXPLAIN SELECT coalesce(j.b, exponent) +EXPLAIN SELECT coalesce(j.b, exponent), coalesce(json_get(j, 'b'), exponent) FROM json2_query_respect_type_hint; +-- Hint injection precedes scalar, aggregate, window and binary-expression inference. +SELECT abs(j.a) AS dotted_abs, abs(json_get(j, 'a')) AS direct_abs, + j.a + 1 AS dotted_plus, json_get(j, 'a') + 1 AS direct_plus +FROM json2_query_respect_type_hint ORDER BY ts; + +SELECT sum(j.a) AS dotted_sum, sum(json_get(j, 'a')) AS direct_sum +FROM json2_query_respect_type_hint; + +SELECT sum(j.a) OVER (ORDER BY ts) AS dotted_sum, + sum(json_get(j, 'a')) OVER (ORDER BY ts) AS direct_sum +FROM json2_query_respect_type_hint ORDER BY ts; + -- A STRING type hint is respected, so numeric arithmetic requires an explicit cast. SELECT j.b + exponent FROM json2_query_respect_type_hint; +SELECT json_get(j, 'b') + exponent +FROM json2_query_respect_type_hint; + -- An explicit cast changes the expression type while preserving the STRING native read type. -- The invalid string value still reports a normal SQL cast error. SELECT j.b::DOUBLE + exponent FROM json2_query_respect_type_hint; +SELECT json_get(j, 'b')::DOUBLE + exponent +FROM json2_query_respect_type_hint; + +-- Resolve the input field through table aliases and a renamed derived-table column. +SELECT t.j.a AS dotted, json_get(t.j, 'a') AS direct +FROM json2_query_respect_type_hint AS t ORDER BY t.ts; + +SELECT q.payload.a AS dotted, json_get(q.payload, 'a') AS direct +FROM (SELECT ts, j AS payload FROM json2_query_respect_type_hint) AS q ORDER BY q.ts; + DROP TABLE json2_query_respect_type_hint; + +CREATE TABLE json2_hint_paths ( + ts TIMESTAMP TIME INDEX, + j JSON2 (nested.a BIGINT, "nested.a" BOOLEAN) +) WITH ( + 'append_mode' = 'true' +); + +INSERT INTO json2_hint_paths VALUES + (1, '{"nested":{"a":42},"nested.a":true,"unhinted":7}'); + +-- Quoted keys are literal names, not nested paths; unhinted paths retain their default type. +SELECT j.nested.a AS dotted_nested, json_get(j, '$.nested.a') AS direct_nested, + j."nested.a" AS dotted_key, json_get(j, '$."nested.a"') AS direct_key, + arrow_typeof(json_get(j, 'nested.a')) AS nested_type, + arrow_typeof(json_get(j, '$."nested.a"')) AS key_type, + arrow_typeof(j.unhinted) AS dotted_unhinted_type, + arrow_typeof(json_get(j, 'unhinted')) AS direct_unhinted_type +FROM json2_hint_paths; + +DROP TABLE json2_hint_paths; + +-- Hint lookup must defer unqualified JOIN USING columns to DataFusion, even for legacy JSON. +SELECT json_get(j, 'a') AS value +FROM (SELECT parse_json('{"a":"x"}') AS j) AS l +JOIN (SELECT parse_json('{"a":"x"}') AS j) AS r USING (j); + +-- Without USING, the unqualified column is genuinely ambiguous and must still fail. +SELECT json_get(j, 'a') AS value +FROM (SELECT parse_json('{"a":"x"}') AS j) AS l +JOIN (SELECT parse_json('{"a":"x"}') AS j) AS r ON l.j = r.j;