diff --git a/.github/scripts/check-version.sh b/.github/scripts/check-version.sh index 28c2812ded..1efa3bb4db 100755 --- a/.github/scripts/check-version.sh +++ b/.github/scripts/check-version.sh @@ -30,13 +30,72 @@ CLEAN_LATEST=$(echo "$LATEST_VERSION" | sed 's/^v//' | sed 's/-nightly-.*//') echo "Current version: $CLEAN_CURRENT" echo "Latest release version: $CLEAN_LATEST" -# Use sort -V to compare versions -HIGHER_VERSION=$(printf "%s\n%s" "$CLEAN_CURRENT" "$CLEAN_LATEST" | sort -V | tail -n1) +# Function to extract base version (without pre-release suffix) +get_base_version() { + echo "$1" | sed -E 's/-(alpha|beta|rc|pre).*//' +} -if [ "$HIGHER_VERSION" = "$CLEAN_CURRENT" ]; then +# Function to check if a version is pre-release +is_prerelease() { + [[ "$1" =~ -(alpha|beta|rc|pre) ]] +} + +# Compare versions properly considering pre-release +compare_versions() { + local current=$1 + local latest=$2 + + # Extract base versions + local current_base=$(get_base_version "$current") + local latest_base=$(get_base_version "$latest") + + # Compare base versions first + HIGHER_BASE=$(printf "%s\n%s" "$current_base" "$latest_base" | sort -V | tail -n1) + + if [ "$HIGHER_BASE" = "$latest_base" ] && [ "$current_base" != "$latest_base" ]; then + # Latest has higher base version + echo "current_older" + return + elif [ "$HIGHER_BASE" = "$current_base" ] && [ "$current_base" != "$latest_base" ]; then + # Current has higher base version + echo "current_newer" + return + fi + + # Base versions are equal, compare pre-release status + if [ "$current_base" = "$latest_base" ]; then + # If current is pre-release and latest is not, current is older + if is_prerelease "$current" && ! is_prerelease "$latest"; then + echo "current_older" + return + fi + + # If latest is pre-release and current is not, current is newer + if ! is_prerelease "$current" && is_prerelease "$latest"; then + echo "current_newer" + return + fi + fi + + # Both are same type or different base versions already handled, use sort -V + HIGHER_VERSION=$(printf "%s\n%s" "$current" "$latest" | sort -V | tail -n1) + if [ "$HIGHER_VERSION" = "$current" ]; then + echo "current_newer_or_equal" + else + echo "current_older" + fi +} + +RESULT=$(compare_versions "$CLEAN_CURRENT" "$CLEAN_LATEST") + +if [ "$RESULT" = "current_newer" ] || [ "$RESULT" = "current_newer_or_equal" ]; then echo "Current version ($CLEAN_CURRENT) is NEWER than or EQUAL to latest ($CLEAN_LATEST)" - echo "is-current-version-latest=true" >> $GITHUB_OUTPUT + if [ -n "$GITHUB_OUTPUT" ]; then + echo "is-current-version-latest=true" >> $GITHUB_OUTPUT + fi else echo "Current version ($CLEAN_CURRENT) is OLDER than latest ($CLEAN_LATEST)" - echo "is-current-version-latest=false" >> $GITHUB_OUTPUT + if [ -n "$GITHUB_OUTPUT" ]; then + echo "is-current-version-latest=false" >> $GITHUB_OUTPUT + fi fi diff --git a/.github/scripts/upload-artifacts-to-s3.sh b/.github/scripts/upload-artifacts-to-s3.sh index 1ddf32044b..9c92859e3b 100755 --- a/.github/scripts/upload-artifacts-to-s3.sh +++ b/.github/scripts/upload-artifacts-to-s3.sh @@ -38,6 +38,11 @@ function upload_artifacts() { curl -X PUT \ -u "$PROXY_USERNAME:$PROXY_PASSWORD" \ -F "file=@$file" \ + --max-time 3600 \ + --connect-timeout 20 \ + --retry 5 \ + --retry-delay 10 \ + --retry-max-time 3000 \ "$TARGET_URL" done } @@ -54,6 +59,11 @@ function update_version_info() { curl -X PUT \ -u "$PROXY_USERNAME:$PROXY_PASSWORD" \ -F "file=@latest-version.txt" \ + --max-time 3600 \ + --connect-timeout 20 \ + --retry 5 \ + --retry-delay 10 \ + --retry-max-time 3000 \ "$TARGET_URL" fi @@ -66,6 +76,11 @@ function update_version_info() { curl -X PUT \ -u "$PROXY_USERNAME:$PROXY_PASSWORD" \ -F "file=@latest-nightly-version.txt" \ + --max-time 3600 \ + --connect-timeout 20 \ + --retry 5 \ + --retry-delay 10 \ + --retry-max-time 3000 \ "$TARGET_URL" fi fi diff --git a/Cargo.lock b/Cargo.lock index edb8ce04d4..d66fa2d686 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -212,7 +212,7 @@ checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c" [[package]] name = "api" -version = "1.0.0" +version = "1.0.1" dependencies = [ "arrow-schema 57.3.0", "common-base", @@ -933,7 +933,7 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "auth" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "async-trait", @@ -1523,7 +1523,7 @@ dependencies = [ [[package]] name = "cache" -version = "1.0.0" +version = "1.0.1" dependencies = [ "catalog", "common-error", @@ -1559,7 +1559,7 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "catalog" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "arrow 57.3.0", @@ -1894,7 +1894,7 @@ checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "cli" -version = "1.0.0" +version = "1.0.1" dependencies = [ "async-stream", "async-trait", @@ -1951,7 +1951,7 @@ dependencies = [ [[package]] name = "client" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "arc-swap", @@ -1983,7 +1983,7 @@ dependencies = [ "serde_json", "snafu 0.8.6", "store-api", - "substrait 1.0.0", + "substrait 1.0.1", "tokio", "tokio-stream", "tonic 0.14.2", @@ -2023,7 +2023,7 @@ dependencies = [ [[package]] name = "cmd" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "async-trait", @@ -2155,7 +2155,7 @@ dependencies = [ [[package]] name = "common-base" -version = "1.0.0" +version = "1.0.1" dependencies = [ "ahash 0.8.12", "anymap2", @@ -2175,14 +2175,14 @@ dependencies = [ [[package]] name = "common-catalog" -version = "1.0.0" +version = "1.0.1" dependencies = [ "const_format", ] [[package]] name = "common-config" -version = "1.0.0" +version = "1.0.1" dependencies = [ "common-base", "common-error", @@ -2206,7 +2206,7 @@ dependencies = [ [[package]] name = "common-datasource" -version = "1.0.0" +version = "1.0.1" dependencies = [ "arrow 57.3.0", "arrow-schema 57.3.0", @@ -2242,7 +2242,7 @@ dependencies = [ [[package]] name = "common-decimal" -version = "1.0.0" +version = "1.0.1" dependencies = [ "bigdecimal 0.4.8", "common-error", @@ -2255,7 +2255,7 @@ dependencies = [ [[package]] name = "common-error" -version = "1.0.0" +version = "1.0.1" dependencies = [ "common-macro", "http 1.3.1", @@ -2266,7 +2266,7 @@ dependencies = [ [[package]] name = "common-event-recorder" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "async-trait", @@ -2289,7 +2289,7 @@ dependencies = [ [[package]] name = "common-frontend" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "async-trait", @@ -2310,7 +2310,7 @@ dependencies = [ [[package]] name = "common-function" -version = "1.0.0" +version = "1.0.1" dependencies = [ "ahash 0.8.12", "api", @@ -2348,6 +2348,7 @@ dependencies = [ "geohash", "h3o", "hyperloglogplus", + "icu_properties", "jsonb", "jsonpath-rust 0.7.5", "memchr", @@ -2373,7 +2374,7 @@ dependencies = [ [[package]] name = "common-greptimedb-telemetry" -version = "1.0.0" +version = "1.0.1" dependencies = [ "async-trait", "common-runtime", @@ -2390,7 +2391,7 @@ dependencies = [ [[package]] name = "common-grpc" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "arrow-flight", @@ -2425,7 +2426,7 @@ dependencies = [ [[package]] name = "common-grpc-expr" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "common-base", @@ -2445,7 +2446,7 @@ dependencies = [ [[package]] name = "common-macro" -version = "1.0.0" +version = "1.0.1" dependencies = [ "greptime-proto", "once_cell", @@ -2456,7 +2457,7 @@ dependencies = [ [[package]] name = "common-mem-prof" -version = "1.0.0" +version = "1.0.1" dependencies = [ "anyhow", "common-error", @@ -2472,7 +2473,7 @@ dependencies = [ [[package]] name = "common-memory-manager" -version = "1.0.0" +version = "1.0.1" dependencies = [ "common-error", "common-macro", @@ -2484,7 +2485,7 @@ dependencies = [ [[package]] name = "common-meta" -version = "1.0.0" +version = "1.0.1" dependencies = [ "anymap2", "api", @@ -2555,7 +2556,7 @@ dependencies = [ [[package]] name = "common-options" -version = "1.0.0" +version = "1.0.1" dependencies = [ "common-grpc", "humantime-serde", @@ -2565,11 +2566,11 @@ dependencies = [ [[package]] name = "common-plugins" -version = "1.0.0" +version = "1.0.1" [[package]] name = "common-pprof" -version = "1.0.0" +version = "1.0.1" dependencies = [ "common-error", "common-macro", @@ -2580,7 +2581,7 @@ dependencies = [ [[package]] name = "common-procedure" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "async-stream", @@ -2609,7 +2610,7 @@ dependencies = [ [[package]] name = "common-procedure-test" -version = "1.0.0" +version = "1.0.1" dependencies = [ "async-trait", "common-procedure", @@ -2619,7 +2620,7 @@ dependencies = [ [[package]] name = "common-query" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "async-trait", @@ -2645,7 +2646,7 @@ dependencies = [ [[package]] name = "common-recordbatch" -version = "1.0.0" +version = "1.0.1" dependencies = [ "arc-swap", "common-base", @@ -2670,7 +2671,7 @@ dependencies = [ [[package]] name = "common-runtime" -version = "1.0.0" +version = "1.0.1" dependencies = [ "async-trait", "clap", @@ -2699,7 +2700,7 @@ dependencies = [ [[package]] name = "common-session" -version = "1.0.0" +version = "1.0.1" dependencies = [ "serde", "strum 0.27.1", @@ -2707,7 +2708,7 @@ dependencies = [ [[package]] name = "common-sql" -version = "1.0.0" +version = "1.0.1" dependencies = [ "arrow-schema 57.3.0", "common-base", @@ -2727,7 +2728,7 @@ dependencies = [ [[package]] name = "common-stat" -version = "1.0.0" +version = "1.0.1" dependencies = [ "common-base", "common-runtime", @@ -2742,7 +2743,7 @@ dependencies = [ [[package]] name = "common-telemetry" -version = "1.0.0" +version = "1.0.1" dependencies = [ "backtrace", "common-base", @@ -2771,7 +2772,7 @@ dependencies = [ [[package]] name = "common-test-util" -version = "1.0.0" +version = "1.0.1" dependencies = [ "client", "common-grpc", @@ -2784,7 +2785,7 @@ dependencies = [ [[package]] name = "common-time" -version = "1.0.0" +version = "1.0.1" dependencies = [ "arrow 57.3.0", "chrono", @@ -2802,7 +2803,7 @@ dependencies = [ [[package]] name = "common-version" -version = "1.0.0" +version = "1.0.1" dependencies = [ "cargo-manifest", "const_format", @@ -2812,7 +2813,7 @@ dependencies = [ [[package]] name = "common-wal" -version = "1.0.0" +version = "1.0.1" dependencies = [ "common-base", "common-error", @@ -2835,7 +2836,7 @@ dependencies = [ [[package]] name = "common-workload" -version = "1.0.0" +version = "1.0.1" dependencies = [ "common-telemetry", "serde", @@ -4197,7 +4198,7 @@ dependencies = [ [[package]] name = "datanode" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "arrow-flight", @@ -4265,7 +4266,7 @@ dependencies = [ [[package]] name = "datatypes" -version = "1.0.0" +version = "1.0.1" dependencies = [ "arrow 57.3.0", "arrow-array 57.3.0", @@ -4943,7 +4944,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "file-engine" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "async-trait", @@ -5075,7 +5076,7 @@ checksum = "8bf7cc16383c4b8d58b9905a8509f02926ce3058053c056376248d958c9df1e8" [[package]] name = "flow" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "arrow 57.3.0", @@ -5144,7 +5145,7 @@ dependencies = [ "sql", "store-api", "strum 0.27.1", - "substrait 1.0.0", + "substrait 1.0.1", "table", "tokio", "tonic 0.14.2", @@ -5205,7 +5206,7 @@ checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619" [[package]] name = "frontend" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "arc-swap", @@ -5681,7 +5682,7 @@ dependencies = [ [[package]] name = "greptime-proto" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=092ba1d01e2da676dca66cca7eebb55009da8ef8#092ba1d01e2da676dca66cca7eebb55009da8ef8" +source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=26a50f4069f50c37d65b45e0d39ae0cb42de5425#26a50f4069f50c37d65b45e0d39ae0cb42de5425" dependencies = [ "prost 0.14.1", "prost-types 0.14.1", @@ -5691,7 +5692,6 @@ dependencies = [ "strum_macros 0.25.3", "tonic 0.14.2", "tonic-prost", - "tonic-prost-build", ] [[package]] @@ -6453,7 +6453,7 @@ dependencies = [ [[package]] name = "index" -version = "1.0.0" +version = "1.0.1" dependencies = [ "async-trait", "asynchronous-codec", @@ -7421,7 +7421,7 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "log-query" -version = "1.0.0" +version = "1.0.1" dependencies = [ "chrono", "common-error", @@ -7433,7 +7433,7 @@ dependencies = [ [[package]] name = "log-store" -version = "1.0.0" +version = "1.0.1" dependencies = [ "async-stream", "async-trait", @@ -7724,7 +7724,7 @@ dependencies = [ [[package]] name = "meta-client" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "async-trait", @@ -7755,7 +7755,7 @@ dependencies = [ [[package]] name = "meta-srv" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "async-trait", @@ -7855,7 +7855,7 @@ dependencies = [ [[package]] name = "metric-engine" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "aquamarine", @@ -7956,7 +7956,7 @@ dependencies = [ [[package]] name = "mito-codec" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "bytes", @@ -7981,7 +7981,7 @@ dependencies = [ [[package]] name = "mito2" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "aquamarine", @@ -8705,7 +8705,7 @@ dependencies = [ [[package]] name = "object-store" -version = "1.0.0" +version = "1.0.1" dependencies = [ "anyhow", "bytes", @@ -8883,7 +8883,7 @@ dependencies = [ [[package]] name = "opensrv-mysql" version = "0.8.0" -source = "git+https://github.com/datafuselabs/opensrv?tag=v0.10.0#074bd8fb81da3c9e6d6a098a482f3380478b9c0b" +source = "git+https://github.com/GreptimeTeam/opensrv?rev=6c5a451544194b7bb60a8318d155d4f892b49f2c#6c5a451544194b7bb60a8318d155d4f892b49f2c" dependencies = [ "async-trait", "byteorder", @@ -9032,7 +9032,7 @@ dependencies = [ [[package]] name = "operator" -version = "1.0.0" +version = "1.0.1" dependencies = [ "ahash 0.8.12", "api", @@ -9092,7 +9092,7 @@ dependencies = [ "sql", "sqlparser", "store-api", - "substrait 1.0.0", + "substrait 1.0.1", "table", "tokio", "tokio-util", @@ -9368,7 +9368,7 @@ checksum = "e3c406c9e2aa74554e662d2c2ee11cd3e73756988800be7e6f5eddb16fed4699" [[package]] name = "partition" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "async-trait", @@ -9724,7 +9724,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pipeline" -version = "1.0.0" +version = "1.0.1" dependencies = [ "ahash 0.8.12", "api", @@ -9881,7 +9881,7 @@ dependencies = [ [[package]] name = "plugins" -version = "1.0.0" +version = "1.0.1" dependencies = [ "auth", "catalog", @@ -10199,7 +10199,7 @@ dependencies = [ [[package]] name = "promql" -version = "1.0.0" +version = "1.0.1" dependencies = [ "ahash 0.8.12", "async-trait", @@ -10551,7 +10551,7 @@ dependencies = [ [[package]] name = "puffin" -version = "1.0.0" +version = "1.0.1" dependencies = [ "async-compression", "async-trait", @@ -10613,7 +10613,7 @@ dependencies = [ [[package]] name = "query" -version = "1.0.0" +version = "1.0.1" dependencies = [ "ahash 0.8.12", "api", @@ -10680,7 +10680,7 @@ dependencies = [ "sql", "sqlparser", "store-api", - "substrait 1.0.0", + "substrait 1.0.1", "table", "tokio", "tokio-stream", @@ -11984,7 +11984,7 @@ dependencies = [ [[package]] name = "servers" -version = "1.0.0" +version = "1.0.1" dependencies = [ "ahash 0.8.12", "api", @@ -12034,6 +12034,7 @@ dependencies = [ "datafusion-pg-catalog", "datatypes", "derive_builder 0.20.2", + "either", "futures", "futures-util", "headers", @@ -12080,6 +12081,7 @@ dependencies = [ "regex", "reqwest", "rust-embed", + "rust_decimal", "rustls", "rustls-pemfile", "rustls-pki-types", @@ -12118,7 +12120,7 @@ dependencies = [ [[package]] name = "session" -version = "1.0.0" +version = "1.0.1" dependencies = [ "ahash 0.8.12", "api", @@ -12450,7 +12452,7 @@ dependencies = [ [[package]] name = "sql" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "arrow-buffer 57.3.0", @@ -12511,7 +12513,7 @@ dependencies = [ [[package]] name = "sqlness-runner" -version = "1.0.0" +version = "1.0.1" dependencies = [ "async-trait", "clap", @@ -12791,7 +12793,7 @@ dependencies = [ [[package]] name = "standalone" -version = "1.0.0" +version = "1.0.1" dependencies = [ "async-trait", "catalog", @@ -12835,7 +12837,7 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "store-api" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "aquamarine", @@ -13027,7 +13029,7 @@ dependencies = [ [[package]] name = "substrait" -version = "1.0.0" +version = "1.0.1" dependencies = [ "async-trait", "bytes", @@ -13149,7 +13151,7 @@ dependencies = [ [[package]] name = "table" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "arc-swap", @@ -13419,7 +13421,7 @@ checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" [[package]] name = "tests-fuzz" -version = "1.0.0" +version = "1.0.1" dependencies = [ "arbitrary", "async-trait", @@ -13463,7 +13465,7 @@ dependencies = [ [[package]] name = "tests-integration" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "arrow-flight", @@ -13501,6 +13503,7 @@ dependencies = [ "datanode", "datatypes", "dotenv", + "either", "flate2", "flow", "frontend", @@ -13516,7 +13519,6 @@ dependencies = [ "meta-client", "meta-srv", "mito2", - "moka", "mysql_async", "object-store", "opentelemetry-proto 0.31.0", @@ -13540,7 +13542,7 @@ dependencies = [ "sqlx", "standalone", "store-api", - "substrait 1.0.0", + "substrait 1.0.1", "table", "tempfile", "time", diff --git a/Cargo.toml b/Cargo.toml index 227608bf64..aa3d3d0c6c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,7 +75,7 @@ members = [ resolver = "2" [workspace.package] -version = "1.0.0" +version = "1.0.1" edition = "2024" license = "Apache-2.0" @@ -154,13 +154,14 @@ etcd-client = { version = "0.17", features = [ fst = "0.4.7" futures = "0.3" futures-util = "0.3" -greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "092ba1d01e2da676dca66cca7eebb55009da8ef8" } +greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "26a50f4069f50c37d65b45e0d39ae0cb42de5425" } hex = "0.4" http = "1" humantime = "2.1" humantime-serde = "1.1" hyper = "1.1" hyper-util = "0.1" +icu_properties = "2.0.1" itertools = "0.14" jsonb = { version = "0.4.4", default-features = false } lazy_static = "1.4" diff --git a/cliff.toml b/cliff.toml index 4245203e92..2b35ddab5c 100644 --- a/cliff.toml +++ b/cliff.toml @@ -12,7 +12,9 @@ footer = "" body = """ # {{ version }} +{% if timestamp -%} Release date: {{ timestamp | date(format="%B %d, %Y") }} +{% endif -%} {%- set breakings = commits | filter(attribute="breaking", value=true) -%} {%- if breakings | length > 0 %} @@ -118,7 +120,10 @@ filter_commits = false # regex for skipping tags # skip_tags = "" # regex for ignoring tags -ignore_tags = ".*-nightly-.*" +# Ignore nightly tags and build-suffixed release tags such as +# v1.0.0-rc.2-13cdfa9b5-20260325-1774407105 so their commits are merged into +# the next visible release section instead of creating extra headings. +ignore_tags = ".*-nightly-.*|^v[0-9]+\\.[0-9]+\\.[0-9]+(-(alpha|beta|rc)\\.[0-9]+)?-[0-9a-f]{7,}-[0-9]{8}-[0-9]+$" # sort the tags topologically topo_order = false # sort the commits inside sections by oldest/newest order diff --git a/src/cache/Cargo.toml b/src/cache/Cargo.toml index 3c128fe1ad..814eb9e484 100644 --- a/src/cache/Cargo.toml +++ b/src/cache/Cargo.toml @@ -9,6 +9,6 @@ catalog.workspace = true common-error.workspace = true common-macro.workspace = true common-meta.workspace = true -moka.workspace = true +moka = { workspace = true, features = ["future"] } partition.workspace = true snafu.workspace = true diff --git a/src/cli/src/common/object_store.rs b/src/cli/src/common/object_store.rs index 0b8372e509..129fb29cb7 100644 --- a/src/cli/src/common/object_store.rs +++ b/src/cli/src/common/object_store.rs @@ -220,18 +220,8 @@ impl PrefixedAzblobConnection { name: "AzBlob", required: [ (&self.azblob_container, "container"), - (&self.azblob_root, "root"), - (&self.azblob_account_name, "account name"), (&self.azblob_endpoint, "endpoint"), - ], - custom_validator: |missing: &mut Vec<&str>| { - // account_key is only required if sas_token is not provided - if self.azblob_sas_token.is_none() - && self.azblob_account_key.is_empty() - { - missing.push("account key (when sas_token is not provided)"); - } - } + ] ) } } diff --git a/src/cli/src/data/export.rs b/src/cli/src/data/export.rs index 051c07da35..148c27316e 100644 --- a/src/cli/src/data/export.rs +++ b/src/cli/src/data/export.rs @@ -1084,7 +1084,7 @@ mod tests { #[tokio::test] async fn test_export_command_build_with_azblob_empty_account_name() { - // Test Azure Blob with empty account_name + // account_name is optional for Azure Blob validation let cmd = ExportCommand::parse_from([ "export", "--addr", @@ -1092,30 +1092,19 @@ mod tests { "--azblob", "--azblob-container", "test-container", - "--azblob-root", - "test-root", "--azblob-account-name", "", // Empty account name - "--azblob-account-key", - MOCK_AZBLOB_ACCOUNT_KEY_B64, "--azblob-endpoint", "https://account.blob.core.windows.net", ]); let result = cmd.build().await; - assert!(result.is_err()); - if let Err(err) = result { - assert!( - err.to_string().contains("AzBlob account name must be set"), - "Actual error: {}", - err - ); - } + assert!(result.is_ok(), "Empty account_name should succeed"); } #[tokio::test] async fn test_export_command_build_with_azblob_missing_account_key() { - // Missing account key + // account_key is optional for Azure Blob validation let cmd = ExportCommand::parse_from([ "export", "--addr", @@ -1123,24 +1112,12 @@ mod tests { "--azblob", "--azblob-container", "test-container", - "--azblob-root", - "test-root", - "--azblob-account-name", - "test-account", "--azblob-endpoint", "https://account.blob.core.windows.net", ]); let result = cmd.build().await; - assert!(result.is_err()); - if let Err(err) = result { - assert!( - err.to_string() - .contains("AzBlob account key (when sas_token is not provided) must be set"), - "Actual error: {}", - err - ); - } + assert!(result.is_ok(), "Missing account_key should succeed"); } // ==================== Gap 3: Boundary cases ==================== @@ -1238,21 +1215,58 @@ mod tests { "--azblob", "--azblob-container", "test-container", - "--azblob-root", - "test-root", - "--azblob-account-name", - "test-account", - "--azblob-account-key", - MOCK_AZBLOB_ACCOUNT_KEY_B64, "--azblob-endpoint", "https://account.blob.core.windows.net", - // No sas_token ]); let result = cmd.build().await; assert!(result.is_ok(), "Minimal AzBlob config should succeed"); } + #[tokio::test] + async fn test_export_command_build_with_azblob_missing_endpoint() { + let cmd = ExportCommand::parse_from([ + "export", + "--addr", + "127.0.0.1:4000", + "--azblob", + "--azblob-container", + "test-container", + ]); + + let result = cmd.build().await; + assert!(result.is_err()); + if let Err(err) = result { + assert!( + err.to_string().contains("AzBlob endpoint must be set"), + "Actual error: {}", + err + ); + } + } + + #[tokio::test] + async fn test_export_command_build_with_azblob_missing_container() { + let cmd = ExportCommand::parse_from([ + "export", + "--addr", + "127.0.0.1:4000", + "--azblob", + "--azblob-endpoint", + "https://account.blob.core.windows.net", + ]); + + let result = cmd.build().await; + assert!(result.is_err()); + if let Err(err) = result { + assert!( + err.to_string().contains("AzBlob container must be set"), + "Actual error: {}", + err + ); + } + } + #[tokio::test] async fn test_export_command_build_with_local_and_s3() { // Both output-dir and S3 - S3 should take precedence @@ -1287,7 +1301,7 @@ mod tests { #[tokio::test] async fn test_export_command_build_with_azblob_only_sas_token() { - // Azure Blob with sas_token but no account_key - should succeed + // Azure Blob with sas_token but no credentials - should still succeed let cmd = ExportCommand::parse_from([ "export", "--addr", @@ -1295,15 +1309,10 @@ mod tests { "--azblob", "--azblob-container", "test-container", - "--azblob-root", - "test-root", - "--azblob-account-name", - "test-account", "--azblob-endpoint", "https://account.blob.core.windows.net", "--azblob-sas-token", "test-sas-token", - // No account_key ]); let result = cmd.build().await; @@ -1324,10 +1333,6 @@ mod tests { "--azblob", "--azblob-container", "test-container", - "--azblob-root", - "test-root", - "--azblob-account-name", - "test-account", "--azblob-account-key", "", // Empty account_key is OK if sas_token is provided "--azblob-endpoint", diff --git a/src/cmd/Cargo.toml b/src/cmd/Cargo.toml index 34619f4f1b..2a5a599a91 100644 --- a/src/cmd/Cargo.toml +++ b/src/cmd/Cargo.toml @@ -72,7 +72,7 @@ meta-client.workspace = true meta-srv.workspace = true metric-engine.workspace = true mito2.workspace = true -moka.workspace = true +moka = { workspace = true, features = ["future"] } object-store.workspace = true parquet = { workspace = true, features = ["object_store"] } plugins.workspace = true diff --git a/src/cmd/src/cli.rs b/src/cmd/src/cli.rs index 84e797c291..95c5f00b77 100644 --- a/src/cmd/src/cli.rs +++ b/src/cmd/src/cli.rs @@ -102,31 +102,79 @@ impl Command { #[cfg(test)] mod tests { + use std::net::TcpListener; + use std::ops::RangeInclusive; + use clap::Parser; use client::{Client, Database}; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_telemetry::logging::LoggingOptions; + use rand::Rng; use crate::error::Result as CmdResult; use crate::options::GlobalOptions; use crate::{App, cli, standalone}; + fn random_standalone_addrs() -> (String, String, String, String) { + let offset = choose_random_unused_port_offset(14000..=24000, 10); + + ( + format!("127.0.0.1:{}", 4000 + offset), + format!("127.0.0.1:{}", 4001 + offset), + format!("127.0.0.1:{}", 4002 + offset), + format!("127.0.0.1:{}", 4003 + offset), + ) + } + + fn choose_random_unused_port_offset( + port_range: RangeInclusive, + max_attempts: usize, + ) -> u16 { + let mut rng = rand::rng(); + + for _ in 0..max_attempts { + let http_port = rng.random_range(port_range.clone()); + let offset = http_port - 4000; + let ports = [4000 + offset, 4001 + offset, 4002 + offset, 4003 + offset]; + + let listeners = ports + .into_iter() + .map(|port| TcpListener::bind(("127.0.0.1", port))) + .collect::, _>>(); + + if listeners.is_ok() { + return offset; + } + } + + panic!("failed to find unused standalone test ports"); + } + #[tokio::test(flavor = "multi_thread")] async fn test_export_create_table_with_quoted_names() -> CmdResult<()> { let output_dir = tempfile::tempdir().unwrap(); + let (http_addr, rpc_addr, mysql_addr, postgres_addr) = random_standalone_addrs(); let standalone = standalone::Command::parse_from([ "standalone", "start", "--data-home", &*output_dir.path().to_string_lossy(), + "--http-addr", + &http_addr, + "--rpc-bind-addr", + &rpc_addr, + "--mysql-addr", + &mysql_addr, + "--postgres-addr", + &postgres_addr, ]); let standalone_opts = standalone.load_options(&GlobalOptions::default()).unwrap(); let mut instance = standalone.build(standalone_opts).await?; instance.start().await?; - let client = Client::with_urls(["127.0.0.1:4001"]); + let client = Client::with_urls([rpc_addr.as_str()]); let database = Database::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, client); database .sql(r#"CREATE DATABASE "cli.export.create_table";"#) @@ -149,7 +197,7 @@ mod tests { "data", "export", "--addr", - "127.0.0.1:4000", + &http_addr, "--output-dir", &*output_dir.path().to_string_lossy(), "--target", diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index 215bea0ec5..a95c1bad1f 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -42,6 +42,7 @@ use common_meta::region_keeper::MemoryRegionKeeper; use common_meta::region_registry::LeaderRegionRegistry; use common_meta::sequence::{Sequence, SequenceBuilder}; use common_meta::wal_provider::{WalProviderRef, build_wal_provider}; +use common_options::plugin_options::StandaloneFlag; use common_procedure::ProcedureManagerRef; use common_query::prelude::set_default_prefix; use common_telemetry::info; @@ -369,6 +370,7 @@ impl StartCommand { creator: InstanceCreator, ) -> Result<(Instance, InstanceCreatorResult)> { let mut plugins = Plugins::new(); + plugins.insert(StandaloneFlag); set_default_prefix(opts.default_column_prefix.as_deref()) .map_err(BoxedError::new) .context(error::BuildCliSnafu)?; diff --git a/src/common/function/Cargo.toml b/src/common/function/Cargo.toml index d164b9285d..43ddf9ae0c 100644 --- a/src/common/function/Cargo.toml +++ b/src/common/function/Cargo.toml @@ -47,6 +47,7 @@ geo-types = { version = "0.7", optional = true } geohash = { version = "0.13", optional = true } h3o = { version = "0.6", optional = true } hyperloglogplus = "0.4" +icu_properties.workspace = true jsonb.workspace = true jsonpath-rust = "0.7.5" memchr = "2.7" diff --git a/src/common/function/src/admin/flush_compact_region.rs b/src/common/function/src/admin/flush_compact_region.rs index 60fd19ef5a..6dcf92117a 100644 --- a/src/common/function/src/admin/flush_compact_region.rs +++ b/src/common/function/src/admin/flush_compact_region.rs @@ -128,7 +128,7 @@ mod tests { }; let result = f.invoke_async_with_args(func_args).await.unwrap_err(); assert_eq!( - "Execution error: Handler error: Missing TableMutationHandler, not expected", + "Execution error: Missing TableMutationHandler, not expected", result.to_string() ); } diff --git a/src/common/function/src/admin/flush_compact_table.rs b/src/common/function/src/admin/flush_compact_table.rs index 3298a95061..20b95e4379 100644 --- a/src/common/function/src/admin/flush_compact_table.rs +++ b/src/common/function/src/admin/flush_compact_table.rs @@ -355,7 +355,7 @@ mod tests { }; let result = f.invoke_async_with_args(func_args).await.unwrap_err(); assert_eq!( - "Execution error: Handler error: Missing TableMutationHandler, not expected", + "Execution error: Missing TableMutationHandler, not expected", result.to_string() ); } diff --git a/src/common/function/src/admin/migrate_region.rs b/src/common/function/src/admin/migrate_region.rs index 91b5540b1a..d958b3cce9 100644 --- a/src/common/function/src/admin/migrate_region.rs +++ b/src/common/function/src/admin/migrate_region.rs @@ -173,7 +173,7 @@ mod tests { }; let result = f.invoke_async_with_args(func_args).await.unwrap_err(); assert_eq!( - "Execution error: Handler error: Missing ProcedureServiceHandler, not expected", + "Execution error: Missing ProcedureServiceHandler, not expected", result.to_string() ); } diff --git a/src/common/function/src/flush_flow.rs b/src/common/function/src/flush_flow.rs index c4ea554585..35f347bcb2 100644 --- a/src/common/function/src/flush_flow.rs +++ b/src/common/function/src/flush_flow.rs @@ -149,7 +149,7 @@ mod test { let result = f.invoke_async_with_args(func_args).await.unwrap_err(); assert_eq!( - "Execution error: Handler error: Missing FlowServiceHandler, not expected", + "Execution error: Missing FlowServiceHandler, not expected", result.to_string() ); } diff --git a/src/common/function/src/scalars/ip/ipv4.rs b/src/common/function/src/scalars/ip/ipv4.rs index dfaab40d68..b03fbf0a85 100644 --- a/src/common/function/src/scalars/ip/ipv4.rs +++ b/src/common/function/src/scalars/ip/ipv4.rs @@ -20,7 +20,10 @@ use common_query::error::InvalidFuncArgsSnafu; use datafusion_common::arrow::array::{Array, AsArray, StringViewBuilder, UInt32Builder}; use datafusion_common::arrow::compute; use datafusion_common::arrow::datatypes::{DataType, UInt32Type}; -use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, Signature, TypeSignature, Volatility}; +use datafusion_expr::{ + Coercion, ColumnarValue, ScalarFunctionArgs, Signature, TypeSignature, TypeSignatureClass, + Volatility, +}; use derive_more::Display; use crate::function::{Function, extract_args}; @@ -44,7 +47,7 @@ impl Default for Ipv4NumToString { fn default() -> Self { Self { signature: Signature::new( - TypeSignature::Exact(vec![DataType::UInt32]), + TypeSignature::Coercible(vec![Coercion::new_exact(TypeSignatureClass::Integer)]), Volatility::Immutable, ), aliases: ["inet_ntoa".to_string()], @@ -70,6 +73,14 @@ impl Function for Ipv4NumToString { args: ScalarFunctionArgs, ) -> datafusion_common::Result { let [arg0] = extract_args(self.name(), &args)?; + let arg0 = compute::cast_with_options( + &arg0, + &DataType::UInt32, + &compute::CastOptions { + safe: false, + ..Default::default() + }, + )?; let uint_vec = arg0.as_primitive::(); let size = uint_vec.len(); @@ -171,7 +182,7 @@ mod tests { use std::sync::Arc; use arrow_schema::Field; - use datafusion_common::arrow::array::{StringViewArray, UInt32Array}; + use datafusion_common::arrow::array::{Int64Array, StringViewArray, UInt32Array}; use super::*; @@ -200,6 +211,51 @@ mod tests { assert_eq!(result.value(3), "255.255.255.255"); } + #[test] + fn test_ipv4_num_to_string_accepts_int64() { + let func = Ipv4NumToString::default(); + + // Test data + let values = vec![167772161i64, 3232235521i64, 0i64, 4294967295i64]; + let input = ColumnarValue::Array(Arc::new(Int64Array::from(values))); + + let args = ScalarFunctionArgs { + args: vec![input], + arg_fields: vec![], + number_rows: 4, + return_field: Arc::new(Field::new("x", DataType::Utf8View, false)), + config_options: Arc::new(Default::default()), + }; + let result = func.invoke_with_args(args).unwrap(); + let result = result.to_array(4).unwrap(); + let result = result.as_string_view(); + + assert_eq!(result.value(0), "10.0.0.1"); + assert_eq!(result.value(1), "192.168.0.1"); + assert_eq!(result.value(2), "0.0.0.0"); + assert_eq!(result.value(3), "255.255.255.255"); + } + + #[test] + fn test_ipv4_num_to_string_rejects_negative_int64() { + let func = Ipv4NumToString::default(); + + // Test data + let values = vec![-1i64]; + let input = ColumnarValue::Array(Arc::new(Int64Array::from(values))); + + let args = ScalarFunctionArgs { + args: vec![input], + arg_fields: vec![], + number_rows: 1, + return_field: Arc::new(Field::new("x", DataType::Utf8View, false)), + config_options: Arc::new(Default::default()), + }; + let result = func.invoke_with_args(args); + + assert!(result.is_err()); + } + #[test] fn test_ipv4_string_to_num() { let func = Ipv4StringToNum::default(); diff --git a/src/common/function/src/scalars/matches_term.rs b/src/common/function/src/scalars/matches_term.rs index 8dfb25cbc0..ec1b34d408 100644 --- a/src/common/function/src/scalars/matches_term.rs +++ b/src/common/function/src/scalars/matches_term.rs @@ -20,6 +20,8 @@ use datafusion_common::arrow::compute; use datafusion_common::arrow::datatypes::DataType; use datafusion_common::{DataFusionError, ScalarValue}; use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, Signature, Volatility}; +use icu_properties::props::Script; +use icu_properties::{CodePointMapData, CodePointMapDataBorrowed}; use memchr::memmem; use crate::function::Function; @@ -27,10 +29,11 @@ use crate::function_registry::FunctionRegistry; /// Exact term/phrase matching function for text columns. /// -/// This function checks if a text column contains exact term/phrase matches -/// with non-alphanumeric boundaries. Designed for: -/// - Whole-word matching (e.g. "cat" in "cat!" but not in "category") +/// This function uses script-aware matching rules: +/// - ASCII-only terms keep whole-word style boundary matching, like Whole-word matching (e.g. "cat" in "cat!" but not in "category") /// - Phrase matching (e.g. "hello world" in "note:hello world!") +/// - Terms containing Han characters match as contiguous substrings +/// - Mixed-script identifiers and numeric terms remain searchable in Chinese text /// /// # Signature /// `matches_term(text: String, term: String) -> Boolean` @@ -43,9 +46,8 @@ use crate::function_registry::FunctionRegistry; /// BooleanVector where each element indicates if the corresponding text /// contains an exact match of the term, following these rules: /// 1. Exact substring match found (case-sensitive) -/// 2. Match boundaries are either: -/// - Start/end of text -/// - Any non-alphanumeric character (including spaces, hyphens, punctuation, etc.) +/// 2. For ASCII-only terms, adjacent ASCII word characters block the match +/// 3. For Han-containing terms, contiguous substring match is sufficient /// /// # Examples /// ``` @@ -60,6 +62,9 @@ use crate::function_registry::FunctionRegistry; /// SELECT matches_term(column, 'critical error') FROM logs; /// -- Match in: "ERROR:critical error!" /// -- No match: "critical_errors" +/// -- Chinese substring examples -- +/// SELECT matches_term(column, '手机') FROM table; +/// -- Text: "登录手机号18888888888的动态key" => true /// /// -- Empty string handling -- /// SELECT matches_term(column, '') FROM table; @@ -204,9 +209,8 @@ impl Function for MatchesTermFunction { /// /// A term is considered matched when: /// 1. The exact sequence appears in the text -/// 2. It is either: -/// - At the start/end of text with adjacent non-alphanumeric character -/// - Surrounded by non-alphanumeric characters +/// 2. ASCII-only terms are not adjacent to ASCII word characters +/// 3. Han-containing terms match as contiguous substrings /// /// # Examples /// ``` @@ -215,28 +219,105 @@ impl Function for MatchesTermFunction { /// assert!(finder.find("dog,cat")); // Term preceded by comma /// assert!(!finder.find("category")); // Partial match rejected /// -/// let finder = MatchesTermFinder::new("world"); -/// assert!(finder.find("hello-world")); // Hyphen boundary +/// let finder = MatchesTermFinder::new("手机"); +/// assert!(finder.find("登录手机号18888888888的动态key")); /// ``` #[derive(Clone, Debug)] pub struct MatchesTermFinder { finder: memmem::Finder<'static>, term: String, - starts_with_non_alnum: bool, - ends_with_non_alnum: bool, + term_kind: TermKind, + starts_with_other: bool, + ends_with_other: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CharClass { + AsciiWord, + Han, + UnicodeWord, + Other, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum TermKind { + AsciiLike, + UnicodeWord, + HanContaining, +} + +fn classify_char(c: char) -> CharClass { + if c.is_ascii_alphanumeric() { + CharClass::AsciiWord + } else if is_han(c) { + CharClass::Han + } else if c.is_alphanumeric() { + CharClass::UnicodeWord + } else { + CharClass::Other + } +} + +static HAN_SCRIPT_DATA: CodePointMapDataBorrowed<'static, Script> = + CodePointMapData::