From 8d2f92c01ae762287a3cac587156519a536ae134 Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Thu, 23 Apr 2026 17:37:27 +0800 Subject: [PATCH] chore: cherry pick fixes and bum version to v1.0.1 (#8024) * fix: remap peer addresses during retries (#7933) * fix: remap peer addresses during retries Signed-off-by: WenyXu * chore: styling Signed-off-by: WenyXu * test: add tests Signed-off-by: WenyXu * chore: apply suggestions from CR Signed-off-by: WenyXu --------- Signed-off-by: WenyXu * fix: using uint64 datatype for postgres prepared statement parameters (#7942) * feat: add support for decimal parameter type, remove string replacement fallback * chore: format * fix: add support for using unsigned bigint in postgres * chore: format toml * refactor: cleanup duplicated code * fix: rescale decimal Signed-off-by: WenyXu * fix: fix current version comparison logic for pre-releases (#7946) Signed-off-by: liyang Signed-off-by: WenyXu * fix(index): intersect bitmaps before early exit in predicates applier (#7867) * fix(index): intersect bitmaps before early exit in predicates applier The loop skipped intersecting when the next bitmap was empty, which left the accumulator unchanged instead of zeroing it. Intersect first, then break when the result is empty. Signed-off-by: Weixie Cui * per gemini * style(index): format predicates applier loop * fix(index): remove unused mut in predicates applier --------- Signed-off-by: Weixie Cui Co-authored-by: discord9 <55937128+discord9@users.noreply.github.com> Co-authored-by: discord9 Signed-off-by: WenyXu * fix: randomize standalone test ports in cli export test (#7955) fix/flaky-test: ### Add Dynamic Port Selection for Standalone Tests - **`cli.rs`**: Implemented functions `random_standalone_addrs` and `choose_random_unused_port_offset` to dynamically select unused ports for standalone tests, enhancing test reliability. - Updated `test_export_create_table_with_quoted_names` to use dynamically assigned ports for HTTP, RPC, MySQL, and PostgreSQL addresses. Signed-off-by: Lei, HUANG Signed-off-by: WenyXu * chore: fix git cliff errors in latest version (#7947) * chore: fix git cliff errors in latest version - Fix errors in v2.12.0 - Do not generate logs for beta/rc tags between the compared commits Signed-off-by: evenyag * chore: preserve blank line before release date in changelog Signed-off-by: evenyag --------- Signed-off-by: evenyag Signed-off-by: WenyXu * fix: match term zh (#7952) * fix: match term zh Signed-off-by: discord9 * chore: per gemini Signed-off-by: discord9 * chore: revert accident change Signed-off-by: discord9 * feat: unicode script han Signed-off-by: discord9 --------- Signed-off-by: discord9 Signed-off-by: WenyXu * ci: set upload timeout for uploading artifacts to S3 (#7958) * ci: set upload timeout for uploading artifacts to S3 Signed-off-by: liyang * Update upload-artifacts-to-s3.sh --------- Signed-off-by: liyang Signed-off-by: WenyXu * fix: cargo check -p common-meta (#7964) fix: moka feature Signed-off-by: WenyXu * fix: always skip field pruning when using merge mode (#7957) * test: add prefilter regressions for last_row null filters Signed-off-by: evenyag * fix: skip fields in all merge mode Signed-off-by: evenyag * refactor: simplify pre-filter skip fields handling Signed-off-by: evenyag * test: update test Signed-off-by: evenyag --------- Signed-off-by: evenyag Signed-off-by: WenyXu * fix: mysql prepare correctly returns error instead of panic (#7963) feat: mysql writer support multiple statement execution Signed-off-by: luofucong Signed-off-by: WenyXu * fix: relax azblob validation requirements (#7970) Signed-off-by: WenyXu * feat(mito2): allow CompactionOutput to succeed independently (#7948) * refactor(mito2): improve compaction error handling and file removal Refactor compaction task execution to enhance error handling and robustness. - Implemented parallel execution of compaction tasks with proper error capture and logging for individual task failures. - Ensured JoinSnafu is no longer directly used in error propagation, instead handling errors within the task processing loop. - Adjusted file removal logic to correctly include expired SSTs after compaction merges. Signed-off-by: Lei, HUANG * refactor(mito2): extract SstMerger trait for testability in compaction Extract SstMerger trait and DefaultSstMerger implementation to improve the testability of DefaultCompactor. The DefaultCompactor is now generic over SstMerger, allowing mock implementations to be injected for unit testing without relying on the full object storage access layer. This refactoring separates the concerns of SST file merging from the overall compaction orchestration logic. Additionally: - Updated CompactionScheduler to use DefaultCompactor::default(). - Added unit tests for DefaultCompactor using a MockMerger. Signed-off-by: Lei, HUANG * fix(compaction): propagate join error during sst flush Correctly propagates the error when joining SST flush handles during compaction. Previously, the error was logged but not returned, leading to potential silent failures. Also reorders some imports for consistency. Signed-off-by: Lei, HUANG * perf(compaction): pre-allocate capacity for compacted_inputs Pre-allocates capacity for the compacted_inputs vector based on the estimated total size of inputs and expired SSTs. This optimization aims to reduce vector reallocations during the compaction process. Signed-off-by: Lei, HUANG * feat/allow-partial-compaction: ### Commit Message Enhance `DefaultCompactor` and `MockMerger` for Improved Flexibility - **`compactor.rs`**: - Added `Clone` trait to `DefaultSstMerger` and `MockMerger` to allow cloning. - Removed `Arc` wrapping from `DefaultCompactor`'s `merger` field for direct usage. - Updated `merge_ssts` method to require `Clone` trait for `SstMerger`. - Modified `MockMerger` to use `Arc` for `results` and `call_idx` to ensure thread safety. - Adjusted error handling to use `error::InvalidMetaSnafu` directly. Signed-off-by: Lei, HUANG --------- Signed-off-by: Lei, HUANG Signed-off-by: WenyXu * feat: propagate staging leader through lease and heartbeat (#7950) * feat(mito): expose staging leader role state * fix(region): clear staging metadata on leader exit * feat: propagate staging leader role through heartbeat and metasrv * chore: update comments Signed-off-by: WenyXu * fix(region): unify staging exit role transitions * chore: update proto Signed-off-by: WenyXu --------- Signed-off-by: WenyXu * feat: cancel local compaction for enter staging (#7885) * feat(mito2): support cancelling active local compaction Signed-off-by: WenyXu * chore: apply suggestions from CR Signed-off-by: WenyXu * test(mito2): cover compaction cancellation return paths Signed-off-by: WenyXu * chore: apply suggestions from CR Signed-off-by: WenyXu * chore: apply suggestions from CR Signed-off-by: WenyXu * chore: apply suggestions from CR Signed-off-by: WenyXu * fix: cancel remaining tasks Signed-off-by: WenyXu * chore: apply suggestions Signed-off-by: WenyXu --------- Signed-off-by: WenyXu * refactor: move group rollback ownership to parent repartition (#7967) * refactor(meta-srv): move group rollback ownership to parent repartition procedure - Parent procedure now owns partial rollback based on failed/unknown subprocedures - rollback order: group metadata first, then allocated-region cleanup - original_target_routes captured during build-plan, persisted in RepartitionPlanEntry - rollback_group_metadata_routes moved to utils as parent-owned helper - Group subprocedure no longer supports rollback (rollback_supported = false) - Removed UpdateMetadata::RollbackStaging from group state machine - Deleted redundant group rollback tests and helpers BREAKING CHANGE: group Procedure no longer handles rollback; parent procedure is responsible for crash recovery and selecting which plans to roll back. Signed-off-by: WenyXu * chore: update comments Signed-off-by: WenyXu * chore: apply suggestions from CR Signed-off-by: WenyXu --------- Signed-off-by: WenyXu * feat: use PreFilterMode::All if only one source in the partition range (#7973) * feat: use PrefilterMode::All if only one source in the partition range Signed-off-by: evenyag * fix: consider append_mode Signed-off-by: evenyag * chore: skip merge if only one source Signed-off-by: evenyag * test: fix test Signed-off-by: evenyag --------- Signed-off-by: evenyag Signed-off-by: WenyXu * fix(meta): renew operating region leases from keeper roles (#7971) * refactor(meta): store operating region roles in memory keeper Signed-off-by: WenyXu * refactor(meta): register operating region roles from region routes Signed-off-by: WenyXu * refactor(meta): require explicit operating region roles Signed-off-by: WenyXu * fix(meta): renew operating region leases from keeper roles Signed-off-by: WenyXu * test(common-meta): cover region route role helpers Signed-off-by: WenyXu * test(meta): cover operating region role propagation Signed-off-by: WenyXu * chore: apply suggestions Signed-off-by: WenyXu * chore: apply suggestions Signed-off-by: WenyXu * fix(meta): prefer metadata roles for region lease renewal Signed-off-by: WenyXu --------- Signed-off-by: WenyXu * feat: add an index page (#7975) * feat: include an index page * fix: address code review * fix: let / auth gated * refactor: rename public-apis to public-api-prefix Signed-off-by: WenyXu * fix: remove redundant error messages in admin functions (#7953) Closes #7938 Signed-off-by: yxrxy Signed-off-by: WenyXu * perf: better jieba cut (#7984) * perf: better jieba cut Signed-off-by: discord9 * fix: also filter pun mark Signed-off-by: discord9 * chore Signed-off-by: discord9 * docs: explain why Signed-off-by: discord9 --------- Signed-off-by: discord9 Signed-off-by: WenyXu * fix: allow ipv4_num_to_string to accept valid integers (#7994) * fix: allow ipv4_num_to_string to accept valid integers Signed-off-by: Johannes Sluis * test: update sqlness result file Signed-off-by: Johannes Sluis * fix: use coercible integer signature for ipv4_num_to_string Signed-off-by: Johannes Sluis --------- Signed-off-by: Johannes Sluis Signed-off-by: WenyXu * fix: update manifest state before deleting delta files (#8001) * fix: update state before deleting deltas Signed-off-by: evenyag * chore: update comment Signed-off-by: evenyag * chore: update log level Signed-off-by: evenyag --------- Signed-off-by: evenyag Signed-off-by: WenyXu * fix: upgrade mysql metadata value limit to mediumblob (#7985) * fix: upgrade mysql metadata values to mediumblob * fix: fail mysql metadata startup on upgrade check errors Signed-off-by: WenyXu --------- Signed-off-by: WenyXu * fix: zh same underscore behavior (#8002) * fix: zh same underscore behavior Signed-off-by: discord9 * fix: only add token with _ from en analyzer Signed-off-by: discord9 * test: neg sqlness case Signed-off-by: discord9 --------- Signed-off-by: discord9 Signed-off-by: WenyXu * fix: manifest recovery scans after last version if possible (#8009) * feat: suppport scan with start after Signed-off-by: evenyag * test: add start_after test Signed-off-by: evenyag * chore: adjust remove dir warning Signed-off-by: evenyag * test: test list_with_start_after Signed-off-by: evenyag * fix: update get_paths call with start_after arg in checkpoint test Signed-off-by: evenyag * feat: log scan metrics Signed-off-by: evenyag * fix: fix start_after on manifest dir Signed-off-by: evenyag --------- Signed-off-by: evenyag Signed-off-by: WenyXu * chore: add a standalone flag in plugins during startup (#7974) * chore: add a standalone flag in plugins during startup Signed-off-by: shuiyisong * chore: add derive Signed-off-by: shuiyisong --------- Signed-off-by: shuiyisong Signed-off-by: WenyXu * chore: bump version to v1.0.1 Signed-off-by: WenyXu --------- Signed-off-by: WenyXu Signed-off-by: liyang Signed-off-by: Weixie Cui Signed-off-by: Lei, HUANG Signed-off-by: evenyag Signed-off-by: discord9 Signed-off-by: luofucong Signed-off-by: yxrxy Signed-off-by: Johannes Sluis Signed-off-by: shuiyisong Co-authored-by: Ning Sun Co-authored-by: liyang Co-authored-by: cui Co-authored-by: discord9 <55937128+discord9@users.noreply.github.com> Co-authored-by: discord9 Co-authored-by: Lei, HUANG <6406592+v0y4g3r@users.noreply.github.com> Co-authored-by: Yingwen Co-authored-by: fys <40801205+fengys1996@users.noreply.github.com> Co-authored-by: LFC <990479+MichaelScofield@users.noreply.github.com> Co-authored-by: yxrxy <1532529704@qq.com> Co-authored-by: Joe Sluis <43276756+JoeS51@users.noreply.github.com> Co-authored-by: shuiyisong <113876041+shuiyisong@users.noreply.github.com> --- .github/scripts/check-version.sh | 69 +- .github/scripts/upload-artifacts-to-s3.sh | 15 + Cargo.lock | 162 ++--- Cargo.toml | 5 +- cliff.toml | 7 +- src/cache/Cargo.toml | 2 +- src/cli/src/common/object_store.rs | 12 +- src/cli/src/data/export.rs | 93 +-- src/cmd/Cargo.toml | 2 +- src/cmd/src/cli.rs | 52 +- src/cmd/src/standalone.rs | 2 + src/common/function/Cargo.toml | 1 + .../src/admin/flush_compact_region.rs | 2 +- .../function/src/admin/flush_compact_table.rs | 2 +- .../function/src/admin/migrate_region.rs | 2 +- src/common/function/src/flush_flow.rs | 2 +- src/common/function/src/scalars/ip/ipv4.rs | 62 +- .../function/src/scalars/matches_term.rs | 170 ++++- src/common/macro/src/admin_fn.rs | 6 +- src/common/meta/Cargo.toml | 2 +- src/common/meta/src/datanode.rs | 23 + src/common/meta/src/ddl/create_table.rs | 35 +- src/common/meta/src/ddl/drop_database.rs | 6 +- .../meta/src/ddl/drop_database/cursor.rs | 3 + .../meta/src/ddl/drop_database/executor.rs | 110 +++- .../meta/src/ddl/drop_database/metadata.rs | 2 + .../meta/src/ddl/drop_database/start.rs | 3 + src/common/meta/src/ddl/drop_table.rs | 34 +- src/common/meta/src/ddl/test_util.rs | 19 + src/common/meta/src/ddl/tests/create_table.rs | 30 +- src/common/meta/src/ddl/tests/drop_table.rs | 55 +- src/common/meta/src/ddl/truncate_table.rs | 19 +- src/common/meta/src/ddl_manager.rs | 40 +- src/common/meta/src/key.rs | 2 +- src/common/meta/src/key/table_route.rs | 53 +- src/common/meta/src/kv_backend/rds/mysql.rs | 278 +++++++- src/common/meta/src/region_keeper.rs | 88 ++- src/common/meta/src/rpc/router.rs | 98 +++ src/common/options/src/plugin_options.rs | 7 + src/common/procedure/src/local/runner.rs | 54 ++ src/common/procedure/src/procedure.rs | 12 + src/datanode/src/alive_keeper.rs | 138 ++++ src/datanode/src/heartbeat.rs | 6 +- src/datanode/src/region_server.rs | 1 + src/index/src/fulltext_index/tokenizer.rs | 374 ++++++++++- .../search/index_apply/predicates_apply.rs | 12 +- .../handler/collect_cluster_info_handler.rs | 2 +- .../handler/collect_leader_region_handler.rs | 2 +- .../src/handler/persist_stats_handler.rs | 5 +- .../src/handler/region_lease_handler.rs | 61 +- .../region_migration/open_candidate_region.rs | 5 +- .../upgrade_candidate_region.rs | 3 +- src/meta-srv/src/procedure/repartition.rs | 594 ++++++++++++++++-- .../procedure/repartition/allocate_region.rs | 92 ++- .../repartition/deallocate_region.rs | 1 + .../src/procedure/repartition/group.rs | 407 +----------- .../repartition/group/update_metadata.rs | 16 +- .../update_metadata/exit_staging_region.rs | 2 +- .../rollback_staging_region.rs | 301 --------- .../src/procedure/repartition/plan.rs | 9 +- .../src/procedure/repartition/utils.rs | 245 +++++++- src/meta-srv/src/procedure/tests.rs | 2 +- src/meta-srv/src/region/lease_keeper.rs | 332 ++++++++-- src/metric-engine/Cargo.toml | 2 +- src/mito2/src/cache/manifest_cache.rs | 2 +- src/mito2/src/compaction.rs | 420 ++++++++++++- src/mito2/src/compaction/compactor.rs | 509 +++++++++++++-- src/mito2/src/compaction/task.rs | 92 ++- src/mito2/src/engine.rs | 10 +- src/mito2/src/engine/alter_test.rs | 154 ++++- .../src/engine/apply_staging_manifest_test.rs | 2 +- src/mito2/src/engine/compaction_test.rs | 90 ++- src/mito2/src/engine/filter_deleted_test.rs | 101 ++- src/mito2/src/engine/prune_test.rs | 16 +- src/mito2/src/engine/set_role_state_test.rs | 266 +++++++- src/mito2/src/engine/staging_test.rs | 2 +- src/mito2/src/error.rs | 5 +- src/mito2/src/manifest/checkpointer.rs | 16 +- src/mito2/src/manifest/manager.rs | 42 +- src/mito2/src/manifest/storage.rs | 157 ++++- src/mito2/src/manifest/storage/delta.rs | 47 +- src/mito2/src/manifest/storage/staging.rs | 4 +- src/mito2/src/manifest/tests/checkpoint.rs | 104 ++- src/mito2/src/memtable/bulk/part.rs | 20 +- src/mito2/src/memtable/bulk/part_reader.rs | 14 +- .../src/memtable/bulk/row_group_reader.rs | 21 - src/mito2/src/read/pruner.rs | 62 +- src/mito2/src/read/scan_region.rs | 45 +- src/mito2/src/read/seq_scan.rs | 24 +- src/mito2/src/region.rs | 173 ++++- src/mito2/src/region/opener.rs | 5 +- src/mito2/src/request.rs | 44 +- src/mito2/src/sst/parquet/file_range.rs | 51 +- src/mito2/src/sst/parquet/reader.rs | 26 +- src/mito2/src/test_util.rs | 13 +- src/mito2/src/worker.rs | 3 + src/mito2/src/worker/handle_apply_staging.rs | 2 +- src/mito2/src/worker/handle_compaction.rs | 25 +- src/mito2/src/worker/handle_enter_staging.rs | 39 +- src/object-store/tests/object_store_test.rs | 109 ++++ src/operator/Cargo.toml | 2 +- src/operator/src/error.rs | 2 +- src/operator/src/statement/admin.rs | 2 +- src/servers/Cargo.toml | 5 +- src/servers/src/http.rs | 4 +- src/servers/src/http/authorize.rs | 4 +- src/servers/src/http/handler.rs | 22 + src/servers/src/mysql/handler.rs | 12 +- src/servers/src/mysql/writer.rs | 213 +++---- src/servers/src/postgres/handler.rs | 15 +- src/servers/src/postgres/types.rs | 191 ++++-- src/servers/tests/mod.rs | 41 +- src/servers/tests/mysql/mysql_server_test.rs | 11 + src/store-api/src/region_engine.rs | 12 +- tests-integration/Cargo.toml | 2 +- tests-integration/tests/main.rs | 1 + tests-integration/tests/mysql.rs | 124 ++++ tests-integration/tests/sql.rs | 52 ++ .../alter/prefilter_last_row_null.result | 69 ++ .../common/alter/prefilter_last_row_null.sql | 26 + .../common/flow/flow_last_non_null.result | 2 +- .../standalone/common/function/ip.result | 9 + tests/cases/standalone/common/function/ip.sql | 3 + .../common/function/matches_term.result | 196 ++++++ .../common/function/matches_term.sql | 46 ++ 125 files changed, 6323 insertions(+), 1711 deletions(-) delete mode 100644 src/meta-srv/src/procedure/repartition/group/update_metadata/rollback_staging_region.rs create mode 100644 tests-integration/tests/mysql.rs create mode 100644 tests/cases/standalone/common/alter/prefilter_last_row_null.result create mode 100644 tests/cases/standalone/common/alter/prefilter_last_row_null.sql diff --git a/.github/scripts/check-version.sh b/.github/scripts/check-version.sh index 28c2812ded..1efa3bb4db 100755 --- a/.github/scripts/check-version.sh +++ b/.github/scripts/check-version.sh @@ -30,13 +30,72 @@ CLEAN_LATEST=$(echo "$LATEST_VERSION" | sed 's/^v//' | sed 's/-nightly-.*//') echo "Current version: $CLEAN_CURRENT" echo "Latest release version: $CLEAN_LATEST" -# Use sort -V to compare versions -HIGHER_VERSION=$(printf "%s\n%s" "$CLEAN_CURRENT" "$CLEAN_LATEST" | sort -V | tail -n1) +# Function to extract base version (without pre-release suffix) +get_base_version() { + echo "$1" | sed -E 's/-(alpha|beta|rc|pre).*//' +} -if [ "$HIGHER_VERSION" = "$CLEAN_CURRENT" ]; then +# Function to check if a version is pre-release +is_prerelease() { + [[ "$1" =~ -(alpha|beta|rc|pre) ]] +} + +# Compare versions properly considering pre-release +compare_versions() { + local current=$1 + local latest=$2 + + # Extract base versions + local current_base=$(get_base_version "$current") + local latest_base=$(get_base_version "$latest") + + # Compare base versions first + HIGHER_BASE=$(printf "%s\n%s" "$current_base" "$latest_base" | sort -V | tail -n1) + + if [ "$HIGHER_BASE" = "$latest_base" ] && [ "$current_base" != "$latest_base" ]; then + # Latest has higher base version + echo "current_older" + return + elif [ "$HIGHER_BASE" = "$current_base" ] && [ "$current_base" != "$latest_base" ]; then + # Current has higher base version + echo "current_newer" + return + fi + + # Base versions are equal, compare pre-release status + if [ "$current_base" = "$latest_base" ]; then + # If current is pre-release and latest is not, current is older + if is_prerelease "$current" && ! is_prerelease "$latest"; then + echo "current_older" + return + fi + + # If latest is pre-release and current is not, current is newer + if ! is_prerelease "$current" && is_prerelease "$latest"; then + echo "current_newer" + return + fi + fi + + # Both are same type or different base versions already handled, use sort -V + HIGHER_VERSION=$(printf "%s\n%s" "$current" "$latest" | sort -V | tail -n1) + if [ "$HIGHER_VERSION" = "$current" ]; then + echo "current_newer_or_equal" + else + echo "current_older" + fi +} + +RESULT=$(compare_versions "$CLEAN_CURRENT" "$CLEAN_LATEST") + +if [ "$RESULT" = "current_newer" ] || [ "$RESULT" = "current_newer_or_equal" ]; then echo "Current version ($CLEAN_CURRENT) is NEWER than or EQUAL to latest ($CLEAN_LATEST)" - echo "is-current-version-latest=true" >> $GITHUB_OUTPUT + if [ -n "$GITHUB_OUTPUT" ]; then + echo "is-current-version-latest=true" >> $GITHUB_OUTPUT + fi else echo "Current version ($CLEAN_CURRENT) is OLDER than latest ($CLEAN_LATEST)" - echo "is-current-version-latest=false" >> $GITHUB_OUTPUT + if [ -n "$GITHUB_OUTPUT" ]; then + echo "is-current-version-latest=false" >> $GITHUB_OUTPUT + fi fi diff --git a/.github/scripts/upload-artifacts-to-s3.sh b/.github/scripts/upload-artifacts-to-s3.sh index 1ddf32044b..9c92859e3b 100755 --- a/.github/scripts/upload-artifacts-to-s3.sh +++ b/.github/scripts/upload-artifacts-to-s3.sh @@ -38,6 +38,11 @@ function upload_artifacts() { curl -X PUT \ -u "$PROXY_USERNAME:$PROXY_PASSWORD" \ -F "file=@$file" \ + --max-time 3600 \ + --connect-timeout 20 \ + --retry 5 \ + --retry-delay 10 \ + --retry-max-time 3000 \ "$TARGET_URL" done } @@ -54,6 +59,11 @@ function update_version_info() { curl -X PUT \ -u "$PROXY_USERNAME:$PROXY_PASSWORD" \ -F "file=@latest-version.txt" \ + --max-time 3600 \ + --connect-timeout 20 \ + --retry 5 \ + --retry-delay 10 \ + --retry-max-time 3000 \ "$TARGET_URL" fi @@ -66,6 +76,11 @@ function update_version_info() { curl -X PUT \ -u "$PROXY_USERNAME:$PROXY_PASSWORD" \ -F "file=@latest-nightly-version.txt" \ + --max-time 3600 \ + --connect-timeout 20 \ + --retry 5 \ + --retry-delay 10 \ + --retry-max-time 3000 \ "$TARGET_URL" fi fi diff --git a/Cargo.lock b/Cargo.lock index edb8ce04d4..d66fa2d686 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -212,7 +212,7 @@ checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c" [[package]] name = "api" -version = "1.0.0" +version = "1.0.1" dependencies = [ "arrow-schema 57.3.0", "common-base", @@ -933,7 +933,7 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "auth" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "async-trait", @@ -1523,7 +1523,7 @@ dependencies = [ [[package]] name = "cache" -version = "1.0.0" +version = "1.0.1" dependencies = [ "catalog", "common-error", @@ -1559,7 +1559,7 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "catalog" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "arrow 57.3.0", @@ -1894,7 +1894,7 @@ checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "cli" -version = "1.0.0" +version = "1.0.1" dependencies = [ "async-stream", "async-trait", @@ -1951,7 +1951,7 @@ dependencies = [ [[package]] name = "client" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "arc-swap", @@ -1983,7 +1983,7 @@ dependencies = [ "serde_json", "snafu 0.8.6", "store-api", - "substrait 1.0.0", + "substrait 1.0.1", "tokio", "tokio-stream", "tonic 0.14.2", @@ -2023,7 +2023,7 @@ dependencies = [ [[package]] name = "cmd" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "async-trait", @@ -2155,7 +2155,7 @@ dependencies = [ [[package]] name = "common-base" -version = "1.0.0" +version = "1.0.1" dependencies = [ "ahash 0.8.12", "anymap2", @@ -2175,14 +2175,14 @@ dependencies = [ [[package]] name = "common-catalog" -version = "1.0.0" +version = "1.0.1" dependencies = [ "const_format", ] [[package]] name = "common-config" -version = "1.0.0" +version = "1.0.1" dependencies = [ "common-base", "common-error", @@ -2206,7 +2206,7 @@ dependencies = [ [[package]] name = "common-datasource" -version = "1.0.0" +version = "1.0.1" dependencies = [ "arrow 57.3.0", "arrow-schema 57.3.0", @@ -2242,7 +2242,7 @@ dependencies = [ [[package]] name = "common-decimal" -version = "1.0.0" +version = "1.0.1" dependencies = [ "bigdecimal 0.4.8", "common-error", @@ -2255,7 +2255,7 @@ dependencies = [ [[package]] name = "common-error" -version = "1.0.0" +version = "1.0.1" dependencies = [ "common-macro", "http 1.3.1", @@ -2266,7 +2266,7 @@ dependencies = [ [[package]] name = "common-event-recorder" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "async-trait", @@ -2289,7 +2289,7 @@ dependencies = [ [[package]] name = "common-frontend" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "async-trait", @@ -2310,7 +2310,7 @@ dependencies = [ [[package]] name = "common-function" -version = "1.0.0" +version = "1.0.1" dependencies = [ "ahash 0.8.12", "api", @@ -2348,6 +2348,7 @@ dependencies = [ "geohash", "h3o", "hyperloglogplus", + "icu_properties", "jsonb", "jsonpath-rust 0.7.5", "memchr", @@ -2373,7 +2374,7 @@ dependencies = [ [[package]] name = "common-greptimedb-telemetry" -version = "1.0.0" +version = "1.0.1" dependencies = [ "async-trait", "common-runtime", @@ -2390,7 +2391,7 @@ dependencies = [ [[package]] name = "common-grpc" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "arrow-flight", @@ -2425,7 +2426,7 @@ dependencies = [ [[package]] name = "common-grpc-expr" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "common-base", @@ -2445,7 +2446,7 @@ dependencies = [ [[package]] name = "common-macro" -version = "1.0.0" +version = "1.0.1" dependencies = [ "greptime-proto", "once_cell", @@ -2456,7 +2457,7 @@ dependencies = [ [[package]] name = "common-mem-prof" -version = "1.0.0" +version = "1.0.1" dependencies = [ "anyhow", "common-error", @@ -2472,7 +2473,7 @@ dependencies = [ [[package]] name = "common-memory-manager" -version = "1.0.0" +version = "1.0.1" dependencies = [ "common-error", "common-macro", @@ -2484,7 +2485,7 @@ dependencies = [ [[package]] name = "common-meta" -version = "1.0.0" +version = "1.0.1" dependencies = [ "anymap2", "api", @@ -2555,7 +2556,7 @@ dependencies = [ [[package]] name = "common-options" -version = "1.0.0" +version = "1.0.1" dependencies = [ "common-grpc", "humantime-serde", @@ -2565,11 +2566,11 @@ dependencies = [ [[package]] name = "common-plugins" -version = "1.0.0" +version = "1.0.1" [[package]] name = "common-pprof" -version = "1.0.0" +version = "1.0.1" dependencies = [ "common-error", "common-macro", @@ -2580,7 +2581,7 @@ dependencies = [ [[package]] name = "common-procedure" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "async-stream", @@ -2609,7 +2610,7 @@ dependencies = [ [[package]] name = "common-procedure-test" -version = "1.0.0" +version = "1.0.1" dependencies = [ "async-trait", "common-procedure", @@ -2619,7 +2620,7 @@ dependencies = [ [[package]] name = "common-query" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "async-trait", @@ -2645,7 +2646,7 @@ dependencies = [ [[package]] name = "common-recordbatch" -version = "1.0.0" +version = "1.0.1" dependencies = [ "arc-swap", "common-base", @@ -2670,7 +2671,7 @@ dependencies = [ [[package]] name = "common-runtime" -version = "1.0.0" +version = "1.0.1" dependencies = [ "async-trait", "clap", @@ -2699,7 +2700,7 @@ dependencies = [ [[package]] name = "common-session" -version = "1.0.0" +version = "1.0.1" dependencies = [ "serde", "strum 0.27.1", @@ -2707,7 +2708,7 @@ dependencies = [ [[package]] name = "common-sql" -version = "1.0.0" +version = "1.0.1" dependencies = [ "arrow-schema 57.3.0", "common-base", @@ -2727,7 +2728,7 @@ dependencies = [ [[package]] name = "common-stat" -version = "1.0.0" +version = "1.0.1" dependencies = [ "common-base", "common-runtime", @@ -2742,7 +2743,7 @@ dependencies = [ [[package]] name = "common-telemetry" -version = "1.0.0" +version = "1.0.1" dependencies = [ "backtrace", "common-base", @@ -2771,7 +2772,7 @@ dependencies = [ [[package]] name = "common-test-util" -version = "1.0.0" +version = "1.0.1" dependencies = [ "client", "common-grpc", @@ -2784,7 +2785,7 @@ dependencies = [ [[package]] name = "common-time" -version = "1.0.0" +version = "1.0.1" dependencies = [ "arrow 57.3.0", "chrono", @@ -2802,7 +2803,7 @@ dependencies = [ [[package]] name = "common-version" -version = "1.0.0" +version = "1.0.1" dependencies = [ "cargo-manifest", "const_format", @@ -2812,7 +2813,7 @@ dependencies = [ [[package]] name = "common-wal" -version = "1.0.0" +version = "1.0.1" dependencies = [ "common-base", "common-error", @@ -2835,7 +2836,7 @@ dependencies = [ [[package]] name = "common-workload" -version = "1.0.0" +version = "1.0.1" dependencies = [ "common-telemetry", "serde", @@ -4197,7 +4198,7 @@ dependencies = [ [[package]] name = "datanode" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "arrow-flight", @@ -4265,7 +4266,7 @@ dependencies = [ [[package]] name = "datatypes" -version = "1.0.0" +version = "1.0.1" dependencies = [ "arrow 57.3.0", "arrow-array 57.3.0", @@ -4943,7 +4944,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "file-engine" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "async-trait", @@ -5075,7 +5076,7 @@ checksum = "8bf7cc16383c4b8d58b9905a8509f02926ce3058053c056376248d958c9df1e8" [[package]] name = "flow" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "arrow 57.3.0", @@ -5144,7 +5145,7 @@ dependencies = [ "sql", "store-api", "strum 0.27.1", - "substrait 1.0.0", + "substrait 1.0.1", "table", "tokio", "tonic 0.14.2", @@ -5205,7 +5206,7 @@ checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619" [[package]] name = "frontend" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "arc-swap", @@ -5681,7 +5682,7 @@ dependencies = [ [[package]] name = "greptime-proto" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=092ba1d01e2da676dca66cca7eebb55009da8ef8#092ba1d01e2da676dca66cca7eebb55009da8ef8" +source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=26a50f4069f50c37d65b45e0d39ae0cb42de5425#26a50f4069f50c37d65b45e0d39ae0cb42de5425" dependencies = [ "prost 0.14.1", "prost-types 0.14.1", @@ -5691,7 +5692,6 @@ dependencies = [ "strum_macros 0.25.3", "tonic 0.14.2", "tonic-prost", - "tonic-prost-build", ] [[package]] @@ -6453,7 +6453,7 @@ dependencies = [ [[package]] name = "index" -version = "1.0.0" +version = "1.0.1" dependencies = [ "async-trait", "asynchronous-codec", @@ -7421,7 +7421,7 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "log-query" -version = "1.0.0" +version = "1.0.1" dependencies = [ "chrono", "common-error", @@ -7433,7 +7433,7 @@ dependencies = [ [[package]] name = "log-store" -version = "1.0.0" +version = "1.0.1" dependencies = [ "async-stream", "async-trait", @@ -7724,7 +7724,7 @@ dependencies = [ [[package]] name = "meta-client" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "async-trait", @@ -7755,7 +7755,7 @@ dependencies = [ [[package]] name = "meta-srv" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "async-trait", @@ -7855,7 +7855,7 @@ dependencies = [ [[package]] name = "metric-engine" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "aquamarine", @@ -7956,7 +7956,7 @@ dependencies = [ [[package]] name = "mito-codec" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "bytes", @@ -7981,7 +7981,7 @@ dependencies = [ [[package]] name = "mito2" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "aquamarine", @@ -8705,7 +8705,7 @@ dependencies = [ [[package]] name = "object-store" -version = "1.0.0" +version = "1.0.1" dependencies = [ "anyhow", "bytes", @@ -8883,7 +8883,7 @@ dependencies = [ [[package]] name = "opensrv-mysql" version = "0.8.0" -source = "git+https://github.com/datafuselabs/opensrv?tag=v0.10.0#074bd8fb81da3c9e6d6a098a482f3380478b9c0b" +source = "git+https://github.com/GreptimeTeam/opensrv?rev=6c5a451544194b7bb60a8318d155d4f892b49f2c#6c5a451544194b7bb60a8318d155d4f892b49f2c" dependencies = [ "async-trait", "byteorder", @@ -9032,7 +9032,7 @@ dependencies = [ [[package]] name = "operator" -version = "1.0.0" +version = "1.0.1" dependencies = [ "ahash 0.8.12", "api", @@ -9092,7 +9092,7 @@ dependencies = [ "sql", "sqlparser", "store-api", - "substrait 1.0.0", + "substrait 1.0.1", "table", "tokio", "tokio-util", @@ -9368,7 +9368,7 @@ checksum = "e3c406c9e2aa74554e662d2c2ee11cd3e73756988800be7e6f5eddb16fed4699" [[package]] name = "partition" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "async-trait", @@ -9724,7 +9724,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pipeline" -version = "1.0.0" +version = "1.0.1" dependencies = [ "ahash 0.8.12", "api", @@ -9881,7 +9881,7 @@ dependencies = [ [[package]] name = "plugins" -version = "1.0.0" +version = "1.0.1" dependencies = [ "auth", "catalog", @@ -10199,7 +10199,7 @@ dependencies = [ [[package]] name = "promql" -version = "1.0.0" +version = "1.0.1" dependencies = [ "ahash 0.8.12", "async-trait", @@ -10551,7 +10551,7 @@ dependencies = [ [[package]] name = "puffin" -version = "1.0.0" +version = "1.0.1" dependencies = [ "async-compression", "async-trait", @@ -10613,7 +10613,7 @@ dependencies = [ [[package]] name = "query" -version = "1.0.0" +version = "1.0.1" dependencies = [ "ahash 0.8.12", "api", @@ -10680,7 +10680,7 @@ dependencies = [ "sql", "sqlparser", "store-api", - "substrait 1.0.0", + "substrait 1.0.1", "table", "tokio", "tokio-stream", @@ -11984,7 +11984,7 @@ dependencies = [ [[package]] name = "servers" -version = "1.0.0" +version = "1.0.1" dependencies = [ "ahash 0.8.12", "api", @@ -12034,6 +12034,7 @@ dependencies = [ "datafusion-pg-catalog", "datatypes", "derive_builder 0.20.2", + "either", "futures", "futures-util", "headers", @@ -12080,6 +12081,7 @@ dependencies = [ "regex", "reqwest", "rust-embed", + "rust_decimal", "rustls", "rustls-pemfile", "rustls-pki-types", @@ -12118,7 +12120,7 @@ dependencies = [ [[package]] name = "session" -version = "1.0.0" +version = "1.0.1" dependencies = [ "ahash 0.8.12", "api", @@ -12450,7 +12452,7 @@ dependencies = [ [[package]] name = "sql" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "arrow-buffer 57.3.0", @@ -12511,7 +12513,7 @@ dependencies = [ [[package]] name = "sqlness-runner" -version = "1.0.0" +version = "1.0.1" dependencies = [ "async-trait", "clap", @@ -12791,7 +12793,7 @@ dependencies = [ [[package]] name = "standalone" -version = "1.0.0" +version = "1.0.1" dependencies = [ "async-trait", "catalog", @@ -12835,7 +12837,7 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "store-api" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "aquamarine", @@ -13027,7 +13029,7 @@ dependencies = [ [[package]] name = "substrait" -version = "1.0.0" +version = "1.0.1" dependencies = [ "async-trait", "bytes", @@ -13149,7 +13151,7 @@ dependencies = [ [[package]] name = "table" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "arc-swap", @@ -13419,7 +13421,7 @@ checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" [[package]] name = "tests-fuzz" -version = "1.0.0" +version = "1.0.1" dependencies = [ "arbitrary", "async-trait", @@ -13463,7 +13465,7 @@ dependencies = [ [[package]] name = "tests-integration" -version = "1.0.0" +version = "1.0.1" dependencies = [ "api", "arrow-flight", @@ -13501,6 +13503,7 @@ dependencies = [ "datanode", "datatypes", "dotenv", + "either", "flate2", "flow", "frontend", @@ -13516,7 +13519,6 @@ dependencies = [ "meta-client", "meta-srv", "mito2", - "moka", "mysql_async", "object-store", "opentelemetry-proto 0.31.0", @@ -13540,7 +13542,7 @@ dependencies = [ "sqlx", "standalone", "store-api", - "substrait 1.0.0", + "substrait 1.0.1", "table", "tempfile", "time", diff --git a/Cargo.toml b/Cargo.toml index 227608bf64..aa3d3d0c6c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,7 +75,7 @@ members = [ resolver = "2" [workspace.package] -version = "1.0.0" +version = "1.0.1" edition = "2024" license = "Apache-2.0" @@ -154,13 +154,14 @@ etcd-client = { version = "0.17", features = [ fst = "0.4.7" futures = "0.3" futures-util = "0.3" -greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "092ba1d01e2da676dca66cca7eebb55009da8ef8" } +greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "26a50f4069f50c37d65b45e0d39ae0cb42de5425" } hex = "0.4" http = "1" humantime = "2.1" humantime-serde = "1.1" hyper = "1.1" hyper-util = "0.1" +icu_properties = "2.0.1" itertools = "0.14" jsonb = { version = "0.4.4", default-features = false } lazy_static = "1.4" diff --git a/cliff.toml b/cliff.toml index 4245203e92..2b35ddab5c 100644 --- a/cliff.toml +++ b/cliff.toml @@ -12,7 +12,9 @@ footer = "" body = """ # {{ version }} +{% if timestamp -%} Release date: {{ timestamp | date(format="%B %d, %Y") }} +{% endif -%} {%- set breakings = commits | filter(attribute="breaking", value=true) -%} {%- if breakings | length > 0 %} @@ -118,7 +120,10 @@ filter_commits = false # regex for skipping tags # skip_tags = "" # regex for ignoring tags -ignore_tags = ".*-nightly-.*" +# Ignore nightly tags and build-suffixed release tags such as +# v1.0.0-rc.2-13cdfa9b5-20260325-1774407105 so their commits are merged into +# the next visible release section instead of creating extra headings. +ignore_tags = ".*-nightly-.*|^v[0-9]+\\.[0-9]+\\.[0-9]+(-(alpha|beta|rc)\\.[0-9]+)?-[0-9a-f]{7,}-[0-9]{8}-[0-9]+$" # sort the tags topologically topo_order = false # sort the commits inside sections by oldest/newest order diff --git a/src/cache/Cargo.toml b/src/cache/Cargo.toml index 3c128fe1ad..814eb9e484 100644 --- a/src/cache/Cargo.toml +++ b/src/cache/Cargo.toml @@ -9,6 +9,6 @@ catalog.workspace = true common-error.workspace = true common-macro.workspace = true common-meta.workspace = true -moka.workspace = true +moka = { workspace = true, features = ["future"] } partition.workspace = true snafu.workspace = true diff --git a/src/cli/src/common/object_store.rs b/src/cli/src/common/object_store.rs index 0b8372e509..129fb29cb7 100644 --- a/src/cli/src/common/object_store.rs +++ b/src/cli/src/common/object_store.rs @@ -220,18 +220,8 @@ impl PrefixedAzblobConnection { name: "AzBlob", required: [ (&self.azblob_container, "container"), - (&self.azblob_root, "root"), - (&self.azblob_account_name, "account name"), (&self.azblob_endpoint, "endpoint"), - ], - custom_validator: |missing: &mut Vec<&str>| { - // account_key is only required if sas_token is not provided - if self.azblob_sas_token.is_none() - && self.azblob_account_key.is_empty() - { - missing.push("account key (when sas_token is not provided)"); - } - } + ] ) } } diff --git a/src/cli/src/data/export.rs b/src/cli/src/data/export.rs index 051c07da35..148c27316e 100644 --- a/src/cli/src/data/export.rs +++ b/src/cli/src/data/export.rs @@ -1084,7 +1084,7 @@ mod tests { #[tokio::test] async fn test_export_command_build_with_azblob_empty_account_name() { - // Test Azure Blob with empty account_name + // account_name is optional for Azure Blob validation let cmd = ExportCommand::parse_from([ "export", "--addr", @@ -1092,30 +1092,19 @@ mod tests { "--azblob", "--azblob-container", "test-container", - "--azblob-root", - "test-root", "--azblob-account-name", "", // Empty account name - "--azblob-account-key", - MOCK_AZBLOB_ACCOUNT_KEY_B64, "--azblob-endpoint", "https://account.blob.core.windows.net", ]); let result = cmd.build().await; - assert!(result.is_err()); - if let Err(err) = result { - assert!( - err.to_string().contains("AzBlob account name must be set"), - "Actual error: {}", - err - ); - } + assert!(result.is_ok(), "Empty account_name should succeed"); } #[tokio::test] async fn test_export_command_build_with_azblob_missing_account_key() { - // Missing account key + // account_key is optional for Azure Blob validation let cmd = ExportCommand::parse_from([ "export", "--addr", @@ -1123,24 +1112,12 @@ mod tests { "--azblob", "--azblob-container", "test-container", - "--azblob-root", - "test-root", - "--azblob-account-name", - "test-account", "--azblob-endpoint", "https://account.blob.core.windows.net", ]); let result = cmd.build().await; - assert!(result.is_err()); - if let Err(err) = result { - assert!( - err.to_string() - .contains("AzBlob account key (when sas_token is not provided) must be set"), - "Actual error: {}", - err - ); - } + assert!(result.is_ok(), "Missing account_key should succeed"); } // ==================== Gap 3: Boundary cases ==================== @@ -1238,21 +1215,58 @@ mod tests { "--azblob", "--azblob-container", "test-container", - "--azblob-root", - "test-root", - "--azblob-account-name", - "test-account", - "--azblob-account-key", - MOCK_AZBLOB_ACCOUNT_KEY_B64, "--azblob-endpoint", "https://account.blob.core.windows.net", - // No sas_token ]); let result = cmd.build().await; assert!(result.is_ok(), "Minimal AzBlob config should succeed"); } + #[tokio::test] + async fn test_export_command_build_with_azblob_missing_endpoint() { + let cmd = ExportCommand::parse_from([ + "export", + "--addr", + "127.0.0.1:4000", + "--azblob", + "--azblob-container", + "test-container", + ]); + + let result = cmd.build().await; + assert!(result.is_err()); + if let Err(err) = result { + assert!( + err.to_string().contains("AzBlob endpoint must be set"), + "Actual error: {}", + err + ); + } + } + + #[tokio::test] + async fn test_export_command_build_with_azblob_missing_container() { + let cmd = ExportCommand::parse_from([ + "export", + "--addr", + "127.0.0.1:4000", + "--azblob", + "--azblob-endpoint", + "https://account.blob.core.windows.net", + ]); + + let result = cmd.build().await; + assert!(result.is_err()); + if let Err(err) = result { + assert!( + err.to_string().contains("AzBlob container must be set"), + "Actual error: {}", + err + ); + } + } + #[tokio::test] async fn test_export_command_build_with_local_and_s3() { // Both output-dir and S3 - S3 should take precedence @@ -1287,7 +1301,7 @@ mod tests { #[tokio::test] async fn test_export_command_build_with_azblob_only_sas_token() { - // Azure Blob with sas_token but no account_key - should succeed + // Azure Blob with sas_token but no credentials - should still succeed let cmd = ExportCommand::parse_from([ "export", "--addr", @@ -1295,15 +1309,10 @@ mod tests { "--azblob", "--azblob-container", "test-container", - "--azblob-root", - "test-root", - "--azblob-account-name", - "test-account", "--azblob-endpoint", "https://account.blob.core.windows.net", "--azblob-sas-token", "test-sas-token", - // No account_key ]); let result = cmd.build().await; @@ -1324,10 +1333,6 @@ mod tests { "--azblob", "--azblob-container", "test-container", - "--azblob-root", - "test-root", - "--azblob-account-name", - "test-account", "--azblob-account-key", "", // Empty account_key is OK if sas_token is provided "--azblob-endpoint", diff --git a/src/cmd/Cargo.toml b/src/cmd/Cargo.toml index 34619f4f1b..2a5a599a91 100644 --- a/src/cmd/Cargo.toml +++ b/src/cmd/Cargo.toml @@ -72,7 +72,7 @@ meta-client.workspace = true meta-srv.workspace = true metric-engine.workspace = true mito2.workspace = true -moka.workspace = true +moka = { workspace = true, features = ["future"] } object-store.workspace = true parquet = { workspace = true, features = ["object_store"] } plugins.workspace = true diff --git a/src/cmd/src/cli.rs b/src/cmd/src/cli.rs index 84e797c291..95c5f00b77 100644 --- a/src/cmd/src/cli.rs +++ b/src/cmd/src/cli.rs @@ -102,31 +102,79 @@ impl Command { #[cfg(test)] mod tests { + use std::net::TcpListener; + use std::ops::RangeInclusive; + use clap::Parser; use client::{Client, Database}; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_telemetry::logging::LoggingOptions; + use rand::Rng; use crate::error::Result as CmdResult; use crate::options::GlobalOptions; use crate::{App, cli, standalone}; + fn random_standalone_addrs() -> (String, String, String, String) { + let offset = choose_random_unused_port_offset(14000..=24000, 10); + + ( + format!("127.0.0.1:{}", 4000 + offset), + format!("127.0.0.1:{}", 4001 + offset), + format!("127.0.0.1:{}", 4002 + offset), + format!("127.0.0.1:{}", 4003 + offset), + ) + } + + fn choose_random_unused_port_offset( + port_range: RangeInclusive, + max_attempts: usize, + ) -> u16 { + let mut rng = rand::rng(); + + for _ in 0..max_attempts { + let http_port = rng.random_range(port_range.clone()); + let offset = http_port - 4000; + let ports = [4000 + offset, 4001 + offset, 4002 + offset, 4003 + offset]; + + let listeners = ports + .into_iter() + .map(|port| TcpListener::bind(("127.0.0.1", port))) + .collect::, _>>(); + + if listeners.is_ok() { + return offset; + } + } + + panic!("failed to find unused standalone test ports"); + } + #[tokio::test(flavor = "multi_thread")] async fn test_export_create_table_with_quoted_names() -> CmdResult<()> { let output_dir = tempfile::tempdir().unwrap(); + let (http_addr, rpc_addr, mysql_addr, postgres_addr) = random_standalone_addrs(); let standalone = standalone::Command::parse_from([ "standalone", "start", "--data-home", &*output_dir.path().to_string_lossy(), + "--http-addr", + &http_addr, + "--rpc-bind-addr", + &rpc_addr, + "--mysql-addr", + &mysql_addr, + "--postgres-addr", + &postgres_addr, ]); let standalone_opts = standalone.load_options(&GlobalOptions::default()).unwrap(); let mut instance = standalone.build(standalone_opts).await?; instance.start().await?; - let client = Client::with_urls(["127.0.0.1:4001"]); + let client = Client::with_urls([rpc_addr.as_str()]); let database = Database::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, client); database .sql(r#"CREATE DATABASE "cli.export.create_table";"#) @@ -149,7 +197,7 @@ mod tests { "data", "export", "--addr", - "127.0.0.1:4000", + &http_addr, "--output-dir", &*output_dir.path().to_string_lossy(), "--target", diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index 215bea0ec5..a95c1bad1f 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -42,6 +42,7 @@ use common_meta::region_keeper::MemoryRegionKeeper; use common_meta::region_registry::LeaderRegionRegistry; use common_meta::sequence::{Sequence, SequenceBuilder}; use common_meta::wal_provider::{WalProviderRef, build_wal_provider}; +use common_options::plugin_options::StandaloneFlag; use common_procedure::ProcedureManagerRef; use common_query::prelude::set_default_prefix; use common_telemetry::info; @@ -369,6 +370,7 @@ impl StartCommand { creator: InstanceCreator, ) -> Result<(Instance, InstanceCreatorResult)> { let mut plugins = Plugins::new(); + plugins.insert(StandaloneFlag); set_default_prefix(opts.default_column_prefix.as_deref()) .map_err(BoxedError::new) .context(error::BuildCliSnafu)?; diff --git a/src/common/function/Cargo.toml b/src/common/function/Cargo.toml index d164b9285d..43ddf9ae0c 100644 --- a/src/common/function/Cargo.toml +++ b/src/common/function/Cargo.toml @@ -47,6 +47,7 @@ geo-types = { version = "0.7", optional = true } geohash = { version = "0.13", optional = true } h3o = { version = "0.6", optional = true } hyperloglogplus = "0.4" +icu_properties.workspace = true jsonb.workspace = true jsonpath-rust = "0.7.5" memchr = "2.7" diff --git a/src/common/function/src/admin/flush_compact_region.rs b/src/common/function/src/admin/flush_compact_region.rs index 60fd19ef5a..6dcf92117a 100644 --- a/src/common/function/src/admin/flush_compact_region.rs +++ b/src/common/function/src/admin/flush_compact_region.rs @@ -128,7 +128,7 @@ mod tests { }; let result = f.invoke_async_with_args(func_args).await.unwrap_err(); assert_eq!( - "Execution error: Handler error: Missing TableMutationHandler, not expected", + "Execution error: Missing TableMutationHandler, not expected", result.to_string() ); } diff --git a/src/common/function/src/admin/flush_compact_table.rs b/src/common/function/src/admin/flush_compact_table.rs index 3298a95061..20b95e4379 100644 --- a/src/common/function/src/admin/flush_compact_table.rs +++ b/src/common/function/src/admin/flush_compact_table.rs @@ -355,7 +355,7 @@ mod tests { }; let result = f.invoke_async_with_args(func_args).await.unwrap_err(); assert_eq!( - "Execution error: Handler error: Missing TableMutationHandler, not expected", + "Execution error: Missing TableMutationHandler, not expected", result.to_string() ); } diff --git a/src/common/function/src/admin/migrate_region.rs b/src/common/function/src/admin/migrate_region.rs index 91b5540b1a..d958b3cce9 100644 --- a/src/common/function/src/admin/migrate_region.rs +++ b/src/common/function/src/admin/migrate_region.rs @@ -173,7 +173,7 @@ mod tests { }; let result = f.invoke_async_with_args(func_args).await.unwrap_err(); assert_eq!( - "Execution error: Handler error: Missing ProcedureServiceHandler, not expected", + "Execution error: Missing ProcedureServiceHandler, not expected", result.to_string() ); } diff --git a/src/common/function/src/flush_flow.rs b/src/common/function/src/flush_flow.rs index c4ea554585..35f347bcb2 100644 --- a/src/common/function/src/flush_flow.rs +++ b/src/common/function/src/flush_flow.rs @@ -149,7 +149,7 @@ mod test { let result = f.invoke_async_with_args(func_args).await.unwrap_err(); assert_eq!( - "Execution error: Handler error: Missing FlowServiceHandler, not expected", + "Execution error: Missing FlowServiceHandler, not expected", result.to_string() ); } diff --git a/src/common/function/src/scalars/ip/ipv4.rs b/src/common/function/src/scalars/ip/ipv4.rs index dfaab40d68..b03fbf0a85 100644 --- a/src/common/function/src/scalars/ip/ipv4.rs +++ b/src/common/function/src/scalars/ip/ipv4.rs @@ -20,7 +20,10 @@ use common_query::error::InvalidFuncArgsSnafu; use datafusion_common::arrow::array::{Array, AsArray, StringViewBuilder, UInt32Builder}; use datafusion_common::arrow::compute; use datafusion_common::arrow::datatypes::{DataType, UInt32Type}; -use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, Signature, TypeSignature, Volatility}; +use datafusion_expr::{ + Coercion, ColumnarValue, ScalarFunctionArgs, Signature, TypeSignature, TypeSignatureClass, + Volatility, +}; use derive_more::Display; use crate::function::{Function, extract_args}; @@ -44,7 +47,7 @@ impl Default for Ipv4NumToString { fn default() -> Self { Self { signature: Signature::new( - TypeSignature::Exact(vec![DataType::UInt32]), + TypeSignature::Coercible(vec![Coercion::new_exact(TypeSignatureClass::Integer)]), Volatility::Immutable, ), aliases: ["inet_ntoa".to_string()], @@ -70,6 +73,14 @@ impl Function for Ipv4NumToString { args: ScalarFunctionArgs, ) -> datafusion_common::Result { let [arg0] = extract_args(self.name(), &args)?; + let arg0 = compute::cast_with_options( + &arg0, + &DataType::UInt32, + &compute::CastOptions { + safe: false, + ..Default::default() + }, + )?; let uint_vec = arg0.as_primitive::(); let size = uint_vec.len(); @@ -171,7 +182,7 @@ mod tests { use std::sync::Arc; use arrow_schema::Field; - use datafusion_common::arrow::array::{StringViewArray, UInt32Array}; + use datafusion_common::arrow::array::{Int64Array, StringViewArray, UInt32Array}; use super::*; @@ -200,6 +211,51 @@ mod tests { assert_eq!(result.value(3), "255.255.255.255"); } + #[test] + fn test_ipv4_num_to_string_accepts_int64() { + let func = Ipv4NumToString::default(); + + // Test data + let values = vec![167772161i64, 3232235521i64, 0i64, 4294967295i64]; + let input = ColumnarValue::Array(Arc::new(Int64Array::from(values))); + + let args = ScalarFunctionArgs { + args: vec![input], + arg_fields: vec![], + number_rows: 4, + return_field: Arc::new(Field::new("x", DataType::Utf8View, false)), + config_options: Arc::new(Default::default()), + }; + let result = func.invoke_with_args(args).unwrap(); + let result = result.to_array(4).unwrap(); + let result = result.as_string_view(); + + assert_eq!(result.value(0), "10.0.0.1"); + assert_eq!(result.value(1), "192.168.0.1"); + assert_eq!(result.value(2), "0.0.0.0"); + assert_eq!(result.value(3), "255.255.255.255"); + } + + #[test] + fn test_ipv4_num_to_string_rejects_negative_int64() { + let func = Ipv4NumToString::default(); + + // Test data + let values = vec![-1i64]; + let input = ColumnarValue::Array(Arc::new(Int64Array::from(values))); + + let args = ScalarFunctionArgs { + args: vec![input], + arg_fields: vec![], + number_rows: 1, + return_field: Arc::new(Field::new("x", DataType::Utf8View, false)), + config_options: Arc::new(Default::default()), + }; + let result = func.invoke_with_args(args); + + assert!(result.is_err()); + } + #[test] fn test_ipv4_string_to_num() { let func = Ipv4StringToNum::default(); diff --git a/src/common/function/src/scalars/matches_term.rs b/src/common/function/src/scalars/matches_term.rs index 8dfb25cbc0..ec1b34d408 100644 --- a/src/common/function/src/scalars/matches_term.rs +++ b/src/common/function/src/scalars/matches_term.rs @@ -20,6 +20,8 @@ use datafusion_common::arrow::compute; use datafusion_common::arrow::datatypes::DataType; use datafusion_common::{DataFusionError, ScalarValue}; use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, Signature, Volatility}; +use icu_properties::props::Script; +use icu_properties::{CodePointMapData, CodePointMapDataBorrowed}; use memchr::memmem; use crate::function::Function; @@ -27,10 +29,11 @@ use crate::function_registry::FunctionRegistry; /// Exact term/phrase matching function for text columns. /// -/// This function checks if a text column contains exact term/phrase matches -/// with non-alphanumeric boundaries. Designed for: -/// - Whole-word matching (e.g. "cat" in "cat!" but not in "category") +/// This function uses script-aware matching rules: +/// - ASCII-only terms keep whole-word style boundary matching, like Whole-word matching (e.g. "cat" in "cat!" but not in "category") /// - Phrase matching (e.g. "hello world" in "note:hello world!") +/// - Terms containing Han characters match as contiguous substrings +/// - Mixed-script identifiers and numeric terms remain searchable in Chinese text /// /// # Signature /// `matches_term(text: String, term: String) -> Boolean` @@ -43,9 +46,8 @@ use crate::function_registry::FunctionRegistry; /// BooleanVector where each element indicates if the corresponding text /// contains an exact match of the term, following these rules: /// 1. Exact substring match found (case-sensitive) -/// 2. Match boundaries are either: -/// - Start/end of text -/// - Any non-alphanumeric character (including spaces, hyphens, punctuation, etc.) +/// 2. For ASCII-only terms, adjacent ASCII word characters block the match +/// 3. For Han-containing terms, contiguous substring match is sufficient /// /// # Examples /// ``` @@ -60,6 +62,9 @@ use crate::function_registry::FunctionRegistry; /// SELECT matches_term(column, 'critical error') FROM logs; /// -- Match in: "ERROR:critical error!" /// -- No match: "critical_errors" +/// -- Chinese substring examples -- +/// SELECT matches_term(column, '手机') FROM table; +/// -- Text: "登录手机号18888888888的动态key" => true /// /// -- Empty string handling -- /// SELECT matches_term(column, '') FROM table; @@ -204,9 +209,8 @@ impl Function for MatchesTermFunction { /// /// A term is considered matched when: /// 1. The exact sequence appears in the text -/// 2. It is either: -/// - At the start/end of text with adjacent non-alphanumeric character -/// - Surrounded by non-alphanumeric characters +/// 2. ASCII-only terms are not adjacent to ASCII word characters +/// 3. Han-containing terms match as contiguous substrings /// /// # Examples /// ``` @@ -215,28 +219,105 @@ impl Function for MatchesTermFunction { /// assert!(finder.find("dog,cat")); // Term preceded by comma /// assert!(!finder.find("category")); // Partial match rejected /// -/// let finder = MatchesTermFinder::new("world"); -/// assert!(finder.find("hello-world")); // Hyphen boundary +/// let finder = MatchesTermFinder::new("手机"); +/// assert!(finder.find("登录手机号18888888888的动态key")); /// ``` #[derive(Clone, Debug)] pub struct MatchesTermFinder { finder: memmem::Finder<'static>, term: String, - starts_with_non_alnum: bool, - ends_with_non_alnum: bool, + term_kind: TermKind, + starts_with_other: bool, + ends_with_other: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CharClass { + AsciiWord, + Han, + UnicodeWord, + Other, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum TermKind { + AsciiLike, + UnicodeWord, + HanContaining, +} + +fn classify_char(c: char) -> CharClass { + if c.is_ascii_alphanumeric() { + CharClass::AsciiWord + } else if is_han(c) { + CharClass::Han + } else if c.is_alphanumeric() { + CharClass::UnicodeWord + } else { + CharClass::Other + } +} + +static HAN_SCRIPT_DATA: CodePointMapDataBorrowed<'static, Script> = + CodePointMapData::