From f6bc5ef6f6cce73f9a002a32aa1237208b36941a Mon Sep 17 00:00:00 2001 From: Yashodhan Joshi Date: Tue, 23 Sep 2025 19:36:49 +0530 Subject: [PATCH] fix: consider all fts fields to ignore for distinct values (#8601) ### **User description** Consider all fts fields including env specified fts fields for ignoring in distinct field setup ___ ### **PR Type** Bug fix ___ ### **Description** - Exclude all FTS fields from distinct setup - Use global and per-stream FTS sources - Add config constant checks for FTS fields - Centralize FTS retrieval via schema helper ___ ### Diagram Walkthrough ```mermaid flowchart LR CFG["SQL_FULL_TEXT_SEARCH_FIELDS (config)"] SCH["get_stream_setting_fts_fields (schema)"] MIG["migration/dashboards.rs"] SVCDB["service/dashboards/mod.rs"] SVCSTR["service/stream.rs"] CFG -- "global FTS list" --> MIG SCH -- "resolve stream FTS" --> MIG SCH -- "resolve stream FTS" --> SVCDB SCH -- "resolve stream FTS" --> SVCSTR MIG -- "skip FTS in distinct" --> Distinct["Distinct setup"] SVCDB -- "skip FTS in distinct" --> Distinct SVCSTR -- "skip FTS in distinct" --> Distinct ```

File Walkthrough
Relevant files
Bug fix
dashboards.rs
Ignore global and stream FTS in migration distincts           
src/migration/dashboards.rs
  • Import SQL_FULL_TEXT_SEARCH_FIELDS and get_stream_setting_fts_fields
  • Skip adding distinct fields if in global or stream FTS
  • Preserve existing reserved field checks
 +6/-3     
mod.rs
Use unified FTS set when updating dashboard distincts       
src/service/dashboards/mod.rs
  • Import SQL_FULL_TEXT_SEARCH_FIELDS and schema FTS helper
  • Use get_stream_setting_fts_fields to filter FTS
  • Continue ignoring reserved fields like timestamp and count
 +9/-5     
stream.rs
Centralize FTS filtering in stream settings update             
src/service/stream.rs
  • Import get_stream_setting_fts_fields
  • Precompute FTS set from settings
  • Skip distinct additions if in FTS or newly added FTS
 +5/-5     
___ --------- Signed-off-by: Yashodhan Joshi Co-authored-by: Yashodhan Joshi --- src/migration/dashboards.rs | 7 +++++-- src/service/dashboards/mod.rs | 12 ++++++++---- src/service/stream.rs | 10 +++++----- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/migration/dashboards.rs b/src/migration/dashboards.rs index 24136e6975..230aaa6bf7 100644 --- a/src/migration/dashboards.rs +++ b/src/migration/dashboards.rs @@ -14,7 +14,7 @@ // along with this program. If not, see . use config::{ - TIMESTAMP_COL_NAME, + SQL_FULL_TEXT_SEARCH_FIELDS, TIMESTAMP_COL_NAME, meta::{ dashboards::Dashboard, stream::{DistinctField, StreamSettings, StreamType}, @@ -82,7 +82,10 @@ async fn add_distinct_from_dashboard( added_ts: now_micros(), }; - if !stream_settings.distinct_value_fields.contains(&temp) { + if !stream_settings.distinct_value_fields.contains(&temp) + && !SQL_FULL_TEXT_SEARCH_FIELDS.contains(&temp.name) + && !stream_settings.full_text_search_keys.contains(&temp.name) + { stream_settings.distinct_value_fields.push(temp); } diff --git a/src/service/dashboards/mod.rs b/src/service/dashboards/mod.rs index 86d0bfa02b..6924a1ee3d 100644 --- a/src/service/dashboards/mod.rs +++ b/src/service/dashboards/mod.rs @@ -24,9 +24,12 @@ use config::{ }; use futures::future::join_all; use hashbrown::HashMap; -use infra::table::{ - self, - distinct_values::{DistinctFieldRecord, OriginType}, +use infra::{ + schema::get_stream_setting_fts_fields, + table::{ + self, + distinct_values::{DistinctFieldRecord, OriginType}, + }, }; use super::{db::distinct_values, folders, stream::save_stream_settings}; @@ -192,9 +195,10 @@ async fn update_distinct_variables( .unwrap_or_default(); let mut _new_added = false; + let _fts = get_stream_setting_fts_fields(&Some(stream_settings.clone())); for f in fields.iter() { // we ignore full text search no matter what - if stream_settings.full_text_search_keys.contains(f) { + if _fts.contains(f) { continue; } diff --git a/src/service/stream.rs b/src/service/stream.rs index 1cd138ed48..3a4d9fa116 100644 --- a/src/service/stream.rs +++ b/src/service/stream.rs @@ -34,8 +34,8 @@ use infra::{ cache::stats, schema::{ STREAM_RECORD_ID_GENERATOR, STREAM_SCHEMAS, STREAM_SCHEMAS_LATEST, STREAM_SETTINGS, - unwrap_partition_time_level, unwrap_stream_created_at, unwrap_stream_is_derived, - unwrap_stream_settings, + get_stream_setting_fts_fields, unwrap_partition_time_level, unwrap_stream_created_at, + unwrap_stream_is_derived, unwrap_stream_settings, }, table::distinct_values::{DistinctFieldRecord, OriginType, check_field_use}, }; @@ -646,6 +646,8 @@ pub async fn update_stream_settings( .retain(|range| !new_settings.extended_retention_days.remove.contains(range)); } + let _fts = get_stream_setting_fts_fields(&Some(settings.clone())); + if !new_settings.distinct_value_fields.add.is_empty() { for f in &new_settings.distinct_value_fields.add { if f == "count" || f == TIMESTAMP_COL_NAME { @@ -659,9 +661,7 @@ pub async fn update_stream_settings( ); } // we ignore full text search fields - if settings.full_text_search_keys.contains(f) - || new_settings.full_text_search_keys.add.contains(f) - { + if _fts.contains(f) || new_settings.full_text_search_keys.add.contains(f) { continue; } let record = DistinctFieldRecord::new(