fix: consider all fts fields to ignore for distinct values (#8601)

### **User description**
Consider all fts fields including env specified fts fields for ignoring
in distinct field setup


___

### **PR Type**
Bug fix


___

### **Description**
- Exclude all FTS fields from distinct setup

- Use global and per-stream FTS sources

- Add config constant checks for FTS fields

- Centralize FTS retrieval via schema helper


___

### Diagram Walkthrough


```mermaid
flowchart LR
  CFG["SQL_FULL_TEXT_SEARCH_FIELDS (config)"]
  SCH["get_stream_setting_fts_fields (schema)"]
  MIG["migration/dashboards.rs"]
  SVCDB["service/dashboards/mod.rs"]
  SVCSTR["service/stream.rs"]

  CFG -- "global FTS list" --> MIG
  SCH -- "resolve stream FTS" --> MIG
  SCH -- "resolve stream FTS" --> SVCDB
  SCH -- "resolve stream FTS" --> SVCSTR

  MIG -- "skip FTS in distinct" --> Distinct["Distinct setup"]
  SVCDB -- "skip FTS in distinct" --> Distinct
  SVCSTR -- "skip FTS in distinct" --> Distinct
```



<details> <summary><h3> File Walkthrough</h3></summary>

<table><thead><tr><th></th><th align="left">Relevant
files</th></tr></thead><tbody><tr><td><strong>Bug
fix</strong></td><td><table>
<tr>
  <td>
    <details>
<summary><strong>dashboards.rs</strong><dd><code>Ignore global and
stream FTS in migration distincts</code>&nbsp; &nbsp; &nbsp; &nbsp;
&nbsp; &nbsp; </dd></summary>
<hr>

src/migration/dashboards.rs

<ul><li>Import <code>SQL_FULL_TEXT_SEARCH_FIELDS</code> and
<code>get_stream_setting_fts_fields</code><br> <li> Skip adding distinct
fields if in global or stream FTS<br> <li> Preserve existing reserved
field checks</ul>


</details>


  </td>
<td><a
href="https://github.com/openobserve/openobserve/pull/8601/files#diff-eba7ddaafafa0efbc35474fd19af320f0cce754dc6048bf4ce3d9ff3f22358b1">+6/-3</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td>
    <details>
<summary><strong>mod.rs</strong><dd><code>Use unified FTS set when
updating dashboard distincts</code>&nbsp; &nbsp; &nbsp; &nbsp;
</dd></summary>
<hr>

src/service/dashboards/mod.rs

<ul><li>Import <code>SQL_FULL_TEXT_SEARCH_FIELDS</code> and schema FTS
helper<br> <li> Use <code>get_stream_setting_fts_fields</code> to filter
FTS<br> <li> Continue ignoring reserved fields like timestamp and
count</ul>


</details>


  </td>
<td><a
href="https://github.com/openobserve/openobserve/pull/8601/files#diff-6ae9e22ced40c50675dde5348b8d696f6559655bbbded915d7e6e88bf64b71a6">+9/-5</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>

<tr>
  <td>
    <details>
<summary><strong>stream.rs</strong><dd><code>Centralize FTS filtering in
stream settings update</code>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
&nbsp; </dd></summary>
<hr>

src/service/stream.rs

<ul><li>Import <code>get_stream_setting_fts_fields</code><br> <li>
Precompute FTS set from settings<br> <li> Skip distinct additions if in
FTS or newly added FTS</ul>


</details>


  </td>
<td><a
href="https://github.com/openobserve/openobserve/pull/8601/files#diff-41452a7fb29b4634437174fd6aecf6a9050dbfdef41b9077a21d59af3d79e9ed">+5/-5</a>&nbsp;
&nbsp; &nbsp; </td>

</tr>
</table></td></tr></tr></tbody></table>

</details>

___

---------

Signed-off-by: Yashodhan Joshi <yjdoc2@gmail.com>
Co-authored-by: Yashodhan Joshi <yjdoc2@gmail.com>

src/migration/dashboards.rs

@@ -14,7 +14,7 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 use config::{
-    TIMESTAMP_COL_NAME,
+    SQL_FULL_TEXT_SEARCH_FIELDS, TIMESTAMP_COL_NAME,
     meta::{
         dashboards::Dashboard,
         stream::{DistinctField, StreamSettings, StreamType},
@@ -82,7 +82,10 @@ async fn add_distinct_from_dashboard(
                 added_ts: now_micros(),
             };
 
-            if !stream_settings.distinct_value_fields.contains(&temp) {
+            if !stream_settings.distinct_value_fields.contains(&temp)
+                && !SQL_FULL_TEXT_SEARCH_FIELDS.contains(&temp.name)
+                && !stream_settings.full_text_search_keys.contains(&temp.name)
+            {
                 stream_settings.distinct_value_fields.push(temp);
             }
 

src/service/dashboards/mod.rs

@@ -24,9 +24,12 @@ use config::{
 };
 use futures::future::join_all;
 use hashbrown::HashMap;
-use infra::table::{
-    self,
-    distinct_values::{DistinctFieldRecord, OriginType},
+use infra::{
+    schema::get_stream_setting_fts_fields,
+    table::{
+        self,
+        distinct_values::{DistinctFieldRecord, OriginType},
+    },
 };
 
 use super::{db::distinct_values, folders, stream::save_stream_settings};
@@ -192,9 +195,10 @@ async fn update_distinct_variables(
                 .unwrap_or_default();
             let mut _new_added = false;
 
+            let _fts = get_stream_setting_fts_fields(&Some(stream_settings.clone()));
             for f in fields.iter() {
                 // we ignore full text search no matter what
-                if stream_settings.full_text_search_keys.contains(f) {
+                if _fts.contains(f) {
                     continue;
                 }
 

src/service/stream.rs

@@ -34,8 +34,8 @@ use infra::{
     cache::stats,
     schema::{
         STREAM_RECORD_ID_GENERATOR, STREAM_SCHEMAS, STREAM_SCHEMAS_LATEST, STREAM_SETTINGS,
-        unwrap_partition_time_level, unwrap_stream_created_at, unwrap_stream_is_derived,
-        unwrap_stream_settings,
+        get_stream_setting_fts_fields, unwrap_partition_time_level, unwrap_stream_created_at,
+        unwrap_stream_is_derived, unwrap_stream_settings,
     },
     table::distinct_values::{DistinctFieldRecord, OriginType, check_field_use},
 };
@@ -646,6 +646,8 @@ pub async fn update_stream_settings(
             .retain(|range| !new_settings.extended_retention_days.remove.contains(range));
     }
 
+    let _fts = get_stream_setting_fts_fields(&Some(settings.clone()));
+
     if !new_settings.distinct_value_fields.add.is_empty() {
         for f in &new_settings.distinct_value_fields.add {
             if f == "count" || f == TIMESTAMP_COL_NAME {
@@ -659,9 +661,7 @@ pub async fn update_stream_settings(
                 );
             }
             // we ignore full text search fields
-            if settings.full_text_search_keys.contains(f)
-                || new_settings.full_text_search_keys.add.contains(f)
-            {
+            if _fts.contains(f) || new_settings.full_text_search_keys.add.contains(f) {
                 continue;
             }
             let record = DistinctFieldRecord::new(