From 2da2e1a23195771e2b3ee3fa6a35a975ab8014eb Mon Sep 17 00:00:00 2001 From: Sai Nikhil Date: Wed, 25 Mar 2026 16:20:38 +0530 Subject: [PATCH] fix: auto suggestion field value on query editor logs and traces (#10943) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds persistent, cross-session field value suggestions to the SQL query editor. When a user types field = , field IN (, field LIKE ', str_match(field, etc., the autocomplete popup now shows real values seen in previous searches — not just keywords and field names. Values are captured passively from search results and field expansions, stored in IndexedDB, and served from an in-memory cache. Zero impact on existing UI performance. What's new Value capture — two sources: Run Query (useStreamFields.ts) — after each search, up to 100 result rows are scanned and field values extracted into IndexedDB in a browser idle callback Field expansion (useFieldValuesStream.ts, IndexList.vue) — when a user clicks > on a field in the left panel, the Values API response is also captured Persistence layer (fieldValueDB.ts — new): IndexedDB database o2FieldValues, store fieldValues Composite key: org|streamType|streamName|fieldName — fully isolated per org, stream type, and stream Merge strategy: new values are unioned with existing, capped at 50 per field 7-day sliding TTL — fields from streams you stop using expire automatically Opportunistic housekeeping (5% chance per search) via by_expires and by_updated indexes Orchestration layer (useFieldValueStore.ts — new): captureFromSearchHits — schedules extraction + batch IDB write via requestIdleCallback captureFromValuesApi — schedules single-field IDB write getFieldValuesForSuggestion — reads with 60s in-memory cache (500-entry cap) so keystrokes never hit IDB Safari fallback: setTimeout(0) when requestIdleCallback is unavailable All IDB errors caught silently — feature degrades gracefully in private browsing Suggestion engine (useSuggestions.ts): New analyzeSqlWhereClause function detects when cursor is after an operator expecting a value using a 4-alternative regex covering =, !=, <>, >=, <=, >, <, IN (, NOT IN (, LIKE, NOT LIKE, str_match(, fuzzy_match( Cursor-sliced matching handles Monaco's auto-closing brackets correctly Smart insertText quoting: numeric/boolean values unquoted, strings wrapped in '...', open-quote detection avoids double-quoting sortText prefixed with \x01 so value suggestions always sort above keywords and functions In-session values (current search) take precedence over stored values (previous sessions) Monaco popup fix (CodeQueryEditor.vue): hideSuggestWidget → nextTick → triggerSuggest pattern ensures the popup always refreshes, fixing cases where it was already in "Showing" state (e.g. typing LIKE ') Traces support (traces/SearchBar.vue): Field expansion in traces was already writing to IDB (via useFieldValuesStream with stream_type: "traces") Added 3 lines to set org/streamType/streamName on autoCompleteData so the read side works too --- web/src/components/CodeQueryEditor.vue | 9 +- web/src/composables/fieldValueDB.spec.ts | 296 ++++++++ web/src/composables/fieldValueDB.ts | 350 +++++++++ .../composables/useFieldValueStore.spec.ts | 219 ++++++ web/src/composables/useFieldValueStore.ts | 284 ++++++++ web/src/composables/useFieldValuesStream.ts | 14 + .../composables/useLogs/useStreamFields.ts | 24 +- web/src/composables/useSuggestions.spec.ts | 676 ++++++++---------- web/src/composables/useSuggestions.ts | 168 ++++- web/src/plugins/logs/IndexList.vue | 14 + web/src/plugins/logs/SearchBar.vue | 7 + web/src/plugins/traces/SearchBar.vue | 9 + web/src/utils/fieldValueUtils.spec.ts | 243 +++++++ web/src/utils/fieldValueUtils.ts | 102 +++ 14 files changed, 1987 insertions(+), 428 deletions(-) create mode 100644 web/src/composables/fieldValueDB.spec.ts create mode 100644 web/src/composables/fieldValueDB.ts create mode 100644 web/src/composables/useFieldValueStore.spec.ts create mode 100644 web/src/composables/useFieldValueStore.ts create mode 100644 web/src/utils/fieldValueUtils.spec.ts create mode 100644 web/src/utils/fieldValueUtils.ts diff --git a/web/src/components/CodeQueryEditor.vue b/web/src/components/CodeQueryEditor.vue index b4c62f3020..2587e60b94 100644 --- a/web/src/components/CodeQueryEditor.vue +++ b/web/src/components/CodeQueryEditor.vue @@ -888,7 +888,14 @@ export default defineComponent({ }; const triggerAutoComplete = async (value: string) => { - disableSuggestionPopup(); + // Close any currently-open suggestion popup before retriggering. + // Monaco's model.trigger() calls cancel() internally, but if a natural + // popup is already in "Showing" state (e.g. opened by typing quote after + // an operator, or from word-based suggest while typing NOT LIKE), the + // widget may not refresh its item list. hideSuggestWidget transitions + // state to Idle cleanly (no "user dismissed" flag) so the following + // triggerSuggest always opens a fresh popup with the latest keywords. + editorObj.trigger(value, "hideSuggestWidget", {}); await nextTick(); editorObj.trigger(value, "editor.action.triggerSuggest", {}); }; diff --git a/web/src/composables/fieldValueDB.spec.ts b/web/src/composables/fieldValueDB.spec.ts new file mode 100644 index 0000000000..de3aa1e518 --- /dev/null +++ b/web/src/composables/fieldValueDB.spec.ts @@ -0,0 +1,296 @@ +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { + openDB, + mergeValues, + mergeMultipleValues, + getValues, + evictExpired, + trimToMaxFields, +} from "./fieldValueDB"; + +// ─── IDB mock helpers ───────────────────────────────────────────────────────── +// jsdom does not ship a real IDB implementation. We use a minimal in-memory +// mock that mirrors the async/event-driven API surface used by fieldValueDB.ts. + +// Track the last db created so we can fire onversionchange between tests to +// reset the _db singleton inside fieldValueDB.ts. +let _lastDb: any = null; + +const makeStore = () => { + const records = new Map(); + + const makeRequest = (fn: () => any) => { + const req: any = { onsuccess: null, onerror: null, result: undefined }; + setTimeout(() => { + try { + req.result = fn(); + req.onsuccess?.(); + } catch (e) { + req.error = e; + req.onerror?.(); + } + }, 0); + return req; + }; + + return { + records, + get: (key: string) => makeRequest(() => records.get(key)), + put: (record: any) => makeRequest(() => records.set(record.key, record)), + delete: (key: string) => makeRequest(() => records.delete(key)), + count: () => makeRequest(() => records.size), + index: (name: string) => ({ + openCursor: (range?: any) => { + const entries = [...records.values()]; + let filtered = entries; + if (range) { + // Simulate upperBound(now, true) for by_expires index + filtered = entries.filter((r) => r.expiresAt < range._upper); + } + // Sort by updatedAt ascending for by_updated index + if (name === "by_updated") { + filtered.sort((a, b) => a.updatedAt - b.updatedAt); + } + let idx = 0; + const req: any = { onsuccess: null }; + const advance = () => { + setTimeout(() => { + if (idx < filtered.length) { + const entry = filtered[idx++]; + req.result = { + value: entry, + delete: () => records.delete(entry.key), + continue: advance, + }; + } else { + req.result = null; + } + req.onsuccess?.({ target: req }); + }, 0); + }; + setTimeout(advance, 0); + return req; + }, + }), + }; +}; + +const setupIDBMock = () => { + // Fire onversionchange on the previous db so fieldValueDB.ts resets _db = null. + // This ensures each test gets a fresh IDB connection rather than reusing the + // cached singleton from the previous test. + if (_lastDb?.onversionchange) { + _lastDb.onversionchange(); + } + + const store = makeStore(); + const txCompleteCbs: (() => void)[] = []; + + const tx: any = { + objectStore: () => store, + oncomplete: null, + onerror: null, + }; + + // Fire oncomplete after a tick so all put/get requests resolve first + const fireTxComplete = () => { + setTimeout(() => tx.oncomplete?.(), 10); + }; + + const db: any = { + transaction: () => { + setTimeout(fireTxComplete, 0); + return tx; + }, + close: () => {}, // fieldValueDB.ts calls _db?.close() inside onversionchange + onclose: null, + onversionchange: null, + }; + + const openReq: any = { onsuccess: null, onerror: null, onupgradeneeded: null }; + setTimeout(() => { + openReq.result = db; + openReq.onsuccess?.(); + }, 0); + + vi.stubGlobal("indexedDB", { + open: () => openReq, + }); + + // Also stub IDBKeyRange + vi.stubGlobal("IDBKeyRange", { + upperBound: (val: any, exclusive: boolean) => ({ _upper: val, exclusive }), + }); + + // Track db so the next setupIDBMock() can fire onversionchange to reset _db + _lastDb = db; + + return store; +}; + +// ─── openDB ─────────────────────────────────────────────────────────────────── + +describe("openDB", () => { + beforeEach(() => { + // Reset the cached _db between tests by re-importing would need module reset. + // Instead we just verify openDB resolves without throwing. + setupIDBMock(); + }); + + it("resolves with a database object", async () => { + const db = await openDB(); + expect(db).toBeDefined(); + expect(typeof db.transaction).toBe("function"); + }); +}); + +// ─── mergeValues ───────────────────────────────────────────────────────────── + +describe("mergeValues", () => { + beforeEach(() => { + setupIDBMock(); + }); + + it("writes a new record when key does not exist", async () => { + await mergeValues("org|logs|stream|status", ["200", "404"], "search_hits", 50, 86400000); + const values = await getValues("org|logs|stream|status"); + expect(values).toEqual(expect.arrayContaining(["200", "404"])); + }); + + it("merges with existing record (deduplicates)", async () => { + await mergeValues("org|logs|stream|env", ["prod"], "search_hits", 50, 86400000); + await mergeValues("org|logs|stream|env", ["prod", "staging"], "search_hits", 50, 86400000); + const values = await getValues("org|logs|stream|env"); + expect(values).toEqual(expect.arrayContaining(["prod", "staging"])); + expect(values).toHaveLength(2); + }); + + it("caps values at maxValues", async () => { + const incoming = Array.from({ length: 10 }, (_, i) => `v${i}`); + await mergeValues("org|logs|stream|service", incoming, "search_hits", 3, 86400000); + const values = await getValues("org|logs|stream|service"); + expect(values).toHaveLength(3); + }); + + it("marks source as mixed when sources differ", async () => { + await mergeValues("org|logs|stream|field1", ["a"], "search_hits", 50, 86400000); + await mergeValues("org|logs|stream|field1", ["b"], "values_api", 50, 86400000); + // We can't directly inspect source via getValues — we trust the internal + // record. The merge shouldn't throw and should return combined values. + const values = await getValues("org|logs|stream|field1"); + expect(values).toEqual(expect.arrayContaining(["a", "b"])); + }); +}); + +// ─── mergeMultipleValues ────────────────────────────────────────────────────── + +describe("mergeMultipleValues", () => { + beforeEach(() => { + setupIDBMock(); + }); + + it("writes multiple fields in one call", async () => { + await mergeMultipleValues( + [ + { key: "org|logs|stream|status", values: ["200"], source: "search_hits" }, + { key: "org|logs|stream|env", values: ["prod"], source: "search_hits" }, + ], + 50, + 86400000, + ); + expect(await getValues("org|logs|stream|status")).toEqual(["200"]); + expect(await getValues("org|logs|stream|env")).toEqual(["prod"]); + }); + + it("returns early when entries is empty", async () => { + // Should not throw + await expect(mergeMultipleValues([], 50, 86400000)).resolves.toBeUndefined(); + }); + + it("deduplicates values within a single entry", async () => { + await mergeMultipleValues( + [{ key: "org|logs|stream|level", values: ["info", "info", "warn"], source: "search_hits" }], + 50, + 86400000, + ); + const values = await getValues("org|logs|stream|level"); + expect(values).toHaveLength(2); + }); + + it("caps each field at maxValues", async () => { + const values = Array.from({ length: 10 }, (_, i) => `val${i}`); + await mergeMultipleValues( + [{ key: "org|logs|stream|capped", values, source: "search_hits" }], + 3, + 86400000, + ); + expect(await getValues("org|logs|stream|capped")).toHaveLength(3); + }); +}); + +// ─── getValues ──────────────────────────────────────────────────────────────── + +describe("getValues", () => { + beforeEach(() => { + setupIDBMock(); + }); + + it("returns empty array for missing key", async () => { + expect(await getValues("nonexistent|key")).toEqual([]); + }); + + it("returns empty array for expired record", async () => { + // Write a record with expiresAt in the past + const store = setupIDBMock(); + store.records.set("org|logs|stream|expired", { + key: "org|logs|stream|expired", + values: ["200"], + updatedAt: Date.now() - 10000, + expiresAt: Date.now() - 1000, // already expired + source: "search_hits", + }); + expect(await getValues("org|logs|stream|expired")).toEqual([]); + }); + + it("returns values for a valid non-expired record", async () => { + await mergeValues("org|logs|stream|valid", ["200", "404"], "search_hits", 50, 86400000); + const values = await getValues("org|logs|stream|valid"); + expect(values).toEqual(expect.arrayContaining(["200", "404"])); + }); + + it("returns [] on IDB error without throwing", async () => { + // Break IDB to simulate unavailability + vi.stubGlobal("indexedDB", undefined); + const values = await getValues("any|key"); + expect(values).toEqual([]); + }); +}); + +// ─── evictExpired ───────────────────────────────────────────────────────────── + +describe("evictExpired", () => { + beforeEach(() => { + setupIDBMock(); + }); + + it("resolves with a number (count of deleted records)", async () => { + const result = await evictExpired(); + expect(typeof result).toBe("number"); + }); +}); + +// ─── trimToMaxFields ───────────────────────────────────────────────────────── + +describe("trimToMaxFields", () => { + beforeEach(() => { + setupIDBMock(); + }); + + it("resolves without error when records are below limit", async () => { + await expect(trimToMaxFields(5000)).resolves.toBeUndefined(); + }); + + it("does not throw when store is empty", async () => { + await expect(trimToMaxFields(0)).resolves.toBeUndefined(); + }); +}); diff --git a/web/src/composables/fieldValueDB.ts b/web/src/composables/fieldValueDB.ts new file mode 100644 index 0000000000..152c7789bb --- /dev/null +++ b/web/src/composables/fieldValueDB.ts @@ -0,0 +1,350 @@ +// Copyright 2026 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +/** + * Low-level IndexedDB operations for field value storage. + * All methods are async and non-blocking to the caller. + * + * DB layout: + * Database : o2FieldValues (version 1) + * Store : fieldValues + * PK : key — composite string "org|streamType|streamName|fieldName" + * Index 1 : by_expires (expiresAt) — used by evictExpired for TTL sweeps + * Index 2 : by_updated (updatedAt) — used by trimToMaxFields to evict LRU + * + * Why two indexes? + * evictExpired needs records sorted by expiry time (delete the expired ones). + * trimToMaxFields needs records sorted by last-used time (delete the oldest). + * These are different orderings so they require separate indexes. + */ + +const DB_NAME = "o2FieldValues"; +const DB_VERSION = 1; +const STORE_NAME = "fieldValues"; + +export interface FieldValueRecord { + key: string; // PK — composite key "org|streamType|streamName|fieldName" + values: string[]; // unique, deduplicated; capped at MAX_VALUES_PER_FIELD + updatedAt: number; // Unix ms — last write timestamp; used for LRU eviction + expiresAt: number; // Unix ms — sliding TTL, reset on every write + // "mixed" means values came from both sources at different points in time. + // Used for debugging only — lets you see in DevTools where values originated. + source: "values_api" | "search_hits" | "mixed"; +} + +// Cached IDB connection — opened once and reused across all calls. +// null means not yet opened (or was closed and needs to be reopened). +let _db: IDBDatabase | null = null; + +/** + * Opens the database lazily — returns the cached connection on subsequent calls. + * + * Why lazy (open on first use, not at app startup)? + * IDB open is async and costs ~5–10ms. Opening eagerly would always pay that + * cost even if the user never uses the query editor or never triggers + * autocomplete. Lazy open means zero cost for users who don't need it. + * + * Why cache the connection in _db? + * IDBDatabase.open() creates a new connection object each time. Connections + * are cheap but not free. More importantly, reusing one connection means all + * operations share the same transaction queue, which is correct IDB behaviour. + * + * onclose / onversionchange handlers: + * If another tab opens a newer version of this DB (DB_VERSION bumped in a + * future release), the browser fires versionchange on existing connections and + * then closes them. We close and null out _db so the next call to openDB() + * reopens cleanly with the new version. + */ +export const openDB = (): Promise => { + if (_db) return Promise.resolve(_db); + return new Promise((resolve, reject) => { + const req = indexedDB.open(DB_NAME, DB_VERSION); + req.onupgradeneeded = (e) => { + const db = (e.target as IDBOpenDBRequest).result; + if (!db.objectStoreNames.contains(STORE_NAME)) { + const store = db.createObjectStore(STORE_NAME, { keyPath: "key" }); + // by_expires: IDBKeyRange.upperBound(now) range scan in evictExpired + store.createIndex("by_expires", "expiresAt", { unique: false }); + // by_updated: ascending cursor in trimToMaxFields (oldest first) + store.createIndex("by_updated", "updatedAt", { unique: false }); + } + }; + req.onsuccess = () => { + _db = req.result; + // Reset the cached reference if the browser closes the connection + // (e.g. another tab opens a newer DB version triggering a versionchange). + _db.onclose = () => { + _db = null; + }; + _db.onversionchange = () => { + _db?.close(); + _db = null; + }; + resolve(_db!); + }; + req.onerror = () => reject(req.error); + }); +}; + +/** + * Merge new values into an existing record for a single field. + * Used by captureFromValuesApi (Values API path — one field at a time). + * + * Merge strategy: + * existing values + incoming values → deduplicate via Set → cap to maxValues + * If total > maxValues, keep the LAST maxValues (most recently seen / freshest). + * + * Why sliding TTL (reset expiresAt on every write)? + * Fields from streams you query frequently stay fresh forever — every new + * search resets their 7-day clock. Fields from streams you stop using expire + * naturally after 7 days of inactivity without any manual cleanup. + * + * Why track source ("values_api" | "search_hits" | "mixed")? + * Purely for debugging. In DevTools → Application → IndexedDB you can see + * whether a field's values came from explicit field expansion (values_api), + * passively from search results (search_hits), or both (mixed). Helps when + * investigating why certain values appear or don't appear in autocomplete. + */ +export const mergeValues = async ( + key: string, + incoming: string[], + source: FieldValueRecord["source"], + maxValues: number, + ttlMs: number, +): Promise => { + const db = await openDB(); + return new Promise((resolve, reject) => { + const tx = db.transaction(STORE_NAME, "readwrite"); + const store = tx.objectStore(STORE_NAME); + const getReq = store.get(key); + getReq.onsuccess = () => { + const existing: FieldValueRecord | undefined = getReq.result; + // Combine existing + incoming into a Set for automatic deduplication. + // No need to manually check "have I seen this value before?". + const valueSet = new Set(existing?.values ?? []); + for (const v of incoming) valueSet.add(String(v)); + const merged = Array.from(valueSet); + // Keep the last maxValues entries — the most recently added values are + // at the end of the array and are most likely to be relevant to the + // user's current query context. + const values = + merged.length > maxValues + ? merged.slice(merged.length - maxValues) + : merged; + const now = Date.now(); + const record: FieldValueRecord = { + key, + values, + updatedAt: now, + expiresAt: now + ttlMs, // sliding window — reset on every write + source: existing + ? existing.source === source + ? source // same source as before — keep it + : "mixed" // different source — mark as mixed + : source, // first write — use the incoming source + }; + store.put(record); + }; + tx.oncomplete = () => resolve(); + tx.onerror = () => reject(tx.error); + }); +}; + +/** + * Merge values for multiple fields in a single IDB transaction. + * Used by captureFromSearchHits — the hot path after every Run Query. + * + * Why one transaction instead of N mergeValues calls? + * IDB transactions have a fixed cost: acquire the store lock, execute + * operations, commit. If a stream has 40 fields, calling mergeValues 40 times + * = 40 separate transactions = 40× that fixed cost. One transaction with 40 + * get+put operations = 1× that cost. On a wide schema this is ~40× cheaper. + * + * The merge logic per field is identical to mergeValues above — the only + * difference is that all operations share one tx.oncomplete callback. + */ +export const mergeMultipleValues = async ( + entries: Array<{ + key: string; + values: string[]; + source: FieldValueRecord["source"]; + }>, + maxValues: number, + ttlMs: number, +): Promise => { + if (entries.length === 0) return; + const db = await openDB(); + return new Promise((resolve, reject) => { + const tx = db.transaction(STORE_NAME, "readwrite"); + const store = tx.objectStore(STORE_NAME); + const now = Date.now(); + + for (const { key, values: incoming, source } of entries) { + const getReq = store.get(key); + getReq.onsuccess = () => { + const existing: FieldValueRecord | undefined = getReq.result; + const valueSet = new Set(existing?.values ?? []); + for (const v of incoming) valueSet.add(String(v)); + const merged = Array.from(valueSet); + const values = + merged.length > maxValues + ? merged.slice(merged.length - maxValues) + : merged; + const record: FieldValueRecord = { + key, + values, + updatedAt: now, + expiresAt: now + ttlMs, + source: existing + ? existing.source === source + ? source + : "mixed" + : source, + }; + store.put(record); + }; + } + + // All get+put operations for every field complete before this fires. + // Resolving here means the caller knows the full batch is safely written. + tx.oncomplete = () => resolve(); + tx.onerror = () => reject(tx.error); + }); +}; + +/** + * Fast point-read for a single field's values. + * Returns empty array if record is missing or expired. + * + * Why check expiresAt at read time (not just rely on evictExpired)? + * evictExpired only runs on ~5% of searches — a record can be expired but not + * yet swept from the DB. Without this check, we could serve stale values to + * autocomplete for up to several searches after the TTL has passed. + * This makes the read authoritative regardless of when cleanup last ran. + */ +export const getValues = async (key: string): Promise => { + let db: IDBDatabase; + try { + db = await openDB(); + } catch { + // IDB unavailable (private browsing, unsupported browser, quota) — return + // empty array so autocomplete silently degrades rather than throwing. + return []; + } + return new Promise((resolve) => { + const tx = db.transaction(STORE_NAME, "readonly"); + const req = tx.objectStore(STORE_NAME).get(key); + req.onsuccess = () => { + const record = req.result as FieldValueRecord | undefined; + if (!record) { + resolve([]); + return; + } + // Check TTL at read time — don't serve expired data even if evictExpired + // hasn't swept this record yet. + if (record.expiresAt < Date.now()) { + resolve([]); + return; + } + resolve(record.values); + }; + req.onerror = () => resolve([]); // never throw — autocomplete is non-critical + }); +}; + +/** + * Evict all records whose expiresAt < now (TTL cleanup). + * Called opportunistically on ~5% of searches to amortize the scan cost. + * + * Why use the by_expires index instead of a full store scan? + * A full scan would read every record regardless of expiry status. + * IDBKeyRange.upperBound(now, true) on the by_expires index reads ONLY records + * with expiresAt < now — exactly the ones we want to delete. On 5000 records + * with 10 expired, this reads 10 records instead of 5000. O(expired) not O(total). + * + * Why "opportunistic" (5% chance) instead of every search? + * When nothing has expired (the common case), this scan still costs ~5–10ms. + * Running it on every search wastes that time for zero benefit. 1-in-20 gives + * periodic cleanup while spreading the cost across many searches. + */ +export const evictExpired = async (): Promise => { + const db = await openDB(); + const now = Date.now(); + return new Promise((resolve, reject) => { + const tx = db.transaction(STORE_NAME, "readwrite"); + const index = tx.objectStore(STORE_NAME).index("by_expires"); + // upperBound(now, true) = expiresAt < now (exclusive upper bound) + const range = IDBKeyRange.upperBound(now, true); + let deleted = 0; + const cursorReq = index.openCursor(range); + cursorReq.onsuccess = (e) => { + const cursor = (e.target as IDBRequest).result; + if (cursor) { + cursor.delete(); + deleted++; + cursor.continue(); // advance to next expired record + } else { + resolve(deleted); // cursor exhausted — done + } + }; + cursorReq.onerror = () => reject(cursorReq.error); + tx.onerror = () => reject(tx.error); + }); +}; + +/** + * Trim total record count to maxFields — evict least-recently-used fields first. + * Called opportunistically alongside evictExpired (~5% of searches). + * + * Why trim by updatedAt (oldest first)? + * Fields with the oldest updatedAt are the ones the user hasn't queried in the + * longest time — least likely to be useful in autocomplete. Evicting them + * preserves the most relevant, recently-used field values. + * + * Why is this a last-resort guard? + * In practice, 5000 fields = ~100 streams × 50 fields each, which is far more + * than most users will encounter. This function is mainly here to prevent + * unbounded growth in edge cases (many orgs, many streams, very long sessions). + */ +export const trimToMaxFields = async (maxFields: number): Promise => { + const db = await openDB(); + return new Promise((resolve) => { + const tx = db.transaction(STORE_NAME, "readwrite"); + const store = tx.objectStore(STORE_NAME); + const countReq = store.count(); + countReq.onsuccess = () => { + const total = countReq.result; + if (total <= maxFields) { + // Already under the cap — nothing to do + resolve(); + return; + } + const toDelete = total - maxFields; + // by_updated cursor starts from the smallest updatedAt (least recently used) + const index = store.index("by_updated"); + let deleted = 0; + index.openCursor().onsuccess = (e) => { + const cursor = (e.target as IDBRequest).result; + if (cursor && deleted < toDelete) { + cursor.delete(); + deleted++; + cursor.continue(); + } else { + resolve(); // deleted enough or ran out of records + } + }; + }; + }); +}; diff --git a/web/src/composables/useFieldValueStore.spec.ts b/web/src/composables/useFieldValueStore.spec.ts new file mode 100644 index 0000000000..6512e805ea --- /dev/null +++ b/web/src/composables/useFieldValueStore.spec.ts @@ -0,0 +1,219 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +// ─── Mock fieldValueDB ──────────────────────────────────────────────────────── +vi.mock("@/composables/fieldValueDB", () => ({ + mergeValues: vi.fn().mockResolvedValue(undefined), + mergeMultipleValues: vi.fn().mockResolvedValue(undefined), + getValues: vi.fn().mockResolvedValue([]), + evictExpired: vi.fn().mockResolvedValue(0), + trimToMaxFields: vi.fn().mockResolvedValue(undefined), +})); + +// ─── Mock extractValuesFromHits ─────────────────────────────────────────────── +vi.mock("@/utils/fieldValueUtils", () => ({ + extractValuesFromHits: vi.fn().mockReturnValue({ + status: ["200", "404"], + env: ["prod", "staging"], + }), +})); + +import * as fieldValueDB from "@/composables/fieldValueDB"; +import { extractValuesFromHits } from "@/utils/fieldValueUtils"; +import { + captureFromSearchHits, + captureFromValuesApi, + getFieldValuesForSuggestion, +} from "./useFieldValueStore"; + +const ctx = { org: "myorg", streamType: "logs", streamName: "http_logs" }; + +// ─── scheduleWrite behaviour ───────────────────────────────────────────────── +// requestIdleCallback is not available in jsdom — our code falls back to +// setTimeout(0). We use vi.useFakeTimers() to control when those fire. + +describe("captureFromSearchHits", () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.clearAllMocks(); + }); + afterEach(() => { + vi.useRealTimers(); + }); + + it("does nothing when hits is empty", async () => { + captureFromSearchHits(ctx, [], []); + await vi.runAllTimersAsync(); + expect(fieldValueDB.mergeMultipleValues).not.toHaveBeenCalled(); + }); + + it("does nothing when hits is null/undefined", async () => { + captureFromSearchHits(ctx, null as any, []); + await vi.runAllTimersAsync(); + expect(fieldValueDB.mergeMultipleValues).not.toHaveBeenCalled(); + }); + + it("calls extractValuesFromHits with up to 100 hits", async () => { + const hits = Array.from({ length: 150 }, (_, i) => ({ status: `${i}` })); + captureFromSearchHits(ctx, hits, []); + await vi.runAllTimersAsync(); + expect(extractValuesFromHits).toHaveBeenCalledWith( + hits.slice(0, 100), + [], + expect.any(Number), + ); + }); + + it("calls mergeMultipleValues with correct composite keys", async () => { + captureFromSearchHits(ctx, [{ status: "200" }], ["status"]); + await vi.runAllTimersAsync(); + expect(fieldValueDB.mergeMultipleValues).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.objectContaining({ key: "myorg|logs|http_logs|status" }), + expect.objectContaining({ key: "myorg|logs|http_logs|env" }), + ]), + expect.any(Number), + expect.any(Number), + ); + }); + + it("passes schemaFields to extractValuesFromHits", async () => { + const schema = ["status", "env"]; + captureFromSearchHits(ctx, [{ status: "200" }], schema); + await vi.runAllTimersAsync(); + expect(extractValuesFromHits).toHaveBeenCalledWith( + expect.any(Array), + schema, + expect.any(Number), + ); + }); + + it("does not call mergeMultipleValues when extractValuesFromHits returns empty", async () => { + vi.mocked(extractValuesFromHits).mockReturnValueOnce({}); + captureFromSearchHits(ctx, [{ status: "200" }], []); + await vi.runAllTimersAsync(); + expect(fieldValueDB.mergeMultipleValues).not.toHaveBeenCalled(); + }); +}); + +// ─── captureFromValuesApi ───────────────────────────────────────────────────── + +describe("captureFromValuesApi", () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.clearAllMocks(); + }); + afterEach(() => { + vi.useRealTimers(); + }); + + it("calls mergeValues with correct key and values", async () => { + captureFromValuesApi(ctx, "status", [ + { key: "200" }, + { key: "404" }, + ]); + await vi.runAllTimersAsync(); + expect(fieldValueDB.mergeValues).toHaveBeenCalledWith( + "myorg|logs|http_logs|status", + ["200", "404"], + "values_api", + expect.any(Number), + expect.any(Number), + ); + }); + + it("filters out empty-string values", async () => { + captureFromValuesApi(ctx, "status", [{ key: "" }, { key: "200" }]); + await vi.runAllTimersAsync(); + expect(fieldValueDB.mergeValues).toHaveBeenCalledWith( + expect.any(String), + ["200"], + expect.any(String), + expect.any(Number), + expect.any(Number), + ); + }); + + it("filters out values longer than 200 chars", async () => { + const long = "x".repeat(201); + captureFromValuesApi(ctx, "status", [{ key: long }, { key: "200" }]); + await vi.runAllTimersAsync(); + expect(fieldValueDB.mergeValues).toHaveBeenCalledWith( + expect.any(String), + ["200"], + expect.any(String), + expect.any(Number), + expect.any(Number), + ); + }); + + it("does nothing when all values are filtered out", async () => { + captureFromValuesApi(ctx, "status", [{ key: "" }]); + await vi.runAllTimersAsync(); + expect(fieldValueDB.mergeValues).not.toHaveBeenCalled(); + }); + + it("does nothing when entries array is empty", async () => { + captureFromValuesApi(ctx, "status", []); + await vi.runAllTimersAsync(); + expect(fieldValueDB.mergeValues).not.toHaveBeenCalled(); + }); + + it("uses correct stream context in composite key", async () => { + const tracesCtx = { org: "acme", streamType: "traces", streamName: "default" }; + captureFromValuesApi(tracesCtx, "service", [{ key: "frontend" }]); + await vi.runAllTimersAsync(); + expect(fieldValueDB.mergeValues).toHaveBeenCalledWith( + "acme|traces|default|service", + ["frontend"], + "values_api", + expect.any(Number), + expect.any(Number), + ); + }); +}); + +// ─── getFieldValuesForSuggestion ────────────────────────────────────────────── + +describe("getFieldValuesForSuggestion", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("returns values from IDB on cache miss", async () => { + vi.mocked(fieldValueDB.getValues).mockResolvedValueOnce(["200", "404"]); + const values = await getFieldValuesForSuggestion(ctx, "status"); + expect(values).toEqual(["200", "404"]); + expect(fieldValueDB.getValues).toHaveBeenCalledWith("myorg|logs|http_logs|status"); + }); + + it("returns cached values on second call (cache hit, no extra IDB read)", async () => { + // Use a unique field name to avoid cache pollution from other tests + const freshCtx = { org: "cachetest", streamType: "logs", streamName: "mystream" }; + vi.mocked(fieldValueDB.getValues).mockResolvedValue(["prod", "staging"]); + + await getFieldValuesForSuggestion(freshCtx, "env"); + await getFieldValuesForSuggestion(freshCtx, "env"); + + // IDB should only be called once — second call served from cache + expect(fieldValueDB.getValues).toHaveBeenCalledTimes(1); + }); + + it("returns empty array when IDB throws", async () => { + vi.mocked(fieldValueDB.getValues).mockRejectedValueOnce(new Error("IDB error")); + const values = await getFieldValuesForSuggestion(ctx, "broken_field"); + expect(values).toEqual([]); + }); + + it("returns empty array when IDB returns empty", async () => { + vi.mocked(fieldValueDB.getValues).mockResolvedValueOnce([]); + const values = await getFieldValuesForSuggestion(ctx, "empty_field"); + expect(values).toEqual([]); + }); + + it("builds correct composite key from context", async () => { + const specificCtx = { org: "acme", streamType: "traces", streamName: "spans" }; + vi.mocked(fieldValueDB.getValues).mockResolvedValueOnce(["frontend"]); + await getFieldValuesForSuggestion(specificCtx, "service"); + expect(fieldValueDB.getValues).toHaveBeenCalledWith("acme|traces|spans|service"); + }); +}); diff --git a/web/src/composables/useFieldValueStore.ts b/web/src/composables/useFieldValueStore.ts new file mode 100644 index 0000000000..a8e4f3e88a --- /dev/null +++ b/web/src/composables/useFieldValueStore.ts @@ -0,0 +1,284 @@ +// Copyright 2026 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +/** + * Composable that orchestrates field value capture from both data sources: + * 1. Values API — field expansion in FieldList + * 2. Search result hits — Run Query results + * + * All IndexedDB writes are scheduled via requestIdleCallback so they have + * zero impact on main-thread rendering. Reads use an in-memory cache + * (1-minute TTL, 500-entry cap) so repeated keystrokes never hit IndexedDB. + */ + +import * as fieldValueDB from "@/composables/fieldValueDB"; +import { extractValuesFromHits } from "@/utils/fieldValueUtils"; + +// ─── Configuration ──────────────────────────────────────────────────────────── + +// Max unique values stored per field in IDB. 50 is enough for all realistic +// observability filter fields (status codes, env names, log levels). More than +// 50 items in a dropdown is noise rather than help. Overridable via .env so +// team can tune without a code change. +const MAX_VALUES_PER_FIELD = Number( + (import.meta as any).env?.VITE_MAX_FIELD_VALUES ?? 50, +); + +// Total IDB record cap across all orgs/streams/fields combined. Prevents the +// browser's IndexedDB storage from growing forever across many sessions. +// 5000 records × ~50 values × ~20 chars avg ≈ ~5MB — well within browser limits. +// Overridable via .env. +const MAX_FIELDS_STORED = Number( + (import.meta as any).env?.VITE_MAX_FIELDS_STORED ?? 5000, +); + +// How long a field's values stay valid in IDB after the last write. +// Sliding window — every new write resets the 7-day clock. Fields from streams +// you stop querying disappear automatically after 7 days of inactivity. +// Overridable via .env (unit: days). +const FIELD_VALUE_TTL_MS = + Number((import.meta as any).env?.VITE_FIELD_VALUE_TTL_DAYS ?? 7) * + 86400 * + 1000; + +// ─── In-memory read cache ───────────────────────────────────────────────────── + +// Two-level cache: memory → IDB → []. +// Without this, every keystroke in the query editor (e.g. "status = ") would +// trigger an async IDB read (~1ms). With it, the first keystroke pays the IDB +// cost; all subsequent keystrokes in the same minute return from memory in ~0ms. +const readCache = new Map(); +const READ_CACHE_TTL_MS = 60_000; // 1 minute — balance between freshness and cost +const READ_CACHE_MAX_ENTRIES = 500; // ~10 streams × ~50 fields each; enough for any session + +// ─── Types ──────────────────────────────────────────────────────────────────── + +export interface StreamContext { + org: string; + streamType: string; // 'logs' | 'metrics' | 'traces' + streamName: string; +} + +// ─── Helpers ───────────────────────────────────────────────────────────────── + +// Composite string key used as the IDB primary key. +// Format: "org|streamType|streamName|fieldName" +// e.g. : "myorg|logs|http_logs|status" +// +// Why a flat string instead of a compound key? +// IDB compound keys require an array and more complex range queries. +// A flat string is simpler, human-readable in DevTools, and works perfectly +// for our point-read access pattern. The "|" separator is safe because +// org/stream names in OO cannot contain "|". +const makeKey = (ctx: StreamContext, fieldName: string): string => + `${ctx.org}|${ctx.streamType}|${ctx.streamName}|${fieldName}`; + +/** + * Schedule a background write using requestIdleCallback. + * Falls back to setTimeout(0) in environments without rIC (e.g. Safari). + * + * Why this wrapper exists: + * - requestIdleCallback runs the work AFTER the browser has finished rendering + * the current frame — zero impact on the search results appearing on screen. + * - timeout: 2000 is a safety valve: if the browser stays busy for 2 full + * seconds without an idle slot, the write is forced anyway so data is never + * permanently lost. + * - Safari does not support requestIdleCallback. setTimeout(0) is the next + * best thing — it defers to the next event loop tick, which still avoids + * blocking the current render cycle. + * - The .catch() wrapper ensures IDB failures (private browsing, quota + * exceeded, IDB unavailable) never crash the app. Autocomplete is a + * nice-to-have, not a critical feature. Errors are logged in dev only. + */ +const scheduleWrite = (fn: () => Promise): void => { + const wrapped = () => + fn().catch((err) => { + if (import.meta.env.DEV) { + console.debug("[fieldValueStore] write error:", err); + } + }); + + if (typeof requestIdleCallback !== "undefined") { + requestIdleCallback(wrapped, { timeout: 2000 }); + } else { + setTimeout(wrapped, 0); + } +}; + +/** + * Write to the in-memory read cache with a bounded size. + * + * Why evict 10% at a time instead of 1 entry? + * Evicting a single entry on every insert-at-cap means paying the eviction + * cost on every new entry once the map is full. Evicting 10% (50 entries) at + * once amortizes that cost — we only pay it once every 50 inserts. + */ +const setCacheEntry = (key: string, values: string[]): void => { + readCache.set(key, { values, ts: Date.now() }); + if (readCache.size > READ_CACHE_MAX_ENTRIES) { + const evict = Math.ceil(READ_CACHE_MAX_ENTRIES * 0.1); + const iter = readCache.keys(); + for (let i = 0; i < evict; i++) { + const { value, done } = iter.next(); + if (done) break; + readCache.delete(value); + } + } +}; + +// ─── Public API ─────────────────────────────────────────────────────────────── + +/** + * Capture values returned by the Values API (field expansion in FieldList). + * Called from useFieldValuesStream.handleResponse() AFTER the existing UI + * state update — returns synchronously in ~1µs. + * + * Why 200-char filter here (vs 150 in extractValuesFromHits)? + * The Values API returns pre-selected top values from the server — they have + * already been ranked and filtered for relevance. The 200-char limit is just + * a safety net against malformed responses. We don't need the aggressive + * filtering used for raw log hits where any field can appear. + * + * Why delete readCache after write? + * If the user had already triggered autocomplete for this field (cache was + * populated), the write would update IDB but the stale cache entry would be + * returned for the next 60 seconds. Deleting it forces a fresh IDB read the + * next time the user types, ensuring they see the newly captured values. + */ +export const captureFromValuesApi = ( + ctx: StreamContext, + fieldName: string, + entries: Array<{ key: string }>, +): void => { + const values = entries + .map((e) => String(e.key)) + .filter((v) => v.length > 0 && v.length <= 200); + + if (values.length === 0) return; + + const key = makeKey(ctx, fieldName); + scheduleWrite(async () => { + await fieldValueDB.mergeValues( + key, + values, + "values_api", + MAX_VALUES_PER_FIELD, + FIELD_VALUE_TTL_MS, + ); + readCache.delete(key); + }); +}; + +/** + * Capture field values from search result hits. + * Called from useStreamFields.updateFieldValues() after existing UI update. + * Limits to first 100 hits; skips high-cardinality and excluded fields. + * Uses a single IDB transaction for all fields to minimise transaction overhead. + * + * Why hits.slice(0, 100) even though the logs page already caps at 100? + * Defensive programming — this function is a public export and could be called + * by future code that passes a larger array. The slice keeps the contract clear. + * + * Why one mergeMultipleValues call instead of N mergeValues calls? + * Each IDB transaction has a fixed overhead (open, commit, IDB lock). If a + * stream has 40 fields, 40 individual transactions = 40× that overhead. + * One transaction with 40 operations = 1× overhead. Substantially cheaper + * for wide schemas. + * + * Why 5% chance for housekeeping (evictExpired + trimToMaxFields)? + * evictExpired scans the entire IDB by_expires index — on 5000 records that + * takes 10–20ms even in an idle callback. Running it on every search wastes + * that time when 99% of searches produce no newly expired records. + * 1-in-20 gives periodic cleanup with 1/20th the cumulative cost. + */ +export const captureFromSearchHits = ( + ctx: StreamContext, + hits: Record[], + schemaFields: string[], +): void => { + // Early return avoids scheduling a no-op idle callback + if (!hits || hits.length === 0) return; + + scheduleWrite(async () => { + const extracted = extractValuesFromHits( + hits.slice(0, 100), + schemaFields, + MAX_VALUES_PER_FIELD, + ); + + const entries = Object.entries(extracted).map(([field, values]) => ({ + key: makeKey(ctx, field), + values, + source: "search_hits" as const, + })); + + if (entries.length === 0) return; + + // Single transaction for all fields — N times cheaper than Promise.all(N writes) + await fieldValueDB.mergeMultipleValues( + entries, + MAX_VALUES_PER_FIELD, + FIELD_VALUE_TTL_MS, + ); + + // Invalidate read-cache for written keys so the next autocomplete read + // returns fresh values instead of the pre-write stale cache entry + for (const { key } of entries) readCache.delete(key); + + // Opportunistically sweep — 5% chance per capture so cost is amortized + if (Math.random() < 0.05) { + await fieldValueDB.evictExpired(); + await fieldValueDB.trimToMaxFields(MAX_FIELDS_STORED); + } + }); +}; + +/** + * Read stored values for a field — used by autocomplete suggestion provider. + * Uses in-memory cache (1-minute TTL, 500-entry cap) to avoid IndexedDB + * round-trips during typing. Never throws — returns empty array on any failure. + * + * Why return [] on error instead of propagating? + * IDB can be unavailable in private browsing mode or certain enterprise + * browser configurations. Users in those environments should still be able to + * use the query editor — they just won't see value suggestions. Crashing or + * showing an error for a non-critical enhancement would be a worse experience. + * + * Why check Date.now() - cached.ts < TTL instead of storing expiresAt? + * Minor: computing expiresAt = Date.now() + TTL at write time and comparing + * at read time is equivalent. The ts approach is slightly more readable + * because the TTL constant appears only in one place (READ_CACHE_TTL_MS). + */ +export const getFieldValuesForSuggestion = async ( + ctx: StreamContext, + fieldName: string, +): Promise => { + const key = makeKey(ctx, fieldName); + const cached = readCache.get(key); + if (cached) { + if (Date.now() - cached.ts < READ_CACHE_TTL_MS) { + return cached.values; + } + // Expired — remove so we don't leak memory + readCache.delete(key); + } + try { + const values = await fieldValueDB.getValues(key); + setCacheEntry(key, values); + return values; + } catch { + return []; + } +}; diff --git a/web/src/composables/useFieldValuesStream.ts b/web/src/composables/useFieldValuesStream.ts index 6a9d84ce4f..90ffad2f8e 100644 --- a/web/src/composables/useFieldValuesStream.ts +++ b/web/src/composables/useFieldValuesStream.ts @@ -17,6 +17,7 @@ import { ref, type Ref } from "vue"; import store from "@/stores"; import { generateTraceContext } from "@/utils/zincutils"; import useHttpStreaming from "@/composables/useStreamingSearch"; +import { captureFromValuesApi } from "@/composables/useFieldValueStore"; export interface FieldValueEntry { key: string; @@ -159,6 +160,19 @@ const useFieldValuesStream = () => { }); }); + // [NEW] Background capture — does not block handleResponse return + if (chunkValues.length > 0 && fieldName) { + captureFromValuesApi( + { + org: store.state.selectedOrganization.identifier, + streamType: payload?.queryReq?.stream_type ?? "logs", + streamName: streamName ?? "", + }, + fieldName, + chunkValues, + ); + } + // Append in paginated (load-more) mode; replace on fresh load. if (isAppend) { streamFieldValues.value[fieldName][streamName].values = [ diff --git a/web/src/composables/useLogs/useStreamFields.ts b/web/src/composables/useLogs/useStreamFields.ts index c253ef8170..ae15985d3e 100644 --- a/web/src/composables/useLogs/useStreamFields.ts +++ b/web/src/composables/useLogs/useStreamFields.ts @@ -22,6 +22,7 @@ import { useRouter } from "vue-router"; import { searchState } from "@/composables/useLogs/searchState"; import useStreams from "@/composables/useStreams"; import useSqlSuggestions from "@/composables/useSuggestions"; +import { captureFromSearchHits } from "@/composables/useFieldValueStore"; import { useLocalLogFilterField, @@ -53,13 +54,34 @@ export const useStreamFields = () => { const updateFieldValues = () => { try { + const hits = searchObj.data.queryResults.hits; + + // [NEW] Background capture into IndexedDB — must be placed BEFORE the + // loop below because that loop contains an early `return` when it + // encounters excluded fields (pre-existing behaviour), which would + // prevent this code from running if placed after. + if (hits?.length > 0) { + const schemaFields = ( + searchObj.data.stream.selectedStreamFields ?? [] + ).map((f: any) => f.name ?? f); + captureFromSearchHits( + { + org: searchObj.organizationIdentifier, + streamType: searchObj.data.stream.streamType ?? "logs", + streamName: searchObj.data.stream.selectedStream?.[0] ?? "", + }, + hits, + schemaFields, + ); + } + const excludedFields = [ store.state.zoConfig.timestamp_column, "log", "msg", ]; // searchObj.data.queryResults.hits.forEach((item: { [x: string]: any }) => { - for (const item of searchObj.data.queryResults.hits) { + for (const item of hits) { // Create set for each field values and add values to corresponding set // Object.keys(item).forEach((key) => { for (const key of Object.keys(item)) { diff --git a/web/src/composables/useSuggestions.spec.ts b/web/src/composables/useSuggestions.spec.ts index 23838b197b..668eca0c3f 100644 --- a/web/src/composables/useSuggestions.spec.ts +++ b/web/src/composables/useSuggestions.spec.ts @@ -1,423 +1,307 @@ -// Copyright 2023 OpenObserve Inc. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - import { describe, it, expect, vi, beforeEach } from "vitest"; -// --- mocks before imports --- +// ─── Mock Vuex useStore ─────────────────────────────────────────────────────── +vi.mock("vuex", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + useStore: vi.fn(() => ({ + state: { zoConfig: { timestamp_column: "_timestamp" } }, + })), + }; +}); -vi.mock("vuex", () => ({ - useStore: vi.fn(() => ({ - state: { - zoConfig: { timestamp_column: "_timestamp" }, - }, - })), +// ─── Mock IDB so no real storage is touched ─────────────────────────────────── +vi.mock("@/composables/useFieldValueStore", () => ({ + getFieldValuesForSuggestion: vi.fn().mockResolvedValue([]), })); -import useSuggestions from "./useSuggestions"; +import { getFieldValuesForSuggestion } from "@/composables/useFieldValueStore"; +import useSqlSuggestions from "./useSuggestions"; -// --------------------------------------------------------------------------- +// ─── helper: build composable with common defaults ──────────────────────────── +const makeComposable = (overrides: { + storedValues?: string[]; + inSessionValues?: Record; +} = {}) => { + const { storedValues = [], inSessionValues = {} } = overrides; + vi.mocked(getFieldValuesForSuggestion).mockResolvedValue(storedValues); -describe("useSuggestions", () => { - beforeEach(() => { - vi.clearAllMocks(); + const c = useSqlSuggestions(); + c.autoCompleteData.value.org = "myorg"; + c.autoCompleteData.value.streamType = "logs"; + c.autoCompleteData.value.streamName = "http_logs"; + c.autoCompleteData.value.fieldValues = Object.fromEntries( + Object.entries(inSessionValues).map(([k, v]) => [k, new Set(v)]), + ); + c.autoCompleteData.value.popup.open = vi.fn(); + return c; +}; + +const run = async ( + c: ReturnType, + query: string, + cursorIndex?: number, +) => { + c.autoCompleteData.value.query = query; + (c.autoCompleteData.value as any).cursorIndex = cursorIndex ?? query.length; + await c.getSuggestions(); + return c.autoCompleteKeywords.value; +}; + +// ─── operator detection ─────────────────────────────────────────────────────── + +describe("analyzeSqlWhereClause — operator detection", () => { + beforeEach(() => vi.clearAllMocks()); + + const operators: [string, string][] = [ + ["=", "status = "], + ["!=", "status != "], + ["<>", "status <> "], + [">=", "code >= "], + ["<=", "code <= "], + [">", "code > "], + ["<", "code < "], + ]; + + it.each(operators)("detects field %s operator", async (_op, query) => { + const c = makeComposable({ storedValues: ["200"] }); + const keywords = await run(c, query); + expect(keywords.some((k: any) => k.kind === "Value")).toBe(true); }); - // ------------------------------------------------------------------------- - // Return value structure - // ------------------------------------------------------------------------- - describe("return value structure", () => { - it("returns all expected properties", () => { - const result = useSuggestions(); - - expect(result).toHaveProperty("autoCompleteData"); - expect(result).toHaveProperty("autoCompleteKeywords"); - expect(result).toHaveProperty("autoCompleteSuggestions"); - expect(result).toHaveProperty("loading"); - expect(result).toHaveProperty("getSuggestions"); - expect(result).toHaveProperty("updateFieldKeywords"); - expect(result).toHaveProperty("updateFunctionKeywords"); - expect(result).toHaveProperty("defaultSuggestions"); - }); - - it("loading is a ref defaulting to false", () => { - const { loading } = useSuggestions(); - expect(loading.value).toBe(false); - }); - - it("autoCompleteKeywords starts as an empty array", () => { - const { autoCompleteKeywords } = useSuggestions(); - expect(autoCompleteKeywords.value).toEqual([]); - }); - - it("autoCompleteSuggestions starts as an empty array", () => { - const { autoCompleteSuggestions } = useSuggestions(); - expect(autoCompleteSuggestions.value).toEqual([]); - }); + it("detects IN (", async () => { + const c = makeComposable({ storedValues: ["200"] }); + const q = "status IN ("; + const keywords = await run(c, q, q.length); + expect(keywords.some((k: any) => k.kind === "Value")).toBe(true); }); - // ------------------------------------------------------------------------- - // defaultSuggestions - // ------------------------------------------------------------------------- - describe("defaultSuggestions", () => { - it("is a non-empty array", () => { - const { defaultSuggestions } = useSuggestions(); - expect(Array.isArray(defaultSuggestions)).toBe(true); - expect(defaultSuggestions.length).toBeGreaterThan(0); - }); - - it("every entry has label, kind, and insertText properties", () => { - const { defaultSuggestions } = useSuggestions(); - defaultSuggestions.forEach((s: any) => { - expect(s).toHaveProperty("label"); - expect(s).toHaveProperty("kind"); - expect(s).toHaveProperty("insertText"); - }); - }); - - it("contains match_all suggestion", () => { - const { defaultSuggestions } = useSuggestions(); - const hasMatchAll = defaultSuggestions.some((s: any) => { - const label = - typeof s.label === "function" ? s.label("test") : s.label; - return label.includes("match_all"); - }); - expect(hasMatchAll).toBe(true); - }); - - it("contains str_match suggestion", () => { - const { defaultSuggestions } = useSuggestions(); - const has = defaultSuggestions.some((s: any) => { - const label = - typeof s.label === "function" ? s.label("test") : s.label; - return label.includes("str_match"); - }); - expect(has).toBe(true); - }); - - it("contains histogram suggestion", () => { - const { defaultSuggestions } = useSuggestions(); - const has = defaultSuggestions.some((s: any) => { - const label = - typeof s.label === "function" ? s.label("field") : s.label; - return label.includes("histogram"); - }); - expect(has).toBe(true); - }); - - it("function-label entries produce correct output when called with a keyword", () => { - const { defaultSuggestions } = useSuggestions(); - const matchAll = defaultSuggestions.find((s: any) => { - const label = - typeof s.label === "function" ? s.label("") : s.label; - return label.startsWith("match_all("); - }); - expect(matchAll).toBeDefined(); - expect(typeof matchAll!.label).toBe("function"); - expect((matchAll!.label as Function)("hello")).toBe("match_all('hello')"); - }); + it("detects NOT IN (", async () => { + const c = makeComposable({ storedValues: ["200"] }); + const q = "status NOT IN ("; + const keywords = await run(c, q, q.length); + expect(keywords.some((k: any) => k.kind === "Value")).toBe(true); }); - // ------------------------------------------------------------------------- - // updateFieldKeywords - // ------------------------------------------------------------------------- - describe("updateFieldKeywords", () => { - it("populates autoCompleteKeywords with field entries", () => { - const { updateFieldKeywords, autoCompleteKeywords } = useSuggestions(); - - updateFieldKeywords([ - { name: "level" }, - { name: "service_name" }, - ]); - - const fieldLabels = autoCompleteKeywords.value - .filter((k: any) => k.kind === "Field") - .map((k: any) => k.label); - - expect(fieldLabels).toContain("level"); - expect(fieldLabels).toContain("service_name"); - }); - - it("excludes the timestamp_column from field keywords", () => { - const { updateFieldKeywords, autoCompleteKeywords } = useSuggestions(); - - updateFieldKeywords([{ name: "_timestamp" }, { name: "level" }]); - - const fieldLabels = autoCompleteKeywords.value - .filter((k: any) => k.kind === "Field") - .map((k: any) => k.label); - - expect(fieldLabels).not.toContain("_timestamp"); - expect(fieldLabels).toContain("level"); - }); - - it("sets kind to Field for each field entry", () => { - const { updateFieldKeywords, autoCompleteKeywords } = useSuggestions(); - - updateFieldKeywords([{ name: "host" }]); - - const fieldEntry = autoCompleteKeywords.value.find( - (k: any) => k.label === "host", - ); - expect(fieldEntry?.kind).toBe("Field"); - }); - - it("sets insertTextRules to InsertAsSnippet for field entries", () => { - const { updateFieldKeywords, autoCompleteKeywords } = useSuggestions(); - - updateFieldKeywords([{ name: "host" }]); - - const fieldEntry = autoCompleteKeywords.value.find( - (k: any) => k.label === "host", - ); - expect(fieldEntry?.insertTextRules).toBe("InsertAsSnippet"); - }); - - it("also appends default SQL keywords to autoCompleteKeywords", () => { - const { updateFieldKeywords, autoCompleteKeywords } = useSuggestions(); - - updateFieldKeywords([{ name: "level" }]); - - const keywordLabels = autoCompleteKeywords.value - .filter((k: any) => k.kind === "Keyword") - .map((k: any) => k.label); - - expect(keywordLabels).toContain("and"); - expect(keywordLabels).toContain("or"); - }); - - it("handles empty fields array without error", () => { - const { updateFieldKeywords, autoCompleteKeywords } = useSuggestions(); - - expect(() => updateFieldKeywords([])).not.toThrow(); - // Only default keywords and operators present (no Field entries) - const fieldEntries = autoCompleteKeywords.value.filter( - (k: any) => k.kind === "Field", - ); - expect(fieldEntries.length).toBe(0); - }); + it("detects LIKE (space)", async () => { + const c = makeComposable({ storedValues: ["api"] }); + const keywords = await run(c, "msg LIKE "); + expect(keywords.some((k: any) => k.kind === "Value")).toBe(true); }); - // ------------------------------------------------------------------------- - // updateFunctionKeywords - // ------------------------------------------------------------------------- - describe("updateFunctionKeywords", () => { - it("adds function entries with kind Function", () => { - const { updateFunctionKeywords, autoCompleteKeywords } = useSuggestions(); - - updateFunctionKeywords([{ name: "myFunc", args: "(x)" }]); - - const funcEntry = autoCompleteKeywords.value.find( - (k: any) => k.kind === "Function", - ); - expect(funcEntry?.label).toBe("myFunc"); - }); - - it("concatenates name and args for insertText", () => { - const { updateFunctionKeywords, autoCompleteKeywords } = useSuggestions(); - - updateFunctionKeywords([{ name: "transform", args: "(field, value)" }]); - - const funcEntry = autoCompleteKeywords.value.find( - (k: any) => k.label === "transform", - ); - expect(funcEntry?.insertText).toBe("transform(field, value)"); - }); - - it("handles empty functions array without error", () => { - const { updateFunctionKeywords, autoCompleteKeywords } = useSuggestions(); - - expect(() => updateFunctionKeywords([])).not.toThrow(); - const funcEntries = autoCompleteKeywords.value.filter( - (k: any) => k.kind === "Function", - ); - expect(funcEntries.length).toBe(0); - }); - - it("replaces previously set function keywords on second call", () => { - const { updateFunctionKeywords, autoCompleteKeywords } = useSuggestions(); - - updateFunctionKeywords([{ name: "funcA", args: "()" }]); - updateFunctionKeywords([{ name: "funcB", args: "()" }]); - - const funcLabels = autoCompleteKeywords.value - .filter((k: any) => k.kind === "Function") - .map((k: any) => k.label); - - expect(funcLabels).not.toContain("funcA"); - expect(funcLabels).toContain("funcB"); - }); + it("detects LIKE with open quote", async () => { + const c = makeComposable({ storedValues: ["api"] }); + const q = "msg LIKE '"; + const keywords = await run(c, q, q.length); + expect(keywords.some((k: any) => k.kind === "Value")).toBe(true); }); - // ------------------------------------------------------------------------- - // getSuggestions — field-value path - // ------------------------------------------------------------------------- - describe("getSuggestions – field-value lookup", () => { - it("populates autoCompleteKeywords with field values when column=value pattern is detected", async () => { - const { - getSuggestions, - autoCompleteData, - autoCompleteKeywords, - } = useSuggestions(); - - // Simulate SQL like: "level = " with cursor at end - autoCompleteData.value.query = "level = "; - autoCompleteData.value.position.cursorIndex = 7; - autoCompleteData.value.fieldValues = { - level: new Set(["error", "warn", "info"]), - }; - autoCompleteData.value.popup = { - open: vi.fn(), - close: vi.fn(), - }; - - await getSuggestions(); - - const valueLabels = autoCompleteKeywords.value.map((k: any) => k.label); - expect(valueLabels).toContain("error"); - expect(valueLabels).toContain("warn"); - expect(valueLabels).toContain("info"); - }); - - it("wraps field values in single quotes for insertText", async () => { - const { - getSuggestions, - autoCompleteData, - autoCompleteKeywords, - } = useSuggestions(); - - autoCompleteData.value.query = "level = "; - autoCompleteData.value.position.cursorIndex = 7; - autoCompleteData.value.fieldValues = { - level: new Set(["error"]), - }; - autoCompleteData.value.popup = { open: vi.fn(), close: vi.fn() }; - - await getSuggestions(); - - const entry = autoCompleteKeywords.value.find( - (k: any) => k.label === "error", - ); - expect(entry?.insertText).toBe("'error'"); - }); - - it("calls popup.open after populating field-value keywords", async () => { - const { - getSuggestions, - autoCompleteData, - } = useSuggestions(); - - const mockOpen = vi.fn(); - autoCompleteData.value.query = "level = "; - autoCompleteData.value.position.cursorIndex = 7; - autoCompleteData.value.fieldValues = { - level: new Set(["error"]), - }; - autoCompleteData.value.popup = { open: mockOpen, close: vi.fn() }; - - await getSuggestions(); - - expect(mockOpen).toHaveBeenCalledWith("level = "); - }); - - it("clears autoCompleteSuggestions when field-value path is taken", async () => { - const { - getSuggestions, - autoCompleteData, - autoCompleteSuggestions, - } = useSuggestions(); - - autoCompleteData.value.query = "level = "; - autoCompleteData.value.position.cursorIndex = 7; - autoCompleteData.value.fieldValues = { - level: new Set(["error"]), - }; - autoCompleteData.value.popup = { open: vi.fn(), close: vi.fn() }; - - await getSuggestions(); - - expect(autoCompleteSuggestions.value).toEqual([]); - }); + it("detects NOT LIKE", async () => { + const c = makeComposable({ storedValues: ["api"] }); + const q = "path NOT LIKE '"; + const keywords = await run(c, q, q.length); + expect(keywords.some((k: any) => k.kind === "Value")).toBe(true); }); - // ------------------------------------------------------------------------- - // getSuggestions — general / non-field-value path - // ------------------------------------------------------------------------- - describe("getSuggestions – general path", () => { - it("falls through to general suggestions when no field-value match", async () => { - const { - getSuggestions, - autoCompleteData, - autoCompleteKeywords, - autoCompleteSuggestions, - } = useSuggestions(); - - autoCompleteData.value.query = "some random query"; - autoCompleteData.value.position.cursorIndex = 17; - autoCompleteData.value.fieldValues = {}; - autoCompleteData.value.popup = { open: vi.fn(), close: vi.fn() }; - - await getSuggestions(); - - // Should populate with default keywords (and, or, etc.) - const keywordLabels = autoCompleteKeywords.value - .filter((k: any) => k.kind === "Keyword") - .map((k: any) => k.label); - expect(keywordLabels).toContain("and"); - // Should also populate autoCompleteSuggestions with function suggestions - expect(autoCompleteSuggestions.value.length).toBeGreaterThan(0); - }); - - it("includes operator suggestions (<, >, =) in general path", async () => { - const { - getSuggestions, - autoCompleteData, - autoCompleteKeywords, - } = useSuggestions(); - - autoCompleteData.value.query = "query text"; - autoCompleteData.value.position.cursorIndex = 10; - autoCompleteData.value.fieldValues = {}; - autoCompleteData.value.popup = { open: vi.fn(), close: vi.fn() }; - - await getSuggestions(); - - const operatorLabels = autoCompleteKeywords.value - .filter((k: any) => k.kind === "Operator") - .map((k: any) => k.label); - expect(operatorLabels).toContain("="); - expect(operatorLabels).toContain("!="); - }); + it("detects str_match(field, )", async () => { + const c = makeComposable({ storedValues: ["frontend"] }); + const q = "str_match(service, '"; + const keywords = await run(c, q, q.length); + expect(keywords.some((k: any) => k.kind === "Value")).toBe(true); }); - // ------------------------------------------------------------------------- - // autoCompleteData default structure - // ------------------------------------------------------------------------- - describe("autoCompleteData default structure", () => { - it("has correct initial fieldValues, query, and position", () => { - const { autoCompleteData } = useSuggestions(); + it("detects fuzzy_match(field, )", async () => { + const c = makeComposable({ storedValues: ["frontend"] }); + const q = "fuzzy_match(service, '"; + const keywords = await run(c, q, q.length); + expect(keywords.some((k: any) => k.kind === "Value")).toBe(true); + }); - expect(autoCompleteData.value.fieldValues).toEqual({}); - expect(autoCompleteData.value.query).toBe(""); - expect(autoCompleteData.value.position.cursorIndex).toBe(0); - }); + it("handles auto-closed bracket IN () — cursor between ( and )", async () => { + const c = makeComposable({ storedValues: ["200"] }); + const full = "status IN ()"; + // cursor is between ( and ) — Monaco offset 11, getCursorIndex = offset-1 = 10. + // slice(0, 10+1) = "status IN (" which matches the IN regex. + const keywords = await run(c, full, 10); + expect(keywords.some((k: any) => k.kind === "Value")).toBe(true); + }); - it("popup.open and popup.close are functions by default", () => { - const { autoCompleteData } = useSuggestions(); + it("detects partial typed value after operator: field = '20", async () => { + const c = makeComposable({ storedValues: ["200"] }); + const q = "status = '20"; + const keywords = await run(c, q, q.length); + expect(keywords.some((k: any) => k.kind === "Value")).toBe(true); + }); - expect(typeof autoCompleteData.value.popup.open).toBe("function"); - expect(typeof autoCompleteData.value.popup.close).toBe("function"); + it("falls through to keywords when no operator context", async () => { + const c = makeComposable({ storedValues: [] }); + const keywords = await run(c, "SELECT * FROM stream WHERE "); + expect(keywords.some((k: any) => k.kind === "Value")).toBe(false); + expect(keywords.length).toBeGreaterThan(0); + }); + + it("shows default keywords for empty query", async () => { + const c = makeComposable({ storedValues: [] }); + const keywords = await run(c, ""); + expect(keywords.some((k: any) => k.kind === "Value")).toBe(false); + expect(keywords.length).toBeGreaterThan(0); + }); +}); + +// ─── insertText quoting logic ───────────────────────────────────────────────── + +describe("getSuggestions — insertText quoting", () => { + beforeEach(() => vi.clearAllMocks()); + + it("wraps string value in single quotes when no open quote", async () => { + const c = makeComposable({ storedValues: ["prod"] }); + const keywords = await run(c, "env = "); + const item = keywords.find((k: any) => k.label === "prod"); + expect(item?.insertText).toBe("'prod'"); + }); + + it("closes only when open quote already typed", async () => { + const c = makeComposable({ storedValues: ["prod"] }); + const q = "env = '"; + const keywords = await run(c, q, q.length); + const item = keywords.find((k: any) => k.label === "prod"); + expect(item?.insertText).toBe("prod'"); + }); + + it("inserts numeric values without quotes", async () => { + const c = makeComposable({ storedValues: ["200", "404"] }); + const keywords = await run(c, "status = "); + const item = keywords.find((k: any) => k.label === "200"); + expect(item?.insertText).toBe("200"); + }); + + it("inserts boolean 'true' without quotes", async () => { + const c = makeComposable({ storedValues: ["true"] }); + const keywords = await run(c, "active = "); + const item = keywords.find((k: any) => k.label === "true"); + expect(item?.insertText).toBe("true"); + }); + + it("inserts boolean 'false' without quotes", async () => { + const c = makeComposable({ storedValues: ["false"] }); + const keywords = await run(c, "active = "); + const item = keywords.find((k: any) => k.label === "false"); + expect(item?.insertText).toBe("false"); + }); + + it("sortText starts with \\x01 so values sort above keywords", async () => { + const c = makeComposable({ storedValues: ["200"] }); + const keywords = await run(c, "status = "); + const item = keywords.find((k: any) => k.kind === "Value"); + expect(item?.sortText.startsWith("\x01")).toBe(true); + }); + + it("all value suggestions have kind = 'Value'", async () => { + const c = makeComposable({ storedValues: ["200", "404", "500"] }); + const keywords = await run(c, "status = "); + const valueItems = keywords.filter((k: any) => k.kind === "Value"); + expect(valueItems).toHaveLength(3); + }); +}); + +// ─── merge: in-session vs stored ───────────────────────────────────────────── + +describe("getSuggestions — in-session and stored value merge", () => { + beforeEach(() => vi.clearAllMocks()); + + it("shows in-session values from fieldValues prop", async () => { + const c = makeComposable({ + storedValues: [], + inSessionValues: { status: ["200", "404"] }, }); + const keywords = await run(c, "status = "); + const labels = keywords.map((k: any) => k.label); + expect(labels).toContain("200"); + expect(labels).toContain("404"); + }); + + it("merges stored values with in-session values", async () => { + const c = makeComposable({ + storedValues: ["500"], + inSessionValues: { status: ["200"] }, + }); + const keywords = await run(c, "status = "); + const labels = keywords.map((k: any) => k.label); + expect(labels).toContain("200"); + expect(labels).toContain("500"); + }); + + it("deduplicates values appearing in both sources", async () => { + const c = makeComposable({ + storedValues: ["200", "500"], + inSessionValues: { status: ["200", "404"] }, + }); + const keywords = await run(c, "status = "); + const labels = keywords.map((k: any) => k.label); + expect(labels.filter((l: string) => l === "200")).toHaveLength(1); + expect(labels).toContain("404"); + expect(labels).toContain("500"); + }); + + it("skips IDB read when stream context is missing", async () => { + const c = useSqlSuggestions(); + // org/streamType/streamName left as empty strings + c.autoCompleteData.value.fieldValues = {}; + c.autoCompleteData.value.popup.open = vi.fn(); + await run(c, "status = "); + expect(getFieldValuesForSuggestion).not.toHaveBeenCalled(); + }); + + it("falls through to updateAutoComplete when merged is empty", async () => { + const c = makeComposable({ storedValues: [] }); + const keywords = await run(c, "status = "); + expect(keywords.some((k: any) => k.kind === "Value")).toBe(false); + expect(keywords.length).toBeGreaterThan(0); + }); +}); + +// ─── IDB context forwarding ─────────────────────────────────────────────────── + +describe("getSuggestions — IDB context forwarding", () => { + it("passes org/streamType/streamName to getFieldValuesForSuggestion", async () => { + vi.mocked(getFieldValuesForSuggestion).mockResolvedValue(["prod"]); + const c = useSqlSuggestions(); + c.autoCompleteData.value.org = "acme"; + c.autoCompleteData.value.streamType = "traces"; + c.autoCompleteData.value.streamName = "default"; + c.autoCompleteData.value.fieldValues = {}; + c.autoCompleteData.value.popup.open = vi.fn(); + await run(c, "env = "); + expect(getFieldValuesForSuggestion).toHaveBeenCalledWith( + { org: "acme", streamType: "traces", streamName: "default" }, + "env", + ); + }); +}); + +// ─── updateFieldKeywords ────────────────────────────────────────────────────── + +describe("updateFieldKeywords", () => { + it("adds field keywords excluding timestamp column", () => { + const c = useSqlSuggestions(); + c.updateFieldKeywords([ + { name: "status" }, + { name: "env" }, + { name: "_timestamp" }, + ]); + const labels = c.autoCompleteKeywords.value.map((k: any) => k.label); + expect(labels).toContain("status"); + expect(labels).toContain("env"); + expect(labels).not.toContain("_timestamp"); + }); + + it("sets kind = Field for all field keywords", () => { + const c = useSqlSuggestions(); + c.updateFieldKeywords([{ name: "status" }, { name: "env" }]); + const fieldItems = c.autoCompleteKeywords.value.filter( + (k: any) => k.kind === "Field", + ); + expect(fieldItems.length).toBeGreaterThanOrEqual(2); }); }); diff --git a/web/src/composables/useSuggestions.ts b/web/src/composables/useSuggestions.ts index 5276cf432b..ad2521b841 100644 --- a/web/src/composables/useSuggestions.ts +++ b/web/src/composables/useSuggestions.ts @@ -1,5 +1,6 @@ import { ref, nextTick } from "vue"; import { useStore } from "vuex"; +import { getFieldValuesForSuggestion } from "@/composables/useFieldValueStore"; const useSqlSuggestions = () => { const defaultKeywords = [ @@ -285,6 +286,12 @@ const useSqlSuggestions = () => { open: (val: string) => {}, close: (val: string) => {}, }, + // [NEW] Stream context — set by SearchBar.vue when a stream is selected. + // Required to build the composite IDB key: "org|streamType|streamName|fieldName". + // Without these three, getFieldValuesForSuggestion cannot look up stored values. + org: "", + streamType: "", + streamName: "", }); const autoCompleteSuggestions: any = ref([]); const loading = ref(false); @@ -302,52 +309,153 @@ const useSqlSuggestions = () => { meta: { label: "", value: "", + hasOpenQuote: false, }, }; - const columnValueRegex = /(\w+)\s*=\s*$|\w+\s+IN\s+\($/i; + // Detects whether the cursor is positioned after an operator that expects + // a value, and extracts the field name to the left of that operator. + // + // 4 alternatives — each captures the field name in a different group: + // match[1]: symbolic operators = != <> >= <= > < + // e.g. "status = ", "code >= ", "env != 'pro" + // match[2]: IN / NOT IN ( + // e.g. "status IN (", "env NOT IN ('pro" + // match[3]: LIKE / NOT LIKE + // e.g. "msg LIKE '", "path NOT LIKE '%api" + // match[4]: str_match / fuzzy_match function second argument + // e.g. "str_match(field, ", "fuzzy_match(field, 'par" + // + // Why >=/<= appear before >/<: + // Regex alternation is left-to-right. If > appeared first, ">=" would + // match on ">" and stop, leaving "=" unmatched. Longer tokens must come first. + // + // Why (?:'[^']*)?$ at the end of each alternative: + // Allows the regex to match even after the user has typed an opening quote + // and a partial value. Without it, "status = 'pro" would not match — we + // would stop showing value suggestions the moment the user starts typing. + const columnValueRegex = + /(\w+)\s*(?:!=|<>|>=|<=|=|>|<)\s*(?:'[^']*)?$|(\w+)\s+(?:NOT\s+)?IN\s+\(\s*(?:'[^']*)?$|(\w+)\s+(?:NOT\s+)?LIKE\s*(?:'[^']*)?$|(?:str_match|fuzzy_match)\s*\(\s*(\w+)\s*,\s*(?:'[^']*)?$/i; - const match = columnValueRegex.exec(whereClause); - if (match !== null) { - const column = match[1]; - - if (cursorIndex <= match.index + match[0].length - 1) { - labelMeta.focusOn = "value"; - labelMeta.isFocused = true; - labelMeta.meta.label = column; - } + // Slice the query at the cursor position before matching, so that the $ + // anchor lands at the cursor — not at the end of the full query string. + // + // Why this matters — auto-closing brackets example: + // User types "status IN (" → editor auto-inserts ")" → full string is "status IN ()" + // Without slicing: $ anchors after ")" → regex does NOT match + // After slicing at cursor (between "(" and ")"): text is "status IN (" → matches + // + // cursorIndex = 0 means uninitialized (position.cursorIndex default). + // Fall back to the full string length so we don't break the pre-existing + // behaviour where no cursor tracking was done. + const endIdx = cursorIndex > 0 ? cursorIndex + 1 : whereClause.length; + const textUpToCursor = whereClause.slice(0, endIdx); + const match = columnValueRegex.exec(textUpToCursor); + if (match) { + labelMeta.focusOn = "value"; + labelMeta.isFocused = true; + // Pick whichever capture group matched — only one will be non-null. + labelMeta.meta.label = match[1] ?? match[2] ?? match[3] ?? match[4]; + // True when the user has already typed an opening quote, e.g. field = 'partial + // In this case insertText should be value' (close only), not 'value' + labelMeta.meta.hasOpenQuote = /'[^']*$/.test(textUpToCursor); } return labelMeta; } const getSuggestions = async () => { autoCompleteKeywords.value = []; - const cursorIndex = autoCompleteData.value.position.cursorIndex; + + // SearchBar sets autoCompleteData.value.cursorIndex at the top level. + // autoCompleteData.value.position.cursorIndex is the legacy field — it + // is never updated by SearchBar and stays 0. We read the top-level one + // first and fall back to position.cursorIndex for safety. + const cursorIndex = + (autoCompleteData.value as any).cursorIndex ?? + autoCompleteData.value.position.cursorIndex; + + // Determine if the cursor is currently after an operator expecting a value. + // If so, sqlWhereClause.meta.label is the field name (e.g. "status"). const sqlWhereClause = analyzeSqlWhereClause( autoCompleteData.value.query, cursorIndex, ); - if ( - sqlWhereClause.meta.label && - autoCompleteData.value.fieldValues[sqlWhereClause.meta.label] - ) { - const values = Array.from( - autoCompleteData.value.fieldValues[sqlWhereClause.meta.label] || - new Set(), - ).map((item) => { - return { - label: item, - insertText: `'${item}'`, - kind: "Keyword", - }; - }); - autoCompleteKeywords.value = [...values]; - autoCompleteSuggestions.value = []; - autoCompleteData.value.popup.open(autoCompleteData.value.query); - } else { - updateAutoComplete(); + if (sqlWhereClause.meta.label) { + const fieldName = sqlWhereClause.meta.label; + + // [EXISTING] In-session values — collected from the current session's + // search result hits and stored in the reactive fieldValues prop. + // These are available immediately (no async) but disappear on page reload. + const inSessionValues = Array.from( + autoCompleteData.value.fieldValues[fieldName] || new Set(), + ) as string[]; + + // [NEW] Persisted values — read from IndexedDB (via in-memory cache). + // These survive page reloads and accumulate across multiple searches. + // Guard: only query IDB if stream context is set — without org/streamType/ + // streamName we cannot build the composite key and would get empty results. + let storedValues: string[] = []; + if ( + autoCompleteData.value.org && + autoCompleteData.value.streamType && + autoCompleteData.value.streamName + ) { + storedValues = await getFieldValuesForSuggestion( + { + org: autoCompleteData.value.org, + streamType: autoCompleteData.value.streamType, + streamName: autoCompleteData.value.streamName, + }, + fieldName, + ); + } + + // Merge in-session + stored, deduplicate via Set. + // inSessionValues come first so they appear at the top of the dropdown + // (they are from the current search context, most relevant). + // storedValues from previous sessions fill in anything not seen today. + const merged = [...new Set([...inSessionValues, ...storedValues])]; + + if (merged.length > 0) { + const hasOpenQuote = sqlWhereClause.meta.hasOpenQuote; + + // Build Monaco suggestion items with smart quoting and sort order. + autoCompleteKeywords.value = merged.map((item, idx) => { + const isNumeric = item !== "" && !isNaN(Number(item)); + const isBoolean = item === "true" || item === "false"; + + // Quoting rules: + // numeric / boolean → no quotes (SQL: status = 200, active = true) + // string, open quote already typed → close only (field = 'val → val') + // string, no open quote → wrap fully (field = → 'val') + let insertText: string; + if (isNumeric || isBoolean) { + insertText = item; + } else if (hasOpenQuote) { + insertText = `${item}'`; // user already typed the opening ' + } else { + insertText = `'${item}'`; + } + + // \x01 (ASCII 1) is the lowest-sorting printable character. + // Prefixing sortText with it ensures value suggestions always appear + // ABOVE keywords ("and", "or", "like") and functions in the Monaco + // dropdown, which sort by their label (starting with a letter > \x01). + // The padded index preserves the order of values as returned from IDB. + const sortText = `\x01${String(idx).padStart(6, "0")}`; + return { label: item, insertText, kind: "Value", sortText }; + }); + + autoCompleteData.value.popup.open(autoCompleteData.value.query); + // Return early — do NOT fall through to updateAutoComplete(). + // We don't want keywords/fields/functions mixed into a value dropdown. + return; + } } + + // No value context detected — show the default keywords, fields, functions. + updateAutoComplete(); }; const updateAutoComplete = () => { diff --git a/web/src/plugins/logs/IndexList.vue b/web/src/plugins/logs/IndexList.vue index 826b557e28..c7842461b5 100644 --- a/web/src/plugins/logs/IndexList.vue +++ b/web/src/plugins/logs/IndexList.vue @@ -195,6 +195,7 @@ import { useSearchBar } from "@/composables/useLogs/useSearchBar"; import { useSearchStream } from "@/composables/useLogs/useSearchStream"; import { searchState } from "@/composables/useLogs/searchState"; import { useStreamFields } from "@/composables/useLogs/useStreamFields"; +import { captureFromValuesApi } from "@/composables/useFieldValueStore"; interface Filter { fieldName: string; @@ -1546,6 +1547,19 @@ export default defineComponent({ }); }); + // [NEW] Background capture into IndexedDB — does not block return + if (streamValues.length > 0 && fieldName) { + captureFromValuesApi( + { + org: store.state.selectedOrganization.identifier, + streamType: searchObj.data.stream.streamType ?? "logs", + streamName: streamName ?? "", + }, + fieldName, + streamValues, + ); + } + // Append to existing stream values in paginated mode; replace on fresh load. if (isAppend) { const existing = diff --git a/web/src/plugins/logs/SearchBar.vue b/web/src/plugins/logs/SearchBar.vue index 30f9fb4016..7c032348e1 100644 --- a/web/src/plugins/logs/SearchBar.vue +++ b/web/src/plugins/logs/SearchBar.vue @@ -2780,6 +2780,13 @@ export default defineComponent({ autoCompleteData.value.fieldValues = props.fieldValues; autoCompleteData.value.popup.open = queryEditorRef?.value?.triggerAutoComplete; + // [NEW] Pass stream context for IndexedDB value lookups + autoCompleteData.value.org = + store.state.selectedOrganization.identifier; + autoCompleteData.value.streamType = + searchObj.data.stream.streamType ?? "logs"; + autoCompleteData.value.streamName = + searchObj.data.stream.selectedStream?.[0] ?? ""; getSuggestions(); }; diff --git a/web/src/plugins/traces/SearchBar.vue b/web/src/plugins/traces/SearchBar.vue index 71c09189b1..562b25e42f 100644 --- a/web/src/plugins/traces/SearchBar.vue +++ b/web/src/plugins/traces/SearchBar.vue @@ -496,6 +496,15 @@ export default defineComponent({ autoCompleteData.value.fieldValues = props.fieldValues; autoCompleteData.value.popup.open = queryEditorRef.value.triggerAutoComplete; + // [NEW] Set stream context so getSuggestions can read stored values from + // IndexedDB. Traces field expansion already writes to IDB via + // captureFromValuesApi (useFieldValuesStream) with stream_type="traces", + // so values are already being captured — this just enables the read side. + autoCompleteData.value.org = + store.state.selectedOrganization.identifier; + autoCompleteData.value.streamType = "traces"; + autoCompleteData.value.streamName = + searchObj.data.stream.selectedStream.value ?? ""; getSuggestions(); }; diff --git a/web/src/utils/fieldValueUtils.spec.ts b/web/src/utils/fieldValueUtils.spec.ts new file mode 100644 index 0000000000..88cdd95b18 --- /dev/null +++ b/web/src/utils/fieldValueUtils.spec.ts @@ -0,0 +1,243 @@ +import { describe, it, expect } from "vitest"; +import { extractValuesFromHits, EXCLUDED_FIELDS } from "./fieldValueUtils"; + +// ─── EXCLUDED_FIELDS ───────────────────────────────────────────────────────── + +describe("EXCLUDED_FIELDS", () => { + it("contains timestamp fields", () => { + expect(EXCLUDED_FIELDS.has("_timestamp")).toBe(true); + expect(EXCLUDED_FIELDS.has("timestamp")).toBe(true); + expect(EXCLUDED_FIELDS.has("@timestamp")).toBe(true); + }); + + it("contains id fields", () => { + expect(EXCLUDED_FIELDS.has("_id")).toBe(true); + expect(EXCLUDED_FIELDS.has("__id")).toBe(true); + }); + + it("contains OO internal fields", () => { + expect(EXCLUDED_FIELDS.has("_stream")).toBe(true); + expect(EXCLUDED_FIELDS.has("_all")).toBe(true); + }); + + it("contains free-text fields", () => { + expect(EXCLUDED_FIELDS.has("log")).toBe(true); + expect(EXCLUDED_FIELDS.has("message")).toBe(true); + expect(EXCLUDED_FIELDS.has("msg")).toBe(true); + expect(EXCLUDED_FIELDS.has("body")).toBe(true); + }); + + it("does not contain regular fields", () => { + expect(EXCLUDED_FIELDS.has("status")).toBe(false); + expect(EXCLUDED_FIELDS.has("env")).toBe(false); + expect(EXCLUDED_FIELDS.has("service")).toBe(false); + }); +}); + +// ─── extractValuesFromHits ──────────────────────────────────────────────────── + +describe("extractValuesFromHits", () => { + const schemaFields = ["status", "env", "service"]; + + // ── basic happy path ──────────────────────────────────────────────────────── + + it("extracts unique values per field from hits", () => { + const hits = [ + { status: "200", env: "prod" }, + { status: "404", env: "staging" }, + { status: "200", env: "prod" }, + ]; + const result = extractValuesFromHits(hits, schemaFields, 50); + expect(result.status).toEqual(expect.arrayContaining(["200", "404"])); + expect(result.status).toHaveLength(2); + expect(result.env).toEqual(expect.arrayContaining(["prod", "staging"])); + }); + + it("deduplicates values across rows", () => { + const hits = [ + { status: "200" }, + { status: "200" }, + { status: "200" }, + ]; + const result = extractValuesFromHits(hits, schemaFields, 50); + expect(result.status).toEqual(["200"]); + }); + + it("returns empty object when hits is empty", () => { + expect(extractValuesFromHits([], schemaFields, 50)).toEqual({}); + }); + + // ── excluded fields ───────────────────────────────────────────────────────── + + it("skips _timestamp field", () => { + const hits = [{ _timestamp: "1711000000", status: "200" }]; + const result = extractValuesFromHits(hits, [], 50); + expect(result._timestamp).toBeUndefined(); + expect(result.status).toEqual(["200"]); + }); + + it("skips all EXCLUDED_FIELDS", () => { + const hit: Record = { + status: "200", + _timestamp: "1711", + timestamp: "1711", + "@timestamp": "1711", + _id: "abc", + __id: "abc", + _stream: "logs", + _all: "all", + log: "some log", + message: "some msg", + msg: "short", + body: "body text", + }; + const result = extractValuesFromHits([hit], [], 50); + expect(Object.keys(result)).toEqual(["status"]); + }); + + // ── schema filtering ──────────────────────────────────────────────────────── + + it("restricts to schema fields when schemaFields is non-empty", () => { + const hits = [{ status: "200", extra_field: "noise", env: "prod" }]; + const result = extractValuesFromHits(hits, ["status"], 50); + expect(result.status).toBeDefined(); + expect(result.extra_field).toBeUndefined(); + expect(result.env).toBeUndefined(); + }); + + it("allows all fields when schemaFields is empty", () => { + const hits = [{ status: "200", custom_field: "value" }]; + const result = extractValuesFromHits(hits, [], 50); + expect(result.status).toBeDefined(); + expect(result.custom_field).toBeDefined(); + }); + + // ── value length filter ───────────────────────────────────────────────────── + + it("skips values longer than MAX_VALUE_LENGTH (150 chars)", () => { + const longValue = "a".repeat(151); + const hits = [{ status: longValue }]; + const result = extractValuesFromHits(hits, [], 50); + expect(result.status).toBeUndefined(); + }); + + it("keeps values exactly at MAX_VALUE_LENGTH (150 chars)", () => { + const exactValue = "a".repeat(150); + const hits = [{ status: exactValue }]; + const result = extractValuesFromHits(hits, [], 50); + expect(result.status).toEqual([exactValue]); + }); + + it("keeps values shorter than MAX_VALUE_LENGTH", () => { + const hits = [{ status: "200" }]; + const result = extractValuesFromHits(hits, [], 50); + expect(result.status).toEqual(["200"]); + }); + + it("skips empty string values", () => { + const hits = [{ status: "" }]; + const result = extractValuesFromHits(hits, [], 50); + expect(result.status).toBeUndefined(); + }); + + // ── junk value filter ─────────────────────────────────────────────────────── + + it("skips 'null' string values", () => { + const hits = [{ status: "null" }]; + const result = extractValuesFromHits(hits, [], 50); + expect(result.status).toBeUndefined(); + }); + + it("skips 'undefined' string values", () => { + const hits = [{ status: "undefined" }]; + const result = extractValuesFromHits(hits, [], 50); + expect(result.status).toBeUndefined(); + }); + + it("skips 'NaN' string values", () => { + const hits = [{ status: "NaN" }]; + const result = extractValuesFromHits(hits, [], 50); + expect(result.status).toBeUndefined(); + }); + + it("converts non-string values to string (numbers, booleans)", () => { + const hits = [{ status: 200, active: true }]; + const result = extractValuesFromHits(hits, [], 50); + expect(result.status).toEqual(["200"]); + expect(result.active).toEqual(["true"]); + }); + + it("handles null rawValue by converting to empty string (skipped)", () => { + const hits = [{ status: null }]; + const result = extractValuesFromHits(hits, [], 50); + expect(result.status).toBeUndefined(); + }); + + // ── maxValuesPerField cap ─────────────────────────────────────────────────── + + it("caps values to maxValuesPerField in final result", () => { + const hits = Array.from({ length: 10 }, (_, i) => ({ status: `v${i}` })); + const result = extractValuesFromHits(hits, [], 3); + expect(result.status).toHaveLength(3); + }); + + it("returns all values when total is below cap", () => { + const hits = [{ status: "200" }, { status: "404" }]; + const result = extractValuesFromHits(hits, [], 50); + expect(result.status).toHaveLength(2); + }); + + // ── multiple fields ───────────────────────────────────────────────────────── + + it("handles multiple fields across multiple hits", () => { + const hits = [ + { status: "200", env: "prod", service: "api" }, + { status: "500", env: "staging", service: "worker" }, + ]; + const result = extractValuesFromHits(hits, schemaFields, 50); + expect(Object.keys(result).sort()).toEqual(["env", "service", "status"]); + expect(result.status).toHaveLength(2); + expect(result.env).toHaveLength(2); + expect(result.service).toHaveLength(2); + }); + + it("handles hits with missing fields gracefully", () => { + const hits = [ + { status: "200" }, + { env: "prod" }, + { status: "404", env: "staging" }, + ]; + const result = extractValuesFromHits(hits, schemaFields, 50); + expect(result.status).toEqual(expect.arrayContaining(["200", "404"])); + expect(result.env).toEqual(expect.arrayContaining(["prod", "staging"])); + }); + + it("handles 100 hits without errors (max page size)", () => { + const hits = Array.from({ length: 100 }, (_, i) => ({ + status: i % 5 === 0 ? "200" : "404", + env: i % 2 === 0 ? "prod" : "staging", + })); + const result = extractValuesFromHits(hits, schemaFields, 50); + expect(result.status).toHaveLength(2); + expect(result.env).toHaveLength(2); + }); + + it("does not store trace_id-like values if they pass length filter", () => { + const hits = [{ trace_id: "4bf92f3577b34da6" }]; + const result = extractValuesFromHits(hits, [], 50); + // 16 chars, within limit, not excluded — should be stored + expect(result.trace_id).toEqual(["4bf92f3577b34da6"]); + }); + + it("skips stack-trace-like values over 150 chars", () => { + const hits = [{ error_detail: "Error: at " + "x".repeat(150) }]; + const result = extractValuesFromHits(hits, [], 50); + expect(result.error_detail).toBeUndefined(); + }); + + it("handles empty schemaFields set with no hits matching schema", () => { + const hits = [{ _timestamp: "1711", log: "text" }]; + const result = extractValuesFromHits(hits, [], 50); + expect(Object.keys(result)).toHaveLength(0); + }); +}); diff --git a/web/src/utils/fieldValueUtils.ts b/web/src/utils/fieldValueUtils.ts new file mode 100644 index 0000000000..9cc7e68464 --- /dev/null +++ b/web/src/utils/fieldValueUtils.ts @@ -0,0 +1,102 @@ +// Copyright 2026 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +/** + * Extracts field → values mapping from search result hits. + * Designed to be called inside an idle callback — must be fast. + */ + +// Fields that should never be stored (meta / high-cardinality) +export const EXCLUDED_FIELDS = new Set([ + "_timestamp", + "_id", + "__id", + "timestamp", + "@timestamp", + "_stream", + "_all", + "body", + "log", + "message", + "msg", +]); + +const MAX_VALUE_LENGTH = 150; // chars — skip values too long to be useful filter candidates + +/** + * Extract deduplicated field→values from an array of search hits. + * + * @param hits - Raw search result records (max ~100 recommended) + * @param schemaFields - Stream schema fields to restrict extraction to. + * Pass an empty array to allow all fields. + * @param maxValuesPerField - Cap on unique values per field + * @returns Map of fieldName → unique values array + */ +export const extractValuesFromHits = ( + hits: Record[], + schemaFields: string[], + maxValuesPerField: number, +): Record => { + // fieldSets accumulates unique values per field using a Set so duplicates are + // automatically ignored — no need to check "have I seen this value before?". + const fieldSets: Map> = new Map(); + // Convert to a Set for O(1) lookups instead of O(n) Array.includes(). + const schemaSet = new Set(schemaFields); + + for (const hit of hits) { + for (const [field, rawValue] of Object.entries(hit)) { + // Skip meta / high-cardinality fields that are never useful as filter + // values: timestamps (every value unique), IDs, internal OO fields, + // and free-text fields like log/message that would just add noise. + if (EXCLUDED_FIELDS.has(field)) continue; + + // If the stream has a known schema, only keep values for schema fields. + // This prevents storing values from dynamic/temporary fields that users + // would never filter on. When schemaFields is empty we allow all fields. + if (schemaSet.size > 0 && !schemaSet.has(field)) continue; + + const value = String(rawValue ?? ""); + // Empty values and very long values (stack traces, JSON blobs, base64) + // are useless as autocomplete suggestions — skip them. + if (value.length === 0 || value.length > MAX_VALUE_LENGTH) continue; + // These are JavaScript serialisation artifacts, not real field values. + if (value === "null" || value === "undefined" || value === "NaN") + continue; + + if (!fieldSets.has(field)) fieldSets.set(field, new Set()); + fieldSets.get(field)!.add(value); + + // Early memory cap: if a field accumulates too many unique values while + // scanning (e.g. a high-cardinality "user_id" field with 100 unique + // values across 100 hits), trim it back to maxValuesPerField immediately + // instead of waiting until the loop finishes. Without this, a single + // high-cardinality field could hold hundreds of strings in memory at once. + // We keep the LAST maxValuesPerField entries (most recently seen hits) + // because they are the freshest. The threshold of ×2 avoids trimming on + // every insert — it fires at most once per field during extraction. + if (fieldSets.get(field)!.size > maxValuesPerField * 2) { + const arr = Array.from(fieldSets.get(field)!); + fieldSets.set(field, new Set(arr.slice(-maxValuesPerField))); + } + } + } + + // Convert each field's Set into a plain capped array for IndexedDB storage. + const result: Record = {}; + for (const [field, valueSet] of fieldSets.entries()) { + result[field] = Array.from(valueSet).slice(0, maxValuesPerField); + } + return result; +};