mirror of
https://github.com/openobserve/openobserve.git
synced 2026-11-05 00:09:35 +00:00
fix: auto suggestion field value on query editor logs and traces (#10943)
Adds persistent, cross-session field value suggestions to the SQL query editor. When a user types field = , field IN (, field LIKE ', str_match(field, etc., the autocomplete popup now shows real values seen in previous searches — not just keywords and field names. Values are captured passively from search results and field expansions, stored in IndexedDB, and served from an in-memory cache. Zero impact on existing UI performance. What's new Value capture — two sources: Run Query (useStreamFields.ts) — after each search, up to 100 result rows are scanned and field values extracted into IndexedDB in a browser idle callback Field expansion (useFieldValuesStream.ts, IndexList.vue) — when a user clicks > on a field in the left panel, the Values API response is also captured Persistence layer (fieldValueDB.ts — new): IndexedDB database o2FieldValues, store fieldValues Composite key: org|streamType|streamName|fieldName — fully isolated per org, stream type, and stream Merge strategy: new values are unioned with existing, capped at 50 per field 7-day sliding TTL — fields from streams you stop using expire automatically Opportunistic housekeeping (5% chance per search) via by_expires and by_updated indexes Orchestration layer (useFieldValueStore.ts — new): captureFromSearchHits — schedules extraction + batch IDB write via requestIdleCallback captureFromValuesApi — schedules single-field IDB write getFieldValuesForSuggestion — reads with 60s in-memory cache (500-entry cap) so keystrokes never hit IDB Safari fallback: setTimeout(0) when requestIdleCallback is unavailable All IDB errors caught silently — feature degrades gracefully in private browsing Suggestion engine (useSuggestions.ts): New analyzeSqlWhereClause function detects when cursor is after an operator expecting a value using a 4-alternative regex covering =, !=, <>, >=, <=, >, <, IN (, NOT IN (, LIKE, NOT LIKE, str_match(, fuzzy_match( Cursor-sliced matching handles Monaco's auto-closing brackets correctly Smart insertText quoting: numeric/boolean values unquoted, strings wrapped in '...', open-quote detection avoids double-quoting sortText prefixed with \x01 so value suggestions always sort above keywords and functions In-session values (current search) take precedence over stored values (previous sessions) Monaco popup fix (CodeQueryEditor.vue): hideSuggestWidget → nextTick → triggerSuggest pattern ensures the popup always refreshes, fixing cases where it was already in "Showing" state (e.g. typing LIKE ') Traces support (traces/SearchBar.vue): Field expansion in traces was already writing to IDB (via useFieldValuesStream with stream_type: "traces") Added 3 lines to set org/streamType/streamName on autoCompleteData so the read side works too
This commit is contained in:
Sai Nikhil
@@ -888,7 +888,14 @@ export default defineComponent({
|
||||
};
|
||||
|
||||
const triggerAutoComplete = async (value: string) => {
|
||||
disableSuggestionPopup();
|
||||
// Close any currently-open suggestion popup before retriggering.
|
||||
// Monaco's model.trigger() calls cancel() internally, but if a natural
|
||||
// popup is already in "Showing" state (e.g. opened by typing quote after
|
||||
// an operator, or from word-based suggest while typing NOT LIKE), the
|
||||
// widget may not refresh its item list. hideSuggestWidget transitions
|
||||
// state to Idle cleanly (no "user dismissed" flag) so the following
|
||||
// triggerSuggest always opens a fresh popup with the latest keywords.
|
||||
editorObj.trigger(value, "hideSuggestWidget", {});
|
||||
await nextTick();
|
||||
editorObj.trigger(value, "editor.action.triggerSuggest", {});
|
||||
};
|
||||
|
||||
@@ -0,0 +1,296 @@
|
||||
import { describe, it, expect, beforeEach, vi } from "vitest";
|
||||
import {
|
||||
openDB,
|
||||
mergeValues,
|
||||
mergeMultipleValues,
|
||||
getValues,
|
||||
evictExpired,
|
||||
trimToMaxFields,
|
||||
} from "./fieldValueDB";
|
||||
|
||||
// ─── IDB mock helpers ─────────────────────────────────────────────────────────
|
||||
// jsdom does not ship a real IDB implementation. We use a minimal in-memory
|
||||
// mock that mirrors the async/event-driven API surface used by fieldValueDB.ts.
|
||||
|
||||
// Track the last db created so we can fire onversionchange between tests to
|
||||
// reset the _db singleton inside fieldValueDB.ts.
|
||||
let _lastDb: any = null;
|
||||
|
||||
const makeStore = () => {
|
||||
const records = new Map<string, any>();
|
||||
|
||||
const makeRequest = (fn: () => any) => {
|
||||
const req: any = { onsuccess: null, onerror: null, result: undefined };
|
||||
setTimeout(() => {
|
||||
try {
|
||||
req.result = fn();
|
||||
req.onsuccess?.();
|
||||
} catch (e) {
|
||||
req.error = e;
|
||||
req.onerror?.();
|
||||
}
|
||||
}, 0);
|
||||
return req;
|
||||
};
|
||||
|
||||
return {
|
||||
records,
|
||||
get: (key: string) => makeRequest(() => records.get(key)),
|
||||
put: (record: any) => makeRequest(() => records.set(record.key, record)),
|
||||
delete: (key: string) => makeRequest(() => records.delete(key)),
|
||||
count: () => makeRequest(() => records.size),
|
||||
index: (name: string) => ({
|
||||
openCursor: (range?: any) => {
|
||||
const entries = [...records.values()];
|
||||
let filtered = entries;
|
||||
if (range) {
|
||||
// Simulate upperBound(now, true) for by_expires index
|
||||
filtered = entries.filter((r) => r.expiresAt < range._upper);
|
||||
}
|
||||
// Sort by updatedAt ascending for by_updated index
|
||||
if (name === "by_updated") {
|
||||
filtered.sort((a, b) => a.updatedAt - b.updatedAt);
|
||||
}
|
||||
let idx = 0;
|
||||
const req: any = { onsuccess: null };
|
||||
const advance = () => {
|
||||
setTimeout(() => {
|
||||
if (idx < filtered.length) {
|
||||
const entry = filtered[idx++];
|
||||
req.result = {
|
||||
value: entry,
|
||||
delete: () => records.delete(entry.key),
|
||||
continue: advance,
|
||||
};
|
||||
} else {
|
||||
req.result = null;
|
||||
}
|
||||
req.onsuccess?.({ target: req });
|
||||
}, 0);
|
||||
};
|
||||
setTimeout(advance, 0);
|
||||
return req;
|
||||
},
|
||||
}),
|
||||
};
|
||||
};
|
||||
|
||||
const setupIDBMock = () => {
|
||||
// Fire onversionchange on the previous db so fieldValueDB.ts resets _db = null.
|
||||
// This ensures each test gets a fresh IDB connection rather than reusing the
|
||||
// cached singleton from the previous test.
|
||||
if (_lastDb?.onversionchange) {
|
||||
_lastDb.onversionchange();
|
||||
}
|
||||
|
||||
const store = makeStore();
|
||||
const txCompleteCbs: (() => void)[] = [];
|
||||
|
||||
const tx: any = {
|
||||
objectStore: () => store,
|
||||
oncomplete: null,
|
||||
onerror: null,
|
||||
};
|
||||
|
||||
// Fire oncomplete after a tick so all put/get requests resolve first
|
||||
const fireTxComplete = () => {
|
||||
setTimeout(() => tx.oncomplete?.(), 10);
|
||||
};
|
||||
|
||||
const db: any = {
|
||||
transaction: () => {
|
||||
setTimeout(fireTxComplete, 0);
|
||||
return tx;
|
||||
},
|
||||
close: () => {}, // fieldValueDB.ts calls _db?.close() inside onversionchange
|
||||
onclose: null,
|
||||
onversionchange: null,
|
||||
};
|
||||
|
||||
const openReq: any = { onsuccess: null, onerror: null, onupgradeneeded: null };
|
||||
setTimeout(() => {
|
||||
openReq.result = db;
|
||||
openReq.onsuccess?.();
|
||||
}, 0);
|
||||
|
||||
vi.stubGlobal("indexedDB", {
|
||||
open: () => openReq,
|
||||
});
|
||||
|
||||
// Also stub IDBKeyRange
|
||||
vi.stubGlobal("IDBKeyRange", {
|
||||
upperBound: (val: any, exclusive: boolean) => ({ _upper: val, exclusive }),
|
||||
});
|
||||
|
||||
// Track db so the next setupIDBMock() can fire onversionchange to reset _db
|
||||
_lastDb = db;
|
||||
|
||||
return store;
|
||||
};
|
||||
|
||||
// ─── openDB ───────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("openDB", () => {
|
||||
beforeEach(() => {
|
||||
// Reset the cached _db between tests by re-importing would need module reset.
|
||||
// Instead we just verify openDB resolves without throwing.
|
||||
setupIDBMock();
|
||||
});
|
||||
|
||||
it("resolves with a database object", async () => {
|
||||
const db = await openDB();
|
||||
expect(db).toBeDefined();
|
||||
expect(typeof db.transaction).toBe("function");
|
||||
});
|
||||
});
|
||||
|
||||
// ─── mergeValues ─────────────────────────────────────────────────────────────
|
||||
|
||||
describe("mergeValues", () => {
|
||||
beforeEach(() => {
|
||||
setupIDBMock();
|
||||
});
|
||||
|
||||
it("writes a new record when key does not exist", async () => {
|
||||
await mergeValues("org|logs|stream|status", ["200", "404"], "search_hits", 50, 86400000);
|
||||
const values = await getValues("org|logs|stream|status");
|
||||
expect(values).toEqual(expect.arrayContaining(["200", "404"]));
|
||||
});
|
||||
|
||||
it("merges with existing record (deduplicates)", async () => {
|
||||
await mergeValues("org|logs|stream|env", ["prod"], "search_hits", 50, 86400000);
|
||||
await mergeValues("org|logs|stream|env", ["prod", "staging"], "search_hits", 50, 86400000);
|
||||
const values = await getValues("org|logs|stream|env");
|
||||
expect(values).toEqual(expect.arrayContaining(["prod", "staging"]));
|
||||
expect(values).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("caps values at maxValues", async () => {
|
||||
const incoming = Array.from({ length: 10 }, (_, i) => `v${i}`);
|
||||
await mergeValues("org|logs|stream|service", incoming, "search_hits", 3, 86400000);
|
||||
const values = await getValues("org|logs|stream|service");
|
||||
expect(values).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("marks source as mixed when sources differ", async () => {
|
||||
await mergeValues("org|logs|stream|field1", ["a"], "search_hits", 50, 86400000);
|
||||
await mergeValues("org|logs|stream|field1", ["b"], "values_api", 50, 86400000);
|
||||
// We can't directly inspect source via getValues — we trust the internal
|
||||
// record. The merge shouldn't throw and should return combined values.
|
||||
const values = await getValues("org|logs|stream|field1");
|
||||
expect(values).toEqual(expect.arrayContaining(["a", "b"]));
|
||||
});
|
||||
});
|
||||
|
||||
// ─── mergeMultipleValues ──────────────────────────────────────────────────────
|
||||
|
||||
describe("mergeMultipleValues", () => {
|
||||
beforeEach(() => {
|
||||
setupIDBMock();
|
||||
});
|
||||
|
||||
it("writes multiple fields in one call", async () => {
|
||||
await mergeMultipleValues(
|
||||
[
|
||||
{ key: "org|logs|stream|status", values: ["200"], source: "search_hits" },
|
||||
{ key: "org|logs|stream|env", values: ["prod"], source: "search_hits" },
|
||||
],
|
||||
50,
|
||||
86400000,
|
||||
);
|
||||
expect(await getValues("org|logs|stream|status")).toEqual(["200"]);
|
||||
expect(await getValues("org|logs|stream|env")).toEqual(["prod"]);
|
||||
});
|
||||
|
||||
it("returns early when entries is empty", async () => {
|
||||
// Should not throw
|
||||
await expect(mergeMultipleValues([], 50, 86400000)).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("deduplicates values within a single entry", async () => {
|
||||
await mergeMultipleValues(
|
||||
[{ key: "org|logs|stream|level", values: ["info", "info", "warn"], source: "search_hits" }],
|
||||
50,
|
||||
86400000,
|
||||
);
|
||||
const values = await getValues("org|logs|stream|level");
|
||||
expect(values).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("caps each field at maxValues", async () => {
|
||||
const values = Array.from({ length: 10 }, (_, i) => `val${i}`);
|
||||
await mergeMultipleValues(
|
||||
[{ key: "org|logs|stream|capped", values, source: "search_hits" }],
|
||||
3,
|
||||
86400000,
|
||||
);
|
||||
expect(await getValues("org|logs|stream|capped")).toHaveLength(3);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── getValues ────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("getValues", () => {
|
||||
beforeEach(() => {
|
||||
setupIDBMock();
|
||||
});
|
||||
|
||||
it("returns empty array for missing key", async () => {
|
||||
expect(await getValues("nonexistent|key")).toEqual([]);
|
||||
});
|
||||
|
||||
it("returns empty array for expired record", async () => {
|
||||
// Write a record with expiresAt in the past
|
||||
const store = setupIDBMock();
|
||||
store.records.set("org|logs|stream|expired", {
|
||||
key: "org|logs|stream|expired",
|
||||
values: ["200"],
|
||||
updatedAt: Date.now() - 10000,
|
||||
expiresAt: Date.now() - 1000, // already expired
|
||||
source: "search_hits",
|
||||
});
|
||||
expect(await getValues("org|logs|stream|expired")).toEqual([]);
|
||||
});
|
||||
|
||||
it("returns values for a valid non-expired record", async () => {
|
||||
await mergeValues("org|logs|stream|valid", ["200", "404"], "search_hits", 50, 86400000);
|
||||
const values = await getValues("org|logs|stream|valid");
|
||||
expect(values).toEqual(expect.arrayContaining(["200", "404"]));
|
||||
});
|
||||
|
||||
it("returns [] on IDB error without throwing", async () => {
|
||||
// Break IDB to simulate unavailability
|
||||
vi.stubGlobal("indexedDB", undefined);
|
||||
const values = await getValues("any|key");
|
||||
expect(values).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── evictExpired ─────────────────────────────────────────────────────────────
|
||||
|
||||
describe("evictExpired", () => {
|
||||
beforeEach(() => {
|
||||
setupIDBMock();
|
||||
});
|
||||
|
||||
it("resolves with a number (count of deleted records)", async () => {
|
||||
const result = await evictExpired();
|
||||
expect(typeof result).toBe("number");
|
||||
});
|
||||
});
|
||||
|
||||
// ─── trimToMaxFields ─────────────────────────────────────────────────────────
|
||||
|
||||
describe("trimToMaxFields", () => {
|
||||
beforeEach(() => {
|
||||
setupIDBMock();
|
||||
});
|
||||
|
||||
it("resolves without error when records are below limit", async () => {
|
||||
await expect(trimToMaxFields(5000)).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not throw when store is empty", async () => {
|
||||
await expect(trimToMaxFields(0)).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,350 @@
|
||||
// Copyright 2026 OpenObserve Inc.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
/**
|
||||
* Low-level IndexedDB operations for field value storage.
|
||||
* All methods are async and non-blocking to the caller.
|
||||
*
|
||||
* DB layout:
|
||||
* Database : o2FieldValues (version 1)
|
||||
* Store : fieldValues
|
||||
* PK : key — composite string "org|streamType|streamName|fieldName"
|
||||
* Index 1 : by_expires (expiresAt) — used by evictExpired for TTL sweeps
|
||||
* Index 2 : by_updated (updatedAt) — used by trimToMaxFields to evict LRU
|
||||
*
|
||||
* Why two indexes?
|
||||
* evictExpired needs records sorted by expiry time (delete the expired ones).
|
||||
* trimToMaxFields needs records sorted by last-used time (delete the oldest).
|
||||
* These are different orderings so they require separate indexes.
|
||||
*/
|
||||
|
||||
const DB_NAME = "o2FieldValues";
|
||||
const DB_VERSION = 1;
|
||||
const STORE_NAME = "fieldValues";
|
||||
|
||||
export interface FieldValueRecord {
|
||||
key: string; // PK — composite key "org|streamType|streamName|fieldName"
|
||||
values: string[]; // unique, deduplicated; capped at MAX_VALUES_PER_FIELD
|
||||
updatedAt: number; // Unix ms — last write timestamp; used for LRU eviction
|
||||
expiresAt: number; // Unix ms — sliding TTL, reset on every write
|
||||
// "mixed" means values came from both sources at different points in time.
|
||||
// Used for debugging only — lets you see in DevTools where values originated.
|
||||
source: "values_api" | "search_hits" | "mixed";
|
||||
}
|
||||
|
||||
// Cached IDB connection — opened once and reused across all calls.
|
||||
// null means not yet opened (or was closed and needs to be reopened).
|
||||
let _db: IDBDatabase | null = null;
|
||||
|
||||
/**
|
||||
* Opens the database lazily — returns the cached connection on subsequent calls.
|
||||
*
|
||||
* Why lazy (open on first use, not at app startup)?
|
||||
* IDB open is async and costs ~5–10ms. Opening eagerly would always pay that
|
||||
* cost even if the user never uses the query editor or never triggers
|
||||
* autocomplete. Lazy open means zero cost for users who don't need it.
|
||||
*
|
||||
* Why cache the connection in _db?
|
||||
* IDBDatabase.open() creates a new connection object each time. Connections
|
||||
* are cheap but not free. More importantly, reusing one connection means all
|
||||
* operations share the same transaction queue, which is correct IDB behaviour.
|
||||
*
|
||||
* onclose / onversionchange handlers:
|
||||
* If another tab opens a newer version of this DB (DB_VERSION bumped in a
|
||||
* future release), the browser fires versionchange on existing connections and
|
||||
* then closes them. We close and null out _db so the next call to openDB()
|
||||
* reopens cleanly with the new version.
|
||||
*/
|
||||
export const openDB = (): Promise<IDBDatabase> => {
|
||||
if (_db) return Promise.resolve(_db);
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = indexedDB.open(DB_NAME, DB_VERSION);
|
||||
req.onupgradeneeded = (e) => {
|
||||
const db = (e.target as IDBOpenDBRequest).result;
|
||||
if (!db.objectStoreNames.contains(STORE_NAME)) {
|
||||
const store = db.createObjectStore(STORE_NAME, { keyPath: "key" });
|
||||
// by_expires: IDBKeyRange.upperBound(now) range scan in evictExpired
|
||||
store.createIndex("by_expires", "expiresAt", { unique: false });
|
||||
// by_updated: ascending cursor in trimToMaxFields (oldest first)
|
||||
store.createIndex("by_updated", "updatedAt", { unique: false });
|
||||
}
|
||||
};
|
||||
req.onsuccess = () => {
|
||||
_db = req.result;
|
||||
// Reset the cached reference if the browser closes the connection
|
||||
// (e.g. another tab opens a newer DB version triggering a versionchange).
|
||||
_db.onclose = () => {
|
||||
_db = null;
|
||||
};
|
||||
_db.onversionchange = () => {
|
||||
_db?.close();
|
||||
_db = null;
|
||||
};
|
||||
resolve(_db!);
|
||||
};
|
||||
req.onerror = () => reject(req.error);
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Merge new values into an existing record for a single field.
|
||||
* Used by captureFromValuesApi (Values API path — one field at a time).
|
||||
*
|
||||
* Merge strategy:
|
||||
* existing values + incoming values → deduplicate via Set → cap to maxValues
|
||||
* If total > maxValues, keep the LAST maxValues (most recently seen / freshest).
|
||||
*
|
||||
* Why sliding TTL (reset expiresAt on every write)?
|
||||
* Fields from streams you query frequently stay fresh forever — every new
|
||||
* search resets their 7-day clock. Fields from streams you stop using expire
|
||||
* naturally after 7 days of inactivity without any manual cleanup.
|
||||
*
|
||||
* Why track source ("values_api" | "search_hits" | "mixed")?
|
||||
* Purely for debugging. In DevTools → Application → IndexedDB you can see
|
||||
* whether a field's values came from explicit field expansion (values_api),
|
||||
* passively from search results (search_hits), or both (mixed). Helps when
|
||||
* investigating why certain values appear or don't appear in autocomplete.
|
||||
*/
|
||||
export const mergeValues = async (
|
||||
key: string,
|
||||
incoming: string[],
|
||||
source: FieldValueRecord["source"],
|
||||
maxValues: number,
|
||||
ttlMs: number,
|
||||
): Promise<void> => {
|
||||
const db = await openDB();
|
||||
return new Promise((resolve, reject) => {
|
||||
const tx = db.transaction(STORE_NAME, "readwrite");
|
||||
const store = tx.objectStore(STORE_NAME);
|
||||
const getReq = store.get(key);
|
||||
getReq.onsuccess = () => {
|
||||
const existing: FieldValueRecord | undefined = getReq.result;
|
||||
// Combine existing + incoming into a Set for automatic deduplication.
|
||||
// No need to manually check "have I seen this value before?".
|
||||
const valueSet = new Set(existing?.values ?? []);
|
||||
for (const v of incoming) valueSet.add(String(v));
|
||||
const merged = Array.from(valueSet);
|
||||
// Keep the last maxValues entries — the most recently added values are
|
||||
// at the end of the array and are most likely to be relevant to the
|
||||
// user's current query context.
|
||||
const values =
|
||||
merged.length > maxValues
|
||||
? merged.slice(merged.length - maxValues)
|
||||
: merged;
|
||||
const now = Date.now();
|
||||
const record: FieldValueRecord = {
|
||||
key,
|
||||
values,
|
||||
updatedAt: now,
|
||||
expiresAt: now + ttlMs, // sliding window — reset on every write
|
||||
source: existing
|
||||
? existing.source === source
|
||||
? source // same source as before — keep it
|
||||
: "mixed" // different source — mark as mixed
|
||||
: source, // first write — use the incoming source
|
||||
};
|
||||
store.put(record);
|
||||
};
|
||||
tx.oncomplete = () => resolve();
|
||||
tx.onerror = () => reject(tx.error);
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Merge values for multiple fields in a single IDB transaction.
|
||||
* Used by captureFromSearchHits — the hot path after every Run Query.
|
||||
*
|
||||
* Why one transaction instead of N mergeValues calls?
|
||||
* IDB transactions have a fixed cost: acquire the store lock, execute
|
||||
* operations, commit. If a stream has 40 fields, calling mergeValues 40 times
|
||||
* = 40 separate transactions = 40× that fixed cost. One transaction with 40
|
||||
* get+put operations = 1× that cost. On a wide schema this is ~40× cheaper.
|
||||
*
|
||||
* The merge logic per field is identical to mergeValues above — the only
|
||||
* difference is that all operations share one tx.oncomplete callback.
|
||||
*/
|
||||
export const mergeMultipleValues = async (
|
||||
entries: Array<{
|
||||
key: string;
|
||||
values: string[];
|
||||
source: FieldValueRecord["source"];
|
||||
}>,
|
||||
maxValues: number,
|
||||
ttlMs: number,
|
||||
): Promise<void> => {
|
||||
if (entries.length === 0) return;
|
||||
const db = await openDB();
|
||||
return new Promise((resolve, reject) => {
|
||||
const tx = db.transaction(STORE_NAME, "readwrite");
|
||||
const store = tx.objectStore(STORE_NAME);
|
||||
const now = Date.now();
|
||||
|
||||
for (const { key, values: incoming, source } of entries) {
|
||||
const getReq = store.get(key);
|
||||
getReq.onsuccess = () => {
|
||||
const existing: FieldValueRecord | undefined = getReq.result;
|
||||
const valueSet = new Set(existing?.values ?? []);
|
||||
for (const v of incoming) valueSet.add(String(v));
|
||||
const merged = Array.from(valueSet);
|
||||
const values =
|
||||
merged.length > maxValues
|
||||
? merged.slice(merged.length - maxValues)
|
||||
: merged;
|
||||
const record: FieldValueRecord = {
|
||||
key,
|
||||
values,
|
||||
updatedAt: now,
|
||||
expiresAt: now + ttlMs,
|
||||
source: existing
|
||||
? existing.source === source
|
||||
? source
|
||||
: "mixed"
|
||||
: source,
|
||||
};
|
||||
store.put(record);
|
||||
};
|
||||
}
|
||||
|
||||
// All get+put operations for every field complete before this fires.
|
||||
// Resolving here means the caller knows the full batch is safely written.
|
||||
tx.oncomplete = () => resolve();
|
||||
tx.onerror = () => reject(tx.error);
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Fast point-read for a single field's values.
|
||||
* Returns empty array if record is missing or expired.
|
||||
*
|
||||
* Why check expiresAt at read time (not just rely on evictExpired)?
|
||||
* evictExpired only runs on ~5% of searches — a record can be expired but not
|
||||
* yet swept from the DB. Without this check, we could serve stale values to
|
||||
* autocomplete for up to several searches after the TTL has passed.
|
||||
* This makes the read authoritative regardless of when cleanup last ran.
|
||||
*/
|
||||
export const getValues = async (key: string): Promise<string[]> => {
|
||||
let db: IDBDatabase;
|
||||
try {
|
||||
db = await openDB();
|
||||
} catch {
|
||||
// IDB unavailable (private browsing, unsupported browser, quota) — return
|
||||
// empty array so autocomplete silently degrades rather than throwing.
|
||||
return [];
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const tx = db.transaction(STORE_NAME, "readonly");
|
||||
const req = tx.objectStore(STORE_NAME).get(key);
|
||||
req.onsuccess = () => {
|
||||
const record = req.result as FieldValueRecord | undefined;
|
||||
if (!record) {
|
||||
resolve([]);
|
||||
return;
|
||||
}
|
||||
// Check TTL at read time — don't serve expired data even if evictExpired
|
||||
// hasn't swept this record yet.
|
||||
if (record.expiresAt < Date.now()) {
|
||||
resolve([]);
|
||||
return;
|
||||
}
|
||||
resolve(record.values);
|
||||
};
|
||||
req.onerror = () => resolve([]); // never throw — autocomplete is non-critical
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Evict all records whose expiresAt < now (TTL cleanup).
|
||||
* Called opportunistically on ~5% of searches to amortize the scan cost.
|
||||
*
|
||||
* Why use the by_expires index instead of a full store scan?
|
||||
* A full scan would read every record regardless of expiry status.
|
||||
* IDBKeyRange.upperBound(now, true) on the by_expires index reads ONLY records
|
||||
* with expiresAt < now — exactly the ones we want to delete. On 5000 records
|
||||
* with 10 expired, this reads 10 records instead of 5000. O(expired) not O(total).
|
||||
*
|
||||
* Why "opportunistic" (5% chance) instead of every search?
|
||||
* When nothing has expired (the common case), this scan still costs ~5–10ms.
|
||||
* Running it on every search wastes that time for zero benefit. 1-in-20 gives
|
||||
* periodic cleanup while spreading the cost across many searches.
|
||||
*/
|
||||
export const evictExpired = async (): Promise<number> => {
|
||||
const db = await openDB();
|
||||
const now = Date.now();
|
||||
return new Promise((resolve, reject) => {
|
||||
const tx = db.transaction(STORE_NAME, "readwrite");
|
||||
const index = tx.objectStore(STORE_NAME).index("by_expires");
|
||||
// upperBound(now, true) = expiresAt < now (exclusive upper bound)
|
||||
const range = IDBKeyRange.upperBound(now, true);
|
||||
let deleted = 0;
|
||||
const cursorReq = index.openCursor(range);
|
||||
cursorReq.onsuccess = (e) => {
|
||||
const cursor = (e.target as IDBRequest<IDBCursorWithValue>).result;
|
||||
if (cursor) {
|
||||
cursor.delete();
|
||||
deleted++;
|
||||
cursor.continue(); // advance to next expired record
|
||||
} else {
|
||||
resolve(deleted); // cursor exhausted — done
|
||||
}
|
||||
};
|
||||
cursorReq.onerror = () => reject(cursorReq.error);
|
||||
tx.onerror = () => reject(tx.error);
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Trim total record count to maxFields — evict least-recently-used fields first.
|
||||
* Called opportunistically alongside evictExpired (~5% of searches).
|
||||
*
|
||||
* Why trim by updatedAt (oldest first)?
|
||||
* Fields with the oldest updatedAt are the ones the user hasn't queried in the
|
||||
* longest time — least likely to be useful in autocomplete. Evicting them
|
||||
* preserves the most relevant, recently-used field values.
|
||||
*
|
||||
* Why is this a last-resort guard?
|
||||
* In practice, 5000 fields = ~100 streams × 50 fields each, which is far more
|
||||
* than most users will encounter. This function is mainly here to prevent
|
||||
* unbounded growth in edge cases (many orgs, many streams, very long sessions).
|
||||
*/
|
||||
export const trimToMaxFields = async (maxFields: number): Promise<void> => {
|
||||
const db = await openDB();
|
||||
return new Promise((resolve) => {
|
||||
const tx = db.transaction(STORE_NAME, "readwrite");
|
||||
const store = tx.objectStore(STORE_NAME);
|
||||
const countReq = store.count();
|
||||
countReq.onsuccess = () => {
|
||||
const total = countReq.result;
|
||||
if (total <= maxFields) {
|
||||
// Already under the cap — nothing to do
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
const toDelete = total - maxFields;
|
||||
// by_updated cursor starts from the smallest updatedAt (least recently used)
|
||||
const index = store.index("by_updated");
|
||||
let deleted = 0;
|
||||
index.openCursor().onsuccess = (e) => {
|
||||
const cursor = (e.target as IDBRequest<IDBCursorWithValue>).result;
|
||||
if (cursor && deleted < toDelete) {
|
||||
cursor.delete();
|
||||
deleted++;
|
||||
cursor.continue();
|
||||
} else {
|
||||
resolve(); // deleted enough or ran out of records
|
||||
}
|
||||
};
|
||||
};
|
||||
});
|
||||
};
|
||||
@@ -0,0 +1,219 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
|
||||
// ─── Mock fieldValueDB ────────────────────────────────────────────────────────
|
||||
vi.mock("@/composables/fieldValueDB", () => ({
|
||||
mergeValues: vi.fn().mockResolvedValue(undefined),
|
||||
mergeMultipleValues: vi.fn().mockResolvedValue(undefined),
|
||||
getValues: vi.fn().mockResolvedValue([]),
|
||||
evictExpired: vi.fn().mockResolvedValue(0),
|
||||
trimToMaxFields: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
// ─── Mock extractValuesFromHits ───────────────────────────────────────────────
|
||||
vi.mock("@/utils/fieldValueUtils", () => ({
|
||||
extractValuesFromHits: vi.fn().mockReturnValue({
|
||||
status: ["200", "404"],
|
||||
env: ["prod", "staging"],
|
||||
}),
|
||||
}));
|
||||
|
||||
import * as fieldValueDB from "@/composables/fieldValueDB";
|
||||
import { extractValuesFromHits } from "@/utils/fieldValueUtils";
|
||||
import {
|
||||
captureFromSearchHits,
|
||||
captureFromValuesApi,
|
||||
getFieldValuesForSuggestion,
|
||||
} from "./useFieldValueStore";
|
||||
|
||||
const ctx = { org: "myorg", streamType: "logs", streamName: "http_logs" };
|
||||
|
||||
// ─── scheduleWrite behaviour ─────────────────────────────────────────────────
|
||||
// requestIdleCallback is not available in jsdom — our code falls back to
|
||||
// setTimeout(0). We use vi.useFakeTimers() to control when those fire.
|
||||
|
||||
describe("captureFromSearchHits", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("does nothing when hits is empty", async () => {
|
||||
captureFromSearchHits(ctx, [], []);
|
||||
await vi.runAllTimersAsync();
|
||||
expect(fieldValueDB.mergeMultipleValues).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does nothing when hits is null/undefined", async () => {
|
||||
captureFromSearchHits(ctx, null as any, []);
|
||||
await vi.runAllTimersAsync();
|
||||
expect(fieldValueDB.mergeMultipleValues).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("calls extractValuesFromHits with up to 100 hits", async () => {
|
||||
const hits = Array.from({ length: 150 }, (_, i) => ({ status: `${i}` }));
|
||||
captureFromSearchHits(ctx, hits, []);
|
||||
await vi.runAllTimersAsync();
|
||||
expect(extractValuesFromHits).toHaveBeenCalledWith(
|
||||
hits.slice(0, 100),
|
||||
[],
|
||||
expect.any(Number),
|
||||
);
|
||||
});
|
||||
|
||||
it("calls mergeMultipleValues with correct composite keys", async () => {
|
||||
captureFromSearchHits(ctx, [{ status: "200" }], ["status"]);
|
||||
await vi.runAllTimersAsync();
|
||||
expect(fieldValueDB.mergeMultipleValues).toHaveBeenCalledWith(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ key: "myorg|logs|http_logs|status" }),
|
||||
expect.objectContaining({ key: "myorg|logs|http_logs|env" }),
|
||||
]),
|
||||
expect.any(Number),
|
||||
expect.any(Number),
|
||||
);
|
||||
});
|
||||
|
||||
it("passes schemaFields to extractValuesFromHits", async () => {
|
||||
const schema = ["status", "env"];
|
||||
captureFromSearchHits(ctx, [{ status: "200" }], schema);
|
||||
await vi.runAllTimersAsync();
|
||||
expect(extractValuesFromHits).toHaveBeenCalledWith(
|
||||
expect.any(Array),
|
||||
schema,
|
||||
expect.any(Number),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not call mergeMultipleValues when extractValuesFromHits returns empty", async () => {
|
||||
vi.mocked(extractValuesFromHits).mockReturnValueOnce({});
|
||||
captureFromSearchHits(ctx, [{ status: "200" }], []);
|
||||
await vi.runAllTimersAsync();
|
||||
expect(fieldValueDB.mergeMultipleValues).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ─── captureFromValuesApi ─────────────────────────────────────────────────────
|
||||
|
||||
describe("captureFromValuesApi", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("calls mergeValues with correct key and values", async () => {
|
||||
captureFromValuesApi(ctx, "status", [
|
||||
{ key: "200" },
|
||||
{ key: "404" },
|
||||
]);
|
||||
await vi.runAllTimersAsync();
|
||||
expect(fieldValueDB.mergeValues).toHaveBeenCalledWith(
|
||||
"myorg|logs|http_logs|status",
|
||||
["200", "404"],
|
||||
"values_api",
|
||||
expect.any(Number),
|
||||
expect.any(Number),
|
||||
);
|
||||
});
|
||||
|
||||
it("filters out empty-string values", async () => {
|
||||
captureFromValuesApi(ctx, "status", [{ key: "" }, { key: "200" }]);
|
||||
await vi.runAllTimersAsync();
|
||||
expect(fieldValueDB.mergeValues).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
["200"],
|
||||
expect.any(String),
|
||||
expect.any(Number),
|
||||
expect.any(Number),
|
||||
);
|
||||
});
|
||||
|
||||
it("filters out values longer than 200 chars", async () => {
|
||||
const long = "x".repeat(201);
|
||||
captureFromValuesApi(ctx, "status", [{ key: long }, { key: "200" }]);
|
||||
await vi.runAllTimersAsync();
|
||||
expect(fieldValueDB.mergeValues).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
["200"],
|
||||
expect.any(String),
|
||||
expect.any(Number),
|
||||
expect.any(Number),
|
||||
);
|
||||
});
|
||||
|
||||
it("does nothing when all values are filtered out", async () => {
|
||||
captureFromValuesApi(ctx, "status", [{ key: "" }]);
|
||||
await vi.runAllTimersAsync();
|
||||
expect(fieldValueDB.mergeValues).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does nothing when entries array is empty", async () => {
|
||||
captureFromValuesApi(ctx, "status", []);
|
||||
await vi.runAllTimersAsync();
|
||||
expect(fieldValueDB.mergeValues).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses correct stream context in composite key", async () => {
|
||||
const tracesCtx = { org: "acme", streamType: "traces", streamName: "default" };
|
||||
captureFromValuesApi(tracesCtx, "service", [{ key: "frontend" }]);
|
||||
await vi.runAllTimersAsync();
|
||||
expect(fieldValueDB.mergeValues).toHaveBeenCalledWith(
|
||||
"acme|traces|default|service",
|
||||
["frontend"],
|
||||
"values_api",
|
||||
expect.any(Number),
|
||||
expect.any(Number),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── getFieldValuesForSuggestion ──────────────────────────────────────────────
|
||||
|
||||
describe("getFieldValuesForSuggestion", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("returns values from IDB on cache miss", async () => {
|
||||
vi.mocked(fieldValueDB.getValues).mockResolvedValueOnce(["200", "404"]);
|
||||
const values = await getFieldValuesForSuggestion(ctx, "status");
|
||||
expect(values).toEqual(["200", "404"]);
|
||||
expect(fieldValueDB.getValues).toHaveBeenCalledWith("myorg|logs|http_logs|status");
|
||||
});
|
||||
|
||||
it("returns cached values on second call (cache hit, no extra IDB read)", async () => {
|
||||
// Use a unique field name to avoid cache pollution from other tests
|
||||
const freshCtx = { org: "cachetest", streamType: "logs", streamName: "mystream" };
|
||||
vi.mocked(fieldValueDB.getValues).mockResolvedValue(["prod", "staging"]);
|
||||
|
||||
await getFieldValuesForSuggestion(freshCtx, "env");
|
||||
await getFieldValuesForSuggestion(freshCtx, "env");
|
||||
|
||||
// IDB should only be called once — second call served from cache
|
||||
expect(fieldValueDB.getValues).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("returns empty array when IDB throws", async () => {
|
||||
vi.mocked(fieldValueDB.getValues).mockRejectedValueOnce(new Error("IDB error"));
|
||||
const values = await getFieldValuesForSuggestion(ctx, "broken_field");
|
||||
expect(values).toEqual([]);
|
||||
});
|
||||
|
||||
it("returns empty array when IDB returns empty", async () => {
|
||||
vi.mocked(fieldValueDB.getValues).mockResolvedValueOnce([]);
|
||||
const values = await getFieldValuesForSuggestion(ctx, "empty_field");
|
||||
expect(values).toEqual([]);
|
||||
});
|
||||
|
||||
it("builds correct composite key from context", async () => {
|
||||
const specificCtx = { org: "acme", streamType: "traces", streamName: "spans" };
|
||||
vi.mocked(fieldValueDB.getValues).mockResolvedValueOnce(["frontend"]);
|
||||
await getFieldValuesForSuggestion(specificCtx, "service");
|
||||
expect(fieldValueDB.getValues).toHaveBeenCalledWith("acme|traces|spans|service");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,284 @@
|
||||
// Copyright 2026 OpenObserve Inc.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
/**
|
||||
* Composable that orchestrates field value capture from both data sources:
|
||||
* 1. Values API — field expansion in FieldList
|
||||
* 2. Search result hits — Run Query results
|
||||
*
|
||||
* All IndexedDB writes are scheduled via requestIdleCallback so they have
|
||||
* zero impact on main-thread rendering. Reads use an in-memory cache
|
||||
* (1-minute TTL, 500-entry cap) so repeated keystrokes never hit IndexedDB.
|
||||
*/
|
||||
|
||||
import * as fieldValueDB from "@/composables/fieldValueDB";
|
||||
import { extractValuesFromHits } from "@/utils/fieldValueUtils";
|
||||
|
||||
// ─── Configuration ────────────────────────────────────────────────────────────
|
||||
|
||||
// Max unique values stored per field in IDB. 50 is enough for all realistic
|
||||
// observability filter fields (status codes, env names, log levels). More than
|
||||
// 50 items in a dropdown is noise rather than help. Overridable via .env so
|
||||
// team can tune without a code change.
|
||||
const MAX_VALUES_PER_FIELD = Number(
|
||||
(import.meta as any).env?.VITE_MAX_FIELD_VALUES ?? 50,
|
||||
);
|
||||
|
||||
// Total IDB record cap across all orgs/streams/fields combined. Prevents the
|
||||
// browser's IndexedDB storage from growing forever across many sessions.
|
||||
// 5000 records × ~50 values × ~20 chars avg ≈ ~5MB — well within browser limits.
|
||||
// Overridable via .env.
|
||||
const MAX_FIELDS_STORED = Number(
|
||||
(import.meta as any).env?.VITE_MAX_FIELDS_STORED ?? 5000,
|
||||
);
|
||||
|
||||
// How long a field's values stay valid in IDB after the last write.
|
||||
// Sliding window — every new write resets the 7-day clock. Fields from streams
|
||||
// you stop querying disappear automatically after 7 days of inactivity.
|
||||
// Overridable via .env (unit: days).
|
||||
const FIELD_VALUE_TTL_MS =
|
||||
Number((import.meta as any).env?.VITE_FIELD_VALUE_TTL_DAYS ?? 7) *
|
||||
86400 *
|
||||
1000;
|
||||
|
||||
// ─── In-memory read cache ─────────────────────────────────────────────────────
|
||||
|
||||
// Two-level cache: memory → IDB → [].
|
||||
// Without this, every keystroke in the query editor (e.g. "status = ") would
|
||||
// trigger an async IDB read (~1ms). With it, the first keystroke pays the IDB
|
||||
// cost; all subsequent keystrokes in the same minute return from memory in ~0ms.
|
||||
const readCache = new Map<string, { values: string[]; ts: number }>();
|
||||
const READ_CACHE_TTL_MS = 60_000; // 1 minute — balance between freshness and cost
|
||||
const READ_CACHE_MAX_ENTRIES = 500; // ~10 streams × ~50 fields each; enough for any session
|
||||
|
||||
// ─── Types ────────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface StreamContext {
|
||||
org: string;
|
||||
streamType: string; // 'logs' | 'metrics' | 'traces'
|
||||
streamName: string;
|
||||
}
|
||||
|
||||
// ─── Helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// Composite string key used as the IDB primary key.
|
||||
// Format: "org|streamType|streamName|fieldName"
|
||||
// e.g. : "myorg|logs|http_logs|status"
|
||||
//
|
||||
// Why a flat string instead of a compound key?
|
||||
// IDB compound keys require an array and more complex range queries.
|
||||
// A flat string is simpler, human-readable in DevTools, and works perfectly
|
||||
// for our point-read access pattern. The "|" separator is safe because
|
||||
// org/stream names in OO cannot contain "|".
|
||||
const makeKey = (ctx: StreamContext, fieldName: string): string =>
|
||||
`${ctx.org}|${ctx.streamType}|${ctx.streamName}|${fieldName}`;
|
||||
|
||||
/**
|
||||
* Schedule a background write using requestIdleCallback.
|
||||
* Falls back to setTimeout(0) in environments without rIC (e.g. Safari).
|
||||
*
|
||||
* Why this wrapper exists:
|
||||
* - requestIdleCallback runs the work AFTER the browser has finished rendering
|
||||
* the current frame — zero impact on the search results appearing on screen.
|
||||
* - timeout: 2000 is a safety valve: if the browser stays busy for 2 full
|
||||
* seconds without an idle slot, the write is forced anyway so data is never
|
||||
* permanently lost.
|
||||
* - Safari does not support requestIdleCallback. setTimeout(0) is the next
|
||||
* best thing — it defers to the next event loop tick, which still avoids
|
||||
* blocking the current render cycle.
|
||||
* - The .catch() wrapper ensures IDB failures (private browsing, quota
|
||||
* exceeded, IDB unavailable) never crash the app. Autocomplete is a
|
||||
* nice-to-have, not a critical feature. Errors are logged in dev only.
|
||||
*/
|
||||
const scheduleWrite = (fn: () => Promise<void>): void => {
|
||||
const wrapped = () =>
|
||||
fn().catch((err) => {
|
||||
if (import.meta.env.DEV) {
|
||||
console.debug("[fieldValueStore] write error:", err);
|
||||
}
|
||||
});
|
||||
|
||||
if (typeof requestIdleCallback !== "undefined") {
|
||||
requestIdleCallback(wrapped, { timeout: 2000 });
|
||||
} else {
|
||||
setTimeout(wrapped, 0);
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Write to the in-memory read cache with a bounded size.
|
||||
*
|
||||
* Why evict 10% at a time instead of 1 entry?
|
||||
* Evicting a single entry on every insert-at-cap means paying the eviction
|
||||
* cost on every new entry once the map is full. Evicting 10% (50 entries) at
|
||||
* once amortizes that cost — we only pay it once every 50 inserts.
|
||||
*/
|
||||
const setCacheEntry = (key: string, values: string[]): void => {
|
||||
readCache.set(key, { values, ts: Date.now() });
|
||||
if (readCache.size > READ_CACHE_MAX_ENTRIES) {
|
||||
const evict = Math.ceil(READ_CACHE_MAX_ENTRIES * 0.1);
|
||||
const iter = readCache.keys();
|
||||
for (let i = 0; i < evict; i++) {
|
||||
const { value, done } = iter.next();
|
||||
if (done) break;
|
||||
readCache.delete(value);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// ─── Public API ───────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Capture values returned by the Values API (field expansion in FieldList).
|
||||
* Called from useFieldValuesStream.handleResponse() AFTER the existing UI
|
||||
* state update — returns synchronously in ~1µs.
|
||||
*
|
||||
* Why 200-char filter here (vs 150 in extractValuesFromHits)?
|
||||
* The Values API returns pre-selected top values from the server — they have
|
||||
* already been ranked and filtered for relevance. The 200-char limit is just
|
||||
* a safety net against malformed responses. We don't need the aggressive
|
||||
* filtering used for raw log hits where any field can appear.
|
||||
*
|
||||
* Why delete readCache after write?
|
||||
* If the user had already triggered autocomplete for this field (cache was
|
||||
* populated), the write would update IDB but the stale cache entry would be
|
||||
* returned for the next 60 seconds. Deleting it forces a fresh IDB read the
|
||||
* next time the user types, ensuring they see the newly captured values.
|
||||
*/
|
||||
export const captureFromValuesApi = (
|
||||
ctx: StreamContext,
|
||||
fieldName: string,
|
||||
entries: Array<{ key: string }>,
|
||||
): void => {
|
||||
const values = entries
|
||||
.map((e) => String(e.key))
|
||||
.filter((v) => v.length > 0 && v.length <= 200);
|
||||
|
||||
if (values.length === 0) return;
|
||||
|
||||
const key = makeKey(ctx, fieldName);
|
||||
scheduleWrite(async () => {
|
||||
await fieldValueDB.mergeValues(
|
||||
key,
|
||||
values,
|
||||
"values_api",
|
||||
MAX_VALUES_PER_FIELD,
|
||||
FIELD_VALUE_TTL_MS,
|
||||
);
|
||||
readCache.delete(key);
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Capture field values from search result hits.
|
||||
* Called from useStreamFields.updateFieldValues() after existing UI update.
|
||||
* Limits to first 100 hits; skips high-cardinality and excluded fields.
|
||||
* Uses a single IDB transaction for all fields to minimise transaction overhead.
|
||||
*
|
||||
* Why hits.slice(0, 100) even though the logs page already caps at 100?
|
||||
* Defensive programming — this function is a public export and could be called
|
||||
* by future code that passes a larger array. The slice keeps the contract clear.
|
||||
*
|
||||
* Why one mergeMultipleValues call instead of N mergeValues calls?
|
||||
* Each IDB transaction has a fixed overhead (open, commit, IDB lock). If a
|
||||
* stream has 40 fields, 40 individual transactions = 40× that overhead.
|
||||
* One transaction with 40 operations = 1× overhead. Substantially cheaper
|
||||
* for wide schemas.
|
||||
*
|
||||
* Why 5% chance for housekeeping (evictExpired + trimToMaxFields)?
|
||||
* evictExpired scans the entire IDB by_expires index — on 5000 records that
|
||||
* takes 10–20ms even in an idle callback. Running it on every search wastes
|
||||
* that time when 99% of searches produce no newly expired records.
|
||||
* 1-in-20 gives periodic cleanup with 1/20th the cumulative cost.
|
||||
*/
|
||||
export const captureFromSearchHits = (
|
||||
ctx: StreamContext,
|
||||
hits: Record<string, any>[],
|
||||
schemaFields: string[],
|
||||
): void => {
|
||||
// Early return avoids scheduling a no-op idle callback
|
||||
if (!hits || hits.length === 0) return;
|
||||
|
||||
scheduleWrite(async () => {
|
||||
const extracted = extractValuesFromHits(
|
||||
hits.slice(0, 100),
|
||||
schemaFields,
|
||||
MAX_VALUES_PER_FIELD,
|
||||
);
|
||||
|
||||
const entries = Object.entries(extracted).map(([field, values]) => ({
|
||||
key: makeKey(ctx, field),
|
||||
values,
|
||||
source: "search_hits" as const,
|
||||
}));
|
||||
|
||||
if (entries.length === 0) return;
|
||||
|
||||
// Single transaction for all fields — N times cheaper than Promise.all(N writes)
|
||||
await fieldValueDB.mergeMultipleValues(
|
||||
entries,
|
||||
MAX_VALUES_PER_FIELD,
|
||||
FIELD_VALUE_TTL_MS,
|
||||
);
|
||||
|
||||
// Invalidate read-cache for written keys so the next autocomplete read
|
||||
// returns fresh values instead of the pre-write stale cache entry
|
||||
for (const { key } of entries) readCache.delete(key);
|
||||
|
||||
// Opportunistically sweep — 5% chance per capture so cost is amortized
|
||||
if (Math.random() < 0.05) {
|
||||
await fieldValueDB.evictExpired();
|
||||
await fieldValueDB.trimToMaxFields(MAX_FIELDS_STORED);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Read stored values for a field — used by autocomplete suggestion provider.
|
||||
* Uses in-memory cache (1-minute TTL, 500-entry cap) to avoid IndexedDB
|
||||
* round-trips during typing. Never throws — returns empty array on any failure.
|
||||
*
|
||||
* Why return [] on error instead of propagating?
|
||||
* IDB can be unavailable in private browsing mode or certain enterprise
|
||||
* browser configurations. Users in those environments should still be able to
|
||||
* use the query editor — they just won't see value suggestions. Crashing or
|
||||
* showing an error for a non-critical enhancement would be a worse experience.
|
||||
*
|
||||
* Why check Date.now() - cached.ts < TTL instead of storing expiresAt?
|
||||
* Minor: computing expiresAt = Date.now() + TTL at write time and comparing
|
||||
* at read time is equivalent. The ts approach is slightly more readable
|
||||
* because the TTL constant appears only in one place (READ_CACHE_TTL_MS).
|
||||
*/
|
||||
export const getFieldValuesForSuggestion = async (
|
||||
ctx: StreamContext,
|
||||
fieldName: string,
|
||||
): Promise<string[]> => {
|
||||
const key = makeKey(ctx, fieldName);
|
||||
const cached = readCache.get(key);
|
||||
if (cached) {
|
||||
if (Date.now() - cached.ts < READ_CACHE_TTL_MS) {
|
||||
return cached.values;
|
||||
}
|
||||
// Expired — remove so we don't leak memory
|
||||
readCache.delete(key);
|
||||
}
|
||||
try {
|
||||
const values = await fieldValueDB.getValues(key);
|
||||
setCacheEntry(key, values);
|
||||
return values;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
@@ -17,6 +17,7 @@ import { ref, type Ref } from "vue";
|
||||
import store from "@/stores";
|
||||
import { generateTraceContext } from "@/utils/zincutils";
|
||||
import useHttpStreaming from "@/composables/useStreamingSearch";
|
||||
import { captureFromValuesApi } from "@/composables/useFieldValueStore";
|
||||
|
||||
export interface FieldValueEntry {
|
||||
key: string;
|
||||
@@ -159,6 +160,19 @@ const useFieldValuesStream = () => {
|
||||
});
|
||||
});
|
||||
|
||||
// [NEW] Background capture — does not block handleResponse return
|
||||
if (chunkValues.length > 0 && fieldName) {
|
||||
captureFromValuesApi(
|
||||
{
|
||||
org: store.state.selectedOrganization.identifier,
|
||||
streamType: payload?.queryReq?.stream_type ?? "logs",
|
||||
streamName: streamName ?? "",
|
||||
},
|
||||
fieldName,
|
||||
chunkValues,
|
||||
);
|
||||
}
|
||||
|
||||
// Append in paginated (load-more) mode; replace on fresh load.
|
||||
if (isAppend) {
|
||||
streamFieldValues.value[fieldName][streamName].values = [
|
||||
|
||||
@@ -22,6 +22,7 @@ import { useRouter } from "vue-router";
|
||||
import { searchState } from "@/composables/useLogs/searchState";
|
||||
import useStreams from "@/composables/useStreams";
|
||||
import useSqlSuggestions from "@/composables/useSuggestions";
|
||||
import { captureFromSearchHits } from "@/composables/useFieldValueStore";
|
||||
|
||||
import {
|
||||
useLocalLogFilterField,
|
||||
@@ -53,13 +54,34 @@ export const useStreamFields = () => {
|
||||
|
||||
const updateFieldValues = () => {
|
||||
try {
|
||||
const hits = searchObj.data.queryResults.hits;
|
||||
|
||||
// [NEW] Background capture into IndexedDB — must be placed BEFORE the
|
||||
// loop below because that loop contains an early `return` when it
|
||||
// encounters excluded fields (pre-existing behaviour), which would
|
||||
// prevent this code from running if placed after.
|
||||
if (hits?.length > 0) {
|
||||
const schemaFields = (
|
||||
searchObj.data.stream.selectedStreamFields ?? []
|
||||
).map((f: any) => f.name ?? f);
|
||||
captureFromSearchHits(
|
||||
{
|
||||
org: searchObj.organizationIdentifier,
|
||||
streamType: searchObj.data.stream.streamType ?? "logs",
|
||||
streamName: searchObj.data.stream.selectedStream?.[0] ?? "",
|
||||
},
|
||||
hits,
|
||||
schemaFields,
|
||||
);
|
||||
}
|
||||
|
||||
const excludedFields = [
|
||||
store.state.zoConfig.timestamp_column,
|
||||
"log",
|
||||
"msg",
|
||||
];
|
||||
// searchObj.data.queryResults.hits.forEach((item: { [x: string]: any }) => {
|
||||
for (const item of searchObj.data.queryResults.hits) {
|
||||
for (const item of hits) {
|
||||
// Create set for each field values and add values to corresponding set
|
||||
// Object.keys(item).forEach((key) => {
|
||||
for (const key of Object.keys(item)) {
|
||||
|
||||
@@ -1,423 +1,307 @@
|
||||
// Copyright 2023 OpenObserve Inc.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
|
||||
// --- mocks before imports ---
|
||||
// ─── Mock Vuex useStore ───────────────────────────────────────────────────────
|
||||
vi.mock("vuex", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("vuex")>();
|
||||
return {
|
||||
...actual,
|
||||
useStore: vi.fn(() => ({
|
||||
state: { zoConfig: { timestamp_column: "_timestamp" } },
|
||||
})),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("vuex", () => ({
|
||||
useStore: vi.fn(() => ({
|
||||
state: {
|
||||
zoConfig: { timestamp_column: "_timestamp" },
|
||||
},
|
||||
})),
|
||||
// ─── Mock IDB so no real storage is touched ───────────────────────────────────
|
||||
vi.mock("@/composables/useFieldValueStore", () => ({
|
||||
getFieldValuesForSuggestion: vi.fn().mockResolvedValue([]),
|
||||
}));
|
||||
|
||||
import useSuggestions from "./useSuggestions";
|
||||
import { getFieldValuesForSuggestion } from "@/composables/useFieldValueStore";
|
||||
import useSqlSuggestions from "./useSuggestions";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ─── helper: build composable with common defaults ────────────────────────────
|
||||
const makeComposable = (overrides: {
|
||||
storedValues?: string[];
|
||||
inSessionValues?: Record<string, string[]>;
|
||||
} = {}) => {
|
||||
const { storedValues = [], inSessionValues = {} } = overrides;
|
||||
vi.mocked(getFieldValuesForSuggestion).mockResolvedValue(storedValues);
|
||||
|
||||
describe("useSuggestions", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
const c = useSqlSuggestions();
|
||||
c.autoCompleteData.value.org = "myorg";
|
||||
c.autoCompleteData.value.streamType = "logs";
|
||||
c.autoCompleteData.value.streamName = "http_logs";
|
||||
c.autoCompleteData.value.fieldValues = Object.fromEntries(
|
||||
Object.entries(inSessionValues).map(([k, v]) => [k, new Set(v)]),
|
||||
);
|
||||
c.autoCompleteData.value.popup.open = vi.fn();
|
||||
return c;
|
||||
};
|
||||
|
||||
const run = async (
|
||||
c: ReturnType<typeof useSqlSuggestions>,
|
||||
query: string,
|
||||
cursorIndex?: number,
|
||||
) => {
|
||||
c.autoCompleteData.value.query = query;
|
||||
(c.autoCompleteData.value as any).cursorIndex = cursorIndex ?? query.length;
|
||||
await c.getSuggestions();
|
||||
return c.autoCompleteKeywords.value;
|
||||
};
|
||||
|
||||
// ─── operator detection ───────────────────────────────────────────────────────
|
||||
|
||||
describe("analyzeSqlWhereClause — operator detection", () => {
|
||||
beforeEach(() => vi.clearAllMocks());
|
||||
|
||||
const operators: [string, string][] = [
|
||||
["=", "status = "],
|
||||
["!=", "status != "],
|
||||
["<>", "status <> "],
|
||||
[">=", "code >= "],
|
||||
["<=", "code <= "],
|
||||
[">", "code > "],
|
||||
["<", "code < "],
|
||||
];
|
||||
|
||||
it.each(operators)("detects field %s operator", async (_op, query) => {
|
||||
const c = makeComposable({ storedValues: ["200"] });
|
||||
const keywords = await run(c, query);
|
||||
expect(keywords.some((k: any) => k.kind === "Value")).toBe(true);
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Return value structure
|
||||
// -------------------------------------------------------------------------
|
||||
describe("return value structure", () => {
|
||||
it("returns all expected properties", () => {
|
||||
const result = useSuggestions();
|
||||
|
||||
expect(result).toHaveProperty("autoCompleteData");
|
||||
expect(result).toHaveProperty("autoCompleteKeywords");
|
||||
expect(result).toHaveProperty("autoCompleteSuggestions");
|
||||
expect(result).toHaveProperty("loading");
|
||||
expect(result).toHaveProperty("getSuggestions");
|
||||
expect(result).toHaveProperty("updateFieldKeywords");
|
||||
expect(result).toHaveProperty("updateFunctionKeywords");
|
||||
expect(result).toHaveProperty("defaultSuggestions");
|
||||
});
|
||||
|
||||
it("loading is a ref defaulting to false", () => {
|
||||
const { loading } = useSuggestions();
|
||||
expect(loading.value).toBe(false);
|
||||
});
|
||||
|
||||
it("autoCompleteKeywords starts as an empty array", () => {
|
||||
const { autoCompleteKeywords } = useSuggestions();
|
||||
expect(autoCompleteKeywords.value).toEqual([]);
|
||||
});
|
||||
|
||||
it("autoCompleteSuggestions starts as an empty array", () => {
|
||||
const { autoCompleteSuggestions } = useSuggestions();
|
||||
expect(autoCompleteSuggestions.value).toEqual([]);
|
||||
});
|
||||
it("detects IN (", async () => {
|
||||
const c = makeComposable({ storedValues: ["200"] });
|
||||
const q = "status IN (";
|
||||
const keywords = await run(c, q, q.length);
|
||||
expect(keywords.some((k: any) => k.kind === "Value")).toBe(true);
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// defaultSuggestions
|
||||
// -------------------------------------------------------------------------
|
||||
describe("defaultSuggestions", () => {
|
||||
it("is a non-empty array", () => {
|
||||
const { defaultSuggestions } = useSuggestions();
|
||||
expect(Array.isArray(defaultSuggestions)).toBe(true);
|
||||
expect(defaultSuggestions.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("every entry has label, kind, and insertText properties", () => {
|
||||
const { defaultSuggestions } = useSuggestions();
|
||||
defaultSuggestions.forEach((s: any) => {
|
||||
expect(s).toHaveProperty("label");
|
||||
expect(s).toHaveProperty("kind");
|
||||
expect(s).toHaveProperty("insertText");
|
||||
});
|
||||
});
|
||||
|
||||
it("contains match_all suggestion", () => {
|
||||
const { defaultSuggestions } = useSuggestions();
|
||||
const hasMatchAll = defaultSuggestions.some((s: any) => {
|
||||
const label =
|
||||
typeof s.label === "function" ? s.label("test") : s.label;
|
||||
return label.includes("match_all");
|
||||
});
|
||||
expect(hasMatchAll).toBe(true);
|
||||
});
|
||||
|
||||
it("contains str_match suggestion", () => {
|
||||
const { defaultSuggestions } = useSuggestions();
|
||||
const has = defaultSuggestions.some((s: any) => {
|
||||
const label =
|
||||
typeof s.label === "function" ? s.label("test") : s.label;
|
||||
return label.includes("str_match");
|
||||
});
|
||||
expect(has).toBe(true);
|
||||
});
|
||||
|
||||
it("contains histogram suggestion", () => {
|
||||
const { defaultSuggestions } = useSuggestions();
|
||||
const has = defaultSuggestions.some((s: any) => {
|
||||
const label =
|
||||
typeof s.label === "function" ? s.label("field") : s.label;
|
||||
return label.includes("histogram");
|
||||
});
|
||||
expect(has).toBe(true);
|
||||
});
|
||||
|
||||
it("function-label entries produce correct output when called with a keyword", () => {
|
||||
const { defaultSuggestions } = useSuggestions();
|
||||
const matchAll = defaultSuggestions.find((s: any) => {
|
||||
const label =
|
||||
typeof s.label === "function" ? s.label("") : s.label;
|
||||
return label.startsWith("match_all(");
|
||||
});
|
||||
expect(matchAll).toBeDefined();
|
||||
expect(typeof matchAll!.label).toBe("function");
|
||||
expect((matchAll!.label as Function)("hello")).toBe("match_all('hello')");
|
||||
});
|
||||
it("detects NOT IN (", async () => {
|
||||
const c = makeComposable({ storedValues: ["200"] });
|
||||
const q = "status NOT IN (";
|
||||
const keywords = await run(c, q, q.length);
|
||||
expect(keywords.some((k: any) => k.kind === "Value")).toBe(true);
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// updateFieldKeywords
|
||||
// -------------------------------------------------------------------------
|
||||
describe("updateFieldKeywords", () => {
|
||||
it("populates autoCompleteKeywords with field entries", () => {
|
||||
const { updateFieldKeywords, autoCompleteKeywords } = useSuggestions();
|
||||
|
||||
updateFieldKeywords([
|
||||
{ name: "level" },
|
||||
{ name: "service_name" },
|
||||
]);
|
||||
|
||||
const fieldLabels = autoCompleteKeywords.value
|
||||
.filter((k: any) => k.kind === "Field")
|
||||
.map((k: any) => k.label);
|
||||
|
||||
expect(fieldLabels).toContain("level");
|
||||
expect(fieldLabels).toContain("service_name");
|
||||
});
|
||||
|
||||
it("excludes the timestamp_column from field keywords", () => {
|
||||
const { updateFieldKeywords, autoCompleteKeywords } = useSuggestions();
|
||||
|
||||
updateFieldKeywords([{ name: "_timestamp" }, { name: "level" }]);
|
||||
|
||||
const fieldLabels = autoCompleteKeywords.value
|
||||
.filter((k: any) => k.kind === "Field")
|
||||
.map((k: any) => k.label);
|
||||
|
||||
expect(fieldLabels).not.toContain("_timestamp");
|
||||
expect(fieldLabels).toContain("level");
|
||||
});
|
||||
|
||||
it("sets kind to Field for each field entry", () => {
|
||||
const { updateFieldKeywords, autoCompleteKeywords } = useSuggestions();
|
||||
|
||||
updateFieldKeywords([{ name: "host" }]);
|
||||
|
||||
const fieldEntry = autoCompleteKeywords.value.find(
|
||||
(k: any) => k.label === "host",
|
||||
);
|
||||
expect(fieldEntry?.kind).toBe("Field");
|
||||
});
|
||||
|
||||
it("sets insertTextRules to InsertAsSnippet for field entries", () => {
|
||||
const { updateFieldKeywords, autoCompleteKeywords } = useSuggestions();
|
||||
|
||||
updateFieldKeywords([{ name: "host" }]);
|
||||
|
||||
const fieldEntry = autoCompleteKeywords.value.find(
|
||||
(k: any) => k.label === "host",
|
||||
);
|
||||
expect(fieldEntry?.insertTextRules).toBe("InsertAsSnippet");
|
||||
});
|
||||
|
||||
it("also appends default SQL keywords to autoCompleteKeywords", () => {
|
||||
const { updateFieldKeywords, autoCompleteKeywords } = useSuggestions();
|
||||
|
||||
updateFieldKeywords([{ name: "level" }]);
|
||||
|
||||
const keywordLabels = autoCompleteKeywords.value
|
||||
.filter((k: any) => k.kind === "Keyword")
|
||||
.map((k: any) => k.label);
|
||||
|
||||
expect(keywordLabels).toContain("and");
|
||||
expect(keywordLabels).toContain("or");
|
||||
});
|
||||
|
||||
it("handles empty fields array without error", () => {
|
||||
const { updateFieldKeywords, autoCompleteKeywords } = useSuggestions();
|
||||
|
||||
expect(() => updateFieldKeywords([])).not.toThrow();
|
||||
// Only default keywords and operators present (no Field entries)
|
||||
const fieldEntries = autoCompleteKeywords.value.filter(
|
||||
(k: any) => k.kind === "Field",
|
||||
);
|
||||
expect(fieldEntries.length).toBe(0);
|
||||
});
|
||||
it("detects LIKE (space)", async () => {
|
||||
const c = makeComposable({ storedValues: ["api"] });
|
||||
const keywords = await run(c, "msg LIKE ");
|
||||
expect(keywords.some((k: any) => k.kind === "Value")).toBe(true);
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// updateFunctionKeywords
|
||||
// -------------------------------------------------------------------------
|
||||
describe("updateFunctionKeywords", () => {
|
||||
it("adds function entries with kind Function", () => {
|
||||
const { updateFunctionKeywords, autoCompleteKeywords } = useSuggestions();
|
||||
|
||||
updateFunctionKeywords([{ name: "myFunc", args: "(x)" }]);
|
||||
|
||||
const funcEntry = autoCompleteKeywords.value.find(
|
||||
(k: any) => k.kind === "Function",
|
||||
);
|
||||
expect(funcEntry?.label).toBe("myFunc");
|
||||
});
|
||||
|
||||
it("concatenates name and args for insertText", () => {
|
||||
const { updateFunctionKeywords, autoCompleteKeywords } = useSuggestions();
|
||||
|
||||
updateFunctionKeywords([{ name: "transform", args: "(field, value)" }]);
|
||||
|
||||
const funcEntry = autoCompleteKeywords.value.find(
|
||||
(k: any) => k.label === "transform",
|
||||
);
|
||||
expect(funcEntry?.insertText).toBe("transform(field, value)");
|
||||
});
|
||||
|
||||
it("handles empty functions array without error", () => {
|
||||
const { updateFunctionKeywords, autoCompleteKeywords } = useSuggestions();
|
||||
|
||||
expect(() => updateFunctionKeywords([])).not.toThrow();
|
||||
const funcEntries = autoCompleteKeywords.value.filter(
|
||||
(k: any) => k.kind === "Function",
|
||||
);
|
||||
expect(funcEntries.length).toBe(0);
|
||||
});
|
||||
|
||||
it("replaces previously set function keywords on second call", () => {
|
||||
const { updateFunctionKeywords, autoCompleteKeywords } = useSuggestions();
|
||||
|
||||
updateFunctionKeywords([{ name: "funcA", args: "()" }]);
|
||||
updateFunctionKeywords([{ name: "funcB", args: "()" }]);
|
||||
|
||||
const funcLabels = autoCompleteKeywords.value
|
||||
.filter((k: any) => k.kind === "Function")
|
||||
.map((k: any) => k.label);
|
||||
|
||||
expect(funcLabels).not.toContain("funcA");
|
||||
expect(funcLabels).toContain("funcB");
|
||||
});
|
||||
it("detects LIKE with open quote", async () => {
|
||||
const c = makeComposable({ storedValues: ["api"] });
|
||||
const q = "msg LIKE '";
|
||||
const keywords = await run(c, q, q.length);
|
||||
expect(keywords.some((k: any) => k.kind === "Value")).toBe(true);
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// getSuggestions — field-value path
|
||||
// -------------------------------------------------------------------------
|
||||
describe("getSuggestions – field-value lookup", () => {
|
||||
it("populates autoCompleteKeywords with field values when column=value pattern is detected", async () => {
|
||||
const {
|
||||
getSuggestions,
|
||||
autoCompleteData,
|
||||
autoCompleteKeywords,
|
||||
} = useSuggestions();
|
||||
|
||||
// Simulate SQL like: "level = " with cursor at end
|
||||
autoCompleteData.value.query = "level = ";
|
||||
autoCompleteData.value.position.cursorIndex = 7;
|
||||
autoCompleteData.value.fieldValues = {
|
||||
level: new Set(["error", "warn", "info"]),
|
||||
};
|
||||
autoCompleteData.value.popup = {
|
||||
open: vi.fn(),
|
||||
close: vi.fn(),
|
||||
};
|
||||
|
||||
await getSuggestions();
|
||||
|
||||
const valueLabels = autoCompleteKeywords.value.map((k: any) => k.label);
|
||||
expect(valueLabels).toContain("error");
|
||||
expect(valueLabels).toContain("warn");
|
||||
expect(valueLabels).toContain("info");
|
||||
});
|
||||
|
||||
it("wraps field values in single quotes for insertText", async () => {
|
||||
const {
|
||||
getSuggestions,
|
||||
autoCompleteData,
|
||||
autoCompleteKeywords,
|
||||
} = useSuggestions();
|
||||
|
||||
autoCompleteData.value.query = "level = ";
|
||||
autoCompleteData.value.position.cursorIndex = 7;
|
||||
autoCompleteData.value.fieldValues = {
|
||||
level: new Set(["error"]),
|
||||
};
|
||||
autoCompleteData.value.popup = { open: vi.fn(), close: vi.fn() };
|
||||
|
||||
await getSuggestions();
|
||||
|
||||
const entry = autoCompleteKeywords.value.find(
|
||||
(k: any) => k.label === "error",
|
||||
);
|
||||
expect(entry?.insertText).toBe("'error'");
|
||||
});
|
||||
|
||||
it("calls popup.open after populating field-value keywords", async () => {
|
||||
const {
|
||||
getSuggestions,
|
||||
autoCompleteData,
|
||||
} = useSuggestions();
|
||||
|
||||
const mockOpen = vi.fn();
|
||||
autoCompleteData.value.query = "level = ";
|
||||
autoCompleteData.value.position.cursorIndex = 7;
|
||||
autoCompleteData.value.fieldValues = {
|
||||
level: new Set(["error"]),
|
||||
};
|
||||
autoCompleteData.value.popup = { open: mockOpen, close: vi.fn() };
|
||||
|
||||
await getSuggestions();
|
||||
|
||||
expect(mockOpen).toHaveBeenCalledWith("level = ");
|
||||
});
|
||||
|
||||
it("clears autoCompleteSuggestions when field-value path is taken", async () => {
|
||||
const {
|
||||
getSuggestions,
|
||||
autoCompleteData,
|
||||
autoCompleteSuggestions,
|
||||
} = useSuggestions();
|
||||
|
||||
autoCompleteData.value.query = "level = ";
|
||||
autoCompleteData.value.position.cursorIndex = 7;
|
||||
autoCompleteData.value.fieldValues = {
|
||||
level: new Set(["error"]),
|
||||
};
|
||||
autoCompleteData.value.popup = { open: vi.fn(), close: vi.fn() };
|
||||
|
||||
await getSuggestions();
|
||||
|
||||
expect(autoCompleteSuggestions.value).toEqual([]);
|
||||
});
|
||||
it("detects NOT LIKE", async () => {
|
||||
const c = makeComposable({ storedValues: ["api"] });
|
||||
const q = "path NOT LIKE '";
|
||||
const keywords = await run(c, q, q.length);
|
||||
expect(keywords.some((k: any) => k.kind === "Value")).toBe(true);
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// getSuggestions — general / non-field-value path
|
||||
// -------------------------------------------------------------------------
|
||||
describe("getSuggestions – general path", () => {
|
||||
it("falls through to general suggestions when no field-value match", async () => {
|
||||
const {
|
||||
getSuggestions,
|
||||
autoCompleteData,
|
||||
autoCompleteKeywords,
|
||||
autoCompleteSuggestions,
|
||||
} = useSuggestions();
|
||||
|
||||
autoCompleteData.value.query = "some random query";
|
||||
autoCompleteData.value.position.cursorIndex = 17;
|
||||
autoCompleteData.value.fieldValues = {};
|
||||
autoCompleteData.value.popup = { open: vi.fn(), close: vi.fn() };
|
||||
|
||||
await getSuggestions();
|
||||
|
||||
// Should populate with default keywords (and, or, etc.)
|
||||
const keywordLabels = autoCompleteKeywords.value
|
||||
.filter((k: any) => k.kind === "Keyword")
|
||||
.map((k: any) => k.label);
|
||||
expect(keywordLabels).toContain("and");
|
||||
// Should also populate autoCompleteSuggestions with function suggestions
|
||||
expect(autoCompleteSuggestions.value.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("includes operator suggestions (<, >, =) in general path", async () => {
|
||||
const {
|
||||
getSuggestions,
|
||||
autoCompleteData,
|
||||
autoCompleteKeywords,
|
||||
} = useSuggestions();
|
||||
|
||||
autoCompleteData.value.query = "query text";
|
||||
autoCompleteData.value.position.cursorIndex = 10;
|
||||
autoCompleteData.value.fieldValues = {};
|
||||
autoCompleteData.value.popup = { open: vi.fn(), close: vi.fn() };
|
||||
|
||||
await getSuggestions();
|
||||
|
||||
const operatorLabels = autoCompleteKeywords.value
|
||||
.filter((k: any) => k.kind === "Operator")
|
||||
.map((k: any) => k.label);
|
||||
expect(operatorLabels).toContain("=");
|
||||
expect(operatorLabels).toContain("!=");
|
||||
});
|
||||
it("detects str_match(field, )", async () => {
|
||||
const c = makeComposable({ storedValues: ["frontend"] });
|
||||
const q = "str_match(service, '";
|
||||
const keywords = await run(c, q, q.length);
|
||||
expect(keywords.some((k: any) => k.kind === "Value")).toBe(true);
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// autoCompleteData default structure
|
||||
// -------------------------------------------------------------------------
|
||||
describe("autoCompleteData default structure", () => {
|
||||
it("has correct initial fieldValues, query, and position", () => {
|
||||
const { autoCompleteData } = useSuggestions();
|
||||
it("detects fuzzy_match(field, )", async () => {
|
||||
const c = makeComposable({ storedValues: ["frontend"] });
|
||||
const q = "fuzzy_match(service, '";
|
||||
const keywords = await run(c, q, q.length);
|
||||
expect(keywords.some((k: any) => k.kind === "Value")).toBe(true);
|
||||
});
|
||||
|
||||
expect(autoCompleteData.value.fieldValues).toEqual({});
|
||||
expect(autoCompleteData.value.query).toBe("");
|
||||
expect(autoCompleteData.value.position.cursorIndex).toBe(0);
|
||||
});
|
||||
it("handles auto-closed bracket IN () — cursor between ( and )", async () => {
|
||||
const c = makeComposable({ storedValues: ["200"] });
|
||||
const full = "status IN ()";
|
||||
// cursor is between ( and ) — Monaco offset 11, getCursorIndex = offset-1 = 10.
|
||||
// slice(0, 10+1) = "status IN (" which matches the IN regex.
|
||||
const keywords = await run(c, full, 10);
|
||||
expect(keywords.some((k: any) => k.kind === "Value")).toBe(true);
|
||||
});
|
||||
|
||||
it("popup.open and popup.close are functions by default", () => {
|
||||
const { autoCompleteData } = useSuggestions();
|
||||
it("detects partial typed value after operator: field = '20", async () => {
|
||||
const c = makeComposable({ storedValues: ["200"] });
|
||||
const q = "status = '20";
|
||||
const keywords = await run(c, q, q.length);
|
||||
expect(keywords.some((k: any) => k.kind === "Value")).toBe(true);
|
||||
});
|
||||
|
||||
expect(typeof autoCompleteData.value.popup.open).toBe("function");
|
||||
expect(typeof autoCompleteData.value.popup.close).toBe("function");
|
||||
it("falls through to keywords when no operator context", async () => {
|
||||
const c = makeComposable({ storedValues: [] });
|
||||
const keywords = await run(c, "SELECT * FROM stream WHERE ");
|
||||
expect(keywords.some((k: any) => k.kind === "Value")).toBe(false);
|
||||
expect(keywords.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("shows default keywords for empty query", async () => {
|
||||
const c = makeComposable({ storedValues: [] });
|
||||
const keywords = await run(c, "");
|
||||
expect(keywords.some((k: any) => k.kind === "Value")).toBe(false);
|
||||
expect(keywords.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── insertText quoting logic ─────────────────────────────────────────────────
|
||||
|
||||
describe("getSuggestions — insertText quoting", () => {
|
||||
beforeEach(() => vi.clearAllMocks());
|
||||
|
||||
it("wraps string value in single quotes when no open quote", async () => {
|
||||
const c = makeComposable({ storedValues: ["prod"] });
|
||||
const keywords = await run(c, "env = ");
|
||||
const item = keywords.find((k: any) => k.label === "prod");
|
||||
expect(item?.insertText).toBe("'prod'");
|
||||
});
|
||||
|
||||
it("closes only when open quote already typed", async () => {
|
||||
const c = makeComposable({ storedValues: ["prod"] });
|
||||
const q = "env = '";
|
||||
const keywords = await run(c, q, q.length);
|
||||
const item = keywords.find((k: any) => k.label === "prod");
|
||||
expect(item?.insertText).toBe("prod'");
|
||||
});
|
||||
|
||||
it("inserts numeric values without quotes", async () => {
|
||||
const c = makeComposable({ storedValues: ["200", "404"] });
|
||||
const keywords = await run(c, "status = ");
|
||||
const item = keywords.find((k: any) => k.label === "200");
|
||||
expect(item?.insertText).toBe("200");
|
||||
});
|
||||
|
||||
it("inserts boolean 'true' without quotes", async () => {
|
||||
const c = makeComposable({ storedValues: ["true"] });
|
||||
const keywords = await run(c, "active = ");
|
||||
const item = keywords.find((k: any) => k.label === "true");
|
||||
expect(item?.insertText).toBe("true");
|
||||
});
|
||||
|
||||
it("inserts boolean 'false' without quotes", async () => {
|
||||
const c = makeComposable({ storedValues: ["false"] });
|
||||
const keywords = await run(c, "active = ");
|
||||
const item = keywords.find((k: any) => k.label === "false");
|
||||
expect(item?.insertText).toBe("false");
|
||||
});
|
||||
|
||||
it("sortText starts with \\x01 so values sort above keywords", async () => {
|
||||
const c = makeComposable({ storedValues: ["200"] });
|
||||
const keywords = await run(c, "status = ");
|
||||
const item = keywords.find((k: any) => k.kind === "Value");
|
||||
expect(item?.sortText.startsWith("\x01")).toBe(true);
|
||||
});
|
||||
|
||||
it("all value suggestions have kind = 'Value'", async () => {
|
||||
const c = makeComposable({ storedValues: ["200", "404", "500"] });
|
||||
const keywords = await run(c, "status = ");
|
||||
const valueItems = keywords.filter((k: any) => k.kind === "Value");
|
||||
expect(valueItems).toHaveLength(3);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── merge: in-session vs stored ─────────────────────────────────────────────
|
||||
|
||||
describe("getSuggestions — in-session and stored value merge", () => {
|
||||
beforeEach(() => vi.clearAllMocks());
|
||||
|
||||
it("shows in-session values from fieldValues prop", async () => {
|
||||
const c = makeComposable({
|
||||
storedValues: [],
|
||||
inSessionValues: { status: ["200", "404"] },
|
||||
});
|
||||
const keywords = await run(c, "status = ");
|
||||
const labels = keywords.map((k: any) => k.label);
|
||||
expect(labels).toContain("200");
|
||||
expect(labels).toContain("404");
|
||||
});
|
||||
|
||||
it("merges stored values with in-session values", async () => {
|
||||
const c = makeComposable({
|
||||
storedValues: ["500"],
|
||||
inSessionValues: { status: ["200"] },
|
||||
});
|
||||
const keywords = await run(c, "status = ");
|
||||
const labels = keywords.map((k: any) => k.label);
|
||||
expect(labels).toContain("200");
|
||||
expect(labels).toContain("500");
|
||||
});
|
||||
|
||||
it("deduplicates values appearing in both sources", async () => {
|
||||
const c = makeComposable({
|
||||
storedValues: ["200", "500"],
|
||||
inSessionValues: { status: ["200", "404"] },
|
||||
});
|
||||
const keywords = await run(c, "status = ");
|
||||
const labels = keywords.map((k: any) => k.label);
|
||||
expect(labels.filter((l: string) => l === "200")).toHaveLength(1);
|
||||
expect(labels).toContain("404");
|
||||
expect(labels).toContain("500");
|
||||
});
|
||||
|
||||
it("skips IDB read when stream context is missing", async () => {
|
||||
const c = useSqlSuggestions();
|
||||
// org/streamType/streamName left as empty strings
|
||||
c.autoCompleteData.value.fieldValues = {};
|
||||
c.autoCompleteData.value.popup.open = vi.fn();
|
||||
await run(c, "status = ");
|
||||
expect(getFieldValuesForSuggestion).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("falls through to updateAutoComplete when merged is empty", async () => {
|
||||
const c = makeComposable({ storedValues: [] });
|
||||
const keywords = await run(c, "status = ");
|
||||
expect(keywords.some((k: any) => k.kind === "Value")).toBe(false);
|
||||
expect(keywords.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── IDB context forwarding ───────────────────────────────────────────────────
|
||||
|
||||
describe("getSuggestions — IDB context forwarding", () => {
|
||||
it("passes org/streamType/streamName to getFieldValuesForSuggestion", async () => {
|
||||
vi.mocked(getFieldValuesForSuggestion).mockResolvedValue(["prod"]);
|
||||
const c = useSqlSuggestions();
|
||||
c.autoCompleteData.value.org = "acme";
|
||||
c.autoCompleteData.value.streamType = "traces";
|
||||
c.autoCompleteData.value.streamName = "default";
|
||||
c.autoCompleteData.value.fieldValues = {};
|
||||
c.autoCompleteData.value.popup.open = vi.fn();
|
||||
await run(c, "env = ");
|
||||
expect(getFieldValuesForSuggestion).toHaveBeenCalledWith(
|
||||
{ org: "acme", streamType: "traces", streamName: "default" },
|
||||
"env",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── updateFieldKeywords ──────────────────────────────────────────────────────
|
||||
|
||||
describe("updateFieldKeywords", () => {
|
||||
it("adds field keywords excluding timestamp column", () => {
|
||||
const c = useSqlSuggestions();
|
||||
c.updateFieldKeywords([
|
||||
{ name: "status" },
|
||||
{ name: "env" },
|
||||
{ name: "_timestamp" },
|
||||
]);
|
||||
const labels = c.autoCompleteKeywords.value.map((k: any) => k.label);
|
||||
expect(labels).toContain("status");
|
||||
expect(labels).toContain("env");
|
||||
expect(labels).not.toContain("_timestamp");
|
||||
});
|
||||
|
||||
it("sets kind = Field for all field keywords", () => {
|
||||
const c = useSqlSuggestions();
|
||||
c.updateFieldKeywords([{ name: "status" }, { name: "env" }]);
|
||||
const fieldItems = c.autoCompleteKeywords.value.filter(
|
||||
(k: any) => k.kind === "Field",
|
||||
);
|
||||
expect(fieldItems.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { ref, nextTick } from "vue";
|
||||
import { useStore } from "vuex";
|
||||
import { getFieldValuesForSuggestion } from "@/composables/useFieldValueStore";
|
||||
|
||||
const useSqlSuggestions = () => {
|
||||
const defaultKeywords = [
|
||||
@@ -285,6 +286,12 @@ const useSqlSuggestions = () => {
|
||||
open: (val: string) => {},
|
||||
close: (val: string) => {},
|
||||
},
|
||||
// [NEW] Stream context — set by SearchBar.vue when a stream is selected.
|
||||
// Required to build the composite IDB key: "org|streamType|streamName|fieldName".
|
||||
// Without these three, getFieldValuesForSuggestion cannot look up stored values.
|
||||
org: "",
|
||||
streamType: "",
|
||||
streamName: "",
|
||||
});
|
||||
const autoCompleteSuggestions: any = ref([]);
|
||||
const loading = ref(false);
|
||||
@@ -302,52 +309,153 @@ const useSqlSuggestions = () => {
|
||||
meta: {
|
||||
label: "",
|
||||
value: "",
|
||||
hasOpenQuote: false,
|
||||
},
|
||||
};
|
||||
|
||||
const columnValueRegex = /(\w+)\s*=\s*$|\w+\s+IN\s+\($/i;
|
||||
// Detects whether the cursor is positioned after an operator that expects
|
||||
// a value, and extracts the field name to the left of that operator.
|
||||
//
|
||||
// 4 alternatives — each captures the field name in a different group:
|
||||
// match[1]: symbolic operators = != <> >= <= > <
|
||||
// e.g. "status = ", "code >= ", "env != 'pro"
|
||||
// match[2]: IN / NOT IN (
|
||||
// e.g. "status IN (", "env NOT IN ('pro"
|
||||
// match[3]: LIKE / NOT LIKE
|
||||
// e.g. "msg LIKE '", "path NOT LIKE '%api"
|
||||
// match[4]: str_match / fuzzy_match function second argument
|
||||
// e.g. "str_match(field, ", "fuzzy_match(field, 'par"
|
||||
//
|
||||
// Why >=/<= appear before >/<:
|
||||
// Regex alternation is left-to-right. If > appeared first, ">=" would
|
||||
// match on ">" and stop, leaving "=" unmatched. Longer tokens must come first.
|
||||
//
|
||||
// Why (?:'[^']*)?$ at the end of each alternative:
|
||||
// Allows the regex to match even after the user has typed an opening quote
|
||||
// and a partial value. Without it, "status = 'pro" would not match — we
|
||||
// would stop showing value suggestions the moment the user starts typing.
|
||||
const columnValueRegex =
|
||||
/(\w+)\s*(?:!=|<>|>=|<=|=|>|<)\s*(?:'[^']*)?$|(\w+)\s+(?:NOT\s+)?IN\s+\(\s*(?:'[^']*)?$|(\w+)\s+(?:NOT\s+)?LIKE\s*(?:'[^']*)?$|(?:str_match|fuzzy_match)\s*\(\s*(\w+)\s*,\s*(?:'[^']*)?$/i;
|
||||
|
||||
const match = columnValueRegex.exec(whereClause);
|
||||
if (match !== null) {
|
||||
const column = match[1];
|
||||
|
||||
if (cursorIndex <= match.index + match[0].length - 1) {
|
||||
labelMeta.focusOn = "value";
|
||||
labelMeta.isFocused = true;
|
||||
labelMeta.meta.label = column;
|
||||
}
|
||||
// Slice the query at the cursor position before matching, so that the $
|
||||
// anchor lands at the cursor — not at the end of the full query string.
|
||||
//
|
||||
// Why this matters — auto-closing brackets example:
|
||||
// User types "status IN (" → editor auto-inserts ")" → full string is "status IN ()"
|
||||
// Without slicing: $ anchors after ")" → regex does NOT match
|
||||
// After slicing at cursor (between "(" and ")"): text is "status IN (" → matches
|
||||
//
|
||||
// cursorIndex = 0 means uninitialized (position.cursorIndex default).
|
||||
// Fall back to the full string length so we don't break the pre-existing
|
||||
// behaviour where no cursor tracking was done.
|
||||
const endIdx = cursorIndex > 0 ? cursorIndex + 1 : whereClause.length;
|
||||
const textUpToCursor = whereClause.slice(0, endIdx);
|
||||
const match = columnValueRegex.exec(textUpToCursor);
|
||||
if (match) {
|
||||
labelMeta.focusOn = "value";
|
||||
labelMeta.isFocused = true;
|
||||
// Pick whichever capture group matched — only one will be non-null.
|
||||
labelMeta.meta.label = match[1] ?? match[2] ?? match[3] ?? match[4];
|
||||
// True when the user has already typed an opening quote, e.g. field = 'partial
|
||||
// In this case insertText should be value' (close only), not 'value'
|
||||
labelMeta.meta.hasOpenQuote = /'[^']*$/.test(textUpToCursor);
|
||||
}
|
||||
return labelMeta;
|
||||
}
|
||||
|
||||
const getSuggestions = async () => {
|
||||
autoCompleteKeywords.value = [];
|
||||
const cursorIndex = autoCompleteData.value.position.cursorIndex;
|
||||
|
||||
// SearchBar sets autoCompleteData.value.cursorIndex at the top level.
|
||||
// autoCompleteData.value.position.cursorIndex is the legacy field — it
|
||||
// is never updated by SearchBar and stays 0. We read the top-level one
|
||||
// first and fall back to position.cursorIndex for safety.
|
||||
const cursorIndex =
|
||||
(autoCompleteData.value as any).cursorIndex ??
|
||||
autoCompleteData.value.position.cursorIndex;
|
||||
|
||||
// Determine if the cursor is currently after an operator expecting a value.
|
||||
// If so, sqlWhereClause.meta.label is the field name (e.g. "status").
|
||||
const sqlWhereClause = analyzeSqlWhereClause(
|
||||
autoCompleteData.value.query,
|
||||
cursorIndex,
|
||||
);
|
||||
if (
|
||||
sqlWhereClause.meta.label &&
|
||||
autoCompleteData.value.fieldValues[sqlWhereClause.meta.label]
|
||||
) {
|
||||
const values = Array.from(
|
||||
autoCompleteData.value.fieldValues[sqlWhereClause.meta.label] ||
|
||||
new Set(),
|
||||
).map((item) => {
|
||||
return {
|
||||
label: item,
|
||||
insertText: `'${item}'`,
|
||||
kind: "Keyword",
|
||||
};
|
||||
});
|
||||
|
||||
autoCompleteKeywords.value = [...values];
|
||||
autoCompleteSuggestions.value = [];
|
||||
autoCompleteData.value.popup.open(autoCompleteData.value.query);
|
||||
} else {
|
||||
updateAutoComplete();
|
||||
if (sqlWhereClause.meta.label) {
|
||||
const fieldName = sqlWhereClause.meta.label;
|
||||
|
||||
// [EXISTING] In-session values — collected from the current session's
|
||||
// search result hits and stored in the reactive fieldValues prop.
|
||||
// These are available immediately (no async) but disappear on page reload.
|
||||
const inSessionValues = Array.from(
|
||||
autoCompleteData.value.fieldValues[fieldName] || new Set(),
|
||||
) as string[];
|
||||
|
||||
// [NEW] Persisted values — read from IndexedDB (via in-memory cache).
|
||||
// These survive page reloads and accumulate across multiple searches.
|
||||
// Guard: only query IDB if stream context is set — without org/streamType/
|
||||
// streamName we cannot build the composite key and would get empty results.
|
||||
let storedValues: string[] = [];
|
||||
if (
|
||||
autoCompleteData.value.org &&
|
||||
autoCompleteData.value.streamType &&
|
||||
autoCompleteData.value.streamName
|
||||
) {
|
||||
storedValues = await getFieldValuesForSuggestion(
|
||||
{
|
||||
org: autoCompleteData.value.org,
|
||||
streamType: autoCompleteData.value.streamType,
|
||||
streamName: autoCompleteData.value.streamName,
|
||||
},
|
||||
fieldName,
|
||||
);
|
||||
}
|
||||
|
||||
// Merge in-session + stored, deduplicate via Set.
|
||||
// inSessionValues come first so they appear at the top of the dropdown
|
||||
// (they are from the current search context, most relevant).
|
||||
// storedValues from previous sessions fill in anything not seen today.
|
||||
const merged = [...new Set([...inSessionValues, ...storedValues])];
|
||||
|
||||
if (merged.length > 0) {
|
||||
const hasOpenQuote = sqlWhereClause.meta.hasOpenQuote;
|
||||
|
||||
// Build Monaco suggestion items with smart quoting and sort order.
|
||||
autoCompleteKeywords.value = merged.map((item, idx) => {
|
||||
const isNumeric = item !== "" && !isNaN(Number(item));
|
||||
const isBoolean = item === "true" || item === "false";
|
||||
|
||||
// Quoting rules:
|
||||
// numeric / boolean → no quotes (SQL: status = 200, active = true)
|
||||
// string, open quote already typed → close only (field = 'val → val')
|
||||
// string, no open quote → wrap fully (field = → 'val')
|
||||
let insertText: string;
|
||||
if (isNumeric || isBoolean) {
|
||||
insertText = item;
|
||||
} else if (hasOpenQuote) {
|
||||
insertText = `${item}'`; // user already typed the opening '
|
||||
} else {
|
||||
insertText = `'${item}'`;
|
||||
}
|
||||
|
||||
// \x01 (ASCII 1) is the lowest-sorting printable character.
|
||||
// Prefixing sortText with it ensures value suggestions always appear
|
||||
// ABOVE keywords ("and", "or", "like") and functions in the Monaco
|
||||
// dropdown, which sort by their label (starting with a letter > \x01).
|
||||
// The padded index preserves the order of values as returned from IDB.
|
||||
const sortText = `\x01${String(idx).padStart(6, "0")}`;
|
||||
return { label: item, insertText, kind: "Value", sortText };
|
||||
});
|
||||
|
||||
autoCompleteData.value.popup.open(autoCompleteData.value.query);
|
||||
// Return early — do NOT fall through to updateAutoComplete().
|
||||
// We don't want keywords/fields/functions mixed into a value dropdown.
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// No value context detected — show the default keywords, fields, functions.
|
||||
updateAutoComplete();
|
||||
};
|
||||
|
||||
const updateAutoComplete = () => {
|
||||
|
||||
@@ -195,6 +195,7 @@ import { useSearchBar } from "@/composables/useLogs/useSearchBar";
|
||||
import { useSearchStream } from "@/composables/useLogs/useSearchStream";
|
||||
import { searchState } from "@/composables/useLogs/searchState";
|
||||
import { useStreamFields } from "@/composables/useLogs/useStreamFields";
|
||||
import { captureFromValuesApi } from "@/composables/useFieldValueStore";
|
||||
|
||||
interface Filter {
|
||||
fieldName: string;
|
||||
@@ -1546,6 +1547,19 @@ export default defineComponent({
|
||||
});
|
||||
});
|
||||
|
||||
// [NEW] Background capture into IndexedDB — does not block return
|
||||
if (streamValues.length > 0 && fieldName) {
|
||||
captureFromValuesApi(
|
||||
{
|
||||
org: store.state.selectedOrganization.identifier,
|
||||
streamType: searchObj.data.stream.streamType ?? "logs",
|
||||
streamName: streamName ?? "",
|
||||
},
|
||||
fieldName,
|
||||
streamValues,
|
||||
);
|
||||
}
|
||||
|
||||
// Append to existing stream values in paginated mode; replace on fresh load.
|
||||
if (isAppend) {
|
||||
const existing =
|
||||
|
||||
@@ -2780,6 +2780,13 @@ export default defineComponent({
|
||||
autoCompleteData.value.fieldValues = props.fieldValues;
|
||||
autoCompleteData.value.popup.open =
|
||||
queryEditorRef?.value?.triggerAutoComplete;
|
||||
// [NEW] Pass stream context for IndexedDB value lookups
|
||||
autoCompleteData.value.org =
|
||||
store.state.selectedOrganization.identifier;
|
||||
autoCompleteData.value.streamType =
|
||||
searchObj.data.stream.streamType ?? "logs";
|
||||
autoCompleteData.value.streamName =
|
||||
searchObj.data.stream.selectedStream?.[0] ?? "";
|
||||
getSuggestions();
|
||||
};
|
||||
|
||||
|
||||
@@ -496,6 +496,15 @@ export default defineComponent({
|
||||
autoCompleteData.value.fieldValues = props.fieldValues;
|
||||
autoCompleteData.value.popup.open =
|
||||
queryEditorRef.value.triggerAutoComplete;
|
||||
// [NEW] Set stream context so getSuggestions can read stored values from
|
||||
// IndexedDB. Traces field expansion already writes to IDB via
|
||||
// captureFromValuesApi (useFieldValuesStream) with stream_type="traces",
|
||||
// so values are already being captured — this just enables the read side.
|
||||
autoCompleteData.value.org =
|
||||
store.state.selectedOrganization.identifier;
|
||||
autoCompleteData.value.streamType = "traces";
|
||||
autoCompleteData.value.streamName =
|
||||
searchObj.data.stream.selectedStream.value ?? "";
|
||||
getSuggestions();
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { extractValuesFromHits, EXCLUDED_FIELDS } from "./fieldValueUtils";
|
||||
|
||||
// ─── EXCLUDED_FIELDS ─────────────────────────────────────────────────────────
|
||||
|
||||
describe("EXCLUDED_FIELDS", () => {
|
||||
it("contains timestamp fields", () => {
|
||||
expect(EXCLUDED_FIELDS.has("_timestamp")).toBe(true);
|
||||
expect(EXCLUDED_FIELDS.has("timestamp")).toBe(true);
|
||||
expect(EXCLUDED_FIELDS.has("@timestamp")).toBe(true);
|
||||
});
|
||||
|
||||
it("contains id fields", () => {
|
||||
expect(EXCLUDED_FIELDS.has("_id")).toBe(true);
|
||||
expect(EXCLUDED_FIELDS.has("__id")).toBe(true);
|
||||
});
|
||||
|
||||
it("contains OO internal fields", () => {
|
||||
expect(EXCLUDED_FIELDS.has("_stream")).toBe(true);
|
||||
expect(EXCLUDED_FIELDS.has("_all")).toBe(true);
|
||||
});
|
||||
|
||||
it("contains free-text fields", () => {
|
||||
expect(EXCLUDED_FIELDS.has("log")).toBe(true);
|
||||
expect(EXCLUDED_FIELDS.has("message")).toBe(true);
|
||||
expect(EXCLUDED_FIELDS.has("msg")).toBe(true);
|
||||
expect(EXCLUDED_FIELDS.has("body")).toBe(true);
|
||||
});
|
||||
|
||||
it("does not contain regular fields", () => {
|
||||
expect(EXCLUDED_FIELDS.has("status")).toBe(false);
|
||||
expect(EXCLUDED_FIELDS.has("env")).toBe(false);
|
||||
expect(EXCLUDED_FIELDS.has("service")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── extractValuesFromHits ────────────────────────────────────────────────────
|
||||
|
||||
describe("extractValuesFromHits", () => {
|
||||
const schemaFields = ["status", "env", "service"];
|
||||
|
||||
// ── basic happy path ────────────────────────────────────────────────────────
|
||||
|
||||
it("extracts unique values per field from hits", () => {
|
||||
const hits = [
|
||||
{ status: "200", env: "prod" },
|
||||
{ status: "404", env: "staging" },
|
||||
{ status: "200", env: "prod" },
|
||||
];
|
||||
const result = extractValuesFromHits(hits, schemaFields, 50);
|
||||
expect(result.status).toEqual(expect.arrayContaining(["200", "404"]));
|
||||
expect(result.status).toHaveLength(2);
|
||||
expect(result.env).toEqual(expect.arrayContaining(["prod", "staging"]));
|
||||
});
|
||||
|
||||
it("deduplicates values across rows", () => {
|
||||
const hits = [
|
||||
{ status: "200" },
|
||||
{ status: "200" },
|
||||
{ status: "200" },
|
||||
];
|
||||
const result = extractValuesFromHits(hits, schemaFields, 50);
|
||||
expect(result.status).toEqual(["200"]);
|
||||
});
|
||||
|
||||
it("returns empty object when hits is empty", () => {
|
||||
expect(extractValuesFromHits([], schemaFields, 50)).toEqual({});
|
||||
});
|
||||
|
||||
// ── excluded fields ─────────────────────────────────────────────────────────
|
||||
|
||||
it("skips _timestamp field", () => {
|
||||
const hits = [{ _timestamp: "1711000000", status: "200" }];
|
||||
const result = extractValuesFromHits(hits, [], 50);
|
||||
expect(result._timestamp).toBeUndefined();
|
||||
expect(result.status).toEqual(["200"]);
|
||||
});
|
||||
|
||||
it("skips all EXCLUDED_FIELDS", () => {
|
||||
const hit: Record<string, any> = {
|
||||
status: "200",
|
||||
_timestamp: "1711",
|
||||
timestamp: "1711",
|
||||
"@timestamp": "1711",
|
||||
_id: "abc",
|
||||
__id: "abc",
|
||||
_stream: "logs",
|
||||
_all: "all",
|
||||
log: "some log",
|
||||
message: "some msg",
|
||||
msg: "short",
|
||||
body: "body text",
|
||||
};
|
||||
const result = extractValuesFromHits([hit], [], 50);
|
||||
expect(Object.keys(result)).toEqual(["status"]);
|
||||
});
|
||||
|
||||
// ── schema filtering ────────────────────────────────────────────────────────
|
||||
|
||||
it("restricts to schema fields when schemaFields is non-empty", () => {
|
||||
const hits = [{ status: "200", extra_field: "noise", env: "prod" }];
|
||||
const result = extractValuesFromHits(hits, ["status"], 50);
|
||||
expect(result.status).toBeDefined();
|
||||
expect(result.extra_field).toBeUndefined();
|
||||
expect(result.env).toBeUndefined();
|
||||
});
|
||||
|
||||
it("allows all fields when schemaFields is empty", () => {
|
||||
const hits = [{ status: "200", custom_field: "value" }];
|
||||
const result = extractValuesFromHits(hits, [], 50);
|
||||
expect(result.status).toBeDefined();
|
||||
expect(result.custom_field).toBeDefined();
|
||||
});
|
||||
|
||||
// ── value length filter ─────────────────────────────────────────────────────
|
||||
|
||||
it("skips values longer than MAX_VALUE_LENGTH (150 chars)", () => {
|
||||
const longValue = "a".repeat(151);
|
||||
const hits = [{ status: longValue }];
|
||||
const result = extractValuesFromHits(hits, [], 50);
|
||||
expect(result.status).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps values exactly at MAX_VALUE_LENGTH (150 chars)", () => {
|
||||
const exactValue = "a".repeat(150);
|
||||
const hits = [{ status: exactValue }];
|
||||
const result = extractValuesFromHits(hits, [], 50);
|
||||
expect(result.status).toEqual([exactValue]);
|
||||
});
|
||||
|
||||
it("keeps values shorter than MAX_VALUE_LENGTH", () => {
|
||||
const hits = [{ status: "200" }];
|
||||
const result = extractValuesFromHits(hits, [], 50);
|
||||
expect(result.status).toEqual(["200"]);
|
||||
});
|
||||
|
||||
it("skips empty string values", () => {
|
||||
const hits = [{ status: "" }];
|
||||
const result = extractValuesFromHits(hits, [], 50);
|
||||
expect(result.status).toBeUndefined();
|
||||
});
|
||||
|
||||
// ── junk value filter ───────────────────────────────────────────────────────
|
||||
|
||||
it("skips 'null' string values", () => {
|
||||
const hits = [{ status: "null" }];
|
||||
const result = extractValuesFromHits(hits, [], 50);
|
||||
expect(result.status).toBeUndefined();
|
||||
});
|
||||
|
||||
it("skips 'undefined' string values", () => {
|
||||
const hits = [{ status: "undefined" }];
|
||||
const result = extractValuesFromHits(hits, [], 50);
|
||||
expect(result.status).toBeUndefined();
|
||||
});
|
||||
|
||||
it("skips 'NaN' string values", () => {
|
||||
const hits = [{ status: "NaN" }];
|
||||
const result = extractValuesFromHits(hits, [], 50);
|
||||
expect(result.status).toBeUndefined();
|
||||
});
|
||||
|
||||
it("converts non-string values to string (numbers, booleans)", () => {
|
||||
const hits = [{ status: 200, active: true }];
|
||||
const result = extractValuesFromHits(hits, [], 50);
|
||||
expect(result.status).toEqual(["200"]);
|
||||
expect(result.active).toEqual(["true"]);
|
||||
});
|
||||
|
||||
it("handles null rawValue by converting to empty string (skipped)", () => {
|
||||
const hits = [{ status: null }];
|
||||
const result = extractValuesFromHits(hits, [], 50);
|
||||
expect(result.status).toBeUndefined();
|
||||
});
|
||||
|
||||
// ── maxValuesPerField cap ───────────────────────────────────────────────────
|
||||
|
||||
it("caps values to maxValuesPerField in final result", () => {
|
||||
const hits = Array.from({ length: 10 }, (_, i) => ({ status: `v${i}` }));
|
||||
const result = extractValuesFromHits(hits, [], 3);
|
||||
expect(result.status).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("returns all values when total is below cap", () => {
|
||||
const hits = [{ status: "200" }, { status: "404" }];
|
||||
const result = extractValuesFromHits(hits, [], 50);
|
||||
expect(result.status).toHaveLength(2);
|
||||
});
|
||||
|
||||
// ── multiple fields ─────────────────────────────────────────────────────────
|
||||
|
||||
it("handles multiple fields across multiple hits", () => {
|
||||
const hits = [
|
||||
{ status: "200", env: "prod", service: "api" },
|
||||
{ status: "500", env: "staging", service: "worker" },
|
||||
];
|
||||
const result = extractValuesFromHits(hits, schemaFields, 50);
|
||||
expect(Object.keys(result).sort()).toEqual(["env", "service", "status"]);
|
||||
expect(result.status).toHaveLength(2);
|
||||
expect(result.env).toHaveLength(2);
|
||||
expect(result.service).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("handles hits with missing fields gracefully", () => {
|
||||
const hits = [
|
||||
{ status: "200" },
|
||||
{ env: "prod" },
|
||||
{ status: "404", env: "staging" },
|
||||
];
|
||||
const result = extractValuesFromHits(hits, schemaFields, 50);
|
||||
expect(result.status).toEqual(expect.arrayContaining(["200", "404"]));
|
||||
expect(result.env).toEqual(expect.arrayContaining(["prod", "staging"]));
|
||||
});
|
||||
|
||||
it("handles 100 hits without errors (max page size)", () => {
|
||||
const hits = Array.from({ length: 100 }, (_, i) => ({
|
||||
status: i % 5 === 0 ? "200" : "404",
|
||||
env: i % 2 === 0 ? "prod" : "staging",
|
||||
}));
|
||||
const result = extractValuesFromHits(hits, schemaFields, 50);
|
||||
expect(result.status).toHaveLength(2);
|
||||
expect(result.env).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("does not store trace_id-like values if they pass length filter", () => {
|
||||
const hits = [{ trace_id: "4bf92f3577b34da6" }];
|
||||
const result = extractValuesFromHits(hits, [], 50);
|
||||
// 16 chars, within limit, not excluded — should be stored
|
||||
expect(result.trace_id).toEqual(["4bf92f3577b34da6"]);
|
||||
});
|
||||
|
||||
it("skips stack-trace-like values over 150 chars", () => {
|
||||
const hits = [{ error_detail: "Error: at " + "x".repeat(150) }];
|
||||
const result = extractValuesFromHits(hits, [], 50);
|
||||
expect(result.error_detail).toBeUndefined();
|
||||
});
|
||||
|
||||
it("handles empty schemaFields set with no hits matching schema", () => {
|
||||
const hits = [{ _timestamp: "1711", log: "text" }];
|
||||
const result = extractValuesFromHits(hits, [], 50);
|
||||
expect(Object.keys(result)).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,102 @@
|
||||
// Copyright 2026 OpenObserve Inc.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
/**
|
||||
* Extracts field → values mapping from search result hits.
|
||||
* Designed to be called inside an idle callback — must be fast.
|
||||
*/
|
||||
|
||||
// Fields that should never be stored (meta / high-cardinality)
|
||||
export const EXCLUDED_FIELDS = new Set([
|
||||
"_timestamp",
|
||||
"_id",
|
||||
"__id",
|
||||
"timestamp",
|
||||
"@timestamp",
|
||||
"_stream",
|
||||
"_all",
|
||||
"body",
|
||||
"log",
|
||||
"message",
|
||||
"msg",
|
||||
]);
|
||||
|
||||
const MAX_VALUE_LENGTH = 150; // chars — skip values too long to be useful filter candidates
|
||||
|
||||
/**
|
||||
* Extract deduplicated field→values from an array of search hits.
|
||||
*
|
||||
* @param hits - Raw search result records (max ~100 recommended)
|
||||
* @param schemaFields - Stream schema fields to restrict extraction to.
|
||||
* Pass an empty array to allow all fields.
|
||||
* @param maxValuesPerField - Cap on unique values per field
|
||||
* @returns Map of fieldName → unique values array
|
||||
*/
|
||||
export const extractValuesFromHits = (
|
||||
hits: Record<string, any>[],
|
||||
schemaFields: string[],
|
||||
maxValuesPerField: number,
|
||||
): Record<string, string[]> => {
|
||||
// fieldSets accumulates unique values per field using a Set so duplicates are
|
||||
// automatically ignored — no need to check "have I seen this value before?".
|
||||
const fieldSets: Map<string, Set<string>> = new Map();
|
||||
// Convert to a Set for O(1) lookups instead of O(n) Array.includes().
|
||||
const schemaSet = new Set(schemaFields);
|
||||
|
||||
for (const hit of hits) {
|
||||
for (const [field, rawValue] of Object.entries(hit)) {
|
||||
// Skip meta / high-cardinality fields that are never useful as filter
|
||||
// values: timestamps (every value unique), IDs, internal OO fields,
|
||||
// and free-text fields like log/message that would just add noise.
|
||||
if (EXCLUDED_FIELDS.has(field)) continue;
|
||||
|
||||
// If the stream has a known schema, only keep values for schema fields.
|
||||
// This prevents storing values from dynamic/temporary fields that users
|
||||
// would never filter on. When schemaFields is empty we allow all fields.
|
||||
if (schemaSet.size > 0 && !schemaSet.has(field)) continue;
|
||||
|
||||
const value = String(rawValue ?? "");
|
||||
// Empty values and very long values (stack traces, JSON blobs, base64)
|
||||
// are useless as autocomplete suggestions — skip them.
|
||||
if (value.length === 0 || value.length > MAX_VALUE_LENGTH) continue;
|
||||
// These are JavaScript serialisation artifacts, not real field values.
|
||||
if (value === "null" || value === "undefined" || value === "NaN")
|
||||
continue;
|
||||
|
||||
if (!fieldSets.has(field)) fieldSets.set(field, new Set());
|
||||
fieldSets.get(field)!.add(value);
|
||||
|
||||
// Early memory cap: if a field accumulates too many unique values while
|
||||
// scanning (e.g. a high-cardinality "user_id" field with 100 unique
|
||||
// values across 100 hits), trim it back to maxValuesPerField immediately
|
||||
// instead of waiting until the loop finishes. Without this, a single
|
||||
// high-cardinality field could hold hundreds of strings in memory at once.
|
||||
// We keep the LAST maxValuesPerField entries (most recently seen hits)
|
||||
// because they are the freshest. The threshold of ×2 avoids trimming on
|
||||
// every insert — it fires at most once per field during extraction.
|
||||
if (fieldSets.get(field)!.size > maxValuesPerField * 2) {
|
||||
const arr = Array.from(fieldSets.get(field)!);
|
||||
fieldSets.set(field, new Set(arr.slice(-maxValuesPerField)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Convert each field's Set into a plain capped array for IndexedDB storage.
|
||||
const result: Record<string, string[]> = {};
|
||||
for (const [field, valueSet] of fieldSets.entries()) {
|
||||
result[field] = Array.from(valueSet).slice(0, maxValuesPerField);
|
||||
}
|
||||
return result;
|
||||
};
|
||||
Reference in New Issue
Block a user