From c2ac5fb89cc5c714fb08e9bb45b66e8ade2b043f Mon Sep 17 00:00:00 2001 From: Ashish Kolhe <35160958+oasisk@users.noreply.github.com> Date: Thu, 9 Jul 2026 20:21:01 +0530 Subject: [PATCH] chore: usage ui cloud changes (#13112) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Adds a **daily usage view** to Cloud Billing → Usage for orgs with usage reporting enabled, layered on top of the existing billing-cycle cards. Cloud-only. ### Frontend (cloud billing components) - **Merged into the existing Usage tab** (no separate tab): the billable usage cards are always shown; when self-usage reporting is on, a dashboards **date-range picker** (in the toolbar next to the GB/MB toggle) drives **both** the cards and a daily chart. - **Daily line chart** — a single query via `PanelSchemaRenderer` over the org's own `usage` stream, broken down by event into **Ingestion / Search / Pipeline / Remote Pipeline** lines (connect-nulls on). - **Six billable usage tiles** on one row with **automatic unit scaling** (GB→TB→PB / MB→GB→…) so large numbers stay readable; AI Credits shown as a count. - Cards re-fetch over the calendar range (derived `days`, hardened against bad values). ### Backend (all cloud-gated) - `usage_stream_enabled` **defaults to `ZO_USAGE_REPORT_TO_OWN_ORG`** in cloud builds, so orgs report to their own `usage` stream by default; a per-org opt-out persists an explicit `false`. OSS / self-hosted default stays `false`. - **Reserved self-reporting streams** (`usage`, `stats`, `triggers`, `errors`, `data_retention_usage`) are blocked from user **create / ingest / delete** — cloud-only; the internal self-reporting writer (`IngestionRequest::Usage`) is exempt. - `get_org_usage`: a **missing `usage` / `data_retention_usage` stream is tolerated** (treated as zero) instead of failing the whole request with a 400. ## Notes - No enterprise-repo or DB-migration changes; backend logic stays in OSS behind `#[cfg(feature = "cloud")]`, per the OSS/Enterprise layer map. - `Cargo.toml` / `Cargo.lock` are intentionally **not** included (local `downloadO2.sh` dev-setup). ## Testing - `cargo check` (OSS) and `cargo check --features cloud` — pass; `cargo fmt` clean. - Frontend: **411 billing unit tests** pass; `vue-tsc` typecheck clean; production build succeeds. --- src/common/meta/organization.rs | 67 +++- src/config/src/meta/self_reporting/usage.rs | 38 ++ src/service/db/organization.rs | 11 +- src/service/logs/bulk.rs | 33 ++ src/service/logs/ingest.rs | 15 + src/service/org_usage.rs | 44 ++- src/service/stream.rs | 39 ++ .../components/billings/Billing.spec.ts | 51 ++- .../components/billings/Billing.vue | 79 +++- .../components/billings/usage.spec.ts | 145 +++++++- .../enterprise/components/billings/usage.vue | 345 +++++++++++------- .../billings/usageDailyPanelSchema.spec.ts | 74 ++++ .../billings/usageDailyPanelSchema.ts | 182 +++++++++ web/src/locales/languages/en.json | 3 + 14 files changed, 945 insertions(+), 181 deletions(-) create mode 100644 web/src/enterprise/components/billings/usageDailyPanelSchema.spec.ts create mode 100644 web/src/enterprise/components/billings/usageDailyPanelSchema.ts diff --git a/src/common/meta/organization.rs b/src/common/meta/organization.rs index f09172e096..5c01107bb8 100644 --- a/src/common/meta/organization.rs +++ b/src/common/meta/organization.rs @@ -387,6 +387,26 @@ fn default_enable_streaming_search() -> bool { false } +/// Default for `usage_stream_enabled`. In **cloud** builds, when self-reporting +/// to the org's own usage stream is turned on globally +/// (`ZO_USAGE_REPORT_TO_OWN_ORG`), orgs opt in by default so their `usage` +/// stream is populated and the daily Usage Trends view works out of the box. +/// Users can still toggle it off per org, which persists an explicit `false` +/// and falls back to the billing-API (cycle-total) view. +/// +/// In OSS and self-hosted enterprise builds — or when the global flag is off — +/// the default stays `false` (nothing writes per-org usage streams by default). +fn default_usage_stream_enabled() -> bool { + #[cfg(feature = "cloud")] + { + config::get_config().common.usage_report_to_own_org + } + #[cfg(not(feature = "cloud"))] + { + false + } +} + #[cfg(feature = "enterprise")] fn default_claim_parser_function() -> String { "".to_string() @@ -453,7 +473,7 @@ pub struct OrganizationSetting { pub dark_mode_theme_color: Option, #[serde(skip_serializing_if = "Option::is_none")] pub max_series_per_query: Option, - #[serde(default)] + #[serde(default = "default_usage_stream_enabled")] pub usage_stream_enabled: bool, #[cfg(feature = "enterprise")] #[serde(default = "default_claim_parser_function")] @@ -490,7 +510,7 @@ impl Default for OrganizationSetting { light_mode_theme_color, dark_mode_theme_color, max_series_per_query: None, - usage_stream_enabled: false, + usage_stream_enabled: default_usage_stream_enabled(), #[cfg(feature = "enterprise")] claim_parser_function: default_claim_parser_function(), cross_links: Vec::new(), @@ -1376,4 +1396,47 @@ mod tests { assert!(obj.contains_key("dark_mode_theme_color")); assert!(obj.contains_key("max_series_per_query")); } + + /// In OSS / self-hosted enterprise builds (cloud feature off), usage stream + /// self-reporting must NOT be on by default — the default helper and the + /// struct default both stay `false`, and a stored setting that omits the + /// field deserializes to `false`. + #[cfg(not(feature = "cloud"))] + #[test] + fn test_usage_stream_enabled_defaults_off_when_not_cloud() { + assert!(!default_usage_stream_enabled()); + assert!(!OrganizationSetting::default().usage_stream_enabled); + + // A stored setting missing the field falls back to the (false) default. + let json = r#"{ + "scrape_interval": 15, + "trace_id_field_name": "trace_id", + "span_id_field_name": "span_id" + }"#; + let parsed: OrganizationSetting = serde_json::from_str(json).unwrap(); + assert!(!parsed.usage_stream_enabled); + } + + /// An explicit value always wins over the default, in every build — this is + /// what makes a per-org opt-out (or opt-in) stick. + #[test] + fn test_usage_stream_enabled_explicit_value_is_respected() { + let json_true = r#"{ + "scrape_interval": 15, + "trace_id_field_name": "trace_id", + "span_id_field_name": "span_id", + "usage_stream_enabled": true + }"#; + let parsed: OrganizationSetting = serde_json::from_str(json_true).unwrap(); + assert!(parsed.usage_stream_enabled); + + let json_false = r#"{ + "scrape_interval": 15, + "trace_id_field_name": "trace_id", + "span_id_field_name": "span_id", + "usage_stream_enabled": false + }"#; + let parsed: OrganizationSetting = serde_json::from_str(json_false).unwrap(); + assert!(!parsed.usage_stream_enabled); + } } diff --git a/src/config/src/meta/self_reporting/usage.rs b/src/config/src/meta/self_reporting/usage.rs index 728e83984c..36b5877d40 100644 --- a/src/config/src/meta/self_reporting/usage.rs +++ b/src/config/src/meta/self_reporting/usage.rs @@ -30,6 +30,25 @@ pub const TRIGGERS_STREAM: &str = "triggers"; pub const ERROR_STREAM: &str = "errors"; pub const DATA_RETENTION_USAGE_STREAM: &str = "data_retention_usage"; +/// The reserved self-reporting stream names. These are written only by internal +/// self-reporting jobs (usage/stats/triggers/errors/data-retention). Users must +/// not be able to create, ingest into, or delete them — doing so would corrupt +/// billing/usage accounting. +pub const RESERVED_SELF_REPORTING_STREAMS: [&str; 5] = [ + USAGE_STREAM, + STATS_STREAM, + TRIGGERS_STREAM, + ERROR_STREAM, + DATA_RETENTION_USAGE_STREAM, +]; + +/// Returns true if `stream_name` is a reserved self-reporting stream that users +/// are not allowed to create, ingest into, or delete. Internal self-reporting +/// writes bypass this via the `IngestionRequest::Usage` channel. +pub fn is_reserved_self_reporting_stream(stream_name: &str) -> bool { + RESERVED_SELF_REPORTING_STREAMS.contains(&stream_name) +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub enum TriggerDataStatus { #[serde(rename = "completed")] @@ -646,6 +665,25 @@ mod tests { assert_eq!(format!("{}", UsageEvent::Other), "Other"); } + #[test] + fn test_is_reserved_self_reporting_stream() { + // All self-reporting streams are reserved. + assert!(is_reserved_self_reporting_stream(USAGE_STREAM)); + assert!(is_reserved_self_reporting_stream(STATS_STREAM)); + assert!(is_reserved_self_reporting_stream(TRIGGERS_STREAM)); + assert!(is_reserved_self_reporting_stream(ERROR_STREAM)); + assert!(is_reserved_self_reporting_stream( + DATA_RETENTION_USAGE_STREAM + )); + assert!(is_reserved_self_reporting_stream("usage")); + + // Ordinary user streams are not. + assert!(!is_reserved_self_reporting_stream("my_logs")); + assert!(!is_reserved_self_reporting_stream("usage_production")); + assert!(!is_reserved_self_reporting_stream("_usage")); + assert!(!is_reserved_self_reporting_stream("")); + } + #[test] fn test_usage_event_from_usage_type() { assert_eq!(UsageEvent::from(UsageType::Bulk), UsageEvent::Ingestion); diff --git a/src/service/db/organization.rs b/src/service/db/organization.rs index 2a73c4b10f..6001ebb56d 100644 --- a/src/service/db/organization.rs +++ b/src/service/db/organization.rs @@ -140,9 +140,13 @@ pub async fn get_org_setting_toggle_ingestion_logs(org_id: &str) -> Result Result { let key = format!("{ORG_SETTINGS_KEY_PREFIX}/{org_id}"); if let Some(v) = ORGANIZATION_SETTING.read().await.get(&key) { @@ -180,6 +184,7 @@ pub async fn org_settings_cache() -> Result<(), anyhow::Error> { .insert(key, json_val); } log::info!("Organization settings Cached"); + Ok(()) } diff --git a/src/service/logs/bulk.rs b/src/service/logs/bulk.rs index c63e591f64..5c7e4e7298 100644 --- a/src/service/logs/bulk.rs +++ b/src/service/logs/bulk.rs @@ -19,6 +19,8 @@ use std::{ }; use axum::body::Bytes; +#[cfg(feature = "cloud")] +use config::meta::self_reporting::usage::is_reserved_self_reporting_stream; use config::{ BLOCKED_STREAMS, TIMESTAMP_COL_NAME, get_config, meta::stream::StreamType, @@ -146,6 +148,37 @@ pub async fn ingest( stream_name = format_stream_name(stream_name); } + // Reject reserved self-reporting streams (usage/stats/triggers/...). + // Bulk is always a user path (internal self-reporting uses the + // non-bulk `IngestionRequest::Usage` channel), so this is safe. + // Cloud-only: OSS / self-hosted may legitimately use these names. + #[cfg(feature = "cloud")] + if is_reserved_self_reporting_stream(&stream_name) { + let err_msg = + format!("stream '{stream_name}' is reserved and cannot be ingested into"); + log::warn!("[LOGS:BULK] {err_msg}"); + bulk_res.errors = true; + let err = BulkResponseError::new( + err_msg.clone(), + stream_name.to_string(), + err_msg, + "0".to_string(), + ); + let mut item = HashMap::new(); + item.insert( + action.to_string(), + BulkResponseItem::new_failed( + stream_name.to_string(), + doc_id.clone().unwrap_or_default(), + err, + Some(value), + stream_name.to_string(), + ), + ); + bulk_res.items.push(item); + continue; // skip + } + // skip blocked streams if !stream_key_cache.contains_key(&stream_name) { let key = format!("{org_id}/{}/{stream_name}", stream_type); diff --git a/src/service/logs/ingest.rs b/src/service/logs/ingest.rs index fbed67f881..227b7bfeaf 100644 --- a/src/service/logs/ingest.rs +++ b/src/service/logs/ingest.rs @@ -20,6 +20,8 @@ use std::{ use axum::http; use chrono::Utc; +#[cfg(feature = "cloud")] +use config::meta::self_reporting::usage::is_reserved_self_reporting_stream; use config::{ ALL_VALUES_COL_NAME, ID_COL_NAME, ORIGINAL_DATA_COL_NAME, TIMESTAMP_COL_NAME, meta::{ @@ -109,6 +111,19 @@ pub async fn ingest( return Err(Error::IngestionError("Stream name is empty".to_string())); } + // Block user ingestion into reserved self-reporting streams + // (usage/stats/triggers/errors/...). The internal self-reporting job writes + // these via `IngestionRequest::Usage` (for which `should_report_usage()` is + // false → `need_usage_report == false`), so it is exempt; any other request + // targeting a reserved stream is a user write and is rejected. Cloud-only: + // OSS / self-hosted may legitimately use these stream names. + #[cfg(feature = "cloud")] + if need_usage_report && is_reserved_self_reporting_stream(&stream_name) { + return Err(Error::IngestionError(format!( + "stream '{stream_name}' is reserved and cannot be ingested into" + ))); + } + // check system resource check_ingestion_allowed(org_id, stream_type, Some(&stream_name)).await?; diff --git a/src/service/org_usage.rs b/src/service/org_usage.rs index d0265f8688..3d10b917b8 100644 --- a/src/service/org_usage.rs +++ b/src/service/org_usage.rs @@ -64,13 +64,20 @@ pub async fn get_org_usage( let (sql, start_time, end_time) = org_usage::create_usage_query_sql_and_time_range(org_id, usage_range, cycle_details); - let mut usage_results = get_usage(sql, start_time, end_time, false) - .await - .map_err(|e| billings::BillingError::PartialUsageResults(e.to_string()))? - .into_iter() - .filter_map(|hit| json::from_value::(hit).ok()) - .filter(|r| r.event.is_billable()) - .collect::>(); + // The main `usage` stream may not exist yet for a brand-new org. As with + // data retention below, a missing stream should read as "no usage" rather + // than failing the whole request. + let mut usage_results = match get_usage(sql, start_time, end_time, false).await { + Ok(hits) => hits + .into_iter() + .filter_map(|hit| json::from_value::(hit).ok()) + .filter(|r| r.event.is_billable()) + .collect::>(), + Err(e) => { + log::warn!("usage query failed for org {org_id}, treating as zero: {e}"); + Vec::new() + } + }; let (data_retention_query, ..) = org_usage::create_data_retention_usage_sql_and_time_range( org_id, @@ -78,12 +85,23 @@ pub async fn get_org_usage( cycle_details, ); - let mut data_retention_results = get_usage(data_retention_query, start_time, end_time, false) - .await - .map_err(|e| billings::BillingError::PartialUsageResults(e.to_string()))? - .into_iter() - .filter_map(|hit| json::from_value::(hit).ok()) - .collect::>(); + // Data-retention usage lives in a separate `data_retention_usage` stream + // that may not exist for an org yet (e.g. no retention usage recorded). A + // failure here (most commonly "stream not found") must NOT fail the whole + // usage response — treat it as zero retention and return the rest. + let mut data_retention_results = + match get_usage(data_retention_query, start_time, end_time, false).await { + Ok(hits) => hits + .into_iter() + .filter_map(|hit| json::from_value::(hit).ok()) + .collect::>(), + Err(e) => { + log::warn!( + "data_retention_usage query failed for org {org_id}, treating as zero: {e}" + ); + Vec::new() + } + }; usage_results.append(&mut data_retention_results); diff --git a/src/service/stream.rs b/src/service/stream.rs index 4724f2fd42..3c7fcd83b5 100644 --- a/src/service/stream.rs +++ b/src/service/stream.rs @@ -20,6 +20,10 @@ use axum::{ response::{IntoResponse, Response as HttpResponse}, }; use chrono::{TimeZone, Timelike, Utc}; +// Reserved self-reporting stream guards are a Cloud-only concern (Cloud manages +// these streams for billing); OSS / self-hosted must not block user streams. +#[cfg(feature = "cloud")] +use config::meta::self_reporting::usage::is_reserved_self_reporting_stream; use config::{ ALL_VALUES_COL_NAME, ORIGINAL_DATA_COL_NAME, SIZE_IN_MB, TIMESTAMP_COL_NAME, get_config, is_local_disk_storage, @@ -243,6 +247,24 @@ pub async fn create_stream( stream_type: StreamType, mut stream: StreamCreate, ) -> Result { + // Reserved self-reporting streams (usage/stats/triggers/errors/...) are + // managed internally by Cloud and must not be user-created — doing so would + // corrupt billing/usage accounting. The internal self-reporting job creates + // its schema directly (not via create_stream), so blocking here is safe. + // Cloud-only: OSS / self-hosted may legitimately use these stream names. + #[cfg(feature = "cloud")] + if is_reserved_self_reporting_stream(stream_name) { + return Ok(( + http::StatusCode::BAD_REQUEST, + [(ERROR_HEADER, "stream name is reserved")], + Json(MetaHttpResponse::error( + http::StatusCode::BAD_REQUEST, + format!("stream name '{stream_name}' is reserved and cannot be created"), + )), + ) + .into_response()); + } + // check if the stream already exists let schema = match infra::schema::get(org_id, stream_name, stream_type).await { Ok(schema) => schema, @@ -949,6 +971,23 @@ pub async fn delete_stream( stream_type: StreamType, del_related_feature_resources: bool, ) -> Result { + // Reserved self-reporting streams (usage/stats/triggers/errors/...) are + // managed internally by Cloud and must not be user-deleted — retention/ + // compaction uses a separate internal path, so blocking this user-facing + // delete is safe and preserves billing/usage accounting. Cloud-only. + #[cfg(feature = "cloud")] + if is_reserved_self_reporting_stream(stream_name) { + return Ok(( + http::StatusCode::BAD_REQUEST, + [(ERROR_HEADER, "stream name is reserved")], + Json(MetaHttpResponse::error( + http::StatusCode::BAD_REQUEST, + format!("stream '{stream_name}' is reserved and cannot be deleted"), + )), + ) + .into_response()); + } + let schema = infra::schema::get_versions(org_id, stream_name, stream_type, None) .await .unwrap(); diff --git a/web/src/enterprise/components/billings/Billing.spec.ts b/web/src/enterprise/components/billings/Billing.spec.ts index acea8a7a42..dfa0c4a8b7 100644 --- a/web/src/enterprise/components/billings/Billing.spec.ts +++ b/web/src/enterprise/components/billings/Billing.spec.ts @@ -5,9 +5,29 @@ import i18n from "@/locales"; import store from "@/test/unit/helpers/store"; -// Mock utils -vi.mock("@/utils/zincutils", () => ({ - getImageURL: vi.fn((imagePath: string) => `img:${imagePath}`) +// Mock utils — partial so the store (pulled in transitively via usage.vue's +// dashboards renderer import) still resolves useLocalOrganization etc. +vi.mock("@/utils/zincutils", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + getImageURL: vi.fn((imagePath: string) => `img:${imagePath}`), + }; +}); + +// usage.vue (rendered by the Billing shell) imports these; stub them so the +// Billing shell test doesn't pull in the dashboards query pipeline. +vi.mock("@/components/dashboards/PanelSchemaRenderer.vue", () => ({ + default: { + name: "PanelSchemaRenderer", + template: "
", + }, +})); +vi.mock("@/components/DateTimePickerDashboard.vue", () => ({ + default: { + name: "DateTimePickerDashboard", + template: "
", + }, })); vi.mock("@/aws-exports", () => ({ @@ -145,8 +165,8 @@ describe("Billing Component", () => { it("should have correct tabs configuration", () => { expect(wrapper.vm.tabs).toEqual( expect.arrayContaining([ - expect.objectContaining({ label: 'Gb', value: "gb" }), - expect.objectContaining({ label: 'Mb', value: "mb" }), + expect.objectContaining({ label: 'GB', value: "gb" }), + expect.objectContaining({ label: 'MB', value: "mb" }), ]) ); }); @@ -657,4 +677,25 @@ describe("Billing Component", () => { expect(() => wrapper.vm.updateActiveTab(undefined)).not.toThrow(); }); }); + + describe("Daily-view date range", () => { + it("exposes usageStreamEnabled from org settings", () => { + store.state.organizationData.organizationSettings.usage_stream_enabled = true; + expect(wrapper.vm.usageStreamEnabled).toBe(true); + store.state.organizationData.organizationSettings.usage_stream_enabled = false; + }); + + it("resolves a relative range into micros and bumps the key", () => { + const before = wrapper.vm.usageStreamEnabled; + wrapper.vm.dateRange = { + valueType: "relative", + relativeTimePeriod: "7d", + }; + wrapper.vm.onRangeChange(); + // getConsumableRelativeTime returns start. v-if="isUsageRoute" class="flex gap-2 items-center justify-end overflow-y-auto shrink-0 mb-2 ml-2 mr-3 px-3 py-2" > -
+ +
.
+ +
@@ -168,7 +180,7 @@ import OTabs from '@/lib/navigation/Tabs/OTabs.vue' import OSelect from '@/lib/forms/Select/OSelect.vue' import OButton from '@/lib/core/Button/OButton.vue' // @ts-ignore -import { defineComponent, ref, onBeforeMount, computed, onMounted, provide, reactive } from "vue"; +import { defineComponent, ref, onBeforeMount, computed, onMounted, provide, reactive, watch } from "vue"; import { useI18n } from "vue-i18n"; import { useStore } from "vuex"; import { useRouter } from "vue-router"; @@ -184,17 +196,22 @@ import BillingService from "@/services/billings"; import OIcon from "@/lib/core/Icon/OIcon.vue"; import OSplitter from "@/lib/core/Splitter/OSplitter.vue"; import UsageMemberList from "./UsageMemberList.vue"; +import DateTimePickerDashboard from "@/components/DateTimePickerDashboard.vue"; +import { getConsumableRelativeTime } from "@/utils/date"; export default defineComponent({ name: "PageIngestion", components: { AppPageHeader, OTabs, ORouteTab, ConfirmDialog, Usage, AppTabs, OSelect, - OIcon, OSplitter, OButton, UsageMemberList }, + OIcon, OSplitter, OButton, UsageMemberList, DateTimePickerDashboard }, setup() { const { t } = useI18n(); const store = useStore(); const router: any = useRouter(); - const billingtab = ref(resolveTab("billings", router.currentRoute.value.name as string, "usage")); + // Default/fallback tab is "plans" — that's where /billings redirects on + // mount, so falling back to "usage" made the Usage tab flash-highlight + // first before the redirect settled. + const billingtab = ref(resolveTab("billings", router.currentRoute.value.name as string, "plans")); const usageDataType = ref(router.currentRoute.value.query.data_type || "gb"); const showSidebar = ref(true); const lastSplitterPosition = ref(200); @@ -311,6 +328,52 @@ export default defineComponent({ const isUsageRoute = computed(() => { return router.currentRoute.value.name == "usage"; }) + // Self-usage reporting on → the Usage tab renders the daily view (which has + // its own date-range picker), so the billing-cycle range dropdown is hidden. + const usageStreamEnabled = computed( + () => + !!store.state?.organizationData?.organizationSettings + ?.usage_stream_enabled, + ); + + // Daily-view date range: the picker lives here (toolbar, next to GB/MB) and + // shares the RESOLVED window (microseconds) with the child usage.vue via + // provide/inject. `key` bumps on every change so the charts re-render. + const dateTimePicker = ref(null); + const dateRange = ref({ + valueType: "relative", + relativeTimePeriod: "7d", + }); + const usageRange = reactive({ + start: (new Date().getTime() - 7 * 24 * 60 * 60 * 1000) * 1000, + end: new Date().getTime() * 1000, + key: 0, + }); + provide("usageRange", usageRange); + const onRangeChange = () => { + const v: any = dateRange.value; + if (v?.valueType === "relative" && v.relativeTimePeriod) { + const r = getConsumableRelativeTime(v.relativeTimePeriod); + if (r) { + usageRange.start = r.startTime; + usageRange.end = r.endTime; + } + } else if (v?.valueType === "absolute" && v.startTime && v.endTime) { + usageRange.start = v.startTime; + usageRange.end = v.endTime; + } else if (dateTimePicker.value?.getConsumableDateTime) { + // Fallback: ask the picker to resolve its current selection. + const d = dateTimePicker.value.getConsumableDateTime(); + if (d?.startTime && d?.endTime) { + usageRange.start = d.startTime; + usageRange.end = d.endTime; + } + } + usageRange.key++; + }; + // Apply on any value change (relative or absolute), not just on @hide, so a + // calendar selection immediately re-renders the charts below. + watch(dateRange, onRangeChange, { deep: true }); const isOrgGroupRoute = computed(() => { return router.currentRoute.value.name == "billing_group"; }) @@ -338,12 +401,12 @@ export default defineComponent({ } const tabs = [ { - label: 'Gb', + label: 'GB', value: "gb", icon: "storage", }, { - label: 'Mb', + label: 'MB', value: "mb", icon: "database", } @@ -359,6 +422,10 @@ export default defineComponent({ splitterModel, headerBasedOnRoute, options, + usageStreamEnabled, + dateTimePicker, + dateRange, + onRangeChange, usageDate, selectUsageDate, isUsageRoute, diff --git a/web/src/enterprise/components/billings/usage.spec.ts b/web/src/enterprise/components/billings/usage.spec.ts index 061bbe2bb4..c335281ed6 100644 --- a/web/src/enterprise/components/billings/usage.spec.ts +++ b/web/src/enterprise/components/billings/usage.spec.ts @@ -47,10 +47,15 @@ vi.mock("vue-router", () => ({ useRouter: () => mockRouter })); -// Mock zincutils -vi.mock("@/utils/zincutils", () => ({ - getImageURL: vi.fn((path) => `mocked-image-url-${path}`), -})); +// Mock zincutils — partial so the store (pulled in transitively via the +// dashboards renderer) still resolves useLocalOrganization etc. +vi.mock("@/utils/zincutils", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + getImageURL: vi.fn((path) => `mocked-image-url-${path}`), + }; +}); // Mock convertBillingData vi.mock("@/utils/billing/convertBillingData", () => ({ @@ -65,6 +70,25 @@ vi.mock("@/components/dashboards/panels/CustomChartRenderer.vue", () => ({ }, })); +// Stub the heavy dashboards renderer + date picker the daily view uses. +vi.mock("@/components/dashboards/PanelSchemaRenderer.vue", () => ({ + default: { + name: "PanelSchemaRenderer", + template: "
", + }, +})); +vi.mock("@/components/DateTimePickerDashboard.vue", () => ({ + default: { + name: "DateTimePickerDashboard", + template: "
", + methods: { + getConsumableDateTime() { + return { startTime: 1000, endTime: 2000 }; + }, + }, + }, +})); + const node = document.createElement("div"); node.setAttribute("id", "app"); document.body.appendChild(node); @@ -95,6 +119,8 @@ describe("Usage Component", () => { provide: { store, $router: mockRouter, + // The daily view injects the resolved range from the Billing shell. + usageRange: { start: 1000, end: 2000, key: 1 }, }, mocks: { $t: (key: string) => key, @@ -234,7 +260,6 @@ describe("Usage Component", () => { expect(wrapper.text()).toContain("Ingestion"); expect(wrapper.text()).toContain("Search"); - expect(wrapper.text()).toContain("Functions"); }); // Test 16: Usage data formatting in tiles @@ -247,9 +272,39 @@ describe("Usage Component", () => { wrapper.vm.dataLoading = false; await nextTick(); - expect(wrapper.text()).toContain("10.50 GB"); - expect(wrapper.text()).toContain("5.25 GB"); - expect(wrapper.text()).toContain("2.75 GB"); + // Value and unit render in separate spans (no literal space in .text()). + expect(wrapper.text()).toContain("10.50"); + expect(wrapper.text()).toContain("5.25"); + expect(wrapper.text()).toContain("GB"); + }); + + it("auto-scales large byte values up from the base unit", async () => { + // base GB: 10930.96 GB → 10.67 TB; 1095652 GB → ~1.05 PB + wrapper.vm.usageData = { + ingestion: "10930.96", + dataretention: "1095652", + search: "5.25", + ai_credits: "42", + }; + wrapper.vm.dataLoading = false; + await nextTick(); + + const tiles = wrapper.vm.usageTiles; + const ing = tiles.find((t: any) => t.key === "ingestion"); + expect(ing.unit).toBe("TB"); + expect(parseFloat(ing.value)).toBeCloseTo(10.67, 1); + + const ret = tiles.find((t: any) => t.key === "dataretention"); + expect(ret.unit).toBe("PB"); + + // small value keeps the base unit + const srch = tiles.find((t: any) => t.key === "search"); + expect(srch.unit).toBe("GB"); + expect(srch.value).toBe("5.25"); + + // AI credits is a plain count, never scaled + const ai = tiles.find((t: any) => t.key === "ai_credits"); + expect(ai.unit).toBe("Credits"); }); @@ -562,7 +617,8 @@ describe("Usage Component", () => { expect(tileTexts).toContain('Ingestion'); expect(tileTexts).toContain('Search'); - expect(tileTexts).toContain('Functions'); + // Functions card was removed (not a billable line item). + expect(tileTexts).not.toContain('Functions'); }); // Test 43: Usage tiles hidden when loading @@ -588,8 +644,10 @@ describe("Usage Component", () => { }); // Test 45: Total usage heading display - it("should display total usage heading", () => { - expect(wrapper.text()).toContain(wrapper.vm.t("billing.totalUsage")); + it("renders the billable usage tiles", () => { + // The "Total Usage" heading was removed; tiles carry the usage now. + const tiles = wrapper.findAll('[data-test="billings-usage-tile"]'); + expect(tiles.length).toBeGreaterThan(0); }); // Test 46: Component container styling @@ -675,4 +733,69 @@ describe("Usage Component", () => { // The member-org selector now lives in Billing.vue (rendered beside this // component) and shares the selection via inject; its formatting/search is // covered in UsageMemberList.spec.ts. + + // --- Daily chart appended below the existing cards (self-usage ON) ----- + describe("Daily chart", () => { + afterEach(() => { + store.state.organizationData.organizationSettings.usage_stream_enabled = false; + }); + + it("hides the daily chart when self-usage is off", async () => { + store.state.organizationData.organizationSettings.usage_stream_enabled = false; + await nextTick(); + expect(wrapper.vm.usageStreamEnabled).toBe(false); + expect(wrapper.find('[data-test="usage-daily-chart"]').exists()).toBe( + false, + ); + }); + + it("shows the daily chart when self-usage is on", async () => { + store.state.organizationData.organizationSettings.usage_stream_enabled = true; + await nextTick(); + expect(wrapper.vm.usageStreamEnabled).toBe(true); + expect(wrapper.find('[data-test="usage-daily-chart"]').exists()).toBe( + true, + ); + }); + + it("builds one combined line panel over the org's usage stream", async () => { + store.state.organizationData.organizationSettings.usage_stream_enabled = true; + store.state.selectedOrganization.identifier = "org-daily"; + await nextTick(); + const schema = wrapper.vm.combinedSchema; + expect(schema.type).toBe("line"); + expect(schema.queries).toHaveLength(1); + expect(schema.queries[0].fields.stream).toBe("usage"); + expect(schema.queries[0].query).toContain("org_id = 'org-daily'"); + expect(schema.queries[0].query).toContain( + "event IN ('Ingestion', 'Search', 'Pipeline', 'RemotePipeline')", + ); + expect(schema.queries[0].query).not.toContain("Functions"); + expect(schema.queries[0].fields.breakdown).toHaveLength(1); + }); + + it("uses the injected range for the chart's time window", async () => { + store.state.organizationData.organizationSettings.usage_stream_enabled = true; + await nextTick(); + // injected usageRange = { start: 1000, end: 2000 } (micros) + expect(wrapper.vm.dailyTimeObj.start_time.getTime()).toBe(1000); + expect(wrapper.vm.dailyTimeObj.end_time.getTime()).toBe(2000); + }); + + it("fetches the cards with a calendar-derived days range when on", async () => { + mockBillingService.get_data_usage.mockClear(); + store.state.organizationData.organizationSettings.usage_stream_enabled = true; + store.state.selectedOrganization.identifier = "org-cal"; + await nextTick(); + await wrapper.vm.getUsage(); + await flushPromises(); + // injected sub-day window rounds up to "1days" + expect(mockBillingService.get_data_usage).toHaveBeenLastCalledWith( + "org-cal", + "1days", + expect.any(String), + undefined, + ); + }); + }); }); diff --git a/web/src/enterprise/components/billings/usage.vue b/web/src/enterprise/components/billings/usage.vue index 8c039724bd..598269872b 100644 --- a/web/src/enterprise/components/billings/usage.vue +++ b/web/src/enterprise/components/billings/usage.vue @@ -16,14 +16,9 @@ along with this program. If not, see . + + diff --git a/web/src/enterprise/components/billings/usageDailyPanelSchema.spec.ts b/web/src/enterprise/components/billings/usageDailyPanelSchema.spec.ts new file mode 100644 index 0000000000..20ed7be25e --- /dev/null +++ b/web/src/enterprise/components/billings/usageDailyPanelSchema.spec.ts @@ -0,0 +1,74 @@ +// Copyright 2026 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +import { describe, it, expect } from "vitest"; +import { + buildUsageCombinedSql, + buildUsageCombinedLinePanelSchema, + USAGE_STREAM_NAME, +} from "./usageDailyPanelSchema"; + +describe("usageDailyPanelSchema", () => { + describe("buildUsageCombinedSql", () => { + it("buckets both metrics in one query, broken down by event", () => { + const sql = buildUsageCombinedSql({ orgId: "org-x", dataType: "mb" }); + expect(sql).toContain("histogram(_timestamp, '1 day')"); + expect(sql).toContain(`FROM "${USAGE_STREAM_NAME}"`); + expect(sql).toContain("org_id = 'org-x'"); + expect(sql).toContain('event as "breakdown_1"'); + expect(sql).toContain( + "event IN ('Ingestion', 'Search', 'Pipeline', 'RemotePipeline')", + ); + expect(sql).toContain("GROUP BY x_axis_1, breakdown_1"); + expect(sql).toContain("ORDER BY x_axis_1 ASC"); + }); + + it("divides size by 1024 for GB, raw MB otherwise", () => { + const gb = buildUsageCombinedSql({ orgId: "o", dataType: "gb" }); + expect(gb).toContain("sum(size) / 1024"); + const mb = buildUsageCombinedSql({ orgId: "o", dataType: "mb" }); + expect(mb).toContain("sum(size)"); + expect(mb).not.toContain("/ 1024"); + }); + + it("escapes single quotes in the org id", () => { + const sql = buildUsageCombinedSql({ orgId: "o'x", dataType: "mb" }); + expect(sql).toContain("org_id = 'o''x'"); + }); + }); + + describe("buildUsageCombinedLinePanelSchema", () => { + it("produces a version-2 line panel with an event breakdown (one query)", () => { + const schema = buildUsageCombinedLinePanelSchema({ + orgId: "org-x", + dataType: "gb", + }); + expect(schema.version).toBe(2); + expect(schema.type).toBe("line"); + expect(schema.queryType).toBe("sql"); + expect(schema.queries).toHaveLength(1); + expect(schema.queries[0].customQuery).toBe(true); + expect(schema.queries[0].fields.stream).toBe(USAGE_STREAM_NAME); + expect(schema.queries[0].fields.breakdown).toHaveLength(1); + }); + + it("labels the axis in the selected unit", () => { + const gb = buildUsageCombinedLinePanelSchema({ orgId: "o", dataType: "gb" }); + expect(gb.config.unit_custom).toBe("GB"); + const mb = buildUsageCombinedLinePanelSchema({ orgId: "o", dataType: "mb" }); + expect(mb.config.unit_custom).toBe("MB"); + }); + }); +}); diff --git a/web/src/enterprise/components/billings/usageDailyPanelSchema.ts b/web/src/enterprise/components/billings/usageDailyPanelSchema.ts new file mode 100644 index 0000000000..03e88d21e2 --- /dev/null +++ b/web/src/enterprise/components/billings/usageDailyPanelSchema.ts @@ -0,0 +1,182 @@ +// Copyright 2026 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +/** + * Builds dashboard panel schemas (version 2) for the Cloud Billing → Usage + * daily view, so they render through the shared `PanelSchemaRenderer` (the same + * engine dashboards use). This is UI-only: the charts fire normal SQL searches + * against the current org's own self-reporting `usage` stream — exactly like a + * hand-built dashboard panel — so no billing API changes are needed. + * + * The `usage` stream carries one row per usage event with fields: + * `_timestamp`, `event` (Ingestion/Search/...), `size`, `stream_type`, + * `org_id`, `unit`. + * + * The daily view shows one bar chart per metric (Ingestion, Search), each + * bucketed by day with `histogram(_timestamp, '1 day')`, so users see a daily + * run-rate and can catch a spike without waiting for the monthly total. + */ + +/** The org's own self-reporting usage stream (populated when usage reporting + * is enabled for the org). Matches the backend constant `USAGE_STREAM` + * (`src/config/src/meta/self_reporting/usage.rs`), which writes to a stream + * literally named `usage` in each org — NOT `_usage`. Single source of truth + * if a deployment names it differently. */ +export const USAGE_STREAM_NAME = "usage"; + +export type UsageMetric = + | "Ingestion" + | "Search" + | "Pipeline" + | "RemotePipeline"; + +/** The byte-based billable events charted daily (all except Functions, and + * except Data Retention / AI Credits which use different units/streams). */ +export const CHART_METRICS: UsageMetric[] = [ + "Ingestion", + "Search", + "Pipeline", + "RemotePipeline", +]; + +/** Per-metric display colour (kept distinct so the lines read apart). */ +const METRIC_COLOR: Record = { + Ingestion: "#5960b2", + Search: "#e6a817", + Pipeline: "#3fb27f", + RemotePipeline: "#c65fb2", +}; + +function safe(s: string): string { + return String(s).replace(/'/g, "''"); +} + +/** MB→GB is done in SQL when needed so values match the card totals exactly. */ +function sizeExpr(dataType: "gb" | "mb"): string { + return dataType === "gb" ? "sum(size) / 1024" : "sum(size)"; +} + +/** + * Daily-bucketed SQL for all byte-based billable metrics in one query, broken + * down by `event` (Ingestion, Search, Pipeline, Remote Pipeline — everything + * except Functions). Powers the single combined line chart. + */ +export function buildUsageCombinedSql(opts: { + orgId: string; + dataType: "gb" | "mb"; +}): string { + const { orgId, dataType } = opts; + const events = CHART_METRICS.map((e) => `'${e}'`).join(", "); + return ( + `SELECT histogram(_timestamp, '1 day') as "x_axis_1", ` + + `${sizeExpr(dataType)} as "y_axis_1", ` + + `event as "breakdown_1" ` + + `FROM "${USAGE_STREAM_NAME}" ` + + `WHERE org_id = '${safe(orgId)}' AND event IN (${events}) ` + + `GROUP BY x_axis_1, breakdown_1 ORDER BY x_axis_1 ASC` + ); +} + +/** + * Build a single combined daily LINE panel showing both metrics as separate + * lines (broken down by `event`). One query for both series — Ingestion + + * Search — day buckets on X, GB/MB on Y (converted in SQL, labelled here). + */ +export function buildUsageCombinedLinePanelSchema(opts: { + orgId: string; + dataType: "gb" | "mb"; +}): any { + const { orgId, dataType } = opts; + const sql = buildUsageCombinedSql({ orgId, dataType }); + const unitLabel = dataType === "gb" ? "GB" : "MB"; + + return { + version: 2, + id: "usage-daily-combined", + title: "", + description: "", + type: "line", + config: { + show_legends: true, + legends_position: "bottom", + unit: "custom", + unit_custom: unitLabel, + decimals: 2, + axis_border_show: true, + // Bridge gaps so each metric's line stays continuous across empty days. + connect_nulls: true, + no_value_replacement: "", + show_symbol: true, + base_map: { type: "osm" }, + map_view: { zoom: 1, lat: 0, lng: 0 }, + mark_line: [], + // Pin the series colours so the metrics read consistently. + color: { + mode: "palette-classic", + fixedColor: CHART_METRICS.map((m) => METRIC_COLOR[m]), + seriesBy: "last", + }, + }, + queryType: "sql", + queries: [ + { + query: sql, + customQuery: true, + vrlFunctionQuery: "", + fields: { + stream: USAGE_STREAM_NAME, + stream_type: "logs", + x: [ + { alias: "x_axis_1", column: "x_axis_1", color: null, label: "Time" }, + ], + y: [ + { + alias: "y_axis_1", + column: "y_axis_1", + color: null, + label: unitLabel, + }, + ], + z: [], + breakdown: [ + { + alias: "breakdown_1", + column: "breakdown_1", + color: null, + label: "Event", + }, + ], + filter: { + filterType: "group", + logicalOperator: "AND", + conditions: [], + }, + latitude: null, + longitude: null, + weight: null, + }, + config: { + promql_legend: "", + layer_type: "scatter", + weight_fixed: 1, + limit: 0, + min: 0, + max: 100, + time_shift: [], + }, + }, + ], + }; +} diff --git a/web/src/locales/languages/en.json b/web/src/locales/languages/en.json index c164edcf71..fe27425c36 100644 --- a/web/src/locales/languages/en.json +++ b/web/src/locales/languages/en.json @@ -2971,6 +2971,9 @@ "enterpriseNote": "High volume needs? Let's talk scale! Contact us for tailored solutions.", "contactSupport": "Contact Support", "totalUsage": "Total Usage", + "usageTrends": { + "dailyUsage": "Daily usage" + }, "aiCredits": "AI Credits", "aiModeFree": "Free", "aiModePayAsYouGo": "Pay as you go",