Network Usage - Ingress Egress per interface and global usage (#229)

* Add network io stats

Add network usage graph and current status

Change network graphs to use network interface from drop down menu

Adjust the type to be able to get general network stats for the general
UI view

Working setup with a working builder

remove changes to these dockerfile

remove lock changes

* change network hashmap to Vector

* adjust all the network_usage_interface to be a vector rather than a hash map

* PR requested changes applied

* Change net_ingress_bytes and egress to network_ingress_bytes egress respectively

* final gen-client types

---------

Co-authored-by: mbecker20 <becker.maxh@gmail.com>
This commit is contained in:
rita7lopes
2024-12-21 08:15:21 -08:00
committed by GitHub
co-authored by mbecker20
parent ab8777460d
commit e274d6f7c8
10 changed files with 273 additions and 72 deletions
+3
View File
@@ -26,6 +26,9 @@ pub async fn record_server_stats(ts: i64) {
disk_total_gb,
disk_used_gb,
disks: stats.disks.clone(),
network_ingress_bytes: stats.network_ingress_bytes,
network_egress_bytes: stats.network_egress_bytes,
network_usage_interface: stats.network_usage_interface.clone(),
})
})
.collect::<Vec<_>>();
+1
View File
@@ -56,3 +56,4 @@ impl ResolveToString<GetSystemProcesses> for State {
.context("failed to serialize response to string")
}
}
+36 -1
View File
@@ -2,7 +2,7 @@ use std::{cmp::Ordering, sync::OnceLock};
use async_timing_util::wait_until_timelength;
use komodo_client::entities::stats::{
SingleDiskUsage, SystemInformation, SystemProcess, SystemStats,
SingleDiskUsage, SystemInformation, SystemProcess, SystemStats, SingleNetworkInterfaceUsage,
};
use sysinfo::{ProcessesToUpdate, System};
use tokio::sync::RwLock;
@@ -48,6 +48,7 @@ pub fn spawn_system_stats_polling_threads() {
});
}
pub struct StatsClient {
/// Cached system stats
pub stats: SystemStats,
@@ -57,6 +58,7 @@ pub struct StatsClient {
// the handles used to get the stats
system: sysinfo::System,
disks: sysinfo::Disks,
networks: sysinfo::Networks,
}
const BYTES_PER_GB: f64 = 1073741824.0;
@@ -67,6 +69,7 @@ impl Default for StatsClient {
fn default() -> Self {
let system = sysinfo::System::new_all();
let disks = sysinfo::Disks::new_with_refreshed_list();
let networks = sysinfo::Networks::new_with_refreshed_list();
let stats = SystemStats {
polling_rate: periphery_config().stats_polling_rate,
..Default::default()
@@ -75,6 +78,7 @@ impl Default for StatsClient {
info: get_system_information(&system),
system,
disks,
networks,
stats,
}
}
@@ -86,20 +90,51 @@ impl StatsClient {
self.system.refresh_memory();
self.system.refresh_processes(ProcessesToUpdate::All, true);
self.disks.refresh();
self.networks.refresh();
}
fn refresh_lists(&mut self) {
self.disks.refresh_list();
self.networks.refresh_list();
}
pub fn get_system_stats(&self) -> SystemStats {
let total_mem = self.system.total_memory();
let available_mem = self.system.available_memory();
let mut total_ingress: u64 = 0;
let mut total_egress: u64 = 0;
// Fetch network data (Ingress and Egress)
let network_usage: Vec<SingleNetworkInterfaceUsage> = self.networks
.iter()
.map(|(interface_name, network)| {
let ingress = network.received();
let egress = network.transmitted();
// Update total ingress and egress
total_ingress += ingress;
total_egress += egress;
// Return per-interface network stats
SingleNetworkInterfaceUsage {
name: interface_name.clone(),
ingress_bytes: ingress as f64,
egress_bytes: egress as f64,
}
})
.collect();
SystemStats {
cpu_perc: self.system.global_cpu_usage(),
mem_free_gb: self.system.free_memory() as f64 / BYTES_PER_GB,
mem_used_gb: (total_mem - available_mem) as f64 / BYTES_PER_GB,
mem_total_gb: total_mem as f64 / BYTES_PER_GB,
// Added total ingress and egress
network_ingress_bytes: total_ingress as f64,
network_egress_bytes: total_egress as f64,
network_usage_interface: network_usage,
disks: self.get_disks(),
polling_rate: self.stats.polling_rate,
refresh_ts: self.stats.refresh_ts,
+30 -1
View File
@@ -51,6 +51,15 @@ pub struct SystemStatsRecord {
pub disk_total_gb: f64,
/// Breakdown of individual disks, ie their usages, sizes, and mount points
pub disks: Vec<SingleDiskUsage>,
/// Network ingress usage in bytes
#[serde(default)]
pub network_ingress_bytes: f64,
/// Network egress usage in bytes
#[serde(default)]
pub network_egress_bytes: f64,
/// Network usage by interface name (ingress, egress in bytes)
#[serde(default)]
pub network_usage_interface: Vec<SingleNetworkInterfaceUsage>, // interface -> (ingress, egress)
}
/// Realtime system stats data.
@@ -71,7 +80,15 @@ pub struct SystemStats {
pub mem_total_gb: f64,
/// Breakdown of individual disks, ie their usages, sizes, and mount points
pub disks: Vec<SingleDiskUsage>,
/// Network ingress usage in MB
#[serde(default)]
pub network_ingress_bytes: f64,
/// Network egress usage in MB
#[serde(default)]
pub network_egress_bytes: f64,
/// Network usage by interface name (ingress, egress in bytes)
#[serde(default)]
pub network_usage_interface: Vec<SingleNetworkInterfaceUsage>, // interface -> (ingress, egress)
// metadata
/// The rate the system stats are being polled from the system
pub polling_rate: Timelength,
@@ -95,6 +112,18 @@ pub struct SingleDiskUsage {
pub total_gb: f64,
}
/// Info for network interface usage.
#[typeshare]
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct SingleNetworkInterfaceUsage {
/// The network interface name
pub name: String,
/// The ingress in bytes
pub ingress_bytes: f64,
/// The egress in bytes
pub egress_bytes: f64,
}
pub fn sum_disk_usage(disks: &[SingleDiskUsage]) -> TotalDiskUsage {
disks
.iter()
+22
View File
@@ -1809,6 +1809,16 @@ export interface SingleDiskUsage {
total_gb: number;
}
/** Info for network interface usage. */
export interface SingleNetworkInterfaceUsage {
/** The network interface name */
name: string;
/** The ingress in bytes */
ingress_bytes: number;
/** The egress in bytes */
egress_bytes: number;
}
export enum Timelength {
OneSecond = "1-sec",
FiveSeconds = "5-sec",
@@ -1850,6 +1860,12 @@ export interface SystemStats {
mem_total_gb: number;
/** Breakdown of individual disks, ie their usages, sizes, and mount points */
disks: SingleDiskUsage[];
/** Network ingress usage in MB */
network_ingress_bytes?: number;
/** Network egress usage in MB */
network_egress_bytes?: number;
/** Network usage by interface name (ingress, egress in bytes) */
network_usage_interface?: SingleNetworkInterfaceUsage[];
/** The rate the system stats are being polled from the system */
polling_rate: Timelength;
/** Unix timestamp in milliseconds when stats were last polled */
@@ -5095,6 +5111,12 @@ export interface SystemStatsRecord {
disk_total_gb: number;
/** Breakdown of individual disks, ie their usages, sizes, and mount points */
disks: SingleDiskUsage[];
/** Network ingress usage in bytes */
network_ingress_bytes?: number;
/** Network egress usage in bytes */
network_egress_bytes?: number;
/** Network usage by interface name (ingress, egress in bytes) */
network_usage_interface?: SingleNetworkInterfaceUsage[];
}
/** Response to [GetHistoricalServerStats]. */
+1 -3
View File
@@ -20,6 +20,4 @@ pub struct GetSystemStats {}
#[derive(Serialize, Deserialize, Debug, Clone, Request)]
#[response(Vec<SystemProcess>)]
pub struct GetSystemProcesses {}
//
pub struct GetSystemProcesses {}
+21
View File
@@ -1894,6 +1894,15 @@ export interface SingleDiskUsage {
/** Total size of the disk in GB */
total_gb: number;
}
/** Info for network interface usage. */
export interface SingleNetworkInterfaceUsage {
/** The network interface name */
name: string;
/** The ingress in bytes */
ingress_bytes: number;
/** The egress in bytes */
egress_bytes: number;
}
export declare enum Timelength {
OneSecond = "1-sec",
FiveSeconds = "5-sec",
@@ -1934,6 +1943,12 @@ export interface SystemStats {
mem_total_gb: number;
/** Breakdown of individual disks, ie their usages, sizes, and mount points */
disks: SingleDiskUsage[];
/** Network ingress usage in MB */
network_ingress_bytes?: number;
/** Network egress usage in MB */
network_egress_bytes?: number;
/** Network usage by interface name (ingress, egress in bytes) */
network_usage_interface?: SingleNetworkInterfaceUsage[];
/** The rate the system stats are being polled from the system */
polling_rate: Timelength;
/** Unix timestamp in milliseconds when stats were last polled */
@@ -4842,6 +4857,12 @@ export interface SystemStatsRecord {
disk_total_gb: number;
/** Breakdown of individual disks, ie their usages, sizes, and mount points */
disks: SingleDiskUsage[];
/** Network ingress usage in bytes */
network_ingress_bytes?: number;
/** Network egress usage in bytes */
network_egress_bytes?: number;
/** Network usage by interface name (ingress, egress in bytes) */
network_usage_interface?: SingleNetworkInterfaceUsage[];
}
/** Response to [GetHistoricalServerStats]. */
export interface GetHistoricalServerStatsResponse {
@@ -8,3 +8,10 @@ const statsGranularityAtom = atomWithStorage<Types.Timelength>(
);
export const useStatsGranularity = () => useAtom(statsGranularityAtom);
const selectedNetworkInterfaceAtom = atomWithStorage<string | undefined>(
"selected-network-interface-v0",
undefined // Default value is `undefined` (Global view)
);
export const useSelectedNetworkInterface = () => useAtom(selectedNetworkInterfaceAtom);
@@ -2,14 +2,14 @@ import { hex_color_by_intention } from "@lib/color";
import { useRead } from "@lib/hooks";
import { Types } from "komodo_client";
import { useMemo } from "react";
import { useStatsGranularity } from "./hooks";
import { useStatsGranularity, useSelectedNetworkInterface } from "./hooks";
import { Loader2 } from "lucide-react";
import { AxisOptions, Chart } from "react-charts";
import { convertTsMsToLocalUnixTsInMs } from "@lib/utils";
import { useTheme } from "@ui/theme";
import { fmt_utc_date } from "@lib/formatting";
type StatType = "cpu" | "mem" | "disk";
type StatType = "cpu" | "mem" | "disk" | "network_ingress" | "network_egress" | "network_interface_ingress" | "network_interface_egress";
type StatDatapoint = { date: number; value: number };
@@ -22,11 +22,13 @@ export const StatChart = ({
type: StatType;
className?: string;
}) => {
const [selectedInterface] = useSelectedNetworkInterface();
const [granularity] = useStatsGranularity();
const { data, isPending } = useRead("GetHistoricalServerStats", {
server: server_id,
granularity,
selectedInterface,
});
const stats = useMemo(
@@ -35,7 +37,7 @@ export const StatChart = ({
.map((stat) => {
return {
date: convertTsMsToLocalUnixTsInMs(stat.ts),
value: getStat(stat, type),
value: getStat(stat, type, selectedInterface),
};
})
.reverse(),
@@ -70,6 +72,9 @@ export const InnerStatChart = ({
? "dark"
: "light"
: _theme;
const BYTES_PER_GB = 1073741824.0;
const BYTES_PER_MB = 1048576.0;
const BYTES_PER_KB = 1024.0;
const min = stats?.[0]?.date ?? 0;
const max = stats?.[stats.length - 1]?.date ?? 0;
const diff = max - min;
@@ -90,25 +95,65 @@ export const InnerStatChart = ({
},
};
}, []);
// Determine the dynamic scaling for network-related types
const maxStatValue = Math.max(...(stats?.map((d) => d.value) ?? [0]));
const { unit, maxUnitValue } = useMemo(() => {
if (type === "network_ingress" || type === "network_egress") {
if (maxStatValue <= BYTES_PER_KB) {
return { unit: "KB", maxUnitValue: BYTES_PER_KB };
} else if (maxStatValue <= BYTES_PER_MB) {
return { unit: "MB", maxUnitValue: BYTES_PER_MB };
} else if (maxStatValue <= BYTES_PER_GB) {
return { unit: "GB", maxUnitValue: BYTES_PER_GB };
} else {
return { unit: "TB", maxUnitValue: BYTES_PER_GB * 1024 }; // Larger scale for high values
}
}
return { unit: "", maxUnitValue: 100 }; // Default for CPU, memory, disk
}, [type, maxStatValue]);
const valueAxis = useMemo(
(): AxisOptions<StatDatapoint>[] => [
{
getValue: (datum) => datum.value,
elementType: "area",
min: 0,
max: 100,
max: maxUnitValue,
formatters: {
tooltip: (value?: number) => (
<div className="text-lg font-mono">
{(value ?? 0) >= 10 ? value?.toFixed(2) : "0" + value?.toFixed(2)}
%
{(type === "network_ingress" || type === "network_egress") && unit
? `${(value ?? 0) / (maxUnitValue / 1024)} ${unit}`
: `${value?.toFixed(2)}%`}
</div>
),
},
},
],
[]
[type, maxUnitValue, unit]
);
// const valueAxis = useMemo(
// (): AxisOptions<StatDatapoint>[] => [
// {
// getValue: (datum) => datum.value,
// elementType: "area",
// min: 0,
// max: 100,
// formatters: {
// tooltip: (value?: number) => (
// <div className="text-lg font-mono">
// {(value ?? 0) >= 10 ? value?.toFixed(2) : "0" + value?.toFixed(2)}
// %
// </div>
// ),
// },
// },
// ],
// []
// );
return (
<Chart
options={{
@@ -133,68 +178,26 @@ export const InnerStatChart = ({
/>
);
// const container_ref = useRef<HTMLDivElement>(null);
// const line_ref = useRef<IChartApi>();
// const series_ref = useRef<ISeriesApi<"Area">>();
// const lineColor = getColor(type);
// const handleResize = () =>
// line_ref.current?.applyOptions({
// width: container_ref.current?.clientWidth,
// });
// useEffect(() => {
// if (!stats) return;
// if (line_ref.current) line_ref.current.remove();
// const init = () => {
// if (!container_ref.current) return;
// // INIT LINE
// line_ref.current = createChart(container_ref.current, {
// width: container_ref.current.clientWidth,
// height: container_ref.current.clientHeight,
// layout: {
// background: { type: ColorType.Solid, color: "transparent" },
// // textColor: "grey",
// fontSize: 12,
// },
// grid: {
// horzLines: { color: "#3f454d25" },
// vertLines: { color: "#3f454d25" },
// },
// timeScale: { timeVisible: true },
// handleScale: false,
// handleScroll: false,
// });
// line_ref.current.timeScale().fitContent();
// // INIT SERIES
// series_ref.current = line_ref.current.addAreaSeries({
// priceLineVisible: false,
// title: `${type} %`,
// lineColor,
// topColor: `${lineColor}B3`,
// bottomColor: `${lineColor}0D`,
// });
// series_ref.current.setData(stats);
// };
// // Run the effect
// init();
// window.addEventListener("resize", handleResize);
// return () => {
// window.removeEventListener("resize", handleResize);
// };
// }, [stats]);
// return <div className="w-full max-w-full h-full" ref={container_ref} />;
};
const getStat = (stat: Types.SystemStatsRecord, type: StatType) => {
const getStat = (stat: Types.SystemStatsRecord, type: StatType, selectedInterface?: string) => {
if (type === "cpu") return stat.cpu_perc || 0;
if (type === "mem") return (100 * stat.mem_used_gb) / stat.mem_total_gb;
if (type === "disk") return (100 * stat.disk_used_gb) / stat.disk_total_gb;
if (type === "network_ingress") return stat.network_ingress_bytes || 0;
if (type === "network_egress") return stat.network_egress_bytes || 0;
if (type === "network_interface_ingress")
return selectedInterface
? stat.network_usage_interface?.find(
(networkInterface) => networkInterface.name === selectedInterface
)?.ingress_bytes || 0
: stat.network_ingress_bytes || 0;
if (type === "network_interface_egress")
return selectedInterface
? stat.network_usage_interface?.find(
(networkInterface) => networkInterface.name === selectedInterface
)?.egress_bytes || 0
: stat.network_egress_bytes || 0;
return 0;
};
@@ -202,5 +205,7 @@ const getColor = (type: StatType) => {
if (type === "cpu") return hex_color_by_intention("Good");
if (type === "mem") return hex_color_by_intention("Warning");
if (type === "disk") return hex_color_by_intention("Neutral");
if (type === "network_interface_ingress") return hex_color_by_intention("Critical");
if (type === "network_interface_egress") return hex_color_by_intention("Unknown");
return hex_color_by_intention("Unknown");
};
};
@@ -14,7 +14,7 @@ import { DataTable, SortableHeader } from "@ui/data-table";
import { ReactNode, useMemo, useState } from "react";
import { Input } from "@ui/input";
import { StatChart } from "./stat-chart";
import { useStatsGranularity } from "./hooks";
import { useStatsGranularity, useSelectedNetworkInterface } from "./hooks";
import {
Select,
SelectContent,
@@ -32,6 +32,8 @@ export const ServerStats = ({
titleOther?: ReactNode;
}) => {
const [interval, setInterval] = useStatsGranularity();
const [networkInterface, setNetworkInterface] = useSelectedNetworkInterface();
const stats = useRead(
"GetSystemStats",
{ server: id },
@@ -94,6 +96,7 @@ export const ServerStats = ({
<div className="flex flex-col lg:flex-row gap-4">
<CPU stats={stats} />
<RAM stats={stats} />
<NETWORK stats={stats} />
<DISK stats={stats} />
</div>
</Section>
@@ -101,7 +104,9 @@ export const ServerStats = ({
<Section
title="Historical"
actions={
<div className="flex gap-2 items-center">
<div className="flex gap-4 items-center">
{/* Granularity Dropdown */}
<div className="flex items-center gap-2">
<div className="text-muted-foreground">Interval:</div>
<Select
value={interval}
@@ -131,6 +136,36 @@ export const ServerStats = ({
</SelectContent>
</Select>
</div>
{/* Network Interface Dropdown */}
<div className="flex items-center gap-2">
<div className="text-muted-foreground">Interface:</div>
<Select
value={networkInterface ?? "all"} // Show "all" if networkInterface is undefined
onValueChange={(interfaceName) => {
if (interfaceName === "all") {
setNetworkInterface(undefined); // Set undefined for "All" option
} else {
setNetworkInterface(interfaceName);
}
}}
>
<SelectTrigger className="w-[150px]">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="all">All</SelectItem>
{/* Iterate over the vector and access the `name` property */}
{(stats?.network_usage_interface ?? []).map((networkInterface) => (
<SelectItem key={networkInterface.name} value={networkInterface.name}>
{networkInterface.name}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
</div>
}
>
<div className="flex flex-col gap-8">
@@ -141,6 +176,8 @@ export const ServerStats = ({
type="disk"
className="w-full h-[250px]"
/>
<StatChart server_id={id} type="network_ingress" className="w-full h-[250px]" />
<StatChart server_id={id} type="network_egress" className="w-full h-[250px]" />
</div>
</Section>
@@ -347,6 +384,49 @@ const RAM = ({ stats }: { stats: Types.SystemStats | undefined }) => {
);
};
const formatBytes = (bytes: number) => {
const BYTES_PER_KB = 1024;
const BYTES_PER_MB = 1024 * BYTES_PER_KB;
const BYTES_PER_GB = 1024 * BYTES_PER_MB;
if (bytes >= BYTES_PER_GB) {
return { value: bytes / BYTES_PER_GB, unit: "GB" };
} else if (bytes >= BYTES_PER_MB) {
return { value: bytes / BYTES_PER_MB, unit: "MB" };
} else if (bytes >= BYTES_PER_KB) {
return { value: bytes / BYTES_PER_KB, unit: "KB" };
} else {
return { value: bytes, unit: "bytes" };
}
};
const NETWORK = ({ stats }: { stats: Types.SystemStats | undefined }) => {
const ingress = stats?.network_ingress_bytes ?? 0;
const egress = stats?.network_egress_bytes ?? 0;
const formattedIngress = formatBytes(ingress);
const formattedEgress = formatBytes(egress);
return (
<Card className="w-full">
<CardHeader className="flex-row justify-between">
<CardTitle>Network Usage</CardTitle>
</CardHeader>
<CardContent>
<div className="flex justify-between items-center mb-4">
<p className="font-medium">Ingress</p>
<span className="text-sm text-gray-600">{formattedIngress.value.toFixed(2)} {formattedIngress.unit}</span>
</div>
<div className="flex justify-between items-center">
<p className="font-medium">Egress</p>
<span className="text-sm text-gray-600">{formattedEgress.value.toFixed(2)} {formattedEgress.unit}</span>
</div>
</CardContent>
</Card>
);
};
const DISK = ({ stats }: { stats: Types.SystemStats | undefined }) => {
const used = stats?.disks.reduce((acc, curr) => (acc += curr.used_gb), 0);
const total = stats?.disks.reduce((acc, curr) => (acc += curr.total_gb), 0);