warmbly/deploy/kubernetes/monitoring/values.yaml

# kube-prometheus-stack Helm values
# Install with: helm install prometheus-stack prometheus-community/kube-prometheus-stack -n monitoring -f values.yaml

# Grafana configuration
grafana:
  enabled: true
  adminPassword: ""  # Leave empty to use existing secret or set via --set grafana.adminPassword=xxx
  adminUser: admin

  persistence:
    enabled: true
    size: 10Gi
    storageClassName: ""  # Use default storage class

  # Resource limits
  resources:
    requests:
      memory: 256Mi
      cpu: 100m
    limits:
      memory: 512Mi
      cpu: 500m

  # Default dashboards
  defaultDashboardsEnabled: true
  defaultDashboardsTimezone: utc

  # Grafana.ini configuration
  grafana.ini:
    server:
      root_url: "%(protocol)s://%(domain)s/"
    auth.anonymous:
      enabled: false

# Prometheus configuration
prometheus:
  enabled: true
  prometheusSpec:
    retention: 15d  # How long to keep metrics
    retentionSize: "45GB"  # Max storage size

    # Scrape interval
    scrapeInterval: 30s
    evaluationInterval: 30s

    # Resource limits
    resources:
      requests:
        memory: 1Gi
        cpu: 500m
      limits:
        memory: 2Gi
        cpu: 1000m

    # Persistent storage
    storageSpec:
      volumeClaimTemplate:
        spec:
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
              storage: 50Gi

    # ServiceMonitor selector - monitor all namespaces
    serviceMonitorSelectorNilUsesHelmValues: false
    podMonitorSelectorNilUsesHelmValues: false
    ruleSelectorNilUsesHelmValues: false

# Alertmanager configuration
alertmanager:
  enabled: true
  alertmanagerSpec:
    retention: 120h
    resources:
      requests:
        memory: 64Mi
        cpu: 10m
      limits:
        memory: 128Mi
        cpu: 100m
    storage:
      volumeClaimTemplate:
        spec:
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
              storage: 5Gi

  config:
    global:
      resolve_timeout: 5m

    route:
      receiver: 'default'
      group_by: ['alertname', 'namespace', 'severity']
      group_wait: 30s
      group_interval: 5m
      repeat_interval: 12h
      routes:
        - receiver: 'critical'
          match:
            severity: critical
          repeat_interval: 1h
        - receiver: 'warning'
          match:
            severity: warning
          repeat_interval: 4h

    receivers:
      - name: 'default'
        # Configure webhook, Slack, or email here
        # Example Slack:
        # slack_configs:
        #   - api_url: 'https://hooks.slack.com/services/xxx/yyy/zzz'
        #     channel: '#alerts'
        #     send_resolved: true

      - name: 'critical'
        # Critical alerts - more aggressive notification
        # slack_configs:
        #   - api_url: 'https://hooks.slack.com/services/xxx/yyy/zzz'
        #     channel: '#alerts-critical'
        #     send_resolved: true

      - name: 'warning'
        # Warning alerts
        # slack_configs:
        #   - api_url: 'https://hooks.slack.com/services/xxx/yyy/zzz'
        #     channel: '#alerts'
        #     send_resolved: true

    inhibit_rules:
      - source_match:
          severity: 'critical'
        target_match:
          severity: 'warning'
        equal: ['alertname', 'namespace']

# Node exporter for host-level metrics
nodeExporter:
  enabled: true
  resources:
    requests:
      memory: 32Mi
      cpu: 10m
    limits:
      memory: 64Mi
      cpu: 100m

# kube-state-metrics for Kubernetes object metrics
kubeStateMetrics:
  enabled: true

# Prometheus Operator
prometheusOperator:
  enabled: true
  resources:
    requests:
      memory: 128Mi
      cpu: 100m
    limits:
      memory: 256Mi
      cpu: 200m

# Custom alerting rules for cluster scaling and health
additionalPrometheusRulesMap:
  cluster-scaling-rules:
    groups:
      - name: cluster-scaling
        rules:
          # High node CPU usage
          - alert: HighNodeCPU
            expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
            for: 10m
            labels:
              severity: warning
            annotations:
              summary: "Node {{ $labels.instance }} CPU > 80%"
              description: "Node {{ $labels.instance }} has sustained CPU usage above 80% for 10 minutes. Consider adding more nodes."

          # Critical node CPU
          - alert: CriticalNodeCPU
            expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
            for: 5m
            labels:
              severity: critical
            annotations:
              summary: "Node {{ $labels.instance }} CPU > 95%"
              description: "Node {{ $labels.instance }} has critical CPU usage above 95%."

          # Low node memory
          - alert: LowNodeMemory
            expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 < 20
            for: 5m
            labels:
              severity: warning
            annotations:
              summary: "Node {{ $labels.instance }} memory < 20% available"
              description: "Node {{ $labels.instance }} has less than 20% memory available. Consider adding more nodes or scaling down workloads."

          # Critical low memory
          - alert: CriticalLowNodeMemory
            expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 < 10
            for: 2m
            labels:
              severity: critical
            annotations:
              summary: "Node {{ $labels.instance }} memory < 10% available"
              description: "Node {{ $labels.instance }} has critical memory shortage."

          # Pods stuck in pending
          - alert: PodsPending
            expr: sum(kube_pod_status_phase{phase="Pending"}) > 0
            for: 15m
            labels:
              severity: warning
            annotations:
              summary: "{{ $value }} pods stuck in Pending state"
              description: "Pods have been pending for more than 15 minutes. Check for resource constraints or scheduling issues."

          # Container CPU throttling
          - alert: ContainerThrottling
            expr: rate(container_cpu_cfs_throttled_seconds_total[5m]) > 0.5
            for: 10m
            labels:
              severity: info
            annotations:
              summary: "Container {{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }} is being throttled"
              description: "Container is experiencing CPU throttling. Consider increasing CPU limits."

          # High container memory usage
          - alert: ContainerMemoryNearLimit
            expr: (container_memory_working_set_bytes / container_spec_memory_limit_bytes) * 100 > 90
            for: 5m
            labels:
              severity: warning
            annotations:
              summary: "Container {{ $labels.namespace }}/{{ $labels.pod }}/{{ $labels.container }} memory > 90% of limit"
              description: "Container is using more than 90% of its memory limit. Risk of OOMKill."

      - name: cluster-health
        rules:
          # Node not ready
          - alert: NodeNotReady
            expr: kube_node_status_condition{condition="Ready",status="true"} == 0
            for: 5m
            labels:
              severity: critical
            annotations:
              summary: "Node {{ $labels.node }} is not ready"
              description: "Node {{ $labels.node }} has been in NotReady state for 5 minutes."

          # High pod restart count
          - alert: PodHighRestartCount
            expr: increase(kube_pod_container_status_restarts_total[1h]) > 5
            for: 0m
            labels:
              severity: warning
            annotations:
              summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} restarting frequently"
              description: "Pod has restarted more than 5 times in the last hour."

          # Pod CrashLoopBackOff
          - alert: PodCrashLoopBackOff
            expr: kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"} > 0
            for: 5m
            labels:
              severity: warning
            annotations:
              summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in CrashLoopBackOff"
              description: "Pod container {{ $labels.container }} is crash looping."

          # Deployment replicas mismatch
          - alert: DeploymentReplicasMismatch
            expr: kube_deployment_spec_replicas != kube_deployment_status_replicas_available
            for: 10m
            labels:
              severity: warning
            annotations:
              summary: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has replica mismatch"
              description: "Deployment has not reached desired replica count for 10 minutes."

          # PVC almost full
          - alert: PersistentVolumeAlmostFull
            expr: (kubelet_volume_stats_used_bytes / kubelet_volume_stats_capacity_bytes) * 100 > 85
            for: 5m
            labels:
              severity: warning
            annotations:
              summary: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is > 85% full"
              description: "Persistent volume claim is running low on space."

      - name: warmbly-application
        rules:
          # Warmbly backend unavailable
          - alert: WarmblyBackendUnavailable
            expr: kube_deployment_status_replicas_available{namespace="warmbly", deployment="backend"} == 0
            for: 1m
            labels:
              severity: critical
            annotations:
              summary: "Warmbly backend is unavailable"
              description: "No backend pods are available in the warmbly namespace."

          # Warmbly realtime service unavailable
          - alert: WarmblyRealtimeUnavailable
            expr: kube_deployment_status_replicas_available{namespace="warmbly", deployment="realtime"} == 0
            for: 1m
            labels:
              severity: critical
            annotations:
              summary: "Warmbly realtime service is unavailable"
              description: "No realtime pods are available in the warmbly namespace."

          # Warmbly tracking service unavailable
          - alert: WarmblyTrackingUnavailable
            expr: kube_deployment_status_replicas_available{namespace="warmbly", deployment="tracking"} == 0
            for: 1m
            labels:
              severity: critical
            annotations:
              summary: "Warmbly tracking service is unavailable"
              description: "No tracking pods are available in the warmbly namespace."

# Disable components not needed for basic monitoring
# Enable these if you need to monitor the control plane
kubeApiServer:
  enabled: true

kubeControllerManager:
  enabled: false  # Usually not accessible in managed K8s

kubeScheduler:
  enabled: false  # Usually not accessible in managed K8s

kubeProxy:
  enabled: true

kubeEtcd:
  enabled: false  # Usually not accessible in managed K8s

coreDns:
  enabled: true