From e7775aa9dca955a8ea4919fd9a9cdfa2008d5f1f Mon Sep 17 00:00:00 2001
From: William P <me@williamtpeebles.com>
Date: Tue, 2 Dec 2025 10:57:46 -0500
Subject: [PATCH] monitoring: additional alert for excessive warnings

---
 system-apps/monitoring/values.yaml | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/system-apps/monitoring/values.yaml b/system-apps/monitoring/values.yaml
index 407dc3d..59cc720 100644
--- a/system-apps/monitoring/values.yaml
+++ b/system-apps/monitoring/values.yaml
@@ -27,6 +27,19 @@ kube-prometheus-stack:
             resources:
               requests:
                 storage: 50Gi
+  additionalPrometheusRulesMap:
+    rule-name:
+      groups:
+        - name: AdditionalAlerts
+          rules:
+            - alert: ExcessiveWarnings
+              expr: count(ALERTS{severity="warning",alertstate="firing"}) >= 5
+              for: 1m
+              labels:
+                severity: critical
+              annotations:
+                summary: Excessive 'warning' alerts are firing in the cluster
+                description: "{{ $value }} alerts with 'warning' severity are firing and could be a sign of catastrophic failure in the cluster"
   prometheusOperator:
     admissionWebhooks:
       certManager: