feat(monitoring): add Argo Workflow Controller failure alert

charts/apps/Chart.yaml

@@ -2,4 +2,4 @@ apiVersion: v2
 name: apps
 description: An argocd app to deploy apps inside the virtual cluster
 type: application
-version: 0.5.18
+version: 0.5.19

charts/apps/templates/monitoring-application.yaml

@@ -9,12 +9,13 @@ metadata:
 spec:
   destination:
     namespace: monitoring
-    server: {{ .Values.destination.server }}
+    targetRevision: {{ .Values.monitoring.targetRevision }}
   project: default
   source:
     repoURL: https://github.com/DiamondLightSource/workflows.git
     path: charts/monitoring
     targetRevision: {{ .Values.monitoring.targetRevision }}
+
     helm:
       valueFiles:
         - values.yaml

charts/monitoring/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: monitoring
 description: A monitoring stack for the workflows deployment
 type: application
-version: 0.1.28
+version: 0.1.29
 dependencies:
   - name: grafana
     repository: https://grafana-community.github.io/helm-charts

charts/monitoring/values.yaml

@@ -168,6 +168,24 @@ thanos:
 prometheus:
   kube-state-metrics:
     enabled: false
+  serverFiles:
+    alerting_rules.yml:
+      groups:
+        - name: argo-workflows
+          rules:
+            - alert: ArgoWorkflowControllerFailing
+              expr: |
+                kube_pod_container_status_waiting_reason{
+                  namespace="workflows",
+                  pod=~"workflows-argo-workflows-workflow-controller-.*",
+                  reason="CrashLoopBackOff"
+                } > 0
+              for: 10m
+              labels:
+                severity: warning
+              annotations:
+                summary: Argo Workflow Controller failing
+                description: Workflow controller has been in CrashLoopBackOff for more than 10 minutes.
   prometheus-node-exporter:
     enabled: false
   prometheus-pushgateway: