Add some alerting rules

2025-01-21 22:47:44 +01:00 · 2025-01-21 22:47:44 +01:00 · 43dfc0ec28
commit 43dfc0ec28
parent 4154fdb9de
1 changed files with 16 additions and 0 deletions
--- a/services/monitoring/prometheus.nix
+++ b/services/monitoring/prometheus.nix
@ -51,6 +51,22 @@
                annotations: 
                  summary: "Instance {{ $labels.instance }} down"
                  description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
+              - alert: low_disk_space
+                expr: node_filesystem_free_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"} * 100 < 10
+                for: 5m
+                labels:
+                  severity: warning
+                annotations: 
+                  summary: "Disk space low on {{ $labels.instance }}"
+                  description: "Disk space is low on {{ $labels.instance }}. Please check."
+              - alert: high_cpu_load
+                expr: node_load1 > 1
+                for: 5m
+                labels:
+                  severity: warning
+                annotations: 
+                  summary: "High CPU load on {{ $labels.instance }}"
+                  description: "CPU load is high on {{ $labels.instance }}. Please check."
          - name: nameserver_rules 
            rules:
              - alert: unbound_down