Improve monitoring stuff

secrets/secrets.yaml

@@ -10,92 +10,101 @@ sops:
         - recipient: age1lznyk4ee7e7x8n92cq2n87kz9920473ks5u9jlhd3dczfzq4wamqept56u
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxQTVka2k2dGpvekhJeWU5
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2bjN4QXozRm1GWDZoajQy
-            M2VFOWh0S3RsbzBqQ2lhL1VQQVVvKzA5QldzCllVZk56SndDQjMyRzljUVdiQ1Bw
+            cWVLbjhSUENxM1dwYXpNYVRZcUVBNFlHU3hVClprSEhDNGZmN1ZZViswREY0dkdh
-            eWx3bnZ5NWg0RTZ3aGtFbHp5RGQvNHcKLS0tIGxFVFA4R3NQcnNnZzR4a0pQdnFs
+            WXoxeWs3MnJ5aVVjejBELzFNS2x4akUKLS0tIHB3VEd6S2gwcWRlcVNUTnJ5S2Z4
-            dERsUjgzaFQrY0VTZFRDbVhISEwwelEKE4LcpxhwEaPOkO7uHqI6DpYNGTNjoRtw
+            SDJZK2Y5VXpwL1ZaeVlGT3JwN0ZoaWsKvzKiwSXdPW7NL2t8u2lw26+CijlDbhiB
-            6IeDTOLlx07CMHQ/9hWbUwKyr51FMJGJ7Q5rgBKaCFgCfKBF1ssGVg==
+            A4WkQAxYud7MPvf5K8mewp2OeCJG35Shw91PIzOhEyJr/HI2G7Jokw==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1hz2lz4k050ru3shrk5j3zk3f8azxmrp54pktw5a7nzjml4saudesx6jsl0
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPcitIbE95dVo2SHBVNEp5
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxTUg2Z3duYzZRMXVvRExF
-            UlVpL21peTRRZWxTcThxaWZQdmdlRmloUHdnCndFQmFTWHdqa1NmZm1vVVRVSWk0
+            dlhsNDVaYStPRlFzbTNuZWZ0OWQ5TmhmNXdjCkpjeUk5b0xwSjcvdHJZd1VkMjhU
-            QVJIanlPSUc5WTZqYVN2ditZVFpLL1kKLS0tIERvTmQwYUdSMWlpWkdtSWhtbVJD
+            ZkFnN2tTUm5IWnVnbU14bzgydFdwbk0KLS0tIFUveDhGZWtpc3VOcFJ6SGgyWi9i
-            LzV1VEdObXRHYnVBaGYrUDhPU0FXU2sK5wPshVZNc+KdOfEv449VSOn81u7MNLZT
+            bkNkd2lKWHZKaXFUV2VaaDJiQ3FYTUkKwoYuLCnlUJR3srD8mTzyrmXM7+ZoUV3H
-            xKtBEwe2H6FOxyauLrpfrTo5dAWnRhSsIHiIR2e6WjXajTjVPOpWcg==
+            Ib6PyF1Ufl47h3RqibuUP3YkoZ1F1BbiIv4LhzLo7+fi2Y2H3Z3B3w==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1w2q4gm2lrcgdzscq8du3ssyvk6qtzm4fcszc92z9ftclq23yyydqdga5um
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxR3FmRnhzZUd0eEoxbmRo
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBndnJETWlrUFNjWGkza2F6
-            YzVaakl0eXA1dy9uRlhyc2RyeUlvT2VkYTFVCm9CcEI1a3FQL2piWHo1NWRvK0Yz
+            Sm5zdk9oU2RxaHpLN2kzM1NPSkUzNmVEZlI4CmU0UTdIWGpCR3c0dUJwM3hXYWpp
-            NGM4SFRtQnZRNHVrZm1Cemw2SWxJNTQKLS0tIHFKYWMwdVltbzhKVTUvOEdXTE9T
+            UmpJdHl6OC9yT0JuQWJtYlFobGJwcE0KLS0tIGJscEx3LzNscUdWeUEzVmpiVEdx
-            aStOZndTL3hITmVZaU5qMlhsLzl4dDgKWm2cGdCeIKjggE/udnuor4xhGVNcb6vk
+            ZDN2K2Z0NUJscmpaSVhOVFM4eDA1OE0KO8YdXVk8/7hhItS3ZAF95Oxi0fOyqX+S
-            yKNXwD/tWMI+fykp3S+G8Aqt8vWXRbQbx4ITg8iVVZudi2SIf50rGQ==
+            zgpvCGxdF8bBjc7z5ysqoa88khug3k8o4CVbUWudAAFwK/zoxUYQhw==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1snmhmpavqy7xddmw4nuny0u4xusqmnqxqarjmghkm5zaluff84eq5xatrd
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBMM0dabXVDVUFJZld5RmxT
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBaTTFwME5qVjNkL1V0MU5I
-            MFJLMm53VW05YVJ5RFk3YkxFUE5aS2UxVndJCmVLMkw4UnBwK205cTJVN3laTmN0
+            SFNjWUZWL1ZWUlEraWlqNVBzRXhjZHV5ODFjCmNtV0xsMDdLY1dyY256ZkIrMWp2
-            YnRqSlpzL3JaUEFWWnR6SmxKa2lHNHMKLS0tIEtsMGR5Y01PRk5RNGVyMWZUc3dW
+            U0grbUdFMU9vUHh3SGNSQ1g0ZW5JMjAKLS0tIEMwSEppODFwNzNOK08xTVJMYWxq
-            ZXVPRFI1eFJLYkJyN0YrU28yenJhNG8KBmM0oIoMMmT9tBmfvaoxumCwwM/X9khd
+            c0dKc2xLbEtjbXpyQ21kOVlXajVmdTQKIPKjVq3W3fzbO6V6tsIfDc2TQ9y1SmRs
-            XkSdNax6HfovIylzoChJ6srIZ5BmTtA2ioKMna/kif57PD5nDU7Kmg==
+            7QuQkFlU3NzNeKSaofgecuzzOi9vHEAd36qC42KZjB0lEa7EFCtNpA==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age12a3nyvjs8jrwmpkf3tgawel3nwcklwsr35ktmytnvhpawqwzrsfqpgcy0q
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2SzVlS2pBZWZGUzIvZ0U3
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnVHJFL2tBNHlTQnV3cTN3
-            VDkrcmRESHh6MG8yNUZBRlEvam9SeUJseXpFCkVMSTFiTmsyMWEwNVg4TmdaeEdo
+            Y0NHT2U0b0l3ZmdUM3Z5TURGQzVGbE1OZjF3Cm01N0xtQ0tsRUpPejFESGhHcFNM
-            K09GdTdkR1g3bUJ6dFpZaGZKTjY3aW8KLS0tIGcxalhyVGlCM2hGdGhCL2dXTDk2
+            Zk9FZ2Z1YjZWRU1BWUVRaDB0WGRaVGsKLS0tIFhsNmxUd0dDTHVQalZtV3QzYi9O
-            N1gxdGZJSDVGUXFwTStoTFpXQkdUT3MKQ8c3oZfGxloS/eJByG2i25Cg3Gg9I65P
+            Q2VkeEZBYS9CcmZMcldGMWQ2OXozN3MK86HFP9IyNqDg9aIrk9xFBUGB1PLLUpn/
-            eqA+Jx5SWaM+DyYIxe7HseoPA0jwK5hUEJfcK+HK4N6ErbnG7n8byg==
+            /V5x3wNUgAOzuRNnKVM45vbdWzg0aHmBefnKNzkXLOxBrflCisPaTw==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1d2w5zece9647qwyq4vas9qyqegg96xwmg6c86440a6eg4uj6dd2qrq0w3l
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrZGt4ci9WcEE5dnN5R3NB
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4dTd1cEg0RHk4eEJPM1RR
-            UC9pZG5iaE5yR0R3dXN5SlFpaHhwclFBVWc0CmFjUnB3QU9neVV6S3haRml4MTJC
+            a3pJYzdxOGxId05jUkJna0xYZDBKR1dlTmhBCkUrQVJFTkRQbEN6TlA3SDBkVExm
-            QVpBYWZnQW0rZUcyWStIUk40SXZpZFUKLS0tIDczeEpMRDdDV25OaXV4aDhoTHFH
+            WVdUT1U3bmxDZjlZdjRpRjJQTDNiU1kKLS0tIHkwQUFNa3JTcEt4TzNRdmUvdnlq
-            NVNyVEFqc2kyMjFtcEUrdjNMdjkrS2cKGOkOhsy/RPlzQJz3vVt934rtg9sFiM1S
+            aktyd2crdG9NcXZyRXR0MzNESDVCbFUKO+02sMl+U5pfzEQuAiDtyQFqqSelplhw
-            3w9YN1VjzNW7RYG7Ro+Jtoli7/2j1So1uHiATS8JBcpTjO1BWEiSMg==
+            uymfZz/Wtt5JMu81VUdasPxiwhBMTAbomuvWsm115CuLXrAVaG2goA==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1gcyfkxh4fq5zdp0dh484aj82ksz66wrly7qhnpv0r0p576sn9ekse8e9ju
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoQjNEWVR5UlVZZ1NMcTEv
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvRFo3bitzbTJVU1M1S1pI
-            ZnUvM3ZaeUNsTGxwVjQwL0kvSTRyUnhhMXgwClpTV2ZwLzRVVlhOTTN5V1VTS2pG
+            eWI0V29HU1FwN2R4ZTlzbTdkdzI3a3dCbzMwClhBeGd0aHFMRjNUZktnaGJKdGJK
-            bHpCZlZ2SU1wTTJiZWQ3eHd2SFE5QmMKLS0tIG0wZFBvNXZlTTVwZHNRRXhEV1Ft
+            NWxqSC9vVXNPMEFlWmZ5eXE5WWg0RWcKLS0tIDZkT2sycUJRdnlWOGlQeDVWWmpW
-            VXhjM1VESWhzcHVDY09kUUdvWE9QbU0KH2sUKQNQg4w9/Te85YfRMM5Cx83I4tEV
+            V01zZkw5THVqdWRieG9oaERIVG9BaG8KsSH64d6a8Xrv/Olm02b697wzCc8ONfY9
-            Am4FMHpf3b9cVyhI+gNds5ROrhvox7VYW1rtLE43ApAnj9Jtj7qcMA==
+            2/7NrrSv9WWg2Rcui6iBpIMSbPvM3hjCLwBMa2BhCmfHXPfWLNyrBg==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1g5luz2rtel3surgzuh62rkvtey7lythrvfenyq954vmeyfpxjqkqdj3wt8
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTc1RhaHdDa0lreklQNXBv
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBOUTRaRDE5dGhQNEVSdmQv
-            cFk4OUxqcENMOW5nMWxtQ2pNeEpYK2ZWczJrCmJzL0NOL2FXa2RyS21oTzZyVTNK
+            ajdaQ3pIa3UvMjk2dGhpaTVualR0WXNaK2pBCjVOemwxY1Y1NURLSFVEeVNvOEQy
-            eExiUktRc3U4YU9hNnFQdko5ZmdMY1kKLS0tIHQwcDFxbjNoSXdWSjdQd281YTJC
+            OGI0KzRYY01Oa1U5b1ZvSTF2K3VXUmcKLS0tIDRVYWdQSkZtMnh5ZXhGaWRwamlL
-            LzN0UFVYTXdrU0pPZDFXODgyMkl4WjAKE0B30QO2gVITg3C8AG2/nW3jZHnEsL02
+            MFJ5ZXc3VHovaFljV1plTWlDbnVYTzAKbs4i40SzygTt1FiL8Mc9bl/k8jVI269U
-            pnlhQSpN4L1awQL6xcKk96d3n57pa+Hz2ah3fNkDGLw9vm8pNwVmiA==
+            MgOZbO+Eki/XC/8CRkKKYIsEczvpwDPO/uZQOmIkhUnjHYLdQMoB0g==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1gq8434ku0xekqmvnseeunv83e779cg03c06gwrusnymdsr3rpufqx6vr3m
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiNUx4UDEwdGRUaVdQc2xp
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpU1FYTkRRVGlrVzZ3cE5i
-            cjM4c1ZEcElOdk1uS0ZwRDFCZDBZQ2oyV1djCm1rU3JLcVJHR0VPV0h0OEFML3Mz
+            Y0ZvVXRyTHdrNDhESHNOMzV5NTE4WU8vSG1JCkVnczhOdGdLbE9oSjduREpxUHE0
-            YkFJZUpvcXA0c0owUGlBcFJYNW1abDAKLS0tIHlUQ283anpwWTI1SklGUDAwVGFZ
+            TnJQd05GdXlIUnBIMUppWmpjQSs2NVkKLS0tIHBOSmp0N1JLTFo3bDE1enVxeGs3
-            QVdTd0lhZklUVC92QnJmU09sd2xybVEKXJ6fb3zFZntL1/WxtHYvamywN08kUplo
+            dXEyV3FHYTkyVERSTHRscU44QklnM0UKUexa2IrP/meLehVA4/DgeLblD5Jt6QE3
-            kIiSRv+mJgRu5h35gih47q9vymcs8FEIdgDotztGS7qr+vV7ULP7KA==
+            zfPODxYON9NTnLKpzuzEMF3SlcFIZU7jDimwf6SHT8T/bBpgFLA9xQ==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1288993th0ge00reg4zqueyvmkrsvk829cs068eekjqfdprsrkeqql7mljk
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxdFBScjZNTnNPZWhHblFm
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3N1pzYmlxOXBRN01MZ0tK
-            YkdlSW4yOFFsUkZZaE5hdE0zcXJmNkV1L0ZjCkJTVk1VQzByUnBGbTZFQkdCdktV
+            VGxVWmNQQmE4b0hlQStDb0JRQ3pYdytvWUZnCk1xdUQ2d05Ed1lVTWhiOGorQjJn
-            VS92WkZkMy82ZCt0bVV1Mjhjekp0MlUKLS0tIFFyZjFvVVhUTnBVUVcveUNzVUpJ
+            TzNBRTlXOC9BRTlMYXJncWJQcDFnMDgKLS0tIDRlV2F6V3JTTTNZdW4wQkNrQ2dw
-            K3J6UDYvajZzeEhFb3M5eFh5OExqOEkK1AjfgMEImokKS7ei9ASMyTNzdhvUznI5
+            aEpTMUY5blJ3dFBsNzhTaXN6YlFqamMKrIWqnrGd4pa/nND53SPTXqeKbJKjvkQ2
-            soMhl7O9P++xyRSzPW+vEMyJ7Nr6YvzjbS3pyQa0eoAibVvUXmFM8w==
+            JLc+q/QsavWqK2CoG1zasIari6i+xZKdfBiJGp8hRIR2o5T/gK2Uqw==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1vpns76ykll8jgdlu3h05cur4ew2t3k7u03kxdg8y6ypfhsfhq9fqyurjey
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvVVAvSWFQMXdXZE04eFUw
+            RWhWOGxpOWhXSC9ETjg1ck9nMlVubThPQzFrCkk2UjNJUmNIMWRMUW96TVpaVUhF
+            ZU5xRVBrdTFNeXhuT1Z3ODFsWVB1UTAKLS0tIFdtb1BoQXdwbGpJYzFnWDQyWk9N
+            UVRyeWlUNDFuMmYzU1pITEwxY3FrK3MKA4uFZKYydoQNMIAF/mCqObOU/ornUvXF
+            XZ3wDScihUoR79Kd/wjEm+Piq1Uaya6MTyXXUvSumN1am5H+XgEJ1w==
             -----END AGE ENCRYPTED FILE-----
     lastmodified: "2024-06-02T22:09:06Z"
     mac: ENC[AES256_GCM,data:cxJq4EMEMVEw0IUXNwtyQj4MaYIJ/Xo4OaY+3VLgIhYw6oBO9CmJxgLuXcSnGnr23oNE5OQF6ALv+vxF46D1pI0V1zhqKL6zMIs0DzPBwo7Arg166w5kGAT274jK7YWymeJ7fafWXYubLlGUthyVJS1BkvlqIhoe2BlTZ3bPyBs=,iv:Z2Uh9Oo4q/ce6DDLShs7JAX3XFNAVOGBmBPvRbGxaaU=,tag:6qZhZ4+tgtXl60b0Lx7Taw==,type:str]

services/monitoring/prometheus.nix

@@ -3,8 +3,100 @@
   services.prometheus = {
     enable = true;
     alertmanager = {
-      enable = false;
+      enable = true;
+      configuration = {
+        global =
+          {
           };
+        route = {
+          receiver = "webhook_gunter";
+          group_wait = "30s";
+          group_interval = "5m";
+          repeat_interval = "12h";
+          group_by = [ "alertname" ];
+        };
+        receivers = [
+          {
+            name = "webhook_gunter";
+            webhook_configs = [
+              {
+                url = "http://gunter.home.2rjus.net:5001/alert";
+              }
+            ];
+          }
+        ];
+      };
+    };
+    rules = [
+      ''
+        groups:
+          - name: common_rules
+            rules:
+              - alert: node_down
+                expr: up == 0
+                for: 5m
+                labels:
+                  severity: critical
+                annotations: 
+                  summary: "Instance {{ $labels.instance }} down"
+                  description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
+          - name: nameserver_rules 
+            rules:
+              - alert: unbound_down
+                expr: node_systemd_unit_state {instance =~ "ns.+", name = "unbound.service", state = "active"} == 0
+                for: 5m
+                labels:
+                  severity: critical
+                annotations: 
+                  summary: "Unbound not running on {{ $labels.instance }}"
+                  description: "Unbound has been down on {{ $labels.instance }} more than 5 minutes."
+              - alert: nsd_down
+                expr: node_systemd_unit_state {instance =~ "ns.+", name = "nsd.service", state = "active"} == 0
+                for: 5m
+                labels:
+                  severity: critical
+                annotations: 
+                  summary: "NSD not running on {{ $labels.instance }}"
+                  description: "NSD has been down on {{ $labels.instance }} more than 5 minutes."
+          - name: http-proxy_rules 
+            rules:
+              - alert: caddy_down
+                expr: node_systemd_unit_state {instance="http-proxy.home.2rjus.net:9100", name = "caddy.service", state = "active"} == 0
+                for: 5m
+                labels:
+                  severity: critical
+                annotations: 
+                  summary: "Caddy not running on {{ $labels.instance }}"
+                  description: "Caddy has been down on {{ $labels.instance }} more than 5 minutes."
+          - name: home_assistant_rules 
+            rules:
+              - alert: home_assistant_down 
+                expr: node_systemd_unit_state {instance="ha1.home.2rjus.net:9100", name="home-assistant.service", state="active"}  == 0
+                for: 5m
+                labels:
+                  severity: critical
+                annotations: 
+                  summary: "Home assistant not running on {{ $labels.instance }}"
+                  description: "Home assistant has been down on {{ $labels.instance }} more than 5 minutes."
+              - alert: zigbee2qmtt_down 
+                expr: node_systemd_unit_state {instance = "ha1.home.2rjus.net:9100", name = "zigbee2mqtt.service", state = "active"} == 0
+                for: 5m
+                labels:
+                  severity: critical
+                annotations: 
+                  summary: "Zigbee2mqtt not running on {{ $labels.instance }}"
+                  description: "Zigbee2mqtt has been down on {{ $labels.instance }} more than 5 minutes."
+              - alert: mosquitto_down
+                expr: node_systemd_unit_state {instance = "ha1.home.2rjus.net:9100", name = "mosquitto.service", state = "active"} == 0
+                for: 5m
+                labels:
+                  severity: critical
+                annotations: 
+                  summary: "Mosquitto not running on {{ $labels.instance }}"
+                  description: "Mosquitto has been down on {{ $labels.instance }} more than 5 minutes."
+      ''
+    ];
+
     scrapeConfigs = [
       {
         job_name = "node-exporter";