2 changed files with 51 additions and 152 deletions
--- a/secrets/secrets.yaml
+++ b/secrets/secrets.yaml
@ -10,101 +10,92 @@ sops:
        - recipient: age1lznyk4ee7e7x8n92cq2n87kz9920473ks5u9jlhd3dczfzq4wamqept56u
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2bjN4QXozRm1GWDZoajQy
-            cWVLbjhSUENxM1dwYXpNYVRZcUVBNFlHU3hVClprSEhDNGZmN1ZZViswREY0dkdh
-            WXoxeWs3MnJ5aVVjejBELzFNS2x4akUKLS0tIHB3VEd6S2gwcWRlcVNUTnJ5S2Z4
-            SDJZK2Y5VXpwL1ZaeVlGT3JwN0ZoaWsKvzKiwSXdPW7NL2t8u2lw26+CijlDbhiB
-            A4WkQAxYud7MPvf5K8mewp2OeCJG35Shw91PIzOhEyJr/HI2G7Jokw==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxQTVka2k2dGpvekhJeWU5
+            M2VFOWh0S3RsbzBqQ2lhL1VQQVVvKzA5QldzCllVZk56SndDQjMyRzljUVdiQ1Bw
+            eWx3bnZ5NWg0RTZ3aGtFbHp5RGQvNHcKLS0tIGxFVFA4R3NQcnNnZzR4a0pQdnFs
+            dERsUjgzaFQrY0VTZFRDbVhISEwwelEKE4LcpxhwEaPOkO7uHqI6DpYNGTNjoRtw
+            6IeDTOLlx07CMHQ/9hWbUwKyr51FMJGJ7Q5rgBKaCFgCfKBF1ssGVg==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1hz2lz4k050ru3shrk5j3zk3f8azxmrp54pktw5a7nzjml4saudesx6jsl0
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxTUg2Z3duYzZRMXVvRExF
-            dlhsNDVaYStPRlFzbTNuZWZ0OWQ5TmhmNXdjCkpjeUk5b0xwSjcvdHJZd1VkMjhU
-            ZkFnN2tTUm5IWnVnbU14bzgydFdwbk0KLS0tIFUveDhGZWtpc3VOcFJ6SGgyWi9i
-            bkNkd2lKWHZKaXFUV2VaaDJiQ3FYTUkKwoYuLCnlUJR3srD8mTzyrmXM7+ZoUV3H
-            Ib6PyF1Ufl47h3RqibuUP3YkoZ1F1BbiIv4LhzLo7+fi2Y2H3Z3B3w==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPcitIbE95dVo2SHBVNEp5
+            UlVpL21peTRRZWxTcThxaWZQdmdlRmloUHdnCndFQmFTWHdqa1NmZm1vVVRVSWk0
+            QVJIanlPSUc5WTZqYVN2ditZVFpLL1kKLS0tIERvTmQwYUdSMWlpWkdtSWhtbVJD
+            LzV1VEdObXRHYnVBaGYrUDhPU0FXU2sK5wPshVZNc+KdOfEv449VSOn81u7MNLZT
+            xKtBEwe2H6FOxyauLrpfrTo5dAWnRhSsIHiIR2e6WjXajTjVPOpWcg==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1w2q4gm2lrcgdzscq8du3ssyvk6qtzm4fcszc92z9ftclq23yyydqdga5um
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBndnJETWlrUFNjWGkza2F6
-            Sm5zdk9oU2RxaHpLN2kzM1NPSkUzNmVEZlI4CmU0UTdIWGpCR3c0dUJwM3hXYWpp
-            UmpJdHl6OC9yT0JuQWJtYlFobGJwcE0KLS0tIGJscEx3LzNscUdWeUEzVmpiVEdx
-            ZDN2K2Z0NUJscmpaSVhOVFM4eDA1OE0KO8YdXVk8/7hhItS3ZAF95Oxi0fOyqX+S
-            zgpvCGxdF8bBjc7z5ysqoa88khug3k8o4CVbUWudAAFwK/zoxUYQhw==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxR3FmRnhzZUd0eEoxbmRo
+            YzVaakl0eXA1dy9uRlhyc2RyeUlvT2VkYTFVCm9CcEI1a3FQL2piWHo1NWRvK0Yz
+            NGM4SFRtQnZRNHVrZm1Cemw2SWxJNTQKLS0tIHFKYWMwdVltbzhKVTUvOEdXTE9T
+            aStOZndTL3hITmVZaU5qMlhsLzl4dDgKWm2cGdCeIKjggE/udnuor4xhGVNcb6vk
+            yKNXwD/tWMI+fykp3S+G8Aqt8vWXRbQbx4ITg8iVVZudi2SIf50rGQ==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1snmhmpavqy7xddmw4nuny0u4xusqmnqxqarjmghkm5zaluff84eq5xatrd
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBaTTFwME5qVjNkL1V0MU5I
-            SFNjWUZWL1ZWUlEraWlqNVBzRXhjZHV5ODFjCmNtV0xsMDdLY1dyY256ZkIrMWp2
-            U0grbUdFMU9vUHh3SGNSQ1g0ZW5JMjAKLS0tIEMwSEppODFwNzNOK08xTVJMYWxq
-            c0dKc2xLbEtjbXpyQ21kOVlXajVmdTQKIPKjVq3W3fzbO6V6tsIfDc2TQ9y1SmRs
-            7QuQkFlU3NzNeKSaofgecuzzOi9vHEAd36qC42KZjB0lEa7EFCtNpA==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBMM0dabXVDVUFJZld5RmxT
+            MFJLMm53VW05YVJ5RFk3YkxFUE5aS2UxVndJCmVLMkw4UnBwK205cTJVN3laTmN0
+            YnRqSlpzL3JaUEFWWnR6SmxKa2lHNHMKLS0tIEtsMGR5Y01PRk5RNGVyMWZUc3dW
+            ZXVPRFI1eFJLYkJyN0YrU28yenJhNG8KBmM0oIoMMmT9tBmfvaoxumCwwM/X9khd
+            XkSdNax6HfovIylzoChJ6srIZ5BmTtA2ioKMna/kif57PD5nDU7Kmg==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age12a3nyvjs8jrwmpkf3tgawel3nwcklwsr35ktmytnvhpawqwzrsfqpgcy0q
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnVHJFL2tBNHlTQnV3cTN3
-            Y0NHT2U0b0l3ZmdUM3Z5TURGQzVGbE1OZjF3Cm01N0xtQ0tsRUpPejFESGhHcFNM
-            Zk9FZ2Z1YjZWRU1BWUVRaDB0WGRaVGsKLS0tIFhsNmxUd0dDTHVQalZtV3QzYi9O
-            Q2VkeEZBYS9CcmZMcldGMWQ2OXozN3MK86HFP9IyNqDg9aIrk9xFBUGB1PLLUpn/
-            /V5x3wNUgAOzuRNnKVM45vbdWzg0aHmBefnKNzkXLOxBrflCisPaTw==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2SzVlS2pBZWZGUzIvZ0U3
+            VDkrcmRESHh6MG8yNUZBRlEvam9SeUJseXpFCkVMSTFiTmsyMWEwNVg4TmdaeEdo
+            K09GdTdkR1g3bUJ6dFpZaGZKTjY3aW8KLS0tIGcxalhyVGlCM2hGdGhCL2dXTDk2
+            N1gxdGZJSDVGUXFwTStoTFpXQkdUT3MKQ8c3oZfGxloS/eJByG2i25Cg3Gg9I65P
+            eqA+Jx5SWaM+DyYIxe7HseoPA0jwK5hUEJfcK+HK4N6ErbnG7n8byg==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1d2w5zece9647qwyq4vas9qyqegg96xwmg6c86440a6eg4uj6dd2qrq0w3l
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4dTd1cEg0RHk4eEJPM1RR
-            a3pJYzdxOGxId05jUkJna0xYZDBKR1dlTmhBCkUrQVJFTkRQbEN6TlA3SDBkVExm
-            WVdUT1U3bmxDZjlZdjRpRjJQTDNiU1kKLS0tIHkwQUFNa3JTcEt4TzNRdmUvdnlq
-            aktyd2crdG9NcXZyRXR0MzNESDVCbFUKO+02sMl+U5pfzEQuAiDtyQFqqSelplhw
-            uymfZz/Wtt5JMu81VUdasPxiwhBMTAbomuvWsm115CuLXrAVaG2goA==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrZGt4ci9WcEE5dnN5R3NB
+            UC9pZG5iaE5yR0R3dXN5SlFpaHhwclFBVWc0CmFjUnB3QU9neVV6S3haRml4MTJC
+            QVpBYWZnQW0rZUcyWStIUk40SXZpZFUKLS0tIDczeEpMRDdDV25OaXV4aDhoTHFH
+            NVNyVEFqc2kyMjFtcEUrdjNMdjkrS2cKGOkOhsy/RPlzQJz3vVt934rtg9sFiM1S
+            3w9YN1VjzNW7RYG7Ro+Jtoli7/2j1So1uHiATS8JBcpTjO1BWEiSMg==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1gcyfkxh4fq5zdp0dh484aj82ksz66wrly7qhnpv0r0p576sn9ekse8e9ju
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvRFo3bitzbTJVU1M1S1pI
-            eWI0V29HU1FwN2R4ZTlzbTdkdzI3a3dCbzMwClhBeGd0aHFMRjNUZktnaGJKdGJK
-            NWxqSC9vVXNPMEFlWmZ5eXE5WWg0RWcKLS0tIDZkT2sycUJRdnlWOGlQeDVWWmpW
-            V01zZkw5THVqdWRieG9oaERIVG9BaG8KsSH64d6a8Xrv/Olm02b697wzCc8ONfY9
-            2/7NrrSv9WWg2Rcui6iBpIMSbPvM3hjCLwBMa2BhCmfHXPfWLNyrBg==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoQjNEWVR5UlVZZ1NMcTEv
+            ZnUvM3ZaeUNsTGxwVjQwL0kvSTRyUnhhMXgwClpTV2ZwLzRVVlhOTTN5V1VTS2pG
+            bHpCZlZ2SU1wTTJiZWQ3eHd2SFE5QmMKLS0tIG0wZFBvNXZlTTVwZHNRRXhEV1Ft
+            VXhjM1VESWhzcHVDY09kUUdvWE9QbU0KH2sUKQNQg4w9/Te85YfRMM5Cx83I4tEV
+            Am4FMHpf3b9cVyhI+gNds5ROrhvox7VYW1rtLE43ApAnj9Jtj7qcMA==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1g5luz2rtel3surgzuh62rkvtey7lythrvfenyq954vmeyfpxjqkqdj3wt8
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBOUTRaRDE5dGhQNEVSdmQv
-            ajdaQ3pIa3UvMjk2dGhpaTVualR0WXNaK2pBCjVOemwxY1Y1NURLSFVEeVNvOEQy
-            OGI0KzRYY01Oa1U5b1ZvSTF2K3VXUmcKLS0tIDRVYWdQSkZtMnh5ZXhGaWRwamlL
-            MFJ5ZXc3VHovaFljV1plTWlDbnVYTzAKbs4i40SzygTt1FiL8Mc9bl/k8jVI269U
-            MgOZbO+Eki/XC/8CRkKKYIsEczvpwDPO/uZQOmIkhUnjHYLdQMoB0g==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTc1RhaHdDa0lreklQNXBv
+            cFk4OUxqcENMOW5nMWxtQ2pNeEpYK2ZWczJrCmJzL0NOL2FXa2RyS21oTzZyVTNK
+            eExiUktRc3U4YU9hNnFQdko5ZmdMY1kKLS0tIHQwcDFxbjNoSXdWSjdQd281YTJC
+            LzN0UFVYTXdrU0pPZDFXODgyMkl4WjAKE0B30QO2gVITg3C8AG2/nW3jZHnEsL02
+            pnlhQSpN4L1awQL6xcKk96d3n57pa+Hz2ah3fNkDGLw9vm8pNwVmiA==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1gq8434ku0xekqmvnseeunv83e779cg03c06gwrusnymdsr3rpufqx6vr3m
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpU1FYTkRRVGlrVzZ3cE5i
-            Y0ZvVXRyTHdrNDhESHNOMzV5NTE4WU8vSG1JCkVnczhOdGdLbE9oSjduREpxUHE0
-            TnJQd05GdXlIUnBIMUppWmpjQSs2NVkKLS0tIHBOSmp0N1JLTFo3bDE1enVxeGs3
-            dXEyV3FHYTkyVERSTHRscU44QklnM0UKUexa2IrP/meLehVA4/DgeLblD5Jt6QE3
-            zfPODxYON9NTnLKpzuzEMF3SlcFIZU7jDimwf6SHT8T/bBpgFLA9xQ==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiNUx4UDEwdGRUaVdQc2xp
+            cjM4c1ZEcElOdk1uS0ZwRDFCZDBZQ2oyV1djCm1rU3JLcVJHR0VPV0h0OEFML3Mz
+            YkFJZUpvcXA0c0owUGlBcFJYNW1abDAKLS0tIHlUQ283anpwWTI1SklGUDAwVGFZ
+            QVdTd0lhZklUVC92QnJmU09sd2xybVEKXJ6fb3zFZntL1/WxtHYvamywN08kUplo
+            kIiSRv+mJgRu5h35gih47q9vymcs8FEIdgDotztGS7qr+vV7ULP7KA==
            -----END AGE ENCRYPTED FILE-----
        - recipient: age1288993th0ge00reg4zqueyvmkrsvk829cs068eekjqfdprsrkeqql7mljk
          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3N1pzYmlxOXBRN01MZ0tK
-            VGxVWmNQQmE4b0hlQStDb0JRQ3pYdytvWUZnCk1xdUQ2d05Ed1lVTWhiOGorQjJn
-            TzNBRTlXOC9BRTlMYXJncWJQcDFnMDgKLS0tIDRlV2F6V3JTTTNZdW4wQkNrQ2dw
-            aEpTMUY5blJ3dFBsNzhTaXN6YlFqamMKrIWqnrGd4pa/nND53SPTXqeKbJKjvkQ2
-            JLc+q/QsavWqK2CoG1zasIari6i+xZKdfBiJGp8hRIR2o5T/gK2Uqw==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1vpns76ykll8jgdlu3h05cur4ew2t3k7u03kxdg8y6ypfhsfhq9fqyurjey
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvVVAvSWFQMXdXZE04eFUw
-            RWhWOGxpOWhXSC9ETjg1ck9nMlVubThPQzFrCkk2UjNJUmNIMWRMUW96TVpaVUhF
-            ZU5xRVBrdTFNeXhuT1Z3ODFsWVB1UTAKLS0tIFdtb1BoQXdwbGpJYzFnWDQyWk9N
-            UVRyeWlUNDFuMmYzU1pITEwxY3FrK3MKA4uFZKYydoQNMIAF/mCqObOU/ornUvXF
-            XZ3wDScihUoR79Kd/wjEm+Piq1Uaya6MTyXXUvSumN1am5H+XgEJ1w==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxdFBScjZNTnNPZWhHblFm
+            YkdlSW4yOFFsUkZZaE5hdE0zcXJmNkV1L0ZjCkJTVk1VQzByUnBGbTZFQkdCdktV
+            VS92WkZkMy82ZCt0bVV1Mjhjekp0MlUKLS0tIFFyZjFvVVhUTnBVUVcveUNzVUpJ
+            K3J6UDYvajZzeEhFb3M5eFh5OExqOEkK1AjfgMEImokKS7ei9ASMyTNzdhvUznI5
+            soMhl7O9P++xyRSzPW+vEMyJ7Nr6YvzjbS3pyQa0eoAibVvUXmFM8w==
            -----END AGE ENCRYPTED FILE-----
    lastmodified: "2024-06-02T22:09:06Z"
    mac: ENC[AES256_GCM,data:cxJq4EMEMVEw0IUXNwtyQj4MaYIJ/Xo4OaY+3VLgIhYw6oBO9CmJxgLuXcSnGnr23oNE5OQF6ALv+vxF46D1pI0V1zhqKL6zMIs0DzPBwo7Arg166w5kGAT274jK7YWymeJ7fafWXYubLlGUthyVJS1BkvlqIhoe2BlTZ3bPyBs=,iv:Z2Uh9Oo4q/ce6DDLShs7JAX3XFNAVOGBmBPvRbGxaaU=,tag:6qZhZ4+tgtXl60b0Lx7Taw==,type:str]
--- a/services/monitoring/prometheus.nix
+++ b/services/monitoring/prometheus.nix
@ -3,100 +3,8 @@
  services.prometheus = {
    enable = true;
    alertmanager = {
-      enable = true;
-      configuration = {
-        global =
-          {
-          };
-        route = {
-          receiver = "webhook_gunter";
-          group_wait = "30s";
-          group_interval = "5m";
-          repeat_interval = "12h";
-          group_by = [ "alertname" ];
-        };
-        receivers = [
-          {
-            name = "webhook_gunter";
-            webhook_configs = [
-              {
-                url = "http://gunter.home.2rjus.net:5001/alert";
-              }
-            ];
-          }
-        ];
-      };
+      enable = false;
    };
-    rules = [
-      ''
-        groups:
-          - name: common_rules
-            rules:
-              - alert: node_down
-                expr: up == 0
-                for: 5m
-                labels:
-                  severity: critical
-                annotations: 
-                  summary: "Instance {{ $labels.instance }} down"
-                  description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
-          - name: nameserver_rules 
-            rules:
-              - alert: unbound_down
-                expr: node_systemd_unit_state {instance =~ "ns.+", name = "unbound.service", state = "active"} == 0
-                for: 5m
-                labels:
-                  severity: critical
-                annotations: 
-                  summary: "Unbound not running on {{ $labels.instance }}"
-                  description: "Unbound has been down on {{ $labels.instance }} more than 5 minutes."
-              - alert: nsd_down
-                expr: node_systemd_unit_state {instance =~ "ns.+", name = "nsd.service", state = "active"} == 0
-                for: 5m
-                labels:
-                  severity: critical
-                annotations: 
-                  summary: "NSD not running on {{ $labels.instance }}"
-                  description: "NSD has been down on {{ $labels.instance }} more than 5 minutes."
-          - name: http-proxy_rules 
-            rules:
-              - alert: caddy_down
-                expr: node_systemd_unit_state {instance="http-proxy.home.2rjus.net:9100", name = "caddy.service", state = "active"} == 0
-                for: 5m
-                labels:
-                  severity: critical
-                annotations: 
-                  summary: "Caddy not running on {{ $labels.instance }}"
-                  description: "Caddy has been down on {{ $labels.instance }} more than 5 minutes."
-          - name: home_assistant_rules 
-            rules:
-              - alert: home_assistant_down 
-                expr: node_systemd_unit_state {instance="ha1.home.2rjus.net:9100", name="home-assistant.service", state="active"}  == 0
-                for: 5m
-                labels:
-                  severity: critical
-                annotations: 
-                  summary: "Home assistant not running on {{ $labels.instance }}"
-                  description: "Home assistant has been down on {{ $labels.instance }} more than 5 minutes."
-              - alert: zigbee2qmtt_down 
-                expr: node_systemd_unit_state {instance = "ha1.home.2rjus.net:9100", name = "zigbee2mqtt.service", state = "active"} == 0
-                for: 5m
-                labels:
-                  severity: critical
-                annotations: 
-                  summary: "Zigbee2mqtt not running on {{ $labels.instance }}"
-                  description: "Zigbee2mqtt has been down on {{ $labels.instance }} more than 5 minutes."
-              - alert: mosquitto_down
-                expr: node_systemd_unit_state {instance = "ha1.home.2rjus.net:9100", name = "mosquitto.service", state = "active"} == 0
-                for: 5m
-                labels:
-                  severity: critical
-                annotations: 
-                  summary: "Mosquitto not running on {{ $labels.instance }}"
-                  description: "Mosquitto has been down on {{ $labels.instance }} more than 5 minutes."
-      ''
-    ];
-
    scrapeConfigs = [
      {
        job_name = "node-exporter";