monitoring02: enable alerting and migrate CNAMEs from http-proxy

- Switch vmalert from blackhole mode to sending alerts to local
  Alertmanager
- Import alerttonotify service so alerts route to NATS notifications
- Move alertmanager and grafana CNAMEs from http-proxy to monitoring02
- Add monitoring CNAME to monitoring02
- Add Caddy reverse proxy entries for alertmanager and grafana

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

hosts/http-proxy/configuration.nix

@@ -18,9 +18,6 @@
     "sonarr"
     "ha"
     "z2m"
-    "grafana"
-    "prometheus"
-    "alertmanager"
     "jelly"
     "pyroscope"
     "pushgw"

hosts/monitoring02/configuration.nix

@@ -18,7 +18,7 @@
     role = "monitoring";
   };
 
-  homelab.dns.cnames = [ "grafana-test" "metrics" "vmalert" "loki" ];
+  homelab.dns.cnames = [ "monitoring" "alertmanager" "grafana" "grafana-test" "metrics" "vmalert" "loki" ];
 
   # Enable Vault integration
   vault.enable = true;

hosts/monitoring02/default.nix

@@ -4,5 +4,6 @@
     ../../services/grafana
     ../../services/victoriametrics
     ../../services/loki
+    ../../services/monitoring/alerttonotify.nix
   ];
 }

services/grafana/default.nix

@@ -91,6 +91,14 @@
       acme_ca https://vault.home.2rjus.net:8200/v1/pki_int/acme/directory
       metrics
     '';
+    virtualHosts."grafana.home.2rjus.net".extraConfig = ''
+      log {
+        output file /var/log/caddy/grafana.log {
+          mode 644
+        }
+      }
+      reverse_proxy http://127.0.0.1:3000
+    '';
     virtualHosts."grafana-test.home.2rjus.net".extraConfig = ''
       log {
         output file /var/log/caddy/grafana.log {

services/victoriametrics/default.nix

@@ -170,15 +170,12 @@ in
     };
   };
 
-  # vmalert for alerting rules - no notifier during parallel operation
+  # vmalert for alerting rules
   services.vmalert.instances.default = {
     enable = true;
     settings = {
       "datasource.url" = "http://localhost:8428";
-      # Blackhole notifications during parallel operation to prevent duplicate alerts.
+      "notifier.url" = [ "http://localhost:9093" ];
-      # Replace with notifier.url after cutover from monitoring01:
-      # "notifier.url" = [ "http://localhost:9093" ];
-      "notifier.blackhole" = true;
       "rule" = [ ../monitoring/rules.yml ];
     };
   };
@@ -191,8 +188,11 @@ in
     reverse_proxy http://127.0.0.1:8880
   '';
 
-  # Alertmanager - same config as monitoring01 but will only receive
+  # Alertmanager
-  # alerts after cutover (vmalert notifier is disabled above)
+  services.caddy.virtualHosts."alertmanager.home.2rjus.net".extraConfig = ''
+    reverse_proxy http://127.0.0.1:9093
+  '';
+
   services.prometheus.alertmanager = {
     enable = true;
     configuration = {