From aff4473a2b93f708cf730f5d2b7737bb6ab3d1bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Torjus=20H=C3=A5kestad?= <torjus@usit.uio.no>
Date: Tue, 17 Feb 2026 20:56:55 +0100
Subject: [PATCH] monitoring02: enable alerting and migrate CNAMEs from
 http-proxy

- Switch vmalert from blackhole mode to sending alerts to local
  Alertmanager
- Import alerttonotify service so alerts route to NATS notifications
- Move alertmanager and grafana CNAMEs from http-proxy to monitoring02
- Add monitoring CNAME to monitoring02
- Add Caddy reverse proxy entries for alertmanager and grafana

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 hosts/http-proxy/configuration.nix   |  3 ---
 hosts/monitoring02/configuration.nix |  2 +-
 hosts/monitoring02/default.nix       |  1 +
 services/grafana/default.nix         |  8 ++++++++
 services/victoriametrics/default.nix | 14 +++++++-------
 5 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/hosts/http-proxy/configuration.nix b/hosts/http-proxy/configuration.nix
index 75364f8..25e080d 100644
--- a/hosts/http-proxy/configuration.nix
+++ b/hosts/http-proxy/configuration.nix
@@ -18,9 +18,6 @@
     "sonarr"
     "ha"
     "z2m"
-    "grafana"
-    "prometheus"
-    "alertmanager"
     "jelly"
     "pyroscope"
     "pushgw"
diff --git a/hosts/monitoring02/configuration.nix b/hosts/monitoring02/configuration.nix
index 2616555..8e792ea 100644
--- a/hosts/monitoring02/configuration.nix
+++ b/hosts/monitoring02/configuration.nix
@@ -18,7 +18,7 @@
     role = "monitoring";
   };
 
-  homelab.dns.cnames = [ "grafana-test" "metrics" "vmalert" "loki" ];
+  homelab.dns.cnames = [ "monitoring" "alertmanager" "grafana" "grafana-test" "metrics" "vmalert" "loki" ];
 
   # Enable Vault integration
   vault.enable = true;
diff --git a/hosts/monitoring02/default.nix b/hosts/monitoring02/default.nix
index a8ef155..252daf0 100644
--- a/hosts/monitoring02/default.nix
+++ b/hosts/monitoring02/default.nix
@@ -4,5 +4,6 @@
     ../../services/grafana
     ../../services/victoriametrics
     ../../services/loki
+    ../../services/monitoring/alerttonotify.nix
   ];
 }
\ No newline at end of file
diff --git a/services/grafana/default.nix b/services/grafana/default.nix
index ed5aece..8fb645f 100644
--- a/services/grafana/default.nix
+++ b/services/grafana/default.nix
@@ -91,6 +91,14 @@
       acme_ca https://vault.home.2rjus.net:8200/v1/pki_int/acme/directory
       metrics
     '';
+    virtualHosts."grafana.home.2rjus.net".extraConfig = ''
+      log {
+        output file /var/log/caddy/grafana.log {
+          mode 644
+        }
+      }
+      reverse_proxy http://127.0.0.1:3000
+    '';
     virtualHosts."grafana-test.home.2rjus.net".extraConfig = ''
       log {
         output file /var/log/caddy/grafana.log {
diff --git a/services/victoriametrics/default.nix b/services/victoriametrics/default.nix
index 02aee75..2c2af1b 100644
--- a/services/victoriametrics/default.nix
+++ b/services/victoriametrics/default.nix
@@ -170,15 +170,12 @@ in
     };
   };
 
-  # vmalert for alerting rules - no notifier during parallel operation
+  # vmalert for alerting rules
   services.vmalert.instances.default = {
     enable = true;
     settings = {
       "datasource.url" = "http://localhost:8428";
-      # Blackhole notifications during parallel operation to prevent duplicate alerts.
-      # Replace with notifier.url after cutover from monitoring01:
-      # "notifier.url" = [ "http://localhost:9093" ];
-      "notifier.blackhole" = true;
+      "notifier.url" = [ "http://localhost:9093" ];
       "rule" = [ ../monitoring/rules.yml ];
     };
   };
@@ -191,8 +188,11 @@ in
     reverse_proxy http://127.0.0.1:8880
   '';
 
-  # Alertmanager - same config as monitoring01 but will only receive
-  # alerts after cutover (vmalert notifier is disabled above)
+  # Alertmanager
+  services.caddy.virtualHosts."alertmanager.home.2rjus.net".extraConfig = ''
+    reverse_proxy http://127.0.0.1:9093
+  '';
+
   services.prometheus.alertmanager = {
     enable = true;
     configuration = {