diff --git a/hosts/http-proxy/configuration.nix b/hosts/http-proxy/configuration.nix index 75364f8..25e080d 100644 --- a/hosts/http-proxy/configuration.nix +++ b/hosts/http-proxy/configuration.nix @@ -18,9 +18,6 @@ "sonarr" "ha" "z2m" - "grafana" - "prometheus" - "alertmanager" "jelly" "pyroscope" "pushgw" diff --git a/hosts/monitoring02/configuration.nix b/hosts/monitoring02/configuration.nix index 2616555..8e792ea 100644 --- a/hosts/monitoring02/configuration.nix +++ b/hosts/monitoring02/configuration.nix @@ -18,7 +18,7 @@ role = "monitoring"; }; - homelab.dns.cnames = [ "grafana-test" "metrics" "vmalert" "loki" ]; + homelab.dns.cnames = [ "monitoring" "alertmanager" "grafana" "grafana-test" "metrics" "vmalert" "loki" ]; # Enable Vault integration vault.enable = true; diff --git a/hosts/monitoring02/default.nix b/hosts/monitoring02/default.nix index a8ef155..252daf0 100644 --- a/hosts/monitoring02/default.nix +++ b/hosts/monitoring02/default.nix @@ -4,5 +4,6 @@ ../../services/grafana ../../services/victoriametrics ../../services/loki + ../../services/monitoring/alerttonotify.nix ]; } \ No newline at end of file diff --git a/services/grafana/default.nix b/services/grafana/default.nix index ed5aece..8fb645f 100644 --- a/services/grafana/default.nix +++ b/services/grafana/default.nix @@ -91,6 +91,14 @@ acme_ca https://vault.home.2rjus.net:8200/v1/pki_int/acme/directory metrics ''; + virtualHosts."grafana.home.2rjus.net".extraConfig = '' + log { + output file /var/log/caddy/grafana.log { + mode 644 + } + } + reverse_proxy http://127.0.0.1:3000 + ''; virtualHosts."grafana-test.home.2rjus.net".extraConfig = '' log { output file /var/log/caddy/grafana.log { diff --git a/services/http-proxy/proxy.nix b/services/http-proxy/proxy.nix index 8756dd4..613a162 100644 --- a/services/http-proxy/proxy.nix +++ b/services/http-proxy/proxy.nix @@ -54,30 +54,7 @@ } reverse_proxy http://ha1.home.2rjus.net:8080 } - prometheus.home.2rjus.net { - log { - output file /var/log/caddy/prometheus.log { - mode 644 - } - } - reverse_proxy http://monitoring01.home.2rjus.net:9090 - } - alertmanager.home.2rjus.net { - log { - output file /var/log/caddy/alertmanager.log { - mode 644 - } - } - reverse_proxy http://monitoring01.home.2rjus.net:9093 - } - grafana.home.2rjus.net { - log { - output file /var/log/caddy/grafana.log { - mode 644 - } - } - reverse_proxy http://monitoring01.home.2rjus.net:3000 - } + jelly.home.2rjus.net { log { output file /var/log/caddy/jelly.log { diff --git a/services/victoriametrics/default.nix b/services/victoriametrics/default.nix index 02aee75..2c2af1b 100644 --- a/services/victoriametrics/default.nix +++ b/services/victoriametrics/default.nix @@ -170,15 +170,12 @@ in }; }; - # vmalert for alerting rules - no notifier during parallel operation + # vmalert for alerting rules services.vmalert.instances.default = { enable = true; settings = { "datasource.url" = "http://localhost:8428"; - # Blackhole notifications during parallel operation to prevent duplicate alerts. - # Replace with notifier.url after cutover from monitoring01: - # "notifier.url" = [ "http://localhost:9093" ]; - "notifier.blackhole" = true; + "notifier.url" = [ "http://localhost:9093" ]; "rule" = [ ../monitoring/rules.yml ]; }; }; @@ -191,8 +188,11 @@ in reverse_proxy http://127.0.0.1:8880 ''; - # Alertmanager - same config as monitoring01 but will only receive - # alerts after cutover (vmalert notifier is disabled above) + # Alertmanager + services.caddy.virtualHosts."alertmanager.home.2rjus.net".extraConfig = '' + reverse_proxy http://127.0.0.1:9093 + ''; + services.prometheus.alertmanager = { enable = true; configuration = {