monitoring02: enable alerting and migrate CNAMEs from http-proxy
Some checks failed
Run nix flake check / flake-check (push) Failing after 6m25s
Run nix flake check / flake-check (pull_request) Failing after 3m52s

- Switch vmalert from blackhole mode to sending alerts to local
  Alertmanager
- Import alerttonotify service so alerts route to NATS notifications
- Move alertmanager and grafana CNAMEs from http-proxy to monitoring02
- Add monitoring CNAME to monitoring02
- Add Caddy reverse proxy entries for alertmanager and grafana
- Remove prometheus, alertmanager, and grafana Caddy entries from
  http-proxy (now served directly by monitoring02)
- Move monitoring02 Vault AppRole to hosts-generated.tf with
  extra_policies support and prometheus-metrics policy
- Update Promtail to use authenticated loki.home.2rjus.net endpoint
  only (remove unauthenticated monitoring01 client)
- Update pipe-to-loki and bootstrap to use loki.home.2rjus.net with
  basic auth from Vault secret
- Move migration plan to completed

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-02-17 20:56:55 +01:00
parent 7f69c0738a
commit a6013d3950
13 changed files with 204 additions and 253 deletions

View File

@@ -18,9 +18,6 @@
"sonarr"
"ha"
"z2m"
"grafana"
"prometheus"
"alertmanager"
"jelly"
"pyroscope"
"pushgw"

View File

@@ -18,7 +18,7 @@
role = "monitoring";
};
homelab.dns.cnames = [ "grafana-test" "metrics" "vmalert" "loki" ];
homelab.dns.cnames = [ "monitoring" "alertmanager" "grafana" "grafana-test" "metrics" "vmalert" "loki" ];
# Enable Vault integration
vault.enable = true;

View File

@@ -4,5 +4,6 @@
../../services/grafana
../../services/victoriametrics
../../services/loki
../../services/monitoring/alerttonotify.nix
];
}

View File

@@ -6,7 +6,8 @@ let
text = ''
set -euo pipefail
LOKI_URL="http://monitoring01.home.2rjus.net:3100/loki/api/v1/push"
LOKI_URL="https://loki.home.2rjus.net/loki/api/v1/push"
LOKI_AUTH_FILE="/run/secrets/promtail-loki-auth"
# Send a log entry to Loki with bootstrap status
# Usage: log_to_loki <stage> <message>
@@ -36,8 +37,14 @@ let
}]
}')
local auth_args=()
if [[ -f "$LOKI_AUTH_FILE" ]]; then
auth_args=(-u "promtail:$(cat "$LOKI_AUTH_FILE")")
fi
curl -s --connect-timeout 2 --max-time 5 \
-X POST \
"''${auth_args[@]}" \
-H "Content-Type: application/json" \
-d "$payload" \
"$LOKI_URL" >/dev/null 2>&1 || true