nixos-servers/services/monitoring/prometheus.nix

184 lines
5.8 KiB
Nix
Raw Normal View History

2024-12-01 00:51:29 +00:00
{ ... }:
{
services.prometheus = {
enable = true;
2024-12-01 19:51:14 +00:00
alertmanager = {
2024-12-02 22:41:46 +00:00
enable = true;
configuration = {
2024-12-22 03:33:00 +00:00
global = {
};
2024-12-02 22:41:46 +00:00
route = {
receiver = "webhook_gunter";
group_wait = "30s";
group_interval = "5m";
repeat_interval = "12h";
group_by = [ "alertname" ];
};
receivers = [
{
name = "webhook_gunter";
webhook_configs = [
{
url = "http://gunter.home.2rjus.net:5001/alert";
}
];
}
];
};
2024-12-01 19:51:14 +00:00
};
2024-12-02 23:47:00 +00:00
alertmanagers = [
{
static_configs = [
{
targets = [ "localhost:9093" ];
}
];
}
];
2024-12-05 18:36:55 +00:00
globalConfig = {
scrape_interval = "15s";
};
2024-12-02 22:41:46 +00:00
rules = [
''
groups:
- name: common_rules
rules:
- alert: node_down
expr: up == 0
for: 5m
labels:
severity: critical
annotations:
summary: "Instance {{ $labels.instance }} down"
description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
- name: nameserver_rules
rules:
- alert: unbound_down
expr: node_systemd_unit_state {instance =~ "ns.+", name = "unbound.service", state = "active"} == 0
for: 5m
labels:
severity: critical
annotations:
summary: "Unbound not running on {{ $labels.instance }}"
description: "Unbound has been down on {{ $labels.instance }} more than 5 minutes."
- alert: nsd_down
expr: node_systemd_unit_state {instance =~ "ns.+", name = "nsd.service", state = "active"} == 0
for: 5m
labels:
severity: critical
annotations:
summary: "NSD not running on {{ $labels.instance }}"
description: "NSD has been down on {{ $labels.instance }} more than 5 minutes."
- name: http-proxy_rules
rules:
- alert: caddy_down
expr: node_systemd_unit_state {instance="http-proxy.home.2rjus.net:9100", name = "caddy.service", state = "active"} == 0
for: 5m
labels:
severity: critical
annotations:
summary: "Caddy not running on {{ $labels.instance }}"
description: "Caddy has been down on {{ $labels.instance }} more than 5 minutes."
- name: home_assistant_rules
rules:
- alert: home_assistant_down
expr: node_systemd_unit_state {instance="ha1.home.2rjus.net:9100", name="home-assistant.service", state="active"} == 0
for: 5m
labels:
severity: critical
annotations:
summary: "Home assistant not running on {{ $labels.instance }}"
description: "Home assistant has been down on {{ $labels.instance }} more than 5 minutes."
- alert: zigbee2qmtt_down
expr: node_systemd_unit_state {instance = "ha1.home.2rjus.net:9100", name = "zigbee2mqtt.service", state = "active"} == 0
for: 5m
labels:
severity: critical
annotations:
summary: "Zigbee2mqtt not running on {{ $labels.instance }}"
description: "Zigbee2mqtt has been down on {{ $labels.instance }} more than 5 minutes."
- alert: mosquitto_down
expr: node_systemd_unit_state {instance = "ha1.home.2rjus.net:9100", name = "mosquitto.service", state = "active"} == 0
for: 5m
labels:
severity: critical
annotations:
summary: "Mosquitto not running on {{ $labels.instance }}"
description: "Mosquitto has been down on {{ $labels.instance }} more than 5 minutes."
''
];
2024-12-01 19:51:14 +00:00
scrapeConfigs = [
{
job_name = "node-exporter";
static_configs = [
{
targets = [
# Node exporter
"gunter.home.2rjus.net:9100"
"ca.home.2rjus.net:9100"
"monitoring01.home.2rjus.net:9100"
"ns1.home.2rjus.net:9100"
"ns2.home.2rjus.net:9100"
"http-proxy.home.2rjus.net:9100"
"ha1.home.2rjus.net:9100"
];
}
];
}
2024-12-05 18:36:55 +00:00
{
job_name = "prometheus";
static_configs = [
{
targets = [ "localhost:9090" ];
}
];
}
2024-12-03 17:01:48 +00:00
{
job_name = "pve-exporter";
static_configs = [
{
targets = [ "10.69.12.75" ];
}
];
metrics_path = "/pve";
params = {
module = [ "default" ];
cluster = [ "1" ];
node = [ "1" ];
};
relabel_configs = [
{
source_labels = [ "__address__" ];
target_label = "__param_target";
}
{
source_labels = [ "__param_target" ];
target_label = "instance";
}
{
target_label = "__address__";
replacement = "127.0.0.1:9221";
}
];
}
2024-12-04 03:02:24 +00:00
{
job_name = "caddy";
static_configs = [
{
2024-12-04 03:09:06 +00:00
targets = [ "http-proxy.home.2rjus.net" ];
2024-12-04 03:02:24 +00:00
}
];
}
2024-12-22 03:33:00 +00:00
{
job_name = "jellyfin";
static_configs = [
{
targets = [ "jelly01.home.2rjus.net:8096" ];
}
];
}
2024-12-01 19:51:14 +00:00
];
2024-12-01 00:51:29 +00:00
};
}