nixos-servers/services/monitoring/prometheus.nix

{ ... }:
{
  services.prometheus = {
    enable = true;
    alertmanager = {
      enable = true;
      configuration = {
        global = {
        };
        route = {
          receiver = "webhook_natstonotify";
          group_wait = "30s";
          group_interval = "5m";
          repeat_interval = "1h";
          group_by = [ "alertname" ];
        };
        receivers = [
          {
            name = "webhook_natstonotify";
            webhook_configs = [
              {
                url = "http://localhost:5001/alert";
              }
            ];
          }
        ];
      };
    };
    alertmanagers = [
      {
        static_configs = [
          {
            targets = [ "localhost:9093" ];
          }
        ];
      }
    ];

    retentionTime = "30d";
    globalConfig = {
      scrape_interval = "15s";
    };
    rules = [
      (builtins.readFile ./rules.yml)
    ];

    scrapeConfigs = [
      {
        job_name = "node-exporter";
        static_configs = [
          {
            targets = [
              "ca.home.2rjus.net:9100"
              "gunter.home.2rjus.net:9100"
              "ha1.home.2rjus.net:9100"
              "http-proxy.home.2rjus.net:9100"
              "jelly01.home.2rjus.net:9100"
              "monitoring01.home.2rjus.net:9100"
              "nix-cache01.home.2rjus.net:9100"
              "ns1.home.2rjus.net:9100"
              "ns2.home.2rjus.net:9100"
              "pgdb1.home.2rjus.net:9100"
              "nats1.home.2rjus.net:9100"
            ];
          }
        ];
      }
      {
        job_name = "prometheus";
        static_configs = [
          {
            targets = [ "localhost:9090" ];
          }
        ];
      }
      {
        job_name = "loki";
        static_configs = [
          {
            targets = [ "localhost:3100" ];
          }
        ];
      }
      {
        job_name = "grafana";
        static_configs = [
          {
            targets = [ "localhost:3100" ];
          }
        ];
      }
      {
        job_name = "alertmanager";
        static_configs = [
          {
            targets = [ "localhost:9093" ];
          }
        ];
      }
      {
        job_name = "restic_rest";
        static_configs = [
          {
            targets = [ "10.69.12.52:8000" ];
          }
        ];
      }
      {
        job_name = "pve-exporter";
        static_configs = [
          {
            targets = [ "10.69.12.75" ];
          }
        ];
        metrics_path = "/pve";
        params = {
          module = [ "default" ];
          cluster = [ "1" ];
          node = [ "1" ];
        };
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
          }
          {
            source_labels = [ "__param_target" ];
            target_label = "instance";
          }
          {
            target_label = "__address__";
            replacement = "127.0.0.1:9221";
          }
        ];
      }
      {
        job_name = "caddy";
        static_configs = [
          {
            targets = [ "http-proxy.home.2rjus.net" ];
          }
        ];
      }
      {
        job_name = "jellyfin";
        static_configs = [
          {
            targets = [ "jelly01.home.2rjus.net:8096" ];
          }
        ];
      }
      {
        job_name = "smartctl";
        static_configs = [
          {
            targets = [ "gunter.home.2rjus.net:9633" ];
          }
        ];
      }
      {
        job_name = "wireguard";
        static_configs = [
          {
            targets = [ "http-proxy.home.2rjus.net:9586" ];
          }
        ];
      }
      {
        job_name = "home-assistant";
        scrape_interval = "60s";
        metrics_path = "/api/prometheus";
        static_configs = [
          {
            targets = [ "ha1.home.2rjus.net:8123" ];
          }
        ];
      }
      {
        job_name = "ghettoptt";
        static_configs = [
          {
            targets = [ "gunter.home.2rjus.net:8989" ];
          }
        ];
      }
      {
        job_name = "step-ca";
        static_configs = [
          {
            targets = [ "ca.home.2rjus.net:9000" ];
          }
        ];
      }
      {
        job_name = "labmon";
        static_configs = [
          {
            targets = [ "monitoring01.home.2rjus.net:9969" ];
          }
        ];
      }
      {
        job_name = "pushgateway";
        honor_labels = true;
        static_configs = [
          {
            targets = [ "localhost:9091" ];
          }
        ];
      }
      {
        job_name = "nix-cache_caddy";
        static_configs = [
          {
            scheme = "https";
            targets = [ "nix-cache.home.2rjus.net" ];
          }
        ];
      }
    ];
    pushgateway = {
      enable = true;
      web = {
        external-url = "https://pushgw.home.2rjus.net";
      };
    };
  };
}