nixos-servers/system/monitoring/logs.nix

{ config, lib, ... }:
let
  hostLabels = {
    hostname = config.networking.hostName;
    tier = config.homelab.host.tier;
  } // lib.optionalAttrs (config.homelab.host.role != null) {
    role = config.homelab.host.role;
  };
in
{
  # Configure journald
  services.journald = {
    rateLimitInterval = "10s";
    extraConfig = ''
      SystemMaxUse=100M
      SystemKeepFree=1G
    '';
  };

  # Fetch Loki push password from Vault (only on hosts with Vault enabled)
  vault.secrets.promtail-loki-auth = lib.mkIf config.vault.enable {
    secretPath = "shared/loki/push-auth";
    extractKey = "password";
    services = [ "promtail" ];
  };

  # Configure promtail
  services.promtail = {
    enable = true;
    configuration = {
      server = {
        http_listen_address = "0.0.0.0";
        http_listen_port = 9099;
        grpc_listen_address = "0.0.0.0";
        grpc_listen_port = 9098;
      };

      clients = [
        {
          url = "http://monitoring01.home.2rjus.net:3100/loki/api/v1/push";
        }
      ] ++ lib.optionals config.vault.enable [
        {
          url = "https://loki.home.2rjus.net/loki/api/v1/push";
          basic_auth = {
            username = "promtail";
            password_file = "/run/secrets/promtail-loki-auth";
          };
        }
      ];

      scrape_configs = [
        {
          job_name = "journal";
          journal = {
            json = true;
            labels = {
              job = "systemd-journal";
            } // hostLabels;
          };
          relabel_configs = [
            {
              source_labels = [ "__journal__systemd_unit" ];
              target_label = "systemd_unit";
            }
          ];
          pipeline_stages = [
            # Extract PRIORITY from journal JSON
            { json.expressions.priority = "PRIORITY"; }
            # Map numeric PRIORITY to level name
            {
              template = {
                source = "priority";
                template = ''{{ if or (eq .Value "0") (eq .Value "1") (eq .Value "2") }}critical{{ else if eq .Value "3" }}error{{ else if eq .Value "4" }}warning{{ else if eq .Value "5" }}notice{{ else if eq .Value "6" }}info{{ else if eq .Value "7" }}debug{{ end }}'';
              };
            }
            # Attach as level label
            { labels.level = "priority"; }
          ];
        }
        {
          job_name = "varlog";
          static_configs = [
            {
              targets = [ "localhost" ];
              labels = {
                job = "varlog";
                __path__ = "/var/log/**/*.log";
              } // hostLabels;
            }
          ];
        }
      ];
    };
  };
}