loki: add basic auth for log push and dual-ship promtail

- Loki bound to localhost, Caddy reverse proxy with basic_auth
- Vault secret (shared/loki/push-auth) for password, bcrypt hash
  generated at boot for Caddy environment
- Promtail dual-ships to monitoring01 (direct) and loki.home.2rjus.net
  (with basic auth), conditional on vault.enable
- Terraform: new shared loki-push policy added to all AppRoles

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

services/loki/default.nix

@@ -1,7 +1,47 @@
-{ ... }:
+{ config, lib, pkgs, ... }:
+let
+  # Script to generate bcrypt hash from Vault password for Caddy basic_auth
+  generateCaddyAuth = pkgs.writeShellApplication {
+    name = "generate-caddy-loki-auth";
+    runtimeInputs = [ config.services.caddy.package ];
+    text = ''
+      PASSWORD=$(cat /run/secrets/loki-push-auth)
+      HASH=$(caddy hash-password --plaintext "$PASSWORD")
+      echo "LOKI_PUSH_HASH=$HASH" > /run/secrets/caddy-loki-auth.env
+      chmod 0400 /run/secrets/caddy-loki-auth.env
+    '';
+  };
+in
 {
-  # Caddy reverse proxy for Loki
+  # Fetch Loki push password from Vault
+  vault.secrets.loki-push-auth = {
+    secretPath = "shared/loki/push-auth";
+    extractKey = "password";
+    services = [ "caddy" ];
+  };
+
+  # Generate bcrypt hash for Caddy before it starts
+  systemd.services.caddy-loki-auth = {
+    description = "Generate Caddy basic auth hash for Loki";
+    after = [ "vault-secret-loki-push-auth.service" ];
+    requires = [ "vault-secret-loki-push-auth.service" ];
+    before = [ "caddy.service" ];
+    requiredBy = [ "caddy.service" ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+      ExecStart = lib.getExe generateCaddyAuth;
+    };
+  };
+
+  # Load the bcrypt hash as environment variable for Caddy
+  services.caddy.environmentFile = "/run/secrets/caddy-loki-auth.env";
+
+  # Caddy reverse proxy for Loki with basic auth
   services.caddy.virtualHosts."loki.home.2rjus.net".extraConfig = ''
+    basic_auth {
+      promtail {env.LOKI_PUSH_HASH}
+    }
     reverse_proxy http://127.0.0.1:3100
   '';
 
@@ -11,6 +51,7 @@
       auth_enabled = false;
 
       server = {
+        http_listen_address = "127.0.0.1";
         http_listen_port = 3100;
       };
       common = {

system/monitoring/logs.nix

@@ -16,6 +16,14 @@ in
       SystemKeepFree=1G
     '';
   };
+
+  # Fetch Loki push password from Vault (only on hosts with Vault enabled)
+  vault.secrets.promtail-loki-auth = lib.mkIf config.vault.enable {
+    secretPath = "shared/loki/push-auth";
+    extractKey = "password";
+    services = [ "promtail" ];
+  };
+
   # Configure promtail
   services.promtail = {
     enable = true;
@@ -31,6 +39,14 @@ in
         {
           url = "http://monitoring01.home.2rjus.net:3100/loki/api/v1/push";
         }
+      ] ++ lib.optionals config.vault.enable [
+        {
+          url = "https://loki.home.2rjus.net/loki/api/v1/push";
+          basic_auth = {
+            username = "promtail";
+            password_file = "/run/secrets/promtail-loki-auth";
+          };
+        }
       ];
 
       scrape_configs = [

terraform/vault/approle.tf

@@ -26,6 +26,17 @@ path "secret/data/shared/nixos-exporter/*" {
 EOT
 }
 
+# Shared policy for Loki push authentication (all hosts push logs)
+resource "vault_policy" "loki_push" {
+  name = "loki-push"
+
+  policy = <<EOT
+path "secret/data/shared/loki/*" {
+  capabilities = ["read", "list"]
+}
+EOT
+}
+
 # Define host access policies
 locals {
   host_policies = {
@@ -138,7 +149,7 @@ resource "vault_approle_auth_backend_role" "hosts" {
   backend   = vault_auth_backend.approle.path
   role_name = each.key
   token_policies = concat(
-    ["${each.key}-policy", "homelab-deploy", "nixos-exporter"],
+    ["${each.key}-policy", "homelab-deploy", "nixos-exporter", "loki-push"],
     lookup(each.value, "extra_policies", [])
   )
 

terraform/vault/secrets.tf

@@ -153,6 +153,12 @@ locals {
       auto_generate   = true
       password_length = 64
     }
+
+    # Loki push authentication (used by Promtail on all hosts)
+    "shared/loki/push-auth" = {
+      auto_generate   = true
+      password_length = 32
+    }
   }
 }