docs/plans/sops-to-openbao-migration.md

@@ -0,0 +1,70 @@
+# Sops to OpenBao Secrets Migration Plan
+
+## Status: In Progress
+
+## Overview
+
+Migrate all hosts from sops-nix secrets to OpenBao (vault) secrets management. Pilot with ha1, then roll out to remaining hosts in waves.
+
+## Pre-requisites (completed)
+
+1. Hardcoded root password hash in `system/root-user.nix` (removes sops dependency for all hosts)
+2. Added `extractKey` option to `system/vault-secrets.nix` (extracts single key as file)
+
+## Deployment Order
+
+### Pilot: ha1
+- Terraform: shared/backup/password secret, ha1 AppRole policy
+- Provision AppRole credentials via `playbooks/provision-approle.yml`
+- NixOS: vault.enable + backup-helper vault secret
+
+### Wave 1: nats1, jelly01, pgdb1
+- No service secrets (only root password, already handled)
+- Just need AppRole policies + credential provisioning
+
+### Wave 2: monitoring01
+- 3 secrets: backup password, nats nkey, pve-exporter config
+- Updates: alerttonotify.nix, pve.nix, configuration.nix
+
+### Wave 3: ns1, then ns2 (critical - deploy ns1 first, verify, then ns2)
+- DNS zone transfer key (shared/dns/xfer-key)
+
+### Wave 4: http-proxy
+- WireGuard private key
+
+### Wave 5: nix-cache01
+- Cache signing key + Gitea Actions token
+
+### Wave 6: ca (DEFERRED - waiting for PKI migration)
+
+### Skipped: auth01 (decommissioned)
+
+## Terraform variables needed
+
+User must extract from sops and add to `terraform/vault/terraform.tfvars`:
+
+| Variable | Source |
+|----------|--------|
+| `backup_helper_secret` | `sops -d secrets/secrets.yaml` |
+| `ns_xfer_key` | `sops -d secrets/secrets.yaml` |
+| `nats_nkey` | `sops -d secrets/secrets.yaml` |
+| `pve_exporter_config` | `sops -d secrets/monitoring01/pve-exporter.yaml` |
+| `wireguard_private_key` | `sops -d secrets/http-proxy/wireguard.yaml` |
+| `cache_signing_key` | `sops -d secrets/nix-cache01/cache-secret` |
+| `actions_token_1` | `sops -d secrets/nix-cache01/actions_token_1` |
+
+## Provisioning AppRole credentials
+
+```bash
+export BAO_ADDR='https://vault01.home.2rjus.net:8200'
+export BAO_TOKEN='<root-token>'
+nix develop -c ansible-playbook playbooks/provision-approle.yml -e hostname=<host>
+```
+
+## Verification (per host)
+
+1. `systemctl status vault-secret-*` - all secret fetch services succeeded
+2. Check secret files exist at expected paths with correct permissions
+3. Verify dependent services are running
+4. Check `/var/lib/vault/cache/` is populated (fallback ready)
+5. Reboot host to verify boot-time secret fetching works

hosts/ha1/configuration.nix

@@ -55,8 +55,16 @@
     git
   ];
 
+  # Vault secrets management
+  vault.enable = true;
+  vault.secrets.backup-helper = {
+    secretPath = "shared/backup/password";
+    extractKey = "password";
+    outputDir = "/run/secrets/backup_helper_secret";
+    services = [ "restic-backups-ha1" ];
+  };
+
   # Backup service dirs
-  sops.secrets."backup_helper_secret" = { };
   services.restic.backups.ha1 = {
     repository = "rest:http://10.69.12.52:8000/backup-nix";
     passwordFile = "/run/secrets/backup_helper_secret";

hosts/http-proxy/configuration.nix

@@ -62,6 +62,8 @@
     "nix-command"
     "flakes"
   ];
+  vault.enable = true;
+
   nix.settings.tarball-ttl = 0;
   environment.systemPackages = with pkgs; [
     vim

hosts/http-proxy/wireguard.nix

@@ -1,9 +1,12 @@
 { config, ... }:
 {
-  sops.secrets.wireguard_private_key = {
+  vault.secrets.wireguard = {
-    sopsFile = ../../secrets/http-proxy/wireguard.yaml;
+    secretPath = "hosts/http-proxy/wireguard";
-    key = "wg_private_key";
+    extractKey = "private_key";
+    outputDir = "/run/secrets/wireguard_private_key";
+    services = [ "wireguard-wg0" ];
   };
+
   networking.wireguard = {
     enable = true;
     useNetworkd = true;
@@ -13,7 +16,7 @@
         ips = [ "10.69.222.3/24" ];
         mtu = 1384;
         listenPort = 51820;
-        privateKeyFile = config.sops.secrets.wireguard_private_key.path;
+        privateKeyFile = "/run/secrets/wireguard_private_key";
         peers = [
           {
             name = "docker2.t-juice.club";

hosts/monitoring01/configuration.nix

@@ -56,7 +56,15 @@
 
   services.qemuGuest.enable = true;
 
-  sops.secrets."backup_helper_secret" = { };
+  # Vault secrets management
+  vault.enable = true;
+  vault.secrets.backup-helper = {
+    secretPath = "shared/backup/password";
+    extractKey = "password";
+    outputDir = "/run/secrets/backup_helper_secret";
+    services = [ "restic-backups-grafana" "restic-backups-grafana-db" ];
+  };
+
   services.restic.backups.grafana = {
     repository = "rest:http://10.69.12.52:8000/backup-nix";
     passwordFile = "/run/secrets/backup_helper_secret";

hosts/nix-cache01/configuration.nix

@@ -52,6 +52,8 @@
     "nix-command"
     "flakes"
   ];
+  vault.enable = true;
+
   nix.settings.tarball-ttl = 0;
   environment.systemPackages = with pkgs; [
     vim

hosts/ns1/configuration.nix

@@ -47,6 +47,8 @@
     "nix-command"
     "flakes"
   ];
+  vault.enable = true;
+
   nix.settings.tarball-ttl = 0;
   environment.systemPackages = with pkgs; [
     vim

hosts/ns2/configuration.nix

@@ -47,6 +47,8 @@
     "nix-command"
     "flakes"
   ];
+  vault.enable = true;
+
   environment.systemPackages = with pkgs; [
     vim
     wget

playbooks/provision-approle.yml

@@ -0,0 +1,78 @@
+---
+# Provision OpenBao AppRole credentials to an existing host
+# Usage: nix develop -c ansible-playbook playbooks/provision-approle.yml -e hostname=ha1
+# Requires: BAO_ADDR and BAO_TOKEN environment variables set
+
+- name: Fetch AppRole credentials from OpenBao
+  hosts: localhost
+  connection: local
+  gather_facts: false
+
+  vars:
+    vault_addr: "{{ lookup('env', 'BAO_ADDR') | default('https://vault01.home.2rjus.net:8200', true) }}"
+    domain: "home.2rjus.net"
+
+  tasks:
+    - name: Validate hostname is provided
+      ansible.builtin.fail:
+        msg: "hostname variable is required. Use: -e hostname=<name>"
+      when: hostname is not defined
+
+    - name: Get role-id for host
+      ansible.builtin.command:
+        cmd: "bao read -field=role_id auth/approle/role/{{ hostname }}/role-id"
+      environment:
+        BAO_ADDR: "{{ vault_addr }}"
+        BAO_SKIP_VERIFY: "1"
+      register: role_id_result
+      changed_when: false
+
+    - name: Generate secret-id for host
+      ansible.builtin.command:
+        cmd: "bao write -field=secret_id -f auth/approle/role/{{ hostname }}/secret-id"
+      environment:
+        BAO_ADDR: "{{ vault_addr }}"
+        BAO_SKIP_VERIFY: "1"
+      register: secret_id_result
+      changed_when: true
+
+    - name: Add target host to inventory
+      ansible.builtin.add_host:
+        name: "{{ hostname }}.{{ domain }}"
+        groups: vault_target
+        ansible_user: root
+        vault_role_id: "{{ role_id_result.stdout }}"
+        vault_secret_id: "{{ secret_id_result.stdout }}"
+
+- name: Deploy AppRole credentials to host
+  hosts: vault_target
+  gather_facts: false
+
+  tasks:
+    - name: Create AppRole directory
+      ansible.builtin.file:
+        path: /var/lib/vault/approle
+        state: directory
+        mode: "0700"
+        owner: root
+        group: root
+
+    - name: Write role-id
+      ansible.builtin.copy:
+        content: "{{ vault_role_id }}"
+        dest: /var/lib/vault/approle/role-id
+        mode: "0600"
+        owner: root
+        group: root
+
+    - name: Write secret-id
+      ansible.builtin.copy:
+        content: "{{ vault_secret_id }}"
+        dest: /var/lib/vault/approle/secret-id
+        mode: "0600"
+        owner: root
+        group: root
+
+    - name: Display success
+      ansible.builtin.debug:
+        msg: "AppRole credentials provisioned to {{ inventory_hostname }}"

scripts/vault-fetch/vault-fetch.sh

@@ -137,9 +137,9 @@ fetch_from_vault() {
 
     # Write each secret key to a separate file
     log "Writing secrets to $OUTPUT_DIR"
-    echo "$SECRET_DATA" | jq -r 'to_entries[] | "\(.key)\n\(.value)"' | while read -r key; read -r value; do
+    for key in $(echo "$SECRET_DATA" | jq -r 'keys[]'); do
-        echo -n "$value" > "$OUTPUT_DIR/$key"
+        echo "$SECRET_DATA" | jq -j --arg k "$key" '.[$k]' > "$OUTPUT_DIR/$key"
-        echo -n "$value" > "$CACHE_DIR/$key"
+        echo "$SECRET_DATA" | jq -j --arg k "$key" '.[$k]' > "$CACHE_DIR/$key"
         chmod 600 "$OUTPUT_DIR/$key"
         chmod 600 "$CACHE_DIR/$key"
         log "  - Wrote secret key: $key"

services/actions-runner/default.nix

@@ -1,8 +1,10 @@
 { pkgs, config, ... }:
 {
-  sops.secrets."actions-token-1" = {
+  vault.secrets.actions-token = {
-    sopsFile = ../../secrets/nix-cache01/actions_token_1;
+    secretPath = "hosts/nix-cache01/actions-token";
-    format = "binary";
+    extractKey = "token";
+    outputDir = "/run/secrets/actions-token-1";
+    services = [ "gitea-runner-actions1" ];
   };
 
   virtualisation.podman = {
@@ -13,7 +15,7 @@
   services.gitea-actions-runner.instances = {
     actions1 = {
       enable = true;
-      tokenFile = config.sops.secrets.actions-token-1.path;
+      tokenFile = "/run/secrets/actions-token-1";
       name = "actions1.home.2rjus.net";
       settings = {
         log = {

services/monitoring/alerttonotify.nix

@@ -1,12 +1,18 @@
 { pkgs, config, ... }:
 {
-  sops.secrets."nats_nkey" = { };
+  vault.secrets.nats-nkey = {
+    secretPath = "shared/nats/nkey";
+    extractKey = "nkey";
+    outputDir = "/run/secrets/nats_nkey";
+    services = [ "alerttonotify" ];
+  };
+
   systemd.services."alerttonotify" = {
     enable = true;
     wants = [ "network-online.target" ];
     after = [
       "network-online.target"
-      "sops-nix.service"
+      "vault-secret-nats-nkey.service"
     ];
     wantedBy = [ "multi-user.target" ];
     restartIfChanged = true;

services/monitoring/pve.nix

@@ -1,14 +1,16 @@
 { config, ... }:
 {
-  sops.secrets.pve_exporter = {
+  vault.secrets.pve-exporter = {
-    format = "yaml";
+    secretPath = "hosts/monitoring01/pve-exporter";
-    sopsFile = ../../secrets/monitoring01/pve-exporter.yaml;
+    extractKey = "config";
-    key = "";
+    outputDir = "/run/secrets/pve_exporter";
     mode = "0444";
+    services = [ "prometheus-pve-exporter" ];
   };
+
   services.prometheus.exporters.pve = {
     enable = true;
-    configFile = config.sops.secrets.pve_exporter.path;
+    configFile = "/run/secrets/pve_exporter";
     collectors = {
       cluster = false;
       replication = false;

services/nix-cache/harmonia.nix

@@ -1,14 +1,16 @@
 { pkgs, config, ... }:
 {
-  sops.secrets."cache-secret" = {
+  vault.secrets.cache-secret = {
-    sopsFile = ../../secrets/nix-cache01/cache-secret;
+    secretPath = "hosts/nix-cache01/cache-secret";
-    format = "binary";
+    extractKey = "key";
+    outputDir = "/run/secrets/cache-secret";
+    services = [ "harmonia" ];
   };
 
   services.harmonia = {
     enable = true;
     package = pkgs.unstable.harmonia;
-    signKeyPaths = [ config.sops.secrets.cache-secret.path ];
+    signKeyPaths = [ "/run/secrets/cache-secret" ];
   };
   systemd.services.harmonia = {
     environment.RUST_LOG = "info,actix_web=debug";

services/ns/master-authorative.nix

@@ -12,8 +12,11 @@ let
   };
 in
 {
-  sops.secrets.ns_xfer_key = {
+  vault.secrets.ns-xfer-key = {
-    path = "/etc/nsd/xfer.key";
+    secretPath = "shared/dns/xfer-key";
+    extractKey = "key";
+    outputDir = "/etc/nsd/xfer.key";
+    services = [ "nsd" ];
   };
 
   networking.firewall.allowedTCPPorts = [ 8053 ];

services/ns/secondary-authorative.nix

@@ -12,8 +12,11 @@ let
   };
 in
 {
-  sops.secrets.ns_xfer_key = {
+  vault.secrets.ns-xfer-key = {
-    path = "/etc/nsd/xfer.key";
+    secretPath = "shared/dns/xfer-key";
+    extractKey = "key";
+    outputDir = "/etc/nsd/xfer.key";
+    services = [ "nsd" ];
   };
   networking.firewall.allowedTCPPorts = [ 8053 ];
   networking.firewall.allowedUDPPorts = [ 8053 ];

system/root-user.nix

@@ -1,11 +1,10 @@
-{ pkgs, config, ... }: {
+{ pkgs, config, ... }:
+{
   programs.zsh.enable = true;
-  sops.secrets.root_password_hash = { };
-  sops.secrets.root_password_hash.neededForUsers = true;
 
   users.users.root = {
     shell = pkgs.zsh;
-    hashedPasswordFile = config.sops.secrets.root_password_hash.path;
+    hashedPassword = "$y$j9T$N09APWqKc4//z9BoGyzSb0$3dMUzojSmo3/10nbIfShd6/IpaYoKdI21bfbWER3jl8";
     openssh.authorizedKeys.keys = [
       "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAwfb2jpKrBnCw28aevnH8HbE5YbcMXpdaVv2KmueDu6 torjus@gunter"
     ];

system/vault-secrets.nix

@@ -73,6 +73,16 @@ let
         '';
       };
 
+      extractKey = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        description = ''
+          Extract a single key from the vault secret JSON and write it as a
+          plain file instead of a directory of files. When set, outputDir
+          becomes a file path rather than a directory path.
+        '';
+      };
+
       services = mkOption {
         type = types.listOf types.str;
         default = [];
@@ -152,14 +162,35 @@ in
           RemainAfterExit = true;
 
           # Fetch the secret
-          ExecStart = pkgs.writeShellScript "fetch-${name}" ''
+          ExecStart = pkgs.writeShellScript "fetch-${name}" (''
             set -euo pipefail
 
             # Set Vault environment variables
             export VAULT_ADDR="${cfg.vaultAddress}"
             export VAULT_SKIP_VERIFY="${if cfg.skipTlsVerify then "1" else "0"}"
+          '' + (if secretCfg.extractKey != null then ''
+            # Fetch to temporary directory, then extract single key
+            TMPDIR=$(mktemp -d)
+            trap "rm -rf $TMPDIR" EXIT
 
-            # Fetch secret using vault-fetch
+            ${vault-fetch}/bin/vault-fetch \
+              "${secretCfg.secretPath}" \
+              "$TMPDIR" \
+              "${secretCfg.cacheDir}"
+
+            # Extract the specified key and write as a single file
+            if [ ! -f "$TMPDIR/${secretCfg.extractKey}" ]; then
+              echo "ERROR: Key '${secretCfg.extractKey}' not found in secret" >&2
+              exit 1
+            fi
+
+            # Ensure parent directory exists
+            mkdir -p "$(dirname "${secretCfg.outputDir}")"
+            cp "$TMPDIR/${secretCfg.extractKey}" "${secretCfg.outputDir}"
+            chown ${secretCfg.owner}:${secretCfg.group} "${secretCfg.outputDir}"
+            chmod ${secretCfg.mode} "${secretCfg.outputDir}"
+          '' else ''
+            # Fetch secret as directory of files
             ${vault-fetch}/bin/vault-fetch \
               "${secretCfg.secretPath}" \
               "${secretCfg.outputDir}" \
@@ -168,7 +199,7 @@ in
             # Set ownership and permissions
             chown -R ${secretCfg.owner}:${secretCfg.group} "${secretCfg.outputDir}"
             chmod ${secretCfg.mode} "${secretCfg.outputDir}"/*
-          '';
+          ''));
 
           # Logging
           StandardOutput = "journal";
@@ -216,7 +247,10 @@ in
       [ "d /run/secrets 0755 root root -" ] ++
       [ "d /var/lib/vault/cache 0700 root root -" ] ++
       flatten (mapAttrsToList (name: secretCfg: [
-        "d ${secretCfg.outputDir} 0755 root root -"
+        # When extractKey is set, outputDir is a file path - create parent dir instead
+        (if secretCfg.extractKey != null
+         then "d ${dirOf secretCfg.outputDir} 0755 root root -"
+         else "d ${secretCfg.outputDir} 0755 root root -")
         "d ${secretCfg.cacheDir} 0700 root root -"
       ]) cfg.secrets);
   };

terraform/vault/approle.tf

@@ -25,17 +25,66 @@ locals {
     #   ]
     # }
 
-    # TODO: actually use this policy
     "ha1" = {
       paths = [
         "secret/data/hosts/ha1/*",
+        "secret/data/shared/backup/*",
       ]
     }
 
-    # TODO: actually use this policy
     "monitoring01" = {
       paths = [
         "secret/data/hosts/monitoring01/*",
+        "secret/data/shared/backup/*",
+        "secret/data/shared/nats/*",
+      ]
+    }
+
+    # Wave 1: hosts with no service secrets (only need vault.enable for future use)
+    "nats1" = {
+      paths = [
+        "secret/data/hosts/nats1/*",
+      ]
+    }
+
+    "jelly01" = {
+      paths = [
+        "secret/data/hosts/jelly01/*",
+      ]
+    }
+
+    "pgdb1" = {
+      paths = [
+        "secret/data/hosts/pgdb1/*",
+      ]
+    }
+
+    # Wave 3: DNS servers
+    "ns1" = {
+      paths = [
+        "secret/data/hosts/ns1/*",
+        "secret/data/shared/dns/*",
+      ]
+    }
+
+    "ns2" = {
+      paths = [
+        "secret/data/hosts/ns2/*",
+        "secret/data/shared/dns/*",
+      ]
+    }
+
+    # Wave 4: http-proxy
+    "http-proxy" = {
+      paths = [
+        "secret/data/hosts/http-proxy/*",
+      ]
+    }
+
+    # Wave 5: nix-cache01
+    "nix-cache01" = {
+      paths = [
+        "secret/data/hosts/nix-cache01/*",
       ]
     }
   }

terraform/vault/secrets.tf

@@ -35,22 +35,63 @@ locals {
     #   }
     # }
 
-    # TODO: actually use the secret
     "hosts/monitoring01/grafana-admin" = {
       auto_generate   = true
       password_length = 32
     }
 
-    # TODO: actually use the secret
     "hosts/ha1/mqtt-password" = {
       auto_generate   = true
       password_length = 24
     }
+
     # TODO: Remove after testing
     "hosts/vaulttest01/test-service" = {
       auto_generate   = true
       password_length = 32
     }
+
+    # Shared backup password (auto-generated, add alongside existing restic key)
+    "shared/backup/password" = {
+      auto_generate   = true
+      password_length = 32
+    }
+
+    # NATS NKey for alerttonotify
+    "shared/nats/nkey" = {
+      auto_generate = false
+      data          = { nkey = var.nats_nkey }
+    }
+
+    # PVE exporter config for monitoring01
+    "hosts/monitoring01/pve-exporter" = {
+      auto_generate = false
+      data          = { config = var.pve_exporter_config }
+    }
+
+    # DNS zone transfer key
+    "shared/dns/xfer-key" = {
+      auto_generate = false
+      data          = { key = var.ns_xfer_key }
+    }
+
+    # WireGuard private key for http-proxy
+    "hosts/http-proxy/wireguard" = {
+      auto_generate = false
+      data          = { private_key = var.wireguard_private_key }
+    }
+
+    # Nix cache signing key
+    "hosts/nix-cache01/cache-secret" = {
+      auto_generate = false
+      data          = { key = var.cache_signing_key }
+    }
+
+    # Gitea Actions runner token
+    "hosts/nix-cache01/actions-token" = {
+      auto_generate = false
+      data          = { token = var.actions_token_1 }
+    }
   }
 }
 

terraform/vault/variables.tf

@@ -16,11 +16,38 @@ variable "vault_skip_tls_verify" {
   default     = true
 }
 
-# Example variables for manual secrets
+variable "nats_nkey" {
-# Uncomment and add to terraform.tfvars as needed
+  description = "NATS NKey for alerttonotify"
+  type        = string
+  sensitive   = true
+}
 
-# variable "smtp_password" {
+variable "pve_exporter_config" {
-#   description = "SMTP password for notifications"
+  description = "PVE exporter YAML configuration"
-#   type        = string
+  type        = string
-#   sensitive   = true
+  sensitive   = true
-# }
+}
+
+variable "ns_xfer_key" {
+  description = "DNS zone transfer TSIG key"
+  type        = string
+  sensitive   = true
+}
+
+variable "wireguard_private_key" {
+  description = "WireGuard private key for http-proxy"
+  type        = string
+  sensitive   = true
+}
+
+variable "cache_signing_key" {
+  description = "Nix binary cache signing key"
+  type        = string
+  sensitive   = true
+}
+
+variable "actions_token_1" {
+  description = "Gitea Actions runner token"
+  type        = string
+  sensitive   = true
+}