vault: add homelab-deploy policy to generated hosts

The homelab-deploy listener requires access to shared/homelab-deploy/*
secrets. Update hosts-generated.tf and the generator script to include
this policy automatically.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

.mcp.json

@@ -22,6 +22,17 @@
         "ALERTMANAGER_URL": "https://alertmanager.home.2rjus.net",
         "LOKI_URL": "http://monitoring01.home.2rjus.net:3100"
       }
+    },
+    "homelab-deploy": {
+      "command": "nix",
+      "args": [
+        "run",
+        "git+https://git.t-juice.club/torjus/homelab-deploy",
+        "--",
+        "mcp",
+        "--nats-url", "nats://nats1.home.2rjus.net:4222",
+        "--nkey-file", "/home/torjus/.config/homelab-deploy/test-deployer.nkey"
+      ]
     }
   }
 }

CLAUDE.md

@@ -194,6 +194,51 @@ node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes
 node_filesystem_avail_bytes{mountpoint="/"}
 ```
 
+### Deploying to Test Hosts
+
+The **homelab-deploy** MCP server enables remote deployments to test-tier hosts via NATS messaging.
+
+**Available Tools:**
+
+- `deploy` - Deploy NixOS configuration to test-tier hosts
+- `list_hosts` - List available deployment targets
+
+**Deploy Parameters:**
+
+- `hostname` - Target a specific host (e.g., `vaulttest01`)
+- `role` - Deploy to all hosts with a specific role (e.g., `vault`)
+- `all` - Deploy to all test-tier hosts
+- `action` - nixos-rebuild action: `switch` (default), `boot`, `test`, `dry-activate`
+- `branch` - Git branch or commit to deploy (default: `master`)
+
+**Examples:**
+
+```
+# List available hosts
+list_hosts()
+
+# Deploy to a specific host
+deploy(hostname="vaulttest01", action="switch")
+
+# Dry-run deployment
+deploy(hostname="vaulttest01", action="dry-activate")
+
+# Deploy to all hosts with a role
+deploy(role="vault", action="switch")
+```
+
+**Note:** Only test-tier hosts with `homelab.deploy.enable = true` and the listener service running will respond to deployments.
+
+**Verifying Deployments:**
+
+After deploying, use the `nixos_flake_info` metric from nixos-exporter to verify the host is running the expected revision:
+
+```promql
+nixos_flake_info{instance=~"vaulttest01.*"}
+```
+
+The `current_rev` label contains the git commit hash of the deployed flake configuration.
+
 ## Architecture
 
 ### Directory Structure

docs/plans/long-term-metrics-storage.md

@@ -0,0 +1,122 @@
+# Long-Term Metrics Storage Options
+
+## Problem Statement
+
+Current Prometheus configuration retains metrics for 30 days (`retentionTime = "30d"`). Extending retention further raises disk usage concerns on the homelab hypervisor with limited local storage.
+
+Prometheus does not support downsampling - it stores all data at full resolution until the retention period expires, then deletes it entirely.
+
+## Current Configuration
+
+Location: `services/monitoring/prometheus.nix`
+
+- **Retention**: 30 days
+- **Scrape interval**: 15s
+- **Features**: Alertmanager, Pushgateway, auto-generated scrape configs from flake hosts
+- **Storage**: Local disk on monitoring01
+
+## Options Evaluated
+
+### Option 1: VictoriaMetrics
+
+VictoriaMetrics is a Prometheus-compatible TSDB with significantly better compression (5-10x smaller storage footprint).
+
+**NixOS Options Available:**
+- `services.victoriametrics.enable`
+- `services.victoriametrics.prometheusConfig` - accepts Prometheus scrape config format
+- `services.victoriametrics.retentionPeriod` - e.g., "6m" for 6 months
+- `services.vmagent` - dedicated scraping agent
+- `services.vmalert` - alerting rules evaluation
+
+**Pros:**
+- Simple migration - single service replacement
+- Same PromQL query language - Grafana dashboards work unchanged
+- Same scrape config format - existing auto-generated configs work as-is
+- 5-10x better compression means 30 days of Prometheus data could become 180+ days
+- Lightweight, single binary
+
+**Cons:**
+- No automatic downsampling (relies on compression alone)
+- Alerting requires switching to vmalert instead of Prometheus alertmanager integration
+- Would need to migrate existing data or start fresh
+
+**Migration Steps:**
+1. Replace `services.prometheus` with `services.victoriametrics`
+2. Move scrape configs to `prometheusConfig`
+3. Set up `services.vmalert` for alerting rules
+4. Update Grafana datasource to VictoriaMetrics port (8428)
+5. Keep Alertmanager for notification routing
+
+### Option 2: Thanos
+
+Thanos extends Prometheus with long-term storage and automatic downsampling by uploading data to object storage.
+
+**NixOS Options Available:**
+- `services.thanos.sidecar` - uploads Prometheus blocks to object storage
+- `services.thanos.compact` - compacts and downsamples data
+- `services.thanos.query` - unified query gateway
+- `services.thanos.query-frontend` - query caching and parallelization
+- `services.thanos.downsample` - dedicated downsampling service
+
+**Downsampling Behavior:**
+- Raw resolution kept for configurable period (default: indefinite)
+- 5-minute resolution created after 40 hours
+- 1-hour resolution created after 10 days
+
+**Retention Configuration (in compactor):**
+```nix
+services.thanos.compact = {
+  retention.resolution-raw = "30d";   # Keep raw for 30 days
+  retention.resolution-5m = "180d";   # Keep 5m samples for 6 months
+  retention.resolution-1h = "2y";     # Keep 1h samples for 2 years
+};
+```
+
+**Pros:**
+- True downsampling - older data uses progressively less storage
+- Keep metrics for years with minimal storage impact
+- Prometheus continues running unchanged
+- Existing Alertmanager integration preserved
+
+**Cons:**
+- Requires object storage (MinIO, S3, or local filesystem)
+- Multiple services to manage (sidecar, compactor, query)
+- More complex architecture
+- Additional infrastructure (MinIO) may be needed
+
+**Required Components:**
+1. Thanos Sidecar (runs alongside Prometheus)
+2. Object storage (MinIO or local filesystem)
+3. Thanos Compactor (handles downsampling)
+4. Thanos Query (provides unified query endpoint)
+
+**Migration Steps:**
+1. Deploy object storage (MinIO or configure filesystem backend)
+2. Add Thanos sidecar pointing to Prometheus data directory
+3. Add Thanos compactor with retention policies
+4. Add Thanos query gateway
+5. Update Grafana datasource to Thanos Query port (10902)
+
+## Comparison
+
+| Aspect | VictoriaMetrics | Thanos |
+|--------|-----------------|--------|
+| Complexity | Low (1 service) | Higher (3-4 services) |
+| Downsampling | No | Yes (automatic) |
+| Storage savings | 5-10x compression | Compression + downsampling |
+| Object storage required | No | Yes |
+| Migration effort | Minimal | Moderate |
+| Grafana changes | Change port only | Change port only |
+| Alerting changes | Need vmalert | Keep existing |
+
+## Recommendation
+
+**Start with VictoriaMetrics** for simplicity. The compression alone may provide 6+ months of retention in the same disk space currently used for 30 days.
+
+If multi-year retention with true downsampling becomes necessary, Thanos can be evaluated later. However, it requires deploying object storage infrastructure (MinIO) which adds operational complexity.
+
+## References
+
+- VictoriaMetrics docs: https://docs.victoriametrics.com/
+- Thanos docs: https://thanos.io/tip/thanos/getting-started.md/
+- NixOS options searched from nixpkgs revision e576e3c9 (NixOS 25.11)

flake.lock

@@ -28,11 +28,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1770437282,
+        "lastModified": 1770447502,
-        "narHash": "sha256-7C6hheIP8JUkK0Aoib/lQ4xbOaXHoqSe9SJjU2u3t/Q=",
+        "narHash": "sha256-xH1PNyE3ydj4udhe1IpK8VQxBPZETGLuORZdSWYRmSU=",
         "ref": "master",
-        "rev": "cf3b1ce2c9e85ad954d8c230161553c5473e9579",
+        "rev": "79db119d1ca6630023947ef0a65896cc3307c2ff",
-        "revCount": 12,
+        "revCount": 22,
         "type": "git",
         "url": "https://git.t-juice.club/torjus/homelab-deploy"
       },

flake.nix

@@ -186,15 +186,6 @@
             ./hosts/nats1
           ];
         };
-        testvm01 = nixpkgs.lib.nixosSystem {
-          inherit system;
-          specialArgs = {
-            inherit inputs self sops-nix;
-          };
-          modules = commonModules ++ [
-            ./hosts/testvm01
-          ];
-        };
         vault01 = nixpkgs.lib.nixosSystem {
           inherit system;
           specialArgs = {
@@ -204,13 +195,31 @@
             ./hosts/vault01
           ];
         };
-        vaulttest01 = nixpkgs.lib.nixosSystem {
+        testvm01 = nixpkgs.lib.nixosSystem {
           inherit system;
           specialArgs = {
             inherit inputs self sops-nix;
           };
           modules = commonModules ++ [
-            ./hosts/vaulttest01
+            ./hosts/testvm01
+          ];
+        };
+        testvm02 = nixpkgs.lib.nixosSystem {
+          inherit system;
+          specialArgs = {
+            inherit inputs self sops-nix;
+          };
+          modules = commonModules ++ [
+            ./hosts/testvm02
+          ];
+        };
+        testvm03 = nixpkgs.lib.nixosSystem {
+          inherit system;
+          specialArgs = {
+            inherit inputs self sops-nix;
+          };
+          modules = commonModules ++ [
+            ./hosts/testvm03
           ];
         };
       };
@@ -225,11 +234,12 @@
         { pkgs }:
         {
           default = pkgs.mkShell {
-            packages = with pkgs; [
+            packages = [
-              ansible
+              pkgs.ansible
-              opentofu
+              pkgs.opentofu
-              openbao
+              pkgs.openbao
               (pkgs.callPackage ./scripts/create-host { })
+              homelab-deploy.packages.${pkgs.system}.default
             ];
           };
         }

hosts/ha1/configuration.nix

@@ -57,6 +57,7 @@
 
   # Vault secrets management
   vault.enable = true;
+  homelab.deploy.enable = true;
   vault.secrets.backup-helper = {
     secretPath = "shared/backup/password";
     extractKey = "password";

hosts/http-proxy/configuration.nix

@@ -61,6 +61,7 @@
     "flakes"
   ];
   vault.enable = true;
+  homelab.deploy.enable = true;
 
   nix.settings.tarball-ttl = 0;
   environment.systemPackages = with pkgs; [

hosts/monitoring01/configuration.nix

@@ -58,6 +58,7 @@
 
   # Vault secrets management
   vault.enable = true;
+  homelab.deploy.enable = true;
   vault.secrets.backup-helper = {
     secretPath = "shared/backup/password";
     extractKey = "password";

hosts/nix-cache01/configuration.nix

@@ -55,6 +55,7 @@
     "flakes"
   ];
   vault.enable = true;
+  homelab.deploy.enable = true;
 
   nix.settings.tarball-ttl = 0;
   environment.systemPackages = with pkgs; [

hosts/ns1/configuration.nix

@@ -48,6 +48,7 @@
     "flakes"
   ];
   vault.enable = true;
+  homelab.deploy.enable = true;
 
   homelab.host = {
     role = "dns";

hosts/ns2/configuration.nix

@@ -48,6 +48,7 @@
     "flakes"
   ];
   vault.enable = true;
+  homelab.deploy.enable = true;
 
   homelab.host = {
     role = "dns";

hosts/testvm01/configuration.nix

@@ -13,14 +13,17 @@
     ../../common/vm
   ];
 
-  # Test VM - exclude from DNS zone generation
+  # Host metadata (adjust as needed)
-  homelab.dns.enable = false;
-
   homelab.host = {
-    tier = "test";
+    tier = "test";  # Start in test tier, move to prod after validation
-    priority = "low";
   };
 
+  # Enable Vault integration
+  vault.enable = true;
+
+  # Enable remote deployment via NATS
+  homelab.deploy.enable = true;
+
   nixpkgs.config.allowUnfree = true;
   boot.loader.grub.enable = true;
   boot.loader.grub.device = "/dev/vda";
@@ -29,7 +32,7 @@
   networking.domain = "home.2rjus.net";
   networking.useNetworkd = true;
   networking.useDHCP = false;
-  services.resolved.enable = false;
+  services.resolved.enable = true;
   networking.nameservers = [
     "10.69.13.5"
     "10.69.13.6"
@@ -39,7 +42,7 @@
   systemd.network.networks."ens18" = {
     matchConfig.Name = "ens18";
     address = [
-      "10.69.13.101/24"
+      "10.69.13.20/24"
     ];
     routes = [
       { Gateway = "10.69.13.1"; }

hosts/testvm02/configuration.nix

@@ -0,0 +1,72 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [
+    ../template2/hardware-configuration.nix
+
+    ../../system
+    ../../common/vm
+  ];
+
+  # Host metadata (adjust as needed)
+  homelab.host = {
+    tier = "test";  # Start in test tier, move to prod after validation
+  };
+
+  # Enable Vault integration
+  vault.enable = true;
+
+  # Enable remote deployment via NATS
+  homelab.deploy.enable = true;
+
+  nixpkgs.config.allowUnfree = true;
+  boot.loader.grub.enable = true;
+  boot.loader.grub.device = "/dev/vda";
+
+  networking.hostName = "testvm02";
+  networking.domain = "home.2rjus.net";
+  networking.useNetworkd = true;
+  networking.useDHCP = false;
+  services.resolved.enable = true;
+  networking.nameservers = [
+    "10.69.13.5"
+    "10.69.13.6"
+  ];
+
+  systemd.network.enable = true;
+  systemd.network.networks."ens18" = {
+    matchConfig.Name = "ens18";
+    address = [
+      "10.69.13.21/24"
+    ];
+    routes = [
+      { Gateway = "10.69.13.1"; }
+    ];
+    linkConfig.RequiredForOnline = "routable";
+  };
+  time.timeZone = "Europe/Oslo";
+
+  nix.settings.experimental-features = [
+    "nix-command"
+    "flakes"
+  ];
+  nix.settings.tarball-ttl = 0;
+  environment.systemPackages = with pkgs; [
+    vim
+    wget
+    git
+  ];
+
+  # Open ports in the firewall.
+  # networking.firewall.allowedTCPPorts = [ ... ];
+  # networking.firewall.allowedUDPPorts = [ ... ];
+  # Or disable the firewall altogether.
+  networking.firewall.enable = false;
+
+  system.stateVersion = "25.11"; # Did you read the comment?
+}

hosts/testvm03/configuration.nix

@@ -0,0 +1,72 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [
+    ../template2/hardware-configuration.nix
+
+    ../../system
+    ../../common/vm
+  ];
+
+  # Host metadata (adjust as needed)
+  homelab.host = {
+    tier = "test";  # Start in test tier, move to prod after validation
+  };
+
+  # Enable Vault integration
+  vault.enable = true;
+
+  # Enable remote deployment via NATS
+  homelab.deploy.enable = true;
+
+  nixpkgs.config.allowUnfree = true;
+  boot.loader.grub.enable = true;
+  boot.loader.grub.device = "/dev/vda";
+
+  networking.hostName = "testvm03";
+  networking.domain = "home.2rjus.net";
+  networking.useNetworkd = true;
+  networking.useDHCP = false;
+  services.resolved.enable = true;
+  networking.nameservers = [
+    "10.69.13.5"
+    "10.69.13.6"
+  ];
+
+  systemd.network.enable = true;
+  systemd.network.networks."ens18" = {
+    matchConfig.Name = "ens18";
+    address = [
+      "10.69.13.22/24"
+    ];
+    routes = [
+      { Gateway = "10.69.13.1"; }
+    ];
+    linkConfig.RequiredForOnline = "routable";
+  };
+  time.timeZone = "Europe/Oslo";
+
+  nix.settings.experimental-features = [
+    "nix-command"
+    "flakes"
+  ];
+  nix.settings.tarball-ttl = 0;
+  environment.systemPackages = with pkgs; [
+    vim
+    wget
+    git
+  ];
+
+  # Open ports in the firewall.
+  # networking.firewall.allowedTCPPorts = [ ... ];
+  # networking.firewall.allowedUDPPorts = [ ... ];
+  # Or disable the firewall altogether.
+  networking.firewall.enable = false;
+
+  system.stateVersion = "25.11"; # Did you read the comment?
+}

hosts/testvm03/default.nix

@@ -0,0 +1,5 @@
+{ ... }: {
+  imports = [
+    ./configuration.nix
+  ];
+}

hosts/vaulttest01/configuration.nix

@@ -1,155 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  ...
-}:
-
-let
-  vault-test-script = pkgs.writeShellApplication {
-    name = "vault-test";
-    text = ''
-      echo "=== Vault Secret Test ==="
-      echo "Secret path: hosts/vaulttest01/test-service"
-
-      if [ -f /run/secrets/test-service/password ]; then
-        echo "✓ Password file exists"
-        echo "Password length: $(wc -c < /run/secrets/test-service/password)"
-      else
-        echo "✗ Password file missing!"
-        exit 1
-      fi
-
-      if [ -d /var/lib/vault/cache/test-service ]; then
-        echo "✓ Cache directory exists"
-      else
-        echo "✗ Cache directory missing!"
-        exit 1
-      fi
-
-      echo "Test successful!"
-    '';
-  };
-in
-{
-  imports = [
-    ../template2/hardware-configuration.nix
-
-    ../../system
-    ../../common/vm
-  ];
-
-  homelab.host = {
-    tier = "test";
-    priority = "low";
-    role = "vault";
-  };
-
-  nixpkgs.config.allowUnfree = true;
-  boot.loader.grub.enable = true;
-  boot.loader.grub.device = "/dev/vda";
-
-  networking.hostName = "vaulttest01";
-  networking.domain = "home.2rjus.net";
-  networking.useNetworkd = true;
-  networking.useDHCP = false;
-  services.resolved.enable = true;
-  networking.nameservers = [
-    "10.69.13.5"
-    "10.69.13.6"
-  ];
-
-  systemd.network.enable = true;
-  systemd.network.networks."ens18" = {
-    matchConfig.Name = "ens18";
-    address = [
-      "10.69.13.150/24"
-    ];
-    routes = [
-      { Gateway = "10.69.13.1"; }
-    ];
-    linkConfig.RequiredForOnline = "routable";
-  };
-  time.timeZone = "Europe/Oslo";
-
-  nix.settings.experimental-features = [
-    "nix-command"
-    "flakes"
-  ];
-  nix.settings.tarball-ttl = 0;
-  environment.systemPackages = with pkgs; [
-    vim
-    wget
-    git
-  ];
-
-  # Open ports in the firewall.
-  # networking.firewall.allowedTCPPorts = [ ... ];
-  # networking.firewall.allowedUDPPorts = [ ... ];
-  # Or disable the firewall altogether.
-  networking.firewall.enable = false;
-
-  # Testing config
-  # Enable Vault secrets management
-  vault.enable = true;
-
-  # Define a test secret
-  vault.secrets.test-service = {
-    secretPath = "hosts/vaulttest01/test-service";
-    restartTrigger = true;
-    restartInterval = "daily";
-    services = [ "vault-test" ];
-  };
-
-  # Homelab-deploy listener NKey
-  vault.secrets.homelab-deploy-nkey = {
-    secretPath = "shared/homelab-deploy/listener-nkey";
-    extractKey = "nkey";
-  };
-
-  # Enable homelab-deploy listener
-  services.homelab-deploy.listener = {
-    enable = true;
-    tier = "test";
-    role = "vault";
-    natsUrl = "nats://nats1.home.2rjus.net:4222";
-    nkeyFile = "/run/secrets/homelab-deploy-nkey";
-    flakeUrl = "git+https://git.t-juice.club/torjus/nixos-servers.git";
-  };
-
-  # Ensure listener starts after vault secret is available
-  systemd.services.homelab-deploy-listener = {
-    after = [ "vault-secret-homelab-deploy-nkey.service" ];
-    requires = [ "vault-secret-homelab-deploy-nkey.service" ];
-  };
-
-  # Create a test service that uses the secret
-  systemd.services.vault-test = {
-    description = "Test Vault secret fetching";
-    wantedBy = [ "multi-user.target" ];
-    after = [ "vault-secret-test-service.service" ];
-
-    serviceConfig = {
-      Type = "oneshot";
-      RemainAfterExit = true;
-
-      ExecStart = lib.getExe vault-test-script;
-
-      StandardOutput = "journal+console";
-    };
-  };
-
-  # Test ACME certificate issuance from OpenBao PKI
-  # Override the global ACME server (from system/acme.nix) to use OpenBao instead of step-ca
-  security.acme.defaults.server = lib.mkForce "https://vault01.home.2rjus.net:8200/v1/pki_int/acme/directory";
-
-  # Request a certificate for this host
-  # Using HTTP-01 challenge with standalone listener on port 80
-  security.acme.certs."vaulttest01.home.2rjus.net" = {
-    listenHTTP = ":80";
-    enableDebugLogs = true;
-  };
-
-  system.stateVersion = "25.11"; # Did you read the comment?
-}
-

modules/homelab/default.nix

@@ -1,6 +1,7 @@
 { ... }:
 {
   imports = [
+    ./deploy.nix
     ./dns.nix
     ./host.nix
     ./monitoring.nix

modules/homelab/deploy.nix

@@ -0,0 +1,16 @@
+{ config, lib, ... }:
+
+{
+  options.homelab.deploy = {
+    enable = lib.mkEnableOption "homelab-deploy listener for NATS-based deployments";
+  };
+
+  config = {
+    assertions = [
+      {
+        assertion = config.homelab.deploy.enable -> config.vault.enable;
+        message = "homelab.deploy.enable requires vault.enable to be true (needed for NKey secret)";
+      }
+    ];
+  };
+}

scripts/create-host/create_host.py

@@ -18,6 +18,8 @@ from manipulators import (
     remove_from_flake_nix,
     remove_from_terraform_vms,
     remove_from_vault_terraform,
+    remove_from_approle_tf,
+    find_host_secrets,
     check_entries_exist,
 )
 from models import HostConfig
@@ -255,7 +257,10 @@ def handle_remove(
         sys.exit(1)
 
     # Check what entries exist
-    flake_exists, terraform_exists, vault_exists = check_entries_exist(hostname, repo_root)
+    flake_exists, terraform_exists, vault_exists, approle_exists = check_entries_exist(hostname, repo_root)
+
+    # Check for secrets in secrets.tf
+    host_secrets = find_host_secrets(hostname, repo_root)
 
     # Collect all files in the host directory recursively
     files_in_host_dir = sorted([f for f in host_dir.rglob("*") if f.is_file()])
@@ -294,6 +299,21 @@ def handle_remove(
     else:
         console.print(f"  • terraform/vault/hosts-generated.tf [dim](not found)[/dim]")
 
+    if approle_exists:
+        console.print(f'  • terraform/vault/approle.tf (host_policies["{hostname}"])')
+    else:
+        console.print(f"  • terraform/vault/approle.tf [dim](not found)[/dim]")
+
+    # Warn about secrets in secrets.tf
+    if host_secrets:
+        console.print(f"\n[yellow]⚠️  Warning: Found {len(host_secrets)} secret(s) in terraform/vault/secrets.tf:[/yellow]")
+        for secret_path in host_secrets:
+            console.print(f'   • "{secret_path}"')
+        console.print(f"\n   [yellow]These will NOT be removed automatically.[/yellow]")
+        console.print(f"   After removal, manually edit secrets.tf and run:")
+        for secret_path in host_secrets:
+            console.print(f"   [white]vault kv delete secret/{secret_path}[/white]")
+
     # Warn about secrets directory
     if secrets_exist:
         console.print(f"\n[yellow]⚠️  Warning: secrets/{hostname}/ directory exists and will NOT be deleted[/yellow]")
@@ -323,6 +343,13 @@ def handle_remove(
         else:
             console.print("[yellow]⚠[/yellow]  Could not remove from terraform/vault/hosts-generated.tf")
 
+    # Remove from terraform/vault/approle.tf
+    if approle_exists:
+        if remove_from_approle_tf(hostname, repo_root):
+            console.print("[green]✓[/green] Removed from terraform/vault/approle.tf")
+        else:
+            console.print("[yellow]⚠[/yellow]  Could not remove from terraform/vault/approle.tf")
+
     # Remove from terraform/vms.tf
     if terraform_exists:
         if remove_from_terraform_vms(hostname, repo_root):
@@ -345,19 +372,34 @@ def handle_remove(
     console.print(f"\n[bold green]✓ Host {hostname} removed successfully![/bold green]\n")
 
     # Display next steps
-    display_removal_next_steps(hostname, vault_exists)
+    display_removal_next_steps(hostname, vault_exists, approle_exists, host_secrets)
 
 
-def display_removal_next_steps(hostname: str, had_vault: bool) -> None:
+def display_removal_next_steps(hostname: str, had_vault: bool, had_approle: bool, host_secrets: list) -> None:
     """Display next steps after successful removal."""
-    vault_file = " terraform/vault/hosts-generated.tf" if had_vault else ""
+    vault_files = ""
-    vault_apply = ""
     if had_vault:
+        vault_files += " terraform/vault/hosts-generated.tf"
+    if had_approle:
+        vault_files += " terraform/vault/approle.tf"
+
+    vault_apply = ""
+    if had_vault or had_approle:
         vault_apply = f"""
 3. Apply Vault changes:
    [white]cd terraform/vault && tofu apply[/white]
 """
 
+    secrets_cleanup = ""
+    if host_secrets:
+        secrets_cleanup = f"""
+5. Clean up secrets (manual):
+   Edit terraform/vault/secrets.tf to remove entries for {hostname}
+   Then delete from Vault:"""
+        for secret_path in host_secrets:
+            secrets_cleanup += f"\n   [white]vault kv delete secret/{secret_path}[/white]"
+        secrets_cleanup += "\n"
+
     next_steps = f"""[bold cyan]Next Steps:[/bold cyan]
 
 1. Review changes:
@@ -367,9 +409,9 @@ def display_removal_next_steps(hostname: str, had_vault: bool) -> None:
    [white]cd terraform && tofu destroy -target='proxmox_vm_qemu.vm["{hostname}"]'[/white]
 {vault_apply}
 4. Commit changes:
-   [white]git add -u hosts/{hostname} flake.nix terraform/vms.tf{vault_file}
+   [white]git add -u hosts/{hostname} flake.nix terraform/vms.tf{vault_files}
    git commit -m "hosts: remove {hostname}"[/white]
-"""
+{secrets_cleanup}"""
     console.print(Panel(next_steps, border_style="cyan"))
 
 

scripts/create-host/generators.py

@@ -144,7 +144,7 @@ resource "vault_approle_auth_backend_role" "generated_hosts" {
 
   backend            = vault_auth_backend.approle.path
   role_name          = each.key
-  token_policies     = ["host-\${each.key}"]
+  token_policies     = ["host-\${each.key}", "homelab-deploy"]
   secret_id_ttl      = 0  # Never expire (wrapped tokens provide time limit)
   token_ttl          = 3600
   token_max_ttl      = 3600

scripts/create-host/manipulators.py

@@ -22,12 +22,12 @@ def remove_from_flake_nix(hostname: str, repo_root: Path) -> bool:
     content = flake_path.read_text()
 
     # Check if hostname exists
-    hostname_pattern = rf"^      {re.escape(hostname)} = nixpkgs\.lib\.nixosSystem"
+    hostname_pattern = rf"^        {re.escape(hostname)} = nixpkgs\.lib\.nixosSystem"
     if not re.search(hostname_pattern, content, re.MULTILINE):
         return False
 
     # Match the entire block from "hostname = " to "};"
-    replace_pattern = rf"^      {re.escape(hostname)} = nixpkgs\.lib\.nixosSystem \{{.*?^      \}};\n"
+    replace_pattern = rf"^        {re.escape(hostname)} = nixpkgs\.lib\.nixosSystem \{{.*?^        \}};\n"
     new_content, count = re.subn(replace_pattern, "", content, flags=re.MULTILINE | re.DOTALL)
 
     if count == 0:
@@ -101,7 +101,68 @@ def remove_from_vault_terraform(hostname: str, repo_root: Path) -> bool:
     return True
 
 
-def check_entries_exist(hostname: str, repo_root: Path) -> Tuple[bool, bool, bool]:
+def remove_from_approle_tf(hostname: str, repo_root: Path) -> bool:
+    """
+    Remove host entry from terraform/vault/approle.tf locals.host_policies.
+
+    Args:
+        hostname: Hostname to remove
+        repo_root: Path to repository root
+
+    Returns:
+        True if found and removed, False if not found
+    """
+    approle_path = repo_root / "terraform" / "vault" / "approle.tf"
+
+    if not approle_path.exists():
+        return False
+
+    content = approle_path.read_text()
+
+    # Check if hostname exists in host_policies
+    hostname_pattern = rf'^\s+"{re.escape(hostname)}" = \{{'
+    if not re.search(hostname_pattern, content, re.MULTILINE):
+        return False
+
+    # Match the entire block from "hostname" = { to closing }
+    # The block contains paths = [ ... ] and possibly extra_policies = [...]
+    replace_pattern = rf'\n?\s+"{re.escape(hostname)}" = \{{[^}}]*\}}\n?'
+    new_content, count = re.subn(replace_pattern, "\n", content, flags=re.DOTALL)
+
+    if count == 0:
+        return False
+
+    approle_path.write_text(new_content)
+    return True
+
+
+def find_host_secrets(hostname: str, repo_root: Path) -> list:
+    """
+    Find secrets in terraform/vault/secrets.tf that belong to a host.
+
+    Args:
+        hostname: Hostname to search for
+        repo_root: Path to repository root
+
+    Returns:
+        List of secret paths found (e.g., ["hosts/hostname/test-service"])
+    """
+    secrets_path = repo_root / "terraform" / "vault" / "secrets.tf"
+
+    if not secrets_path.exists():
+        return []
+
+    content = secrets_path.read_text()
+
+    # Find all secret paths matching hosts/{hostname}/
+    pattern = rf'"(hosts/{re.escape(hostname)}/[^"]+)"'
+    matches = re.findall(pattern, content)
+
+    # Return unique paths, preserving order
+    return list(dict.fromkeys(matches))
+
+
+def check_entries_exist(hostname: str, repo_root: Path) -> Tuple[bool, bool, bool, bool]:
     """
     Check which entries exist for a hostname.
 
@@ -110,12 +171,12 @@ def check_entries_exist(hostname: str, repo_root: Path) -> Tuple[bool, bool, boo
         repo_root: Path to repository root
 
     Returns:
-        Tuple of (flake_exists, terraform_vms_exists, vault_exists)
+        Tuple of (flake_exists, terraform_vms_exists, vault_generated_exists, approle_exists)
     """
     # Check flake.nix
     flake_path = repo_root / "flake.nix"
     flake_content = flake_path.read_text()
-    flake_pattern = rf"^      {re.escape(hostname)} = nixpkgs\.lib\.nixosSystem"
+    flake_pattern = rf"^        {re.escape(hostname)} = nixpkgs\.lib\.nixosSystem"
     flake_exists = bool(re.search(flake_pattern, flake_content, re.MULTILINE))
 
     # Check terraform/vms.tf
@@ -131,7 +192,15 @@ def check_entries_exist(hostname: str, repo_root: Path) -> Tuple[bool, bool, boo
         vault_content = vault_tf_path.read_text()
         vault_exists = f'"{hostname}"' in vault_content
 
-    return (flake_exists, terraform_exists, vault_exists)
+    # Check terraform/vault/approle.tf
+    approle_path = repo_root / "terraform" / "vault" / "approle.tf"
+    approle_exists = False
+    if approle_path.exists():
+        approle_content = approle_path.read_text()
+        approle_pattern = rf'^\s+"{re.escape(hostname)}" = \{{'
+        approle_exists = bool(re.search(approle_pattern, approle_content, re.MULTILINE))
+
+    return (flake_exists, terraform_exists, vault_exists, approle_exists)
 
 
 def update_flake_nix(config: HostConfig, repo_root: Path, force: bool = False) -> None:
@@ -147,32 +216,25 @@ def update_flake_nix(config: HostConfig, repo_root: Path, force: bool = False) -
     content = flake_path.read_text()
 
     # Create new entry
-    new_entry = f"""      {config.hostname} = nixpkgs.lib.nixosSystem {{
+    new_entry = f"""        {config.hostname} = nixpkgs.lib.nixosSystem {{
-        inherit system;
+          inherit system;
-        specialArgs = {{
+          specialArgs = {{
-          inherit inputs self sops-nix;
+            inherit inputs self sops-nix;
+          }};
+          modules = commonModules ++ [
+            ./hosts/{config.hostname}
+          ];
         }};
-        modules = [
-          (
-            {{ config, pkgs, ... }}:
-            {{
-              nixpkgs.overlays = commonOverlays;
-            }}
-          )
-          ./hosts/{config.hostname}
-          sops-nix.nixosModules.sops
-        ];
-      }};
 """
 
     # Check if hostname already exists
-    hostname_pattern = rf"^      {re.escape(config.hostname)} = nixpkgs\.lib\.nixosSystem"
+    hostname_pattern = rf"^        {re.escape(config.hostname)} = nixpkgs\.lib\.nixosSystem"
     existing_match = re.search(hostname_pattern, content, re.MULTILINE)
 
     if existing_match and force:
         # Replace existing entry
         # Match the entire block from "hostname = " to "};"
-        replace_pattern = rf"^      {re.escape(config.hostname)} = nixpkgs\.lib\.nixosSystem \{{.*?^      \}};\n"
+        replace_pattern = rf"^        {re.escape(config.hostname)} = nixpkgs\.lib\.nixosSystem \{{.*?^        \}};\n"
         new_content, count = re.subn(replace_pattern, new_entry, content, flags=re.MULTILINE | re.DOTALL)
 
         if count == 0:

scripts/create-host/templates/configuration.nix.j2

@@ -18,6 +18,12 @@
     tier = "test";  # Start in test tier, move to prod after validation
   };
 
+  # Enable Vault integration
+  vault.enable = true;
+
+  # Enable remote deployment via NATS
+  homelab.deploy.enable = true;
+
   nixpkgs.config.allowUnfree = true;
   boot.loader.grub.enable = true;
   boot.loader.grub.device = "/dev/vda";

system/default.nix

@@ -3,6 +3,7 @@
   imports = [
     ./acme.nix
     ./autoupgrade.nix
+    ./homelab-deploy.nix
     ./monitoring
     ./motd.nix
     ./packages.nix

system/homelab-deploy.nix

@@ -0,0 +1,37 @@
+{ config, lib, ... }:
+
+let
+  hostCfg = config.homelab.host;
+in
+{
+  config = lib.mkIf config.homelab.deploy.enable {
+    # Fetch listener NKey from Vault
+    vault.secrets.homelab-deploy-nkey = {
+      secretPath = "shared/homelab-deploy/listener-nkey";
+      extractKey = "nkey";
+    };
+
+    # Enable homelab-deploy listener
+    services.homelab-deploy.listener = {
+      enable = true;
+      tier = hostCfg.tier;
+      role = hostCfg.role;
+      natsUrl = "nats://nats1.home.2rjus.net:4222";
+      nkeyFile = "/run/secrets/homelab-deploy-nkey";
+      flakeUrl = "git+https://git.t-juice.club/torjus/nixos-servers.git";
+      metrics.enable = true;
+    };
+
+    # Expose metrics for Prometheus scraping
+    homelab.monitoring.scrapeTargets = [{
+      job_name = "homelab-deploy";
+      port = 9972;
+    }];
+
+    # Ensure listener starts after vault secret is available
+    systemd.services.homelab-deploy-listener = {
+      after = [ "vault-secret-homelab-deploy-nkey.service" ];
+      requires = [ "vault-secret-homelab-deploy-nkey.service" ];
+    };
+  };
+}

terraform/vault/approle.tf

@@ -4,6 +4,17 @@ resource "vault_auth_backend" "approle" {
   path = "approle"
 }
 
+# Shared policy for homelab-deploy (all hosts need this for NATS-based deployments)
+resource "vault_policy" "homelab_deploy" {
+  name = "homelab-deploy"
+
+  policy = <<EOT
+path "secret/data/shared/homelab-deploy/*" {
+  capabilities = ["read", "list"]
+}
+EOT
+}
+
 # Define host access policies
 locals {
   host_policies = {
@@ -90,13 +101,6 @@ locals {
       ]
     }
 
-    # Vault test host with homelab-deploy access
-    "vaulttest01" = {
-      paths = [
-        "secret/data/hosts/vaulttest01/*",
-        "secret/data/shared/homelab-deploy/*",
-      ]
-    }
   }
 }
 
@@ -122,7 +126,7 @@ resource "vault_approle_auth_backend_role" "hosts" {
   backend   = vault_auth_backend.approle.path
   role_name = each.key
   token_policies = concat(
-    ["${each.key}-policy"],
+    ["${each.key}-policy", "homelab-deploy"],
     lookup(each.value, "extra_policies", [])
   )
 

terraform/vault/hosts-generated.tf

@@ -5,12 +5,22 @@
 # Each host gets access to its own secrets under hosts/<hostname>/*
 locals {
   generated_host_policies = {
-    "vaulttest01" = {
+    "testvm01" = {
       paths = [
-        "secret/data/hosts/vaulttest01/*",
+        "secret/data/hosts/testvm01/*",
       ]
     }
-
+    "testvm02" = {
+      paths = [
+        "secret/data/hosts/testvm02/*",
+      ]
+    }
+    "testvm03" = {
+      paths = [
+        "secret/data/hosts/testvm03/*",
+      ]
+    }
+  
   }
 
   # Placeholder secrets - user should add actual secrets manually or via tofu
@@ -40,7 +50,7 @@ resource "vault_approle_auth_backend_role" "generated_hosts" {
 
   backend            = vault_auth_backend.approle.path
   role_name          = each.key
-  token_policies     = ["host-${each.key}"]
+  token_policies     = ["host-${each.key}", "homelab-deploy"]
   secret_id_ttl      = 0 # Never expire (wrapped tokens provide time limit)
   token_ttl          = 3600
   token_max_ttl      = 3600

terraform/vault/secrets.tf

@@ -45,12 +45,6 @@ locals {
       password_length = 24
     }
 
-    # TODO: Remove after testing
-    "hosts/vaulttest01/test-service" = {
-      auto_generate   = true
-      password_length = 32
-    }
-
     # Shared backup password (auto-generated, add alongside existing restic key)
     "shared/backup/password" = {
       auto_generate   = true

terraform/vms.tf

@@ -31,13 +31,6 @@ locals {
     # Example Minimal VM using all defaults (uncomment to deploy):
     # "minimal-vm" = {}
     # "bootstrap-verify-test" = {}
-    "testvm01" = {
-      ip           = "10.69.13.101/24"
-      cpu_cores    = 2
-      memory       = 2048
-      disk_size    = "20G"
-      flake_branch = "pipeline-testing-improvements"
-    }
     "vault01" = {
       ip           = "10.69.13.19/24"
       cpu_cores    = 2
@@ -45,13 +38,29 @@ locals {
       disk_size    = "20G"
       flake_branch = "vault-setup" # Bootstrap from this branch instead of master
     }
-    "vaulttest01" = {
+    "testvm01" = {
-      ip                  = "10.69.13.150/24"
+      ip                  = "10.69.13.20/24"
       cpu_cores           = 2
       memory              = 2048
       disk_size           = "20G"
-      flake_branch        = "pki-migration"
+      flake_branch        = "deploy-test-hosts"
-      vault_wrapped_token = "s.UCpQCOp7cOKDdtGGBvfRWwAt"
+      vault_wrapped_token = "s.YRGRpAZVVtSYEa3wOYOqFmjt"
+    }
+    "testvm02" = {
+      ip                  = "10.69.13.21/24"
+      cpu_cores           = 2
+      memory              = 2048
+      disk_size           = "20G"
+      flake_branch        = "deploy-test-hosts"
+      vault_wrapped_token = "s.tvs8yhJOkLjBs548STs6DBw7"
+    }
+    "testvm03" = {
+      ip                  = "10.69.13.22/24"
+      cpu_cores           = 2
+      memory              = 2048
+      disk_size           = "20G"
+      flake_branch        = "deploy-test-hosts"
+      vault_wrapped_token = "s.sQ80FZGeG3z6jgrsuh74IopC"
     }
   }