docs: mark pgdb1 for decommissioning instead of migration

Only consumer was Open WebUI on gunter, which will migrate to local
PostgreSQL. Removed pgdb1 backup/migration phases and added to
decommission list.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

.claude/skills/observability/SKILL.md

@@ -185,21 +185,60 @@ Common job names:
 - `home-assistant` - Home automation
 - `step-ca` - Internal CA
 
-### Instance Label Format
+### Target Labels
 
-The `instance` label uses FQDN format:
+All scrape targets have these labels:
 
-```
-<hostname>.home.2rjus.net:<port>
-```
+**Standard labels:**
+- `instance` - Full target address (`<hostname>.home.2rjus.net:<port>`)
+- `job` - Job name (e.g., `node-exporter`, `unbound`, `nixos-exporter`)
+- `hostname` - Short hostname (e.g., `ns1`, `monitoring01`) - use this for host filtering
 
-Example queries filtering by host:
+**Host metadata labels** (when configured in `homelab.host`):
+- `role` - Host role (e.g., `dns`, `build-host`, `vault`)
+- `tier` - Deployment tier (`test` for test VMs, absent for prod)
+- `dns_role` - DNS-specific role (`primary` or `secondary` for ns1/ns2)
+
+### Filtering by Host
+
+Use the `hostname` label for easy host filtering across all jobs:
 
 ```promql
-up{instance=~"monitoring01.*"}
-node_load1{instance=~"ns1.*"}
+{hostname="ns1"}                    # All metrics from ns1
+node_load1{hostname="monitoring01"} # Specific metric by hostname
+up{hostname="ha1"}                  # Check if ha1 is up
 ```
 
+This is simpler than wildcarding the `instance` label:
+
+```promql
+# Old way (still works but verbose)
+up{instance=~"monitoring01.*"}
+
+# New way (preferred)
+up{hostname="monitoring01"}
+```
+
+### Filtering by Role/Tier
+
+Filter hosts by their role or tier:
+
+```promql
+up{role="dns"}                      # All DNS servers (ns1, ns2)
+node_cpu_seconds_total{role="build-host"}  # Build hosts only (nix-cache01)
+up{tier="test"}                     # All test-tier VMs
+up{dns_role="primary"}              # Primary DNS only (ns1)
+```
+
+Current host labels:
+| Host | Labels |
+|------|--------|
+| ns1 | `role=dns`, `dns_role=primary` |
+| ns2 | `role=dns`, `dns_role=secondary` |
+| nix-cache01 | `role=build-host` |
+| vault01 | `role=vault` |
+| testvm01/02/03 | `tier=test` |
+
 ---
 
 ## Troubleshooting Workflows
@@ -212,11 +251,12 @@ node_load1{instance=~"ns1.*"}
 
 ### Investigate Service Issues
 
-1. Check `up{job="<service>"}` for scrape failures
+1. Check `up{job="<service>"}` or `up{hostname="<host>"}` for scrape failures
 2. Use `list_targets` to see target health details
 3. Query service logs: `{host="<host>", systemd_unit="<service>.service"}`
 4. Search for errors: `{host="<host>"} |= "error"`
 5. Check `list_alerts` for related alerts
+6. Use role filters for group issues: `up{role="dns"}` to check all DNS servers
 
 ### After Deploying Changes
 
@@ -246,5 +286,6 @@ With `start: "24h"` to see last 24 hours of upgrades across all hosts.
 - Default scrape interval is 15s for most metrics targets
 - Default log lookback is 1h - use `start` parameter for older logs
 - Use `rate()` for counter metrics, direct queries for gauges
-- The `instance` label includes the port, use regex matching (`=~`) for hostname-only filters
+- Use the `hostname` label to filter metrics by host (simpler than regex on `instance`)
+- Host metadata labels (`role`, `tier`, `dns_role`) are propagated to all scrape targets
 - Log `MESSAGE` field contains the actual log content in JSON format

.sops.yaml

@@ -1,52 +0,0 @@
-keys:
-  - &admin_torjus age1lznyk4ee7e7x8n92cq2n87kz9920473ks5u9jlhd3dczfzq4wamqept56u
-  - &server_ns1 age1hz2lz4k050ru3shrk5j3zk3f8azxmrp54pktw5a7nzjml4saudesx6jsl0
-  - &server_ns2 age1w2q4gm2lrcgdzscq8du3ssyvk6qtzm4fcszc92z9ftclq23yyydqdga5um
-  - &server_ha1 age1d2w5zece9647qwyq4vas9qyqegg96xwmg6c86440a6eg4uj6dd2qrq0w3l
-  - &server_http-proxy age1gq8434ku0xekqmvnseeunv83e779cg03c06gwrusnymdsr3rpufqx6vr3m
-  - &server_ca age1288993th0ge00reg4zqueyvmkrsvk829cs068eekjqfdprsrkeqql7mljk
-  - &server_monitoring01 age1vpns76ykll8jgdlu3h05cur4ew2t3k7u03kxdg8y6ypfhsfhq9fqyurjey
-  - &server_jelly01 age1hchvlf3apn8g8jq2743pw53sd6v6ay6xu6lqk0qufrjeccan9vzsc7hdfq
-  - &server_nix-cache01 age1w029fksjv0edrff9p7s03tgk3axecdkppqymfpwfn2nu2gsqqefqc37sxq
-  - &server_pgdb1 age1ha34qeksr4jeaecevqvv2afqem67eja2mvawlmrqsudch0e7fe7qtpsekv
-  - &server_nats1 age1cxt8kwqzx35yuldazcc49q88qvgy9ajkz30xu0h37uw3ts97jagqgmn2ga
-creation_rules:
-  - path_regex: secrets/[^/]+\.(yaml|json|env|ini)
-    key_groups:
-      - age:
-        - *admin_torjus
-        - *server_ns1
-        - *server_ns2
-        - *server_ha1
-        - *server_http-proxy
-        - *server_ca
-        - *server_monitoring01
-        - *server_jelly01
-        - *server_nix-cache01
-        - *server_pgdb1
-        - *server_nats1
-  - path_regex: secrets/ca/[^/]+\.(yaml|json|env|ini|)
-    key_groups:
-      - age:
-        - *admin_torjus
-        - *server_ca
-  - path_regex: secrets/monitoring01/[^/]+\.(yaml|json|env|ini)
-    key_groups:
-      - age:
-        - *admin_torjus
-        - *server_monitoring01
-  - path_regex: secrets/ca/keys/.+
-    key_groups:
-      - age:
-        - *admin_torjus
-        - *server_ca
-  - path_regex: secrets/nix-cache01/.+
-    key_groups:
-      - age:
-        - *admin_torjus
-        - *server_nix-cache01
-  - path_regex: secrets/http-proxy/.+
-    key_groups:
-      - age:
-        - *admin_torjus
-        - *server_http-proxy

CLAUDE.md

@@ -61,25 +61,45 @@ Do not run `nix flake update`. Should only be done manually by user.
 ### Development Environment
 
 ```bash
-# Enter development shell (provides ansible, python3)
+# Enter development shell
 nix develop
 ```
 
+The devshell provides: `ansible`, `tofu` (OpenTofu), `bao` (OpenBao CLI), `create-host`, and `homelab-deploy`.
+
+**Important:** When suggesting commands that use devshell tools, always use `nix develop -c <command>` syntax rather than assuming the user is already in a devshell. For example:
+```bash
+# Good - works regardless of current shell
+nix develop -c tofu plan
+
+# Avoid - requires user to be in devshell
+tofu plan
+```
+
+**OpenTofu:** Use the `-chdir` option instead of `cd` when running tofu commands in subdirectories:
+```bash
+# Good - uses -chdir option
+nix develop -c tofu -chdir=terraform plan
+nix develop -c tofu -chdir=terraform/vault apply
+
+# Avoid - changing directories
+cd terraform && tofu plan
+```
+
 ### Secrets Management
 
 Secrets are managed by OpenBao (Vault) using AppRole authentication. Most hosts use the
 `vault.secrets` option defined in `system/vault-secrets.nix` to fetch secrets at boot.
 Terraform manages the secrets and AppRole policies in `terraform/vault/`.
 
-Legacy sops-nix is still present but only actively used by the `ca` host. Do not edit any
-`.sops.yaml` or any file within `secrets/`. Ask the user to modify if necessary.
-
 ### Git Workflow
 
 **Important:** Never commit directly to `master` unless the user explicitly asks for it. Always create a feature branch for changes.
 
 **Important:** Never amend commits to `master` unless the user explicitly asks for it. Amending rewrites history and causes issues for deployed configurations.
 
+**Important:** Do not use `gh pr create` to create pull requests. The git server does not support GitHub CLI for PR creation. Instead, push the branch and let the user create the PR manually via the web interface.
+
 When starting a new plan or task, the first step should typically be to create and checkout a new branch with an appropriate name (e.g., `git checkout -b dns-automation` or `git checkout -b fix-nginx-config`).
 
 ### Plan Management
@@ -140,11 +160,27 @@ The **lab-monitoring** MCP server can query logs from Loki. All hosts ship syste
 
 - `host` - Hostname (e.g., `ns1`, `ns2`, `monitoring01`, `ha1`). Use this label, not `hostname`.
 - `systemd_unit` - Systemd unit name (e.g., `nsd.service`, `prometheus.service`, `nixos-upgrade.service`)
-- `job` - Either `systemd-journal` (most logs) or `varlog` (file-based logs like caddy access logs)
+- `job` - Either `systemd-journal` (most logs), `varlog` (file-based logs), or `bootstrap` (VM bootstrap logs)
 - `filename` - For `varlog` job, the log file path (e.g., `/var/log/caddy/nix-cache.log`)
 
 Journal log entries are JSON-formatted with the actual log message in the `MESSAGE` field. Other useful fields include `PRIORITY` and `SYSLOG_IDENTIFIER`.
 
+**Bootstrap Logs:**
+
+VMs provisioned from template2 send bootstrap progress directly to Loki via curl (before promtail is available). These logs use `job="bootstrap"` with additional labels:
+
+- `host` - Target hostname
+- `branch` - Git branch being deployed
+- `stage` - Bootstrap stage: `starting`, `network_ok`, `vault_ok`/`vault_skip`/`vault_warn`, `building`, `success`, `failed`
+
+Query bootstrap status:
+```
+{job="bootstrap"}                              # All bootstrap logs
+{job="bootstrap", host="testvm01"}             # Specific host
+{job="bootstrap", stage="failed"}              # All failures
+{job="bootstrap", stage=~"building|success"}   # Track build progress
+```
+
 **Example LogQL queries:**
 ```
 # Logs from a specific service on a host
@@ -171,13 +207,12 @@ The **lab-monitoring** MCP server can query Prometheus metrics via PromQL. The `
 - `home-assistant` - Home automation metrics
 - `jellyfin` - Media server metrics
 - `loki` / `prometheus` / `grafana` - Monitoring stack self-metrics
-- `step-ca` - Internal CA metrics
 - `pve-exporter` - Proxmox hypervisor metrics
 - `smartctl` - Disk SMART health (gunter)
 - `wireguard` - VPN metrics (http-proxy)
 - `pushgateway` - Push-based metrics (e.g., backup results)
 - `restic_rest` - Backup server metrics
-- `labmon` / `ghettoptt` / `alertmanager` - Other service metrics
+- `ghettoptt` / `alertmanager` - Other service metrics
 
 **Example PromQL queries:**
 ```
@@ -229,6 +264,21 @@ deploy(role="vault", action="switch")
 
 **Note:** Only test-tier hosts with `homelab.deploy.enable = true` and the listener service running will respond to deployments.
 
+**Deploying to Prod Hosts:**
+
+The MCP server only deploys to test-tier hosts. For prod hosts, use the CLI directly:
+
+```bash
+nix develop -c homelab-deploy -- deploy \
+  --nats-url nats://nats1.home.2rjus.net:4222 \
+  --nkey-file ~/.config/homelab-deploy/admin-deployer.nkey \
+  --branch <branch-name> \
+  --action switch \
+  deploy.prod.<hostname>
+```
+
+Subject format: `deploy.<tier>.<hostname>` (e.g., `deploy.prod.monitoring01`, `deploy.test.testvm01`)
+
 **Verifying Deployments:**
 
 After deploying, use the `nixos_flake_info` metric from nixos-exporter to verify the host is running the expected revision:
@@ -248,10 +298,11 @@ The `current_rev` label contains the git commit hash of the deployed flake confi
   - `default.nix` - Entry point, imports configuration.nix and services
   - `configuration.nix` - Host-specific settings (networking, hardware, users)
 - `/system/` - Shared system-level configurations applied to ALL hosts
-  - Core modules: nix.nix, sshd.nix, sops.nix (legacy), vault-secrets.nix, acme.nix, autoupgrade.nix
+  - Core modules: nix.nix, sshd.nix, vault-secrets.nix, acme.nix, autoupgrade.nix
+  - Additional modules: motd.nix (dynamic MOTD), packages.nix (base packages), root-user.nix (root config), homelab-deploy.nix (NATS listener)
   - Monitoring: node-exporter and promtail on every host
 - `/modules/` - Custom NixOS modules
-  - `homelab/` - Homelab-specific options (DNS automation, monitoring scrape targets)
+  - `homelab/` - Homelab-specific options (see "Homelab Module Options" section below)
 - `/lib/` - Nix library functions
   - `dns-zone.nix` - DNS zone generation functions
   - `monitoring.nix` - Prometheus scrape target generation functions
@@ -259,14 +310,14 @@ The `current_rev` label contains the git commit hash of the deployed flake confi
   - `home-assistant/` - Home automation stack
   - `monitoring/` - Observability stack (Prometheus, Grafana, Loki, Tempo)
   - `ns/` - DNS services (authoritative, resolver, zone generation)
-  - `http-proxy/`, `ca/`, `postgres/`, `nats/`, `jellyfin/`, etc.
-- `/secrets/` - SOPS-encrypted secrets with age encryption (legacy, only used by ca)
+  - `vault/` - OpenBao (Vault) secrets server
+  - `actions-runner/` - GitHub Actions runner
+  - `http-proxy/`, `postgres/`, `nats/`, `jellyfin/`, etc.
 - `/common/` - Shared configurations (e.g., VM guest agent)
 - `/docs/` - Documentation and plans
   - `plans/` - Future plans and proposals
   - `plans/completed/` - Completed plans (moved here when done)
 - `/playbooks/` - Ansible playbooks for fleet management
-- `/.sops.yaml` - SOPS configuration with age keys (legacy, only used by ca)
 
 ### Configuration Inheritance
 
@@ -283,7 +334,7 @@ All hosts automatically get:
 - Nix binary cache (nix-cache.home.2rjus.net)
 - SSH with root login enabled
 - OpenBao (Vault) secrets management via AppRole
-- Internal ACME CA integration (ca.home.2rjus.net)
+- Internal ACME CA integration (OpenBao PKI at vault.home.2rjus.net)
 - Daily auto-upgrades with auto-reboot
 - Prometheus node-exporter + Promtail (logs to monitoring01)
 - Monitoring scrape target auto-registration via `homelab.monitoring` options
@@ -292,28 +343,31 @@ All hosts automatically get:
 
 ### Active Hosts
 
-Production servers managed by `rebuild-all.sh`:
+Production servers:
 - `ns1`, `ns2` - Primary/secondary DNS servers (10.69.13.5/6)
-- `ca` - Internal Certificate Authority
+- `vault01` - OpenBao (Vault) secrets server + PKI CA
 - `ha1` - Home Assistant + Zigbee2MQTT + Mosquitto
 - `http-proxy` - Reverse proxy
 - `monitoring01` - Full observability stack (Prometheus, Grafana, Loki, Tempo, Pyroscope)
 - `jelly01` - Jellyfin media server
-- `nix-cache01` - Binary cache server
+- `nix-cache01` - Binary cache server + GitHub Actions runner
 - `pgdb1` - PostgreSQL database
 - `nats1` - NATS messaging server
 
-Template/test hosts:
-- `template1` - Base template for cloning new hosts
+Test/staging hosts:
+- `testvm01`, `testvm02`, `testvm03` - Test-tier VMs for branch testing and deployment validation
+
+Template hosts:
+- `template1`, `template2` - Base templates for cloning new hosts
 
 ### Flake Inputs
 
 - `nixpkgs` - NixOS 25.11 stable (primary)
 - `nixpkgs-unstable` - Unstable channel (available via overlay as `pkgs.unstable.<package>`)
-- `sops-nix` - Secrets management (legacy, only used by ca)
+- `nixos-exporter` - NixOS module for exposing flake revision metrics (used to verify deployments)
+- `homelab-deploy` - NATS-based remote deployment tool for test-tier hosts
 - Custom packages from git.t-juice.club:
   - `alerttonotify` - Alert routing
-  - `labmon` - Lab monitoring
 
 ### Network Architecture
 
@@ -337,11 +391,6 @@ Most hosts use OpenBao (Vault) for secrets:
 - Fallback to cached secrets in `/var/lib/vault/cache/` when Vault is unreachable
 - Provision AppRole credentials: `nix develop -c ansible-playbook playbooks/provision-approle.yml -e hostname=<host>`
 
-Legacy SOPS (only used by `ca` host):
-- SOPS with age encryption, keys in `.sops.yaml`
-- Shared secrets: `/secrets/secrets.yaml`
-- Per-host secrets: `/secrets/<hostname>/`
-
 ### Auto-Upgrade System
 
 All hosts pull updates daily from:
@@ -402,9 +451,21 @@ Example VM deployment includes:
 - Custom CPU/memory/disk sizing
 - VLAN tagging
 - QEMU guest agent
+- Automatic Vault credential provisioning via `vault_wrapped_token`
 
 OpenTofu outputs the VM's IP address after deployment for easy SSH access.
 
+**Automatic Vault Credential Provisioning:**
+
+VMs can receive Vault (OpenBao) credentials automatically during bootstrap:
+
+1. OpenTofu generates a wrapped token via `terraform/vault/` and stores it in the VM configuration
+2. Cloud-init passes `VAULT_WRAPPED_TOKEN` and `NIXOS_FLAKE_BRANCH` to the bootstrap script
+3. The bootstrap script unwraps the token to obtain AppRole credentials
+4. Credentials are written to `/var/lib/vault/approle/` before the NixOS rebuild
+
+This eliminates the need for manual `provision-approle.yml` playbook runs on new VMs. Bootstrap progress is logged to Loki with `job="bootstrap"` labels.
+
 #### Template Rebuilding and Terraform State
 
 When the Proxmox template is rebuilt (via `build-and-deploy-template.yml`), the template name may change. This would normally cause Terraform to want to recreate all existing VMs, but that's unnecessary since VMs are independent once cloned.
@@ -484,11 +545,7 @@ Prometheus scrape targets are automatically generated from host configurations,
 - **External targets**: Non-flake hosts defined in `/services/monitoring/external-targets.nix`
 - **Library**: `lib/monitoring.nix` provides `generateNodeExporterTargets` and `generateScrapeConfigs`
 
-Host monitoring options (`homelab.monitoring.*`):
-- `enable` (default: `true`) - Include host in Prometheus node-exporter scrape targets
-- `scrapeTargets` (default: `[]`) - Additional scrape targets exposed by this host (job_name, port, metrics_path, scheme, scrape_interval, honor_labels)
-
-Service modules declare their scrape targets directly (e.g., `services/ca/default.nix` declares step-ca on port 9000). The Prometheus config on monitoring01 auto-generates scrape configs from all hosts.
+Service modules declare their scrape targets directly via `homelab.monitoring.scrapeTargets`. The Prometheus config on monitoring01 auto-generates scrape configs from all hosts. See "Homelab Module Options" section for available options.
 
 To add monitoring targets for non-NixOS hosts, edit `/services/monitoring/external-targets.nix`.
 
@@ -507,13 +564,30 @@ DNS zone entries are automatically generated from host configurations:
 - **External hosts**: Non-flake hosts defined in `/services/ns/external-hosts.nix`
 - **Serial number**: Uses `self.sourceInfo.lastModified` (git commit timestamp)
 
-Host DNS options (`homelab.dns.*`):
-- `enable` (default: `true`) - Include host in DNS zone generation
-- `cnames` (default: `[]`) - List of CNAME aliases pointing to this host
-
 Hosts are automatically excluded from DNS if:
 - `homelab.dns.enable = false` (e.g., template hosts)
 - No static IP configured (e.g., DHCP-only hosts)
 - Network interface is a VPN/tunnel (wg*, tun*, tap*)
 
 To add DNS entries for non-NixOS hosts, edit `/services/ns/external-hosts.nix`.
+
+### Homelab Module Options
+
+The `modules/homelab/` directory defines custom options used across hosts for automation and metadata.
+
+**Host options (`homelab.host.*`):**
+- `tier` - Deployment tier: `test` or `prod`. Test-tier hosts can receive remote deployments and have different credential access.
+- `priority` - Alerting priority: `high` or `low`. Controls alerting thresholds for the host.
+- `role` - Primary role designation (e.g., `dns`, `database`, `bastion`, `vault`)
+- `labels` - Free-form key-value metadata for host categorization
+
+**DNS options (`homelab.dns.*`):**
+- `enable` (default: `true`) - Include host in DNS zone generation
+- `cnames` (default: `[]`) - List of CNAME aliases pointing to this host
+
+**Monitoring options (`homelab.monitoring.*`):**
+- `enable` (default: `true`) - Include host in Prometheus node-exporter scrape targets
+- `scrapeTargets` (default: `[]`) - Additional scrape targets exposed by this host
+
+**Deploy options (`homelab.deploy.*`):**
+- `enable` (default: `false`) - Enable NATS-based remote deployment listener. When enabled, the host listens for deployment commands via NATS and can be targeted by the `homelab-deploy` MCP server.

docs/plans/bootstrap-cache.md

@@ -0,0 +1,35 @@
+# Plan: Configure Template2 to Use Nix Cache
+
+## Problem
+
+New VMs bootstrapped from template2 don't use our local nix cache (nix-cache.home.2rjus.net) during the initial `nixos-rebuild boot`. This means the first build downloads everything from cache.nixos.org, which is slower and uses more bandwidth.
+
+## Solution
+
+Update the template2 base image to include the nix cache configuration, so new VMs immediately benefit from cached builds during bootstrap.
+
+## Implementation
+
+1. Add nix cache configuration to `hosts/template2/configuration.nix`:
+   ```nix
+   nix.settings = {
+     substituters = [ "https://nix-cache.home.2rjus.net" "https://cache.nixos.org" ];
+     trusted-public-keys = [
+       "nix-cache.home.2rjus.net:..."  # Add the cache's public key
+       "cache.nixos.org-1:..."
+     ];
+   };
+   ```
+
+2. Rebuild and redeploy the Proxmox template:
+   ```bash
+   nix develop -c ansible-playbook -i playbooks/inventory.ini playbooks/build-and-deploy-template.yml
+   ```
+
+3. Update `default_template_name` in `terraform/variables.tf` if the template name changed
+
+## Benefits
+
+- Faster VM bootstrap times
+- Reduced bandwidth to external cache
+- Most derivations will already be cached from other hosts

docs/plans/cert-monitoring.md

@@ -0,0 +1,72 @@
+# Certificate Monitoring Plan
+
+## Summary
+
+This document describes the removal of labmon certificate monitoring and outlines future needs for certificate monitoring in the homelab.
+
+## What Was Removed
+
+### labmon Service
+
+The `labmon` service was a custom Go application that provided:
+
+1. **StepMonitor**: Monitoring for step-ca (Smallstep CA) certificate provisioning and health
+2. **TLSConnectionMonitor**: Periodic TLS connection checks to verify certificate validity and expiration
+
+The service exposed Prometheus metrics at `:9969` including:
+- `labmon_tlsconmon_certificate_seconds_left` - Time until certificate expiration
+- `labmon_tlsconmon_certificate_check_error` - Whether the TLS check failed
+- `labmon_stepmon_certificate_seconds_left` - Step-CA internal certificate expiration
+
+### Affected Files
+
+- `hosts/monitoring01/configuration.nix` - Removed labmon configuration block
+- `services/monitoring/prometheus.nix` - Removed labmon scrape target
+- `services/monitoring/rules.yml` - Removed `certificate_rules` alert group
+- `services/monitoring/alloy.nix` - Deleted (was only used for labmon profiling)
+- `services/monitoring/default.nix` - Removed alloy.nix import
+
+### Removed Alerts
+
+- `certificate_expiring_soon` - Warned when any monitored TLS cert had < 24h validity
+- `step_ca_serving_cert_expiring` - Critical alert for step-ca's own serving certificate
+- `certificate_check_error` - Warned when TLS connection check failed
+- `step_ca_certificate_expiring` - Critical alert for step-ca issued certificates
+
+## Why It Was Removed
+
+1. **step-ca decommissioned**: The primary monitoring target (step-ca) is no longer in use
+2. **Outdated codebase**: labmon was a custom tool that required maintenance
+3. **Limited value**: With ACME auto-renewal, certificates should renew automatically
+
+## Current State
+
+ACME certificates are now issued by OpenBao PKI at `vault.home.2rjus.net:8200`. The ACME protocol handles automatic renewal, and certificates are typically renewed well before expiration.
+
+## Future Needs
+
+While ACME handles renewal automatically, we should consider monitoring for:
+
+1. **ACME renewal failures**: Alert when a certificate fails to renew
+   - Could monitor ACME client logs (via Loki queries)
+   - Could check certificate file modification times
+
+2. **Certificate expiration as backup**: Even with auto-renewal, a last-resort alert for certificates approaching expiration would catch renewal failures
+
+3. **Certificate transparency**: Monitor for unexpected certificate issuance
+
+### Potential Solutions
+
+1. **Prometheus blackbox_exporter**: Can probe TLS endpoints and export certificate expiration metrics
+   - `probe_ssl_earliest_cert_expiry` metric
+   - Already a standard tool, well-maintained
+
+2. **Custom Loki alerting**: Query ACME service logs for renewal failures
+   - Works with existing infrastructure
+   - No additional services needed
+
+3. **Node-exporter textfile collector**: Script that checks local certificate files and writes expiration metrics
+
+## Status
+
+**Not yet implemented.** This document serves as a placeholder for future work on certificate monitoring.

docs/plans/completed/prometheus-scrape-target-labels.md

@@ -1,10 +1,38 @@
 # Prometheus Scrape Target Labels
 
+## Implementation Status
+
+| Step | Status | Notes |
+|------|--------|-------|
+| 1. Create `homelab.host` module | ✅ Complete | `modules/homelab/host.nix` |
+| 2. Update `lib/monitoring.nix` | ✅ Complete | Labels extracted and propagated |
+| 3. Update Prometheus config | ✅ Complete | Uses structured static_configs |
+| 4. Set metadata on hosts | ✅ Complete | All relevant hosts configured |
+| 5. Update alert rules | ✅ Complete | Role-based filtering implemented |
+| 6. Labels for service targets | ✅ Complete | Host labels propagated to all services |
+| 7. Add hostname label | ✅ Complete | All targets have `hostname` label for easy filtering |
+
+**Hosts with metadata configured:**
+- `ns1`, `ns2`: `role = "dns"`, `labels.dns_role = "primary"/"secondary"`
+- `nix-cache01`: `role = "build-host"`
+- `vault01`: `role = "vault"`
+- `testvm01/02/03`: `tier = "test"`
+
+**Implementation complete.** Branch: `prometheus-scrape-target-labels`
+
+**Query examples:**
+- `{hostname="ns1"}` - all metrics from ns1 (any job/port)
+- `node_cpu_seconds_total{hostname="monitoring01"}` - specific metric by hostname
+- `up{role="dns"}` - all DNS servers
+- `up{tier="test"}` - all test-tier hosts
+
+---
+
 ## Goal
 
 Add support for custom per-host labels on Prometheus scrape targets, enabling alert rules to reference host metadata (priority, role) instead of hardcoding instance names.
 
-**Related:** This plan shares the `homelab.host` module with `docs/plans/nats-deploy-service.md`, which uses the same metadata for deployment tier assignment.
+**Related:** This plan shares the `homelab.host` module with `docs/plans/completed/nats-deploy-service.md`, which uses the same metadata for deployment tier assignment.
 
 ## Motivation
 
@@ -54,12 +82,11 @@ or
 
 ## Implementation
 
-This implementation uses a shared `homelab.host` module that provides host metadata for multiple consumers (Prometheus labels, deployment tiers, etc.). See also `docs/plans/nats-deploy-service.md` which uses the same module for deployment tier assignment.
+This implementation uses a shared `homelab.host` module that provides host metadata for multiple consumers (Prometheus labels, deployment tiers, etc.). See also `docs/plans/completed/nats-deploy-service.md` which uses the same module for deployment tier assignment.
 
 ### 1. Create `homelab.host` module
 
-**Status:** Step 1 (Create `homelab.host` module) is complete. The module is in
-`modules/homelab/host.nix` with tier, priority, role, and labels options.
+✅ **Complete.** The module is in `modules/homelab/host.nix`.
 
 Create `modules/homelab/host.nix` with shared host metadata options:
 
@@ -98,6 +125,8 @@ Import this module in `modules/homelab/default.nix`.
 
 ### 2. Update `lib/monitoring.nix`
 
+✅ **Complete.** Labels are now extracted and propagated.
+
 - `extractHostMonitoring` should also extract `homelab.host` values (priority, role, labels).
 - Build the combined label set from `homelab.host`:
 
@@ -126,6 +155,8 @@ This requires grouping hosts by their label attrset and producing one `static_co
 
 ### 3. Update `services/monitoring/prometheus.nix`
 
+✅ **Complete.** Now uses structured static_configs output.
+
 Change the node-exporter scrape config to use the new structured output:
 
 ```nix
@@ -138,36 +169,37 @@ static_configs = nodeExporterTargets;
 
 ### 4. Set metadata on hosts
 
+✅ **Complete.** All relevant hosts have metadata configured. Note: The implementation filters by `role` rather than `priority`, which matches the existing nix-cache01 configuration.
+
 Example in `hosts/nix-cache01/configuration.nix`:
 
 ```nix
 homelab.host = {
-  tier = "test";       # can be deployed by MCP (used by homelab-deploy)
   priority = "low";    # relaxed alerting thresholds
   role = "build-host";
 };
 ```
 
+**Note:** Current implementation only sets `role = "build-host"`. Consider adding `priority = "low"` when label propagation is implemented.
+
 Example in `hosts/ns1/configuration.nix`:
 
 ```nix
 homelab.host = {
-  tier = "prod";
-  priority = "high";
   role = "dns";
   labels.dns_role = "primary";
 };
 ```
 
+**Note:** `tier` and `priority` use defaults ("prod" and "high"), which is the intended behavior. The current ns1/ns2 configurations match this pattern.
+
 ### 5. Update alert rules
 
-After implementing labels, review and update `services/monitoring/rules.yml`:
+✅ **Complete.** Updated `services/monitoring/rules.yml`:
 
-- Replace instance-name exclusions with label-based filters (e.g. `{priority!="low"}` instead of `{instance!="nix-cache01.home.2rjus.net:9100"}`).
-- Consider whether any other rules should differentiate by priority or role.
+- `high_cpu_load`: Replaced `instance!="nix-cache01..."` with `role!="build-host"` for standard hosts (15m duration) and `role="build-host"` for build hosts (2h duration).
+- `unbound_low_cache_hit_ratio`: Added `dns_role="primary"` filter to only alert on the primary DNS resolver (secondary has a cold cache).
 
-Specifically, the `high_cpu_load` rule currently has a nix-cache01 exclusion that should be replaced with a `priority`-based filter.
+### 6. Labels for `generateScrapeConfigs` (service targets)
 
-### 6. Consider labels for `generateScrapeConfigs` (service targets)
-
-The same label propagation could be applied to service-level scrape targets. This is optional and can be deferred -- service targets are more specialized and less likely to need generic label-based filtering.
+✅ **Complete.** Host labels are now propagated to all auto-generated service scrape targets (unbound, homelab-deploy, nixos-exporter, etc.). This enables semantic filtering on any service metric, such as using `dns_role="primary"` with the unbound job.

docs/plans/host-migration-to-opentofu.md

@@ -9,24 +9,23 @@ hosts are decommissioned or deferred.
 
 ## Current State
 
-Hosts already managed by OpenTofu: `vault01`, `testvm01`, `vaulttest01`
+Hosts already managed by OpenTofu: `vault01`, `testvm01`, `testvm02`, `testvm03`, `ns2`
 
 Hosts to migrate:
 
 | Host | Category | Notes |
 |------|----------|-------|
 | ns1 | Stateless | Primary DNS, recreate |
-| ns2 | Stateless | Secondary DNS, recreate |
 | nix-cache01 | Stateless | Binary cache, recreate |
 | http-proxy | Stateless | Reverse proxy, recreate |
 | nats1 | Stateless | Messaging, recreate |
-| auth01 | Decommission | No longer in use |
 | ha1 | Stateful | Home Assistant + Zigbee2MQTT + Mosquitto |
 | monitoring01 | Stateful | Prometheus, Grafana, Loki |
 | jelly01 | Stateful | Jellyfin metadata, watch history, config |
-| pgdb1 | Stateful | PostgreSQL databases |
-| jump | Decommission | No longer needed |
-| ca | Deferred | Pending Phase 4c PKI migration to OpenBao |
+| pgdb1 | Decommission | Only used by Open WebUI on gunter, migrating to local postgres |
+| ~~jump~~ | ~~Decommission~~ | ✓ Complete |
+| ~~auth01~~ | ~~Decommission~~ | ✓ Complete |
+| ~~ca~~ | ~~Deferred~~ | ✓ Complete |
 
 ## Phase 1: Backup Preparation
 
@@ -46,39 +45,19 @@ No backup currently exists. Add a restic backup job for `/var/lib/jellyfin/` whi
 Media files are on the NAS (`nas.home.2rjus.net:/mnt/hdd-pool/media`) and do not need backup.
 The cache directory (`/var/cache/jellyfin/`) does not need backup — it regenerates.
 
-### 1c. Add PostgreSQL Backup to pgdb1
-
-No backup currently exists. Add a restic backup job with a `pg_dumpall` pre-hook to capture
-all databases and roles. The dump should be piped through restic's stdin backup (similar to
-the Grafana DB dump pattern on monitoring01).
-
-### 1d. Verify Existing ha1 Backup
+### 1c. Verify Existing ha1 Backup
 
 ha1 already backs up `/var/lib/hass`, `/var/lib/zigbee2mqtt`, `/var/lib/mosquitto`. Verify
 these backups are current and restorable before proceeding with migration.
 
-### 1e. Verify All Backups
+### 1d. Verify All Backups
 
 After adding/expanding backup jobs:
 1. Trigger a manual backup run on each host
 2. Verify backup integrity with `restic check`
 3. Test a restore to a temporary location to confirm data is recoverable
 
-## Phase 2: Declare pgdb1 Databases in Nix
-
-Before migrating pgdb1, audit the manually-created databases and users on the running
-instance, then declare them in the Nix configuration using `ensureDatabases` and
-`ensureUsers`. This makes the PostgreSQL setup reproducible on the new host.
-
-Steps:
-1. SSH to pgdb1, run `\l` and `\du` in psql to list databases and roles
-2. Add `ensureDatabases` and `ensureUsers` to `services/postgres/postgres.nix`
-3. Document any non-default PostgreSQL settings or extensions per database
-
-After reprovisioning, the databases will be created by NixOS, and data restored from the
-`pg_dumpall` backup.
-
-## Phase 3: Stateless Host Migration
+## Phase 2: Stateless Host Migration
 
 These hosts have no meaningful state and can be recreated fresh. For each host:
 
@@ -95,13 +74,14 @@ Migrate stateless hosts in an order that minimizes disruption:
 
 1. **nix-cache01** — low risk, no downstream dependencies during migration
 2. **nats1** — low risk, verify no persistent JetStream streams first
-4. **http-proxy** — brief disruption to proxied services, migrate during low-traffic window
-5. **ns1, ns2** — migrate one at a time, verify DNS resolution between each
+3. **http-proxy** — brief disruption to proxied services, migrate during low-traffic window
+4. **ns1** — ns2 already migrated, verify AXFR works after ns1 migration
 
-For ns1/ns2: migrate ns2 first (secondary), verify AXFR works, then migrate ns1. All hosts
-use both ns1 and ns2 as resolvers, so one being down briefly is tolerable.
+~~For ns1/ns2: migrate ns2 first (secondary), verify AXFR works, then migrate ns1.~~ ns2
+migration complete. All hosts use both ns1 and ns2 as resolvers, so ns1 being down briefly
+during migration is tolerable.
 
-## Phase 4: Stateful Host Migration
+## Phase 3: Stateful Host Migration
 
 For each stateful host, the procedure is:
 
@@ -114,17 +94,7 @@ For each stateful host, the procedure is:
 7. Start services and verify functionality
 8. Decommission the old VM
 
-### 4a. pgdb1
-
-1. Run final `pg_dumpall` backup via restic
-2. Stop PostgreSQL on the old host
-3. Provision new pgdb1 via OpenTofu
-4. After bootstrap, NixOS creates the declared databases/users
-5. Restore data with `pg_restore` or `psql < dumpall.sql`
-6. Verify database connectivity from gunter (`10.69.30.105`)
-7. Decommission old VM
-
-### 4b. monitoring01
+### 3a. monitoring01
 
 1. Run final Grafana backup
 2. Provision new monitoring01 via OpenTofu
@@ -134,7 +104,7 @@ For each stateful host, the procedure is:
 6. Verify all scrape targets are being collected
 7. Decommission old VM
 
-### 4c. jelly01
+### 3b. jelly01
 
 1. Run final Jellyfin backup
 2. Provision new jelly01 via OpenTofu
@@ -143,7 +113,7 @@ For each stateful host, the procedure is:
 5. Start Jellyfin, verify watch history and library metadata are present
 6. Decommission old VM
 
-### 4d. ha1
+### 3c. ha1
 
 1. Verify latest restic backup is current
 2. Stop Home Assistant, Zigbee2MQTT, and Mosquitto on old host
@@ -167,47 +137,67 @@ OpenTofu/Proxmox. Verify the USB device ID on the hypervisor and add the appropr
 `usb` block to the VM definition in `terraform/vms.tf`. The USB device must be passed
 through before starting Zigbee2MQTT on the new host.
 
-## Phase 5: Decommission jump and auth01 Hosts
+## Phase 4: Decommission Hosts
 
-### jump
-1. Verify nothing depends on the jump host (no SSH proxy configs pointing to it, etc.)
-2. Remove host configuration from `hosts/jump/`
-3. Remove from `flake.nix`
-4. Remove any secrets in `secrets/jump/`
-5. Remove from `.sops.yaml`
+### jump ✓ COMPLETE
+
+~~1. Verify nothing depends on the jump host (no SSH proxy configs pointing to it, etc.)~~
+~~2. Remove host configuration from `hosts/jump/`~~
+~~3. Remove from `flake.nix`~~
+~~4. Remove any secrets in `secrets/jump/`~~
+~~5. Remove from `.sops.yaml`~~
+~~6. Destroy the VM in Proxmox~~
+~~7. Commit cleanup~~
+
+Host was already removed from flake.nix and VM destroyed. Configuration cleaned up in ba9f47f.
+
+### auth01 ✓ COMPLETE
+
+~~1. Remove host configuration from `hosts/auth01/`~~
+~~2. Remove from `flake.nix`~~
+~~3. Remove any secrets in `secrets/auth01/`~~
+~~4. Remove from `.sops.yaml`~~
+~~5. Remove `services/authelia/` and `services/lldap/` (only used by auth01)~~
+~~6. Destroy the VM in Proxmox~~
+~~7. Commit cleanup~~
+
+Host configuration, services, and VM already removed.
+
+### pgdb1
+
+Only consumer was Open WebUI on gunter, which is being migrated to use local PostgreSQL.
+
+1. Verify Open WebUI on gunter is using local PostgreSQL (not pgdb1)
+2. Remove host configuration from `hosts/pgdb1/`
+3. Remove `services/postgres/` (only used by pgdb1)
+4. Remove from `flake.nix`
+5. Remove Vault AppRole from `terraform/vault/approle.tf`
 6. Destroy the VM in Proxmox
 7. Commit cleanup
 
-### auth01
-1. Remove host configuration from `hosts/auth01/`
-2. Remove from `flake.nix`
-3. Remove any secrets in `secrets/auth01/`
-4. Remove from `.sops.yaml`
-5. Remove `services/authelia/` and `services/lldap/` (only used by auth01)
-6. Destroy the VM in Proxmox
-7. Commit cleanup
+## Phase 5: Decommission ca Host ✓ COMPLETE
 
-## Phase 6: Decommission ca Host (Deferred)
-
-Deferred until Phase 4c (PKI migration to OpenBao) is complete. Once all hosts use the
+~~Deferred until Phase 4c (PKI migration to OpenBao) is complete. Once all hosts use the
 OpenBao ACME endpoint for certificates, the step-ca host can be decommissioned following
-the same cleanup steps as the jump host.
+the same cleanup steps as the jump host.~~
 
-## Phase 7: Remove sops-nix
+PKI migration to OpenBao complete. Host configuration, `services/ca/`, and VM removed.
 
-Once `ca` is decommissioned (Phase 6), `sops-nix` is no longer used by any host. Remove
-all remnants:
-- `sops-nix` input from `flake.nix` and `flake.lock`
-- `sops-nix.nixosModules.sops` from all host module lists in `flake.nix`
-- `inherit sops-nix` from all specialArgs in `flake.nix`
-- `system/sops.nix` and its import in `system/default.nix`
-- `.sops.yaml`
-- `secrets/` directory
-- All `sops.secrets.*` declarations in `services/ca/`, `services/authelia/`, `services/lldap/`
-- Template scripts that generate age keys for sops (`hosts/template/scripts.nix`,
-  `hosts/template2/scripts.nix`)
+## Phase 6: Remove sops-nix ✓ COMPLETE
 
-See `docs/plans/completed/sops-to-openbao-migration.md` for full context.
+~~Once `ca` is decommissioned (Phase 6), `sops-nix` is no longer used by any host. Remove
+all remnants:~~
+~~- `sops-nix` input from `flake.nix` and `flake.lock`~~
+~~- `sops-nix.nixosModules.sops` from all host module lists in `flake.nix`~~
+~~- `inherit sops-nix` from all specialArgs in `flake.nix`~~
+~~- `system/sops.nix` and its import in `system/default.nix`~~
+~~- `.sops.yaml`~~
+~~- `secrets/` directory~~
+~~- All `sops.secrets.*` declarations in `services/ca/`, `services/authelia/`, `services/lldap/`~~
+~~- Template scripts that generate age keys for sops (`hosts/template/scripts.nix`,
+  `hosts/template2/scripts.nix`)~~
+
+All sops-nix remnants removed. See `docs/plans/completed/sops-to-openbao-migration.md` for context.
 
 ## Notes
 
@@ -216,7 +206,7 @@ See `docs/plans/completed/sops-to-openbao-migration.md` for full context.
 - The old VMs use IPs that the new VMs need, so the old VM must be shut down before
   the new one is provisioned (or use a temporary IP and swap after verification)
 - Stateful migrations should be done during low-usage windows
-- After all migrations are complete, the only hosts not in OpenTofu will be ca (deferred)
+- After all migrations are complete, all decommissioned hosts (jump, auth01, ca) have been removed
 - Since many hosts are being recreated, this is a good opportunity to establish consistent
   hostname naming conventions before provisioning the new VMs. Current naming is inconsistent
   (e.g. `ns1` vs `nix-cache01`, `ha1` vs `auth01`, `pgdb1` vs `http-proxy`). Decide on a

docs/plans/nix-cache-reprovision.md

@@ -0,0 +1,212 @@
+# Nix Cache Host Reprovision
+
+## Overview
+
+Reprovision `nix-cache01` using the OpenTofu workflow, and improve the build/cache system with:
+1. NATS-based remote build triggering (replacing the current bash script)
+2. Safer flake update workflow that validates builds before pushing to master
+
+## Current State
+
+### Host Configuration
+- `nix-cache01` at 10.69.13.15 serves the binary cache via Harmonia
+- Runs Gitea Actions runner for CI workflows
+- Has `homelab.deploy.enable = true` (already supports NATS-based deployment)
+- Uses a dedicated XFS volume at `/nix` for cache storage
+
+### Current Build System (`services/nix-cache/build-flakes.sh`)
+- Runs every 30 minutes via systemd timer
+- Clones/pulls two repos: `nixos-servers` and `nixos` (gunter)
+- Builds all hosts with `nixos-rebuild build` (no blacklist despite docs mentioning it)
+- Pushes success/failure metrics to pushgateway
+- Simple but has no filtering, no parallelism, no remote triggering
+
+### Current Flake Update Workflow (`.github/workflows/flake-update.yaml`)
+- Runs daily at midnight via cron
+- Runs `nix flake update --commit-lock-file`
+- Pushes directly to master
+- No build validation — can push broken inputs
+
+## Improvement 1: NATS-Based Remote Build Triggering
+
+### Design
+
+Extend the existing `homelab-deploy` tool to support a "build" command that triggers builds on the cache host. This reuses the NATS infrastructure already in place.
+
+| Approach | Pros | Cons |
+|----------|------|------|
+| Extend homelab-deploy | Reuses existing NATS auth, NKey handling, CLI | Adds scope to existing tool |
+| New nix-cache-tool | Clean separation | Duplicate NATS boilerplate, new credentials |
+| Gitea Actions webhook | No custom tooling | Less flexible, tied to Gitea |
+
+**Recommendation:** Extend `homelab-deploy` with a build subcommand. The tool already has NATS client code, authentication handling, and a listener module in NixOS.
+
+### Implementation
+
+1. Add new message type to homelab-deploy: `build.<host>` subject
+2. Listener on nix-cache01 subscribes to `build.>` wildcard
+3. On message receipt, builds the specified host and returns success/failure
+4. CLI command: `homelab-deploy build <hostname>` or `homelab-deploy build --all`
+
+### Benefits
+- Trigger rebuild for specific host to ensure it's cached
+- Could be called from CI after merging PRs
+- Reuses existing NATS infrastructure and auth
+- Progress/status could stream back via NATS reply
+
+## Improvement 2: Smarter Flake Update Workflow
+
+### Current Problems
+1. Updates can push breaking changes to master
+2. No visibility into what broke when it does
+3. Hosts that auto-update can pull broken configs
+
+### Proposed Workflow
+
+```
+┌─────────────────────────────────────────────────────────────────┐
+│                    Flake Update Workflow                         │
+├─────────────────────────────────────────────────────────────────┤
+│  1. nix flake update (on feature branch)                        │
+│  2. Build ALL hosts locally                                      │
+│  3. If all pass → fast-forward merge to master                  │
+│  4. If any fail → create PR with failure logs attached          │
+└─────────────────────────────────────────────────────────────────┘
+```
+
+### Implementation Options
+
+| Option | Description | Pros | Cons |
+|--------|-------------|------|------|
+| **A: Self-hosted runner** | Build on nix-cache01 | Fast (local cache), simple | Ties up cache host during build |
+| **B: Gitea Actions only** | Use container runner | Clean separation | Slow (no cache), resource limits |
+| **C: Hybrid** | Trigger builds on nix-cache01 via NATS from Actions | Best of both | More complex |
+
+**Recommendation:** Option A with nix-cache01 as the runner. The host is already running Gitea Actions runner and has the cache. Building all ~16 hosts is disk I/O heavy but feasible on dedicated hardware.
+
+### Workflow Steps
+
+1. Workflow runs on schedule (daily or weekly)
+2. Creates branch `flake-update/YYYY-MM-DD`
+3. Runs `nix flake update --commit-lock-file`
+4. Builds each host: `nix build .#nixosConfigurations.<host>.config.system.build.toplevel`
+5. If all succeed:
+   - Fast-forward merge to master
+   - Delete feature branch
+6. If any fail:
+   - Create PR from the update branch
+   - Attach build logs as PR comment
+   - Label PR with `needs-review` or `build-failure`
+   - Do NOT merge automatically
+
+### Workflow File Changes
+
+```yaml
+# New: .github/workflows/flake-update-safe.yaml
+name: Safe flake update
+on:
+  schedule:
+    - cron: "0 2 * * 0"  # Weekly on Sunday at 2 AM
+  workflow_dispatch:  # Manual trigger
+
+jobs:
+  update-and-validate:
+    runs-on: homelab  # Use self-hosted runner on nix-cache01
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: master
+          fetch-depth: 0  # Need full history for merge
+
+      - name: Create update branch
+        run: |
+          BRANCH="flake-update/$(date +%Y-%m-%d)"
+          git checkout -b "$BRANCH"
+
+      - name: Update flake
+        run: nix flake update --commit-lock-file
+
+      - name: Build all hosts
+        id: build
+        run: |
+          FAILED=""
+          for host in $(nix flake show --json | jq -r '.nixosConfigurations | keys[]'); do
+            echo "Building $host..."
+            if ! nix build ".#nixosConfigurations.$host.config.system.build.toplevel" 2>&1 | tee "build-$host.log"; then
+              FAILED="$FAILED $host"
+            fi
+          done
+          echo "failed=$FAILED" >> $GITHUB_OUTPUT
+
+      - name: Merge to master (if all pass)
+        if: steps.build.outputs.failed == ''
+        run: |
+          git checkout master
+          git merge --ff-only "$BRANCH"
+          git push origin master
+          git push origin --delete "$BRANCH"
+
+      - name: Create PR (if any fail)
+        if: steps.build.outputs.failed != ''
+        run: |
+          git push origin "$BRANCH"
+          # Create PR via Gitea API with build logs
+          # ... (PR creation with log attachment)
+```
+
+## Migration Steps
+
+### Phase 1: Reprovision Host via OpenTofu
+
+1. Add `nix-cache01` to `terraform/vms.tf`:
+   ```hcl
+   "nix-cache01" = {
+     ip        = "10.69.13.15/24"
+     cpu_cores = 4
+     memory    = 8192
+     disk_size = "100G"  # Larger for nix store
+   }
+   ```
+
+2. Shut down existing nix-cache01 VM
+3. Run `tofu apply` to provision new VM
+4. Verify bootstrap completes and cache is serving
+
+**Note:** The cache will be cold after reprovision. Run initial builds to populate.
+
+### Phase 2: Add Build Triggering to homelab-deploy
+
+1. Add `build` command to homelab-deploy CLI
+2. Add listener handler in NixOS module for `build.*` subjects
+3. Update nix-cache01 config to enable build listener
+4. Test with `homelab-deploy build testvm01`
+
+### Phase 3: Implement Safe Flake Update Workflow
+
+1. Create `.github/workflows/flake-update-safe.yaml`
+2. Disable or remove old `flake-update.yaml`
+3. Test manually with `workflow_dispatch`
+4. Monitor first automated run
+
+### Phase 4: Remove Old Build Script
+
+1. After new workflow is stable, remove:
+   - `services/nix-cache/build-flakes.nix`
+   - `services/nix-cache/build-flakes.sh`
+2. The new workflow handles scheduled builds
+
+## Open Questions
+
+- [ ] What runner labels should the self-hosted runner use for the update workflow?
+- [ ] Should we build hosts in parallel (faster) or sequentially (easier to debug)?
+- [ ] How long to keep flake-update PRs open before auto-closing stale ones?
+- [ ] Should successful updates trigger a NATS notification to rebuild all hosts?
+- [ ] What to do about `gunter` (external nixos repo) - include in validation?
+- [ ] Disk size for new nix-cache01 - is 100G enough for cache + builds?
+
+## Notes
+
+- The existing `homelab.deploy.enable = true` on nix-cache01 means it already has NATS connectivity
+- The Harmonia service and cache signing key will work the same after reprovision
+- Actions runner token is in Vault, will be provisioned automatically
+- Consider adding a `homelab.host.role = "build-host"` label for monitoring/filtering

flake.lock

@@ -28,11 +28,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1770447502,
-        "narHash": "sha256-xH1PNyE3ydj4udhe1IpK8VQxBPZETGLuORZdSWYRmSU=",
+        "lastModified": 1770481834,
+        "narHash": "sha256-Xx9BYnI0C/qgPbwr9nj6NoAdQTbYLunrdbNSaUww9oY=",
         "ref": "master",
-        "rev": "79db119d1ca6630023947ef0a65896cc3307c2ff",
-        "revCount": 22,
+        "rev": "fd0d63b103dfaf21d1c27363266590e723021c67",
+        "revCount": 24,
         "type": "git",
         "url": "https://git.t-juice.club/torjus/homelab-deploy"
       },
@@ -42,27 +42,6 @@
         "url": "https://git.t-juice.club/torjus/homelab-deploy"
       }
     },
-    "labmon": {
-      "inputs": {
-        "nixpkgs": [
-          "nixpkgs-unstable"
-        ]
-      },
-      "locked": {
-        "lastModified": 1748983975,
-        "narHash": "sha256-DA5mOqxwLMj/XLb4hvBU1WtE6cuVej7PjUr8N0EZsCE=",
-        "ref": "master",
-        "rev": "040a73e891a70ff06ec7ab31d7167914129dbf7d",
-        "revCount": 17,
-        "type": "git",
-        "url": "https://git.t-juice.club/torjus/labmon"
-      },
-      "original": {
-        "ref": "master",
-        "type": "git",
-        "url": "https://git.t-juice.club/torjus/labmon"
-      }
-    },
     "nixos-exporter": {
       "inputs": {
         "nixpkgs": [
@@ -119,31 +98,9 @@
       "inputs": {
         "alerttonotify": "alerttonotify",
         "homelab-deploy": "homelab-deploy",
-        "labmon": "labmon",
         "nixos-exporter": "nixos-exporter",
         "nixpkgs": "nixpkgs",
-        "nixpkgs-unstable": "nixpkgs-unstable",
-        "sops-nix": "sops-nix"
-      }
-    },
-    "sops-nix": {
-      "inputs": {
-        "nixpkgs": [
-          "nixpkgs-unstable"
-        ]
-      },
-      "locked": {
-        "lastModified": 1770145881,
-        "narHash": "sha256-ktjWTq+D5MTXQcL9N6cDZXUf9kX8JBLLBLT0ZyOTSYY=",
-        "owner": "Mic92",
-        "repo": "sops-nix",
-        "rev": "17eea6f3816ba6568b8c81db8a4e6ca438b30b7c",
-        "type": "github"
-      },
-      "original": {
-        "owner": "Mic92",
-        "repo": "sops-nix",
-        "type": "github"
+        "nixpkgs-unstable": "nixpkgs-unstable"
       }
     }
   },

flake.nix

@@ -5,18 +5,10 @@
     nixpkgs.url = "github:nixos/nixpkgs?ref=nixos-25.11";
     nixpkgs-unstable.url = "github:nixos/nixpkgs?ref=nixos-unstable";
 
-    sops-nix = {
-      url = "github:Mic92/sops-nix";
-      inputs.nixpkgs.follows = "nixpkgs-unstable";
-    };
     alerttonotify = {
       url = "git+https://git.t-juice.club/torjus/alerttonotify?ref=master";
       inputs.nixpkgs.follows = "nixpkgs-unstable";
     };
-    labmon = {
-      url = "git+https://git.t-juice.club/torjus/labmon?ref=master";
-      inputs.nixpkgs.follows = "nixpkgs-unstable";
-    };
     nixos-exporter = {
       url = "git+https://git.t-juice.club/torjus/nixos-exporter";
       inputs.nixpkgs.follows = "nixpkgs-unstable";
@@ -32,9 +24,7 @@
       self,
       nixpkgs,
       nixpkgs-unstable,
-      sops-nix,
       alerttonotify,
-      labmon,
       nixos-exporter,
       homelab-deploy,
       ...
@@ -50,7 +40,6 @@
       commonOverlays = [
         overlay-unstable
         alerttonotify.overlays.default
-        labmon.overlays.default
       ];
       # Common modules applied to all hosts
       commonModules = [
@@ -61,7 +50,6 @@
             system.configurationRevision = self.rev or self.dirtyRev or "dirty";
           }
         )
-        sops-nix.nixosModules.sops
         nixos-exporter.nixosModules.default
         homelab-deploy.nixosModules.default
         ./modules/homelab
@@ -80,43 +68,25 @@
         ns1 = nixpkgs.lib.nixosSystem {
           inherit system;
           specialArgs = {
-            inherit inputs self sops-nix;
+            inherit inputs self;
           };
           modules = commonModules ++ [
             ./hosts/ns1
           ];
         };
-        ns2 = nixpkgs.lib.nixosSystem {
-          inherit system;
-          specialArgs = {
-            inherit inputs self sops-nix;
-          };
-          modules = commonModules ++ [
-            ./hosts/ns2
-          ];
-        };
         ha1 = nixpkgs.lib.nixosSystem {
           inherit system;
           specialArgs = {
-            inherit inputs self sops-nix;
+            inherit inputs self;
           };
           modules = commonModules ++ [
             ./hosts/ha1
           ];
         };
-        template1 = nixpkgs.lib.nixosSystem {
-          inherit system;
-          specialArgs = {
-            inherit inputs self sops-nix;
-          };
-          modules = commonModules ++ [
-            ./hosts/template
-          ];
-        };
         template2 = nixpkgs.lib.nixosSystem {
           inherit system;
           specialArgs = {
-            inherit inputs self sops-nix;
+            inherit inputs self;
           };
           modules = commonModules ++ [
             ./hosts/template2
@@ -125,35 +95,25 @@
         http-proxy = nixpkgs.lib.nixosSystem {
           inherit system;
           specialArgs = {
-            inherit inputs self sops-nix;
+            inherit inputs self;
           };
           modules = commonModules ++ [
             ./hosts/http-proxy
           ];
         };
-        ca = nixpkgs.lib.nixosSystem {
-          inherit system;
-          specialArgs = {
-            inherit inputs self sops-nix;
-          };
-          modules = commonModules ++ [
-            ./hosts/ca
-          ];
-        };
         monitoring01 = nixpkgs.lib.nixosSystem {
           inherit system;
           specialArgs = {
-            inherit inputs self sops-nix;
+            inherit inputs self;
           };
           modules = commonModules ++ [
             ./hosts/monitoring01
-            labmon.nixosModules.labmon
           ];
         };
         jelly01 = nixpkgs.lib.nixosSystem {
           inherit system;
           specialArgs = {
-            inherit inputs self sops-nix;
+            inherit inputs self;
           };
           modules = commonModules ++ [
             ./hosts/jelly01
@@ -162,7 +122,7 @@
         nix-cache01 = nixpkgs.lib.nixosSystem {
           inherit system;
           specialArgs = {
-            inherit inputs self sops-nix;
+            inherit inputs self;
           };
           modules = commonModules ++ [
             ./hosts/nix-cache01
@@ -171,7 +131,7 @@
         pgdb1 = nixpkgs.lib.nixosSystem {
           inherit system;
           specialArgs = {
-            inherit inputs self sops-nix;
+            inherit inputs self;
           };
           modules = commonModules ++ [
             ./hosts/pgdb1
@@ -180,37 +140,55 @@
         nats1 = nixpkgs.lib.nixosSystem {
           inherit system;
           specialArgs = {
-            inherit inputs self sops-nix;
+            inherit inputs self;
           };
           modules = commonModules ++ [
             ./hosts/nats1
           ];
         };
-        testvm01 = nixpkgs.lib.nixosSystem {
-          inherit system;
-          specialArgs = {
-            inherit inputs self sops-nix;
-          };
-          modules = commonModules ++ [
-            ./hosts/testvm01
-          ];
-        };
         vault01 = nixpkgs.lib.nixosSystem {
           inherit system;
           specialArgs = {
-            inherit inputs self sops-nix;
+            inherit inputs self;
           };
           modules = commonModules ++ [
             ./hosts/vault01
           ];
         };
-        vaulttest01 = nixpkgs.lib.nixosSystem {
+        testvm01 = nixpkgs.lib.nixosSystem {
           inherit system;
           specialArgs = {
-            inherit inputs self sops-nix;
+            inherit inputs self;
           };
           modules = commonModules ++ [
-            ./hosts/vaulttest01
+            ./hosts/testvm01
+          ];
+        };
+        testvm02 = nixpkgs.lib.nixosSystem {
+          inherit system;
+          specialArgs = {
+            inherit inputs self;
+          };
+          modules = commonModules ++ [
+            ./hosts/testvm02
+          ];
+        };
+        testvm03 = nixpkgs.lib.nixosSystem {
+          inherit system;
+          specialArgs = {
+            inherit inputs self;
+          };
+          modules = commonModules ++ [
+            ./hosts/testvm03
+          ];
+        };
+        ns2 = nixpkgs.lib.nixosSystem {
+          inherit system;
+          specialArgs = {
+            inherit inputs self;
+          };
+          modules = commonModules ++ [
+            ./hosts/ns2
           ];
         };
       };

hosts/ca/default.nix

@@ -1,7 +0,0 @@
-{ ... }:
-{
-  imports = [
-    ./configuration.nix
-    ../../services/ca
-  ];
-}

hosts/ha1/configuration.nix

@@ -7,7 +7,7 @@
 
 {
   imports = [
-    ../template/hardware-configuration.nix
+    ./hardware-configuration.nix
 
     ../../system
     ../../common/vm

hosts/http-proxy/configuration.nix

@@ -5,7 +5,7 @@
 
 {
   imports = [
-    ../template/hardware-configuration.nix
+    ./hardware-configuration.nix
 
     ../../system
     ../../common/vm

hosts/http-proxy/hardware-configuration.nix

@@ -0,0 +1,42 @@
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/profiles/qemu-guest.nix")
+  ];
+  boot.initrd.availableKernelModules = [
+    "ata_piix"
+    "uhci_hcd"
+    "virtio_pci"
+    "virtio_scsi"
+    "sd_mod"
+    "sr_mod"
+  ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+  boot.kernelModules = [
+    "ptp_kvm"
+  ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" = {
+    device = "/dev/disk/by-label/root";
+    fsType = "xfs";
+  };
+
+  swapDevices = [ { device = "/dev/disk/by-label/swap"; } ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.ens18.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

hosts/jelly01/configuration.nix

@@ -5,7 +5,7 @@
 
 {
   imports = [
-    ../template/hardware-configuration.nix
+    ./hardware-configuration.nix
 
     ../../system
     ../../common/vm
@@ -61,6 +61,9 @@
   # Or disable the firewall altogether.
   networking.firewall.enable = false;
 
+  vault.enable = true;
+  homelab.deploy.enable = true;
+
   zramSwap = {
     enable = true;
   };

hosts/jelly01/hardware-configuration.nix

@@ -0,0 +1,42 @@
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/profiles/qemu-guest.nix")
+  ];
+  boot.initrd.availableKernelModules = [
+    "ata_piix"
+    "uhci_hcd"
+    "virtio_pci"
+    "virtio_scsi"
+    "sd_mod"
+    "sr_mod"
+  ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+  boot.kernelModules = [
+    "ptp_kvm"
+  ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" = {
+    device = "/dev/disk/by-label/root";
+    fsType = "xfs";
+  };
+
+  swapDevices = [ { device = "/dev/disk/by-label/swap"; } ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.ens18.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

hosts/jump/configuration.nix

@@ -1,56 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-{
-  imports =
-    [
-      ../template/hardware-configuration.nix
-      ../../system
-    ];
-
-  nixpkgs.config.allowUnfree = true;
-
-  homelab.host.role = "bastion";
-
-  # Use the systemd-boot EFI boot loader.
-  boot.loader.grub.enable = true;
-  boot.loader.grub.device = "/dev/sda";
-
-  networking.hostName = "jump";
-  networking.domain = "home.2rjus.net";
-  networking.useNetworkd = true;
-  networking.useDHCP = false;
-  services.resolved.enable = false;
-  networking.nameservers = [
-    "10.69.13.5"
-    "10.69.13.6"
-  ];
-
-  systemd.network.enable = true;
-  systemd.network.networks."ens18" = {
-    matchConfig.Name = "ens18";
-    address = [
-      "10.69.13.10/24"
-    ];
-    routes = [
-      { Gateway = "10.69.13.1"; }
-    ];
-    linkConfig.RequiredForOnline = "routable";
-  };
-  time.timeZone = "Europe/Oslo";
-
-  nix.settings.experimental-features = [ "nix-command" "flakes" ];
-  environment.systemPackages = with pkgs; [
-    vim
-    wget
-    git
-  ];
-
-  # Open ports in the firewall.
-  # networking.firewall.allowedTCPPorts = [ ... ];
-  # networking.firewall.allowedUDPPorts = [ ... ];
-  # Or disable the firewall altogether.
-  networking.firewall.enable = false;
-
-  system.stateVersion = "23.11"; # Did you read the comment?
-}
-

hosts/jump/hardware-configuration.nix

@@ -1,36 +0,0 @@
-{ config, lib, pkgs, modulesPath, ... }:
-
-{
-  imports =
-    [
-      (modulesPath + "/profiles/qemu-guest.nix")
-    ];
-
-  boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "virtio_pci" "virtio_scsi" "sd_mod" "sr_mod" ];
-  boot.initrd.kernelModules = [ ];
-  # boot.kernelModules = [ ];
-  # boot.extraModulePackages = [ ];
-
-  fileSystems."/" =
-    {
-      device = "/dev/disk/by-uuid/6889aba9-61ed-4687-ab10-e5cf4017ac8d";
-      fsType = "xfs";
-    };
-
-  fileSystems."/boot" =
-    {
-      device = "/dev/disk/by-uuid/BC07-3B7A";
-      fsType = "vfat";
-    };
-
-  swapDevices =
-    [{ device = "/dev/disk/by-uuid/64e5757b-6625-4dd2-aa2a-66ca93444d23"; }];
-
-  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
-  # (the default) this is the recommended approach. When using systemd-networkd it's
-  # still possible to use this option, but it's recommended to use it in conjunction
-  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
-  # networking.interfaces.ens18.useDHCP = lib.mkDefault true;
-
-  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
-}

hosts/monitoring01/configuration.nix

@@ -5,7 +5,7 @@
 
 {
   imports = [
-    ../template/hardware-configuration.nix
+    ./hardware-configuration.nix
 
     ../../system
     ../../common/vm
@@ -100,61 +100,6 @@
     ];
   };
 
-  labmon = {
-    enable = true;
-
-    settings = {
-      ListenAddr = ":9969";
-      Profiling = true;
-      StepMonitors = [
-        {
-          Enabled = true;
-          BaseURL = "https://ca.home.2rjus.net";
-          RootID = "3381bda8015a86b9a3cd1851439d1091890a79005e0f1f7c4301fe4bccc29d80";
-        }
-      ];
-
-      TLSConnectionMonitors = [
-        {
-          Enabled = true;
-          Address = "ca.home.2rjus.net:443";
-          Verify = true;
-          Duration = "12h";
-        }
-        {
-          Enabled = true;
-          Address = "jelly.home.2rjus.net:443";
-          Verify = true;
-          Duration = "12h";
-        }
-        {
-          Enabled = true;
-          Address = "grafana.home.2rjus.net:443";
-          Verify = true;
-          Duration = "12h";
-        }
-        {
-          Enabled = true;
-          Address = "prometheus.home.2rjus.net:443";
-          Verify = true;
-          Duration = "12h";
-        }
-        {
-          Enabled = true;
-          Address = "alertmanager.home.2rjus.net:443";
-          Verify = true;
-          Duration = "12h";
-        }
-        {
-          Enabled = true;
-          Address = "pyroscope.home.2rjus.net:443";
-          Verify = true;
-          Duration = "12h";
-        }
-      ];
-    };
-  };
-
   # Open ports in the firewall.
   # networking.firewall.allowedTCPPorts = [ ... ];
   # networking.firewall.allowedUDPPorts = [ ... ];

hosts/monitoring01/hardware-configuration.nix

@@ -0,0 +1,42 @@
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/profiles/qemu-guest.nix")
+  ];
+  boot.initrd.availableKernelModules = [
+    "ata_piix"
+    "uhci_hcd"
+    "virtio_pci"
+    "virtio_scsi"
+    "sd_mod"
+    "sr_mod"
+  ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+  boot.kernelModules = [
+    "ptp_kvm"
+  ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" = {
+    device = "/dev/disk/by-label/root";
+    fsType = "xfs";
+  };
+
+  swapDevices = [ { device = "/dev/disk/by-label/swap"; } ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.ens18.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

hosts/nats1/configuration.nix

@@ -5,7 +5,7 @@
 
 {
   imports = [
-    ../template/hardware-configuration.nix
+    ./hardware-configuration.nix
 
     ../../system
     ../../common/vm
@@ -59,5 +59,8 @@
   # Or disable the firewall altogether.
   networking.firewall.enable = false;
 
+  vault.enable = true;
+  homelab.deploy.enable = true;
+
   system.stateVersion = "23.11"; # Did you read the comment?
 }

hosts/nats1/hardware-configuration.nix

@@ -0,0 +1,42 @@
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/profiles/qemu-guest.nix")
+  ];
+  boot.initrd.availableKernelModules = [
+    "ata_piix"
+    "uhci_hcd"
+    "virtio_pci"
+    "virtio_scsi"
+    "sd_mod"
+    "sr_mod"
+  ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+  boot.kernelModules = [
+    "ptp_kvm"
+  ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" = {
+    device = "/dev/disk/by-label/root";
+    fsType = "xfs";
+  };
+
+  swapDevices = [ { device = "/dev/disk/by-label/swap"; } ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.ens18.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

hosts/nix-cache01/configuration.nix

@@ -5,7 +5,7 @@
 
 {
   imports = [
-    ../template/hardware-configuration.nix
+    ./hardware-configuration.nix
 
     ../../system
     ../../common/vm

hosts/nix-cache01/hardware-configuration.nix

@@ -0,0 +1,42 @@
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/profiles/qemu-guest.nix")
+  ];
+  boot.initrd.availableKernelModules = [
+    "ata_piix"
+    "uhci_hcd"
+    "virtio_pci"
+    "virtio_scsi"
+    "sd_mod"
+    "sr_mod"
+  ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+  boot.kernelModules = [
+    "ptp_kvm"
+  ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" = {
+    device = "/dev/disk/by-label/root";
+    fsType = "xfs";
+  };
+
+  swapDevices = [ { device = "/dev/disk/by-label/swap"; } ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.ens18.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

hosts/ns1/configuration.nix

@@ -7,7 +7,7 @@
 
 {
   imports = [
-    ../template/hardware-configuration.nix
+    ./hardware-configuration.nix
 
     ../../system
     ../../services/ns/master-authorative.nix

hosts/ns2/configuration.nix

@@ -7,23 +7,38 @@
 
 {
   imports = [
-    ../template/hardware-configuration.nix
+    ../template2/hardware-configuration.nix
 
     ../../system
+    ../../common/vm
+
+    # DNS services
     ../../services/ns/secondary-authorative.nix
     ../../services/ns/resolver.nix
-    ../../common/vm
   ];
 
+  # Host metadata
+  homelab.host = {
+    tier = "prod";
+    role = "dns";
+    labels.dns_role = "secondary";
+  };
+
+  # Enable Vault integration
+  vault.enable = true;
+
+  # Enable remote deployment via NATS
+  homelab.deploy.enable = true;
+
   nixpkgs.config.allowUnfree = true;
-  # Use the systemd-boot EFI boot loader.
   boot.loader.grub.enable = true;
-  boot.loader.grub.device = "/dev/sda";
+  boot.loader.grub.device = "/dev/vda";
 
   networking.hostName = "ns2";
   networking.domain = "home.2rjus.net";
   networking.useNetworkd = true;
   networking.useDHCP = false;
+  # Disable resolved - conflicts with Unbound resolver
   services.resolved.enable = false;
   networking.nameservers = [
     "10.69.13.5"
@@ -47,14 +62,7 @@
     "nix-command"
     "flakes"
   ];
-  vault.enable = true;
-  homelab.deploy.enable = true;
-
-  homelab.host = {
-    role = "dns";
-    labels.dns_role = "secondary";
-  };
-
+  nix.settings.tarball-ttl = 0;
   environment.systemPackages = with pkgs; [
     vim
     wget
@@ -67,5 +75,5 @@
   # Or disable the firewall altogether.
   networking.firewall.enable = false;
 
-  system.stateVersion = "23.11"; # Did you read the comment?
-}
+  system.stateVersion = "25.11"; # Did you read the comment?
+}

hosts/ns2/default.nix

@@ -2,4 +2,4 @@
   imports = [
     ./configuration.nix
   ];
-}
+}

hosts/ns2/hardware-configuration.nix

@@ -1,36 +0,0 @@
-{ config, lib, pkgs, modulesPath, ... }:
-
-{
-  imports =
-    [
-      (modulesPath + "/profiles/qemu-guest.nix")
-    ];
-
-  boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "virtio_pci" "virtio_scsi" "sd_mod" "sr_mod" ];
-  boot.initrd.kernelModules = [ ];
-  # boot.kernelModules = [ ];
-  # boot.extraModulePackages = [ ];
-
-  fileSystems."/" =
-    {
-      device = "/dev/disk/by-uuid/6889aba9-61ed-4687-ab10-e5cf4017ac8d";
-      fsType = "xfs";
-    };
-
-  fileSystems."/boot" =
-    {
-      device = "/dev/disk/by-uuid/BC07-3B7A";
-      fsType = "vfat";
-    };
-
-  swapDevices =
-    [{ device = "/dev/disk/by-uuid/64e5757b-6625-4dd2-aa2a-66ca93444d23"; }];
-
-  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
-  # (the default) this is the recommended approach. When using systemd-networkd it's
-  # still possible to use this option, but it's recommended to use it in conjunction
-  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
-  # networking.interfaces.ens18.useDHCP = lib.mkDefault true;
-
-  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
-}

hosts/pgdb1/configuration.nix

@@ -5,7 +5,7 @@
 
 {
   imports = [
-    ../template/hardware-configuration.nix
+    ./hardware-configuration.nix
 
     ../../system
     ../../common/vm
@@ -59,5 +59,8 @@
   # Or disable the firewall altogether.
   networking.firewall.enable = false;
 
+  vault.enable = true;
+  homelab.deploy.enable = true;
+
   system.stateVersion = "23.11"; # Did you read the comment?
 }

hosts/pgdb1/hardware-configuration.nix

@@ -0,0 +1,42 @@
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/profiles/qemu-guest.nix")
+  ];
+  boot.initrd.availableKernelModules = [
+    "ata_piix"
+    "uhci_hcd"
+    "virtio_pci"
+    "virtio_scsi"
+    "sd_mod"
+    "sr_mod"
+  ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+  boot.kernelModules = [
+    "ptp_kvm"
+  ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" = {
+    device = "/dev/disk/by-label/root";
+    fsType = "xfs";
+  };
+
+  swapDevices = [ { device = "/dev/disk/by-label/swap"; } ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.ens18.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

hosts/template/default.nix

@@ -1,7 +0,0 @@
-{ ... }: {
-  imports = [
-    ./hardware-configuration.nix
-    ./configuration.nix
-    ./scripts.nix
-  ];
-}

hosts/template/scripts.nix

@@ -1,36 +0,0 @@
-{ pkgs, ... }:
-let
-  prepare-host-script = pkgs.writeShellApplication {
-    name = "prepare-host.sh";
-    runtimeInputs = [ pkgs.age ];
-    text = ''
-      echo "Removing machine-id"
-      rm -f /etc/machine-id || true
-
-      echo "Removing SSH host keys"
-      rm -f /etc/ssh/ssh_host_* || true
-
-      echo "Restarting SSH"
-      systemctl restart sshd
-
-      echo "Removing temporary files"
-      rm -rf /tmp/* || true
-
-      echo "Removing logs"
-      journalctl --rotate || true
-      journalctl --vacuum-time=1s || true
-
-      echo "Removing cache"
-      rm -rf /var/cache/* || true
-
-      echo "Generate age key"
-      rm -rf /var/lib/sops-nix || true
-      mkdir -p /var/lib/sops-nix
-      age-keygen -o /var/lib/sops-nix/key.txt
-    '';
-  };
-in
-{
-  environment.systemPackages = [ prepare-host-script ];
-  users.motd = "Prepare host by running 'prepare-host.sh'.";
-}

hosts/template2/bootstrap.nix

@@ -6,22 +6,72 @@ let
     text = ''
       set -euo pipefail
 
+      LOKI_URL="http://monitoring01.home.2rjus.net:3100/loki/api/v1/push"
+
+      # Send a log entry to Loki with bootstrap status
+      # Usage: log_to_loki <stage> <message>
+      # Fails silently if Loki is unreachable
+      log_to_loki() {
+        local stage="$1"
+        local message="$2"
+        local timestamp_ns
+        timestamp_ns="$(date +%s)000000000"
+
+        local payload
+        payload=$(jq -n \
+          --arg host "$HOSTNAME" \
+          --arg stage "$stage" \
+          --arg branch "''${BRANCH:-master}" \
+          --arg ts "$timestamp_ns" \
+          --arg msg "$message" \
+          '{
+            streams: [{
+              stream: {
+                job: "bootstrap",
+                host: $host,
+                stage: $stage,
+                branch: $branch
+              },
+              values: [[$ts, $msg]]
+            }]
+          }')
+
+        curl -s --connect-timeout 2 --max-time 5 \
+          -X POST \
+          -H "Content-Type: application/json" \
+          -d "$payload" \
+          "$LOKI_URL" >/dev/null 2>&1 || true
+      }
+
+      echo "================================================================================"
+      echo "                     NIXOS BOOTSTRAP IN PROGRESS"
+      echo "================================================================================"
+      echo ""
+
       # Read hostname set by cloud-init (from Terraform VM name via user-data)
       # Cloud-init sets the system hostname from user-data.txt, so we read it from hostnamectl
       HOSTNAME=$(hostnamectl hostname)
-      echo "DEBUG: Hostname from hostnamectl: '$HOSTNAME'"
+      # Read git branch from environment, default to master
+      BRANCH="''${NIXOS_FLAKE_BRANCH:-master}"
 
+      echo "Hostname: $HOSTNAME"
+      echo ""
       echo "Starting NixOS bootstrap for host: $HOSTNAME"
+
+      log_to_loki "starting" "Bootstrap starting for $HOSTNAME (branch: $BRANCH)"
+
       echo "Waiting for network connectivity..."
 
       # Verify we can reach the git server via HTTPS (doesn't respond to ping)
       if ! curl -s --connect-timeout 5 --max-time 10 https://git.t-juice.club >/dev/null 2>&1; then
         echo "ERROR: Cannot reach git.t-juice.club via HTTPS"
         echo "Check network configuration and DNS settings"
+        log_to_loki "failed" "Network check failed - cannot reach git.t-juice.club"
         exit 1
       fi
 
       echo "Network connectivity confirmed"
+      log_to_loki "network_ok" "Network connectivity confirmed"
 
       # Unwrap Vault token and store AppRole credentials (if provided)
       if [ -n "''${VAULT_WRAPPED_TOKEN:-}" ]; then
@@ -50,6 +100,7 @@ let
           chmod 600 /var/lib/vault/approle/secret-id
 
           echo "Vault credentials unwrapped and stored successfully"
+          log_to_loki "vault_ok" "Vault credentials unwrapped and stored"
         else
           echo "WARNING: Failed to unwrap Vault token"
           if [ -n "$UNWRAP_RESPONSE" ]; then
@@ -63,17 +114,17 @@ let
           echo "To regenerate token, run: create-host --hostname $HOSTNAME --force"
           echo ""
           echo "Vault secrets will not be available, but continuing bootstrap..."
+          log_to_loki "vault_warn" "Failed to unwrap Vault token - continuing without secrets"
         fi
       else
         echo "No Vault wrapped token provided (VAULT_WRAPPED_TOKEN not set)"
         echo "Skipping Vault credential setup"
+        log_to_loki "vault_skip" "No Vault token provided - skipping credential setup"
       fi
 
       echo "Fetching and building NixOS configuration from flake..."
-
-      # Read git branch from environment, default to master
-      BRANCH="''${NIXOS_FLAKE_BRANCH:-master}"
       echo "Using git branch: $BRANCH"
+      log_to_loki "building" "Starting nixos-rebuild boot"
 
       # Build and activate the host-specific configuration
       FLAKE_URL="git+https://git.t-juice.club/torjus/nixos-servers.git?ref=$BRANCH#''${HOSTNAME}"
@@ -81,18 +132,30 @@ let
       if nixos-rebuild boot --flake "$FLAKE_URL"; then
         echo "Successfully built configuration for $HOSTNAME"
         echo "Rebooting into new configuration..."
+        log_to_loki "success" "Build successful - rebooting into new configuration"
         sleep 2
         systemctl reboot
       else
         echo "ERROR: nixos-rebuild failed for $HOSTNAME"
         echo "Check that flake has configuration for this hostname"
         echo "Manual intervention required - system will not reboot"
+        log_to_loki "failed" "nixos-rebuild failed - manual intervention required"
         exit 1
       fi
     '';
   };
 in
 {
+  # Custom greeting line to indicate this is a bootstrap image
+  services.getty.greetingLine = lib.mkForce ''
+    ================================================================================
+                          BOOTSTRAP IMAGE - NixOS \V (\l)
+    ================================================================================
+
+    Bootstrap service is running. Logs are displayed on tty1.
+    Check status: journalctl -fu nixos-bootstrap
+  '';
+
   systemd.services."nixos-bootstrap" = {
     description = "Bootstrap NixOS configuration from flake on first boot";
 
@@ -107,12 +170,12 @@ in
     serviceConfig = {
       Type = "oneshot";
       RemainAfterExit = true;
-      ExecStart = "${bootstrap-script}/bin/nixos-bootstrap";
+      ExecStart = lib.getExe bootstrap-script;
 
       # Read environment variables from cloud-init (set by cloud-init write_files)
       EnvironmentFile = "-/run/cloud-init-env";
 
-      # Logging to journald
+      # Log to journal and console
       StandardOutput = "journal+console";
       StandardError = "journal+console";
     };

hosts/template2/scripts.nix

@@ -2,7 +2,6 @@
 let
   prepare-host-script = pkgs.writeShellApplication {
     name = "prepare-host.sh";
-    runtimeInputs = [ pkgs.age ];
     text = ''
       echo "Removing machine-id"
       rm -f /etc/machine-id || true
@@ -22,11 +21,6 @@ let
 
       echo "Removing cache"
       rm -rf /var/cache/* || true
-
-      echo "Generate age key"
-      rm -rf /var/lib/sops-nix || true
-      mkdir -p /var/lib/sops-nix
-      age-keygen -o /var/lib/sops-nix/key.txt
     '';
   };
 in

hosts/testvm01/configuration.nix

@@ -13,14 +13,17 @@
     ../../common/vm
   ];
 
-  # Test VM - exclude from DNS zone generation
-  homelab.dns.enable = false;
-
+  # Host metadata (adjust as needed)
   homelab.host = {
-    tier = "test";
-    priority = "low";
+    tier = "test";  # Start in test tier, move to prod after validation
   };
 
+  # Enable Vault integration
+  vault.enable = true;
+
+  # Enable remote deployment via NATS
+  homelab.deploy.enable = true;
+
   nixpkgs.config.allowUnfree = true;
   boot.loader.grub.enable = true;
   boot.loader.grub.device = "/dev/vda";
@@ -29,7 +32,7 @@
   networking.domain = "home.2rjus.net";
   networking.useNetworkd = true;
   networking.useDHCP = false;
-  services.resolved.enable = false;
+  services.resolved.enable = true;
   networking.nameservers = [
     "10.69.13.5"
     "10.69.13.6"
@@ -39,7 +42,7 @@
   systemd.network.networks."ens18" = {
     matchConfig.Name = "ens18";
     address = [
-      "10.69.13.101/24"
+      "10.69.13.20/24"
     ];
     routes = [
       { Gateway = "10.69.13.1"; }
@@ -59,6 +62,39 @@
     git
   ];
 
+  # Test nginx with ACME certificate from OpenBao PKI
+  services.nginx = {
+    enable = true;
+    virtualHosts."testvm01.home.2rjus.net" = {
+      forceSSL = true;
+      enableACME = true;
+      locations."/" = {
+        root = pkgs.writeTextDir "index.html" ''
+          <!DOCTYPE html>
+          <html>
+          <head>
+            <title>testvm01 - ACME Test</title>
+            <style>
+              body { font-family: monospace; max-width: 600px; margin: 50px auto; padding: 20px; }
+              .joke { background: #f0f0f0; padding: 20px; border-radius: 8px; margin: 20px 0; }
+              .punchline { margin-top: 15px; font-weight: bold; }
+            </style>
+          </head>
+          <body>
+            <h1>OpenBao PKI ACME Test</h1>
+            <p>If you're seeing this over HTTPS, the migration worked!</p>
+            <div class="joke">
+              <p>Why do programmers prefer dark mode?</p>
+              <p class="punchline">Because light attracts bugs.</p>
+            </div>
+            <p><small>Certificate issued by: vault.home.2rjus.net</small></p>
+          </body>
+          </html>
+        '';
+      };
+    };
+  };
+
   # Open ports in the firewall.
   # networking.firewall.allowedTCPPorts = [ ... ];
   # networking.firewall.allowedUDPPorts = [ ... ];

hosts/testvm02/configuration.nix

@@ -1,25 +1,34 @@
 {
+  config,
+  lib,
   pkgs,
   ...
 }:
 
 {
   imports = [
-    ../template/hardware-configuration.nix
+    ../template2/hardware-configuration.nix
 
     ../../system
     ../../common/vm
   ];
 
-  nixpkgs.config.allowUnfree = true;
-  # Use the systemd-boot EFI boot loader.
-  boot.loader.grub = {
-    enable = true;
-    device = "/dev/sda";
-    configurationLimit = 3;
+  # Host metadata (adjust as needed)
+  homelab.host = {
+    tier = "test";  # Start in test tier, move to prod after validation
   };
 
-  networking.hostName = "ca";
+  # Enable Vault integration
+  vault.enable = true;
+
+  # Enable remote deployment via NATS
+  homelab.deploy.enable = true;
+
+  nixpkgs.config.allowUnfree = true;
+  boot.loader.grub.enable = true;
+  boot.loader.grub.device = "/dev/vda";
+
+  networking.hostName = "testvm02";
   networking.domain = "home.2rjus.net";
   networking.useNetworkd = true;
   networking.useDHCP = false;
@@ -33,7 +42,7 @@
   systemd.network.networks."ens18" = {
     matchConfig.Name = "ens18";
     address = [
-      "10.69.13.12/24"
+      "10.69.13.21/24"
     ];
     routes = [
       { Gateway = "10.69.13.1"; }
@@ -59,5 +68,5 @@
   # Or disable the firewall altogether.
   networking.firewall.enable = false;
 
-  system.stateVersion = "23.11"; # Did you read the comment?
-}
+  system.stateVersion = "25.11"; # Did you read the comment?
+}

hosts/testvm03/configuration.nix

@@ -1,25 +1,34 @@
-{ config, lib, pkgs, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
 
 {
-  imports =
-    [
-      ./hardware-configuration.nix
+  imports = [
+    ../template2/hardware-configuration.nix
 
-      ../../system
-    ];
-
-  # Template host - exclude from DNS zone generation
-  homelab.dns.enable = false;
+    ../../system
+    ../../common/vm
+  ];
 
+  # Host metadata (adjust as needed)
   homelab.host = {
-    tier = "test";
-    priority = "low";
+    tier = "test";  # Start in test tier, move to prod after validation
   };
 
+  # Enable Vault integration
+  vault.enable = true;
 
+  # Enable remote deployment via NATS
+  homelab.deploy.enable = true;
+
+  nixpkgs.config.allowUnfree = true;
   boot.loader.grub.enable = true;
-  boot.loader.grub.device = "/dev/sda";
-  networking.hostName = "nixos-template";
+  boot.loader.grub.device = "/dev/vda";
+
+  networking.hostName = "testvm03";
   networking.domain = "home.2rjus.net";
   networking.useNetworkd = true;
   networking.useDHCP = false;
@@ -33,19 +42,21 @@
   systemd.network.networks."ens18" = {
     matchConfig.Name = "ens18";
     address = [
-      "10.69.8.250/24"
+      "10.69.13.22/24"
     ];
     routes = [
-      { Gateway = "10.69.8.1"; }
+      { Gateway = "10.69.13.1"; }
     ];
     linkConfig.RequiredForOnline = "routable";
   };
   time.timeZone = "Europe/Oslo";
 
-  nix.settings.experimental-features = [ "nix-command" "flakes" ];
+  nix.settings.experimental-features = [
+    "nix-command"
+    "flakes"
+  ];
   nix.settings.tarball-ttl = 0;
   environment.systemPackages = with pkgs; [
-    age
     vim
     wget
     git
@@ -57,6 +68,5 @@
   # Or disable the firewall altogether.
   networking.firewall.enable = false;
 
-  system.stateVersion = "23.11"; # Did you read the comment?
-}
-
+  system.stateVersion = "25.11"; # Did you read the comment?
+}

hosts/testvm03/default.nix

@@ -2,4 +2,4 @@
   imports = [
     ./configuration.nix
   ];
-}
+}

hosts/vault01/configuration.nix

@@ -62,6 +62,16 @@
   # Or disable the firewall altogether.
   networking.firewall.enable = false;
 
+  # Vault fetches secrets from itself (after unseal)
+  vault.enable = true;
+  homelab.deploy.enable = true;
+
+  # Ensure vault-secret services wait for openbao to be unsealed
+  systemd.services.vault-secret-homelab-deploy-nkey = {
+    after = [ "openbao.service" ];
+    wants = [ "openbao.service" ];
+  };
+
   system.stateVersion = "25.11"; # Did you read the comment?
 }
 

hosts/vaulttest01/configuration.nix

@@ -1,135 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  ...
-}:
-
-let
-  vault-test-script = pkgs.writeShellApplication {
-    name = "vault-test";
-    text = ''
-      echo "=== Vault Secret Test ==="
-      echo "Secret path: hosts/vaulttest01/test-service"
-
-      if [ -f /run/secrets/test-service/password ]; then
-        echo "✓ Password file exists"
-        echo "Password length: $(wc -c < /run/secrets/test-service/password)"
-      else
-        echo "✗ Password file missing!"
-        exit 1
-      fi
-
-      if [ -d /var/lib/vault/cache/test-service ]; then
-        echo "✓ Cache directory exists"
-      else
-        echo "✗ Cache directory missing!"
-        exit 1
-      fi
-
-      echo "Test successful!"
-    '';
-  };
-in
-{
-  imports = [
-    ../template2/hardware-configuration.nix
-
-    ../../system
-    ../../common/vm
-  ];
-
-  homelab.host = {
-    tier = "test";
-    priority = "low";
-    role = "vault";
-  };
-
-  nixpkgs.config.allowUnfree = true;
-  boot.loader.grub.enable = true;
-  boot.loader.grub.device = "/dev/vda";
-
-  networking.hostName = "vaulttest01";
-  networking.domain = "home.2rjus.net";
-  networking.useNetworkd = true;
-  networking.useDHCP = false;
-  services.resolved.enable = true;
-  networking.nameservers = [
-    "10.69.13.5"
-    "10.69.13.6"
-  ];
-
-  systemd.network.enable = true;
-  systemd.network.networks."ens18" = {
-    matchConfig.Name = "ens18";
-    address = [
-      "10.69.13.150/24"
-    ];
-    routes = [
-      { Gateway = "10.69.13.1"; }
-    ];
-    linkConfig.RequiredForOnline = "routable";
-  };
-  time.timeZone = "Europe/Oslo";
-
-  nix.settings.experimental-features = [
-    "nix-command"
-    "flakes"
-  ];
-  nix.settings.tarball-ttl = 0;
-  environment.systemPackages = with pkgs; [
-    vim
-    wget
-    git
-    htop # test deploy verification
-  ];
-
-  # Open ports in the firewall.
-  # networking.firewall.allowedTCPPorts = [ ... ];
-  # networking.firewall.allowedUDPPorts = [ ... ];
-  # Or disable the firewall altogether.
-  networking.firewall.enable = false;
-
-  # Testing config
-  # Enable Vault secrets management
-  vault.enable = true;
-  homelab.deploy.enable = true;
-
-  # Define a test secret
-  vault.secrets.test-service = {
-    secretPath = "hosts/vaulttest01/test-service";
-    restartTrigger = true;
-    restartInterval = "daily";
-    services = [ "vault-test" ];
-  };
-
-  # Create a test service that uses the secret
-  systemd.services.vault-test = {
-    description = "Test Vault secret fetching";
-    wantedBy = [ "multi-user.target" ];
-    after = [ "vault-secret-test-service.service" ];
-
-    serviceConfig = {
-      Type = "oneshot";
-      RemainAfterExit = true;
-
-      ExecStart = lib.getExe vault-test-script;
-
-      StandardOutput = "journal+console";
-    };
-  };
-
-  # Test ACME certificate issuance from OpenBao PKI
-  # Override the global ACME server (from system/acme.nix) to use OpenBao instead of step-ca
-  security.acme.defaults.server = lib.mkForce "https://vault01.home.2rjus.net:8200/v1/pki_int/acme/directory";
-
-  # Request a certificate for this host
-  # Using HTTP-01 challenge with standalone listener on port 80
-  security.acme.certs."vaulttest01.home.2rjus.net" = {
-    listenHTTP = ":80";
-    enableDebugLogs = true;
-  };
-
-  system.stateVersion = "25.11"; # Did you read the comment?
-}
-

lib/monitoring.nix

@@ -21,6 +21,7 @@ let
       cfg = hostConfig.config;
       monConfig = (cfg.homelab or { }).monitoring or { enable = true; scrapeTargets = [ ]; };
       dnsConfig = (cfg.homelab or { }).dns or { enable = true; };
+      hostConfig' = (cfg.homelab or { }).host or { };
       hostname = cfg.networking.hostName;
       networks = cfg.systemd.network.networks or { };
 
@@ -49,20 +50,73 @@ let
         inherit hostname;
         ip = extractIP firstAddress;
         scrapeTargets = monConfig.scrapeTargets or [ ];
+        # Host metadata for label propagation
+        tier = hostConfig'.tier or "prod";
+        priority = hostConfig'.priority or "high";
+        role = hostConfig'.role or null;
+        labels = hostConfig'.labels or { };
       };
 
+  # Build effective labels for a host
+  # Always includes hostname; only includes tier/priority/role if non-default
+  buildEffectiveLabels = host:
+    { hostname = host.hostname; }
+    // (lib.optionalAttrs (host.tier != "prod") { tier = host.tier; })
+    // (lib.optionalAttrs (host.priority != "high") { priority = host.priority; })
+    // (lib.optionalAttrs (host.role != null) { role = host.role; })
+    // host.labels;
+
   # Generate node-exporter targets from all flake hosts
+  # Returns a list of static_configs entries with labels
   generateNodeExporterTargets = self: externalTargets:
     let
       nixosConfigs = self.nixosConfigurations or { };
       hostList = lib.filter (x: x != null) (
         lib.mapAttrsToList extractHostMonitoring nixosConfigs
       );
-      flakeTargets = map (host: "${host.hostname}.home.2rjus.net:9100") hostList;
+
+      # Extract hostname from a target string like "gunter.home.2rjus.net:9100"
+      extractHostnameFromTarget = target:
+        builtins.head (lib.splitString "." target);
+
+      # Build target entries with labels for each host
+      flakeEntries = map
+        (host: {
+          target = "${host.hostname}.home.2rjus.net:9100";
+          labels = buildEffectiveLabels host;
+        })
+        hostList;
+
+      # External targets get hostname extracted from the target string
+      externalEntries = map
+        (target: {
+          inherit target;
+          labels = { hostname = extractHostnameFromTarget target; };
+        })
+        (externalTargets.nodeExporter or [ ]);
+
+      allEntries = flakeEntries ++ externalEntries;
+
+      # Group entries by their label set for efficient static_configs
+      # Convert labels attrset to a string key for grouping
+      labelKey = entry: builtins.toJSON entry.labels;
+      grouped = lib.groupBy labelKey allEntries;
+
+      # Convert groups to static_configs format
+      # Every flake host now has at least a hostname label
+      staticConfigs = lib.mapAttrsToList
+        (key: entries:
+          let
+            labels = (builtins.head entries).labels;
+          in
+          { targets = map (e: e.target) entries; labels = labels; }
+        )
+        grouped;
     in
-    flakeTargets ++ (externalTargets.nodeExporter or [ ]);
+    staticConfigs;
 
   # Generate scrape configs from all flake hosts and external targets
+  # Host labels are propagated to service targets for semantic alert filtering
   generateScrapeConfigs = self: externalTargets:
     let
       nixosConfigs = self.nixosConfigurations or { };
@@ -70,13 +124,14 @@ let
         lib.mapAttrsToList extractHostMonitoring nixosConfigs
       );
 
-      # Collect all scrapeTargets from all hosts, grouped by job_name
+      # Collect all scrapeTargets from all hosts, including host labels
       allTargets = lib.flatten (map
         (host:
           map
             (target: {
               inherit (target) job_name port metrics_path scheme scrape_interval honor_labels;
               hostname = host.hostname;
+              hostLabels = buildEffectiveLabels host;
             })
             host.scrapeTargets
         )
@@ -87,22 +142,32 @@ let
       grouped = lib.groupBy (t: t.job_name) allTargets;
 
       # Generate a scrape config for each job
+      # Within each job, group targets by their host labels for efficient static_configs
       flakeScrapeConfigs = lib.mapAttrsToList
         (jobName: targets:
           let
             first = builtins.head targets;
-            targetAddrs = map
-              (t:
+
+            # Group targets within this job by their host labels
+            labelKey = t: builtins.toJSON t.hostLabels;
+            groupedByLabels = lib.groupBy labelKey targets;
+
+            # Every flake host now has at least a hostname label
+            staticConfigs = lib.mapAttrsToList
+              (key: labelTargets:
                 let
-                  portStr = toString t.port;
+                  labels = (builtins.head labelTargets).hostLabels;
+                  targetAddrs = map
+                    (t: "${t.hostname}.home.2rjus.net:${toString t.port}")
+                    labelTargets;
                 in
-                "${t.hostname}.home.2rjus.net:${portStr}")
-              targets;
+                { targets = targetAddrs; labels = labels; }
+              )
+              groupedByLabels;
+
             config = {
               job_name = jobName;
-              static_configs = [{
-                targets = targetAddrs;
-              }];
+              static_configs = staticConfigs;
             }
             // (lib.optionalAttrs (first.metrics_path != "/metrics") {
               metrics_path = first.metrics_path;

playbooks/build-and-deploy-template.yml

@@ -99,3 +99,48 @@
     - name: Display success message
       ansible.builtin.debug:
         msg: "Template VM {{ template_vmid }} created successfully on {{ storage }}"
+
+- name: Update Terraform template name
+  hosts: localhost
+  gather_facts: false
+
+  vars:
+    terraform_dir: "{{ playbook_dir }}/../terraform"
+
+  tasks:
+    - name: Get image filename from earlier play
+      ansible.builtin.set_fact:
+        image_filename: "{{ hostvars['localhost']['image_filename'] }}"
+
+    - name: Extract template name from image filename
+      ansible.builtin.set_fact:
+        new_template_name: "{{ image_filename | regex_replace('\\.vma\\.zst$', '') | regex_replace('^vzdump-qemu-', '') }}"
+
+    - name: Read current Terraform variables file
+      ansible.builtin.slurp:
+        src: "{{ terraform_dir }}/variables.tf"
+      register: variables_tf_content
+
+    - name: Extract current template name from variables.tf
+      ansible.builtin.set_fact:
+        current_template_name: "{{ (variables_tf_content.content | b64decode) | regex_search('variable \"default_template_name\"[^}]+default\\s*=\\s*\"([^\"]+)\"', '\\1') | first }}"
+
+    - name: Check if template name has changed
+      ansible.builtin.set_fact:
+        template_name_changed: "{{ current_template_name != new_template_name }}"
+
+    - name: Display template name status
+      ansible.builtin.debug:
+        msg: "Template name: {{ current_template_name }} -> {{ new_template_name }} ({{ 'changed' if template_name_changed else 'unchanged' }})"
+
+    - name: Update default_template_name in variables.tf
+      ansible.builtin.replace:
+        path: "{{ terraform_dir }}/variables.tf"
+        regexp: '(variable "default_template_name"[^}]+default\s*=\s*)"[^"]+"'
+        replace: '\1"{{ new_template_name }}"'
+      when: template_name_changed
+
+    - name: Display update result
+      ansible.builtin.debug:
+        msg: "Updated terraform/variables.tf with new template name: {{ new_template_name }}"
+      when: template_name_changed

rebuild-all.sh

@@ -5,7 +5,6 @@ set -euo pipefail
 HOSTS=(
     "ns1"
     "ns2"
-    "ca"
     "ha1"
     "http-proxy"
     "jelly01"

scripts/create-host/create_host.py

@@ -18,6 +18,8 @@ from manipulators import (
     remove_from_flake_nix,
     remove_from_terraform_vms,
     remove_from_vault_terraform,
+    remove_from_approle_tf,
+    find_host_secrets,
     check_entries_exist,
 )
 from models import HostConfig
@@ -255,7 +257,10 @@ def handle_remove(
         sys.exit(1)
 
     # Check what entries exist
-    flake_exists, terraform_exists, vault_exists = check_entries_exist(hostname, repo_root)
+    flake_exists, terraform_exists, vault_exists, approle_exists = check_entries_exist(hostname, repo_root)
+
+    # Check for secrets in secrets.tf
+    host_secrets = find_host_secrets(hostname, repo_root)
 
     # Collect all files in the host directory recursively
     files_in_host_dir = sorted([f for f in host_dir.rglob("*") if f.is_file()])
@@ -294,11 +299,25 @@ def handle_remove(
     else:
         console.print(f"  • terraform/vault/hosts-generated.tf [dim](not found)[/dim]")
 
-    # Warn about secrets directory
+    if approle_exists:
+        console.print(f'  • terraform/vault/approle.tf (host_policies["{hostname}"])')
+    else:
+        console.print(f"  • terraform/vault/approle.tf [dim](not found)[/dim]")
+
+    # Warn about secrets in secrets.tf
+    if host_secrets:
+        console.print(f"\n[yellow]⚠️  Warning: Found {len(host_secrets)} secret(s) in terraform/vault/secrets.tf:[/yellow]")
+        for secret_path in host_secrets:
+            console.print(f'   • "{secret_path}"')
+        console.print(f"\n   [yellow]These will NOT be removed automatically.[/yellow]")
+        console.print(f"   After removal, manually edit secrets.tf and run:")
+        for secret_path in host_secrets:
+            console.print(f"   [white]vault kv delete secret/{secret_path}[/white]")
+
+    # Warn about legacy secrets directory
     if secrets_exist:
-        console.print(f"\n[yellow]⚠️  Warning: secrets/{hostname}/ directory exists and will NOT be deleted[/yellow]")
+        console.print(f"\n[yellow]⚠️  Warning: secrets/{hostname}/ directory exists (legacy SOPS)[/yellow]")
         console.print(f"   Manually remove if no longer needed: [white]rm -rf secrets/{hostname}/[/white]")
-        console.print(f"   Also update .sops.yaml to remove the host's age key")
 
     # Exit if dry run
     if dry_run:
@@ -323,6 +342,13 @@ def handle_remove(
         else:
             console.print("[yellow]⚠[/yellow]  Could not remove from terraform/vault/hosts-generated.tf")
 
+    # Remove from terraform/vault/approle.tf
+    if approle_exists:
+        if remove_from_approle_tf(hostname, repo_root):
+            console.print("[green]✓[/green] Removed from terraform/vault/approle.tf")
+        else:
+            console.print("[yellow]⚠[/yellow]  Could not remove from terraform/vault/approle.tf")
+
     # Remove from terraform/vms.tf
     if terraform_exists:
         if remove_from_terraform_vms(hostname, repo_root):
@@ -345,19 +371,34 @@ def handle_remove(
     console.print(f"\n[bold green]✓ Host {hostname} removed successfully![/bold green]\n")
 
     # Display next steps
-    display_removal_next_steps(hostname, vault_exists)
+    display_removal_next_steps(hostname, vault_exists, approle_exists, host_secrets)
 
 
-def display_removal_next_steps(hostname: str, had_vault: bool) -> None:
+def display_removal_next_steps(hostname: str, had_vault: bool, had_approle: bool, host_secrets: list) -> None:
     """Display next steps after successful removal."""
-    vault_file = " terraform/vault/hosts-generated.tf" if had_vault else ""
-    vault_apply = ""
+    vault_files = ""
     if had_vault:
+        vault_files += " terraform/vault/hosts-generated.tf"
+    if had_approle:
+        vault_files += " terraform/vault/approle.tf"
+
+    vault_apply = ""
+    if had_vault or had_approle:
         vault_apply = f"""
 3. Apply Vault changes:
    [white]cd terraform/vault && tofu apply[/white]
 """
 
+    secrets_cleanup = ""
+    if host_secrets:
+        secrets_cleanup = f"""
+5. Clean up secrets (manual):
+   Edit terraform/vault/secrets.tf to remove entries for {hostname}
+   Then delete from Vault:"""
+        for secret_path in host_secrets:
+            secrets_cleanup += f"\n   [white]vault kv delete secret/{secret_path}[/white]"
+        secrets_cleanup += "\n"
+
     next_steps = f"""[bold cyan]Next Steps:[/bold cyan]
 
 1. Review changes:
@@ -367,9 +408,9 @@ def display_removal_next_steps(hostname: str, had_vault: bool) -> None:
    [white]cd terraform && tofu destroy -target='proxmox_vm_qemu.vm["{hostname}"]'[/white]
 {vault_apply}
 4. Commit changes:
-   [white]git add -u hosts/{hostname} flake.nix terraform/vms.tf{vault_file}
+   [white]git add -u hosts/{hostname} flake.nix terraform/vms.tf{vault_files}
    git commit -m "hosts: remove {hostname}"[/white]
-"""
+{secrets_cleanup}"""
     console.print(Panel(next_steps, border_style="cyan"))
 
 

scripts/create-host/generators.py

@@ -144,7 +144,7 @@ resource "vault_approle_auth_backend_role" "generated_hosts" {
 
   backend            = vault_auth_backend.approle.path
   role_name          = each.key
-  token_policies     = ["host-\${each.key}"]
+  token_policies     = ["host-\${each.key}", "homelab-deploy"]
   secret_id_ttl      = 0  # Never expire (wrapped tokens provide time limit)
   token_ttl          = 3600
   token_max_ttl      = 3600

scripts/create-host/manipulators.py

@@ -22,12 +22,12 @@ def remove_from_flake_nix(hostname: str, repo_root: Path) -> bool:
     content = flake_path.read_text()
 
     # Check if hostname exists
-    hostname_pattern = rf"^      {re.escape(hostname)} = nixpkgs\.lib\.nixosSystem"
+    hostname_pattern = rf"^        {re.escape(hostname)} = nixpkgs\.lib\.nixosSystem"
     if not re.search(hostname_pattern, content, re.MULTILINE):
         return False
 
     # Match the entire block from "hostname = " to "};"
-    replace_pattern = rf"^      {re.escape(hostname)} = nixpkgs\.lib\.nixosSystem \{{.*?^      \}};\n"
+    replace_pattern = rf"^        {re.escape(hostname)} = nixpkgs\.lib\.nixosSystem \{{.*?^        \}};\n"
     new_content, count = re.subn(replace_pattern, "", content, flags=re.MULTILINE | re.DOTALL)
 
     if count == 0:
@@ -101,7 +101,68 @@ def remove_from_vault_terraform(hostname: str, repo_root: Path) -> bool:
     return True
 
 
-def check_entries_exist(hostname: str, repo_root: Path) -> Tuple[bool, bool, bool]:
+def remove_from_approle_tf(hostname: str, repo_root: Path) -> bool:
+    """
+    Remove host entry from terraform/vault/approle.tf locals.host_policies.
+
+    Args:
+        hostname: Hostname to remove
+        repo_root: Path to repository root
+
+    Returns:
+        True if found and removed, False if not found
+    """
+    approle_path = repo_root / "terraform" / "vault" / "approle.tf"
+
+    if not approle_path.exists():
+        return False
+
+    content = approle_path.read_text()
+
+    # Check if hostname exists in host_policies
+    hostname_pattern = rf'^\s+"{re.escape(hostname)}" = \{{'
+    if not re.search(hostname_pattern, content, re.MULTILINE):
+        return False
+
+    # Match the entire block from "hostname" = { to closing }
+    # The block contains paths = [ ... ] and possibly extra_policies = [...]
+    replace_pattern = rf'\n?\s+"{re.escape(hostname)}" = \{{[^}}]*\}}\n?'
+    new_content, count = re.subn(replace_pattern, "\n", content, flags=re.DOTALL)
+
+    if count == 0:
+        return False
+
+    approle_path.write_text(new_content)
+    return True
+
+
+def find_host_secrets(hostname: str, repo_root: Path) -> list:
+    """
+    Find secrets in terraform/vault/secrets.tf that belong to a host.
+
+    Args:
+        hostname: Hostname to search for
+        repo_root: Path to repository root
+
+    Returns:
+        List of secret paths found (e.g., ["hosts/hostname/test-service"])
+    """
+    secrets_path = repo_root / "terraform" / "vault" / "secrets.tf"
+
+    if not secrets_path.exists():
+        return []
+
+    content = secrets_path.read_text()
+
+    # Find all secret paths matching hosts/{hostname}/
+    pattern = rf'"(hosts/{re.escape(hostname)}/[^"]+)"'
+    matches = re.findall(pattern, content)
+
+    # Return unique paths, preserving order
+    return list(dict.fromkeys(matches))
+
+
+def check_entries_exist(hostname: str, repo_root: Path) -> Tuple[bool, bool, bool, bool]:
     """
     Check which entries exist for a hostname.
 
@@ -110,12 +171,12 @@ def check_entries_exist(hostname: str, repo_root: Path) -> Tuple[bool, bool, boo
         repo_root: Path to repository root
 
     Returns:
-        Tuple of (flake_exists, terraform_vms_exists, vault_exists)
+        Tuple of (flake_exists, terraform_vms_exists, vault_generated_exists, approle_exists)
     """
     # Check flake.nix
     flake_path = repo_root / "flake.nix"
     flake_content = flake_path.read_text()
-    flake_pattern = rf"^      {re.escape(hostname)} = nixpkgs\.lib\.nixosSystem"
+    flake_pattern = rf"^        {re.escape(hostname)} = nixpkgs\.lib\.nixosSystem"
     flake_exists = bool(re.search(flake_pattern, flake_content, re.MULTILINE))
 
     # Check terraform/vms.tf
@@ -131,7 +192,15 @@ def check_entries_exist(hostname: str, repo_root: Path) -> Tuple[bool, bool, boo
         vault_content = vault_tf_path.read_text()
         vault_exists = f'"{hostname}"' in vault_content
 
-    return (flake_exists, terraform_exists, vault_exists)
+    # Check terraform/vault/approle.tf
+    approle_path = repo_root / "terraform" / "vault" / "approle.tf"
+    approle_exists = False
+    if approle_path.exists():
+        approle_content = approle_path.read_text()
+        approle_pattern = rf'^\s+"{re.escape(hostname)}" = \{{'
+        approle_exists = bool(re.search(approle_pattern, approle_content, re.MULTILINE))
+
+    return (flake_exists, terraform_exists, vault_exists, approle_exists)
 
 
 def update_flake_nix(config: HostConfig, repo_root: Path, force: bool = False) -> None:
@@ -147,32 +216,25 @@ def update_flake_nix(config: HostConfig, repo_root: Path, force: bool = False) -
     content = flake_path.read_text()
 
     # Create new entry
-    new_entry = f"""      {config.hostname} = nixpkgs.lib.nixosSystem {{
-        inherit system;
-        specialArgs = {{
-          inherit inputs self sops-nix;
+    new_entry = f"""        {config.hostname} = nixpkgs.lib.nixosSystem {{
+          inherit system;
+          specialArgs = {{
+            inherit inputs self;
+          }};
+          modules = commonModules ++ [
+            ./hosts/{config.hostname}
+          ];
         }};
-        modules = [
-          (
-            {{ config, pkgs, ... }}:
-            {{
-              nixpkgs.overlays = commonOverlays;
-            }}
-          )
-          ./hosts/{config.hostname}
-          sops-nix.nixosModules.sops
-        ];
-      }};
 """
 
     # Check if hostname already exists
-    hostname_pattern = rf"^      {re.escape(config.hostname)} = nixpkgs\.lib\.nixosSystem"
+    hostname_pattern = rf"^        {re.escape(config.hostname)} = nixpkgs\.lib\.nixosSystem"
     existing_match = re.search(hostname_pattern, content, re.MULTILINE)
 
     if existing_match and force:
         # Replace existing entry
         # Match the entire block from "hostname = " to "};"
-        replace_pattern = rf"^      {re.escape(config.hostname)} = nixpkgs\.lib\.nixosSystem \{{.*?^      \}};\n"
+        replace_pattern = rf"^        {re.escape(config.hostname)} = nixpkgs\.lib\.nixosSystem \{{.*?^        \}};\n"
         new_content, count = re.subn(replace_pattern, new_entry, content, flags=re.MULTILINE | re.DOTALL)
 
         if count == 0:

scripts/create-host/templates/configuration.nix.j2

@@ -18,6 +18,12 @@
     tier = "test";  # Start in test tier, move to prod after validation
   };
 
+  # Enable Vault integration
+  vault.enable = true;
+
+  # Enable remote deployment via NATS
+  homelab.deploy.enable = true;
+
   nixpkgs.config.allowUnfree = true;
   boot.loader.grub.enable = true;
   boot.loader.grub.device = "/dev/vda";

scripts/create-host/validators.py

@@ -140,20 +140,22 @@ def validate_ip_unique(ip: Optional[str], repo_root: Path) -> None:
     ip_part = ip.split("/")[0]
 
     # Check all hosts/*/configuration.nix files
+    # Search for IP with CIDR notation to match static IP assignments
+    # (e.g., "10.69.13.5/24") but not DNS resolver entries (e.g., "10.69.13.5")
     hosts_dir = repo_root / "hosts"
     if hosts_dir.exists():
         for config_file in hosts_dir.glob("*/configuration.nix"):
             content = config_file.read_text()
-            if ip_part in content:
+            if ip in content:
                 raise ValueError(
                     f"IP address {ip_part} already in use in {config_file}"
                 )
 
-    # Check terraform/vms.tf
+    # Check terraform/vms.tf - search for full IP with CIDR
     terraform_file = repo_root / "terraform" / "vms.tf"
     if terraform_file.exists():
         content = terraform_file.read_text()
-        if ip_part in content:
+        if ip in content:
             raise ValueError(
                 f"IP address {ip_part} already in use in {terraform_file}"
             )

secrets/ca/keys/intermediate_ca_key

@@ -1,24 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:TgGIuklFPUSCBosD86NFnkAtRvYijQNQP4vvTkKu3dRAOjdDa2li5djZDUS4NEEPEihpOcMXqHBb+ABk3LmoU5nLmsKCeylUp7+DhcGi9f3xw2h1zbHV37mt40OVLTF3cYufRdydIkCGQA3td3q1ue/wCna2ewe73xwGg5j6ZVJCZAtW4VCNZM+rcG+YxPUC0gmBH59+O0VSrZrkvSnifbr+K0dGwg4i17KwAukI4Ac7YMkQoeuAPXq38+ZftlRx4tq9xBUko6wpPY9zOaFzeagWYMF0n1UYqDt+/3XZI/mukPhJc9tzbWneqgkQBOx3OiDwrNglCHvEpnb+bZePIRLOnNHd1ShETgBqhsHGp9OAwwbAt4tO+HFpCQtVz7s2LWQFLbWiN0SCGzYUkFGCgoXae5H58lxFav8=,iv:UzaWlJ+M+VQx3CcPSGbFZh5/rGbKpS2Rq2XVZAIDFiQ=,tag:F3waoAMuEKTvN2xANReSww==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age1lznyk4ee7e7x8n92cq2n87kz9920473ks5u9jlhd3dczfzq4wamqept56u",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpRGZSVHRSMGlyazAwQU5j\nd1o1L0Y1ckhQMkh4MVZiRmZlR2ozcmdsUW1vCk4xZ1ZibDBrUWZhYmxVVjBUczRn\nYlJtUWF3Y1lHWG56NkhmK2JOUHVGajQKLS0tIDN2S2doQURpTis2U3lWV0NxdWEz\ncjNZaEl1dEQwOXhsNE9xbHhYUzNTV3cKVmVIe05JwgXKSku7AJmrujYXrbBSbpBJ\nnqCuDIhok1w/fiff+XXn8udbgPVq5bC2SOhHbtVxImgBCFzrj5hQ0A==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1288993th0ge00reg4zqueyvmkrsvk829cs068eekjqfdprsrkeqql7mljk",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA4V3NaUEdvMmJvakQ0L1F0\nUnkvQ2F5dEVlZ2pMdlBZcjJac0tERnF5ZWljCmFrdU1NZ29jMkJ1a1ZLdURmVWI0\ncm1vNytFVzZjbVY2aVd2N3laMWNRNFEKLS0tIGgzOTFZY0lxc0JyVmd5cFBlNkRr\nVDBWc0t4c3pVV3RhSTB1UUVpNHd6NUkKNn6Sxb5oxP7iWqTF1+X9nOiYum3U+Rzk\nkryxVnf9EvQIVIFKDaTb+yAEO8otjqj+C4mHA9fannnNEJduOiPWOg==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2024-11-30T13:18:08Z",
-		"mac": "ENC[AES256_GCM,data:9R9RJzPMr9Bv8aeCDxhExTfbr+R2hjap6FGSk5QxBdbNpOcNS78ica0CLEmkAYVAfjmx/X2jC5ZnsAueSPUK7nAgNX2gJXbUTpY0F+oKt35GJziLrFLl3u/ahpF9lQ50EL9OqqgS+igDqtodJhKme5DXH5/GXQHhz++O3VZkR78=,iv:XgN3PiowiEosi2DmrjP82HhJMvnwaV530tsBE8GQfjs=,tag:U243BrtH7H/DU9LcjN/MMg==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.1"
-	}
-}

secrets/ca/keys/root_ca_key

@@ -1,24 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:5AePh5uXcUseYBGWvlztgmg8mGBGy3ngKRa6+QxOaT0/fzSB1pKkaMtZJo76tV9wwjdL6/b6VVUI7GIaCBD5kgdZuA8RdBTXguHyjjdxAlI9xcrQaWWdATd8JJt+eQp/m2Y+0dioyXKaDV2ukI3GtHYjp/ixMoHHWEocnEEb40wG6c3CZcvsLWJvKTkFc2OvcjcU2RTfuNlYtEETidiD9iC/dtCakNQHmLP1UFYgcn0ebXBKmlqD6+x2o7BVT1SLwVCyGNvH3eKA2AWvddZChnhaNCUIXcRwBFCgS8lPs4iXhAhly+nwuj7ssFpuu3sjm5pq196tRS8WQl2iNUEJ2tzoOpceg1kZZ7KHX3wCbdBlCRqhy9Q4JMvWPDssO+zz2aU21+BDEySDTCnTYX9Hu2/iFvZejt++mKY=,iv:u/Ukye0BAj2ka++AA72W8WfXJAZZ/YJ3RC/aydxdoUc=,tag:ihTP5bCCigWEPcLFaYOhMA==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age1lznyk4ee7e7x8n92cq2n87kz9920473ks5u9jlhd3dczfzq4wamqept56u",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0VElDNHArZXlXa2JRQjd0\nQmVIbGpPWk43NDdiTkFtcEd1bDhRdXJWOUY0CndITHdKTFNJQXFOVFdyUGNtQ09k\nN2hnQmFYR0ZORWtxcUN0ZFhsM0U3N2cKLS0tIFh1TTBpMjFIZ2NYM1QxeDRjYlJx\nYkdrUDZmMUpGbjk3REJCVVRpeFk5Z28KJcia0Bk+3ZoifZnRLwqAko526ODPnkSS\nzymtOj/QYTA0++NP3B1aScIyhWITMEZX1iSoWDmgHj8ZQoNMdkM7AQ==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1288993th0ge00reg4zqueyvmkrsvk829cs068eekjqfdprsrkeqql7mljk",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBZNlNHRWNEcUZGNXNBMDFR\nTzE5RnNMQUMvU1k2OS9XMlpvUktMRzQ5RmxvCnlCS3lzRVpGUHJLRGZ6SWZ2ZktR\na3l0TVN2NUlRVEQwRHByYkNEMDQyWUkKLS0tIEh3RjBWT3c5K2RWeDRjWFpsU1lP\ncStqY2xta3RSNkR6Vkt5YXhYUTZmbDgKvVKmZc8S/RwurJGsGiJ5LhM4waLO9B9k\n2cawxHmcYM3KfXDFwp9UZWhIwF7SRkG56ZE4OjGI3sOL+74ixnePxA==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2024-11-30T13:18:16Z",
-		"mac": "ENC[AES256_GCM,data:JwjbQ129cYCBNA5Fb8lN9rW7/y4wuVOqLeajIMcYyCzlBcjzCZAV1DKN5n75xMamb/hb1AUkmtp/K82PKM0Vg5X4/lpWTUZXZOzn/TrwHx+yqlJjL9mUdGuHnSY5DwME38Dde3UxdtUa0CVgQOxvMIycW27w8+8NNfO2zxGxkzc=,iv:ZMZASOsqXZOb0NkBqG3GGaqqKgQdjZLiku2yU5QonB8=,tag:/lb/HMxsYOV5XX/5kWnFHA==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.1"
-	}
-}

secrets/ca/keys/ssh_host_ca_key

@@ -1,24 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:vqQ3HwSmuDlI4UwraLWvwkBSj9zTFeNEWI1xzhVrO/gpx8+WBZOt2F0J7/LSTGAWsWW/9Gov+XXXAOtfnKfjYVzizyT/jE8EQwMuItWiFEVA6hohgwtsk7YKJjXdJIxmiv+WKs73gWb0uFVGh1ArMzsVkGPj1W1AKMFAneDPgsfSCy9aVOMuF8zQwypFC8eaxqOQhLpiN2ncRm8e7khwGurSgYfHDgFghaDr8torgUrZTOPNFk+LEdxB3WcC17+4a8ZyuBapmYdRTrP73czTAuxOF8lMwddJhO99SF7nWuOYVF1FOKLGtK04oKci5/xRIzvWo3I0pGajkxtuF5CyWbd1KblcPfBALIU/J5hU/puGJ7M2sE/qsg/4kaTFxnhq32rPZj291jFb4evDdOhVodfC1axOQUbzAC0=,iv:yOeQ384ikqgDqfthl7GIVSIMNA/n0BYTSIqFN3T9MAY=,tag:Y6nhOCrkWx7MnVpEeKN0Jg==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age1lznyk4ee7e7x8n92cq2n87kz9920473ks5u9jlhd3dczfzq4wamqept56u",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFTjRMWlNtYVQ2WnJEaGFN\nVFU2TXRTK2FHREpqREhOWHBKemxNc2U4WW44CnV4OWlBdXlFUWhJYi9jTTRuUWJV\nOWFPV2I4UytDRFo3blN3bUtFQ1NGU0kKLS0tIGp2VHlDc1JMMUdDUjlNNDFwUUxj\nVnhHbCtrNVNpZXo0K2dDVU5YTVJJUEkKk9mVTbzQVGZo3RKDLPDwtENknh+in1Q5\njf4DA1cGDDNzcEIWOOYyS+1mzT9WY8gU0hWqihX/bAx7CVsNUallZw==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1288993th0ge00reg4zqueyvmkrsvk829cs068eekjqfdprsrkeqql7mljk",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrVFNwUGpkOUhkUXFWWERq\nMVdueC9VSE9KbGZkenBVK3NRMjRNVXVmcVRRCjNLa0QzbWVCQks3ZmV3eFVjcEp0\nRmxDSlZIZU1IbEdnbE83WlkxV3VZV1EKLS0tICtsRXArajQ4Um9mNEV5OWZBdS85\nVGFSU2wwODZ3Zm44M3pWcTdDV1dxejQKM2BK5Axb1cF344ea89gkzCLzEX6j4amK\nzxf+boBK7JUX7F6QaPB0sRU8J4Cei9mALz96C8xNHjX00KcD3O2QOA==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2024-11-30T13:18:20Z",
-		"mac": "ENC[AES256_GCM,data:AllgcWxHnr3igPi/JbfJCbEa6hKtmILnAjiaMojRZNO4p6zYSoF0s8lo9XX05/vIrFUo+YaCtsuacv+kfz9f6vQafPn7Vulbh6PeH1VlAmzyVfJOTmHP3YX8ic3uM56A4+III1jOERCFOIcc/CKsnRLFhLCRQRMgtgT0hTl5aPw=,iv:60dOYhoUTu1HIHzY36eJeRZ66/v6JmRRpIW99W2D+CI=,tag:F7nLSFm933K5M+JE4IvNYw==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.1"
-	}
-}

secrets/ca/keys/ssh_user_ca_key

@@ -1,24 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:YRdPrTLQH0xdWiIzOyjfEGpvfmuj6me6GzZZcauh9bUUywyA1ranDnWqbJYgawQQxIXsq9dhXD0uco+7mmXq2598kF1NI9jh6uLf3k0H494zZOalRBv/k8u9oJDLIiVAkg9eNNLbGX0PMZr/Yue/qdkuXx2Hg9E7bQJwpU/NXF+jKKs+3NmKT5NBlegwAzUs530D4DUoaq5AhvVvdC6a1UcE+KJzQ8pRiz1GjFIxAB7qX+GVwa3yNdLgo2tlAbOzjGtaDfJnhZIHSNEq+4TEhjlF9lCmFCGFDUVupvMOWs0kBywJEzIrDmxmvGHlPj3FfyytPb7qhlsOXDDDS67IoiwluKOnw+sALAG0Iv9LMrDZ3z8MXeEGvRWu0VDMuGXN905/9kGx/A40mPjcfnZvI+qSRIKjER5R8aU=,iv:qiP2Ml59AnK24MBbs7N/HqJIylf+fXGqJAo2N8iFNB0=,tag:0Dj5fVs6OB07kvV4qzuvfw==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age1lznyk4ee7e7x8n92cq2n87kz9920473ks5u9jlhd3dczfzq4wamqept56u",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBUFlvNmRNYUlJSHZYUkpJ\nMEloQXFSdENIWGJVVDNIOVY5MS9SYWRoL0FrCnRJc05wZUZBSDRvMHNUUEhNRXQ4\nTWhYOUp6YUNGZFNWUFRrSmlJM1c4aWcKLS0tIFc1b3NlSEo2eFJhdDgwejRqcHlT\nZE5wN01uaE04cTlIbVJMVWQvQ1pXajgKQ1n6UmP7LEBsnIBXVc0BceOqvwCqQzBP\ncI8C5Io4ILgMjY4dr6sd0SeJG6mfDdiMA+k7c6jqoyZCW/Pkd3LANQ==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1288993th0ge00reg4zqueyvmkrsvk829cs068eekjqfdprsrkeqql7mljk",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBtM2lyeXVzdE9nL1k5L3dC\nTkl2MjhMb1FKMFdCeXFPSmNST0pvOTRUaEVvCmdwMnhjSFFHVFhidmIySS9jMEJu\nNTJpRjdFOWpZZ3ZuZFJwZUUrRFU5NnMKLS0tIDJ1UjdVQkpMNm5Pd01JRnZNOEtr\nb1lpMlBkVHpiT2lYdWtZaUQrRW1HUDgKq/JVMf5gdu6lNEmqY6zU2SymbT+jklem\nnUQ9yieJGF+PanutNW6BCJH8jb/fH+Y6AeJ9S+kKCB4Yi75i4d+oHg==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2024-11-30T13:18:24Z",
-		"mac": "ENC[AES256_GCM,data:6FJTKEdIpCm+Dz7Ua8dZOMZQFaGU0oU/HRP6ly5mWbXCv81LRbZXRBd+5RDY3z9g9nb0PXZrOMNps63F6SKxK52VfzLIOap3UGeMNQn5P4/yyFj7JQHQ5Gjcf2l2z2VZ7NhUdNoSCV/6lwjValbKtids48Q5c3sFX997ZiqIUnY=,iv:nUeyJd/v8d9v7QsLLckziD9K5qjOZKK4vOQJw/ymi18=,tag:6n5EE3oklWdVcedvB2J/zA==,type:str]",
-		"pgp": null,
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.1"
-	}
-}

secrets/ca/secrets.yaml

@@ -1,30 +0,0 @@
-ca_root_pw: ENC[AES256_GCM,data:jS5BHS9i/pOykus5aGsW+w==,iv:aQIU7uXnNKaeNXv1UjRpBoSYcRpHo8RjnvCaIw4yCqc=,tag:lkjGm5/Ve93nizqGDQ0ByA==,type:str]
-sops:
-    kms: []
-    gcp_kms: []
-    azure_kv: []
-    hc_vault: []
-    age:
-        - recipient: age1lznyk4ee7e7x8n92cq2n87kz9920473ks5u9jlhd3dczfzq4wamqept56u
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA5anlORWxJalhRWkJPeGIy
-            OStyVG8vMFRTTEZOWHR3Q3N1UWJQbFlxV3pBCmVKQVM1SlJ2L0JOb3U3cTh3YkZ4
-            WHAxSUpTT1dyRHJHYVd1Qkh1ZWxwYW8KLS0tIEhXeklsSmlGaFlaaWF5L0Nodk5a
-            clZ4M3hFSlFqaEZ0UWREdHpTQ29GVUEKAxj5P05Ilpwis2oKFe54mJX+1LfTwfUv
-            2XRFOrEQbFNcK5WFu46p1mc/AAjKTeHWuvb2Yq43CO+sh1+kqKz0XA==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1288993th0ge00reg4zqueyvmkrsvk829cs068eekjqfdprsrkeqql7mljk
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBaS0dqQ1p4MEE2d2JaeFRx
-            UnB4ejhrS3hLekpqeWJhcEJGdnpzMTZDelVRCmFjVGswd3VtRUloWG1WbWY5N0s3
-            cG9aV2hGU3lFZkkvcUJNWE1rWUIwMmMKLS0tIG1KdlhoQzREWDhPbXVSZVBUQkdE
-            N1hmcEwxWXBIWkQ3a3BrdGhvUFoxbzgKX6hLoz7o/Du6ymrYwmGDkXp2XT+0+7QE
-            YhD5qQzGLVQSh3XM/wWExj2Ue5/gw/NqNziHezOh2r9gQljbHjG2/g==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-10-21T09:12:26Z"
-    mac: ENC[AES256_GCM,data:hfPRIXt/kZJa6lsj7rz+5xGlrWhR/LX895S2d8auP/4t3V//80YE/ofIsHeAY9M7eSFsW9ce2Vp0C/WiCQefVWNaNN7nVAwskCfQ6vTWzs23oYz4NYIeCtZggBG3uGgJxb7ZnAFUJWmLwCxkKTQyoVVnn8i/rUDIBrkilbeLWNI=,iv:lm1HVbWtAifHjqKP0D3sxRadsE9+82ugbA2x54yRBTo=,tag:averxmPLa131lJtFrNxcEA==,type:str]
-    pgp: []
-    unencrypted_suffix: _unencrypted
-    version: 3.9.1

secrets/http-proxy/wireguard.yaml

@@ -1,25 +0,0 @@
-wg_private_key: ENC[AES256_GCM,data:DlC9txcLkTnb7FoEd249oJV/Ehcp50P8uulbE4rY/xU16fkTlnKvPmYZ7u8=,iv:IsiTzdrh+BNSVgx1mfjpMGNV2J0c88q6AoP0kHX2aGY=,tag:OqFsOIyE71SBD1mcNS/PeQ==,type:str]
-sops:
-    age:
-        - recipient: age1lznyk4ee7e7x8n92cq2n87kz9920473ks5u9jlhd3dczfzq4wamqept56u
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzdm9HTTN1amwxQ2Z6MUQv
-            dGJ0cEgyaHNOZWtWSWlXNXc5bGhUdSsvVlVzCkJkc3ZQdzlBNDNxb3Avdi96bXFt
-            TExZY29nUDI3RE5vanh6TVBRME1Fa1UKLS0tIG8vSHdCYzkvWmJpd0hNbnRtUmtk
-            aVcwaFJJclZ3YUlUTTNwR2VESmVyZWMKHvKUJBDuNCqacEcRlapetCXHKRb0Js09
-            sqxLfEDwiN2LQQjYHZOmnMfCOt/b2rwXVKEHdTcIsXbdIdKOJwuAIQ==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1gq8434ku0xekqmvnseeunv83e779cg03c06gwrusnymdsr3rpufqx6vr3m
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBEeU01UTc2V1UyZXRadE5I
-            VE1aakVZUEZUNnJxbzJ1K3J1R3ZQdFdMbUhBCjZBMDM3ZkYvQWlyNHBtaDZRWkd4
-            VzY0L3l4N2RNZjJRTDJWZTZyZVhHbW8KLS0tIGVNZ0N0emVmaVRCV09jNmVKRlla
-            cWVSNkJqWHh5c21KcWFac2FlZTVaMTAK1UvfPgZAZYtwiONKIAo5HlaDpN+UT/S/
-            JfPUfjxgRQid8P20Eh/jUepxrDY8iXRZdsUMON+OoQ8mpwoAh5eN1A==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2025-05-15T18:56:55Z"
-    mac: ENC[AES256_GCM,data:J2kHY7pXBJZ0UuNCZOhkU11M8rDqCYNzY71NyuDRmzzRCC9ZiNIbavyQAWj2Dpk1pjGsYjXsVoZvP7ti1wTFqahpaR/YWI5gmphrzAe32b9qFVEWTC3YTnmItnY0YxQZYehYghspBjnJtfUK0BvZxSb17egpoFnvHmAq+u5dyxg=,iv:/aLg02RLuJZ1bRzZfOD74pJuE7gppCBztQvUEt557mU=,tag:toxHHBuv3WRblyc9Sth6Iw==,type:str]
-    unencrypted_suffix: _unencrypted
-    version: 3.10.2

secrets/monitoring01/pve-exporter.yaml

@@ -1,33 +0,0 @@
-default:
-    user: ENC[AES256_GCM,data:4Zzjm6/e8GCKSPNivnY=,iv:Y3gR+JSH/GLYvkVu3CN4T/chM5mjGjwVPI0iMB4p1t4=,tag:auyG8iWsd/YGjDnnTC21Ew==,type:str]
-    password: ENC[AES256_GCM,data:9cyM9U8VnzXBBA==,iv:YMHNNUoQ9Az5+81Df07tjC+LaEWPHV6frUjd4PZrQOs=,tag:3hKR+BhLJODJp19nn4ppkA==,type:str]
-    verify_ssl: ENC[AES256_GCM,data:Cu5Ucf0=,iv:QFfdV7gDBQ+L2kSZZqlVqCrn9CRg5RNG5DNTFWtVf5Y=,tag:u24ZbpWA65wj3WOwqU1v+g==,type:bool]
-sops:
-    kms: []
-    gcp_kms: []
-    azure_kv: []
-    hc_vault: []
-    age:
-        - recipient: age1lznyk4ee7e7x8n92cq2n87kz9920473ks5u9jlhd3dczfzq4wamqept56u
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuUXdMMG5YaHRJbThQZW9u
-            RHVBbXFiSHNiUWdLTDdPajIyQjN3OGR0dGpzCm9ZVkdNWjhBakU3dVdhRU9kbU81
-            aDlCNzJBQ1hvQ3FnTUk2N2RWQkZpUUEKLS0tIEZacTNqa3FWc2p1NXVtRWhwVExj
-            cUJtYXNjb2Z4QkF4MjlidEZxSUFNa3MKAGHGksPc9oJheSlUQ3ARK5MuR5NFbPmD
-            kmSDSgRmzbarxT8eJnK8/K4ii3hX5E9vGOohUkyc03w4ENsh/dw43g==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1vpns76ykll8jgdlu3h05cur4ew2t3k7u03kxdg8y6ypfhsfhq9fqyurjey
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBOVGhvdGE5Mzl0ckhBM21D
-            RXJwb09OS25PMGViblViM21wTVZiZWhtWmhFCnAzL1NqeUVyOGZFVDFvdXFPbklQ
-            ZkJPWDVIdUdCdjZGUjcrcmtvak5CWG8KLS0tIDhLUHJNN2VqNy9CdVh0K0N0b0k1
-            RUE4U0E0aGxiRkF0NWdwSEIrQTU4MjgKeOU6bIWO6ke9YcG+1E3brnC21sSQxZ9b
-            SiG2QEnFnTeJ5P50XQoYHqUY3B0qx7nDLvyzatYEi6sDkfLXhmHGbw==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-12-03T16:25:12Z"
-    mac: ENC[AES256_GCM,data:gemq8YpMZQC+gY7lmMM3tfZh9XxL40qdGlLiB2CD4SIG49w0V6E/vY7xygt0WW0zHbhMI9yUIqlRc/PaXn+QfyxJEr3IjaT05rrWUqQAeRP9Zss74Y3NtQehh8fM8SgeyU4j2CQ9f9B/lW9IgdOW/TNgQZVXGg1vXZPEzl7AZ4A=,iv:LG5ojv3hAqk+EvFa/xEn43MBqL457uKFDE3dG5lSgZo=,tag:AxzcUzmdhO411Sw7Vg1itA==,type:str]
-    pgp: []
-    unencrypted_suffix: _unencrypted
-    version: 3.9.1

secrets/nix-cache01/actions_token_1

@@ -1,19 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:P84qHFU+xQjwQGK8I1gIdcBsHrskuUg0M1nGMMaA+hFjAdFYUhdhmAN/+y0CO28=,iv:zJtk01zNMTBDQdVtZBTM34CHRaNYDkabolxh7PWGKUI=,tag:8AS80AbZJbh9B3Av3zuI1w==,type:str]",
-	"sops": {
-		"age": [
-			{
-				"recipient": "age1lznyk4ee7e7x8n92cq2n87kz9920473ks5u9jlhd3dczfzq4wamqept56u",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkRFB6QTIyWWdwVkV4ZXNB\nWkdSdEhMc0s4cnByWVZXTGhnSWZ0MTdEUWhJCnFlOFQ5TU1hcE91azVyZXVXRCtu\nZjIxalRLYlEreGZ6ZDNoeXNPaFN4b28KLS0tIHY5WVFXN1k4NFVmUjh6VURkcEpv\ncklGcWVhdTdBRnlOdm1qM2h5SS9UUkEKq2RyxSVymDqcsZ+yiNRujDCwk1WOWYRW\nDa4TRKg3FCe7TcCEPkIaev1aBqjLg9J9c/70SYpUm6Zgeps7v5yl3A==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1w029fksjv0edrff9p7s03tgk3axecdkppqymfpwfn2nu2gsqqefqc37sxq",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArTGVuckp2NlhMZXRNMVhO\naUV3K0h3cmZ5ZGx4Q3dJWHNqZXFJeE1kM0dFCmF4TUFUMm9mTHJlYzlYWVhNa1RH\nR29VNDIrL1IvYUpQYm5SZEYzbWhhbkkKLS0tIEJsK1dwZVdaaHpWQkpOOS90dkhx\nbGhvRXhqdFdqQmhZZmhCdmw4NUtSVG8K3z2do+/cIjAqg6EMJnubOWid1sMeTxvo\nrq6eGJ7YzdgZr2JBVtJdDRtk/KeHXu9In4efbBXwLAPIfn1pU0gm1w==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-08-21T19:08:48Z",
-		"mac": "ENC[AES256_GCM,data:5CkO09NIqttb4UZPB9iGym8avhTsMeUkTFTKZJlNGjgB1qWyGQNeKCa50A1+SbBCCWE5EwxoynB1so7bi8vnq7k8CPUHbiWG8rLOJSYHQcZ9Tu7ZGtpeWPcCw1zPWJ/PTBsFVeaT5/ufdx/6ut+sTtRoKHOZZtO9oStHmu/Rlfg=,iv:z9iJJlbvhgxJaART5QoCrqvrqlgoVlGj8jlndCALmKU=,tag:ldjmND4NVVQrHUldLrB4Jg==,type:str]",
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.10.2"
-	}
-}

secrets/nix-cache01/cache-secret

@@ -1,19 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:MQkR6FQGHK2AuhOmy2was49RY2XlLO5NwaXnUFzFo5Ata/2ufVoAj4Jvotw/dSrKL7f62A6s+2BPAyWrvACJ+pwYFlfyj3T9bNwhxwZPkEmiHEubJjWSiD6jkSW0gOxbY8ib6g/GbyF8I1cPeYr/hJD5qQ==,iv:eBL2Y3MOt9gYTETUZqsHo1D5hPOHxb4JR6Z/DFlzzqI=,tag:Qqbt39xZvQz/QhsggsArsw==,type:str]",
-	"sops": {
-		"age": [
-			{
-				"recipient": "age1lznyk4ee7e7x8n92cq2n87kz9920473ks5u9jlhd3dczfzq4wamqept56u",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwZzFXaEsyUkZGNFV0bVlW\nRkpPRHpUK2VwUHpOQXZCUUpoVzFGa3hycnhvCndTN0toVFdoU2E5N3V3UFhTTjU0\nNDByWTkrV0o3T295dE0zS08rVGpyQjAKLS0tIC96M0VEcWpjRk5DMjJnMFB4ZHI3\nM2Jod2x4ZzMyZm1pbDhZNTFuWGNRUlEKHs5jBSfjml09JOeKiT9vFR0Fykg6OxKG\njhFU/J2+fWB22G7dBc4PI60SNqhxIheUbGTdcz4Yp4BPL6vW3eArIw==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1w029fksjv0edrff9p7s03tgk3axecdkppqymfpwfn2nu2gsqqefqc37sxq",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJT3lxamcrQUpFdjZteFlF\nYUQ3aGdadGpuNXd2Z3RtZ3dQU0cvMlFUMUNRClBDR3U0OXZJU0NDamVMSlR5NitN\nYlhvNVlvUE0wRjErYzkwVHFOdGVCVjgKLS0tIEttR1BLTGpDYTRSQ0lUZmVEcnNi\nWkNaMEViUHVBcExVOEpjNE5CZHpjVkEKuX/Rf8kaB3apr1UhAnq3swS6fXiVmwm8\n7Key+SUAPNstbWbz0u6B9m1ev5QcXB2lx2/+Cm7cjW+6VE2gLHjTsQ==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2025-01-24T12:19:16Z",
-		"mac": "ENC[AES256_GCM,data:X8X91LVP1MMJ8ZYeSNPRO6XHN+NuswLZcHpAkbvoY+E9aTteO8UqS+fsStbNDlpF5jz/mhdMsKElnU8Z/CIWImwolI4GGE6blKy6gyqRkn4VeZotUoXcJadYV/5COud3XP2uSTb694JyQEZnBXFNeYeiHpN0y38zLxoX8kXHFbc=,iv:fFCRfv+Y1Nt2zgJNKsxElrYcuKkATJ3A/jvheUY2IK4=,tag:hYojbMGUAQvx7I4qkO7o9w==,type:str]",
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.9.3"
-	}
-}

secrets/secrets.yaml

@@ -1,109 +0,0 @@
-root_password_hash: ENC[AES256_GCM,data:wk/xEuf+qU3ezmondq9y3OIotXPI/L+TOErTjgJz58wEvQkApYkjc3bHaUTzOrmWjQBgDUENObzPmvQ8WKawUSJRVlpfOEr5TQ==,iv:I8Z3xJz3qoXBD7igx087A1fMwf8d29hQ4JEI3imRXdY=,tag:M80osQeWGG9AAA8BrMfhHA==,type:str]
-ns_xfer_key: ENC[AES256_GCM,data:VFpK7GChgFeUgQm31tTvVC888bN0yt6BAnHQa6KUTg4iZGP1WL5Bx6Zp8dY=,iv:9RF1eEc7JBxBebDOKfcDjGS2U7XsHkOW/l52yIP+1LA=,tag:L6DR2QlHOfo02kzfWWCrvg==,type:str]
-backup_helper_secret: ENC[AES256_GCM,data:EvXEJnDilbfALQ==,iv:Q3dkZ8Ee3qbcjcoi5GxfbaVB4uRIvkIB6ioKVV/dL2Y=,tag:T/UgZvQgYGa740Wh7D0b7Q==,type:str]
-nats_nkey: ENC[AES256_GCM,data:N2CVXjdwiE7eSPUtXe+NeKSTzA9eFwK2igxaCdYsXd4Ps0/DjYb/ggnQziQzSy8viESZYjXhJ2VtNw==,iv:Xhcf5wPB01Wu0A+oMw0wzTEHATp+uN+wsaYshxIzy1w=,tag:IauTIOHqfiM75Ufml/JXbg==,type:str]
-sops:
-    age:
-        - recipient: age1lznyk4ee7e7x8n92cq2n87kz9920473ks5u9jlhd3dczfzq4wamqept56u
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuWXhzQWFmeCt1R05jREcz
-            Ui9HZFN5dkxHNVE0RVJGZUJUa3hKK2sxdkhBCktYcGpLeGZIQzZIV3ZZWGs3YzF1
-            T09sUEhPWkRkOWZFWkltQXBlM1lQV1UKLS0tIERRSlRUYW5QeW9TVjJFSmorOWNI
-            ZytmaEhzMjVhRXI1S0hielF0NlBrMmcK4I1PtSf7tSvSIJxWBjTnfBCO8GEFHbuZ
-            BkZskr5fRnWUIs72ZOGoTAVSO5ZNiBglOZ8YChl4Vz1U7bvdOCt0bw==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1hz2lz4k050ru3shrk5j3zk3f8azxmrp54pktw5a7nzjml4saudesx6jsl0
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQcXM0RHlGcmZrYW4yNGZs
-            S1ZqQzVaYmQ4MGhGaTFMUVIwOTk5K0tZZjB3ClN0QkhVeHRrNXZHdmZWMzFBRnJ6
-            WTFtaWZyRmx2TitkOXkrVkFiYVd3RncKLS0tIExpeGUvY1VpODNDL2NCaUhtZkp0
-            cGNVZTI3UGxlNWdFWVZMd3FlS3pDR3cKBulaMeonV++pArXOg3ilgKnW/51IyT6Z
-            vH9HOJUix+ryEwDIcjv4aWx9pYDHthPFZUDC25kLYG91WrJFQOo2oA==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1w2q4gm2lrcgdzscq8du3ssyvk6qtzm4fcszc92z9ftclq23yyydqdga5um
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBabTdsZWxZQjV2TGx2YjNM
-            ZTgzWktqTjY0S0M3bFpNZXlDRDk5TSt3V2k0CjdWWTN0TlRlK1RpUm9xYW03MFFG
-            aWN4a3o4VUVnYzBDd2FrelUraWtrMTAKLS0tIE1vTGpKYkhzcWErWDRreml2QmE2
-            ZkNIWERKb1drdVR6MTBSTnVmdm51VEkKVNDYdyBSrUT7dUn6a4eF7ELQ2B2Pk6V9
-            Z5fbT75ibuyX1JO315/gl2P/FhxmlRW1K6e+04gQe2R/t/3H11Q7YQ==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1d2w5zece9647qwyq4vas9qyqegg96xwmg6c86440a6eg4uj6dd2qrq0w3l
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBVSFhDOFRVbnZWbVlQaG5G
-            U0NWekU0NzI1SlpRN0NVS1hPN210MXY3Z244CmtFemR5OUpzdlBzMHBUV3g0SFFo
-            eUtqNThXZDJ2b01yVVVuOFdwQVo2Qm8KLS0tIHpXRWd3OEpPRkpaVDNDTEJLMWEv
-            ZlZtaFpBdzF0YXFmdjNkNUR3YkxBZU0KAub+HF/OBZQR9bx/SVadZcL6Ms+NQ7yq
-            21HCcDTWyWHbN4ymUrIYXci1A/0tTOrQL9Mkvaz7IJh4VdHLPZrwwA==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1gq8434ku0xekqmvnseeunv83e779cg03c06gwrusnymdsr3rpufqx6vr3m
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBWkhBL1NTdjFDeEhQcEgv
-            Z3c3Z213L2ZhWGo0Qm5Zd1A1RTBDY3plUkh3CkNWV2ZtNWkrUjB0eWFzUlVtbHlk
-            WTdTQjN4eDIzY0c0dyt6ajVXZ0krd1UKLS0tIHB4aEJqTTRMenV3UkFkTGEySjQ2
-            YVM1a3ZPdUU4T244UU0rc3hVQ3NYczQK10wug4kTjsvv/iOPWi5WrVZMOYUq4/Mf
-            oXS4sikXeUsqH1T2LUBjVnUieSneQVn7puYZlN+cpDQ0XdK/RZ+91A==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1288993th0ge00reg4zqueyvmkrsvk829cs068eekjqfdprsrkeqql7mljk
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYcEtHbjNWRkdodUxYdHRn
-            MDBMU08zWDlKa0Z4cHJvc28rZk5pUjhnMjE0CmdzRmVGWDlYQ052Wm1zWnlYSFV6
-            dURQK3JSbThxQlg3M2ZaL1hGRzVuL0UKLS0tIEI3UGZvbEpvRS9aR2J2Tnc1YmxZ
-            aUY5Q2MrdHNQWDJNaGt5MWx6MVRrRVEKRPxyAekGHFMKs0Z6spVDayBA4EtPk18e
-            jiFc97BGVtC5IoSu4icq3ZpKOdxymnkqKEt0YP/p/JTC+8MKvTJFQw==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1vpns76ykll8jgdlu3h05cur4ew2t3k7u03kxdg8y6ypfhsfhq9fqyurjey
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQL3ZMUkI1dUV1T2tTSHhn
-            SjhyQ3dKTytoaDBNcit1VHpwVGUzWVNpdjBnCklYZWtBYzBpcGxZSDBvM2tIZm9H
-            bTFjb1ZCaDkrOU1JODVBVTBTbmxFbmcKLS0tIGtGcS9kejZPZlhHRXI5QnI5Wm9Q
-            VjMxTDdWZEltWThKVDl0S24yWHJxZHcKgzH79zT2I7ZgyTbbbvIhLN/rEcfiomJH
-            oSZDFvPiXlhPgy8bRyyq3l47CVpWbUI2Y7DFXRuODpLUirt3K3TmCA==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1hchvlf3apn8g8jq2743pw53sd6v6ay6xu6lqk0qufrjeccan9vzsc7hdfq
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPcm9zUm1XUkpLWm1Jb3Uw
-            RncveGozOW5SRThEM1Y4SFF5RDdxUEhZTUE4CjVESHE5R3JZK0krOXZDL0RHR0oy
-            Z3JKaEpydjRjeFFHck1ic2JTRU5yZTQKLS0tIGY2ck56eG95YnpDYlNqUDh5RVp1
-            U3dRYkNleUtsQU1LMWpDbitJbnRIem8K+27HRtZihG8+k7ZC33XVfuXDFjC1e8lA
-            kffmxp9kOEShZF3IKmAjVHFBiPXRyGk3fGPyQLmSMK2UOOfCy/a/qA==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1w029fksjv0edrff9p7s03tgk3axecdkppqymfpwfn2nu2gsqqefqc37sxq
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTZHlldDdSOEhjTklCSXQr
-            U2pXajFwZnNqQzZOTzY5b3lkMzlyREhXRWo4CmxId2F6NkNqeHNCSWNrcUJIY0Nw
-            cGF6NXJaQnovK1FYSXQ2TkJSTFloTUEKLS0tIHRhWk5aZ0lDVkZaZEJobm9FTDNw
-            a29sZE1GL2ZQSk0vUEc1ZGhkUlpNRkEK9tfe7cNOznSKgxshd5Z6TQiNKp+XW6XH
-            VvPgMqMitgiDYnUPj10bYo3kqhd0xZH2IhLXMnZnqqQ0I23zfPiNaw==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1ha34qeksr4jeaecevqvv2afqem67eja2mvawlmrqsudch0e7fe7qtpsekv
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5bk9NVjJNWmMxUGd3cXRx
-            amZ5SWJ3dHpHcnM4UHJxdmh6NnhFVmJQdldzCm95dHN3R21qSkE4Vm9VTnVPREp3
-            dUQyS1B4MWhhdmd3dk5LQ0htZEtpTWMKLS0tIGFaa3MxVExFYk1MY2loOFBvWm1o
-            L0NoRStkeW9VZVdpWlhteC8yTnRmMUkKMYjUdE1rGgVR29FnhJ5OEVjTB1Rh5Mtu
-            M/DvlhW3a7tZU8nDF3IgG2GE5xOXZMDO9QWGdB8zO2RJZAr3Q+YIlA==
-            -----END AGE ENCRYPTED FILE-----
-        - recipient: age1cxt8kwqzx35yuldazcc49q88qvgy9ajkz30xu0h37uw3ts97jagqgmn2ga
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBU0xYMnhqOE0wdXdleStF
-            THcrY2NBQzNoRHdYTXY3ZmM5YXRZZkQ4aUZnCm9ad0IxSWxYT1JBd2RseUdVT1pi
-            UXBuNzFxVlN0OWNTQU5BV2NiVEV0RUUKLS0tIGJHY0dzSDczUzcrV0RpTjE0czEy
-            cWZMNUNlTzBRcEV5MjlRV1BsWGhoaUUKGhYaH8I0oPCfrbs7HbQKVOF/99rg3HXv
-            RRTXUI71/ejKIuxehOvifClQc3nUW73bWkASFQ0guUvO4R+c0xOgUg==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2025-02-11T21:18:22Z"
-    mac: ENC[AES256_GCM,data:5//boMp1awc/2XAkSASSCuobpkxa0E6IKf3GR8xHpMoCD30FJsCwV7PgX3fR8OuLEhOJ7UguqMNQdNqG37RMacreuDmI1J8oCFKp+3M2j4kCbXaEo8bw7WAtyjUez+SAXKzZWYmBibH0KOy6jdt+v0fdgy5hMBT4IFDofYRsyD0=,iv:6pD+SLwncpmal/FR4U8It2njvaQfUzzpALBCxa0NyME=,tag:4QN8ZFjdqck5ZgulF+FtbA==,type:str]
-    unencrypted_suffix: _unencrypted
-    version: 3.9.4

services/ca/default.nix

@@ -1,169 +0,0 @@
-{ pkgs, unstable, ... }:
-{
-  homelab.monitoring.scrapeTargets = [{
-    job_name = "step-ca";
-    port = 9000;
-  }];
-  sops.secrets."ca_root_pw" = {
-    sopsFile = ../../secrets/ca/secrets.yaml;
-    owner = "step-ca";
-    path = "/var/lib/step-ca/secrets/ca_root_pw";
-  };
-  sops.secrets."intermediate_ca_key" = {
-    sopsFile = ../../secrets/ca/keys/intermediate_ca_key;
-    format = "binary";
-    owner = "step-ca";
-    path = "/var/lib/step-ca/secrets/intermediate_ca_key";
-  };
-  sops.secrets."root_ca_key" = {
-    sopsFile = ../../secrets/ca/keys/root_ca_key;
-    format = "binary";
-    owner = "step-ca";
-    path = "/var/lib/step-ca/secrets/root_ca_key";
-  };
-  sops.secrets."ssh_host_ca_key" = {
-    sopsFile = ../../secrets/ca/keys/ssh_host_ca_key;
-    format = "binary";
-    owner = "step-ca";
-    path = "/var/lib/step-ca/secrets/ssh_host_ca_key";
-  };
-  sops.secrets."ssh_user_ca_key" = {
-    sopsFile = ../../secrets/ca/keys/ssh_user_ca_key;
-    format = "binary";
-    owner = "step-ca";
-    path = "/var/lib/step-ca/secrets/ssh_user_ca_key";
-  };
-
-  services.step-ca = {
-    enable = true;
-    package = pkgs.step-ca;
-    intermediatePasswordFile = "/var/lib/step-ca/secrets/ca_root_pw";
-    address = "0.0.0.0";
-    port = 443;
-    settings = {
-      metricsAddress = ":9000";
-      authority = {
-        provisioners = [
-          {
-            claims = {
-              enableSSHCA = true;
-              maxTLSCertDuration = "3600h";
-              defaultTLSCertDuration = "48h";
-            };
-            encryptedKey = "eyJhbGciOiJQQkVTMi1IUzI1NitBMTI4S1ciLCJjdHkiOiJqd2sranNvbiIsImVuYyI6IkEyNTZHQ00iLCJwMmMiOjYwMDAwMCwicDJzIjoiY1lWOFJPb3lteXFLMWpzcS1WM1ZXQSJ9.WS8tPK-Q4gtnSsw7MhpTzYT_oi-SQx-CsRLh7KwdZnpACtd4YbcOYg.zeyDkmKRx8BIp-eB.OQ8c-KDW07gqJFtEMqHacRBkttrbJRRz0sYR47vQWDCoWhodaXsxM_Bj2pGvUrR26ij1t7irDeypnJoh6WXvUg3n_JaIUL4HgTwKSBrXZKTscXmY7YVmRMionhAb6oS9Jgus9K4QcFDHacC9_WgtGI7dnu3m0G7c-9Ur9dcDfROfyrnAByJp1rSZMzvriQr4t9bNYjDa8E8yu9zq6aAQqF0Xg_AxwiqYqesT-sdcfrxKS61appApRgPlAhW-uuzyY0wlWtsiyLaGlWM7WMfKdHsq-VqcVrI7Gi2i77vi7OqPEberqSt8D04tIri9S_sArKqWEDnBJsL07CC41IY.CqtYfbSa_wlmIsKgNj5u7g";
-            key = {
-              alg = "ES256";
-              crv = "P-256";
-              kid = "CIjtIe7FNhsNQe1qKGD9Rpj-lrf2ExyTYCXAOd3YDjE";
-              kty = "EC";
-              use = "sig";
-              x = "XRMX-BeobZ-R5-xb-E9YlaRjJUfd7JQxpscaF1NMgFo";
-              y = "bF9xLp5-jywRD-MugMaOGbpbniPituWSLMlXRJnUUl0";
-            };
-            name = "ca@home.2rjus.net";
-            type = "JWK";
-          }
-          {
-            name = "acme";
-            type = "ACME";
-            claims = {
-              maxTLSCertDuration = "3600h";
-              defaultTLSCertDuration = "1800h";
-            };
-          }
-          {
-            claims = {
-              enableSSHCA = true;
-            };
-            name = "sshpop";
-            type = "SSHPOP";
-          }
-        ];
-      };
-      crt = "/var/lib/step-ca/certs/intermediate_ca.crt";
-      db = {
-        badgerFileLoadingMode = "";
-        dataSource = "/var/lib/step-ca/db";
-        type = "badgerv2";
-      };
-      dnsNames = [
-        "ca.home.2rjus.net"
-        "10.69.13.12"
-      ];
-      federatedRoots = null;
-      insecureAddress = "";
-      key = "/var/lib/step-ca/secrets/intermediate_ca_key";
-      logger = {
-        format = "text";
-      };
-      root = "/var/lib/step-ca/certs/root_ca.crt";
-      ssh = {
-        hostKey = "/var/lib/step-ca/secrets/ssh_host_ca_key";
-        userKey = "/var/lib/step-ca/secrets/ssh_user_ca_key";
-      };
-      templates = {
-        ssh = {
-          host = [
-            {
-              comment = "#";
-              name = "sshd_config.tpl";
-              path = "/etc/ssh/sshd_config";
-              requires = [
-                "Certificate"
-                "Key"
-              ];
-              template = ./templates/ssh/sshd_config.tpl;
-              type = "snippet";
-            }
-            {
-              comment = "#";
-              name = "ca.tpl";
-              path = "/etc/ssh/ca.pub";
-              template = ./templates/ssh/ca.tpl;
-              type = "snippet";
-            }
-          ];
-          user = [
-            {
-              comment = "#";
-              name = "config.tpl";
-              path = "~/.ssh/config";
-              template = ./templates/ssh/config.tpl;
-              type = "snippet";
-            }
-            {
-              comment = "#";
-              name = "step_includes.tpl";
-              path = "\${STEPPATH}/ssh/includes";
-              template = ./templates/ssh/step_includes.tpl;
-              type = "prepend-line";
-            }
-            {
-              comment = "#";
-              name = "step_config.tpl";
-              path = "ssh/config";
-              template = ./templates/ssh/step_config.tpl;
-              type = "file";
-            }
-            {
-              comment = "#";
-              name = "known_hosts.tpl";
-              path = "ssh/known_hosts";
-              template = ./templates/ssh/known_hosts.tpl;
-              type = "file";
-            }
-          ];
-        };
-      };
-      tls = {
-        cipherSuites = [
-          "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256"
-          "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"
-        ];
-        maxVersion = 1.3;
-        minVersion = 1.2;
-        renegotiation = false;
-      };
-    };
-  };
-}

services/ca/templates/ssh/config.tpl

@@ -1,14 +0,0 @@
-Host *
-{{- if or .User.GOOS "none" | eq "windows" }}
-{{- if .User.StepBasePath }}
-	Include "{{ .User.StepBasePath | replace "\\" "/" | trimPrefix "C:" }}/ssh/includes"
-{{- else }}
-	Include "{{ .User.StepPath | replace "\\" "/" | trimPrefix "C:" }}/ssh/includes"
-{{- end }}
-{{- else }}
-{{- if .User.StepBasePath }}
-	Include "{{.User.StepBasePath}}/ssh/includes"
-{{- else }}
-	Include "{{.User.StepPath}}/ssh/includes"
-{{- end }}
-{{- end }}

services/ca/templates/ssh/known_hosts.tpl

@@ -1,4 +0,0 @@
-@cert-authority * {{.Step.SSH.HostKey.Type}} {{.Step.SSH.HostKey.Marshal | toString | b64enc}}
-{{- range .Step.SSH.HostFederatedKeys}}
-@cert-authority * {{.Type}} {{.Marshal | toString | b64enc}}
-{{- end }}

services/ca/templates/ssh/sshd_config.tpl

@@ -1,4 +0,0 @@
-Match all
-	TrustedUserCAKeys /etc/ssh/ca.pub
-	HostCertificate /etc/ssh/{{.User.Certificate}}
-	HostKey /etc/ssh/{{.User.Key}}

services/ca/templates/ssh/step_config.tpl

@@ -1,11 +0,0 @@
-Match exec "step ssh check-host{{- if .User.Context }} --context {{ .User.Context }}{{- end }} %h"
-{{- if .User.User }}
-	User {{.User.User}}
-{{- end }}
-{{- if or .User.GOOS "none" | eq "windows" }}
-	UserKnownHostsFile "{{.User.StepPath}}\ssh\known_hosts"
-	ProxyCommand C:\Windows\System32\cmd.exe /c step ssh proxycommand{{- if .User.Context }} --context {{ .User.Context }}{{- end }}{{- if .User.Provisioner }} --provisioner {{ .User.Provisioner }}{{- end }} %r %h %p
-{{- else }}
-	UserKnownHostsFile "{{.User.StepPath}}/ssh/known_hosts"
-	ProxyCommand step ssh proxycommand{{- if .User.Context }} --context {{ .User.Context }}{{- end }}{{- if .User.Provisioner }} --provisioner {{ .User.Provisioner }}{{- end }} %r %h %p
-{{- end }}

services/ca/templates/ssh/step_includes.tpl

@@ -1 +0,0 @@
-{{- if or .User.GOOS "none" | eq "windows" }}Include "{{ .User.StepPath | replace "\\" "/" | trimPrefix "C:" }}/ssh/config"{{- else }}Include "{{.User.StepPath}}/ssh/config"{{- end }}

services/http-proxy/proxy.nix

@@ -5,7 +5,7 @@
     package = pkgs.unstable.caddy;
     configFile = pkgs.writeText "Caddyfile" ''
       {
-        acme_ca https://ca.home.2rjus.net/acme/acme/directory
+        acme_ca https://vault.home.2rjus.net:8200/v1/pki_int/acme/directory
 
         metrics {
           per_host

services/monitoring/alloy.nix

@@ -1,41 +0,0 @@
-{ ... }:
-{
-  services.alloy = {
-    enable = true;
-  };
-
-  environment.etc."alloy/config.alloy" = {
-    enable = true;
-    mode = "0644";
-    text = ''
-      pyroscope.write "local_pyroscope" {
-        endpoint {
-          url = "http://localhost:4040"
-        }
-      }
-
-      pyroscope.scrape "labmon" {
-        targets    = [{"__address__" = "localhost:9969", "service_name" = "labmon"}]
-        forward_to = [pyroscope.write.local_pyroscope.receiver]
-
-        profiling_config {
-          profile.process_cpu {
-            enabled = true
-          }
-          profile.memory {
-            enabled = true
-          }
-          profile.mutex {
-            enabled = true
-          }
-          profile.block {
-            enabled = true
-          }
-          profile.goroutine {
-            enabled = true
-          }
-        }
-      }
-    '';
-  };
-}

services/monitoring/default.nix

@@ -7,7 +7,6 @@
     ./pve.nix
     ./alerttonotify.nix
     ./pyroscope.nix
-    ./alloy.nix
     ./tempo.nix
   ];
 }

services/monitoring/prometheus.nix

@@ -121,22 +121,20 @@ in
 
     scrapeConfigs = [
       # Auto-generated node-exporter targets from flake hosts + external
+      # Each static_config entry may have labels from homelab.host metadata
       {
         job_name = "node-exporter";
-        static_configs = [
-          {
-            targets = nodeExporterTargets;
-          }
-        ];
+        static_configs = nodeExporterTargets;
       }
       # Systemd exporter on all hosts (same targets, different port)
+      # Preserves the same label grouping as node-exporter
       {
         job_name = "systemd-exporter";
-        static_configs = [
-          {
-            targets = map (t: builtins.replaceStrings [":9100"] [":9558"] t) nodeExporterTargets;
-          }
-        ];
+        static_configs = map
+          (cfg: cfg // {
+            targets = map (t: builtins.replaceStrings [ ":9100" ] [ ":9558" ] t) cfg.targets;
+          })
+          nodeExporterTargets;
       }
       # Local monitoring services (not auto-generated)
       {
@@ -180,14 +178,6 @@ in
           }
         ];
       }
-      {
-        job_name = "labmon";
-        static_configs = [
-          {
-            targets = [ "monitoring01.home.2rjus.net:9969" ];
-          }
-        ];
-      }
       # TODO: nix-cache_caddy can't be auto-generated because the cert is issued
       # for nix-cache.home.2rjus.net (service CNAME), not nix-cache01 (hostname).
       # Consider adding a target override to homelab.monitoring.scrapeTargets.

services/monitoring/rules.yml

@@ -17,8 +17,9 @@ groups:
         annotations:
           summary: "Disk space low on {{ $labels.instance }}"
           description: "Disk space is low on {{ $labels.instance }}. Please check."
+      # Build hosts (e.g., nix-cache01) are expected to have high CPU during builds
       - alert: high_cpu_load
-        expr: max(node_load5{instance!="nix-cache01.home.2rjus.net:9100"}) by (instance) > (count by (instance)(node_cpu_seconds_total{instance!="nix-cache01.home.2rjus.net:9100", mode="idle"}) * 0.7)
+        expr: max(node_load5{role!="build-host"}) by (instance) > (count by (instance)(node_cpu_seconds_total{role!="build-host", mode="idle"}) * 0.7)
         for: 15m
         labels:
           severity: warning
@@ -26,7 +27,7 @@ groups:
           summary: "High CPU load on {{ $labels.instance }}"
           description: "CPU load is high on {{ $labels.instance }}. Please check."
       - alert: high_cpu_load
-        expr: max(node_load5{instance="nix-cache01.home.2rjus.net:9100"}) by (instance) > (count by (instance)(node_cpu_seconds_total{instance="nix-cache01.home.2rjus.net:9100", mode="idle"}) * 0.7)
+        expr: max(node_load5{role="build-host"}) by (instance) > (count by (instance)(node_cpu_seconds_total{role="build-host", mode="idle"}) * 0.7)
         for: 2h
         labels:
           severity: warning
@@ -115,8 +116,9 @@ groups:
         annotations:
           summary: "NSD not running on {{ $labels.instance }}"
           description: "NSD has been down on {{ $labels.instance }} more than 5 minutes."
+      # Only alert on primary DNS (secondary has cold cache after failover)
       - alert: unbound_low_cache_hit_ratio
-        expr: (rate(unbound_cache_hits_total[5m]) / (rate(unbound_cache_hits_total[5m]) + rate(unbound_cache_misses_total[5m]))) < 0.5
+        expr: (rate(unbound_cache_hits_total{dns_role="primary"}[5m]) / (rate(unbound_cache_hits_total{dns_role="primary"}[5m]) + rate(unbound_cache_misses_total{dns_role="primary"}[5m]))) < 0.5
         for: 15m
         labels:
           severity: warning
@@ -336,40 +338,6 @@ groups:
         annotations:
           summary: "Pyroscope service not running on {{ $labels.instance }}"
           description: "Pyroscope service not running on {{ $labels.instance }}"
-  - name: certificate_rules
-    rules:
-      - alert: certificate_expiring_soon
-        expr: labmon_tlsconmon_certificate_seconds_left{address!="ca.home.2rjus.net:443"} < 86400
-        for: 5m
-        labels:
-          severity: warning
-        annotations:
-          summary: "TLS certificate expiring soon for {{ $labels.instance }}"
-          description: "TLS certificate for {{ $labels.address }} is expiring within 24 hours."
-      - alert: step_ca_serving_cert_expiring
-        expr: labmon_tlsconmon_certificate_seconds_left{address="ca.home.2rjus.net:443"} < 3600
-        for: 5m
-        labels:
-          severity: critical
-        annotations:
-          summary: "Step-CA serving certificate expiring"
-          description: "The step-ca serving certificate (24h auto-renewed) has less than 1 hour of validity left. Renewal may have failed."
-      - alert: certificate_check_error
-        expr: labmon_tlsconmon_certificate_check_error == 1
-        for: 5m
-        labels:
-          severity: warning
-        annotations:
-          summary: "Error checking certificate for {{ $labels.address }}"
-          description: "Certificate check is failing for {{ $labels.address }} on {{ $labels.instance }}."
-      - alert: step_ca_certificate_expiring
-        expr: labmon_stepmon_certificate_seconds_left < 3600
-        for: 5m
-        labels:
-          severity: critical
-        annotations:
-          summary: "Step-CA certificate expiring for {{ $labels.instance }}"
-          description: "Step-CA certificate is expiring within 1 hour on {{ $labels.instance }}."
   - name: proxmox_rules
     rules:
       - alert: pve_node_down

services/nix-cache/proxy.nix

@@ -5,7 +5,7 @@
     package = pkgs.unstable.caddy;
     configFile = pkgs.writeText "Caddyfile" ''
       {
-        acme_ca https://ca.home.2rjus.net/acme/acme/directory
+        acme_ca https://vault.home.2rjus.net:8200/v1/pki_int/acme/directory
         metrics
       }
 

services/ns/resolver.nix

@@ -45,7 +45,11 @@
       };
       stub-zone = {
         name = "home.2rjus.net";
-        stub-addr = "127.0.0.1@8053";
+        stub-addr = [
+          "127.0.0.1@8053"   # Local NSD
+          "10.69.13.5@8053"  # ns1
+          "10.69.13.6@8053"  # ns2
+        ];
       };
       forward-zone = {
         name = ".";

system/acme.nix

@@ -3,7 +3,7 @@
   security.acme = {
     acceptTerms = true;
     defaults = {
-      server = "https://ca.home.2rjus.net/acme/acme/directory";
+      server = "https://vault.home.2rjus.net:8200/v1/pki_int/acme/directory";
       email = "root@home.2rjus.net";
       dnsPropagationCheck = false;
     };

system/default.nix

@@ -10,7 +10,6 @@
     ./nix.nix
     ./root-user.nix
     ./pki/root-ca.nix
-    ./sops.nix
     ./sshd.nix
     ./vault-secrets.nix
   ];

system/sops.nix

@@ -1,7 +0,0 @@
-{ ... }: {
-  sops = {
-    defaultSopsFile = ../secrets/secrets.yaml;
-    age.keyFile = "/var/lib/sops-nix/key.txt";
-    age.generateKey = true;
-  };
-}

terraform/variables.tf

@@ -33,7 +33,7 @@ variable "default_target_node" {
 variable "default_template_name" {
   description = "Default template VM name to clone from"
   type        = string
-  default     = "nixos-25.11.20260131.41e216c"
+  default     = "nixos-25.11.20260203.e576e3c"
 }
 
 variable "default_ssh_public_key" {

terraform/vault/approle.tf

@@ -80,13 +80,6 @@ locals {
       ]
     }
 
-    "ns2" = {
-      paths = [
-        "secret/data/hosts/ns2/*",
-        "secret/data/shared/dns/*",
-      ]
-    }
-
     # Wave 4: http-proxy
     "http-proxy" = {
       paths = [
@@ -101,11 +94,13 @@ locals {
       ]
     }
 
-    "vaulttest01" = {
+    # vault01: Vault server itself (fetches secrets from itself)
+    "vault01" = {
       paths = [
-        "secret/data/hosts/vaulttest01/*",
+        "secret/data/hosts/vault01/*",
       ]
     }
+
   }
 }
 

terraform/vault/hosts-generated.tf

@@ -5,12 +5,28 @@
 # Each host gets access to its own secrets under hosts/<hostname>/*
 locals {
   generated_host_policies = {
-    "vaulttest01" = {
+    "testvm01" = {
       paths = [
-        "secret/data/hosts/vaulttest01/*",
+        "secret/data/hosts/testvm01/*",
       ]
     }
-
+    "testvm02" = {
+      paths = [
+        "secret/data/hosts/testvm02/*",
+      ]
+    }
+    "testvm03" = {
+      paths = [
+        "secret/data/hosts/testvm03/*",
+      ]
+    }
+    "ns2" = {
+      paths = [
+        "secret/data/hosts/ns2/*",
+        "secret/data/shared/dns/*",
+      ]
+    }
+  
   }
 
   # Placeholder secrets - user should add actual secrets manually or via tofu
@@ -40,7 +56,7 @@ resource "vault_approle_auth_backend_role" "generated_hosts" {
 
   backend            = vault_auth_backend.approle.path
   role_name          = each.key
-  token_policies     = ["host-${each.key}"]
+  token_policies     = ["host-${each.key}", "homelab-deploy"]
   secret_id_ttl      = 0 # Never expire (wrapped tokens provide time limit)
   token_ttl          = 3600
   token_max_ttl      = 3600

terraform/vault/secrets.tf

@@ -45,12 +45,6 @@ locals {
       password_length = 24
     }
 
-    # TODO: Remove after testing
-    "hosts/vaulttest01/test-service" = {
-      auto_generate   = true
-      password_length = 32
-    }
-
     # Shared backup password (auto-generated, add alongside existing restic key)
     "shared/backup/password" = {
       auto_generate   = true

terraform/vms.tf

@@ -31,13 +31,6 @@ locals {
     # Example Minimal VM using all defaults (uncomment to deploy):
     # "minimal-vm" = {}
     # "bootstrap-verify-test" = {}
-    "testvm01" = {
-      ip           = "10.69.13.101/24"
-      cpu_cores    = 2
-      memory       = 2048
-      disk_size    = "20G"
-      flake_branch = "pipeline-testing-improvements"
-    }
     "vault01" = {
       ip           = "10.69.13.19/24"
       cpu_cores    = 2
@@ -45,13 +38,32 @@ locals {
       disk_size    = "20G"
       flake_branch = "vault-setup" # Bootstrap from this branch instead of master
     }
-    "vaulttest01" = {
-      ip                  = "10.69.13.150/24"
+    "testvm01" = {
+      ip                  = "10.69.13.20/24"
       cpu_cores           = 2
       memory              = 2048
       disk_size           = "20G"
-      flake_branch        = "pki-migration"
-      vault_wrapped_token = "s.UCpQCOp7cOKDdtGGBvfRWwAt"
+      flake_branch        = "improve-bootstrap-visibility"
+      vault_wrapped_token = "s.l5q88wzXfEcr5SMDHmO6o96b"
+    }
+    "testvm02" = {
+      ip        = "10.69.13.21/24"
+      cpu_cores = 2
+      memory    = 2048
+      disk_size = "20G"
+    }
+    "testvm03" = {
+      ip        = "10.69.13.22/24"
+      cpu_cores = 2
+      memory    = 2048
+      disk_size = "20G"
+    }
+    "ns2" = {
+      ip        = "10.69.13.6/24"
+      cpu_cores = 2
+      memory    = 2048
+      disk_size = "20G"
+      vault_wrapped_token = "s.3nran1e1Uim4B1OomIWCoS4T"
     }
   }