nix-cache02: add scheduled builds timer

Add a systemd timer that triggers builds for all hosts every 2 hours
via NATS, keeping the binary cache warm.

- Add scheduler.nix with timer (every 2h) and oneshot service
- Add scheduler NATS user to DEPLOY account
- Add Vault secret and variable for scheduler NKey
- Increase nix-cache02 memory from 16GB to 20GB

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

docs/plans/nix-cache-reprovision.md

@@ -12,6 +12,7 @@ Reprovision `nix-cache01` using the OpenTofu workflow, and improve the build/cac
 **Phase 2: NATS Build Triggering** - COMPLETE
 **Phase 3: Safe Flake Update Workflow** - NOT STARTED
 **Phase 4: Complete Migration** - COMPLETE
+**Phase 5: Scheduled Builds** - COMPLETE
 
 ## Completed Work
 
@@ -103,24 +104,20 @@ The `homelab-deploy` tool was extended with a builder mode:
    - Removed from `flake.nix`
    - Deleted VM from Proxmox
 
-### Phase 5: Scheduled Builds (Optional)
+### Phase 5: Scheduled Builds ✅
 
-Add a systemd timer on nix-cache02 to trigger periodic builds via NATS:
+Implemented a systemd timer on nix-cache02 that triggers builds every 2 hours:
 
-```nix
+- **Timer**: `scheduled-build.timer` runs every 2 hours with 5m random jitter
-systemd.services.scheduled-build = {
+- **Service**: `scheduled-build.service` calls `homelab-deploy build` for both repos
-  script = ''
+- **Authentication**: Dedicated scheduler NKey stored in Vault
-    homelab-deploy build nixos-servers --all
+- **NATS user**: Added to DEPLOY account with publish `build.>` and subscribe `build.responses.>`
-    homelab-deploy build nixos --all
-  '';
-};
-systemd.timers.scheduled-build = {
-  wantedBy = [ "timers.target" ];
-  timerConfig.OnCalendar = "*-*-* *:30:00";
-};
-```
 
-Or trigger builds from CI after merges to master.
+Files:
+- `hosts/nix-cache02/scheduler.nix` - Timer and service configuration
+- `services/nats/default.nix` - Scheduler NATS user
+- `terraform/vault/secrets.tf` - Scheduler NKey secret
+- `terraform/vault/variables.tf` - Variable for scheduler NKey
 
 ## Resolved Questions
 

hosts/nix-cache02/default.nix

@@ -2,6 +2,7 @@
   imports = [
     ./configuration.nix
     ./builder.nix
+    ./scheduler.nix
     ../../services/nix-cache
   ];
 }

hosts/nix-cache02/scheduler.nix

@@ -0,0 +1,61 @@
+{ config, pkgs, lib, inputs, ... }:
+let
+  homelab-deploy = inputs.homelab-deploy.packages.${pkgs.system}.default;
+
+  scheduledBuildScript = pkgs.writeShellApplication {
+    name = "scheduled-build";
+    runtimeInputs = [ homelab-deploy ];
+    text = ''
+      NATS_URL="nats://nats1.home.2rjus.net:4222"
+      NKEY_FILE="/run/secrets/scheduler-nkey"
+
+      echo "Starting scheduled builds at $(date)"
+
+      # Build all nixos-servers hosts
+      homelab-deploy build \
+        --nats-url "$NATS_URL" \
+        --nkey-file "$NKEY_FILE" \
+        nixos-servers --all
+
+      # Build all nixos (gunter) hosts
+      homelab-deploy build \
+        --nats-url "$NATS_URL" \
+        --nkey-file "$NKEY_FILE" \
+        nixos --all
+
+      echo "Scheduled builds completed at $(date)"
+    '';
+  };
+in
+{
+  # Fetch scheduler NKey from Vault
+  vault.secrets.scheduler-nkey = {
+    secretPath = "shared/homelab-deploy/scheduler-nkey";
+    extractKey = "nkey";
+    outputDir = "/run/secrets/scheduler-nkey";
+    services = [ "scheduled-build" ];
+  };
+
+  # Timer: every 2 hours
+  systemd.timers.scheduled-build = {
+    description = "Trigger scheduled Nix builds";
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnCalendar = "*-*-* 00/2:00:00"; # Every 2 hours at :00
+      Persistent = true; # Run missed builds on boot
+      RandomizedDelaySec = "5m"; # Slight jitter
+    };
+  };
+
+  # Service: oneshot that triggers builds
+  systemd.services.scheduled-build = {
+    description = "Trigger builds for all hosts via NATS";
+    after = [ "network-online.target" "vault-secret-scheduler-nkey.service" ];
+    requires = [ "vault-secret-scheduler-nkey.service" ];
+    wants = [ "network-online.target" ];
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = lib.getExe scheduledBuildScript;
+    };
+  };
+}

services/nats/default.nix

@@ -105,6 +105,14 @@
                 publish = [ "build.responses.>" ];
               };
             }
+            # Scheduler (publishes build requests, subscribes to responses)
+            {
+              nkey = "UDQ5SFEGDM66AQGLK7KQDW6ZOC2QCXE2P6EJQ6VPBSR2CRCABPOVWRI4";
+              permissions = {
+                publish = [ "build.>" ];
+                subscribe = [ "build.responses.>" ];
+              };
+            }
           ];
         };
       };

terraform/vault/secrets.tf

@@ -102,6 +102,11 @@ locals {
       data          = { nkey = var.homelab_deploy_builder_nkey }
     }
 
+    "shared/homelab-deploy/scheduler-nkey" = {
+      auto_generate = false
+      data          = { nkey = var.homelab_deploy_scheduler_nkey }
+    }
+
     # Kanidm idm_admin password
     "kanidm/idm-admin-password" = {
       auto_generate   = true

terraform/vault/variables.tf

@@ -74,6 +74,13 @@ variable "homelab_deploy_builder_nkey" {
   sensitive   = true
 }
 
+variable "homelab_deploy_scheduler_nkey" {
+  description = "NKey seed for scheduled build triggering"
+  type        = string
+  default     = "PLACEHOLDER"
+  sensitive   = true
+}
+
 variable "nixos_exporter_nkey" {
   description = "NKey seed for nixos-exporter NATS authentication"
   type        = string

terraform/vms.tf

@@ -89,7 +89,7 @@ locals {
     "nix-cache02" = {
       ip        = "10.69.13.25/24"
       cpu_cores = 8
-      memory    = 16384
+      memory    = 20480
       disk_size = "200G"
       vault_wrapped_token = "s.C5EuHFyULACEqZgsLqMC3cJB"
     }