diff --git a/docs/plans/nix-cache-reprovision.md b/docs/plans/nix-cache-reprovision.md index 4f4a925..ea8f9c6 100644 --- a/docs/plans/nix-cache-reprovision.md +++ b/docs/plans/nix-cache-reprovision.md @@ -12,6 +12,7 @@ Reprovision `nix-cache01` using the OpenTofu workflow, and improve the build/cac **Phase 2: NATS Build Triggering** - COMPLETE **Phase 3: Safe Flake Update Workflow** - NOT STARTED **Phase 4: Complete Migration** - COMPLETE +**Phase 5: Scheduled Builds** - COMPLETE ## Completed Work @@ -103,24 +104,20 @@ The `homelab-deploy` tool was extended with a builder mode: - Removed from `flake.nix` - Deleted VM from Proxmox -### Phase 5: Scheduled Builds (Optional) +### Phase 5: Scheduled Builds ✅ -Add a systemd timer on nix-cache02 to trigger periodic builds via NATS: +Implemented a systemd timer on nix-cache02 that triggers builds every 2 hours: -```nix -systemd.services.scheduled-build = { - script = '' - homelab-deploy build nixos-servers --all - homelab-deploy build nixos --all - ''; -}; -systemd.timers.scheduled-build = { - wantedBy = [ "timers.target" ]; - timerConfig.OnCalendar = "*-*-* *:30:00"; -}; -``` +- **Timer**: `scheduled-build.timer` runs every 2 hours with 5m random jitter +- **Service**: `scheduled-build.service` calls `homelab-deploy build` for both repos +- **Authentication**: Dedicated scheduler NKey stored in Vault +- **NATS user**: Added to DEPLOY account with publish `build.>` and subscribe `build.responses.>` -Or trigger builds from CI after merges to master. +Files: +- `hosts/nix-cache02/scheduler.nix` - Timer and service configuration +- `services/nats/default.nix` - Scheduler NATS user +- `terraform/vault/secrets.tf` - Scheduler NKey secret +- `terraform/vault/variables.tf` - Variable for scheduler NKey ## Resolved Questions diff --git a/hosts/nix-cache02/default.nix b/hosts/nix-cache02/default.nix index e3d1fad..96e0439 100644 --- a/hosts/nix-cache02/default.nix +++ b/hosts/nix-cache02/default.nix @@ -2,6 +2,7 @@ imports = [ ./configuration.nix ./builder.nix + ./scheduler.nix ../../services/nix-cache ]; } \ No newline at end of file diff --git a/hosts/nix-cache02/scheduler.nix b/hosts/nix-cache02/scheduler.nix new file mode 100644 index 0000000..909097d --- /dev/null +++ b/hosts/nix-cache02/scheduler.nix @@ -0,0 +1,61 @@ +{ config, pkgs, lib, inputs, ... }: +let + homelab-deploy = inputs.homelab-deploy.packages.${pkgs.system}.default; + + scheduledBuildScript = pkgs.writeShellApplication { + name = "scheduled-build"; + runtimeInputs = [ homelab-deploy ]; + text = '' + NATS_URL="nats://nats1.home.2rjus.net:4222" + NKEY_FILE="/run/secrets/scheduler-nkey" + + echo "Starting scheduled builds at $(date)" + + # Build all nixos-servers hosts + homelab-deploy build \ + --nats-url "$NATS_URL" \ + --nkey-file "$NKEY_FILE" \ + nixos-servers --all + + # Build all nixos (gunter) hosts + homelab-deploy build \ + --nats-url "$NATS_URL" \ + --nkey-file "$NKEY_FILE" \ + nixos --all + + echo "Scheduled builds completed at $(date)" + ''; + }; +in +{ + # Fetch scheduler NKey from Vault + vault.secrets.scheduler-nkey = { + secretPath = "shared/homelab-deploy/scheduler-nkey"; + extractKey = "nkey"; + outputDir = "/run/secrets/scheduler-nkey"; + services = [ "scheduled-build" ]; + }; + + # Timer: every 2 hours + systemd.timers.scheduled-build = { + description = "Trigger scheduled Nix builds"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "*-*-* 00/2:00:00"; # Every 2 hours at :00 + Persistent = true; # Run missed builds on boot + RandomizedDelaySec = "5m"; # Slight jitter + }; + }; + + # Service: oneshot that triggers builds + systemd.services.scheduled-build = { + description = "Trigger builds for all hosts via NATS"; + after = [ "network-online.target" "vault-secret-scheduler-nkey.service" ]; + requires = [ "vault-secret-scheduler-nkey.service" ]; + wants = [ "network-online.target" ]; + serviceConfig = { + Type = "oneshot"; + ExecStart = lib.getExe scheduledBuildScript; + }; + }; +} diff --git a/services/nats/default.nix b/services/nats/default.nix index b657f0a..e7f9ccf 100644 --- a/services/nats/default.nix +++ b/services/nats/default.nix @@ -105,6 +105,14 @@ publish = [ "build.responses.>" ]; }; } + # Scheduler (publishes build requests, subscribes to responses) + { + nkey = "UDQ5SFEGDM66AQGLK7KQDW6ZOC2QCXE2P6EJQ6VPBSR2CRCABPOVWRI4"; + permissions = { + publish = [ "build.>" ]; + subscribe = [ "build.responses.>" ]; + }; + } ]; }; }; diff --git a/terraform/vault/secrets.tf b/terraform/vault/secrets.tf index f29e607..4ac9ec8 100644 --- a/terraform/vault/secrets.tf +++ b/terraform/vault/secrets.tf @@ -102,6 +102,11 @@ locals { data = { nkey = var.homelab_deploy_builder_nkey } } + "shared/homelab-deploy/scheduler-nkey" = { + auto_generate = false + data = { nkey = var.homelab_deploy_scheduler_nkey } + } + # Kanidm idm_admin password "kanidm/idm-admin-password" = { auto_generate = true diff --git a/terraform/vault/variables.tf b/terraform/vault/variables.tf index 10b9f72..a0ea1a3 100644 --- a/terraform/vault/variables.tf +++ b/terraform/vault/variables.tf @@ -74,6 +74,13 @@ variable "homelab_deploy_builder_nkey" { sensitive = true } +variable "homelab_deploy_scheduler_nkey" { + description = "NKey seed for scheduled build triggering" + type = string + default = "PLACEHOLDER" + sensitive = true +} + variable "nixos_exporter_nkey" { description = "NKey seed for nixos-exporter NATS authentication" type = string diff --git a/terraform/vms.tf b/terraform/vms.tf index e738935..ef1551d 100644 --- a/terraform/vms.tf +++ b/terraform/vms.tf @@ -89,7 +89,7 @@ locals { "nix-cache02" = { ip = "10.69.13.25/24" cpu_cores = 8 - memory = 16384 + memory = 20480 disk_size = "200G" vault_wrapped_token = "s.C5EuHFyULACEqZgsLqMC3cJB" }