diff --git a/TODO.md b/TODO.md index 70e81a5..5ee72c5 100644 --- a/TODO.md +++ b/TODO.md @@ -54,6 +54,7 @@ Automate the entire process of creating, configuring, and deploying new NixOS ho **Status:** ✅ Fully implemented and tested **Completed:** 2025-02-01 +**Enhanced:** 2025-02-01 (added --force flag) **Goal:** Automate creation of host configuration files @@ -64,6 +65,7 @@ Automate the entire process of creating, configuring, and deploying new NixOS ho - Comprehensive validation (hostname format/uniqueness, IP subnet/uniqueness) - Jinja2 templates for NixOS configurations - Automatic updates to flake.nix and terraform/vms.tf +- `--force` flag for regenerating existing configurations (useful for testing) **Tasks:** - [x] Create Python CLI with typer framework @@ -109,6 +111,7 @@ create-host \ **Status:** ✅ Fully implemented and tested **Completed:** 2025-02-01 +**Enhanced:** 2025-02-01 (added branch support for testing) **Goal:** Get freshly deployed VM to apply its specific host configuration @@ -118,7 +121,8 @@ create-host \ - Systemd service `nixos-bootstrap.service` runs on first boot - Depends on `cloud-config.service` to ensure hostname is set - Reads hostname from `hostnamectl` (set by cloud-init via Terraform) -- Runs `nixos-rebuild boot --flake git+https://git.t-juice.club/torjus/nixos-servers.git#${hostname}` +- Supports custom git branch via `NIXOS_FLAKE_BRANCH` environment variable +- Runs `nixos-rebuild boot --flake git+https://git.t-juice.club/torjus/nixos-servers.git?ref=$BRANCH#${hostname}` - Reboots into new configuration on success - Fails gracefully without reboot on errors (network issues, missing config) - Service self-destructs after successful bootstrap (not in new config) @@ -240,10 +244,80 @@ Since most hosts use static IPs defined in their NixOS configurations, we can ex ### Phase 7: Testing & Documentation -**Tasks:** -- [ ] Test full pipeline end-to-end -- [ ] Create test host and verify all steps -- [ ] Document the new workflow in CLAUDE.md +**Status:** 🚧 In Progress (testing improvements completed) + +**Testing Improvements Implemented (2025-02-01):** + +The pipeline now supports efficient testing without polluting master branch: + +**1. --force Flag for create-host** +- Re-run `create-host` to regenerate existing configurations +- Updates existing entries in flake.nix and terraform/vms.tf (no duplicates) +- Skip uniqueness validation checks +- Useful for iterating on configuration templates during testing + +**2. Branch Support for Bootstrap** +- Bootstrap service reads `NIXOS_FLAKE_BRANCH` environment variable +- Defaults to `master` if not set +- Allows testing pipeline changes on feature branches +- Cloud-init passes branch via `/etc/environment` + +**3. Cloud-init Disk for Branch Configuration** +- Terraform generates custom cloud-init snippets for test VMs +- Set `flake_branch` field in VM definition to use non-master branch +- Production VMs omit this field and use master (default) +- Files automatically uploaded to Proxmox via SSH + +**Testing Workflow:** + +```bash +# 1. Create test branch +git checkout -b test-pipeline + +# 2. Generate or update host config +create-host --hostname testvm01 --ip 10.69.13.100/24 + +# 3. Edit terraform/vms.tf to add test VM with branch +# vms = { +# "testvm01" = { +# ip = "10.69.13.100/24" +# flake_branch = "test-pipeline" # Bootstrap from this branch +# } +# } + +# 4. Commit and push test branch +git add -A && git commit -m "test: add testvm01" +git push origin test-pipeline + +# 5. Deploy VM +cd terraform && tofu apply + +# 6. Watch bootstrap (VM fetches from test-pipeline branch) +ssh root@10.69.13.100 +journalctl -fu nixos-bootstrap.service + +# 7. Iterate: modify templates and regenerate with --force +cd .. && create-host --hostname testvm01 --ip 10.69.13.100/24 --force +git commit -am "test: update config" && git push + +# Redeploy to test fresh bootstrap +cd terraform +tofu destroy -target=proxmox_vm_qemu.vm[\"testvm01\"] && tofu apply + +# 8. Clean up when done: squash commits, merge to master, remove test VM +``` + +**Files:** +- `scripts/create-host/create_host.py` - Added --force parameter +- `scripts/create-host/manipulators.py` - Update vs insert logic +- `hosts/template2/bootstrap.nix` - Branch support via environment variable +- `terraform/vms.tf` - flake_branch field support +- `terraform/cloud-init.tf` - Custom cloud-init disk generation +- `terraform/variables.tf` - proxmox_host variable for SSH uploads + +**Remaining Tasks:** +- [ ] Test full pipeline end-to-end on feature branch +- [ ] Update CLAUDE.md with testing workflow - [ ] Add troubleshooting section - [ ] Create examples for common scenarios (DHCP host, static IP host, etc.) diff --git a/flake.nix b/flake.nix index f585f52..05d7a18 100644 --- a/flake.nix +++ b/flake.nix @@ -334,6 +334,22 @@ sops-nix.nixosModules.sops ]; }; + testvm01 = nixpkgs.lib.nixosSystem { + inherit system; + specialArgs = { + inherit inputs self sops-nix; + }; + modules = [ + ( + { config, pkgs, ... }: + { + nixpkgs.overlays = commonOverlays; + } + ) + ./hosts/testvm01 + sops-nix.nixosModules.sops + ]; + }; }; packages = forAllSystems ( { pkgs }: diff --git a/hosts/template2/bootstrap.nix b/hosts/template2/bootstrap.nix index da660f0..bbbe8cc 100644 --- a/hosts/template2/bootstrap.nix +++ b/hosts/template2/bootstrap.nix @@ -24,8 +24,12 @@ let echo "Network connectivity confirmed" echo "Fetching and building NixOS configuration from flake..." + # Read git branch from environment, default to master + BRANCH="''${NIXOS_FLAKE_BRANCH:-master}" + echo "Using git branch: $BRANCH" + # Build and activate the host-specific configuration - FLAKE_URL="git+https://git.t-juice.club/torjus/nixos-servers.git#''${HOSTNAME}" + FLAKE_URL="git+https://git.t-juice.club/torjus/nixos-servers.git?ref=$BRANCH#''${HOSTNAME}" if nixos-rebuild boot --flake "$FLAKE_URL"; then echo "Successfully built configuration for $HOSTNAME" @@ -58,6 +62,9 @@ in RemainAfterExit = true; ExecStart = "${bootstrap-script}/bin/nixos-bootstrap"; + # Read environment variables from /etc/environment (set by cloud-init) + EnvironmentFile = "-/etc/environment"; + # Logging to journald StandardOutput = "journal+console"; StandardError = "journal+console"; diff --git a/hosts/testvm01/configuration.nix b/hosts/testvm01/configuration.nix new file mode 100644 index 0000000..f5b0fdf --- /dev/null +++ b/hosts/testvm01/configuration.nix @@ -0,0 +1,61 @@ +{ + config, + lib, + pkgs, + ... +}: + +{ + imports = [ + ../template2/hardware-configuration.nix + + ../../system + ../../common/vm + ]; + + nixpkgs.config.allowUnfree = true; + boot.loader.grub.enable = true; + boot.loader.grub.device = "/dev/vda"; + + networking.hostName = "testvm01"; + networking.domain = "home.2rjus.net"; + networking.useNetworkd = true; + networking.useDHCP = false; + services.resolved.enable = false; + networking.nameservers = [ + "10.69.13.5" + "10.69.13.6" + ]; + + systemd.network.enable = true; + systemd.network.networks."ens18" = { + matchConfig.Name = "ens18"; + address = [ + "10.69.13.101/24" + ]; + routes = [ + { Gateway = "10.69.13.1"; } + ]; + linkConfig.RequiredForOnline = "routable"; + }; + time.timeZone = "Europe/Oslo"; + + nix.settings.experimental-features = [ + "nix-command" + "flakes" + ]; + nix.settings.tarball-ttl = 0; + environment.systemPackages = with pkgs; [ + vim + wget + git + ]; + + # Open ports in the firewall. + # networking.firewall.allowedTCPPorts = [ ... ]; + # networking.firewall.allowedUDPPorts = [ ... ]; + # Or disable the firewall altogether. + networking.firewall.enable = false; + + system.stateVersion = "25.11"; # Did you read the comment? +} \ No newline at end of file diff --git a/hosts/testvm01/default.nix b/hosts/testvm01/default.nix new file mode 100644 index 0000000..57ed4b4 --- /dev/null +++ b/hosts/testvm01/default.nix @@ -0,0 +1,5 @@ +{ ... }: { + imports = [ + ./configuration.nix + ]; +} \ No newline at end of file diff --git a/scripts/create-host/README.md b/scripts/create-host/README.md index 3169287..18c4b2d 100644 --- a/scripts/create-host/README.md +++ b/scripts/create-host/README.md @@ -50,6 +50,23 @@ python -m scripts.create_host.create_host create \ --dry-run ``` +### Force Mode (Regenerate Existing Configuration) + +Overwrite an existing host configuration (useful for testing): + +```bash +python -m scripts.create_host.create_host create \ + --hostname test01 \ + --ip 10.69.13.50/24 \ + --force +``` + +This mode: +- Skips hostname and IP uniqueness validation +- Overwrites files in `hosts//` +- Updates existing entries in `flake.nix` and `terraform/vms.tf` (doesn't duplicate) +- Useful for iterating on configuration templates during testing + ### Options - `--hostname` (required): Hostname for the new host @@ -73,6 +90,10 @@ python -m scripts.create_host.create_host create \ - `--dry-run` (flag): Preview changes without creating files +- `--force` (flag): Overwrite existing host configuration + - Skips uniqueness validation + - Updates existing entries instead of creating duplicates + ## What It Does The tool performs the following actions: diff --git a/scripts/create-host/create_host.py b/scripts/create-host/create_host.py index 7756444..22ad641 100644 --- a/scripts/create-host/create_host.py +++ b/scripts/create-host/create_host.py @@ -45,6 +45,7 @@ def main( memory: int = typer.Option(2048, "--memory", help="Memory in MB"), disk: str = typer.Option("20G", "--disk", help="Disk size (e.g., 20G, 50G, 100G)"), dry_run: bool = typer.Option(False, "--dry-run", help="Preview changes without creating files"), + force: bool = typer.Option(False, "--force", help="Overwrite existing host configuration"), ) -> None: """ Create a new NixOS host configuration. @@ -75,11 +76,20 @@ def main( config.validate() validate_hostname_format(hostname) - validate_hostname_unique(hostname, repo_root) + + # Skip uniqueness checks in force mode + if not force: + validate_hostname_unique(hostname, repo_root) + if ip: + validate_ip_unique(ip, repo_root) + else: + # Check if we're actually overwriting something + host_dir = repo_root / "hosts" / hostname + if host_dir.exists(): + console.print(f"[yellow]⚠[/yellow] Updating existing host configuration for {hostname}") if ip: validate_ip_subnet(ip) - validate_ip_unique(ip, repo_root) console.print("[green]✓[/green] All validations passed\n") @@ -96,13 +106,14 @@ def main( console.print("\n[bold blue]Generating host configuration...[/bold blue]") generate_host_files(config, repo_root) - console.print(f"[green]✓[/green] Created hosts/{hostname}/default.nix") - console.print(f"[green]✓[/green] Created hosts/{hostname}/configuration.nix") + action = "Updated" if force else "Created" + console.print(f"[green]✓[/green] {action} hosts/{hostname}/default.nix") + console.print(f"[green]✓[/green] {action} hosts/{hostname}/configuration.nix") - update_flake_nix(config, repo_root) + update_flake_nix(config, repo_root, force=force) console.print("[green]✓[/green] Updated flake.nix") - update_terraform_vms(config, repo_root) + update_terraform_vms(config, repo_root, force=force) console.print("[green]✓[/green] Updated terraform/vms.tf") # Success message diff --git a/scripts/create-host/manipulators.py b/scripts/create-host/manipulators.py index b0dca17..366e215 100644 --- a/scripts/create-host/manipulators.py +++ b/scripts/create-host/manipulators.py @@ -6,21 +6,18 @@ from pathlib import Path from models import HostConfig -def update_flake_nix(config: HostConfig, repo_root: Path) -> None: +def update_flake_nix(config: HostConfig, repo_root: Path, force: bool = False) -> None: """ - Add new host entry to flake.nix nixosConfigurations. + Add or update host entry in flake.nix nixosConfigurations. Args: config: Host configuration repo_root: Path to repository root + force: If True, replace existing entry; if False, insert new entry """ flake_path = repo_root / "flake.nix" content = flake_path.read_text() - # Find the closing of nixosConfigurations block - # Pattern: " };\n packages =" - pattern = r"( \};)\n( packages =)" - # Create new entry new_entry = f""" {config.hostname} = nixpkgs.lib.nixosSystem {{ inherit system; @@ -40,35 +37,47 @@ def update_flake_nix(config: HostConfig, repo_root: Path) -> None: }}; """ - # Insert new entry before closing brace - replacement = rf"\g<1>\n{new_entry}\g<2>" + # Check if hostname already exists + hostname_pattern = rf"^ {re.escape(config.hostname)} = nixpkgs\.lib\.nixosSystem" + existing_match = re.search(hostname_pattern, content, re.MULTILINE) - new_content, count = re.subn(pattern, replacement, content) + if existing_match and force: + # Replace existing entry + # Match the entire block from "hostname = " to "};" + replace_pattern = rf"^ {re.escape(config.hostname)} = nixpkgs\.lib\.nixosSystem \{{.*?^ \}};\n" + new_content, count = re.subn(replace_pattern, new_entry, content, flags=re.MULTILINE | re.DOTALL) - if count == 0: - raise ValueError( - "Could not find insertion point in flake.nix. " - "Looking for pattern: ' };\\n devShells ='" - ) + if count == 0: + raise ValueError(f"Could not find existing entry for {config.hostname} in flake.nix") + else: + # Insert new entry before closing brace of nixosConfigurations + # Pattern: " };\n packages = forAllSystems" + pattern = r"( \};)\n( packages = forAllSystems)" + replacement = rf"{new_entry}\g<1>\n\g<2>" + + new_content, count = re.subn(pattern, replacement, content) + + if count == 0: + raise ValueError( + "Could not find insertion point in flake.nix. " + "Looking for pattern: ' };\\n packages = forAllSystems'" + ) flake_path.write_text(new_content) -def update_terraform_vms(config: HostConfig, repo_root: Path) -> None: +def update_terraform_vms(config: HostConfig, repo_root: Path, force: bool = False) -> None: """ - Add new VM entry to terraform/vms.tf locals.vms map. + Add or update VM entry in terraform/vms.tf locals.vms map. Args: config: Host configuration repo_root: Path to repository root + force: If True, replace existing entry; if False, insert new entry """ terraform_path = repo_root / "terraform" / "vms.tf" content = terraform_path.read_text() - # Find the closing of locals.vms block - # Pattern: " }\n\n # Compute VM configurations" - pattern = r"( \})\n\n( # Compute VM configurations)" - # Create new entry based on whether we have static IP or DHCP if config.is_static_ip: new_entry = f''' "{config.hostname}" = {{ @@ -86,15 +95,30 @@ def update_terraform_vms(config: HostConfig, repo_root: Path) -> None: }} ''' - # Insert new entry before closing brace - replacement = rf"{new_entry}\g<1>\n\n\g<2>" + # Check if hostname already exists + hostname_pattern = rf'^\s+"{re.escape(config.hostname)}" = \{{' + existing_match = re.search(hostname_pattern, content, re.MULTILINE) - new_content, count = re.subn(pattern, replacement, content) + if existing_match and force: + # Replace existing entry + # Match the entire block from "hostname" = { to } + replace_pattern = rf'^\s+"{re.escape(config.hostname)}" = \{{.*?^\s+\}}\n' + new_content, count = re.subn(replace_pattern, new_entry, content, flags=re.MULTILINE | re.DOTALL) - if count == 0: - raise ValueError( - "Could not find insertion point in terraform/vms.tf. " - "Looking for pattern: ' }\\n\\n # Compute VM configurations'" - ) + if count == 0: + raise ValueError(f"Could not find existing entry for {config.hostname} in terraform/vms.tf") + else: + # Insert new entry before closing brace + # Pattern: " }\n\n # Compute VM configurations" + pattern = r"( \})\n\n( # Compute VM configurations)" + replacement = rf"{new_entry}\g<1>\n\n\g<2>" + + new_content, count = re.subn(pattern, replacement, content) + + if count == 0: + raise ValueError( + "Could not find insertion point in terraform/vms.tf. " + "Looking for pattern: ' }\\n\\n # Compute VM configurations'" + ) terraform_path.write_text(new_content) diff --git a/scripts/create-host/templates/configuration.nix.j2 b/scripts/create-host/templates/configuration.nix.j2 index 1665e67..30e830f 100644 --- a/scripts/create-host/templates/configuration.nix.j2 +++ b/scripts/create-host/templates/configuration.nix.j2 @@ -7,16 +7,15 @@ { imports = [ - ../template/hardware-configuration.nix + ../template2/hardware-configuration.nix ../../system ../../common/vm ]; nixpkgs.config.allowUnfree = true; - # Use the systemd-boot EFI boot loader. boot.loader.grub.enable = true; - boot.loader.grub.device = "/dev/sda"; + boot.loader.grub.device = "/dev/vda"; networking.hostName = "{{ hostname }}"; networking.domain = "{{ domain }}"; diff --git a/terraform/README.md b/terraform/README.md index 2cca37f..d695c54 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -87,6 +87,21 @@ vms = { } ``` +### Example: Test VM with Custom Git Branch + +For testing pipeline changes without polluting master: + +```hcl +vms = { + "test-vm" = { + ip = "10.69.13.100/24" + flake_branch = "test-pipeline" # Bootstrap from this branch + } +} +``` + +This VM will bootstrap from the `test-pipeline` branch instead of `master`. Production VMs should omit the `flake_branch` field. + ## Configuration Options Each VM in the `vms` map supports the following fields (all optional): @@ -98,6 +113,7 @@ Each VM in the `vms` map supports the following fields (all optional): | `cpu_cores` | Number of CPU cores | `2` | | `memory` | Memory in MB | `2048` | | `disk_size` | Disk size (e.g., "20G", "100G") | `"20G"` | +| `flake_branch` | Git branch for bootstrap (for testing, omit for production) | `master` | | `target_node` | Proxmox node to deploy to | `"pve1"` | | `template_name` | Template VM to clone from | `"nixos-25.11.20260128.fa83fd8"` | | `storage` | Storage backend | `"local-zfs"` | @@ -182,6 +198,7 @@ deployment_summary = { - `main.tf` - Provider configuration - `variables.tf` - Variable definitions and defaults - `vms.tf` - VM definitions and deployment logic +- `cloud-init.tf` - Cloud-init disk management (SSH keys, networking, branch config) - `outputs.tf` - Output definitions for deployed VMs - `terraform.tfvars.example` - Example credentials file - `terraform.tfvars` - Your actual credentials (gitignored) diff --git a/terraform/cloud-init.tf b/terraform/cloud-init.tf new file mode 100644 index 0000000..5ca7fe7 --- /dev/null +++ b/terraform/cloud-init.tf @@ -0,0 +1,58 @@ +# Cloud-init configuration for all VMs +# +# This file manages cloud-init disks for all VMs using the proxmox_cloud_init_disk resource. +# VMs with flake_branch set will include NIXOS_FLAKE_BRANCH environment variable. + +resource "proxmox_cloud_init_disk" "ci" { + for_each = local.vm_configs + + name = each.key + pve_node = each.value.target_node + storage = "local" # Cloud-init disks must be on storage that supports ISO/snippets + + # User data includes SSH keys and optionally NIXOS_FLAKE_BRANCH + user_data = <<-EOT + #cloud-config + ssh_authorized_keys: + - ${each.value.ssh_public_key} + ${each.value.flake_branch != null ? <<-BRANCH + write_files: + - path: /etc/environment + content: | + NIXOS_FLAKE_BRANCH=${each.value.flake_branch} + append: true + BRANCH +: ""} + EOT + + # Network configuration - static IP or DHCP + network_config = each.value.ip != null ? yamlencode({ + version = 1 + config = [{ + type = "physical" + name = "ens18" + subnets = [{ + type = "static" + address = each.value.ip + gateway = each.value.gateway + dns_nameservers = split(" ", each.value.nameservers) + dns_search = [each.value.search_domain] + }] + }] + }) : yamlencode({ + version = 1 + config = [{ + type = "physical" + name = "ens18" + subnets = [{ + type = "dhcp" + }] + }] + }) + + # Instance metadata + meta_data = yamlencode({ + instance_id = sha1(each.key) + local-hostname = each.key + }) +} diff --git a/terraform/vms.tf b/terraform/vms.tf index 6570e70..340b2cb 100644 --- a/terraform/vms.tf +++ b/terraform/vms.tf @@ -22,9 +22,22 @@ locals { # disk_size = "50G" # } + # Example Test VM with custom git branch (for testing pipeline changes): + # "test-vm" = { + # ip = "10.69.13.100/24" + # flake_branch = "test-pipeline" # Bootstrap from this branch instead of master + # } + # Example Minimal VM using all defaults (uncomment to deploy): # "minimal-vm" = {} # "bootstrap-verify-test" = {} + "testvm01" = { + ip = "10.69.13.101/24" + cpu_cores = 2 + memory = 2048 + disk_size = "20G" + flake_branch = "pipeline-testing-improvements" + } } # Compute VM configurations with defaults applied @@ -44,6 +57,8 @@ locals { # Network configuration - detect DHCP vs static ip = lookup(vm, "ip", null) gateway = lookup(vm, "gateway", var.default_gateway) + # Branch configuration for bootstrap (optional, uses master if not set) + flake_branch = lookup(vm, "flake_branch", null) } } } @@ -89,8 +104,9 @@ resource "proxmox_vm_qemu" "vm" { } ide { ide2 { - cloudinit { - storage = each.value.storage + # Reference the custom cloud-init disk created in cloud-init.tf + cdrom { + iso = proxmox_cloud_init_disk.ci[each.key].id } } } @@ -102,15 +118,6 @@ resource "proxmox_vm_qemu" "vm" { # Agent agent = 1 - # Cloud-init configuration - ciuser = "root" - sshkeys = each.value.ssh_public_key - nameserver = each.value.nameservers - searchdomain = each.value.search_domain - - # Network configuration - DHCP or static IP - ipconfig0 = each.value.ip != null ? "ip=${each.value.ip},gw=${each.value.gateway}" : "ip=dhcp" - # Skip IPv6 since we don't use it skip_ipv6 = true