diff --git a/.gitignore b/.gitignore index 8068363..11ce9bc 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ terraform/terraform.tfvars terraform/*.auto.tfvars terraform/crash.log terraform/crash.*.log +terraform/.generated/ diff --git a/TODO.md b/TODO.md index 70e81a5..5ee72c5 100644 --- a/TODO.md +++ b/TODO.md @@ -54,6 +54,7 @@ Automate the entire process of creating, configuring, and deploying new NixOS ho **Status:** ✅ Fully implemented and tested **Completed:** 2025-02-01 +**Enhanced:** 2025-02-01 (added --force flag) **Goal:** Automate creation of host configuration files @@ -64,6 +65,7 @@ Automate the entire process of creating, configuring, and deploying new NixOS ho - Comprehensive validation (hostname format/uniqueness, IP subnet/uniqueness) - Jinja2 templates for NixOS configurations - Automatic updates to flake.nix and terraform/vms.tf +- `--force` flag for regenerating existing configurations (useful for testing) **Tasks:** - [x] Create Python CLI with typer framework @@ -109,6 +111,7 @@ create-host \ **Status:** ✅ Fully implemented and tested **Completed:** 2025-02-01 +**Enhanced:** 2025-02-01 (added branch support for testing) **Goal:** Get freshly deployed VM to apply its specific host configuration @@ -118,7 +121,8 @@ create-host \ - Systemd service `nixos-bootstrap.service` runs on first boot - Depends on `cloud-config.service` to ensure hostname is set - Reads hostname from `hostnamectl` (set by cloud-init via Terraform) -- Runs `nixos-rebuild boot --flake git+https://git.t-juice.club/torjus/nixos-servers.git#${hostname}` +- Supports custom git branch via `NIXOS_FLAKE_BRANCH` environment variable +- Runs `nixos-rebuild boot --flake git+https://git.t-juice.club/torjus/nixos-servers.git?ref=$BRANCH#${hostname}` - Reboots into new configuration on success - Fails gracefully without reboot on errors (network issues, missing config) - Service self-destructs after successful bootstrap (not in new config) @@ -240,10 +244,80 @@ Since most hosts use static IPs defined in their NixOS configurations, we can ex ### Phase 7: Testing & Documentation -**Tasks:** -- [ ] Test full pipeline end-to-end -- [ ] Create test host and verify all steps -- [ ] Document the new workflow in CLAUDE.md +**Status:** 🚧 In Progress (testing improvements completed) + +**Testing Improvements Implemented (2025-02-01):** + +The pipeline now supports efficient testing without polluting master branch: + +**1. --force Flag for create-host** +- Re-run `create-host` to regenerate existing configurations +- Updates existing entries in flake.nix and terraform/vms.tf (no duplicates) +- Skip uniqueness validation checks +- Useful for iterating on configuration templates during testing + +**2. Branch Support for Bootstrap** +- Bootstrap service reads `NIXOS_FLAKE_BRANCH` environment variable +- Defaults to `master` if not set +- Allows testing pipeline changes on feature branches +- Cloud-init passes branch via `/etc/environment` + +**3. Cloud-init Disk for Branch Configuration** +- Terraform generates custom cloud-init snippets for test VMs +- Set `flake_branch` field in VM definition to use non-master branch +- Production VMs omit this field and use master (default) +- Files automatically uploaded to Proxmox via SSH + +**Testing Workflow:** + +```bash +# 1. Create test branch +git checkout -b test-pipeline + +# 2. Generate or update host config +create-host --hostname testvm01 --ip 10.69.13.100/24 + +# 3. Edit terraform/vms.tf to add test VM with branch +# vms = { +# "testvm01" = { +# ip = "10.69.13.100/24" +# flake_branch = "test-pipeline" # Bootstrap from this branch +# } +# } + +# 4. Commit and push test branch +git add -A && git commit -m "test: add testvm01" +git push origin test-pipeline + +# 5. Deploy VM +cd terraform && tofu apply + +# 6. Watch bootstrap (VM fetches from test-pipeline branch) +ssh root@10.69.13.100 +journalctl -fu nixos-bootstrap.service + +# 7. Iterate: modify templates and regenerate with --force +cd .. && create-host --hostname testvm01 --ip 10.69.13.100/24 --force +git commit -am "test: update config" && git push + +# Redeploy to test fresh bootstrap +cd terraform +tofu destroy -target=proxmox_vm_qemu.vm[\"testvm01\"] && tofu apply + +# 8. Clean up when done: squash commits, merge to master, remove test VM +``` + +**Files:** +- `scripts/create-host/create_host.py` - Added --force parameter +- `scripts/create-host/manipulators.py` - Update vs insert logic +- `hosts/template2/bootstrap.nix` - Branch support via environment variable +- `terraform/vms.tf` - flake_branch field support +- `terraform/cloud-init.tf` - Custom cloud-init disk generation +- `terraform/variables.tf` - proxmox_host variable for SSH uploads + +**Remaining Tasks:** +- [ ] Test full pipeline end-to-end on feature branch +- [ ] Update CLAUDE.md with testing workflow - [ ] Add troubleshooting section - [ ] Create examples for common scenarios (DHCP host, static IP host, etc.) diff --git a/hosts/template2/bootstrap.nix b/hosts/template2/bootstrap.nix index da660f0..bbbe8cc 100644 --- a/hosts/template2/bootstrap.nix +++ b/hosts/template2/bootstrap.nix @@ -24,8 +24,12 @@ let echo "Network connectivity confirmed" echo "Fetching and building NixOS configuration from flake..." + # Read git branch from environment, default to master + BRANCH="''${NIXOS_FLAKE_BRANCH:-master}" + echo "Using git branch: $BRANCH" + # Build and activate the host-specific configuration - FLAKE_URL="git+https://git.t-juice.club/torjus/nixos-servers.git#''${HOSTNAME}" + FLAKE_URL="git+https://git.t-juice.club/torjus/nixos-servers.git?ref=$BRANCH#''${HOSTNAME}" if nixos-rebuild boot --flake "$FLAKE_URL"; then echo "Successfully built configuration for $HOSTNAME" @@ -58,6 +62,9 @@ in RemainAfterExit = true; ExecStart = "${bootstrap-script}/bin/nixos-bootstrap"; + # Read environment variables from /etc/environment (set by cloud-init) + EnvironmentFile = "-/etc/environment"; + # Logging to journald StandardOutput = "journal+console"; StandardError = "journal+console"; diff --git a/scripts/create-host/README.md b/scripts/create-host/README.md index 3169287..18c4b2d 100644 --- a/scripts/create-host/README.md +++ b/scripts/create-host/README.md @@ -50,6 +50,23 @@ python -m scripts.create_host.create_host create \ --dry-run ``` +### Force Mode (Regenerate Existing Configuration) + +Overwrite an existing host configuration (useful for testing): + +```bash +python -m scripts.create_host.create_host create \ + --hostname test01 \ + --ip 10.69.13.50/24 \ + --force +``` + +This mode: +- Skips hostname and IP uniqueness validation +- Overwrites files in `hosts//` +- Updates existing entries in `flake.nix` and `terraform/vms.tf` (doesn't duplicate) +- Useful for iterating on configuration templates during testing + ### Options - `--hostname` (required): Hostname for the new host @@ -73,6 +90,10 @@ python -m scripts.create_host.create_host create \ - `--dry-run` (flag): Preview changes without creating files +- `--force` (flag): Overwrite existing host configuration + - Skips uniqueness validation + - Updates existing entries instead of creating duplicates + ## What It Does The tool performs the following actions: diff --git a/scripts/create-host/create_host.py b/scripts/create-host/create_host.py index 7756444..22ad641 100644 --- a/scripts/create-host/create_host.py +++ b/scripts/create-host/create_host.py @@ -45,6 +45,7 @@ def main( memory: int = typer.Option(2048, "--memory", help="Memory in MB"), disk: str = typer.Option("20G", "--disk", help="Disk size (e.g., 20G, 50G, 100G)"), dry_run: bool = typer.Option(False, "--dry-run", help="Preview changes without creating files"), + force: bool = typer.Option(False, "--force", help="Overwrite existing host configuration"), ) -> None: """ Create a new NixOS host configuration. @@ -75,11 +76,20 @@ def main( config.validate() validate_hostname_format(hostname) - validate_hostname_unique(hostname, repo_root) + + # Skip uniqueness checks in force mode + if not force: + validate_hostname_unique(hostname, repo_root) + if ip: + validate_ip_unique(ip, repo_root) + else: + # Check if we're actually overwriting something + host_dir = repo_root / "hosts" / hostname + if host_dir.exists(): + console.print(f"[yellow]⚠[/yellow] Updating existing host configuration for {hostname}") if ip: validate_ip_subnet(ip) - validate_ip_unique(ip, repo_root) console.print("[green]✓[/green] All validations passed\n") @@ -96,13 +106,14 @@ def main( console.print("\n[bold blue]Generating host configuration...[/bold blue]") generate_host_files(config, repo_root) - console.print(f"[green]✓[/green] Created hosts/{hostname}/default.nix") - console.print(f"[green]✓[/green] Created hosts/{hostname}/configuration.nix") + action = "Updated" if force else "Created" + console.print(f"[green]✓[/green] {action} hosts/{hostname}/default.nix") + console.print(f"[green]✓[/green] {action} hosts/{hostname}/configuration.nix") - update_flake_nix(config, repo_root) + update_flake_nix(config, repo_root, force=force) console.print("[green]✓[/green] Updated flake.nix") - update_terraform_vms(config, repo_root) + update_terraform_vms(config, repo_root, force=force) console.print("[green]✓[/green] Updated terraform/vms.tf") # Success message diff --git a/scripts/create-host/manipulators.py b/scripts/create-host/manipulators.py index b0dca17..4d821cc 100644 --- a/scripts/create-host/manipulators.py +++ b/scripts/create-host/manipulators.py @@ -6,21 +6,18 @@ from pathlib import Path from models import HostConfig -def update_flake_nix(config: HostConfig, repo_root: Path) -> None: +def update_flake_nix(config: HostConfig, repo_root: Path, force: bool = False) -> None: """ - Add new host entry to flake.nix nixosConfigurations. + Add or update host entry in flake.nix nixosConfigurations. Args: config: Host configuration repo_root: Path to repository root + force: If True, replace existing entry; if False, insert new entry """ flake_path = repo_root / "flake.nix" content = flake_path.read_text() - # Find the closing of nixosConfigurations block - # Pattern: " };\n packages =" - pattern = r"( \};)\n( packages =)" - # Create new entry new_entry = f""" {config.hostname} = nixpkgs.lib.nixosSystem {{ inherit system; @@ -40,35 +37,47 @@ def update_flake_nix(config: HostConfig, repo_root: Path) -> None: }}; """ - # Insert new entry before closing brace - replacement = rf"\g<1>\n{new_entry}\g<2>" + # Check if hostname already exists + hostname_pattern = rf"^ {re.escape(config.hostname)} = nixpkgs\.lib\.nixosSystem" + existing_match = re.search(hostname_pattern, content, re.MULTILINE) - new_content, count = re.subn(pattern, replacement, content) + if existing_match and force: + # Replace existing entry + # Match the entire block from "hostname = " to "};" + replace_pattern = rf"^ {re.escape(config.hostname)} = nixpkgs\.lib\.nixosSystem \{{.*?^ \}};\n" + new_content, count = re.subn(replace_pattern, new_entry, content, flags=re.MULTILINE | re.DOTALL) - if count == 0: - raise ValueError( - "Could not find insertion point in flake.nix. " - "Looking for pattern: ' };\\n devShells ='" - ) + if count == 0: + raise ValueError(f"Could not find existing entry for {config.hostname} in flake.nix") + else: + # Insert new entry before closing brace + # Pattern: " };\n packages =" + pattern = r"( \};)\n( packages =)" + replacement = rf"\g<1>\n{new_entry}\g<2>" + + new_content, count = re.subn(pattern, replacement, content) + + if count == 0: + raise ValueError( + "Could not find insertion point in flake.nix. " + "Looking for pattern: ' };\\n packages ='" + ) flake_path.write_text(new_content) -def update_terraform_vms(config: HostConfig, repo_root: Path) -> None: +def update_terraform_vms(config: HostConfig, repo_root: Path, force: bool = False) -> None: """ - Add new VM entry to terraform/vms.tf locals.vms map. + Add or update VM entry in terraform/vms.tf locals.vms map. Args: config: Host configuration repo_root: Path to repository root + force: If True, replace existing entry; if False, insert new entry """ terraform_path = repo_root / "terraform" / "vms.tf" content = terraform_path.read_text() - # Find the closing of locals.vms block - # Pattern: " }\n\n # Compute VM configurations" - pattern = r"( \})\n\n( # Compute VM configurations)" - # Create new entry based on whether we have static IP or DHCP if config.is_static_ip: new_entry = f''' "{config.hostname}" = {{ @@ -86,15 +95,30 @@ def update_terraform_vms(config: HostConfig, repo_root: Path) -> None: }} ''' - # Insert new entry before closing brace - replacement = rf"{new_entry}\g<1>\n\n\g<2>" + # Check if hostname already exists + hostname_pattern = rf'^\s+"{re.escape(config.hostname)}" = \{{' + existing_match = re.search(hostname_pattern, content, re.MULTILINE) - new_content, count = re.subn(pattern, replacement, content) + if existing_match and force: + # Replace existing entry + # Match the entire block from "hostname" = { to } + replace_pattern = rf'^\s+"{re.escape(config.hostname)}" = \{{.*?^\s+\}}\n' + new_content, count = re.subn(replace_pattern, new_entry, content, flags=re.MULTILINE | re.DOTALL) - if count == 0: - raise ValueError( - "Could not find insertion point in terraform/vms.tf. " - "Looking for pattern: ' }\\n\\n # Compute VM configurations'" - ) + if count == 0: + raise ValueError(f"Could not find existing entry for {config.hostname} in terraform/vms.tf") + else: + # Insert new entry before closing brace + # Pattern: " }\n\n # Compute VM configurations" + pattern = r"( \})\n\n( # Compute VM configurations)" + replacement = rf"{new_entry}\g<1>\n\n\g<2>" + + new_content, count = re.subn(pattern, replacement, content) + + if count == 0: + raise ValueError( + "Could not find insertion point in terraform/vms.tf. " + "Looking for pattern: ' }\\n\\n # Compute VM configurations'" + ) terraform_path.write_text(new_content) diff --git a/terraform/README.md b/terraform/README.md index 2cca37f..38fffaa 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -87,6 +87,21 @@ vms = { } ``` +### Example: Test VM with Custom Git Branch + +For testing pipeline changes without polluting master: + +```hcl +vms = { + "test-vm" = { + ip = "10.69.13.100/24" + flake_branch = "test-pipeline" # Bootstrap from this branch + } +} +``` + +This VM will bootstrap from the `test-pipeline` branch instead of `master`. Production VMs should omit the `flake_branch` field. + ## Configuration Options Each VM in the `vms` map supports the following fields (all optional): @@ -98,6 +113,7 @@ Each VM in the `vms` map supports the following fields (all optional): | `cpu_cores` | Number of CPU cores | `2` | | `memory` | Memory in MB | `2048` | | `disk_size` | Disk size (e.g., "20G", "100G") | `"20G"` | +| `flake_branch` | Git branch for bootstrap (for testing, omit for production) | `master` | | `target_node` | Proxmox node to deploy to | `"pve1"` | | `template_name` | Template VM to clone from | `"nixos-25.11.20260128.fa83fd8"` | | `storage` | Storage backend | `"local-zfs"` | @@ -182,9 +198,11 @@ deployment_summary = { - `main.tf` - Provider configuration - `variables.tf` - Variable definitions and defaults - `vms.tf` - VM definitions and deployment logic +- `cloud-init.tf` - Custom cloud-init configuration for branch-specific bootstrap - `outputs.tf` - Output definitions for deployed VMs - `terraform.tfvars.example` - Example credentials file - `terraform.tfvars` - Your actual credentials (gitignored) +- `.generated/` - Auto-generated cloud-init files (gitignored) - `vm.tf.old` - Archived single-VM configuration (reference) ## Notes diff --git a/terraform/cloud-init.tf b/terraform/cloud-init.tf new file mode 100644 index 0000000..2779145 --- /dev/null +++ b/terraform/cloud-init.tf @@ -0,0 +1,55 @@ +# Cloud-init configuration for branch-specific bootstrap +# +# This file manages custom cloud-init snippets for VMs that need to bootstrap +# from a specific git branch (non-master). Production VMs omit flake_branch +# and use the default master branch. + +# Generate cloud-init snippets for VMs with custom branch configuration +resource "local_file" "cloud_init_branch" { + for_each = { + for name, vm in local.vm_configs : name => vm + if vm.flake_branch != null + } + + filename = "${path.module}/.generated/cloud-init-${each.key}.yml" + content = yamlencode({ + # Write NIXOS_FLAKE_BRANCH to /etc/environment + # This will be read by bootstrap.nix service via EnvironmentFile + write_files = [{ + path = "/etc/environment" + content = "NIXOS_FLAKE_BRANCH=${each.value.flake_branch}\n" + append = true + }] + }) + + file_permission = "0644" +} + +# Upload cloud-init snippets to Proxmox +# Note: This requires SSH access to the Proxmox host +# Alternative: Manually copy files or use Proxmox API if available +resource "null_resource" "upload_cloud_init" { + for_each = { + for name, vm in local.vm_configs : name => vm + if vm.flake_branch != null + } + + # Trigger re-upload when content changes + triggers = { + content_hash = local_file.cloud_init_branch[each.key].content + } + + # Upload the cloud-init file to Proxmox snippets directory + provisioner "local-exec" { + command = <<-EOT + scp -o StrictHostKeyChecking=no \ + ${local_file.cloud_init_branch[each.key].filename} \ + ${var.proxmox_host}:/var/lib/vz/snippets/cloud-init-${each.key}.yml + EOT + } + + depends_on = [local_file.cloud_init_branch] +} + +# Ensure VMs depend on cloud-init being uploaded +# This is handled implicitly by the cicustom reference in vms.tf diff --git a/terraform/variables.tf b/terraform/variables.tf index fe13cb0..c6acc26 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -21,6 +21,12 @@ variable "proxmox_tls_insecure" { default = true } +variable "proxmox_host" { + description = "Proxmox host for SSH access (used to upload cloud-init snippets)" + type = string + default = "pve1.home.2rjus.net" +} + # Default values for VM configurations # These can be overridden per-VM in vms.tf diff --git a/terraform/vms.tf b/terraform/vms.tf index 6570e70..eecac99 100644 --- a/terraform/vms.tf +++ b/terraform/vms.tf @@ -22,6 +22,12 @@ locals { # disk_size = "50G" # } + # Example Test VM with custom git branch (for testing pipeline changes): + # "test-vm" = { + # ip = "10.69.13.100/24" + # flake_branch = "test-pipeline" # Bootstrap from this branch instead of master + # } + # Example Minimal VM using all defaults (uncomment to deploy): # "minimal-vm" = {} # "bootstrap-verify-test" = {} @@ -44,6 +50,8 @@ locals { # Network configuration - detect DHCP vs static ip = lookup(vm, "ip", null) gateway = lookup(vm, "gateway", var.default_gateway) + # Branch configuration for bootstrap (optional, uses master if not set) + flake_branch = lookup(vm, "flake_branch", null) } } } @@ -111,6 +119,9 @@ resource "proxmox_vm_qemu" "vm" { # Network configuration - DHCP or static IP ipconfig0 = each.value.ip != null ? "ip=${each.value.ip},gw=${each.value.gateway}" : "ip=dhcp" + # Custom cloud-init disk for branch configuration (if flake_branch is set) + cicustom = each.value.flake_branch != null ? "user=${each.value.storage}:snippets/cloud-init-${each.key}.yml" : null + # Skip IPv6 since we don't use it skip_ipv6 = true