.gitignore

@@ -10,3 +10,4 @@ terraform/terraform.tfvars
 terraform/*.auto.tfvars
 terraform/crash.log
 terraform/crash.*.log
+terraform/.generated/

TODO.md

@@ -54,6 +54,7 @@ Automate the entire process of creating, configuring, and deploying new NixOS ho
 
 **Status:** ✅ Fully implemented and tested
 **Completed:** 2025-02-01
+**Enhanced:** 2025-02-01 (added --force flag)
 
 **Goal:** Automate creation of host configuration files
 
@@ -64,6 +65,7 @@ Automate the entire process of creating, configuring, and deploying new NixOS ho
 - Comprehensive validation (hostname format/uniqueness, IP subnet/uniqueness)
 - Jinja2 templates for NixOS configurations
 - Automatic updates to flake.nix and terraform/vms.tf
+- `--force` flag for regenerating existing configurations (useful for testing)
 
 **Tasks:**
 - [x] Create Python CLI with typer framework
@@ -109,6 +111,7 @@ create-host \
 
 **Status:** ✅ Fully implemented and tested
 **Completed:** 2025-02-01
+**Enhanced:** 2025-02-01 (added branch support for testing)
 
 **Goal:** Get freshly deployed VM to apply its specific host configuration
 
@@ -118,7 +121,8 @@ create-host \
 - Systemd service `nixos-bootstrap.service` runs on first boot
 - Depends on `cloud-config.service` to ensure hostname is set
 - Reads hostname from `hostnamectl` (set by cloud-init via Terraform)
-- Runs `nixos-rebuild boot --flake git+https://git.t-juice.club/torjus/nixos-servers.git#${hostname}`
+- Supports custom git branch via `NIXOS_FLAKE_BRANCH` environment variable
+- Runs `nixos-rebuild boot --flake git+https://git.t-juice.club/torjus/nixos-servers.git?ref=$BRANCH#${hostname}`
 - Reboots into new configuration on success
 - Fails gracefully without reboot on errors (network issues, missing config)
 - Service self-destructs after successful bootstrap (not in new config)
@@ -240,10 +244,80 @@ Since most hosts use static IPs defined in their NixOS configurations, we can ex
 
 ### Phase 7: Testing & Documentation
 
-**Tasks:**
+**Status:** 🚧 In Progress (testing improvements completed)
-- [ ] Test full pipeline end-to-end
+
-- [ ] Create test host and verify all steps
+**Testing Improvements Implemented (2025-02-01):**
-- [ ] Document the new workflow in CLAUDE.md
+
+The pipeline now supports efficient testing without polluting master branch:
+
+**1. --force Flag for create-host**
+- Re-run `create-host` to regenerate existing configurations
+- Updates existing entries in flake.nix and terraform/vms.tf (no duplicates)
+- Skip uniqueness validation checks
+- Useful for iterating on configuration templates during testing
+
+**2. Branch Support for Bootstrap**
+- Bootstrap service reads `NIXOS_FLAKE_BRANCH` environment variable
+- Defaults to `master` if not set
+- Allows testing pipeline changes on feature branches
+- Cloud-init passes branch via `/etc/environment`
+
+**3. Cloud-init Disk for Branch Configuration**
+- Terraform generates custom cloud-init snippets for test VMs
+- Set `flake_branch` field in VM definition to use non-master branch
+- Production VMs omit this field and use master (default)
+- Files automatically uploaded to Proxmox via SSH
+
+**Testing Workflow:**
+
+```bash
+# 1. Create test branch
+git checkout -b test-pipeline
+
+# 2. Generate or update host config
+create-host --hostname testvm01 --ip 10.69.13.100/24
+
+# 3. Edit terraform/vms.tf to add test VM with branch
+# vms = {
+#   "testvm01" = {
+#     ip = "10.69.13.100/24"
+#     flake_branch = "test-pipeline"  # Bootstrap from this branch
+#   }
+# }
+
+# 4. Commit and push test branch
+git add -A && git commit -m "test: add testvm01"
+git push origin test-pipeline
+
+# 5. Deploy VM
+cd terraform && tofu apply
+
+# 6. Watch bootstrap (VM fetches from test-pipeline branch)
+ssh root@10.69.13.100
+journalctl -fu nixos-bootstrap.service
+
+# 7. Iterate: modify templates and regenerate with --force
+cd .. && create-host --hostname testvm01 --ip 10.69.13.100/24 --force
+git commit -am "test: update config" && git push
+
+# Redeploy to test fresh bootstrap
+cd terraform
+tofu destroy -target=proxmox_vm_qemu.vm[\"testvm01\"] && tofu apply
+
+# 8. Clean up when done: squash commits, merge to master, remove test VM
+```
+
+**Files:**
+- `scripts/create-host/create_host.py` - Added --force parameter
+- `scripts/create-host/manipulators.py` - Update vs insert logic
+- `hosts/template2/bootstrap.nix` - Branch support via environment variable
+- `terraform/vms.tf` - flake_branch field support
+- `terraform/cloud-init.tf` - Custom cloud-init disk generation
+- `terraform/variables.tf` - proxmox_host variable for SSH uploads
+
+**Remaining Tasks:**
+- [ ] Test full pipeline end-to-end on feature branch
+- [ ] Update CLAUDE.md with testing workflow
 - [ ] Add troubleshooting section
 - [ ] Create examples for common scenarios (DHCP host, static IP host, etc.)
 

hosts/template2/bootstrap.nix

@@ -24,8 +24,12 @@ let
       echo "Network connectivity confirmed"
       echo "Fetching and building NixOS configuration from flake..."
 
+      # Read git branch from environment, default to master
+      BRANCH="''${NIXOS_FLAKE_BRANCH:-master}"
+      echo "Using git branch: $BRANCH"
+
       # Build and activate the host-specific configuration
-      FLAKE_URL="git+https://git.t-juice.club/torjus/nixos-servers.git#''${HOSTNAME}"
+      FLAKE_URL="git+https://git.t-juice.club/torjus/nixos-servers.git?ref=$BRANCH#''${HOSTNAME}"
 
       if nixos-rebuild boot --flake "$FLAKE_URL"; then
         echo "Successfully built configuration for $HOSTNAME"
@@ -58,6 +62,9 @@ in
       RemainAfterExit = true;
       ExecStart = "${bootstrap-script}/bin/nixos-bootstrap";
 
+      # Read environment variables from /etc/environment (set by cloud-init)
+      EnvironmentFile = "-/etc/environment";
+
       # Logging to journald
       StandardOutput = "journal+console";
       StandardError = "journal+console";

scripts/create-host/README.md

@@ -50,6 +50,23 @@ python -m scripts.create_host.create_host create \
   --dry-run
 ```
 
+### Force Mode (Regenerate Existing Configuration)
+
+Overwrite an existing host configuration (useful for testing):
+
+```bash
+python -m scripts.create_host.create_host create \
+  --hostname test01 \
+  --ip 10.69.13.50/24 \
+  --force
+```
+
+This mode:
+- Skips hostname and IP uniqueness validation
+- Overwrites files in `hosts/<hostname>/`
+- Updates existing entries in `flake.nix` and `terraform/vms.tf` (doesn't duplicate)
+- Useful for iterating on configuration templates during testing
+
 ### Options
 
 - `--hostname` (required): Hostname for the new host
@@ -73,6 +90,10 @@ python -m scripts.create_host.create_host create \
 
 - `--dry-run` (flag): Preview changes without creating files
 
+- `--force` (flag): Overwrite existing host configuration
+  - Skips uniqueness validation
+  - Updates existing entries instead of creating duplicates
+
 ## What It Does
 
 The tool performs the following actions:

scripts/create-host/create_host.py

@@ -45,6 +45,7 @@ def main(
     memory: int = typer.Option(2048, "--memory", help="Memory in MB"),
     disk: str = typer.Option("20G", "--disk", help="Disk size (e.g., 20G, 50G, 100G)"),
     dry_run: bool = typer.Option(False, "--dry-run", help="Preview changes without creating files"),
+    force: bool = typer.Option(False, "--force", help="Overwrite existing host configuration"),
 ) -> None:
     """
     Create a new NixOS host configuration.
@@ -75,11 +76,20 @@ def main(
 
         config.validate()
         validate_hostname_format(hostname)
-        validate_hostname_unique(hostname, repo_root)
+
+        # Skip uniqueness checks in force mode
+        if not force:
+            validate_hostname_unique(hostname, repo_root)
+            if ip:
+                validate_ip_unique(ip, repo_root)
+        else:
+            # Check if we're actually overwriting something
+            host_dir = repo_root / "hosts" / hostname
+            if host_dir.exists():
+                console.print(f"[yellow]⚠[/yellow]  Updating existing host configuration for {hostname}")
 
         if ip:
             validate_ip_subnet(ip)
-            validate_ip_unique(ip, repo_root)
 
         console.print("[green]✓[/green] All validations passed\n")
 
@@ -96,13 +106,14 @@ def main(
         console.print("\n[bold blue]Generating host configuration...[/bold blue]")
 
         generate_host_files(config, repo_root)
-        console.print(f"[green]✓[/green] Created hosts/{hostname}/default.nix")
+        action = "Updated" if force else "Created"
-        console.print(f"[green]✓[/green] Created hosts/{hostname}/configuration.nix")
+        console.print(f"[green]✓[/green] {action} hosts/{hostname}/default.nix")
+        console.print(f"[green]✓[/green] {action} hosts/{hostname}/configuration.nix")
 
-        update_flake_nix(config, repo_root)
+        update_flake_nix(config, repo_root, force=force)
         console.print("[green]✓[/green] Updated flake.nix")
 
-        update_terraform_vms(config, repo_root)
+        update_terraform_vms(config, repo_root, force=force)
         console.print("[green]✓[/green] Updated terraform/vms.tf")
 
         # Success message

scripts/create-host/manipulators.py

@@ -6,21 +6,18 @@ from pathlib import Path
 from models import HostConfig
 
 
-def update_flake_nix(config: HostConfig, repo_root: Path) -> None:
+def update_flake_nix(config: HostConfig, repo_root: Path, force: bool = False) -> None:
     """
-    Add new host entry to flake.nix nixosConfigurations.
+    Add or update host entry in flake.nix nixosConfigurations.
 
     Args:
         config: Host configuration
         repo_root: Path to repository root
+        force: If True, replace existing entry; if False, insert new entry
     """
     flake_path = repo_root / "flake.nix"
     content = flake_path.read_text()
 
-    # Find the closing of nixosConfigurations block
-    # Pattern: "      };\n      packages ="
-    pattern = r"(      \};)\n(      packages =)"
-
     # Create new entry
     new_entry = f"""      {config.hostname} = nixpkgs.lib.nixosSystem {{
         inherit system;
@@ -40,35 +37,47 @@ def update_flake_nix(config: HostConfig, repo_root: Path) -> None:
       }};
 """
 
-    # Insert new entry before closing brace
+    # Check if hostname already exists
-    replacement = rf"\g<1>\n{new_entry}\g<2>"
+    hostname_pattern = rf"^      {re.escape(config.hostname)} = nixpkgs\.lib\.nixosSystem"
+    existing_match = re.search(hostname_pattern, content, re.MULTILINE)
 
-    new_content, count = re.subn(pattern, replacement, content)
+    if existing_match and force:
+        # Replace existing entry
+        # Match the entire block from "hostname = " to "};"
+        replace_pattern = rf"^      {re.escape(config.hostname)} = nixpkgs\.lib\.nixosSystem \{{.*?^      \}};\n"
+        new_content, count = re.subn(replace_pattern, new_entry, content, flags=re.MULTILINE | re.DOTALL)
 
-    if count == 0:
+        if count == 0:
-        raise ValueError(
+            raise ValueError(f"Could not find existing entry for {config.hostname} in flake.nix")
-            "Could not find insertion point in flake.nix. "
+    else:
-            "Looking for pattern: '      };\\n      devShells ='"
+        # Insert new entry before closing brace
-        )
+        # Pattern: "      };\n      packages ="
+        pattern = r"(      \};)\n(      packages =)"
+        replacement = rf"\g<1>\n{new_entry}\g<2>"
+
+        new_content, count = re.subn(pattern, replacement, content)
+
+        if count == 0:
+            raise ValueError(
+                "Could not find insertion point in flake.nix. "
+                "Looking for pattern: '      };\\n      packages ='"
+            )
 
     flake_path.write_text(new_content)
 
 
-def update_terraform_vms(config: HostConfig, repo_root: Path) -> None:
+def update_terraform_vms(config: HostConfig, repo_root: Path, force: bool = False) -> None:
     """
-    Add new VM entry to terraform/vms.tf locals.vms map.
+    Add or update VM entry in terraform/vms.tf locals.vms map.
 
     Args:
         config: Host configuration
         repo_root: Path to repository root
+        force: If True, replace existing entry; if False, insert new entry
     """
     terraform_path = repo_root / "terraform" / "vms.tf"
     content = terraform_path.read_text()
 
-    # Find the closing of locals.vms block
-    # Pattern: "  }\n\n  # Compute VM configurations"
-    pattern = r"(  \})\n\n(  # Compute VM configurations)"
-
     # Create new entry based on whether we have static IP or DHCP
     if config.is_static_ip:
         new_entry = f'''    "{config.hostname}" = {{
@@ -86,15 +95,30 @@ def update_terraform_vms(config: HostConfig, repo_root: Path) -> None:
     }}
 '''
 
-    # Insert new entry before closing brace
+    # Check if hostname already exists
-    replacement = rf"{new_entry}\g<1>\n\n\g<2>"
+    hostname_pattern = rf'^\s+"{re.escape(config.hostname)}" = \{{'
+    existing_match = re.search(hostname_pattern, content, re.MULTILINE)
 
-    new_content, count = re.subn(pattern, replacement, content)
+    if existing_match and force:
+        # Replace existing entry
+        # Match the entire block from "hostname" = { to }
+        replace_pattern = rf'^\s+"{re.escape(config.hostname)}" = \{{.*?^\s+\}}\n'
+        new_content, count = re.subn(replace_pattern, new_entry, content, flags=re.MULTILINE | re.DOTALL)
 
-    if count == 0:
+        if count == 0:
-        raise ValueError(
+            raise ValueError(f"Could not find existing entry for {config.hostname} in terraform/vms.tf")
-            "Could not find insertion point in terraform/vms.tf. "
+    else:
-            "Looking for pattern: '  }\\n\\n  # Compute VM configurations'"
+        # Insert new entry before closing brace
-        )
+        # Pattern: "  }\n\n  # Compute VM configurations"
+        pattern = r"(  \})\n\n(  # Compute VM configurations)"
+        replacement = rf"{new_entry}\g<1>\n\n\g<2>"
+
+        new_content, count = re.subn(pattern, replacement, content)
+
+        if count == 0:
+            raise ValueError(
+                "Could not find insertion point in terraform/vms.tf. "
+                "Looking for pattern: '  }\\n\\n  # Compute VM configurations'"
+            )
 
     terraform_path.write_text(new_content)

terraform/README.md

@@ -87,6 +87,21 @@ vms = {
 }
 ```
 
+### Example: Test VM with Custom Git Branch
+
+For testing pipeline changes without polluting master:
+
+```hcl
+vms = {
+  "test-vm" = {
+    ip = "10.69.13.100/24"
+    flake_branch = "test-pipeline"  # Bootstrap from this branch
+  }
+}
+```
+
+This VM will bootstrap from the `test-pipeline` branch instead of `master`. Production VMs should omit the `flake_branch` field.
+
 ## Configuration Options
 
 Each VM in the `vms` map supports the following fields (all optional):
@@ -98,6 +113,7 @@ Each VM in the `vms` map supports the following fields (all optional):
 | `cpu_cores` | Number of CPU cores | `2` |
 | `memory` | Memory in MB | `2048` |
 | `disk_size` | Disk size (e.g., "20G", "100G") | `"20G"` |
+| `flake_branch` | Git branch for bootstrap (for testing, omit for production) | `master` |
 | `target_node` | Proxmox node to deploy to | `"pve1"` |
 | `template_name` | Template VM to clone from | `"nixos-25.11.20260128.fa83fd8"` |
 | `storage` | Storage backend | `"local-zfs"` |
@@ -182,9 +198,11 @@ deployment_summary = {
 - `main.tf` - Provider configuration
 - `variables.tf` - Variable definitions and defaults
 - `vms.tf` - VM definitions and deployment logic
+- `cloud-init.tf` - Custom cloud-init configuration for branch-specific bootstrap
 - `outputs.tf` - Output definitions for deployed VMs
 - `terraform.tfvars.example` - Example credentials file
 - `terraform.tfvars` - Your actual credentials (gitignored)
+- `.generated/` - Auto-generated cloud-init files (gitignored)
 - `vm.tf.old` - Archived single-VM configuration (reference)
 
 ## Notes

terraform/cloud-init.tf

@@ -0,0 +1,55 @@
+# Cloud-init configuration for branch-specific bootstrap
+#
+# This file manages custom cloud-init snippets for VMs that need to bootstrap
+# from a specific git branch (non-master). Production VMs omit flake_branch
+# and use the default master branch.
+
+# Generate cloud-init snippets for VMs with custom branch configuration
+resource "local_file" "cloud_init_branch" {
+  for_each = {
+    for name, vm in local.vm_configs : name => vm
+    if vm.flake_branch != null
+  }
+
+  filename = "${path.module}/.generated/cloud-init-${each.key}.yml"
+  content = yamlencode({
+    # Write NIXOS_FLAKE_BRANCH to /etc/environment
+    # This will be read by bootstrap.nix service via EnvironmentFile
+    write_files = [{
+      path    = "/etc/environment"
+      content = "NIXOS_FLAKE_BRANCH=${each.value.flake_branch}\n"
+      append  = true
+    }]
+  })
+
+  file_permission = "0644"
+}
+
+# Upload cloud-init snippets to Proxmox
+# Note: This requires SSH access to the Proxmox host
+# Alternative: Manually copy files or use Proxmox API if available
+resource "null_resource" "upload_cloud_init" {
+  for_each = {
+    for name, vm in local.vm_configs : name => vm
+    if vm.flake_branch != null
+  }
+
+  # Trigger re-upload when content changes
+  triggers = {
+    content_hash = local_file.cloud_init_branch[each.key].content
+  }
+
+  # Upload the cloud-init file to Proxmox snippets directory
+  provisioner "local-exec" {
+    command = <<-EOT
+      scp -o StrictHostKeyChecking=no \
+        ${local_file.cloud_init_branch[each.key].filename} \
+        ${var.proxmox_host}:/var/lib/vz/snippets/cloud-init-${each.key}.yml
+    EOT
+  }
+
+  depends_on = [local_file.cloud_init_branch]
+}
+
+# Ensure VMs depend on cloud-init being uploaded
+# This is handled implicitly by the cicustom reference in vms.tf

terraform/variables.tf

@@ -21,6 +21,12 @@ variable "proxmox_tls_insecure" {
   default     = true
 }
 
+variable "proxmox_host" {
+  description = "Proxmox host for SSH access (used to upload cloud-init snippets)"
+  type        = string
+  default     = "pve1.home.2rjus.net"
+}
+
 # Default values for VM configurations
 # These can be overridden per-VM in vms.tf
 

terraform/vms.tf

@@ -22,6 +22,12 @@ locals {
     #   disk_size = "50G"
     # }
 
+    # Example Test VM with custom git branch (for testing pipeline changes):
+    # "test-vm" = {
+    #   ip          = "10.69.13.100/24"
+    #   flake_branch = "test-pipeline"  # Bootstrap from this branch instead of master
+    # }
+
     # Example Minimal VM using all defaults (uncomment to deploy):
     # "minimal-vm" = {}
     # "bootstrap-verify-test" = {}
@@ -44,6 +50,8 @@ locals {
       # Network configuration - detect DHCP vs static
       ip      = lookup(vm, "ip", null)
       gateway = lookup(vm, "gateway", var.default_gateway)
+      # Branch configuration for bootstrap (optional, uses master if not set)
+      flake_branch = lookup(vm, "flake_branch", null)
     }
   }
 }
@@ -111,6 +119,9 @@ resource "proxmox_vm_qemu" "vm" {
   # Network configuration - DHCP or static IP
   ipconfig0 = each.value.ip != null ? "ip=${each.value.ip},gw=${each.value.gateway}" : "ip=dhcp"
 
+  # Custom cloud-init disk for branch configuration (if flake_branch is set)
+  cicustom = each.value.flake_branch != null ? "user=${each.value.storage}:snippets/cloud-init-${each.key}.yml" : null
+
   # Skip IPv6 since we don't use it
   skip_ipv6 = true