TODO.md

@@ -54,6 +54,7 @@ Automate the entire process of creating, configuring, and deploying new NixOS ho
 
 **Status:** ✅ Fully implemented and tested
 **Completed:** 2025-02-01
+**Enhanced:** 2025-02-01 (added --force flag)
 
 **Goal:** Automate creation of host configuration files
 
@@ -64,6 +65,7 @@ Automate the entire process of creating, configuring, and deploying new NixOS ho
 - Comprehensive validation (hostname format/uniqueness, IP subnet/uniqueness)
 - Jinja2 templates for NixOS configurations
 - Automatic updates to flake.nix and terraform/vms.tf
+- `--force` flag for regenerating existing configurations (useful for testing)
 
 **Tasks:**
 - [x] Create Python CLI with typer framework
@@ -109,6 +111,7 @@ create-host \
 
 **Status:** ✅ Fully implemented and tested
 **Completed:** 2025-02-01
+**Enhanced:** 2025-02-01 (added branch support for testing)
 
 **Goal:** Get freshly deployed VM to apply its specific host configuration
 
@@ -118,7 +121,8 @@ create-host \
 - Systemd service `nixos-bootstrap.service` runs on first boot
 - Depends on `cloud-config.service` to ensure hostname is set
 - Reads hostname from `hostnamectl` (set by cloud-init via Terraform)
-- Runs `nixos-rebuild boot --flake git+https://git.t-juice.club/torjus/nixos-servers.git#${hostname}`
+- Supports custom git branch via `NIXOS_FLAKE_BRANCH` environment variable
+- Runs `nixos-rebuild boot --flake git+https://git.t-juice.club/torjus/nixos-servers.git?ref=$BRANCH#${hostname}`
 - Reboots into new configuration on success
 - Fails gracefully without reboot on errors (network issues, missing config)
 - Service self-destructs after successful bootstrap (not in new config)
@@ -240,10 +244,80 @@ Since most hosts use static IPs defined in their NixOS configurations, we can ex
 
 ### Phase 7: Testing & Documentation
 
-**Tasks:**
+**Status:** 🚧 In Progress (testing improvements completed)
-- [ ] Test full pipeline end-to-end
+
-- [ ] Create test host and verify all steps
+**Testing Improvements Implemented (2025-02-01):**
-- [ ] Document the new workflow in CLAUDE.md
+
+The pipeline now supports efficient testing without polluting master branch:
+
+**1. --force Flag for create-host**
+- Re-run `create-host` to regenerate existing configurations
+- Updates existing entries in flake.nix and terraform/vms.tf (no duplicates)
+- Skip uniqueness validation checks
+- Useful for iterating on configuration templates during testing
+
+**2. Branch Support for Bootstrap**
+- Bootstrap service reads `NIXOS_FLAKE_BRANCH` environment variable
+- Defaults to `master` if not set
+- Allows testing pipeline changes on feature branches
+- Cloud-init passes branch via `/etc/environment`
+
+**3. Cloud-init Disk for Branch Configuration**
+- Terraform generates custom cloud-init snippets for test VMs
+- Set `flake_branch` field in VM definition to use non-master branch
+- Production VMs omit this field and use master (default)
+- Files automatically uploaded to Proxmox via SSH
+
+**Testing Workflow:**
+
+```bash
+# 1. Create test branch
+git checkout -b test-pipeline
+
+# 2. Generate or update host config
+create-host --hostname testvm01 --ip 10.69.13.100/24
+
+# 3. Edit terraform/vms.tf to add test VM with branch
+# vms = {
+#   "testvm01" = {
+#     ip = "10.69.13.100/24"
+#     flake_branch = "test-pipeline"  # Bootstrap from this branch
+#   }
+# }
+
+# 4. Commit and push test branch
+git add -A && git commit -m "test: add testvm01"
+git push origin test-pipeline
+
+# 5. Deploy VM
+cd terraform && tofu apply
+
+# 6. Watch bootstrap (VM fetches from test-pipeline branch)
+ssh root@10.69.13.100
+journalctl -fu nixos-bootstrap.service
+
+# 7. Iterate: modify templates and regenerate with --force
+cd .. && create-host --hostname testvm01 --ip 10.69.13.100/24 --force
+git commit -am "test: update config" && git push
+
+# Redeploy to test fresh bootstrap
+cd terraform
+tofu destroy -target=proxmox_vm_qemu.vm[\"testvm01\"] && tofu apply
+
+# 8. Clean up when done: squash commits, merge to master, remove test VM
+```
+
+**Files:**
+- `scripts/create-host/create_host.py` - Added --force parameter
+- `scripts/create-host/manipulators.py` - Update vs insert logic
+- `hosts/template2/bootstrap.nix` - Branch support via environment variable
+- `terraform/vms.tf` - flake_branch field support
+- `terraform/cloud-init.tf` - Custom cloud-init disk generation
+- `terraform/variables.tf` - proxmox_host variable for SSH uploads
+
+**Remaining Tasks:**
+- [ ] Test full pipeline end-to-end on feature branch
+- [ ] Update CLAUDE.md with testing workflow
 - [ ] Add troubleshooting section
 - [ ] Create examples for common scenarios (DHCP host, static IP host, etc.)
 

flake.nix

@@ -334,6 +334,22 @@
             sops-nix.nixosModules.sops
           ];
         };
+      testvm01 = nixpkgs.lib.nixosSystem {
+        inherit system;
+        specialArgs = {
+          inherit inputs self sops-nix;
+        };
+        modules = [
+          (
+            { config, pkgs, ... }:
+            {
+              nixpkgs.overlays = commonOverlays;
+            }
+          )
+          ./hosts/testvm01
+          sops-nix.nixosModules.sops
+        ];
+      };
       };
       packages = forAllSystems (
         { pkgs }:

hosts/template2/bootstrap.nix

@@ -24,8 +24,12 @@ let
       echo "Network connectivity confirmed"
       echo "Fetching and building NixOS configuration from flake..."
 
+      # Read git branch from environment, default to master
+      BRANCH="''${NIXOS_FLAKE_BRANCH:-master}"
+      echo "Using git branch: $BRANCH"
+
       # Build and activate the host-specific configuration
-      FLAKE_URL="git+https://git.t-juice.club/torjus/nixos-servers.git#''${HOSTNAME}"
+      FLAKE_URL="git+https://git.t-juice.club/torjus/nixos-servers.git?ref=$BRANCH#''${HOSTNAME}"
 
       if nixos-rebuild boot --flake "$FLAKE_URL"; then
         echo "Successfully built configuration for $HOSTNAME"
@@ -58,6 +62,9 @@ in
       RemainAfterExit = true;
       ExecStart = "${bootstrap-script}/bin/nixos-bootstrap";
 
+      # Read environment variables from /etc/environment (set by cloud-init)
+      EnvironmentFile = "-/etc/environment";
+
       # Logging to journald
       StandardOutput = "journal+console";
       StandardError = "journal+console";

hosts/testvm01/configuration.nix

@@ -0,0 +1,61 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [
+    ../template2/hardware-configuration.nix
+
+    ../../system
+    ../../common/vm
+  ];
+
+  nixpkgs.config.allowUnfree = true;
+  boot.loader.grub.enable = true;
+  boot.loader.grub.device = "/dev/vda";
+
+  networking.hostName = "testvm01";
+  networking.domain = "home.2rjus.net";
+  networking.useNetworkd = true;
+  networking.useDHCP = false;
+  services.resolved.enable = false;
+  networking.nameservers = [
+    "10.69.13.5"
+    "10.69.13.6"
+  ];
+
+  systemd.network.enable = true;
+  systemd.network.networks."ens18" = {
+    matchConfig.Name = "ens18";
+    address = [
+      "10.69.13.101/24"
+    ];
+    routes = [
+      { Gateway = "10.69.13.1"; }
+    ];
+    linkConfig.RequiredForOnline = "routable";
+  };
+  time.timeZone = "Europe/Oslo";
+
+  nix.settings.experimental-features = [
+    "nix-command"
+    "flakes"
+  ];
+  nix.settings.tarball-ttl = 0;
+  environment.systemPackages = with pkgs; [
+    vim
+    wget
+    git
+  ];
+
+  # Open ports in the firewall.
+  # networking.firewall.allowedTCPPorts = [ ... ];
+  # networking.firewall.allowedUDPPorts = [ ... ];
+  # Or disable the firewall altogether.
+  networking.firewall.enable = false;
+
+  system.stateVersion = "25.11"; # Did you read the comment?
+}

hosts/testvm01/default.nix

@@ -0,0 +1,5 @@
+{ ... }: {
+  imports = [
+    ./configuration.nix
+  ];
+}

scripts/create-host/README.md

@@ -50,6 +50,23 @@ python -m scripts.create_host.create_host create \
   --dry-run
 ```
 
+### Force Mode (Regenerate Existing Configuration)
+
+Overwrite an existing host configuration (useful for testing):
+
+```bash
+python -m scripts.create_host.create_host create \
+  --hostname test01 \
+  --ip 10.69.13.50/24 \
+  --force
+```
+
+This mode:
+- Skips hostname and IP uniqueness validation
+- Overwrites files in `hosts/<hostname>/`
+- Updates existing entries in `flake.nix` and `terraform/vms.tf` (doesn't duplicate)
+- Useful for iterating on configuration templates during testing
+
 ### Options
 
 - `--hostname` (required): Hostname for the new host
@@ -73,6 +90,10 @@ python -m scripts.create_host.create_host create \
 
 - `--dry-run` (flag): Preview changes without creating files
 
+- `--force` (flag): Overwrite existing host configuration
+  - Skips uniqueness validation
+  - Updates existing entries instead of creating duplicates
+
 ## What It Does
 
 The tool performs the following actions:

scripts/create-host/create_host.py

@@ -45,6 +45,7 @@ def main(
     memory: int = typer.Option(2048, "--memory", help="Memory in MB"),
     disk: str = typer.Option("20G", "--disk", help="Disk size (e.g., 20G, 50G, 100G)"),
     dry_run: bool = typer.Option(False, "--dry-run", help="Preview changes without creating files"),
+    force: bool = typer.Option(False, "--force", help="Overwrite existing host configuration"),
 ) -> None:
     """
     Create a new NixOS host configuration.
@@ -75,11 +76,20 @@ def main(
 
         config.validate()
         validate_hostname_format(hostname)
-        validate_hostname_unique(hostname, repo_root)
+
+        # Skip uniqueness checks in force mode
+        if not force:
+            validate_hostname_unique(hostname, repo_root)
+            if ip:
+                validate_ip_unique(ip, repo_root)
+        else:
+            # Check if we're actually overwriting something
+            host_dir = repo_root / "hosts" / hostname
+            if host_dir.exists():
+                console.print(f"[yellow]⚠[/yellow]  Updating existing host configuration for {hostname}")
 
         if ip:
             validate_ip_subnet(ip)
-            validate_ip_unique(ip, repo_root)
 
         console.print("[green]✓[/green] All validations passed\n")
 
@@ -96,13 +106,14 @@ def main(
         console.print("\n[bold blue]Generating host configuration...[/bold blue]")
 
         generate_host_files(config, repo_root)
-        console.print(f"[green]✓[/green] Created hosts/{hostname}/default.nix")
+        action = "Updated" if force else "Created"
-        console.print(f"[green]✓[/green] Created hosts/{hostname}/configuration.nix")
+        console.print(f"[green]✓[/green] {action} hosts/{hostname}/default.nix")
+        console.print(f"[green]✓[/green] {action} hosts/{hostname}/configuration.nix")
 
-        update_flake_nix(config, repo_root)
+        update_flake_nix(config, repo_root, force=force)
         console.print("[green]✓[/green] Updated flake.nix")
 
-        update_terraform_vms(config, repo_root)
+        update_terraform_vms(config, repo_root, force=force)
         console.print("[green]✓[/green] Updated terraform/vms.tf")
 
         # Success message

scripts/create-host/manipulators.py

@@ -6,21 +6,18 @@ from pathlib import Path
 from models import HostConfig
 
 
-def update_flake_nix(config: HostConfig, repo_root: Path) -> None:
+def update_flake_nix(config: HostConfig, repo_root: Path, force: bool = False) -> None:
     """
-    Add new host entry to flake.nix nixosConfigurations.
+    Add or update host entry in flake.nix nixosConfigurations.
 
     Args:
         config: Host configuration
         repo_root: Path to repository root
+        force: If True, replace existing entry; if False, insert new entry
     """
     flake_path = repo_root / "flake.nix"
     content = flake_path.read_text()
 
-    # Find the closing of nixosConfigurations block
-    # Pattern: "      };\n      packages ="
-    pattern = r"(      \};)\n(      packages =)"
-
     # Create new entry
     new_entry = f"""      {config.hostname} = nixpkgs.lib.nixosSystem {{
         inherit system;
@@ -40,35 +37,47 @@ def update_flake_nix(config: HostConfig, repo_root: Path) -> None:
       }};
 """
 
-    # Insert new entry before closing brace
+    # Check if hostname already exists
-    replacement = rf"\g<1>\n{new_entry}\g<2>"
+    hostname_pattern = rf"^      {re.escape(config.hostname)} = nixpkgs\.lib\.nixosSystem"
+    existing_match = re.search(hostname_pattern, content, re.MULTILINE)
 
-    new_content, count = re.subn(pattern, replacement, content)
+    if existing_match and force:
+        # Replace existing entry
+        # Match the entire block from "hostname = " to "};"
+        replace_pattern = rf"^      {re.escape(config.hostname)} = nixpkgs\.lib\.nixosSystem \{{.*?^      \}};\n"
+        new_content, count = re.subn(replace_pattern, new_entry, content, flags=re.MULTILINE | re.DOTALL)
 
-    if count == 0:
+        if count == 0:
-        raise ValueError(
+            raise ValueError(f"Could not find existing entry for {config.hostname} in flake.nix")
-            "Could not find insertion point in flake.nix. "
+    else:
-            "Looking for pattern: '      };\\n      devShells ='"
+        # Insert new entry before closing brace of nixosConfigurations
-        )
+        # Pattern: "      };\n      packages = forAllSystems"
+        pattern = r"(      \};)\n(      packages = forAllSystems)"
+        replacement = rf"{new_entry}\g<1>\n\g<2>"
+
+        new_content, count = re.subn(pattern, replacement, content)
+
+        if count == 0:
+            raise ValueError(
+                "Could not find insertion point in flake.nix. "
+                "Looking for pattern: '      };\\n      packages = forAllSystems'"
+            )
 
     flake_path.write_text(new_content)
 
 
-def update_terraform_vms(config: HostConfig, repo_root: Path) -> None:
+def update_terraform_vms(config: HostConfig, repo_root: Path, force: bool = False) -> None:
     """
-    Add new VM entry to terraform/vms.tf locals.vms map.
+    Add or update VM entry in terraform/vms.tf locals.vms map.
 
     Args:
         config: Host configuration
         repo_root: Path to repository root
+        force: If True, replace existing entry; if False, insert new entry
     """
     terraform_path = repo_root / "terraform" / "vms.tf"
     content = terraform_path.read_text()
 
-    # Find the closing of locals.vms block
-    # Pattern: "  }\n\n  # Compute VM configurations"
-    pattern = r"(  \})\n\n(  # Compute VM configurations)"
-
     # Create new entry based on whether we have static IP or DHCP
     if config.is_static_ip:
         new_entry = f'''    "{config.hostname}" = {{
@@ -86,15 +95,30 @@ def update_terraform_vms(config: HostConfig, repo_root: Path) -> None:
     }}
 '''
 
-    # Insert new entry before closing brace
+    # Check if hostname already exists
-    replacement = rf"{new_entry}\g<1>\n\n\g<2>"
+    hostname_pattern = rf'^\s+"{re.escape(config.hostname)}" = \{{'
+    existing_match = re.search(hostname_pattern, content, re.MULTILINE)
 
-    new_content, count = re.subn(pattern, replacement, content)
+    if existing_match and force:
+        # Replace existing entry
+        # Match the entire block from "hostname" = { to }
+        replace_pattern = rf'^\s+"{re.escape(config.hostname)}" = \{{.*?^\s+\}}\n'
+        new_content, count = re.subn(replace_pattern, new_entry, content, flags=re.MULTILINE | re.DOTALL)
 
-    if count == 0:
+        if count == 0:
-        raise ValueError(
+            raise ValueError(f"Could not find existing entry for {config.hostname} in terraform/vms.tf")
-            "Could not find insertion point in terraform/vms.tf. "
+    else:
-            "Looking for pattern: '  }\\n\\n  # Compute VM configurations'"
+        # Insert new entry before closing brace
-        )
+        # Pattern: "  }\n\n  # Compute VM configurations"
+        pattern = r"(  \})\n\n(  # Compute VM configurations)"
+        replacement = rf"{new_entry}\g<1>\n\n\g<2>"
+
+        new_content, count = re.subn(pattern, replacement, content)
+
+        if count == 0:
+            raise ValueError(
+                "Could not find insertion point in terraform/vms.tf. "
+                "Looking for pattern: '  }\\n\\n  # Compute VM configurations'"
+            )
 
     terraform_path.write_text(new_content)

scripts/create-host/templates/configuration.nix.j2

@@ -7,16 +7,15 @@
 
 {
   imports = [
-    ../template/hardware-configuration.nix
+    ../template2/hardware-configuration.nix
 
     ../../system
     ../../common/vm
   ];
 
   nixpkgs.config.allowUnfree = true;
-  # Use the systemd-boot EFI boot loader.
   boot.loader.grub.enable = true;
-  boot.loader.grub.device = "/dev/sda";
+  boot.loader.grub.device = "/dev/vda";
 
   networking.hostName = "{{ hostname }}";
   networking.domain = "{{ domain }}";

terraform/README.md

@@ -87,6 +87,21 @@ vms = {
 }
 ```
 
+### Example: Test VM with Custom Git Branch
+
+For testing pipeline changes without polluting master:
+
+```hcl
+vms = {
+  "test-vm" = {
+    ip = "10.69.13.100/24"
+    flake_branch = "test-pipeline"  # Bootstrap from this branch
+  }
+}
+```
+
+This VM will bootstrap from the `test-pipeline` branch instead of `master`. Production VMs should omit the `flake_branch` field.
+
 ## Configuration Options
 
 Each VM in the `vms` map supports the following fields (all optional):
@@ -98,6 +113,7 @@ Each VM in the `vms` map supports the following fields (all optional):
 | `cpu_cores` | Number of CPU cores | `2` |
 | `memory` | Memory in MB | `2048` |
 | `disk_size` | Disk size (e.g., "20G", "100G") | `"20G"` |
+| `flake_branch` | Git branch for bootstrap (for testing, omit for production) | `master` |
 | `target_node` | Proxmox node to deploy to | `"pve1"` |
 | `template_name` | Template VM to clone from | `"nixos-25.11.20260128.fa83fd8"` |
 | `storage` | Storage backend | `"local-zfs"` |
@@ -182,6 +198,7 @@ deployment_summary = {
 - `main.tf` - Provider configuration
 - `variables.tf` - Variable definitions and defaults
 - `vms.tf` - VM definitions and deployment logic
+- `cloud-init.tf` - Cloud-init disk management (SSH keys, networking, branch config)
 - `outputs.tf` - Output definitions for deployed VMs
 - `terraform.tfvars.example` - Example credentials file
 - `terraform.tfvars` - Your actual credentials (gitignored)

terraform/cloud-init.tf

@@ -0,0 +1,58 @@
+# Cloud-init configuration for all VMs
+#
+# This file manages cloud-init disks for all VMs using the proxmox_cloud_init_disk resource.
+# VMs with flake_branch set will include NIXOS_FLAKE_BRANCH environment variable.
+
+resource "proxmox_cloud_init_disk" "ci" {
+  for_each = local.vm_configs
+
+  name     = each.key
+  pve_node = each.value.target_node
+  storage  = "local"  # Cloud-init disks must be on storage that supports ISO/snippets
+
+  # User data includes SSH keys and optionally NIXOS_FLAKE_BRANCH
+  user_data = <<-EOT
+  #cloud-config
+  ssh_authorized_keys:
+    - ${each.value.ssh_public_key}
+  ${each.value.flake_branch != null ? <<-BRANCH
+  write_files:
+    - path: /etc/environment
+      content: |
+        NIXOS_FLAKE_BRANCH=${each.value.flake_branch}
+      append: true
+  BRANCH
+: ""}
+  EOT
+
+  # Network configuration - static IP or DHCP
+  network_config = each.value.ip != null ? yamlencode({
+    version = 1
+    config = [{
+      type = "physical"
+      name = "ens18"
+      subnets = [{
+        type            = "static"
+        address         = each.value.ip
+        gateway         = each.value.gateway
+        dns_nameservers = split(" ", each.value.nameservers)
+        dns_search      = [each.value.search_domain]
+      }]
+    }]
+    }) : yamlencode({
+    version = 1
+    config = [{
+      type = "physical"
+      name = "ens18"
+      subnets = [{
+        type = "dhcp"
+      }]
+    }]
+  })
+
+  # Instance metadata
+  meta_data = yamlencode({
+    instance_id    = sha1(each.key)
+    local-hostname = each.key
+  })
+}

terraform/vms.tf

@@ -22,9 +22,22 @@ locals {
     #   disk_size = "50G"
     # }
 
+    # Example Test VM with custom git branch (for testing pipeline changes):
+    # "test-vm" = {
+    #   ip          = "10.69.13.100/24"
+    #   flake_branch = "test-pipeline"  # Bootstrap from this branch instead of master
+    # }
+
     # Example Minimal VM using all defaults (uncomment to deploy):
     # "minimal-vm" = {}
     # "bootstrap-verify-test" = {}
+    "testvm01" = {
+      ip           = "10.69.13.101/24"
+      cpu_cores    = 2
+      memory       = 2048
+      disk_size    = "20G"
+      flake_branch = "pipeline-testing-improvements"
+    }
   }
 
   # Compute VM configurations with defaults applied
@@ -44,6 +57,8 @@ locals {
       # Network configuration - detect DHCP vs static
       ip      = lookup(vm, "ip", null)
       gateway = lookup(vm, "gateway", var.default_gateway)
+      # Branch configuration for bootstrap (optional, uses master if not set)
+      flake_branch = lookup(vm, "flake_branch", null)
     }
   }
 }
@@ -89,8 +104,9 @@ resource "proxmox_vm_qemu" "vm" {
     }
     ide {
       ide2 {
-        cloudinit {
+        # Reference the custom cloud-init disk created in cloud-init.tf
-          storage = each.value.storage
+        cdrom {
+          iso = proxmox_cloud_init_disk.ci[each.key].id
         }
       }
     }
@@ -102,15 +118,6 @@ resource "proxmox_vm_qemu" "vm" {
   # Agent
   agent = 1
 
-  # Cloud-init configuration
-  ciuser       = "root"
-  sshkeys      = each.value.ssh_public_key
-  nameserver   = each.value.nameservers
-  searchdomain = each.value.search_domain
-
-  # Network configuration - DHCP or static IP
-  ipconfig0 = each.value.ip != null ? "ip=${each.value.ip},gw=${each.value.gateway}" : "ip=dhcp"
-
   # Skip IPv6 since we don't use it
   skip_ipv6 = true