CLAUDE.md

@@ -35,6 +35,21 @@ nix build .#create-host
 
 Do not automatically deploy changes. Deployments are usually done by updating the master branch, and then triggering the auto update on the specific host.
 
+### Testing Feature Branches on Hosts
+
+All hosts have the `nixos-rebuild-test` helper script for testing feature branches before merging:
+
+```bash
+# On the target host, test a feature branch
+nixos-rebuild-test boot <branch-name>
+nixos-rebuild-test switch <branch-name>
+
+# Additional arguments are passed through to nixos-rebuild
+nixos-rebuild-test boot my-feature --show-trace
+```
+
+When working on a feature branch that requires testing on a live host, suggest using this command instead of the full flake URL syntax.
+
 ### Flake Management
 
 ```bash
@@ -402,6 +417,8 @@ This means:
 
 **Firewall**: Disabled on most hosts (trusted network). Enable selectively in host configuration if needed.
 
+**Shell scripts**: Use `pkgs.writeShellApplication` instead of `pkgs.writeShellScript` or `pkgs.writeShellScriptBin` for creating shell scripts. `writeShellApplication` provides automatic shellcheck validation, sets strict bash options (`set -euo pipefail`), and allows declaring `runtimeInputs` for dependencies.
+
 ### Monitoring Stack
 
 All hosts ship metrics and logs to `monitoring01`:

hosts/template/scripts.nix

@@ -1,7 +1,9 @@
 { pkgs, ... }:
 let
-  prepare-host-script = pkgs.writeShellScriptBin "prepare-host.sh"
+  prepare-host-script = pkgs.writeShellApplication {
-    ''
+    name = "prepare-host.sh";
+    runtimeInputs = [ pkgs.age ];
+    text = ''
       echo "Removing machine-id"
       rm -f /etc/machine-id || true
 
@@ -24,8 +26,9 @@ let
       echo "Generate age key"
       rm -rf /var/lib/sops-nix || true
       mkdir -p /var/lib/sops-nix
-      ${pkgs.age}/bin/age-keygen -o /var/lib/sops-nix/key.txt
+      age-keygen -o /var/lib/sops-nix/key.txt
     '';
+  };
 in
 {
   environment.systemPackages = [ prepare-host-script ];

hosts/template2/scripts.nix

@@ -1,7 +1,9 @@
 { pkgs, ... }:
 let
-  prepare-host-script = pkgs.writeShellScriptBin "prepare-host.sh"
+  prepare-host-script = pkgs.writeShellApplication {
-    ''
+    name = "prepare-host.sh";
+    runtimeInputs = [ pkgs.age ];
+    text = ''
       echo "Removing machine-id"
       rm -f /etc/machine-id || true
 
@@ -24,8 +26,9 @@ let
       echo "Generate age key"
       rm -rf /var/lib/sops-nix || true
       mkdir -p /var/lib/sops-nix
-      ${pkgs.age}/bin/age-keygen -o /var/lib/sops-nix/key.txt
+      age-keygen -o /var/lib/sops-nix/key.txt
     '';
+  };
 in
 {
   environment.systemPackages = [ prepare-host-script ];

hosts/vaulttest01/configuration.nix

@@ -5,6 +5,32 @@
   ...
 }:
 
+let
+  vault-test-script = pkgs.writeShellApplication {
+    name = "vault-test";
+    text = ''
+      echo "=== Vault Secret Test ==="
+      echo "Secret path: hosts/vaulttest01/test-service"
+
+      if [ -f /run/secrets/test-service/password ]; then
+        echo "✓ Password file exists"
+        echo "Password length: $(wc -c < /run/secrets/test-service/password)"
+      else
+        echo "✗ Password file missing!"
+        exit 1
+      fi
+
+      if [ -d /var/lib/vault/cache/test-service ]; then
+        echo "✓ Cache directory exists"
+      else
+        echo "✗ Cache directory missing!"
+        exit 1
+      fi
+
+      echo "Test successful!"
+    '';
+  };
+in
 {
   imports = [
     ../template2/hardware-configuration.nix
@@ -79,27 +105,7 @@
       Type = "oneshot";
       RemainAfterExit = true;
 
-      ExecStart = pkgs.writeShellScript "vault-test" ''
+      ExecStart = lib.getExe vault-test-script;
-        echo "=== Vault Secret Test ==="
-        echo "Secret path: hosts/vaulttest01/test-service"
-
-        if [ -f /run/secrets/test-service/password ]; then
-          echo "✓ Password file exists"
-          echo "Password length: $(wc -c < /run/secrets/test-service/password)"
-        else
-          echo "✗ Password file missing!"
-          exit 1
-        fi
-
-        if [ -d /var/lib/vault/cache/test-service ]; then
-          echo "✓ Cache directory exists"
-        else
-          echo "✗ Cache directory missing!"
-          exit 1
-        fi
-
-        echo "Test successful!"
-      '';
 
       StandardOutput = "journal+console";
     };

services/monitoring/prometheus.nix

@@ -7,42 +7,44 @@ let
   autoScrapeConfigs = monLib.generateScrapeConfigs self externalTargets;
 
   # Script to fetch AppRole token for Prometheus to use when scraping OpenBao metrics
-  fetchOpenbaoToken = pkgs.writeShellScript "fetch-openbao-token" ''
+  fetchOpenbaoToken = pkgs.writeShellApplication {
-    set -euo pipefail
+    name = "fetch-openbao-token";
+    runtimeInputs = [ pkgs.curl pkgs.jq ];
+    text = ''
+      VAULT_ADDR="https://vault01.home.2rjus.net:8200"
+      APPROLE_DIR="/var/lib/vault/approle"
+      OUTPUT_FILE="/run/secrets/prometheus/openbao-token"
 
-    VAULT_ADDR="https://vault01.home.2rjus.net:8200"
+      # Read AppRole credentials
-    APPROLE_DIR="/var/lib/vault/approle"
+      if [ ! -f "$APPROLE_DIR/role-id" ] || [ ! -f "$APPROLE_DIR/secret-id" ]; then
-    OUTPUT_FILE="/run/secrets/prometheus/openbao-token"
+        echo "AppRole credentials not found at $APPROLE_DIR" >&2
+        exit 1
+      fi
 
-    # Read AppRole credentials
+      ROLE_ID=$(cat "$APPROLE_DIR/role-id")
-    if [ ! -f "$APPROLE_DIR/role-id" ] || [ ! -f "$APPROLE_DIR/secret-id" ]; then
+      SECRET_ID=$(cat "$APPROLE_DIR/secret-id")
-      echo "AppRole credentials not found at $APPROLE_DIR" >&2
-      exit 1
-    fi
 
-    ROLE_ID=$(cat "$APPROLE_DIR/role-id")
+      # Authenticate to Vault
-    SECRET_ID=$(cat "$APPROLE_DIR/secret-id")
+      AUTH_RESPONSE=$(curl -sf -k -X POST \
+        -d "{\"role_id\":\"$ROLE_ID\",\"secret_id\":\"$SECRET_ID\"}" \
+        "$VAULT_ADDR/v1/auth/approle/login")
 
-    # Authenticate to Vault
+      # Extract token
-    AUTH_RESPONSE=$(${pkgs.curl}/bin/curl -sf -k -X POST \
+      VAULT_TOKEN=$(echo "$AUTH_RESPONSE" | jq -r '.auth.client_token')
-      -d "{\"role_id\":\"$ROLE_ID\",\"secret_id\":\"$SECRET_ID\"}" \
+      if [ -z "$VAULT_TOKEN" ] || [ "$VAULT_TOKEN" = "null" ]; then
-      "$VAULT_ADDR/v1/auth/approle/login")
+        echo "Failed to extract Vault token from response" >&2
+        exit 1
+      fi
 
-    # Extract token
+      # Write token to file
-    VAULT_TOKEN=$(echo "$AUTH_RESPONSE" | ${pkgs.jq}/bin/jq -r '.auth.client_token')
+      mkdir -p "$(dirname "$OUTPUT_FILE")"
-    if [ -z "$VAULT_TOKEN" ] || [ "$VAULT_TOKEN" = "null" ]; then
+      echo -n "$VAULT_TOKEN" > "$OUTPUT_FILE"
-      echo "Failed to extract Vault token from response" >&2
+      chown prometheus:prometheus "$OUTPUT_FILE"
-      exit 1
+      chmod 0400 "$OUTPUT_FILE"
-    fi
 
-    # Write token to file
+      echo "Successfully fetched OpenBao token"
-    mkdir -p "$(dirname "$OUTPUT_FILE")"
+    '';
-    echo -n "$VAULT_TOKEN" > "$OUTPUT_FILE"
+  };
-    chown prometheus:prometheus "$OUTPUT_FILE"
-    chmod 0400 "$OUTPUT_FILE"
-
-    echo "Successfully fetched OpenBao token"
-  '';
 in
 {
   # Systemd service to fetch AppRole token for Prometheus OpenBao scraping

system/default.nix

@@ -4,6 +4,7 @@
     ./acme.nix
     ./autoupgrade.nix
     ./monitoring
+    ./motd.nix
     ./packages.nix
     ./nix.nix
     ./root-user.nix

system/motd.nix

@@ -0,0 +1,28 @@
+{ config, lib, self, ... }:
+
+let
+  hostname = config.networking.hostName;
+  domain = config.networking.domain or "";
+  fqdn = if domain != "" then "${hostname}.${domain}" else hostname;
+
+  # Get commit hash (handles both clean and dirty trees)
+  shortRev = self.shortRev or self.dirtyShortRev or "unknown";
+
+  # Format timestamp from lastModified (Unix timestamp)
+  # lastModifiedDate is in format "YYYYMMDDHHMMSS"
+  dateStr = self.sourceInfo.lastModifiedDate or "unknown";
+  formattedDate = if dateStr != "unknown" then
+    "${builtins.substring 0 4 dateStr}-${builtins.substring 4 2 dateStr}-${builtins.substring 6 2 dateStr} ${builtins.substring 8 2 dateStr}:${builtins.substring 10 2 dateStr}"
+  else
+    "unknown";
+
+  banner = ''
+    ####################################
+     ${fqdn}
+     Commit: ${shortRev} (${formattedDate})
+    ####################################
+  '';
+in
+{
+  users.motd = lib.mkDefault banner;
+}

system/nix.nix

@@ -1,5 +1,25 @@
-{ lib, ... }:
+{ lib, pkgs, ... }:
+let
+  nixos-rebuild-test = pkgs.writeShellApplication {
+    name = "nixos-rebuild-test";
+    runtimeInputs = [ pkgs.nixos-rebuild ];
+    text = ''
+      if [ $# -lt 2 ]; then
+        echo "Usage: nixos-rebuild-test <action> <branch>"
+        echo "Example: nixos-rebuild-test boot my-feature-branch"
+        exit 1
+      fi
+
+      action="$1"
+      branch="$2"
+      shift 2
+
+      exec nixos-rebuild "$action" --flake "git+https://git.t-juice.club/torjus/nixos-servers.git?ref=$branch" "$@"
+    '';
+  };
+in
 {
+  environment.systemPackages = [ nixos-rebuild-test ];
   nix = {
     gc = {
       automatic = true;

system/vault-secrets.nix

@@ -8,6 +8,48 @@ let
   # Import vault-fetch package
   vault-fetch = pkgs.callPackage ../scripts/vault-fetch { };
 
+  # Helper to create fetch scripts using writeShellApplication
+  mkFetchScript = name: secretCfg: pkgs.writeShellApplication {
+    name = "fetch-${name}";
+    runtimeInputs = [ vault-fetch ];
+    text = ''
+      # Set Vault environment variables
+      export VAULT_ADDR="${cfg.vaultAddress}"
+      export VAULT_SKIP_VERIFY="${if cfg.skipTlsVerify then "1" else "0"}"
+    '' + (if secretCfg.extractKey != null then ''
+      # Fetch to temporary directory, then extract single key
+      TMPDIR=$(mktemp -d)
+      trap 'rm -rf $TMPDIR' EXIT
+
+      vault-fetch \
+        "${secretCfg.secretPath}" \
+        "$TMPDIR" \
+        "${secretCfg.cacheDir}"
+
+      # Extract the specified key and write as a single file
+      if [ ! -f "$TMPDIR/${secretCfg.extractKey}" ]; then
+        echo "ERROR: Key '${secretCfg.extractKey}' not found in secret" >&2
+        exit 1
+      fi
+
+      # Ensure parent directory exists
+      mkdir -p "$(dirname "${secretCfg.outputDir}")"
+      cp "$TMPDIR/${secretCfg.extractKey}" "${secretCfg.outputDir}"
+      chown ${secretCfg.owner}:${secretCfg.group} "${secretCfg.outputDir}"
+      chmod ${secretCfg.mode} "${secretCfg.outputDir}"
+    '' else ''
+      # Fetch secret as directory of files
+      vault-fetch \
+        "${secretCfg.secretPath}" \
+        "${secretCfg.outputDir}" \
+        "${secretCfg.cacheDir}"
+
+      # Set ownership and permissions
+      chown -R ${secretCfg.owner}:${secretCfg.group} "${secretCfg.outputDir}"
+      chmod ${secretCfg.mode} "${secretCfg.outputDir}"/*
+    '');
+  };
+
   # Secret configuration type
   secretType = types.submodule ({ name, config, ... }: {
     options = {
@@ -162,44 +204,7 @@ in
           RemainAfterExit = true;
 
           # Fetch the secret
-          ExecStart = pkgs.writeShellScript "fetch-${name}" (''
+          ExecStart = lib.getExe (mkFetchScript name secretCfg);
-            set -euo pipefail
-
-            # Set Vault environment variables
-            export VAULT_ADDR="${cfg.vaultAddress}"
-            export VAULT_SKIP_VERIFY="${if cfg.skipTlsVerify then "1" else "0"}"
-          '' + (if secretCfg.extractKey != null then ''
-            # Fetch to temporary directory, then extract single key
-            TMPDIR=$(mktemp -d)
-            trap "rm -rf $TMPDIR" EXIT
-
-            ${vault-fetch}/bin/vault-fetch \
-              "${secretCfg.secretPath}" \
-              "$TMPDIR" \
-              "${secretCfg.cacheDir}"
-
-            # Extract the specified key and write as a single file
-            if [ ! -f "$TMPDIR/${secretCfg.extractKey}" ]; then
-              echo "ERROR: Key '${secretCfg.extractKey}' not found in secret" >&2
-              exit 1
-            fi
-
-            # Ensure parent directory exists
-            mkdir -p "$(dirname "${secretCfg.outputDir}")"
-            cp "$TMPDIR/${secretCfg.extractKey}" "${secretCfg.outputDir}"
-            chown ${secretCfg.owner}:${secretCfg.group} "${secretCfg.outputDir}"
-            chmod ${secretCfg.mode} "${secretCfg.outputDir}"
-          '' else ''
-            # Fetch secret as directory of files
-            ${vault-fetch}/bin/vault-fetch \
-              "${secretCfg.secretPath}" \
-              "${secretCfg.outputDir}" \
-              "${secretCfg.cacheDir}"
-
-            # Set ownership and permissions
-            chown -R ${secretCfg.owner}:${secretCfg.group} "${secretCfg.outputDir}"
-            chmod ${secretCfg.mode} "${secretCfg.outputDir}"/*
-          ''));
 
           # Logging
           StandardOutput = "journal";