monitoring: separate node-exporter-only external targets

Add nodeExporterOnly list to external-targets.nix for hosts that
have node-exporter but not systemd-exporter (e.g. pve1). This
prevents a down target in the systemd-exporter scrape job.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

lib/monitoring.nix

@@ -94,7 +94,15 @@ let
         })
         (externalTargets.nodeExporter or [ ]);
 
-      allEntries = flakeEntries ++ externalEntries;
+      # Node-exporter-only external targets (no systemd-exporter)
+      externalOnlyEntries = map
+        (target: {
+          inherit target;
+          labels = { hostname = extractHostnameFromTarget target; };
+        })
+        (externalTargets.nodeExporterOnly or [ ]);
+
+      allEntries = flakeEntries ++ externalEntries ++ externalOnlyEntries;
 
       # Group entries by their label set for efficient static_configs
       # Convert labels attrset to a string key for grouping
@@ -203,7 +211,18 @@ let
     in
     flakeScrapeConfigs ++ externalScrapeConfigs;
 
+  # Generate systemd-exporter targets (excludes nodeExporterOnly hosts)
+  generateSystemdExporterTargets = self: externalTargets:
+    let
+      nodeTargets = generateNodeExporterTargets self (externalTargets // { nodeExporterOnly = [ ]; });
+    in
+    map
+      (cfg: cfg // {
+        targets = map (t: builtins.replaceStrings [ ":9100" ] [ ":9558" ] t) cfg.targets;
+      })
+      nodeTargets;
+
 in
 {
-  inherit extractHostMonitoring generateNodeExporterTargets generateScrapeConfigs;
+  inherit extractHostMonitoring generateNodeExporterTargets generateScrapeConfigs generateSystemdExporterTargets;
 }

services/monitoring/external-targets.nix

@@ -3,6 +3,9 @@
 {
   nodeExporter = [
     "gunter.home.2rjus.net:9100"
+  ];
+  # Hosts with node-exporter but no systemd-exporter
+  nodeExporterOnly = [
     "pve1.home.2rjus.net:9100"
   ];
   scrapeConfigs = [

services/victoriametrics/default.nix

@@ -4,6 +4,7 @@ let
   externalTargets = import ../monitoring/external-targets.nix;
 
   nodeExporterTargets = monLib.generateNodeExporterTargets self externalTargets;
+  systemdExporterTargets = monLib.generateSystemdExporterTargets self externalTargets;
   autoScrapeConfigs = monLib.generateScrapeConfigs self externalTargets;
 
   # TLS endpoints to monitor for certificate expiration via blackbox exporter
@@ -70,14 +71,10 @@ let
       job_name = "node-exporter";
       static_configs = nodeExporterTargets;
     }
-    # Systemd exporter on all hosts (same targets, different port)
+    # Systemd exporter on hosts that have it (excludes nodeExporterOnly hosts)
     {
       job_name = "systemd-exporter";
-      static_configs = map
-        (cfg: cfg // {
-          targets = map (t: builtins.replaceStrings [ ":9100" ] [ ":9558" ] t) cfg.targets;
-        })
-        nodeExporterTargets;
+      static_configs = systemdExporterTargets;
     }
     # Local monitoring services
     {