monitoring: add hostname label to all scrape targets

Add a `hostname` label to all Prometheus scrape targets, making it easy
to query all metrics for a host without wildcarding the instance label.

Example queries:
- {hostname="ns1"} - all metrics from ns1
- node_cpu_seconds_total{hostname="monitoring01"} - specific metric

For external targets (like gunter), the hostname is extracted from the
target string.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

lib/monitoring.nix

@@ -57,9 +57,11 @@ let
         labels = hostConfig'.labels or { };
       };
 
-  # Build effective labels for a host (only include non-default values)
+  # Build effective labels for a host
+  # Always includes hostname; only includes tier/priority/role if non-default
   buildEffectiveLabels = host:
-    (lib.optionalAttrs (host.tier != "prod") { tier = host.tier; })
+    { hostname = host.hostname; }
+    // (lib.optionalAttrs (host.tier != "prod") { tier = host.tier; })
     // (lib.optionalAttrs (host.priority != "high") { priority = host.priority; })
     // (lib.optionalAttrs (host.role != null) { role = host.role; })
     // host.labels;
@@ -73,6 +75,10 @@ let
         lib.mapAttrsToList extractHostMonitoring nixosConfigs
       );
 
+      # Extract hostname from a target string like "gunter.home.2rjus.net:9100"
+      extractHostnameFromTarget = target:
+        builtins.head (lib.splitString "." target);
+
       # Build target entries with labels for each host
       flakeEntries = map
         (host: {
@@ -81,9 +87,12 @@ let
         })
         hostList;
 
-      # External targets have no labels
+      # External targets get hostname extracted from the target string
       externalEntries = map
-        (target: { inherit target; labels = { }; })
+        (target: {
+          inherit target;
+          labels = { hostname = extractHostnameFromTarget target; };
+        })
         (externalTargets.nodeExporter or [ ]);
 
       allEntries = flakeEntries ++ externalEntries;
@@ -94,13 +103,13 @@ let
       grouped = lib.groupBy labelKey allEntries;
 
       # Convert groups to static_configs format
+      # Every flake host now has at least a hostname label
       staticConfigs = lib.mapAttrsToList
         (key: entries:
           let
             labels = (builtins.head entries).labels;
           in
-          { targets = map (e: e.target) entries; }
+          { targets = map (e: e.target) entries; labels = labels; }
-          // (lib.optionalAttrs (labels != { }) { inherit labels; })
         )
         grouped;
     in
@@ -143,6 +152,7 @@ let
             labelKey = t: builtins.toJSON t.hostLabels;
             groupedByLabels = lib.groupBy labelKey targets;
 
+            # Every flake host now has at least a hostname label
             staticConfigs = lib.mapAttrsToList
               (key: labelTargets:
                 let
@@ -151,8 +161,7 @@ let
                     (t: "${t.hostname}.home.2rjus.net:${toString t.port}")
                     labelTargets;
                 in
-                { targets = targetAddrs; }
+                { targets = targetAddrs; labels = labels; }
-                // (lib.optionalAttrs (labels != { }) { inherit labels; })
               )
               groupedByLabels;