flake.lock: Update

Flake lock file updates:

• Updated input 'nixpkgs':
    'github:nixos/nixpkgs/d179d77c139e0a3f5c416477f7747e9d6b7ec315?narHash=sha256-aSgK4BLNFFGvDTNKPeB28lVXYqVn8RdyXDNAvgGq%2Bk0%3D' (2025-09-09)
  → 'github:nixos/nixpkgs/8cd5ce828d5d1d16feff37340171a98fc3bf6526?narHash=sha256-mCxPABZ6jRjUQx3bPP4vjA68ETbPLNz9V2pk9tO7pRQ%3D' (2025-09-10)
• Updated input 'nixpkgs-unstable':
    'github:nixos/nixpkgs/b599843bad24621dcaa5ab60dac98f9b0eb1cabe?narHash=sha256-tLdkkC6XnsY9EOZW9TlpesTclELy8W7lL2ClL%2Bnma8o%3D' (2025-09-08)
  → 'github:nixos/nixpkgs/ab0f3607a6c7486ea22229b92ed2d355f1482ee0?narHash=sha256-zwE/e7CuPJUWKdvvTCB7iunV4E/%2BG0lKfv4kk/5Izdg%3D' (2025-09-10)

flake.lock

@@ -49,11 +49,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1748050424,
+        "lastModified": 1748983975,
-        "narHash": "sha256-RnFEe8mjcaSa5jhEgIcZabuzj3TmflSwt1qK7HBe/4A=",
+        "narHash": "sha256-DA5mOqxwLMj/XLb4hvBU1WtE6cuVej7PjUr8N0EZsCE=",
         "ref": "master",
-        "rev": "586440fe8d55db3e94ee623180bb65472ca04a29",
+        "rev": "040a73e891a70ff06ec7ab31d7167914129dbf7d",
-        "revCount": 10,
+        "revCount": 17,
         "type": "git",
         "url": "https://git.t-juice.club/torjus/labmon"
       },
@@ -65,11 +65,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1747953325,
+        "lastModified": 1757545623,
-        "narHash": "sha256-y2ZtlIlNTuVJUZCqzZAhIw5rrKP4DOSklev6c8PyCkQ=",
+        "narHash": "sha256-mCxPABZ6jRjUQx3bPP4vjA68ETbPLNz9V2pk9tO7pRQ=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "55d1f923c480dadce40f5231feb472e81b0bab48",
+        "rev": "8cd5ce828d5d1d16feff37340171a98fc3bf6526",
         "type": "github"
       },
       "original": {
@@ -81,11 +81,11 @@
     },
     "nixpkgs-unstable": {
       "locked": {
-        "lastModified": 1747744144,
+        "lastModified": 1757487488,
-        "narHash": "sha256-W7lqHp0qZiENCDwUZ5EX/lNhxjMdNapFnbErcbnP11Q=",
+        "narHash": "sha256-zwE/e7CuPJUWKdvvTCB7iunV4E/+G0lKfv4kk/5Izdg=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "2795c506fe8fb7b03c36ccb51f75b6df0ab2553f",
+        "rev": "ab0f3607a6c7486ea22229b92ed2d355f1482ee0",
         "type": "github"
       },
       "original": {
@@ -112,11 +112,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1747603214,
+        "lastModified": 1757503115,
-        "narHash": "sha256-lAblXm0VwifYCJ/ILPXJwlz0qNY07DDYdLD+9H+Wc8o=",
+        "narHash": "sha256-S9F6bHUBh+CFEUalv/qxNImRapCxvSnOzWBUZgK1zDU=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "8d215e1c981be3aa37e47aeabd4e61bb069548fd",
+        "rev": "0bf793823386187dff101ee2a9d4ed26de8bbf8c",
         "type": "github"
       },
       "original": {

flake.nix

@@ -156,40 +156,6 @@
             backup-helper.nixosModules.backup-helper
           ];
         };
-        inc1 = nixpkgs.lib.nixosSystem {
-          inherit system;
-          specialArgs = {
-            inherit inputs self sops-nix;
-          };
-          modules = [
-            (
-              { config, pkgs, ... }:
-              {
-                nixpkgs.overlays = commonOverlays;
-              }
-            )
-            ./hosts/inc1
-            sops-nix.nixosModules.sops
-            # backup-helper.nixosModules.backup-helper
-          ];
-        };
-        inc2 = nixpkgs.lib.nixosSystem {
-          inherit system;
-          specialArgs = {
-            inherit inputs self sops-nix;
-          };
-          modules = [
-            (
-              { config, pkgs, ... }:
-              {
-                nixpkgs.overlays = commonOverlays;
-              }
-            )
-            ./hosts/inc2
-            sops-nix.nixosModules.sops
-            # backup-helper.nixosModules.backup-helper
-          ];
-        };
         template1 = nixpkgs.lib.nixosSystem {
           inherit system;
           specialArgs = {

hosts/inc1/configuration.nix

@@ -1,96 +0,0 @@
-# Edit this configuration file to define what should be installed on
-# your system. Help is available in the configuration.nix(5) man page, on
-# https://search.nixos.org/options and in the NixOS manual (`nixos-help`).
-
-{ config, lib, pkgs, ... }:
-
-{
-  imports =
-    [
-      # Include the results of the hardware scan.
-      ./hardware-configuration.nix
-      ../../system
-      ../../services/incus
-    ];
-
-  # Use the systemd-boot EFI boot loader.
-  boot.loader.systemd-boot.enable = true;
-  boot.loader.efi.canTouchEfiVariables = true;
-
-  boot.kernel.sysctl = {
-    "net.ipv4.ip_forward" = 1;
-  };
-
-  networking.hostName = "inc1";
-  networking.domain = "home.2rjus.net";
-  networking.useNetworkd = true;
-  networking.useDHCP = false;
-  networking.nftables.enable = true;
-  networking.firewall.trustedInterfaces = [ "vlan13" ];
-
-  services.resolved.enable = true;
-  networking.nameservers = [
-    "10.69.13.5"
-    "10.69.13.6"
-  ];
-
-  systemd.network.enable = true;
-  # Primary interface
-  systemd.network.networks."enp2s0" = {
-    matchConfig.Name = "enp2s0";
-    address = [
-      "10.69.12.80/24"
-    ];
-    networkConfig = {
-      VLAN = [ "enp2s0.13" ];
-    };
-    routes = [
-      { Gateway = "10.69.12.1"; }
-    ];
-    linkConfig.RequiredForOnline = "routable";
-  };
-
-  # VLAN 13 netdev
-  systemd.network.netdevs."enp2s0.13" = {
-    enable = true;
-    netdevConfig = {
-      Kind = "vlan";
-      Name = "enp2s0.13";
-    };
-    vlanConfig = {
-      Id = 13;
-    };
-  };
-
-  # # Bridge netdev
-  # systemd.network.netdevs."br13" = {
-  #   netdevConfig = {
-  #     Name = "br13";
-  #     Kind = "bridge";
-  #   };
-  # };
-
-  # # Bridge network
-  # systemd.network.networks."br13" = {
-  #   matchConfig.Name = "enp2s0.13";
-  #   networkConfig.Bridge = "br13";
-  # };
-
-  time.timeZone = "Europe/Oslo";
-
-  nix.settings.experimental-features = [ "nix-command" "flakes" ];
-  nix.settings.tarball-ttl = 0;
-  environment.systemPackages = with pkgs; [
-    tcpdump
-    vim
-    wget
-    git
-  ];
-
-  # Enable the OpenSSH daemon.
-  # services.openssh.enable = true;
-  # services.openssh.settings.PermitRootLogin = "yes";
-
-  system.stateVersion = "24.05"; # Did you read the comment?
-}
-

hosts/inc1/default.nix

@@ -1,5 +0,0 @@
-{ ... }: {
-  imports = [
-    ./configuration.nix
-  ];
-}

hosts/inc1/hardware-configuration.nix

@@ -1,41 +0,0 @@
-# Do not modify this file!  It was generated by ‘nixos-generate-config’
-# and may be overwritten by future invocations.  Please make changes
-# to /etc/nixos/configuration.nix instead.
-{ config, lib, pkgs, modulesPath, ... }:
-
-{
-  imports =
-    [ (modulesPath + "/installer/scan/not-detected.nix")
-    ];
-
-  boot.initrd.availableKernelModules = [ "xhci_pci" "nvme" "ahci" "usbhid" "usb_storage" "sd_mod" "rtsx_usb_sdmmc" ];
-  boot.initrd.kernelModules = [ ];
-  boot.kernelModules = [ "kvm-amd" ];
-  boot.extraModulePackages = [ ];
-
-  fileSystems."/" =
-    { device = "/dev/disk/by-uuid/faa60038-b3a4-448a-8909-49857818c955";
-      fsType = "xfs";
-    };
-
-  fileSystems."/boot" =
-    { device = "/dev/disk/by-uuid/7A94-A91C";
-      fsType = "vfat";
-      options = [ "fmask=0077" "dmask=0077" ];
-    };
-
-  swapDevices =
-    [ { device = "/dev/disk/by-uuid/f7a4f85e-0b4b-492d-a611-f50d2b915c2c"; }
-    ];
-
-  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
-  # (the default) this is the recommended approach. When using systemd-networkd it's
-  # still possible to use this option, but it's recommended to use it in conjunction
-  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
-  networking.useDHCP = lib.mkDefault true;
-  # networking.interfaces.enp2s0.useDHCP = lib.mkDefault true;
-  # networking.interfaces.wlp3s0.useDHCP = lib.mkDefault true;
-
-  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
-  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
-}

hosts/inc2/configuration.nix

@@ -1,96 +0,0 @@
-# Edit this configuration file to define what should be installed on
-# your system. Help is available in the configuration.nix(5) man page, on
-# https://search.nixos.org/options and in the NixOS manual (`nixos-help`).
-
-{ config, lib, pkgs, ... }:
-
-{
-  imports =
-    [
-      # Include the results of the hardware scan.
-      ./hardware-configuration.nix
-      ../../system
-      ../../services/incus
-    ];
-
-  # Use the systemd-boot EFI boot loader.
-  boot.loader.systemd-boot.enable = true;
-  boot.loader.efi.canTouchEfiVariables = true;
-
-  boot.kernel.sysctl = {
-    "net.ipv4.ip_forward" = 1;
-  };
-
-  networking.hostName = "inc2";
-  networking.domain = "home.2rjus.net";
-  networking.useNetworkd = true;
-  networking.useDHCP = false;
-  networking.nftables.enable = true;
-  networking.firewall.trustedInterfaces = [ "vlan13" ];
-
-  services.resolved.enable = true;
-  networking.nameservers = [
-    "10.69.13.5"
-    "10.69.13.6"
-  ];
-
-  systemd.network.enable = true;
-  # Primary interface
-  systemd.network.networks."enp2s0" = {
-    matchConfig.Name = "enp2s0";
-    address = [
-      "10.69.12.81/24"
-    ];
-    networkConfig = {
-      VLAN = [ "enp2s0.13" ];
-    };
-    routes = [
-      { Gateway = "10.69.12.1"; }
-    ];
-    linkConfig.RequiredForOnline = "routable";
-  };
-
-  # VLAN 13 netdev
-  systemd.network.netdevs."enp2s0.13" = {
-    enable = true;
-    netdevConfig = {
-      Kind = "vlan";
-      Name = "enp2s0.13";
-    };
-    vlanConfig = {
-      Id = 13;
-    };
-  };
-
-  # # Bridge netdev
-  # systemd.network.netdevs."br13" = {
-  #   netdevConfig = {
-  #     Name = "br13";
-  #     Kind = "bridge";
-  #   };
-  # };
-
-  # # Bridge network
-  # systemd.network.networks."br13" = {
-  #   matchConfig.Name = "enp2s0.13";
-  #   networkConfig.Bridge = "br13";
-  # };
-
-  time.timeZone = "Europe/Oslo";
-
-  nix.settings.experimental-features = [ "nix-command" "flakes" ];
-  nix.settings.tarball-ttl = 0;
-  environment.systemPackages = with pkgs; [
-    tcpdump
-    vim
-    wget
-    git
-  ];
-
-  # Enable the OpenSSH daemon.
-  # services.openssh.enable = true;
-  # services.openssh.settings.PermitRootLogin = "yes";
-
-  system.stateVersion = "24.05"; # Did you read the comment?
-}
-

hosts/inc2/default.nix

@@ -1,5 +0,0 @@
-{ ... }: {
-  imports = [
-    ./configuration.nix
-  ];
-}

hosts/inc2/hardware-configuration.nix

@@ -1,33 +0,0 @@
-{ config, lib, pkgs, modulesPath, ... }:
-
-{
-  imports =
-    [
-      (modulesPath + "/installer/scan/not-detected.nix")
-    ];
-
-  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usb_storage" "usbhid" "sd_mod" "rtsx_usb_sdmmc" ];
-  boot.initrd.kernelModules = [ ];
-  boot.kernelModules = [ "kvm-amd" ];
-  boot.extraModulePackages = [ ];
-
-  fileSystems."/" =
-    {
-      device = "/dev/disk/by-uuid/3e7c311c-b1a3-4be7-b8bf-e497cba64302";
-      fsType = "btrfs";
-    };
-
-  fileSystems."/boot" =
-    {
-      device = "/dev/disk/by-uuid/F0D7-E5C1";
-      fsType = "vfat";
-      options = [ "fmask=0022" "dmask=0022" ];
-    };
-
-  swapDevices =
-    [{ device = "/dev/disk/by-uuid/1a06a36f-da61-4d36-b94e-b852836c328a"; }];
-
-  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
-  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
-}
-

hosts/nix-cache01/default.nix

@@ -3,5 +3,7 @@
   imports = [
     ./configuration.nix
     ../../services/nix-cache
+    ../../services/actions-runner
+    ./zram.nix
   ];
 }

hosts/nix-cache01/zram.nix

@@ -0,0 +1,6 @@
+{ ... }:
+{
+  zramSwap = {
+    enable = true;
+  };
+}

secrets/nix-cache01/actions_token_1

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:P84qHFU+xQjwQGK8I1gIdcBsHrskuUg0M1nGMMaA+hFjAdFYUhdhmAN/+y0CO28=,iv:zJtk01zNMTBDQdVtZBTM34CHRaNYDkabolxh7PWGKUI=,tag:8AS80AbZJbh9B3Av3zuI1w==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age1lznyk4ee7e7x8n92cq2n87kz9920473ks5u9jlhd3dczfzq4wamqept56u",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSdHozVnN0YXExTWZBbEVK\nT0FoR3NNREtHZzhseE5jb0JzNCtCSDBtT0RVCkQvVFB5aDlxbzVDVE85Q3d5TEw4\nelpuVnY1bXc4YU4vQ0RxbW15SWpwMVkKLS0tIG1HZ2NhR0plSVlZdVNZZUZydjBv\nSjEyREIrL1Z5bkpOM2ZiRnhmRlk0MVUKRowIdTtV7B+me9cdpC0Kmnz3FIQQvCt2\nxVltjChE4N954aa1j8KpXYELkr0rge2/ka9JdI54VxgrACPSbtVqGg==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1a0477laj9sdh79wdas5v7hzk6au8fach74njg8epfw2rdht90qjsakkwd6",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2RlVRbGJiSjNkNXg3T3FU\nZFVzeTNSdGxSWEVzVTFDR1g5ZzI5RHZkMmhRCkZGbE1ZSTJDZ2NLZG5QU25OU3Z4\nRlYyU0N5ZTlQQmgrZ3dBVTgvYTBHR1kKLS0tIENRTmFWYzY2d2t4VTRISEtxZkNL\nZ2IzdVZWNWowZ2hlcS9xM29UaUgzcFkKMSzJh8tVqLUE8joiynqqHlZD5wMne0Ti\n/RE3d8JEwlQZnaxd8ZYfOA9CHIYhsMgWV8YG+hDR0nFPBah6sjRsrw==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-08-21T19:08:48Z",
+		"mac": "ENC[AES256_GCM,data:5CkO09NIqttb4UZPB9iGym8avhTsMeUkTFTKZJlNGjgB1qWyGQNeKCa50A1+SbBCCWE5EwxoynB1so7bi8vnq7k8CPUHbiWG8rLOJSYHQcZ9Tu7ZGtpeWPcCw1zPWJ/PTBsFVeaT5/ufdx/6ut+sTtRoKHOZZtO9oStHmu/Rlfg=,iv:z9iJJlbvhgxJaART5QoCrqvrqlgoVlGj8jlndCALmKU=,tag:ldjmND4NVVQrHUldLrB4Jg==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}

services/actions-runner/default.nix

@@ -0,0 +1,55 @@
+{ pkgs, config, ... }:
+{
+  sops.secrets."actions-token-1" = {
+    sopsFile = ../../secrets/nix-cache01/actions_token_1;
+    format = "binary";
+  };
+
+  virtualisation.podman = {
+    enable = true;
+    dockerCompat = true;
+  };
+
+  services.gitea-actions-runner.instances = {
+    actions1 = {
+      enable = true;
+      tokenFile = config.sops.secrets.actions-token-1.path;
+      name = "actions1.home.2rjus.net";
+      settings = {
+        log = {
+          level = "debug";
+        };
+
+        runner = {
+          file = ".runner";
+          capacity = 4;
+          timeout = "2h";
+          shutdown_timeout = "10m";
+          insecure = false;
+          fetch_timeout = "10s";
+          fetch_interval = "30s";
+        };
+
+        cache = {
+          enabled = true;
+          dir = "/var/cache/gitea-actions1";
+        };
+
+        container = {
+          privileged = false;
+        };
+      };
+      labels =
+        builtins.map (n: "${n}:docker://gitea/runner-images:${n}") [
+          "ubuntu-latest"
+          "ubuntu-latest-slim"
+          "ubuntu-latest-full"
+        ]
+        ++ [
+          "homelab"
+        ];
+
+      url = "https://git.t-juice.club";
+    };
+  };
+}

services/home-assistant/default.nix

@@ -12,6 +12,7 @@
         aioshelly
         bellows
         gtts
+        ha-silabs-firmware-client
         isal
         paho-mqtt
         prometheus-client

services/http-proxy/proxy.nix

@@ -6,6 +6,10 @@
     configFile = pkgs.writeText "Caddyfile" ''
       {
         acme_ca https://ca.home.2rjus.net/acme/acme/directory
+
+        metrics {
+          per_host
+        }
       }
 
       nzbget.home.2rjus.net {
@@ -106,6 +110,14 @@
         }
         reverse_proxy http://monitoring01.home.2rjus.net:4040
       }
+      pushgw.home.2rjus.net {
+        log {
+          output file /var/log/caddy/pushgw.log {
+            mode 644
+          }
+        }
+        reverse_proxy http://monitoring01.home.2rjus.net:9091
+      }
       http://http-proxy.home.2rjus.net/metrics {
         log {
           output file /var/log/caddy/caddy-metrics.log {

services/incus/default.nix

@@ -1,7 +0,0 @@
-{ pkgs, config, ... }:
-{
-  virtualisation.incus = {
-    enable = true;
-  };
-  networking.firewall.allowedTCPPorts = [ 8443 ];
-}

services/monitoring/alloy.nix

@@ -10,7 +10,7 @@
     text = ''
       pyroscope.write "local_pyroscope" {
         endpoint {
-          url = "http://localhost:4040";
+          url = "http://localhost:4040"
         }
       }
 
@@ -20,19 +20,19 @@
 
         profiling_config {
           profile.process_cpu {
-            enabled = true;
+            enabled = true
           }
           profile.memory {
-            enabled = true;
+            enabled = true
           }
           profile.mutex {
-            enabled = true;
+            enabled = true
           }
           profile.block {
-            enabled = true;
+            enabled = true
           }
           profile.goroutine {
-            enabled = true;
+            enabled = true
           }
         }
       }

services/monitoring/default.nix

@@ -8,5 +8,6 @@
     ./alerttonotify.nix
     ./pyroscope.nix
     ./alloy.nix
+    ./tempo.nix
   ];
 }

services/monitoring/prometheus.nix

@@ -199,6 +199,30 @@
           }
         ];
       }
+      {
+        job_name = "pushgateway";
+        honor_labels = true;
+        static_configs = [
+          {
+            targets = [ "localhost:9091" ];
+          }
+        ];
+      }
+      {
+        job_name = "nix-cache_caddy";
+        scheme = "https";
+        static_configs = [
+          {
+            targets = [ "nix-cache.home.2rjus.net" ];
+          }
+        ];
+      }
     ];
+    pushgateway = {
+      enable = true;
+      web = {
+        external-url = "https://pushgw.home.2rjus.net";
+      };
+    };
   };
 }

services/monitoring/rules.yml

@@ -97,15 +97,21 @@ groups:
           description: "NATS has been down on {{ $labels.instance }} more than 5 minutes."
   - name: nix_cache_rules
     rules:
-      - alert: build-flakes_service_failed
+      - alert: build-flakes_service_not_active_recently
-        expr: node_systemd_unit_state{instance="nix-cache01.home.2rjus.net:9100", name="build-flakes.service", state="failed"} == 1
+        expr: count_over_time(node_systemd_unit_state{instance="nix-cache01.home.2rjus.net:9100", name="build-flakes.service", state="active"}[1h]) < 1
         for: 0m
-        keep_firing_for: 10m
         labels:
           severity: critical
         annotations:
-          summary: "The build-flakes service on {{ $labels.instance }} has failed"
+          summary: "The build-flakes service on {{ $labels.instance }} has not run recently"
-          description: "The build-flakes service on {{ $labels.instance }} has failed"
+          description: "The build-flakes service on {{ $labels.instance }} has not run recently"
+      - alert: build_flakes_error
+        expr: build_flakes_error == 1
+        labels:
+          severity: warning
+        annotations:
+          summary: "The build-flakes job has failed for host {{ $labels.host }}."
+          description: "The build-flakes job has failed for host {{ $labels.host }}."
       - alert: harmonia_down
         expr: node_systemd_unit_state {instance="nix-cache01.home.2rjus.net:9100", name = "harmonia.service", state = "active"} == 0
         for: 5m
@@ -184,3 +190,61 @@ groups:
         annotations:
           summary: "Wireguard handshake timeout on {{ $labels.instance }}"
           description: "Wireguard handshake timeout on {{ $labels.instance }} for more than 1 minutes."
+  - name: monitoring_rules
+    rules:
+      - alert: prometheus_not_running
+        expr: node_systemd_unit_state{instance="monitoring01.home.2rjus.net:9100", name="prometheus.service", state="active"} == 0
+        labels:
+          severity: critical
+        annotations:
+          summary: "Prometheus service not running on {{ $labels.instance }}"
+          description: "Prometheus service not running on {{ $labels.instance }}"
+      - alert: alertmanager_not_running
+        expr: node_systemd_unit_state{instance="monitoring01.home.2rjus.net:9100", name="alertmanager.service", state="active"} == 0
+        labels:
+          severity: critical
+        annotations:
+          summary: "Alertmanager service not running on {{ $labels.instance }}"
+          description: "Alertmanager service not running on {{ $labels.instance }}"
+      - alert: pushgateway_not_running
+        expr: node_systemd_unit_state{instance="monitoring01.home.2rjus.net:9100", name="pushgateway.service", state="active"} == 0
+        labels:
+          severity: critical
+        annotations:
+          summary: "Pushgateway service not running on {{ $labels.instance }}"
+          description: "Pushgateway service not running on {{ $labels.instance }}"
+      - alert: pushgateway_not_running
+        expr: node_systemd_unit_state{instance="monitoring01.home.2rjus.net:9100", name="pushgateway.service", state="active"} == 0
+        labels:
+          severity: critical
+        annotations:
+          summary: "Pushgateway service not running on {{ $labels.instance }}"
+          description: "Pushgateway service not running on {{ $labels.instance }}"
+      - alert: loki_not_running
+        expr: node_systemd_unit_state{instance="monitoring01.home.2rjus.net:9100", name="loki.service", state="active"} == 0
+        labels:
+          severity: critical
+        annotations:
+          summary: "Loki service not running on {{ $labels.instance }}"
+          description: "Loki service not running on {{ $labels.instance }}"
+      - alert: grafana_not_running
+        expr: node_systemd_unit_state{instance="monitoring01.home.2rjus.net:9100", name="grafana.service", state="active"} == 0
+        labels:
+          severity: warning
+        annotations:
+          summary: "Grafana service not running on {{ $labels.instance }}"
+          description: "Grafana service not running on {{ $labels.instance }}"
+      - alert: tempo_not_running
+        expr: node_systemd_unit_state{instance="monitoring01.home.2rjus.net:9100", name="tempo.service", state="active"} == 0
+        labels:
+          severity: warning
+        annotations:
+          summary: "Tempo service not running on {{ $labels.instance }}"
+          description: "Tempo service not running on {{ $labels.instance }}"
+      - alert: pyroscope_not_running
+        expr: node_systemd_unit_state{instance="monitoring01.home.2rjus.net:9100", name="podman-pyroscope.service", state="active"} == 0
+        labels:
+          severity: warning
+        annotations:
+          summary: "Pyroscope service not running on {{ $labels.instance }}"
+          description: "Pyroscope service not running on {{ $labels.instance }}"

services/monitoring/tempo.nix

@@ -0,0 +1,37 @@
+{ ... }:
+{
+  services.tempo = {
+    enable = true;
+    settings = {
+      server = {
+        http_listen_port = 3200;
+        grpc_listen_port = 3201;
+      };
+      distributor = {
+        receivers = {
+          otlp = {
+            protocols = {
+              http = {
+                endpoint = ":4318";
+                cors = {
+                  allowed_origins = [ "*.home.2rjus.net" ];
+                };
+              };
+            };
+          };
+        };
+      };
+      storage = {
+        trace = {
+          backend = "local";
+          local = {
+            path = "/var/lib/tempo";
+          };
+          wal = {
+            path = "/var/lib/tempo/wal";
+          };
+        };
+      };
+    };
+  };
+}

services/nix-cache/build-flakes.nix

@@ -7,22 +7,9 @@ let
       nix
       nixos-rebuild
       jq
+      curl
     ];
-    text = ''
+    text = builtins.readFile ./build-flakes.sh;
-      cd /root/nixos-servers
-      git pull
-      echo "Starting nixos-servers builds"
-      for host in $(nix flake show --json| jq -r '.nixosConfigurations | keys[]'); do
-        echo "Building $host"
-        nixos-rebuild --verbose -L --flake ".#$host" build
-      done
-      echo "All nixos-servers builds complete"
-
-      echo "Building gunter"
-      cd /root/nixos
-      git pull
-      nixos-rebuild --verbose -L --flake ".#gunter" build
-    '';
   };
 in
 {

services/nix-cache/build-flakes.sh

@@ -0,0 +1,44 @@
+JOB_NAME="build_flakes"
+
+cd /root/nixos-servers
+git pull
+echo "Starting nixos-servers builds"
+for host in $(nix flake show --json| jq -r '.nixosConfigurations | keys[]'); do
+  echo "Building $host"
+  if ! nixos-rebuild --verbose -L --flake ".#$host" build; then
+    echo "Build failed for $host"
+    cat <<EOF | curl -sS -X PUT --data-binary @- "https://pushgw.home.2rjus.net/metrics/job/$JOB_NAME/host/$host"
+# TYPE build_flakes_error gauge
+# HELP build_flakes_error 0 if the build was successful, 1 if it failed
+build_flakes_error{instance="$HOSTNAME"} 1
+EOF
+  else 
+    echo "Build successful for $host"
+    cat <<EOF | curl -sS -X PUT --data-binary @- "https://pushgw.home.2rjus.net/metrics/job/$JOB_NAME/host/$host"
+# TYPE build_flakes_error gauge
+# HELP build_flakes_error 0 if the build was successful, 1 if it failed
+build_flakes_error{instance="$HOSTNAME"} 0
+EOF
+  fi
+done
+echo "All nixos-servers builds complete"
+
+echo "Building gunter"
+cd /root/nixos
+git pull
+host="gunter"
+if ! nixos-rebuild --verbose -L --flake ".#gunter" build; then
+    echo "Build failed for $host"
+    cat <<EOF | curl -sS -X PUT --data-binary @- "https://pushgw.home.2rjus.net/metrics/job/$JOB_NAME/host/$host"
+# TYPE build_flakes_error gauge
+# HELP build_flakes_error 0 if the build was successful, 1 if it failed
+build_flakes_error{instance="$HOSTNAME"} 1
+EOF
+else 
+    echo "Build successful for $host"
+    cat <<EOF | curl -sS -X PUT --data-binary @- "https://pushgw.home.2rjus.net/metrics/job/$JOB_NAME/host/$host"
+# TYPE build_flakes_error gauge
+# HELP build_flakes_error 0 if the build was successful, 1 if it failed
+build_flakes_error{instance="$HOSTNAME"} 0
+EOF
+fi

services/nix-cache/proxy.nix

@@ -6,14 +6,18 @@
     configFile = pkgs.writeText "Caddyfile" ''
       {
         acme_ca https://ca.home.2rjus.net/acme/acme/directory
+        metrics
       }
 
+
       nix-cache.home.2rjus.net {
         log {
           output file /var/log/caddy/nix-cache.log {
             mode 644
           }
         }
+        metrics /metrics
+
         reverse_proxy http://localhost:5000
       }
     '';

services/ns/zones-home-2rjus-net.conf

@@ -1,7 +1,7 @@
 $ORIGIN home.2rjus.net.
 $TTL 1800
 @       IN      SOA     ns1.home.2rjus.net.      admin.test.2rjus.net. (
-                        2061                    ; serial number
+                        2063                    ; serial number
                         3600                    ; refresh
                         900                     ; retry
                         1209600                 ; expire
@@ -59,6 +59,7 @@ monitoring01        IN      A       10.69.13.13
 jelly01             IN      A       10.69.13.14
 nix-cache01         IN      A       10.69.13.15
 nix-cache           IN      CNAME   nix-cache01
+actions1            IN      CNAME   nix-cache01
 pgdb1               IN      A       10.69.13.16
 nats1               IN      A       10.69.13.17
 auth01              IN      A       10.69.13.18
@@ -76,6 +77,7 @@ jelly               IN      CNAME   http-proxy
 auth                IN      CNAME   http-proxy
 lldap               IN      CNAME   http-proxy
 pyroscope           IN      CNAME   http-proxy
+pushgw              IN      CNAME   http-proxy
 
 ldap                IN      CNAME   auth01