From 2ca2509083be8b0e226eec77090692517fca8244 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torjus=20H=C3=A5kestad?= Date: Fri, 20 Feb 2026 09:36:27 +0100 Subject: [PATCH] monitoring: increase filesystem_filling_up prediction window to 24h Reduces false positives from transient Nix store growth by basing the linear prediction on a 24h trend instead of 6h. Co-Authored-By: Claude Opus 4.6 --- services/monitoring/rules.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/monitoring/rules.yml b/services/monitoring/rules.yml index 14b9baf..6379458 100644 --- a/services/monitoring/rules.yml +++ b/services/monitoring/rules.yml @@ -67,13 +67,13 @@ groups: summary: "Promtail service not running on {{ $labels.instance }}" description: "The promtail service has not been active on {{ $labels.instance }} for 5 minutes." - alert: filesystem_filling_up - expr: predict_linear(node_filesystem_free_bytes{mountpoint="/"}[6h], 24*3600) < 0 + expr: predict_linear(node_filesystem_free_bytes{mountpoint="/"}[24h], 24*3600) < 0 for: 1h labels: severity: warning annotations: summary: "Filesystem predicted to fill within 24h on {{ $labels.instance }}" - description: "Based on the last 6h trend, the root filesystem on {{ $labels.instance }} is predicted to run out of space within 24 hours." + description: "Based on the last 24h trend, the root filesystem on {{ $labels.instance }} is predicted to run out of space within 24 hours." - alert: systemd_not_running expr: node_systemd_system_running == 0 for: 10m