From abb4cf58ea8dc8cc4b65a76e3a40d6c6f7196525 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torjus=20H=C3=A5kestad?= Date: Tue, 11 Feb 2025 22:25:54 +0100 Subject: [PATCH] Add alerttonotify to monitoring host --- .gitignore | 1 + flake.lock | 8 ++--- flake.nix | 43 ++++++++++++++++----------- result | 1 + secrets/secrets.yaml | 7 +++-- services/monitoring/alerttonotify.nix | 21 +++++++++++++ services/monitoring/default.nix | 1 + services/monitoring/prometheus.nix | 6 ++-- 8 files changed, 61 insertions(+), 27 deletions(-) create mode 120000 result create mode 100644 services/monitoring/alerttonotify.nix diff --git a/.gitignore b/.gitignore index 9b42106..d53e06f 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .direnv/ +result diff --git a/flake.lock b/flake.lock index ebd7376..e4288e4 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1739308211, - "narHash": "sha256-XWHHbnyeF00cNCSO+dH4QxQ6Vm2Pye6jc1CFLuA03+M=", + "lastModified": 1739309053, + "narHash": "sha256-sJErNNnDRxfZXn3Xoyyg3GIxiGTghhP8wjwaN25BJGo=", "ref": "master", - "rev": "360109d68446ac1047f25d6480972eb3f25368fb", - "revCount": 5, + "rev": "d9e2147ecac4d8558b6c0fea65f1308775af7db9", + "revCount": 6, "type": "git", "url": "https://git.t-juice.club/torjus/alerttonotify" }, diff --git a/flake.nix b/flake.nix index 049285b..2cb1daa 100644 --- a/flake.nix +++ b/flake.nix @@ -13,6 +13,10 @@ url = "git+https://git.t-juice.club/torjus/backup-helper?ref=master"; inputs.nixpkgs.follows = "nixpkgs-unstable"; }; + alerttonotify = { + url = "git+https://git.t-juice.club/torjus/alerttonotify?ref=master"; + inputs.nixpkgs.follows = "nixpkgs-unstable"; + }; }; outputs = @@ -22,6 +26,7 @@ nixpkgs-unstable, sops-nix, backup-helper, + alerttonotify, ... }@inputs: let @@ -32,6 +37,10 @@ config.allowUnfree = true; }; }; + commonOverlays = [ + overlay-unstable + alerttonotify.overlays.default + ]; allSystems = [ "x86_64-linux" "aarch64-linux" @@ -52,7 +61,7 @@ ( { config, pkgs, ... }: { - nixpkgs.overlays = [ overlay-unstable ]; + nixpkgs.overlays = commonOverlays; } ) ./hosts/ns1 @@ -68,7 +77,7 @@ ( { config, pkgs, ... }: { - nixpkgs.overlays = [ overlay-unstable ]; + nixpkgs.overlays = commonOverlays; } ) ./hosts/ns2 @@ -84,7 +93,7 @@ ( { config, pkgs, ... }: { - nixpkgs.overlays = [ overlay-unstable ]; + nixpkgs.overlays = commonOverlays; } ) ./hosts/ns3 @@ -100,7 +109,7 @@ ( { config, pkgs, ... }: { - nixpkgs.overlays = [ overlay-unstable ]; + nixpkgs.overlays = commonOverlays; } ) ./hosts/ns4 @@ -116,7 +125,7 @@ ( { config, pkgs, ... }: { - nixpkgs.overlays = [ overlay-unstable ]; + nixpkgs.overlays = commonOverlays; } ) ./hosts/nixos-test1 @@ -133,7 +142,7 @@ ( { config, pkgs, ... }: { - nixpkgs.overlays = [ overlay-unstable ]; + nixpkgs.overlays = commonOverlays; } ) ./hosts/ha1 @@ -150,7 +159,7 @@ ( { config, pkgs, ... }: { - nixpkgs.overlays = [ overlay-unstable ]; + nixpkgs.overlays = commonOverlays; } ) ./hosts/inc1 @@ -167,7 +176,7 @@ ( { config, pkgs, ... }: { - nixpkgs.overlays = [ overlay-unstable ]; + nixpkgs.overlays = commonOverlays; } ) ./hosts/inc2 @@ -184,7 +193,7 @@ ( { config, pkgs, ... }: { - nixpkgs.overlays = [ overlay-unstable ]; + nixpkgs.overlays = commonOverlays; } ) ./hosts/template @@ -200,7 +209,7 @@ ( { config, pkgs, ... }: { - nixpkgs.overlays = [ overlay-unstable ]; + nixpkgs.overlays = commonOverlays; } ) ./hosts/http-proxy @@ -216,7 +225,7 @@ ( { config, pkgs, ... }: { - nixpkgs.overlays = [ overlay-unstable ]; + nixpkgs.overlays = commonOverlays; } ) ./hosts/ca @@ -232,7 +241,7 @@ ( { config, pkgs, ... }: { - nixpkgs.overlays = [ overlay-unstable ]; + nixpkgs.overlays = commonOverlays; } ) ./hosts/monitoring01 @@ -249,7 +258,7 @@ ( { config, pkgs, ... }: { - nixpkgs.overlays = [ overlay-unstable ]; + nixpkgs.overlays = commonOverlays; } ) ./hosts/jelly01 @@ -265,7 +274,7 @@ ( { config, pkgs, ... }: { - nixpkgs.overlays = [ overlay-unstable ]; + nixpkgs.overlays = commonOverlays; } ) ./hosts/nix-cache01 @@ -281,7 +290,7 @@ ( { config, pkgs, ... }: { - nixpkgs.overlays = [ overlay-unstable ]; + nixpkgs.overlays = commonOverlays; } ) ./hosts/media1 @@ -297,7 +306,7 @@ ( { config, pkgs, ... }: { - nixpkgs.overlays = [ overlay-unstable ]; + nixpkgs.overlays = commonOverlays; } ) ./hosts/pgdb1 @@ -313,7 +322,7 @@ ( { config, pkgs, ... }: { - nixpkgs.overlays = [ overlay-unstable ]; + nixpkgs.overlays = commonOverlays; } ) ./hosts/nats1 diff --git a/result b/result new file mode 120000 index 0000000..d248575 --- /dev/null +++ b/result @@ -0,0 +1 @@ +/nix/store/5acmmy4l87pd6c6q9mq6j73m6hh9lc5z-nixos-system-monitoring01-24.11.20250210.44534bc \ No newline at end of file diff --git a/secrets/secrets.yaml b/secrets/secrets.yaml index 836d476..f223d8a 100644 --- a/secrets/secrets.yaml +++ b/secrets/secrets.yaml @@ -1,6 +1,7 @@ root_password_hash: ENC[AES256_GCM,data:wk/xEuf+qU3ezmondq9y3OIotXPI/L+TOErTjgJz58wEvQkApYkjc3bHaUTzOrmWjQBgDUENObzPmvQ8WKawUSJRVlpfOEr5TQ==,iv:I8Z3xJz3qoXBD7igx087A1fMwf8d29hQ4JEI3imRXdY=,tag:M80osQeWGG9AAA8BrMfhHA==,type:str] ns_xfer_key: ENC[AES256_GCM,data:VFpK7GChgFeUgQm31tTvVC888bN0yt6BAnHQa6KUTg4iZGP1WL5Bx6Zp8dY=,iv:9RF1eEc7JBxBebDOKfcDjGS2U7XsHkOW/l52yIP+1LA=,tag:L6DR2QlHOfo02kzfWWCrvg==,type:str] backup_helper_secret: ENC[AES256_GCM,data:EvXEJnDilbfALQ==,iv:Q3dkZ8Ee3qbcjcoi5GxfbaVB4uRIvkIB6ioKVV/dL2Y=,tag:T/UgZvQgYGa740Wh7D0b7Q==,type:str] +nats_nkey: ENC[AES256_GCM,data:N2CVXjdwiE7eSPUtXe+NeKSTzA9eFwK2igxaCdYsXd4Ps0/DjYb/ggnQziQzSy8viESZYjXhJ2VtNw==,iv:Xhcf5wPB01Wu0A+oMw0wzTEHATp+uN+wsaYshxIzy1w=,tag:IauTIOHqfiM75Ufml/JXbg==,type:str] sops: kms: [] gcp_kms: [] @@ -142,8 +143,8 @@ sops: UWg1ZmFIMlk4STlMdzBOd1dLOW9ZY2sK8BYqBM/0YZ6fjgQAqSCYM9Cnh2IqP4QD NQDBErJf0AQ8qU+CXjBSxTLBBJPnibdBJPCcOfnym16gFgMuHsqMdg== -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-06-02T22:09:06Z" - mac: ENC[AES256_GCM,data:cxJq4EMEMVEw0IUXNwtyQj4MaYIJ/Xo4OaY+3VLgIhYw6oBO9CmJxgLuXcSnGnr23oNE5OQF6ALv+vxF46D1pI0V1zhqKL6zMIs0DzPBwo7Arg166w5kGAT274jK7YWymeJ7fafWXYubLlGUthyVJS1BkvlqIhoe2BlTZ3bPyBs=,iv:Z2Uh9Oo4q/ce6DDLShs7JAX3XFNAVOGBmBPvRbGxaaU=,tag:6qZhZ4+tgtXl60b0Lx7Taw==,type:str] + lastmodified: "2025-02-11T21:18:22Z" + mac: ENC[AES256_GCM,data:5//boMp1awc/2XAkSASSCuobpkxa0E6IKf3GR8xHpMoCD30FJsCwV7PgX3fR8OuLEhOJ7UguqMNQdNqG37RMacreuDmI1J8oCFKp+3M2j4kCbXaEo8bw7WAtyjUez+SAXKzZWYmBibH0KOy6jdt+v0fdgy5hMBT4IFDofYRsyD0=,iv:6pD+SLwncpmal/FR4U8It2njvaQfUzzpALBCxa0NyME=,tag:4QN8ZFjdqck5ZgulF+FtbA==,type:str] pgp: [] unencrypted_suffix: _unencrypted - version: 3.8.1 + version: 3.9.4 diff --git a/services/monitoring/alerttonotify.nix b/services/monitoring/alerttonotify.nix new file mode 100644 index 0000000..b808ac6 --- /dev/null +++ b/services/monitoring/alerttonotify.nix @@ -0,0 +1,21 @@ +{ pkgs, config, ... }: +{ + sops.secrets."nats_nkey" = { }; + systemd.services."alerttonotify" = { + wants = [ "network-online.target" ]; + after = [ + "network-online.target" + "sops-nix.service" + ]; + + environment = { + NATS_URL = "nats://nats1.home.2rjus.net:4222"; + NATS_NKEY_FILE = "${config.sops.secrets.nats_nkey.path}"; + }; + + serviceConfig = { + Type = "exec"; + ExecStart = "${pkgs.alerttonotify}/bin/alerttonotify"; + }; + }; +} diff --git a/services/monitoring/default.nix b/services/monitoring/default.nix index 139b1d6..1336256 100644 --- a/services/monitoring/default.nix +++ b/services/monitoring/default.nix @@ -5,5 +5,6 @@ ./grafana.nix ./prometheus.nix ./pve.nix + ./alerttonotify.nix ]; } diff --git a/services/monitoring/prometheus.nix b/services/monitoring/prometheus.nix index 27f3b53..cf35eba 100644 --- a/services/monitoring/prometheus.nix +++ b/services/monitoring/prometheus.nix @@ -8,7 +8,7 @@ global = { }; route = { - receiver = "webhook_gunter"; + receiver = "webhook_natstonotify"; group_wait = "30s"; group_interval = "5m"; repeat_interval = "12h"; @@ -16,10 +16,10 @@ }; receivers = [ { - name = "webhook_gunter"; + name = "webhook_natstonotify"; webhook_configs = [ { - url = "http://gunter.home.2rjus.net:5001/alert"; + url = "http://localhost:5001/alert"; } ]; }