mcp: move config to .mcp.json.example, gitignore real config

The real .mcp.json now contains Loki credentials for basic auth,
so it should not be committed. The example file has placeholders.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

.claude/agents/auditor.md

@@ -19,7 +19,7 @@ You may receive:
 ## Audit Log Structure
 
 Logs are shipped to Loki via promtail. Audit events use these labels:
-- `host` - hostname
+- `hostname` - hostname
 - `systemd_unit` - typically `auditd.service` for audit logs
 - `job` - typically `systemd-journal`
 
@@ -36,7 +36,7 @@ Audit log entries contain structured data:
 
 Find SSH logins and session activity:
 ```logql
-{host="<hostname>", systemd_unit="sshd.service"}
+{hostname="<hostname>", systemd_unit="sshd.service"}
 ```
 
 Look for:
@@ -48,7 +48,7 @@ Look for:
 
 Query executed commands (filter out noise):
 ```logql
-{host="<hostname>"} |= "EXECVE" != "PATH item" != "PROCTITLE" != "SYSCALL" != "BPF"
+{hostname="<hostname>"} |= "EXECVE" != "PATH item" != "PROCTITLE" != "SYSCALL" != "BPF"
 ```
 
 Further filtering:
@@ -60,28 +60,28 @@ Further filtering:
 
 Check for privilege escalation:
 ```logql
-{host="<hostname>"} |= "sudo" |= "COMMAND"
+{hostname="<hostname>"} |= "sudo" |= "COMMAND"
 ```
 
 Or via audit:
 ```logql
-{host="<hostname>"} |= "USER_CMD"
+{hostname="<hostname>"} |= "USER_CMD"
 ```
 
 ### 4. Service Manipulation
 
 Check if services were manually stopped/started:
 ```logql
-{host="<hostname>"} |= "EXECVE" |= "systemctl"
+{hostname="<hostname>"} |= "EXECVE" |= "systemctl"
 ```
 
 ### 5. File Operations
 
 Look for file modifications (if auditd rules are configured):
 ```logql
-{host="<hostname>"} |= "EXECVE" |= "vim"
+{hostname="<hostname>"} |= "EXECVE" |= "vim"
-{host="<hostname>"} |= "EXECVE" |= "nano"
+{hostname="<hostname>"} |= "EXECVE" |= "nano"
-{host="<hostname>"} |= "EXECVE" |= "rm"
+{hostname="<hostname>"} |= "EXECVE" |= "rm"
 ```
 
 ## Query Guidelines
@@ -99,7 +99,7 @@ Look for file modifications (if auditd rules are configured):
 **Time-bounded queries:**
 When investigating around a specific event:
 ```logql
-{host="<hostname>"} |= "EXECVE" != "systemd"
+{hostname="<hostname>"} |= "EXECVE" != "systemd"
 ```
 With `start: "2026-02-08T14:30:00Z"` and `end: "2026-02-08T14:35:00Z"`
 

.claude/agents/investigate-alarm.md

@@ -41,13 +41,13 @@ Search for relevant log entries using `query_logs`. Focus on service-specific lo
 **Query strategies (start narrow, expand if needed):**
 - Start with `limit: 20-30`, increase only if needed
 - Use tight time windows: `start: "15m"` or `start: "30m"` initially
-- Filter to specific services: `{host="<hostname>", systemd_unit="<service>.service"}`
+- Filter to specific services: `{hostname="<hostname>", systemd_unit="<service>.service"}`
-- Search for errors: `{host="<hostname>"} |= "error"` or `|= "failed"`
+- Search for errors: `{hostname="<hostname>"} |= "error"` or `|= "failed"`
 
 **Common patterns:**
-- Service logs: `{host="<hostname>", systemd_unit="<service>.service"}`
+- Service logs: `{hostname="<hostname>", systemd_unit="<service>.service"}`
-- All errors on host: `{host="<hostname>"} |= "error"`
+- All errors on host: `{hostname="<hostname>"} |= "error"`
-- Journal for a unit: `{host="<hostname>", systemd_unit="nginx.service"} |= "failed"`
+- Journal for a unit: `{hostname="<hostname>", systemd_unit="nginx.service"} |= "failed"`
 
 **Avoid:**
 - Using `start: "1h"` with no filters on busy hosts

.claude/skills/observability/SKILL.md

@@ -30,11 +30,13 @@ Use the `lab-monitoring` MCP server tools:
 ### Label Reference
 
 Available labels for log queries:
-- `host` - Hostname (e.g., `ns1`, `monitoring01`, `ha1`)
+- `hostname` - Hostname (e.g., `ns1`, `monitoring01`, `ha1`) - matches the Prometheus `hostname` label
 - `systemd_unit` - Systemd unit name (e.g., `nsd.service`, `nixos-upgrade.service`)
 - `job` - Either `systemd-journal` (most logs), `varlog` (file-based logs), or `bootstrap` (VM bootstrap logs)
 - `filename` - For `varlog` job, the log file path
-- `hostname` - Alternative to `host` for some streams
+- `tier` - Deployment tier (`test` or `prod`)
+- `role` - Host role (e.g., `dns`, `vault`, `monitoring`) - matches the Prometheus `role` label
+- `level` - Log level mapped from journal PRIORITY (`critical`, `error`, `warning`, `notice`, `info`, `debug`) - journal scrape only
 
 ### Log Format
 
@@ -47,12 +49,12 @@ Journal logs are JSON-formatted. Key fields:
 
 **Logs from a specific service on a host:**
 ```logql
-{host="ns1", systemd_unit="nsd.service"}
+{hostname="ns1", systemd_unit="nsd.service"}
 ```
 
 **All logs from a host:**
 ```logql
-{host="monitoring01"}
+{hostname="monitoring01"}
 ```
 
 **Logs from a service across all hosts:**
@@ -62,12 +64,12 @@ Journal logs are JSON-formatted. Key fields:
 
 **Substring matching (case-sensitive):**
 ```logql
-{host="ha1"} |= "error"
+{hostname="ha1"} |= "error"
 ```
 
 **Exclude pattern:**
 ```logql
-{host="ns1"} != "routine"
+{hostname="ns1"} != "routine"
 ```
 
 **Regex matching:**
@@ -75,6 +77,20 @@ Journal logs are JSON-formatted. Key fields:
 {systemd_unit="prometheus.service"} |~ "scrape.*failed"
 ```
 
+**Filter by level (journal scrape only):**
+```logql
+{level="error"}                                  # All errors across the fleet
+{level=~"critical|error", tier="prod"}           # Prod errors and criticals
+{hostname="ns1", level="warning"}                # Warnings from a specific host
+```
+
+**Filter by tier/role:**
+```logql
+{tier="prod"} |= "error"                        # All errors on prod hosts
+{role="dns"}                                     # All DNS server logs
+{tier="test", job="systemd-journal"}             # Journal logs from test hosts
+```
+
 **File-based logs (caddy access logs, etc):**
 ```logql
 {job="varlog", hostname="nix-cache01"}
@@ -106,7 +122,7 @@ Useful systemd units for troubleshooting:
 
 VMs provisioned from template2 send bootstrap progress directly to Loki via curl (before promtail is available). These logs use `job="bootstrap"` with additional labels:
 
-- `host` - Target hostname
+- `hostname` - Target hostname
 - `branch` - Git branch being deployed
 - `stage` - Bootstrap stage (see table below)
 
@@ -127,7 +143,7 @@ VMs provisioned from template2 send bootstrap progress directly to Loki via curl
 
 ```logql
 {job="bootstrap"}                              # All bootstrap logs
-{job="bootstrap", host="myhost"}               # Specific host
+{job="bootstrap", hostname="myhost"}            # Specific host
 {job="bootstrap", stage="failed"}              # All failures
 {job="bootstrap", stage=~"building|success"}   # Track build progress
 ```
@@ -308,8 +324,8 @@ Current host labels:
 
 1. Check `up{job="<service>"}` or `up{hostname="<host>"}` for scrape failures
 2. Use `list_targets` to see target health details
-3. Query service logs: `{host="<host>", systemd_unit="<service>.service"}`
+3. Query service logs: `{hostname="<host>", systemd_unit="<service>.service"}`
-4. Search for errors: `{host="<host>"} |= "error"`
+4. Search for errors: `{hostname="<host>"} |= "error"`
 5. Check `list_alerts` for related alerts
 6. Use role filters for group issues: `up{role="dns"}` to check all DNS servers
 
@@ -324,17 +340,17 @@ Current host labels:
 
 When provisioning new VMs, track bootstrap progress:
 
-1. Watch bootstrap logs: `{job="bootstrap", host="<hostname>"}`
+1. Watch bootstrap logs: `{job="bootstrap", hostname="<hostname>"}`
-2. Check for failures: `{job="bootstrap", host="<hostname>", stage="failed"}`
+2. Check for failures: `{job="bootstrap", hostname="<hostname>", stage="failed"}`
 3. After success, verify host appears in metrics: `up{hostname="<hostname>"}`
-4. Check logs are flowing: `{host="<hostname>"}`
+4. Check logs are flowing: `{hostname="<hostname>"}`
 
 See [docs/host-creation.md](../../../docs/host-creation.md) for the full host creation pipeline.
 
 ### Debug SSH/Access Issues
 
 ```logql
-{host="<host>", systemd_unit="sshd.service"}
+{hostname="<host>", systemd_unit="sshd.service"}
 ```
 
 ### Check Recent Upgrades

.gitignore

@@ -2,6 +2,9 @@
 result
 result-*
 
+# MCP config (contains secrets)
+.mcp.json
+
 # Terraform/OpenTofu
 terraform/.terraform/
 terraform/.terraform.lock.hcl

.mcp.json.example

@@ -20,7 +20,9 @@
       "env": {
         "PROMETHEUS_URL": "https://prometheus.home.2rjus.net",
         "ALERTMANAGER_URL": "https://alertmanager.home.2rjus.net",
-        "LOKI_URL": "http://monitoring01.home.2rjus.net:3100"
+        "LOKI_URL": "https://loki.home.2rjus.net",
+        "LOKI_USERNAME": "promtail",
+        "LOKI_PASSWORD": "<password from: bao kv get -field=password secret/shared/loki/push-auth>"
       }
     },
     "homelab-deploy": {
@@ -44,4 +46,3 @@
     }
   }
 }
-

CLAUDE.md

@@ -59,7 +59,7 @@ The script prints the session ID which the user can share. Query results with:
 ```logql
 {job="pipe-to-loki"}                           # All entries
 {job="pipe-to-loki", id="my-test"}             # Specific ID
-{job="pipe-to-loki", host="testvm01"}          # From specific host
+{job="pipe-to-loki", hostname="testvm01"}       # From specific host
 {job="pipe-to-loki", type="session"}           # Only sessions
 ```
 

docs/host-creation.md

@@ -50,7 +50,7 @@ homelab.host.tier = "test";  # or "prod"
 During the bootstrap process, status updates are sent to Loki. Query bootstrap logs with:
 
 ```
-{job="bootstrap", host="<hostname>"}
+{job="bootstrap", hostname="<hostname>"}
 ```
 
 ### Bootstrap Stages
@@ -72,7 +72,7 @@ The bootstrap process reports these stages via the `stage` label:
 
 ```
 # All bootstrap activity for a host
-{job="bootstrap", host="myhost"}
+{job="bootstrap", hostname="myhost"}
 
 # Track all failures
 {job="bootstrap", stage="failed"}
@@ -87,7 +87,7 @@ Once the VM reboots with its full configuration, it will start publishing metric
 
 1. Check bootstrap completed successfully:
    ```
-   {job="bootstrap", host="<hostname>", stage="success"}
+   {job="bootstrap", hostname="<hostname>", stage="success"}
    ```
 
 2. Verify the host is up and reporting metrics:
@@ -102,7 +102,7 @@ Once the VM reboots with its full configuration, it will start publishing metric
 
 4. Check logs are flowing:
    ```
-   {host="<hostname>"}
+   {hostname="<hostname>"}
    ```
 
 5. Confirm expected services are running and producing logs
@@ -119,7 +119,7 @@ Once the VM reboots with its full configuration, it will start publishing metric
 
 1. Check bootstrap logs in Loki - if they never progress past `building`, the rebuild likely consumed all resources:
    ```
-   {job="bootstrap", host="<hostname>"}
+   {job="bootstrap", hostname="<hostname>"}
    ```
 
 2. **USER**: SSH into the host and check the bootstrap service:
@@ -149,7 +149,7 @@ Usually caused by running the `create-host` script without proper credentials, o
 
 2. Check bootstrap logs for vault-related stages:
    ```
-   {job="bootstrap", host="<hostname>", stage=~"vault.*"}
+   {job="bootstrap", hostname="<hostname>", stage=~"vault.*"}
    ```
 
 3. **USER**: Regenerate and provision credentials manually:

docs/plans/loki-improvements.md

@@ -0,0 +1,196 @@
+# Loki Setup Improvements
+
+## Overview
+
+The current Loki deployment on monitoring01 is functional but minimal. It lacks retention policies, rate limiting, and uses local filesystem storage. This plan evaluates improvement options across several dimensions: retention management, storage backend, resource limits, and operational improvements.
+
+## Current State
+
+**Loki** on monitoring01 (`services/monitoring/loki.nix`):
+- Single-node deployment, no HA
+- Filesystem storage at `/var/lib/loki/chunks` (~6.8 GB as of 2026-02-13)
+- TSDB index (v13 schema, 24h period)
+- 30-day compactor-based retention with basic rate limits
+- No caching layer
+- Auth disabled (trusted network)
+
+**Promtail** on all 16 hosts (`system/monitoring/logs.nix`):
+- Ships systemd journal (JSON) + `/var/log/**/*.log`
+- Labels: `hostname`, `tier`, `role`, `level`, `job` (systemd-journal/varlog), `systemd_unit`
+- `level` label mapped from journal PRIORITY (critical/error/warning/notice/info/debug)
+- Hardcoded to `http://monitoring01.home.2rjus.net:3100`
+
+**Additional log sources:**
+- `pipe-to-loki` script (manual log submission, `job=pipe-to-loki`)
+- Bootstrap logs from template2 (`job=bootstrap`)
+
+**Context:** The VictoriaMetrics migration plan (`docs/plans/monitoring-migration-victoriametrics.md`) includes moving Loki to monitoring02 with "same configuration as current". These improvements could be applied either before or after that migration.
+
+## Improvement Areas
+
+### 1. Retention Policy
+
+**Implemented.** Compactor-based retention with 30-day period. Note: Loki 3.6.3 requires `delete_request_store = "filesystem"` when retention is enabled (not documented in older guides).
+
+```nix
+compactor = {
+  working_directory = "/var/lib/loki/compactor";
+  compaction_interval = "10m";
+  retention_enabled = true;
+  retention_delete_delay = "2h";
+  retention_delete_worker_count = 150;
+  delete_request_store = "filesystem";
+};
+
+limits_config = {
+  retention_period = "30d";
+};
+```
+
+### 2. Storage Backend
+
+**Decision:** Stay with filesystem storage for now. Garage S3 was considered but ruled out - the current single-node Garage (replication_factor=1) offers no real durability benefit over local disk. S3 storage can be revisited after the NAS migration, when a more robust S3-compatible solution will likely be available.
+
+### 3. Limits Configuration
+
+**Implemented.** Basic guardrails added alongside retention in `limits_config`:
+
+```nix
+limits_config = {
+  retention_period = "30d";
+  ingestion_rate_mb = 10;           # MB/s per tenant
+  ingestion_burst_size_mb = 20;     # Burst allowance
+  max_streams_per_user = 10000;     # Prevent label explosion
+  max_query_series = 500;           # Limit query resource usage
+  max_query_parallelism = 8;
+};
+```
+
+### 4. Promtail Label Improvements
+
+**Problem:** Label inconsistencies and missing useful metadata:
+- The `varlog` scrape config uses `hostname` while journal uses `host` (different label name)
+- No `tier` or `role` labels, making it hard to filter logs by deployment tier or host function
+
+**Implemented:** Standardized on `hostname` to match Prometheus labels. The journal scrape previously used a relabel from `__journal__hostname` to `host`; now both scrape configs use a static `hostname` label from `config.networking.hostName`. Also updated `pipe-to-loki` and bootstrap scripts to use `hostname` instead of `host`.
+
+1. **Standardized label:** Both scrape configs use `hostname` (matching Prometheus) via shared `hostLabels`
+2. **Added `tier` label:** Static label from `config.homelab.host.tier` (`test`/`prod`) on both scrape configs
+3. **Added `role` label:** Static label from `config.homelab.host.role` on both scrape configs (conditionally, only when non-null)
+
+No cardinality impact - `tier` and `role` are 1:1 with `hostname`, so they add metadata to existing streams without creating new ones.
+
+This enables queries like:
+- `{tier="prod"} |= "error"` - all errors on prod hosts
+- `{role="dns"}` - all DNS server logs
+- `{tier="test", job="systemd-journal"}` - journal logs from test hosts
+
+### 5. Journal Priority → Level Label
+
+**Implemented.** Promtail pipeline stages map journal `PRIORITY` to a `level` label:
+
+| PRIORITY | level |
+|----------|-------|
+| 0-2 | critical |
+| 3 | error |
+| 4 | warning |
+| 5 | notice |
+| 6 | info |
+| 7 | debug |
+
+Uses a `json` stage to extract PRIORITY, `template` to map to level name, and `labels` to attach it. This gives reliable level filtering for all journal logs, unlike Loki's `detected_level` which only works for apps that embed level keywords in message text.
+
+Example queries:
+- `{level="error"}` - all errors across the fleet
+- `{level=~"critical|error", tier="prod"}` - prod errors and criticals
+- `{level="warning", role="dns"}` - warnings from DNS servers
+
+### 6. Enable JSON Logging on Services
+
+**Problem:** Many services support structured JSON log output but may be using plain text by default. JSON logs are significantly easier to query in Loki - `| json` cleanly extracts all fields, whereas plain text requires fragile regex or pattern matching.
+
+**Audit results (2026-02-13):**
+
+**Already logging JSON:**
+- Caddy (all instances) - JSON by default for access logs
+- homelab-deploy (listener/builder) - Go app, logs structured JSON
+
+**Supports JSON, not configured (high value):**
+
+| Service | How to enable | Config file |
+|---------|--------------|-------------|
+| Prometheus | `--log.format=json` | `services/monitoring/prometheus.nix` |
+| Alertmanager | `--log.format=json` | `services/monitoring/prometheus.nix` |
+| Loki | `--log.format=json` | `services/monitoring/loki.nix` |
+| Grafana | `log.console.format = "json"` | `services/monitoring/grafana.nix` |
+| Tempo | `log_format: json` in config | `services/monitoring/tempo.nix` |
+| OpenBao | `log_format = "json"` | `services/vault/default.nix` |
+
+**Supports JSON, not configured (lower value - minimal log output):**
+
+| Service | How to enable |
+|---------|--------------|
+| Pyroscope | `--log.format=json` (OCI container) |
+| Blackbox Exporter | `--log.format=json` |
+| Node Exporter | `--log.format=json` (all 16 hosts) |
+| Systemd Exporter | `--log.format=json` (all 16 hosts) |
+
+**No JSON support (syslog/text only):**
+- NSD, Unbound, OpenSSH, Mosquitto
+
+**Needs verification:**
+- Kanidm, Jellyfin, Home Assistant, Harmonia, Zigbee2MQTT, NATS
+
+**Recommendation:** Start with the monitoring stack (Prometheus, Alertmanager, Loki, Grafana, Tempo) since they're all Go apps with the same `--log.format=json` flag. Then OpenBao. The exporters are lower priority since they produce minimal log output.
+
+### 7. Monitoring CNAME for Promtail Target
+
+**Problem:** Promtail hardcodes `monitoring01.home.2rjus.net:3100`. The VictoriaMetrics migration plan already addresses this by switching to a `monitoring` CNAME.
+
+**Recommendation:** This should happen as part of the monitoring02 migration, not independently. If we do Loki improvements before that migration, keep pointing to monitoring01.
+
+## Priority Ranking
+
+| # | Improvement | Effort | Impact | Status |
+|---|-------------|--------|--------|--------|
+| 1 | **Retention policy** | Low | High | Done (30d compactor retention) |
+| 2 | **Limits config** | Low | Medium | Done (rate limits + stream guards) |
+| 3 | **Promtail labels** | Trivial | Low | Done (hostname/tier/role/level) |
+| 4 | **Journal priority → level** | Low-medium | Medium | Done (pipeline stages) |
+| 5 | **JSON logging audit** | Low-medium | Medium | Audited, not yet enabled |
+| 6 | **Monitoring CNAME** | Low | Medium | Part of monitoring02 migration |
+
+## Implementation Steps
+
+### Phase 1: Retention + Labels (done 2026-02-13)
+
+1. ~~Add `compactor` section to `services/monitoring/loki.nix`~~ Done
+2. ~~Add `limits_config` with 30-day retention and basic rate limits~~ Done
+3. ~~Update `system/monitoring/logs.nix`~~ Done:
+   - Standardized on `hostname` label (matching Prometheus) for both scrape configs
+   - Added `tier` and `role` static labels from `homelab.host` options
+   - Added pipeline stages for journal PRIORITY → `level` label mapping
+4. ~~Update `pipe-to-loki` and bootstrap scripts to use `hostname`~~ Done
+5. ~~Deploy and verify labels~~ Done - all 15 hosts reporting with correct labels
+
+### Phase 2: JSON Logging (not started)
+
+Enable JSON logging on services that support it, starting with the monitoring stack:
+1. Prometheus, Alertmanager, Loki, Grafana, Tempo (`--log.format=json`)
+2. OpenBao (`log_format = "json"`)
+3. Lower priority: exporters (node-exporter, systemd-exporter, blackbox)
+
+### Phase 3 (future): S3 Storage Migration
+
+Revisit after NAS migration when a proper S3-compatible storage solution is available. At that point, add a new schema period with `object_store = "s3"` - the old filesystem period will continue serving historical data until it ages out past retention.
+
+## Open Questions
+
+- [ ] Do we want per-stream retention (e.g., keep bootstrap/pipe-to-loki longer)?
+
+## Notes
+
+- Loki schema changes require adding a new period entry (not modifying existing ones). The old period continues serving historical data.
+- Loki 3.6.3 requires `delete_request_store = "filesystem"` in the compactor config when retention is enabled.
+- S3 storage deferred until post-NAS migration when a proper solution is available.
+- As of 2026-02-13, Loki uses ~6.8 GB for ~30 days of logs from 16 hosts. Prometheus uses ~7.6 GB on the same disk (33 GB total, ~8 GB free).

docs/plans/monitoring-migration-victoriametrics.md

@@ -14,8 +14,8 @@ a `monitoring` CNAME for seamless transition.
 - Alertmanager (routes to alerttonotify webhook)
 - Grafana (dashboards, datasources)
 - Loki (log aggregation from all hosts via Promtail)
-- Tempo (distributed tracing)
+- Tempo (distributed tracing) - not actively used
-- Pyroscope (continuous profiling)
+- Pyroscope (continuous profiling) - not actively used
 
 **Hardcoded References to monitoring01:**
 - `system/monitoring/logs.nix` - Promtail sends logs to `http://monitoring01.home.2rjus.net:3100`
@@ -44,9 +44,7 @@ If multi-year retention with downsampling becomes necessary later, Thanos can be
                      │  VictoriaMetrics│
                      │  + Grafana      │
      monitoring      │  + Loki         │
-     CNAME ──────────│  + Tempo        │
+     CNAME ──────────│  + Alertmanager │
-                     │  + Pyroscope    │
-                     │  + Alertmanager │
                      │  (vmalert)      │
                      └─────────────────┘
                             ▲
@@ -61,53 +59,48 @@ If multi-year retention with downsampling becomes necessary later, Thanos can be
 
 ## Implementation Plan
 
-### Phase 1: Create monitoring02 Host
+### Phase 1: Create monitoring02 Host [COMPLETE]
 
-Use `create-host` script which handles flake.nix and terraform/vms.tf automatically.
+Host created and deployed at 10.69.13.24 (prod tier) with:
-
+- 4 CPU cores, 8GB RAM, 60GB disk
-1. **Run create-host**: `nix develop -c create-host monitoring02 10.69.13.24`
+- Vault integration enabled
-2. **Update VM resources** in `terraform/vms.tf`:
+- NATS-based remote deployment enabled
-   - 4 cores (same as monitoring01)
+- Grafana with Kanidm OIDC deployed as test instance (`grafana-test.home.2rjus.net`)
-   - 8GB RAM (double, for VictoriaMetrics headroom)
-   - 100GB disk (for 3+ months retention with compression)
-3. **Update host configuration**: Import monitoring services
-4. **Create Vault AppRole**: Add to `terraform/vault/approle.tf`
 
 ### Phase 2: Set Up VictoriaMetrics Stack
 
-Create new service module at `services/monitoring/victoriametrics/` for testing alongside existing
+New service module at `services/victoriametrics/` for VictoriaMetrics + vmalert + Alertmanager.
-Prometheus config. Once validated, this can replace the Prometheus module.
+Imported by monitoring02 alongside the existing Grafana service.
 
-1. **VictoriaMetrics** (port 8428):
+1. **VictoriaMetrics** (port 8428): [DONE]
    - `services.victoriametrics.enable = true`
-   - `services.victoriametrics.retentionPeriod = "3m"` (3 months, increase later based on disk usage)
+   - `retentionPeriod = "3"` (3 months)
-   - Migrate scrape configs via `prometheusConfig`
+   - All scrape configs migrated from Prometheus (22 jobs including auto-generated)
-   - Use native push support (replaces Pushgateway)
+   - Static user override (DynamicUser disabled) for credential file access
+   - OpenBao token fetch service + 30min refresh timer
+   - Apiary bearer token via vault.secrets
 
-2. **vmalert** for alerting rules:
+2. **vmalert** for alerting rules: [DONE]
-   - `services.vmalert.enable = true`
+   - Points to VictoriaMetrics datasource at localhost:8428
-   - Point to VictoriaMetrics for metrics evaluation
+   - Reuses existing `services/monitoring/rules.yml` directly via `settings.rule`
-   - Keep rules in separate `rules.yml` file (same format as Prometheus)
+   - No notifier configured during parallel operation (prevents duplicate alerts)
-   - No receiver configured during parallel operation (prevents duplicate alerts)
 
-3. **Alertmanager** (port 9093):
+3. **Alertmanager** (port 9093): [DONE]
-   - Keep existing configuration (alerttonotify webhook routing)
+   - Same configuration as monitoring01 (alerttonotify webhook routing)
-   - Only enable receiver after cutover from monitoring01
+   - Will only receive alerts after cutover (vmalert notifier disabled)
 
-4. **Loki** (port 3100):
+4. **Grafana** (port 3000): [DONE]
-   - Same configuration as current
+   - VictoriaMetrics datasource (localhost:8428) as default
+   - monitoring01 Prometheus datasource kept for comparison during parallel operation
+   - Loki datasource pointing to localhost (after Loki migrated to monitoring02)
 
-5. **Grafana** (port 3000):
+5. **Loki** (port 3100): [DONE]
-   - Define dashboards declaratively via NixOS options (not imported from monitoring01)
+   - Same configuration as monitoring01 in standalone `services/loki/` module
-   - Reference existing dashboards on monitoring01 for content inspiration
+   - Grafana datasource updated to localhost:3100
-   - Configure VictoriaMetrics datasource (port 8428)
-   - Configure Loki datasource
 
-6. **Tempo** (ports 3200, 3201):
+**Note:** pve-exporter and pushgateway scrape targets are not included on monitoring02.
-   - Same configuration
+pve-exporter requires a local exporter instance; pushgateway is replaced by VictoriaMetrics
-
+native push support.
-7. **Pyroscope** (port 4040):
-   - Same Docker-based deployment
 
 ### Phase 3: Parallel Operation
 
@@ -147,7 +140,6 @@ Update hardcoded references to use the CNAME:
    - prometheus.home.2rjus.net -> monitoring.home.2rjus.net:8428
    - alertmanager.home.2rjus.net -> monitoring.home.2rjus.net:9093
    - grafana.home.2rjus.net -> monitoring.home.2rjus.net:3000
-   - pyroscope.home.2rjus.net -> monitoring.home.2rjus.net:4040
 
 Note: `hosts/template2/bootstrap.nix` stays pointed at monitoring01 until decommission.
 
@@ -171,57 +163,26 @@ Once ready to cut over:
 
 ## Current Progress
 
-### monitoring02 Host Created (2026-02-08)
+- **Phase 1** complete (2026-02-08): monitoring02 host created, Grafana with Kanidm OIDC validated
-
+- **Phase 2** complete (2026-02-17): VictoriaMetrics, vmalert, Alertmanager, Loki, Grafana datasources configured
-Host deployed at 10.69.13.24 (test tier) with:
+  - Tempo and Pyroscope deferred (not actively used; can be added later if needed)
-- 4 CPU cores, 8GB RAM, 60GB disk
-- Vault integration enabled
-- NATS-based remote deployment enabled
-
-### Grafana with Kanidm OIDC (2026-02-08)
-
-Grafana deployed on monitoring02 as a test instance (`grafana-test.home.2rjus.net`):
-- Kanidm OIDC authentication (PKCE enabled)
-- Role mapping: `admins` → Admin, others → Viewer
-- Declarative datasources pointing to monitoring01 (Prometheus, Loki)
-- Local Caddy for TLS termination via internal ACME CA
-
-This validates the Grafana + OIDC pattern before the full VictoriaMetrics migration. The existing
-`services/monitoring/grafana.nix` on monitoring01 can be replaced with the new `services/grafana/`
-module once monitoring02 becomes the primary monitoring host.
 
 ## Open Questions
 
 - [ ] What disk size for monitoring02? Current 60GB may need expansion for 3+ months with VictoriaMetrics
 - [ ] Which dashboards to recreate declaratively? (Review monitoring01 Grafana for current set)
+- [ ] Consider replacing Promtail with Grafana Alloy (`services.alloy`, v1.12.2 in nixpkgs). Promtail is in maintenance mode and Grafana recommends Alloy as the successor. Alloy is a unified collector (logs, metrics, traces, profiles) but uses its own "River" config format instead of YAML, so less Nix-native ergonomics. Could bundle the migration with monitoring02 to consolidate disruption.
 
 ## VictoriaMetrics Service Configuration
 
-Example NixOS configuration for monitoring02:
+Implemented in `services/victoriametrics/default.nix`. Key design decisions:
 
-```nix
+- **Static user**: VictoriaMetrics NixOS module uses `DynamicUser`, overridden with a static
-# VictoriaMetrics replaces Prometheus
+  `victoriametrics` user so vault.secrets and credential files work correctly
-services.victoriametrics = {
+- **Shared rules**: vmalert reuses `services/monitoring/rules.yml` via `settings.rule` path
-  enable = true;
+  reference (no YAML-to-Nix conversion needed)
-  retentionPeriod = "3m";  # 3 months, increase based on disk usage
+- **Scrape config reuse**: Uses the same `lib/monitoring.nix` functions and
-  prometheusConfig = {
+  `services/monitoring/external-targets.nix` as Prometheus for auto-generated targets
-    global.scrape_interval = "15s";
-    scrape_configs = [
-      # Auto-generated node-exporter targets
-      # Service-specific scrape targets
-      # External targets
-    ];
-  };
-};
-
-# vmalert for alerting rules (no receiver during parallel operation)
-services.vmalert = {
-  enable = true;
-  datasource.url = "http://localhost:8428";
-  # notifier.alertmanager.url = "http://localhost:9093";  # Enable after cutover
-  rule = [ ./rules.yml ];
-};
-```
 
 ## Rollback Plan
 

docs/plans/new-services.md

@@ -0,0 +1,145 @@
+# New Service Candidates
+
+Ideas for additional services to deploy in the homelab. These lean more enterprise/obscure
+than the typical self-hosted fare.
+
+## Litestream
+
+Continuous SQLite replication to S3-compatible storage. Streams WAL changes in near-real-time,
+providing point-in-time recovery without scheduled backup jobs.
+
+**Why:** Several services use SQLite (Home Assistant, potentially others). Litestream would
+give continuous backup to Garage S3 with minimal resource overhead and near-zero configuration.
+Replaces cron-based backup scripts with a small daemon per database.
+
+**Integration points:**
+- Garage S3 as replication target (already deployed)
+- Home Assistant SQLite database is the primary candidate
+- Could also cover any future SQLite-backed services
+
+**Complexity:** Low. Single Go binary, minimal config (source DB path + S3 endpoint).
+
+**NixOS packaging:** Available in nixpkgs as `litestream`.
+
+---
+
+## ntopng
+
+Deep network traffic analysis and flow monitoring. Provides real-time visibility into bandwidth
+usage, protocol distribution, top talkers, and anomaly detection via a web UI.
+
+**Why:** We have host-level metrics (node-exporter) and logs (Loki) but no network-level
+visibility. ntopng would show traffic patterns across the infrastructure — NFS throughput to
+the NAS, DNS query volume, inter-host traffic, and bandwidth anomalies. Useful for capacity
+planning and debugging network issues.
+
+**Integration points:**
+- Could export metrics to Prometheus via its built-in exporter
+- Web UI behind http-proxy with Kanidm OIDC (if supported) or Pomerium
+- NetFlow/sFlow from managed switches (if available)
+- Passive traffic capture on a mirror port or the monitoring host itself
+
+**Complexity:** Medium. Needs network tap or mirror port for full visibility, or can run
+in host-local mode. May need a dedicated interface or VLAN mirror.
+
+**NixOS packaging:** Available in nixpkgs as `ntopng`.
+
+---
+
+## Renovate
+
+Automated dependency update bot that understands Nix flakes natively. Creates branches/PRs
+to bump flake inputs on a configurable schedule.
+
+**Why:** Currently `nix flake update` is manual. Renovate can automatically propose updates
+to individual flake inputs (nixpkgs, homelab-deploy, nixos-exporter, etc.), group related
+updates, and respect schedules. More granular than updating everything at once — can bump
+nixpkgs weekly but hold back other inputs, auto-merge patch-level changes, etc.
+
+**Integration points:**
+- Runs against git.t-juice.club repositories
+- Understands `flake.lock` format natively
+- Could target both `nixos-servers` and `nixos` repos
+- Update branches would be validated by homelab-deploy builder
+
+**Complexity:** Medium. Needs git forge integration (Gitea/Forgejo API). Self-hosted runner
+mode available. Configuration via `renovate.json` in each repo.
+
+**NixOS packaging:** Available in nixpkgs as `renovate`.
+
+---
+
+## Pomerium
+
+Identity-aware reverse proxy implementing zero-trust access. Every request is authenticated
+and authorized based on identity, device, and context — not just network location.
+
+**Why:** Currently Caddy terminates TLS but doesn't enforce authentication on most services.
+Pomerium would put Kanidm OIDC authentication in front of every internal service, with
+per-route authorization policies (e.g., "only admins can access Prometheus," "require re-auth
+for Vault UI"). Directly addresses the security hardening plan's goals.
+
+**Integration points:**
+- Kanidm as OIDC identity provider (already deployed)
+- Could replace or sit in front of Caddy for internal services
+- Per-route policies based on Kanidm groups (admins, users, ssh-users)
+- Centralizes access logging and audit trail
+
+**Complexity:** Medium-high. Needs careful integration with existing Caddy reverse proxy.
+Decision needed on whether Pomerium replaces Caddy or works alongside it (Pomerium for
+auth, Caddy for TLS termination and routing, or Pomerium handles everything).
+
+**NixOS packaging:** Available in nixpkgs as `pomerium`.
+
+---
+
+## Apache Guacamole
+
+Clientless remote desktop and SSH gateway. Provides browser-based access to hosts via
+RDP, VNC, SSH, and Telnet with no client software required. Supports session recording
+and playback.
+
+**Why:** Provides an alternative remote access path that doesn't require VPN software or
+SSH keys on the client device. Useful for accessing hosts from untrusted machines (phone,
+borrowed laptop) or providing temporary access to others. Session recording gives an audit
+trail. Could complement the WireGuard remote access plan rather than replace it.
+
+**Integration points:**
+- Kanidm for authentication (OIDC or LDAP)
+- Behind http-proxy or Pomerium for TLS
+- SSH access to all hosts in the fleet
+- Session recordings could be stored on Garage S3
+- Could serve as the "emergency access" path when VPN is unavailable
+
+**Complexity:** Medium. Java-based (guacd + web app), typically needs PostgreSQL for
+connection/user storage (already available). Docker is the common deployment method but
+native packaging exists.
+
+**NixOS packaging:** Available in nixpkgs as `guacamole-server` and `guacamole-client`.
+
+---
+
+## CrowdSec
+
+Collaborative intrusion prevention system with crowd-sourced threat intelligence.
+Parses logs to detect attack patterns, applies remediation (firewall bans, CAPTCHA),
+and shares/receives threat signals from a global community network.
+
+**Why:** Goes beyond fail2ban with behavioral detection, crowd-sourced IP reputation,
+and a scenario-based engine. Fits the security hardening plan. The community blocklist
+means we benefit from threat intelligence gathered across thousands of deployments.
+Could parse SSH logs, HTTP access logs, and other service logs to detect and block
+malicious activity.
+
+**Integration points:**
+- Could consume logs from Loki or directly from journald/log files
+- Firewall bouncer for iptables/nftables remediation
+- Caddy bouncer for HTTP-level blocking
+- Prometheus metrics exporter for alert integration
+- Scenarios available for SSH brute force, HTTP scanning, and more
+- Feeds into existing alerting pipeline (Alertmanager -> alerttonotify)
+
+**Complexity:** Medium. Agent (log parser + decision engine) on each host or centralized.
+Bouncers (enforcement) on edge hosts. Free community tier includes threat intel access.
+
+**NixOS packaging:** Available in nixpkgs as `crowdsec`.

docs/plans/nixos-router.md

@@ -0,0 +1,162 @@
+# NixOS Router — Replace EdgeRouter
+
+Replace the aging Ubiquiti EdgeRouter (gw, 10.69.10.1) with a NixOS-based router.
+The EdgeRouter is suspected to be a throughput bottleneck. A NixOS router integrates
+naturally with the existing fleet: same config management, same monitoring pipeline,
+same deployment workflow.
+
+## Goals
+
+- Eliminate the EdgeRouter throughput bottleneck
+- Full integration with existing monitoring (node-exporter, promtail, Prometheus, Loki)
+- Declarative firewall and routing config managed in the flake
+- Inter-VLAN routing for all existing subnets
+- DHCP server for client subnets
+- NetFlow/traffic accounting for future ntopng integration
+- Foundation for WireGuard remote access (see remote-access.md)
+
+## Current Network Topology
+
+**Subnets (known VLANs):**
+| VLAN/Subnet    | Purpose          | Notable hosts                          |
+|----------------|------------------|----------------------------------------|
+| 10.69.10.0/24  | Gateway          | gw (10.69.10.1)                        |
+| 10.69.12.0/24  | Core services    | nas, pve1, arr jails, restic           |
+| 10.69.13.0/24  | Infrastructure   | All NixOS servers (static IPs)         |
+| 10.69.22.0/24  | WLAN             | unifi-ctrl                             |
+| 10.69.30.0/24  | Workstations     | gunter                                 |
+| 10.69.31.0/24  | Media            | media                                  |
+| 10.69.99.0/24  | Management       | sw1 (MikroTik CRS326-24G-2S+)         |
+
+**DNS:** ns1 (10.69.13.5) and ns2 (10.69.13.6) handle all resolution. Upstream is
+Cloudflare/Google over DoT via Unbound.
+
+**Switch:** MikroTik CRS326-24G-2S+ — L2 switching with VLAN trunking. Capable of
+L3 routing via RouterOS but not ideal for sustained routing throughput.
+
+## Hardware
+
+Needs a small x86 box with:
+- At least 2 NICs (WAN + LAN trunk). Dual 2.5GbE preferred.
+- Enough CPU for nftables NAT at line rate (any modern x86 is fine)
+- 4-8 GB RAM (plenty for routing + DHCP + NetFlow accounting)
+- Low power consumption, fanless preferred for always-on use
+
+Candidates:
+- Topton / CWWK mini PC with dual/quad Intel 2.5GbE (~100-150 EUR)
+- Protectli Vault (more expensive, ~200-300 EUR, proven in pfSense/OPNsense community)
+- Any mini PC with one onboard NIC + one USB 2.5GbE adapter (cheapest, less ideal)
+
+The LAN port would carry a VLAN trunk to the MikroTik switch, with sub-interfaces
+for each VLAN. WAN port connects to the ISP uplink.
+
+## NixOS Configuration
+
+### Stability Policy
+
+The router is treated differently from the rest of the fleet:
+- **No auto-upgrade** — `system.autoUpgrade.enable = false`
+- **No homelab-deploy listener** — `homelab.deploy.enable = false`
+- **Manual updates only** — update every few months, test-build first
+- **Use `nixos-rebuild boot`** — changes take effect on next deliberate reboot
+- **Tier: prod, priority: high** — alerts treated with highest priority
+
+### Core Services
+
+**Routing & NAT:**
+- `systemd-networkd` for all interface config (consistent with rest of fleet)
+- VLAN sub-interfaces on the LAN trunk (one per subnet)
+- `networking.nftables` for stateful firewall and NAT
+- IP forwarding enabled (`net.ipv4.ip_forward = 1`)
+- Masquerade outbound traffic on WAN interface
+
+**DHCP:**
+- Kea or dnsmasq for DHCP on client subnets (WLAN, workstations, media)
+- Infrastructure subnet (10.69.13.0/24) stays static — no DHCP needed
+- Static leases for known devices
+
+**Firewall (nftables):**
+- Default deny between VLANs
+- Explicit allow rules for known cross-VLAN traffic:
+  - All subnets → ns1/ns2 (DNS)
+  - All subnets → monitoring01 (metrics/logs)
+  - Infrastructure → all (management access)
+  - Workstations → media, core services
+- NAT masquerade on WAN
+- Rate limiting on WAN-facing services
+
+**Traffic Accounting:**
+- nftables flow accounting or softflowd for NetFlow export
+- Export to future ntopng instance (see new-services.md)
+
+### Monitoring Integration
+
+Since this is a NixOS host in the flake, it gets the standard monitoring stack for free:
+- node-exporter for system metrics (CPU, memory, NIC throughput per interface)
+- promtail shipping logs to Loki
+- Prometheus scrape target auto-registration
+- Alertmanager alerts for host-down, high CPU, etc.
+
+Additional router-specific monitoring:
+- Per-VLAN interface traffic metrics via node-exporter (automatic for all interfaces)
+- NAT connection tracking table size
+- WAN uplink status and throughput
+- DHCP lease metrics (if Kea, it has a Prometheus exporter)
+
+This is a significant advantage over the EdgeRouter — full observability through
+the existing Grafana dashboards and Loki log search, debuggable via the monitoring
+MCP tools.
+
+### WireGuard Integration
+
+The remote access plan (remote-access.md) currently proposes a separate `extgw01`
+gateway host. With a NixOS router, there's a decision to make:
+
+**Option A:** WireGuard terminates on the router itself. Simplest topology — the
+router is already the gateway, so VPN traffic doesn't need extra hops or firewall
+rules. But adds complexity to the router, which should stay simple.
+
+**Option B:** Keep extgw01 as a separate host (original plan). Router just routes
+traffic to it. Better separation of concerns, router stays minimal.
+
+Recommendation: Start with option B (keep it separate). The router should do routing
+and nothing else. WireGuard can move to the router later if extgw01 feels redundant.
+
+## Migration Plan
+
+### Phase 1: Build and lab test
+- Acquire hardware
+- Create host config in the flake (routing, NAT, DHCP, firewall)
+- Test-build on workstation: `nix build .#nixosConfigurations.router01.config.system.build.toplevel`
+- Lab test with a temporary setup if possible (two NICs, isolated VLAN)
+
+### Phase 2: Prepare cutover
+- Pre-configure the MikroTik switch trunk port for the new router
+- Document current EdgeRouter config (port forwarding, NAT rules, DHCP leases)
+- Replicate all rules in the NixOS config
+- Verify DNS, DHCP, and inter-VLAN routing work in test
+
+### Phase 3: Cutover
+- Schedule a maintenance window (brief downtime expected)
+- Swap WAN cable from EdgeRouter to new router
+- Swap LAN trunk from EdgeRouter to new router
+- Verify connectivity from each VLAN
+- Verify internet access, DNS resolution, inter-VLAN routing
+- Monitor via Prometheus/Loki (immediately available since it's a fleet host)
+
+### Phase 4: Decommission EdgeRouter
+- Keep EdgeRouter available as fallback for a few weeks
+- Remove `gw` entry from external-hosts.nix, replace with flake-managed host
+- Update any references to 10.69.10.1 if the router IP changes
+
+## Open Questions
+
+- **Router IP:** Keep 10.69.10.1 or move to a different address? Each VLAN
+  sub-interface needs an IP (the gateway address for that subnet).
+- **ISP uplink:** What type of WAN connection? PPPoE, DHCP, static IP?
+- **Port forwarding:** What ports are currently forwarded on the EdgeRouter?
+  These need to be replicated in nftables.
+- **DHCP scope:** Which subnets currently get DHCP from the EdgeRouter vs
+  other sources (UniFi controller for WLAN?)?
+- **UPnP/NAT-PMP:** Needed for any devices? (gaming consoles, etc.)
+- **Hardware preference:** Fanless mini PC budget and preferred vendor?

docs/plans/remote-access.md

@@ -4,119 +4,127 @@
 
 ## Goal
 
-Enable remote access to some or all homelab services from outside the internal network, without exposing anything directly to the internet.
+Enable personal remote access to selected homelab services from outside the internal network, without exposing anything directly to the internet.
 
 ## Current State
 
 - All services are only accessible from the internal 10.69.13.x network
-- Exception: jelly01 has a WireGuard link to an external VPS
+- http-proxy has a WireGuard tunnel (`wg0`, `10.69.222.0/24`) to a VPS (`docker2.t-juice.club`) on an OpenStack cluster
-- No services are directly exposed to the public internet
+- VPS runs Traefik which proxies selected services (including Jellyfin) back through the tunnel to http-proxy's Caddy
+- No other services are directly exposed to the public internet
 
-## Constraints
+## Decision: WireGuard Gateway
 
-- Nothing should be directly accessible from the outside
+After evaluating WireGuard gateway vs Headscale (self-hosted Tailscale), the **WireGuard gateway** approach was chosen:
-- Must use VPN or overlay network (no port forwarding of services)
-- Self-hosted solutions preferred over managed services
 
-## Options
+- Only 2 client devices (laptop + phone), so Headscale's device management UX isn't needed
+- Split DNS works fine on Linux laptop via systemd-resolved; all-or-nothing DNS on phone is acceptable for occasional use
+- Simpler infrastructure - no control server to maintain
+- Builds on existing WireGuard experience and setup
 
-### 1. WireGuard Gateway (Internal Router)
+## Architecture
 
-A dedicated NixOS host on the internal network with a WireGuard tunnel out to the VPS. The VPS becomes the public entry point, and the gateway routes traffic to internal services. Firewall rules on the gateway control which services are reachable.
+```
+                    ┌─────────────────────────────────┐
+                    │  VPS (OpenStack)                │
+  Laptop/Phone ──→ │  WireGuard endpoint             │
+  (WireGuard)      │  Client peers: laptop, phone    │
+                    │  Routes 10.69.13.0/24 via tunnel│
+                    └──────────┬──────────────────────┘
+                               │ WireGuard tunnel
+                               ▼
+                    ┌─────────────────────────────────┐
+                    │  extgw01 (gateway + bastion)    │
+                    │  - WireGuard tunnel to VPS      │
+                    │  - Firewall (allowlist only)    │
+                    │  - SSH + 2FA (full access)      │
+                    └──────────┬──────────────────────┘
+                               │ allowed traffic only
+                               ▼
+                    ┌─────────────────────────────────┐
+                    │  Internal network 10.69.13.0/24 │
+                    │  - monitoring01:3000 (Grafana)  │
+                    │  - jelly01:8096 (Jellyfin)      │
+                    │  - *-jail hosts (arr stack)     │
+                    └─────────────────────────────────┘
+```
 
-**Pros:**
+### Existing path (unchanged)
-- Simple, well-understood technology
-- Already running WireGuard for jelly01
-- Full control over routing and firewall rules
-- Excellent NixOS module support
-- No extra dependencies
 
-**Cons:**
+The current public access path stays as-is:
-- Hub-and-spoke topology (all traffic goes through VPS)
-- Manual peer management
-- Adding a new client device means editing configs on both VPS and gateway
 
-### 2. WireGuard Mesh (No Relay)
+```
+Internet → VPS (Traefik) → WireGuard → http-proxy (Caddy) → internal services
+```
 
-Each client device connects directly to a WireGuard endpoint. Could be on the VPS which forwards to the homelab, or if there is a routable IP at home, directly to an internal host.
+This handles public Jellyfin access and any other publicly-exposed services.
 
-**Pros:**
+### New path (personal VPN)
-- Simple and fast
-- No extra software
 
-**Cons:**
+A separate WireGuard tunnel for personal remote access with restricted firewall rules:
-- Manual key and endpoint management for every peer
-- Doesn't scale well
-- If behind CGNAT, still needs the VPS as intermediary
 
-### 3. Headscale (Self-Hosted Tailscale)
+```
+Laptop/Phone → VPS (WireGuard peers) → tunnel → extgw01 (firewall) → allowed services
+```
 
-Run a Headscale control server (on the VPS or internally) and install the Tailscale client on homelab hosts and personal devices. Gets the Tailscale mesh networking UX without depending on Tailscale's infrastructure.
+### Access tiers
 
-**Pros:**
+1. **VPN (default)**: Laptop/phone connect to VPS WireGuard endpoint, traffic routed through extgw01 firewall. Only whitelisted services are reachable.
-- Mesh topology - devices communicate directly via NAT traversal (DERP relay as fallback)
+2. **SSH + 2FA (escalated)**: SSH into extgw01 for full network access when needed.
-- Easy to add/remove devices
-- ACL support for granular access control
-- MagicDNS for service discovery
-- Good NixOS support for both headscale server and tailscale client
-- Subnet routing lets you expose the entire 10.69.13.x network or specific hosts without installing tailscale on every host
 
-**Cons:**
+## New Host: extgw01
-- More moving parts than plain WireGuard
-- Headscale is a third-party reimplementation, can lag behind Tailscale features
-- Need to run and maintain the control server
 
-### 4. Tailscale (Managed)
+A NixOS host on the internal network acting as both WireGuard gateway and SSH bastion.
 
-Same as Headscale but using Tailscale's hosted control plane.
+### Responsibilities
 
-**Pros:**
+- **WireGuard tunnel** to the VPS for client traffic
-- Zero infrastructure to manage on the control plane side
+- **Firewall** with allowlist controlling which internal services are reachable through the VPN
-- Polished UX, well-maintained clients
+- **SSH bastion** with 2FA for full network access when needed
-- Free tier covers personal use
+- **DNS**: Clients get split DNS config (laptop via systemd-resolved routing domain, phone uses internal DNS for all queries)
 
-**Cons:**
+### Firewall allowlist (initial)
-- Dependency on Tailscale's service
-- Less aligned with self-hosting preference
-- Coordination metadata goes through their servers (data plane is still peer-to-peer)
 
-### 5. Netbird (Self-Hosted)
+| Service    | Destination                  | Port  |
+|------------|------------------------------|-------|
+| Grafana    | monitoring01.home.2rjus.net  | 3000  |
+| Jellyfin   | jelly01.home.2rjus.net       | 8096  |
+| Sonarr     | sonarr-jail.home.2rjus.net   | 8989  |
+| Radarr     | radarr-jail.home.2rjus.net   | 7878  |
+| NZBget     | nzbget-jail.home.2rjus.net   | 6789  |
 
-Open-source alternative to Tailscale with a self-hostable management server. WireGuard-based, supports ACLs and NAT traversal.
+### SSH 2FA options (to be decided)
 
-**Pros:**
+- **Kanidm**: Already deployed on kanidm01, supports RADIUS/OAuth2 for PAM integration
-- Fully self-hostable
+- **SSH certificates via OpenBao**: Fits existing Vault infrastructure, short-lived certs
-- Web UI for management
+- **TOTP via PAM**: Simplest fallback, Google Authenticator / similar
-- ACL and peer grouping support
 
-**Cons:**
+## VPS Configuration
-- Heavier to self-host (needs multiple components: management server, signal server, TURN relay)
-- Less mature NixOS module support compared to Tailscale/Headscale
 
-### 6. Nebula (by Defined Networking)
+The VPS needs a new WireGuard interface (separate from the existing http-proxy tunnel):
 
-Certificate-based mesh VPN. Each node gets a certificate from a CA you control. No central coordination server needed at runtime.
+- WireGuard endpoint listening on a public UDP port
+- 2 peers: laptop, phone
+- Routes client traffic through tunnel to extgw01
+- Minimal config - just routing, no firewall policy (that lives on extgw01)
 
-**Pros:**
+## Implementation Steps
-- No always-on control plane
-- Certificate-based identity
-- Lightweight
 
-**Cons:**
+1. **Create extgw01 host configuration** in this repo
-- Less convenient for ad-hoc device addition (need to issue certs)
+   - VM provisioned via OpenTofu (same as other hosts)
-- NAT traversal less mature than Tailscale's
+   - WireGuard interface for VPS tunnel
-- Smaller community/ecosystem
+   - nftables/iptables firewall with service allowlist
-
+   - IP forwarding enabled
-## Key Decision Points
+2. **Configure VPS WireGuard** for client peers
-
+   - New WireGuard interface with laptop + phone peers
-- **Static public IP vs CGNAT?** Determines whether clients can connect directly to home network or need VPS relay.
+   - Routing for 10.69.13.0/24 through extgw01 tunnel
-- **Number of client devices?** If just phone and laptop, plain WireGuard via VPS is fine. More devices favors Headscale.
+3. **Set up client configs**
-- **Per-service vs per-network access?** Gateway with firewall rules gives per-service control. Headscale ACLs can also do this. Plain WireGuard gives network-level access with gateway firewall for finer control.
+   - Laptop: WireGuard config + systemd-resolved split DNS for `home.2rjus.net`
-- **Subnet routing vs per-host agents?** With Headscale/Tailscale, can either install client on every host, or use a single subnet router that advertises the 10.69.13.x range. The latter is closer to the gateway approach and avoids touching every host.
+   - Phone: WireGuard app config with DNS pointing at internal nameservers
-
+4. **Set up SSH 2FA** on extgw01
-## Leading Candidates
+   - Evaluate Kanidm integration vs OpenBao SSH certs vs TOTP
-
+5. **Test and verify**
-Based on existing WireGuard experience, self-hosting preference, and NixOS stack:
+   - VPN access to allowed services only
-
+   - Firewall blocks everything else
-1. **Headscale with a subnet router** - Best balance of convenience and self-hosting
+   - SSH + 2FA grants full access
-2. **WireGuard gateway via VPS** - Simplest, most transparent, builds on existing setup
+   - Existing public access path unaffected

flake.lock

@@ -64,11 +64,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1770770419,
+        "lastModified": 1771043024,
-        "narHash": "sha256-iKZMkr6Cm9JzWlRYW/VPoL0A9jVKtZYiU4zSrVeetIs=",
+        "narHash": "sha256-O1XDr7EWbRp+kHrNNgLWgIrB0/US5wvw9K6RERWAj6I=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "6c5e707c6b5339359a9a9e215c5e66d6d802fd7a",
+        "rev": "3aadb7ca9eac2891d52a9dec199d9580a6e2bf44",
         "type": "github"
       },
       "original": {
@@ -80,11 +80,11 @@
     },
     "nixpkgs-unstable": {
       "locked": {
-        "lastModified": 1770562336,
+        "lastModified": 1771008912,
-        "narHash": "sha256-ub1gpAONMFsT/GU2hV6ZWJjur8rJ6kKxdm9IlCT0j84=",
+        "narHash": "sha256-gf2AmWVTs8lEq7z/3ZAsgnZDhWIckkb+ZnAo5RzSxJg=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "d6c71932130818840fc8fe9509cf50be8c64634f",
+        "rev": "a82ccc39b39b621151d6732718e3e250109076fa",
         "type": "github"
       },
       "original": {

flake.nix

@@ -200,6 +200,15 @@
             ./hosts/nix-cache02
           ];
         };
+        garage01 = nixpkgs.lib.nixosSystem {
+          inherit system;
+          specialArgs = {
+            inherit inputs self;
+          };
+          modules = commonModules ++ [
+            ./hosts/garage01
+          ];
+        };
       };
       packages = forAllSystems (
         { pkgs }:

hosts/garage01/configuration.nix

@@ -0,0 +1,75 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  imports = [
+    ../template2/hardware-configuration.nix
+
+    ../../system
+    ../../common/vm
+  ];
+
+  # Host metadata (adjust as needed)
+  homelab.host = {
+    tier = "test";  # Start in test tier, move to prod after validation
+    role = "storage";
+  };
+
+  homelab.dns.cnames = [ "s3" ];
+
+  # Enable Vault integration
+  vault.enable = true;
+
+  # Enable remote deployment via NATS
+  homelab.deploy.enable = true;
+
+  nixpkgs.config.allowUnfree = true;
+  boot.loader.grub.enable = true;
+  boot.loader.grub.device = "/dev/vda";
+
+  networking.hostName = "garage01";
+  networking.domain = "home.2rjus.net";
+  networking.useNetworkd = true;
+  networking.useDHCP = false;
+  services.resolved.enable = true;
+  networking.nameservers = [
+    "10.69.13.5"
+    "10.69.13.6"
+  ];
+
+  systemd.network.enable = true;
+  systemd.network.networks."ens18" = {
+    matchConfig.Name = "ens18";
+    address = [
+      "10.69.13.26/24"
+    ];
+    routes = [
+      { Gateway = "10.69.13.1"; }
+    ];
+    linkConfig.RequiredForOnline = "routable";
+  };
+  time.timeZone = "Europe/Oslo";
+
+  nix.settings.experimental-features = [
+    "nix-command"
+    "flakes"
+  ];
+  nix.settings.tarball-ttl = 0;
+  environment.systemPackages = with pkgs; [
+    vim
+    wget
+    git
+  ];
+
+  # Open ports in the firewall.
+  # networking.firewall.allowedTCPPorts = [ ... ];
+  # networking.firewall.allowedUDPPorts = [ ... ];
+  # Or disable the firewall altogether.
+  networking.firewall.enable = false;
+
+  system.stateVersion = "25.11"; # Did you read the comment?
+}

hosts/garage01/default.nix

@@ -0,0 +1,6 @@
+{ ... }: {
+  imports = [
+    ./configuration.nix
+    ../../services/garage
+  ];
+}

hosts/monitoring02/configuration.nix

@@ -18,8 +18,7 @@
     role = "monitoring";
   };
 
-  # DNS CNAME for Grafana test instance
+  homelab.dns.cnames = [ "grafana-test" "metrics" "vmalert" "loki" ];
-  homelab.dns.cnames = [ "grafana-test" ];
 
   # Enable Vault integration
   vault.enable = true;

hosts/monitoring02/default.nix

@@ -2,5 +2,7 @@
   imports = [
     ./configuration.nix
     ../../services/grafana
+    ../../services/victoriametrics
+    ../../services/loki
   ];
 }

hosts/template2/bootstrap.nix

@@ -28,7 +28,7 @@ let
             streams: [{
               stream: {
                 job: "bootstrap",
-                host: $host,
+                hostname: $host,
                 stage: $stage,
                 branch: $branch
               },

services/garage/default.nix

@@ -0,0 +1,64 @@
+{ config, pkgs, ... }:
+{
+  homelab.monitoring.scrapeTargets = [
+    {
+      job_name = "garage";
+      port = 3903;
+      metrics_path = "/metrics";
+    }
+    {
+      job_name = "caddy";
+      port = 9117;
+    }
+  ];
+
+  vault.secrets.garage-env = {
+    secretPath = "hosts/${config.networking.hostName}/garage";
+    extractKey = "env";
+    outputDir = "/run/secrets/garage-env";
+    services = [ "garage" ];
+  };
+
+  services.garage = {
+    enable = true;
+    package = pkgs.garage;
+    environmentFile = "/run/secrets/garage-env";
+    settings = {
+      metadata_dir = "/var/lib/garage/meta";
+      data_dir = "/var/lib/garage/data";
+      replication_factor = 1;
+      rpc_bind_addr = "[::]:3901";
+      rpc_public_addr = "garage01.home.2rjus.net:3901";
+      s3_api = {
+        api_bind_addr = "[::]:3900";
+        s3_region = "garage";
+        root_domain = ".s3.home.2rjus.net";
+      };
+      admin = {
+        api_bind_addr = "[::]:3903";
+      };
+    };
+  };
+
+  services.caddy = {
+    enable = true;
+    package = pkgs.unstable.caddy;
+    configFile = pkgs.writeText "Caddyfile" ''
+      {
+        acme_ca https://vault.home.2rjus.net:8200/v1/pki_int/acme/directory
+        metrics
+      }
+
+      s3.home.2rjus.net {
+        reverse_proxy http://localhost:3900
+      }
+
+      http://garage01.home.2rjus.net:9117 {
+        handle /metrics {
+          metrics
+        }
+        respond 404
+      }
+    '';
+  };
+}

services/grafana/dashboards/apiary.json

@@ -0,0 +1,391 @@
+{
+  "uid": "apiary-homelab",
+  "title": "Apiary - Honeypot",
+  "tags": ["apiary", "honeypot", "prometheus", "homelab"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "1m",
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "templating": {
+    "list": []
+  },
+  "panels": [
+    {
+      "id": 1,
+      "title": "SSH Connections",
+      "type": "stat",
+      "gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "targets": [
+        {
+          "expr": "sum(oubliette_ssh_connections_total{job=\"apiary\"})",
+          "legendFormat": "Total",
+          "refId": "A",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "blue", "value": null}
+            ]
+          }
+        }
+      },
+      "options": {
+        "reduceOptions": {"calcs": ["lastNotNull"]},
+        "colorMode": "value",
+        "graphMode": "none",
+        "textMode": "auto"
+      },
+      "description": "Total SSH connections across all outcomes"
+    },
+    {
+      "id": 2,
+      "title": "Active Sessions",
+      "type": "stat",
+      "gridPos": {"h": 4, "w": 6, "x": 6, "y": 0},
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "targets": [
+        {
+          "expr": "oubliette_sessions_active{job=\"apiary\"}",
+          "legendFormat": "Active",
+          "refId": "A",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green", "value": null},
+              {"color": "yellow", "value": 5},
+              {"color": "red", "value": 20}
+            ]
+          },
+          "noValue": "0"
+        }
+      },
+      "options": {
+        "reduceOptions": {"calcs": ["lastNotNull"]},
+        "colorMode": "value",
+        "graphMode": "none",
+        "textMode": "auto"
+      },
+      "description": "Currently active honeypot sessions"
+    },
+    {
+      "id": 3,
+      "title": "Unique IPs",
+      "type": "stat",
+      "gridPos": {"h": 4, "w": 6, "x": 12, "y": 0},
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "targets": [
+        {
+          "expr": "oubliette_storage_unique_ips{job=\"apiary\"}",
+          "legendFormat": "IPs",
+          "refId": "A",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "purple", "value": null}
+            ]
+          }
+        }
+      },
+      "options": {
+        "reduceOptions": {"calcs": ["lastNotNull"]},
+        "colorMode": "value",
+        "graphMode": "none",
+        "textMode": "auto"
+      },
+      "description": "Total unique source IPs observed"
+    },
+    {
+      "id": 4,
+      "title": "Total Login Attempts",
+      "type": "stat",
+      "gridPos": {"h": 4, "w": 6, "x": 18, "y": 0},
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "targets": [
+        {
+          "expr": "oubliette_storage_login_attempts_total{job=\"apiary\"}",
+          "legendFormat": "Attempts",
+          "refId": "A",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "orange", "value": null}
+            ]
+          }
+        }
+      },
+      "options": {
+        "reduceOptions": {"calcs": ["lastNotNull"]},
+        "colorMode": "value",
+        "graphMode": "none",
+        "textMode": "auto"
+      },
+      "description": "Total login attempts stored"
+    },
+    {
+      "id": 5,
+      "title": "SSH Connections Over Time",
+      "type": "timeseries",
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 4},
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "interval": "60s",
+      "targets": [
+        {
+          "expr": "rate(oubliette_ssh_connections_total{job=\"apiary\"}[$__rate_interval])",
+          "legendFormat": "{{outcome}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "cps",
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "fillOpacity": 20,
+            "pointSize": 5,
+            "showPoints": "auto",
+            "stacking": {"mode": "none"}
+          }
+        }
+      },
+      "options": {
+        "legend": {"displayMode": "list", "placement": "bottom"},
+        "tooltip": {"mode": "multi", "sort": "desc"}
+      },
+      "description": "SSH connection rate by outcome"
+    },
+    {
+      "id": 6,
+      "title": "Auth Attempts Over Time",
+      "type": "timeseries",
+      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 4},
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "interval": "60s",
+      "targets": [
+        {
+          "expr": "rate(oubliette_auth_attempts_total{job=\"apiary\"}[$__rate_interval])",
+          "legendFormat": "{{reason}} - {{result}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "cps",
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "fillOpacity": 20,
+            "pointSize": 5,
+            "showPoints": "auto",
+            "stacking": {"mode": "none"}
+          }
+        }
+      },
+      "options": {
+        "legend": {"displayMode": "list", "placement": "bottom"},
+        "tooltip": {"mode": "multi", "sort": "desc"}
+      },
+      "description": "Authentication attempt rate by reason and result"
+    },
+    {
+      "id": 7,
+      "title": "Sessions by Shell",
+      "type": "timeseries",
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 22},
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "interval": "60s",
+      "targets": [
+        {
+          "expr": "rate(oubliette_sessions_total{job=\"apiary\"}[$__rate_interval])",
+          "legendFormat": "{{shell}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "cps",
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "fillOpacity": 20,
+            "pointSize": 5,
+            "showPoints": "auto",
+            "stacking": {"mode": "normal"}
+          }
+        }
+      },
+      "options": {
+        "legend": {"displayMode": "list", "placement": "bottom"},
+        "tooltip": {"mode": "multi", "sort": "desc"}
+      },
+      "description": "Session creation rate by shell type"
+    },
+    {
+      "id": 8,
+      "title": "Attempts by Country",
+      "type": "geomap",
+      "gridPos": {"h": 10, "w": 24, "x": 0, "y": 12},
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "targets": [
+        {
+          "expr": "oubliette_auth_attempts_by_country_total{job=\"apiary\"}",
+          "legendFormat": "{{country}}",
+          "refId": "A",
+          "instant": true,
+          "format": "table"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green", "value": null},
+              {"color": "yellow", "value": 10},
+              {"color": "orange", "value": 50},
+              {"color": "red", "value": 200}
+            ]
+          }
+        }
+      },
+      "options": {
+        "view": {
+          "id": "zero",
+          "lat": 30,
+          "lon": 10,
+          "zoom": 2
+        },
+        "basemap": {
+          "type": "default"
+        },
+        "layers": [
+          {
+            "type": "markers",
+            "name": "Auth Attempts",
+            "config": {
+              "showLegend": true,
+              "style": {
+                "size": {
+                  "field": "Value",
+                  "min": 3,
+                  "max": 20
+                },
+                "color": {
+                  "field": "Value"
+                },
+                "symbol": {
+                  "mode": "fixed",
+                  "fixed": "img/icons/marker/circle.svg"
+                }
+              }
+            },
+            "location": {
+              "mode": "lookup",
+              "lookup": "country",
+              "gazetteer": "public/gazetteer/countries.json"
+            }
+          }
+        ]
+      },
+      "description": "Authentication attempts by country (geo lookup from country code)"
+    },
+    {
+      "id": 9,
+      "title": "Session Duration Distribution",
+      "type": "heatmap",
+      "gridPos": {"h": 8, "w": 24, "x": 0, "y": 30},
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "interval": "60s",
+      "targets": [
+        {
+          "expr": "rate(oubliette_session_duration_seconds_bucket{job=\"apiary\"}[$__rate_interval])",
+          "legendFormat": "{{le}}",
+          "refId": "A",
+          "format": "heatmap"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "scaleDistribution": {
+              "type": "log",
+              "log": 2
+            }
+          }
+        }
+      },
+      "options": {
+        "calculate": false,
+        "yAxis": {
+          "unit": "s"
+        },
+        "color": {
+          "scheme": "Oranges",
+          "mode": "scheme"
+        },
+        "cellGap": 1,
+        "tooltip": {
+          "show": true
+        }
+      },
+      "description": "Distribution of session durations"
+    },
+    {
+      "id": 10,
+      "title": "Commands Executed by Shell",
+      "type": "timeseries",
+      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 22},
+      "datasource": {"type": "prometheus", "uid": "prometheus"},
+      "interval": "60s",
+      "targets": [
+        {
+          "expr": "rate(oubliette_commands_executed_total{job=\"apiary\"}[$__rate_interval])",
+          "legendFormat": "{{shell}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "cps",
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "fillOpacity": 20,
+            "pointSize": 5,
+            "showPoints": "auto",
+            "stacking": {"mode": "normal"}
+          }
+        }
+      },
+      "options": {
+        "legend": {"displayMode": "list", "placement": "bottom"},
+        "tooltip": {"mode": "multi", "sort": "desc"}
+      },
+      "description": "Rate of commands executed in honeypot shells"
+    }
+  ]
+}

services/grafana/default.nix

@@ -34,21 +34,27 @@
       };
     };
 
-    # Declarative datasources pointing to monitoring01
+    # Declarative datasources
     provision.datasources.settings = {
       apiVersion = 1;
       datasources = [
         {
-          name = "Prometheus";
+          name = "VictoriaMetrics";
+          type = "prometheus";
+          url = "http://localhost:8428";
+          isDefault = true;
+          uid = "victoriametrics";
+        }
+        {
+          name = "Prometheus (monitoring01)";
           type = "prometheus";
           url = "http://monitoring01.home.2rjus.net:9090";
-          isDefault = true;
           uid = "prometheus";
         }
         {
           name = "Loki";
           type = "loki";
-          url = "http://monitoring01.home.2rjus.net:3100";
+          url = "http://localhost:3100";
           uid = "loki";
         }
       ];
@@ -81,22 +87,20 @@
   services.caddy = {
     enable = true;
     package = pkgs.unstable.caddy;
-    configFile = pkgs.writeText "Caddyfile" ''
+    globalConfig = ''
-      {
+      acme_ca https://vault.home.2rjus.net:8200/v1/pki_int/acme/directory
-        acme_ca https://vault.home.2rjus.net:8200/v1/pki_int/acme/directory
+      metrics
-        metrics
+    '';
-      }
+    virtualHosts."grafana-test.home.2rjus.net".extraConfig = ''
-
+      log {
-      grafana-test.home.2rjus.net {
+        output file /var/log/caddy/grafana.log {
-        log {
+          mode 644
-          output file /var/log/caddy/grafana.log {
-            mode 644
-          }
         }
-
-        reverse_proxy http://127.0.0.1:3000
       }
-
+      reverse_proxy http://127.0.0.1:3000
+    '';
+    # Metrics endpoint on plain HTTP for Prometheus scraping
+    extraConfig = ''
       http://${config.networking.hostName}.home.2rjus.net/metrics {
         metrics
       }

services/loki/default.nix

@@ -0,0 +1,104 @@
+{ config, lib, pkgs, ... }:
+let
+  # Script to generate bcrypt hash from Vault password for Caddy basic_auth
+  generateCaddyAuth = pkgs.writeShellApplication {
+    name = "generate-caddy-loki-auth";
+    runtimeInputs = [ config.services.caddy.package ];
+    text = ''
+      PASSWORD=$(cat /run/secrets/loki-push-auth)
+      HASH=$(caddy hash-password --plaintext "$PASSWORD")
+      echo "LOKI_PUSH_HASH=$HASH" > /run/secrets/caddy-loki-auth.env
+      chmod 0400 /run/secrets/caddy-loki-auth.env
+    '';
+  };
+in
+{
+  # Fetch Loki push password from Vault
+  vault.secrets.loki-push-auth = {
+    secretPath = "shared/loki/push-auth";
+    extractKey = "password";
+    services = [ "caddy" ];
+  };
+
+  # Generate bcrypt hash for Caddy before it starts
+  systemd.services.caddy-loki-auth = {
+    description = "Generate Caddy basic auth hash for Loki";
+    after = [ "vault-secret-loki-push-auth.service" ];
+    requires = [ "vault-secret-loki-push-auth.service" ];
+    before = [ "caddy.service" ];
+    requiredBy = [ "caddy.service" ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+      ExecStart = lib.getExe generateCaddyAuth;
+    };
+  };
+
+  # Load the bcrypt hash as environment variable for Caddy
+  services.caddy.environmentFile = "/run/secrets/caddy-loki-auth.env";
+
+  # Caddy reverse proxy for Loki with basic auth
+  services.caddy.virtualHosts."loki.home.2rjus.net".extraConfig = ''
+    basic_auth {
+      promtail {env.LOKI_PUSH_HASH}
+    }
+    reverse_proxy http://127.0.0.1:3100
+  '';
+
+  services.loki = {
+    enable = true;
+    configuration = {
+      auth_enabled = false;
+
+      server = {
+        http_listen_address = "127.0.0.1";
+        http_listen_port = 3100;
+      };
+      common = {
+        ring = {
+          instance_addr = "127.0.0.1";
+          kvstore = {
+            store = "inmemory";
+          };
+        };
+        replication_factor = 1;
+        path_prefix = "/var/lib/loki";
+      };
+      schema_config = {
+        configs = [
+          {
+            from = "2024-01-01";
+            store = "tsdb";
+            object_store = "filesystem";
+            schema = "v13";
+            index = {
+              prefix = "loki_index_";
+              period = "24h";
+            };
+          }
+        ];
+      };
+      storage_config = {
+        filesystem = {
+          directory = "/var/lib/loki/chunks";
+        };
+      };
+      compactor = {
+        working_directory = "/var/lib/loki/compactor";
+        compaction_interval = "10m";
+        retention_enabled = true;
+        retention_delete_delay = "2h";
+        retention_delete_worker_count = 150;
+        delete_request_store = "filesystem";
+      };
+      limits_config = {
+        retention_period = "30d";
+        ingestion_rate_mb = 10;
+        ingestion_burst_size_mb = 20;
+        max_streams_per_user = 10000;
+        max_query_series = 500;
+        max_query_parallelism = 8;
+      };
+    };
+  };
+}

services/monitoring/loki.nix

@@ -37,6 +37,22 @@
           directory = "/var/lib/loki/chunks";
         };
       };
+      compactor = {
+        working_directory = "/var/lib/loki/compactor";
+        compaction_interval = "10m";
+        retention_enabled = true;
+        retention_delete_delay = "2h";
+        retention_delete_worker_count = 150;
+        delete_request_store = "filesystem";
+      };
+      limits_config = {
+        retention_period = "30d";
+        ingestion_rate_mb = 10;
+        ingestion_burst_size_mb = 20;
+        max_streams_per_user = 10000;
+        max_query_series = 500;
+        max_query_parallelism = 8;
+      };
     };
   };
 }

services/monitoring/prometheus.nix

@@ -73,6 +73,15 @@ in
     };
   };
 
+  # Fetch apiary bearer token from Vault
+  vault.secrets.prometheus-apiary-token = {
+    secretPath = "hosts/monitoring01/apiary-token";
+    extractKey = "password";
+    owner = "prometheus";
+    group = "prometheus";
+    services = [ "prometheus" ];
+  };
+
   services.prometheus = {
     enable = true;
     # syntax-only check because we use external credential files (e.g., openbao-token)
@@ -233,6 +242,19 @@ in
           credentials_file = "/run/secrets/prometheus/openbao-token";
         };
       }
+      # Apiary external service
+      {
+        job_name = "apiary";
+        scheme = "https";
+        scrape_interval = "60s";
+        static_configs = [{
+          targets = [ "apiary.t-juice.club" ];
+        }];
+        authorization = {
+          type = "Bearer";
+          credentials_file = "/run/secrets/prometheus-apiary-token";
+        };
+      }
     ] ++ autoScrapeConfigs;
 
     pushgateway = {

services/victoriametrics/default.nix

@@ -0,0 +1,219 @@
+{ self, config, lib, pkgs, ... }:
+let
+  monLib = import ../../lib/monitoring.nix { inherit lib; };
+  externalTargets = import ../monitoring/external-targets.nix;
+
+  nodeExporterTargets = monLib.generateNodeExporterTargets self externalTargets;
+  autoScrapeConfigs = monLib.generateScrapeConfigs self externalTargets;
+
+  # Script to fetch AppRole token for VictoriaMetrics to use when scraping OpenBao metrics
+  fetchOpenbaoToken = pkgs.writeShellApplication {
+    name = "fetch-openbao-token-vm";
+    runtimeInputs = [ pkgs.curl pkgs.jq ];
+    text = ''
+      VAULT_ADDR="https://vault01.home.2rjus.net:8200"
+      APPROLE_DIR="/var/lib/vault/approle"
+      OUTPUT_FILE="/run/secrets/victoriametrics/openbao-token"
+
+      # Read AppRole credentials
+      if [ ! -f "$APPROLE_DIR/role-id" ] || [ ! -f "$APPROLE_DIR/secret-id" ]; then
+        echo "AppRole credentials not found at $APPROLE_DIR" >&2
+        exit 1
+      fi
+
+      ROLE_ID=$(cat "$APPROLE_DIR/role-id")
+      SECRET_ID=$(cat "$APPROLE_DIR/secret-id")
+
+      # Authenticate to Vault
+      AUTH_RESPONSE=$(curl -sf -k -X POST \
+        -d "{\"role_id\":\"$ROLE_ID\",\"secret_id\":\"$SECRET_ID\"}" \
+        "$VAULT_ADDR/v1/auth/approle/login")
+
+      # Extract token
+      VAULT_TOKEN=$(echo "$AUTH_RESPONSE" | jq -r '.auth.client_token')
+      if [ -z "$VAULT_TOKEN" ] || [ "$VAULT_TOKEN" = "null" ]; then
+        echo "Failed to extract Vault token from response" >&2
+        exit 1
+      fi
+
+      # Write token to file
+      mkdir -p "$(dirname "$OUTPUT_FILE")"
+      echo -n "$VAULT_TOKEN" > "$OUTPUT_FILE"
+      chown victoriametrics:victoriametrics "$OUTPUT_FILE"
+      chmod 0400 "$OUTPUT_FILE"
+
+      echo "Successfully fetched OpenBao token"
+    '';
+  };
+
+  scrapeConfigs = [
+    # Auto-generated node-exporter targets from flake hosts + external
+    {
+      job_name = "node-exporter";
+      static_configs = nodeExporterTargets;
+    }
+    # Systemd exporter on all hosts (same targets, different port)
+    {
+      job_name = "systemd-exporter";
+      static_configs = map
+        (cfg: cfg // {
+          targets = map (t: builtins.replaceStrings [ ":9100" ] [ ":9558" ] t) cfg.targets;
+        })
+        nodeExporterTargets;
+    }
+    # Local monitoring services
+    {
+      job_name = "victoriametrics";
+      static_configs = [{ targets = [ "localhost:8428" ]; }];
+    }
+    {
+      job_name = "loki";
+      static_configs = [{ targets = [ "localhost:3100" ]; }];
+    }
+    {
+      job_name = "grafana";
+      static_configs = [{ targets = [ "localhost:3000" ]; }];
+    }
+    {
+      job_name = "alertmanager";
+      static_configs = [{ targets = [ "localhost:9093" ]; }];
+    }
+    # Caddy metrics from nix-cache02
+    {
+      job_name = "nix-cache_caddy";
+      scheme = "https";
+      static_configs = [{ targets = [ "nix-cache.home.2rjus.net" ]; }];
+    }
+    # OpenBao metrics with bearer token auth
+    {
+      job_name = "openbao";
+      scheme = "https";
+      metrics_path = "/v1/sys/metrics";
+      params = { format = [ "prometheus" ]; };
+      static_configs = [{ targets = [ "vault01.home.2rjus.net:8200" ]; }];
+      authorization = {
+        type = "Bearer";
+        credentials_file = "/run/secrets/victoriametrics/openbao-token";
+      };
+    }
+    # Apiary external service
+    {
+      job_name = "apiary";
+      scheme = "https";
+      scrape_interval = "60s";
+      static_configs = [{ targets = [ "apiary.t-juice.club" ]; }];
+      authorization = {
+        type = "Bearer";
+        credentials_file = "/run/secrets/victoriametrics-apiary-token";
+      };
+    }
+  ] ++ autoScrapeConfigs;
+in
+{
+  # Static user for VictoriaMetrics (overrides DynamicUser) so vault.secrets
+  # and credential files can be owned by this user
+  users.users.victoriametrics = {
+    isSystemUser = true;
+    group = "victoriametrics";
+  };
+  users.groups.victoriametrics = { };
+
+  # Override DynamicUser since we need a static user for credential file access
+  systemd.services.victoriametrics.serviceConfig = {
+    DynamicUser = lib.mkForce false;
+    User = "victoriametrics";
+    Group = "victoriametrics";
+  };
+
+  # Systemd service to fetch AppRole token for OpenBao scraping
+  systemd.services.victoriametrics-openbao-token = {
+    description = "Fetch OpenBao token for VictoriaMetrics metrics scraping";
+    after = [ "network-online.target" ];
+    wants = [ "network-online.target" ];
+    before = [ "victoriametrics.service" ];
+    requiredBy = [ "victoriametrics.service" ];
+
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = lib.getExe fetchOpenbaoToken;
+    };
+  };
+
+  # Timer to periodically refresh the token (AppRole tokens have 1-hour TTL)
+  systemd.timers.victoriametrics-openbao-token = {
+    description = "Refresh OpenBao token for VictoriaMetrics";
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnBootSec = "5min";
+      OnUnitActiveSec = "30min";
+      RandomizedDelaySec = "5min";
+    };
+  };
+
+  # Fetch apiary bearer token from Vault
+  vault.secrets.victoriametrics-apiary-token = {
+    secretPath = "hosts/monitoring01/apiary-token";
+    extractKey = "password";
+    owner = "victoriametrics";
+    group = "victoriametrics";
+    services = [ "victoriametrics" ];
+  };
+
+  services.victoriametrics = {
+    enable = true;
+    retentionPeriod = "3"; # 3 months
+    # Disable config check since we reference external credential files
+    checkConfig = false;
+    prometheusConfig = {
+      global.scrape_interval = "15s";
+      scrape_configs = scrapeConfigs;
+    };
+  };
+
+  # vmalert for alerting rules - no notifier during parallel operation
+  services.vmalert.instances.default = {
+    enable = true;
+    settings = {
+      "datasource.url" = "http://localhost:8428";
+      # Blackhole notifications during parallel operation to prevent duplicate alerts.
+      # Replace with notifier.url after cutover from monitoring01:
+      # "notifier.url" = [ "http://localhost:9093" ];
+      "notifier.blackhole" = true;
+      "rule" = [ ../monitoring/rules.yml ];
+    };
+  };
+
+  # Caddy reverse proxy for VictoriaMetrics and vmalert
+  services.caddy.virtualHosts."metrics.home.2rjus.net".extraConfig = ''
+    reverse_proxy http://127.0.0.1:8428
+  '';
+  services.caddy.virtualHosts."vmalert.home.2rjus.net".extraConfig = ''
+    reverse_proxy http://127.0.0.1:8880
+  '';
+
+  # Alertmanager - same config as monitoring01 but will only receive
+  # alerts after cutover (vmalert notifier is disabled above)
+  services.prometheus.alertmanager = {
+    enable = true;
+    configuration = {
+      global = { };
+      route = {
+        receiver = "webhook_natstonotify";
+        group_wait = "30s";
+        group_interval = "5m";
+        repeat_interval = "1h";
+        group_by = [ "alertname" ];
+      };
+      receivers = [
+        {
+          name = "webhook_natstonotify";
+          webhook_configs = [
+            {
+              url = "http://localhost:5001/alert";
+            }
+          ];
+        }
+      ];
+    };
+  };
+}

system/monitoring/logs.nix

@@ -1,4 +1,12 @@
-{ config, ... }:
+{ config, lib, ... }:
+let
+  hostLabels = {
+    hostname = config.networking.hostName;
+    tier = config.homelab.host.tier;
+  } // lib.optionalAttrs (config.homelab.host.role != null) {
+    role = config.homelab.host.role;
+  };
+in
 {
   # Configure journald
   services.journald = {
@@ -8,6 +16,16 @@
       SystemKeepFree=1G
     '';
   };
+
+  # Fetch Loki push password from Vault (only on hosts with Vault enabled)
+  vault.secrets.promtail-loki-auth = lib.mkIf config.vault.enable {
+    secretPath = "shared/loki/push-auth";
+    extractKey = "password";
+    owner = "promtail";
+    group = "promtail";
+    services = [ "promtail" ];
+  };
+
   # Configure promtail
   services.promtail = {
     enable = true;
@@ -23,6 +41,14 @@
         {
           url = "http://monitoring01.home.2rjus.net:3100/loki/api/v1/push";
         }
+      ] ++ lib.optionals config.vault.enable [
+        {
+          url = "https://loki.home.2rjus.net/loki/api/v1/push";
+          basic_auth = {
+            username = "promtail";
+            password_file = "/run/secrets/promtail-loki-auth";
+          };
+        }
       ];
 
       scrape_configs = [
@@ -32,17 +58,26 @@
             json = true;
             labels = {
               job = "systemd-journal";
-            };
+            } // hostLabels;
           };
           relabel_configs = [
             {
               source_labels = [ "__journal__systemd_unit" ];
               target_label = "systemd_unit";
             }
+          ];
+          pipeline_stages = [
+            # Extract PRIORITY from journal JSON
+            { json.expressions.priority = "PRIORITY"; }
+            # Map numeric PRIORITY to level name
             {
-              source_labels = [ "__journal__hostname" ];
+              template = {
-              target_label = "host";
+                source = "priority";
+                template = ''{{ if or (eq .Value "0") (eq .Value "1") (eq .Value "2") }}critical{{ else if eq .Value "3" }}error{{ else if eq .Value "4" }}warning{{ else if eq .Value "5" }}notice{{ else if eq .Value "6" }}info{{ else if eq .Value "7" }}debug{{ end }}'';
+              };
             }
+            # Attach as level label
+            { labels.level = "priority"; }
           ];
         }
         {
@@ -53,8 +88,7 @@
               labels = {
                 job = "varlog";
                 __path__ = "/var/log/**/*.log";
-                hostname = "${config.networking.hostName}";
+              } // hostLabels;
-              };
             }
           ];
         }

system/pipe-to-loki.nix

@@ -61,7 +61,7 @@ let
             streams: [{
               stream: {
                 job: $job,
-                host: $host,
+                hostname: $host,
                 type: $type,
                 id: $id
               },

terraform/vault/approle.tf

@@ -26,6 +26,17 @@ path "secret/data/shared/nixos-exporter/*" {
 EOT
 }
 
+# Shared policy for Loki push authentication (all hosts push logs)
+resource "vault_policy" "loki_push" {
+  name = "loki-push"
+
+  policy = <<EOT
+path "secret/data/shared/loki/*" {
+  capabilities = ["read", "list"]
+}
+EOT
+}
+
 # Define host access policies
 locals {
   host_policies = {
@@ -78,7 +89,7 @@ locals {
       ]
     }
 
-    # Wave 3: DNS servers
+    # Wave 3: DNS servers (managed in hosts-generated.tf)
 
     # Wave 4: http-proxy
     "http-proxy" = {
@@ -104,10 +115,11 @@ locals {
       ]
     }
 
-    # monitoring02: Grafana test instance
+    # monitoring02: Grafana + VictoriaMetrics
     "monitoring02" = {
       paths = [
         "secret/data/hosts/monitoring02/*",
+        "secret/data/hosts/monitoring01/apiary-token",
         "secret/data/services/grafana/*",
       ]
     }
@@ -137,7 +149,7 @@ resource "vault_approle_auth_backend_role" "hosts" {
   backend   = vault_auth_backend.approle.path
   role_name = each.key
   token_policies = concat(
-    ["${each.key}-policy", "homelab-deploy", "nixos-exporter"],
+    ["${each.key}-policy", "homelab-deploy", "nixos-exporter", "loki-push"],
     lookup(each.value, "extra_policies", [])
   )
 

terraform/vault/hosts-generated.tf

@@ -39,6 +39,11 @@ locals {
         "secret/data/shared/homelab-deploy/*",
       ]
     }
+    "garage01" = {
+      paths = [
+        "secret/data/hosts/garage01/*",
+      ]
+    }
   
   }
 
@@ -69,7 +74,7 @@ resource "vault_approle_auth_backend_role" "generated_hosts" {
 
   backend            = vault_auth_backend.approle.path
   role_name          = each.key
-  token_policies     = ["host-${each.key}", "homelab-deploy", "nixos-exporter"]
+  token_policies     = ["host-${each.key}", "homelab-deploy", "nixos-exporter", "loki-push"]
   secret_id_ttl      = 0 # Never expire (wrapped tokens provide time limit)
   token_ttl          = 3600
   token_max_ttl      = 3600

terraform/vault/secrets.tf

@@ -107,6 +107,12 @@ locals {
       data          = { nkey = var.homelab_deploy_scheduler_nkey }
     }
 
+    # Garage S3 environment (RPC secret + admin token)
+    "hosts/garage01/garage" = {
+      auto_generate = false
+      data          = { env = var.garage_env }
+    }
+
     # Kanidm idm_admin password
     "kanidm/idm-admin-password" = {
       auto_generate   = true
@@ -141,6 +147,18 @@ locals {
       auto_generate = false
       data          = { api_key = var.sonarr_api_key }
     }
+
+    # Bearer token for scraping apiary metrics
+    "hosts/monitoring01/apiary-token" = {
+      auto_generate   = true
+      password_length = 64
+    }
+
+    # Loki push authentication (used by Promtail on all hosts)
+    "shared/loki/push-auth" = {
+      auto_generate   = true
+      password_length = 32
+    }
   }
 }
 

terraform/vault/variables.tf

@@ -88,6 +88,13 @@ variable "nixos_exporter_nkey" {
   sensitive   = true
 }
 
+variable "garage_env" {
+  description = "Garage environment file contents (GARAGE_RPC_SECRET and GARAGE_ADMIN_TOKEN)"
+  type        = string
+  default     = "PLACEHOLDER"
+  sensitive   = true
+}
+
 variable "radarr_api_key" {
   description = "Radarr API key for exportarr metrics"
   type        = string

terraform/vms.tf

@@ -93,6 +93,13 @@ locals {
       disk_size = "200G"
       vault_wrapped_token = "s.C5EuHFyULACEqZgsLqMC3cJB"
     }
+    "garage01" = {
+      ip        = "10.69.13.26/24"
+      cpu_cores = 2
+      memory    = 2048
+      disk_size = "30G"
+      vault_wrapped_token = "s.dtMKPT35AIrbyEiHf9c2UcsB"
+    }
   }
 
   # Compute VM configurations with defaults applied