Add 3 opt-in Loki tools (query_logs, list_labels, list_label_values) that are registered when LOKI_URL is configured. Includes Loki HTTP client, CLI commands (logs, labels), NixOS module option, formatting, and tests. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
438 lines
11 KiB
Go
438 lines
11 KiB
Go
package monitoring
|
|
|
|
import (
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
const maxRows = 100
|
|
|
|
// formatInstantVector formats instant vector results as a markdown table.
|
|
func formatInstantVector(results []PromInstantVector) string {
|
|
if len(results) == 0 {
|
|
return "No results."
|
|
}
|
|
|
|
// Collect all label keys across results (excluding __name__)
|
|
labelKeys := collectLabelKeys(results)
|
|
|
|
var sb strings.Builder
|
|
|
|
// Header
|
|
sb.WriteString("| ")
|
|
if _, ok := results[0].Metric["__name__"]; ok {
|
|
sb.WriteString("Metric | ")
|
|
}
|
|
for _, key := range labelKeys {
|
|
sb.WriteString(key)
|
|
sb.WriteString(" | ")
|
|
}
|
|
sb.WriteString("Value |\n")
|
|
|
|
// Separator
|
|
sb.WriteString("| ")
|
|
if _, ok := results[0].Metric["__name__"]; ok {
|
|
sb.WriteString("--- | ")
|
|
}
|
|
for range labelKeys {
|
|
sb.WriteString("--- | ")
|
|
}
|
|
sb.WriteString("--- |\n")
|
|
|
|
// Rows
|
|
truncated := false
|
|
for i, r := range results {
|
|
if i >= maxRows {
|
|
truncated = true
|
|
break
|
|
}
|
|
sb.WriteString("| ")
|
|
if _, ok := results[0].Metric["__name__"]; ok {
|
|
sb.WriteString(r.Metric["__name__"])
|
|
sb.WriteString(" | ")
|
|
}
|
|
for _, key := range labelKeys {
|
|
sb.WriteString(r.Metric[key])
|
|
sb.WriteString(" | ")
|
|
}
|
|
// Value is at index 1 of the value tuple
|
|
if len(r.Value) >= 2 {
|
|
if v, ok := r.Value[1].(string); ok {
|
|
sb.WriteString(v)
|
|
}
|
|
}
|
|
sb.WriteString(" |\n")
|
|
}
|
|
|
|
if truncated {
|
|
sb.WriteString(fmt.Sprintf("\n*Showing %d of %d results (truncated)*\n", maxRows, len(results)))
|
|
}
|
|
|
|
return sb.String()
|
|
}
|
|
|
|
// collectLabelKeys returns sorted label keys across all results, excluding __name__.
|
|
func collectLabelKeys(results []PromInstantVector) []string {
|
|
keySet := make(map[string]struct{})
|
|
for _, r := range results {
|
|
for k := range r.Metric {
|
|
if k != "__name__" {
|
|
keySet[k] = struct{}{}
|
|
}
|
|
}
|
|
}
|
|
keys := make([]string, 0, len(keySet))
|
|
for k := range keySet {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
return keys
|
|
}
|
|
|
|
// formatAlerts formats alerts as grouped markdown.
|
|
func formatAlerts(alerts []Alert) string {
|
|
if len(alerts) == 0 {
|
|
return "No alerts found."
|
|
}
|
|
|
|
// Group by alertname
|
|
groups := make(map[string][]Alert)
|
|
var order []string
|
|
for _, a := range alerts {
|
|
name := a.Labels["alertname"]
|
|
if _, exists := groups[name]; !exists {
|
|
order = append(order, name)
|
|
}
|
|
groups[name] = append(groups[name], a)
|
|
}
|
|
|
|
var sb strings.Builder
|
|
sb.WriteString(fmt.Sprintf("**%d alert(s)**\n\n", len(alerts)))
|
|
|
|
for _, name := range order {
|
|
group := groups[name]
|
|
sb.WriteString(fmt.Sprintf("## %s (%d)\n\n", name, len(group)))
|
|
|
|
for i, a := range group {
|
|
if i >= maxRows {
|
|
sb.WriteString(fmt.Sprintf("*... and %d more*\n", len(group)-maxRows))
|
|
break
|
|
}
|
|
|
|
sb.WriteString(fmt.Sprintf("**State:** %s | **Severity:** %s\n", a.Status.State, a.Labels["severity"]))
|
|
|
|
// Labels (excluding alertname and severity)
|
|
var labels []string
|
|
for k, v := range a.Labels {
|
|
if k != "alertname" && k != "severity" {
|
|
labels = append(labels, fmt.Sprintf("%s=%s", k, v))
|
|
}
|
|
}
|
|
sort.Strings(labels)
|
|
if len(labels) > 0 {
|
|
sb.WriteString(fmt.Sprintf("**Labels:** %s\n", strings.Join(labels, ", ")))
|
|
}
|
|
|
|
// Annotations
|
|
for k, v := range a.Annotations {
|
|
sb.WriteString(fmt.Sprintf("**%s:** %s\n", k, v))
|
|
}
|
|
|
|
sb.WriteString(fmt.Sprintf("**Fingerprint:** %s\n", a.Fingerprint))
|
|
sb.WriteString(fmt.Sprintf("**Started:** %s\n", a.StartsAt.Format(time.RFC3339)))
|
|
|
|
if len(a.Status.SilencedBy) > 0 {
|
|
sb.WriteString(fmt.Sprintf("**Silenced by:** %s\n", strings.Join(a.Status.SilencedBy, ", ")))
|
|
}
|
|
if len(a.Status.InhibitedBy) > 0 {
|
|
sb.WriteString(fmt.Sprintf("**Inhibited by:** %s\n", strings.Join(a.Status.InhibitedBy, ", ")))
|
|
}
|
|
|
|
sb.WriteString("\n")
|
|
}
|
|
}
|
|
|
|
return sb.String()
|
|
}
|
|
|
|
// formatTargets formats targets as grouped markdown.
|
|
func formatTargets(targets *PromTargetsData) string {
|
|
if targets == nil || len(targets.ActiveTargets) == 0 {
|
|
return "No active targets."
|
|
}
|
|
|
|
// Group by job
|
|
groups := make(map[string][]PromTarget)
|
|
var order []string
|
|
for _, t := range targets.ActiveTargets {
|
|
job := t.Labels["job"]
|
|
if _, exists := groups[job]; !exists {
|
|
order = append(order, job)
|
|
}
|
|
groups[job] = append(groups[job], t)
|
|
}
|
|
sort.Strings(order)
|
|
|
|
var sb strings.Builder
|
|
sb.WriteString(fmt.Sprintf("**%d active target(s)**\n\n", len(targets.ActiveTargets)))
|
|
|
|
// Count health statuses
|
|
healthCounts := make(map[string]int)
|
|
for _, t := range targets.ActiveTargets {
|
|
healthCounts[t.Health]++
|
|
}
|
|
var healthParts []string
|
|
for h, c := range healthCounts {
|
|
healthParts = append(healthParts, fmt.Sprintf("%s: %d", h, c))
|
|
}
|
|
sort.Strings(healthParts)
|
|
sb.WriteString(fmt.Sprintf("**Health summary:** %s\n\n", strings.Join(healthParts, ", ")))
|
|
|
|
for _, job := range order {
|
|
group := groups[job]
|
|
sb.WriteString(fmt.Sprintf("## %s (%d targets)\n\n", job, len(group)))
|
|
|
|
sb.WriteString("| Instance | Health | Last Scrape | Duration | Error |\n")
|
|
sb.WriteString("| --- | --- | --- | --- | --- |\n")
|
|
|
|
for _, t := range group {
|
|
instance := t.Labels["instance"]
|
|
lastScrape := t.LastScrape.Format("15:04:05")
|
|
duration := fmt.Sprintf("%.3fs", t.LastScrapeDuration)
|
|
lastErr := t.LastError
|
|
if lastErr == "" {
|
|
lastErr = "-"
|
|
}
|
|
sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s | %s |\n",
|
|
instance, t.Health, lastScrape, duration, lastErr))
|
|
}
|
|
sb.WriteString("\n")
|
|
}
|
|
|
|
return sb.String()
|
|
}
|
|
|
|
// formatSilences formats silences as markdown.
|
|
func formatSilences(silences []Silence) string {
|
|
if len(silences) == 0 {
|
|
return "No silences found."
|
|
}
|
|
|
|
var sb strings.Builder
|
|
sb.WriteString(fmt.Sprintf("**%d silence(s)**\n\n", len(silences)))
|
|
|
|
for _, s := range silences {
|
|
state := "unknown"
|
|
if s.Status != nil {
|
|
state = s.Status.State
|
|
}
|
|
sb.WriteString(fmt.Sprintf("## Silence %s [%s]\n\n", s.ID, state))
|
|
|
|
// Matchers
|
|
var matchers []string
|
|
for _, m := range s.Matchers {
|
|
op := "="
|
|
if m.IsRegex {
|
|
op = "=~"
|
|
}
|
|
if m.IsEqual != nil && !*m.IsEqual {
|
|
if m.IsRegex {
|
|
op = "!~"
|
|
} else {
|
|
op = "!="
|
|
}
|
|
}
|
|
matchers = append(matchers, fmt.Sprintf("%s%s%s", m.Name, op, m.Value))
|
|
}
|
|
sb.WriteString(fmt.Sprintf("**Matchers:** %s\n", strings.Join(matchers, ", ")))
|
|
sb.WriteString(fmt.Sprintf("**Created by:** %s\n", s.CreatedBy))
|
|
sb.WriteString(fmt.Sprintf("**Comment:** %s\n", s.Comment))
|
|
sb.WriteString(fmt.Sprintf("**Starts:** %s\n", s.StartsAt.Format(time.RFC3339)))
|
|
sb.WriteString(fmt.Sprintf("**Ends:** %s\n", s.EndsAt.Format(time.RFC3339)))
|
|
sb.WriteString("\n")
|
|
}
|
|
|
|
return sb.String()
|
|
}
|
|
|
|
// formatMetricSearch formats metric search results.
|
|
func formatMetricSearch(names []string, metadata map[string][]PromMetadata) string {
|
|
if len(names) == 0 {
|
|
return "No metrics found matching the search."
|
|
}
|
|
|
|
var sb strings.Builder
|
|
sb.WriteString(fmt.Sprintf("**%d metric(s) found**\n\n", len(names)))
|
|
|
|
sb.WriteString("| Metric | Type | Help |\n")
|
|
sb.WriteString("| --- | --- | --- |\n")
|
|
|
|
truncated := false
|
|
for i, name := range names {
|
|
if i >= maxRows {
|
|
truncated = true
|
|
break
|
|
}
|
|
metaType := ""
|
|
help := ""
|
|
if metas, ok := metadata[name]; ok && len(metas) > 0 {
|
|
metaType = metas[0].Type
|
|
help = metas[0].Help
|
|
if len(help) > 100 {
|
|
help = help[:100] + "..."
|
|
}
|
|
}
|
|
sb.WriteString(fmt.Sprintf("| %s | %s | %s |\n", name, metaType, help))
|
|
}
|
|
|
|
if truncated {
|
|
sb.WriteString(fmt.Sprintf("\n*Showing %d of %d metrics (truncated)*\n", maxRows, len(names)))
|
|
}
|
|
|
|
return sb.String()
|
|
}
|
|
|
|
const maxLabelValues = 100
|
|
const maxLineLength = 500
|
|
|
|
// formatLogStreams formats Loki log query results as grouped markdown.
|
|
func formatLogStreams(data *LokiQueryData) string {
|
|
if data == nil || len(data.Result) == 0 {
|
|
return "No log results."
|
|
}
|
|
|
|
var sb strings.Builder
|
|
|
|
totalEntries := 0
|
|
for _, s := range data.Result {
|
|
totalEntries += len(s.Values)
|
|
}
|
|
sb.WriteString(fmt.Sprintf("**%d stream(s), %d total log entries**\n\n", len(data.Result), totalEntries))
|
|
|
|
for _, stream := range data.Result {
|
|
// Stream labels header
|
|
var labels []string
|
|
for k, v := range stream.Stream {
|
|
labels = append(labels, fmt.Sprintf("%s=%q", k, v))
|
|
}
|
|
sort.Strings(labels)
|
|
sb.WriteString(fmt.Sprintf("## {%s}\n\n", strings.Join(labels, ", ")))
|
|
|
|
if len(stream.Values) == 0 {
|
|
sb.WriteString("No entries.\n\n")
|
|
continue
|
|
}
|
|
|
|
sb.WriteString("| Timestamp | Log Line |\n")
|
|
sb.WriteString("| --- | --- |\n")
|
|
|
|
truncated := false
|
|
for i, entry := range stream.Values {
|
|
if i >= maxRows {
|
|
truncated = true
|
|
break
|
|
}
|
|
|
|
ts := formatNanosecondTimestamp(entry[0])
|
|
line := entry[1]
|
|
if len(line) > maxLineLength {
|
|
line = line[:maxLineLength] + "..."
|
|
}
|
|
// Escape pipe characters in log lines for markdown table
|
|
line = strings.ReplaceAll(line, "|", "\\|")
|
|
// Replace newlines with spaces for table compatibility
|
|
line = strings.ReplaceAll(line, "\n", " ")
|
|
|
|
sb.WriteString(fmt.Sprintf("| %s | %s |\n", ts, line))
|
|
}
|
|
|
|
if truncated {
|
|
sb.WriteString(fmt.Sprintf("\n*Showing %d of %d entries (truncated)*\n", maxRows, len(stream.Values)))
|
|
}
|
|
sb.WriteString("\n")
|
|
}
|
|
|
|
return sb.String()
|
|
}
|
|
|
|
// formatLabels formats a list of label names as a bullet list.
|
|
func formatLabels(labels []string) string {
|
|
if len(labels) == 0 {
|
|
return "No labels found."
|
|
}
|
|
|
|
sort.Strings(labels)
|
|
|
|
var sb strings.Builder
|
|
sb.WriteString(fmt.Sprintf("**%d label(s)**\n\n", len(labels)))
|
|
|
|
for _, label := range labels {
|
|
sb.WriteString(fmt.Sprintf("- `%s`\n", label))
|
|
}
|
|
|
|
return sb.String()
|
|
}
|
|
|
|
// formatLabelValues formats label values as a bullet list.
|
|
func formatLabelValues(label string, values []string) string {
|
|
if len(values) == 0 {
|
|
return fmt.Sprintf("No values found for label '%s'.", label)
|
|
}
|
|
|
|
sort.Strings(values)
|
|
|
|
var sb strings.Builder
|
|
sb.WriteString(fmt.Sprintf("**%d value(s) for label `%s`**\n\n", len(values), label))
|
|
|
|
truncated := false
|
|
for i, v := range values {
|
|
if i >= maxLabelValues {
|
|
truncated = true
|
|
break
|
|
}
|
|
sb.WriteString(fmt.Sprintf("- `%s`\n", v))
|
|
}
|
|
|
|
if truncated {
|
|
sb.WriteString(fmt.Sprintf("\n*Showing %d of %d values (truncated)*\n", maxLabelValues, len(values)))
|
|
}
|
|
|
|
return sb.String()
|
|
}
|
|
|
|
// formatNanosecondTimestamp converts a nanosecond Unix timestamp string to RFC3339.
|
|
func formatNanosecondTimestamp(nsStr string) string {
|
|
var ns int64
|
|
for _, c := range nsStr {
|
|
if c >= '0' && c <= '9' {
|
|
ns = ns*10 + int64(c-'0')
|
|
}
|
|
}
|
|
t := time.Unix(0, ns)
|
|
return t.UTC().Format(time.RFC3339)
|
|
}
|
|
|
|
// formatMetricMetadata formats metadata for a single metric.
|
|
func formatMetricMetadata(name string, metas []PromMetadata) string {
|
|
if len(metas) == 0 {
|
|
return fmt.Sprintf("No metadata found for metric '%s'.", name)
|
|
}
|
|
|
|
var sb strings.Builder
|
|
sb.WriteString(fmt.Sprintf("# %s\n\n", name))
|
|
|
|
for _, m := range metas {
|
|
sb.WriteString(fmt.Sprintf("**Type:** %s\n", m.Type))
|
|
if m.Help != "" {
|
|
sb.WriteString(fmt.Sprintf("**Help:** %s\n", m.Help))
|
|
}
|
|
if m.Unit != "" {
|
|
sb.WriteString(fmt.Sprintf("**Unit:** %s\n", m.Unit))
|
|
}
|
|
}
|
|
|
|
return sb.String()
|
|
}
|