Compare commits

6 Commits

Author SHA1 Message Date
713d1e7584 chore: migrate module path from git.t-juice.club to code.t-juice.club
Gitea to Forgejo host migration — update Go module path and all
import references.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-09 19:37:47 +01:00
2d26de5055 fix(metrics): adjust build duration histogram buckets for better resolution
Add lower buckets (5s, 10s) for cached builds and higher buckets (7200s, 14400s)
for cold builds up to the 4h timeout.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-19 09:03:15 +01:00
e5e8be86ec Merge pull request 'feat(builder): log build failure output as separate lines' (#3) from feat/builder-logging into master
Reviewed-on: #3
2026-02-13 17:35:23 +00:00
3ac5d9777f feat(builder): log build failure output as separate lines
Log each line of build failure output as a separate structured log entry
at WARN level, making output readable and queryable in Loki/Grafana.
Add repo and rev fields to all build-related log entries. Add
truncateOutputLines helper that returns a []string for per-line logging.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-13 18:34:21 +01:00
1a23847d31 fix(builder): separate build output from error to preserve timeout messages
When a build timed out, the timeout error was silently replaced by
truncated stderr output. Split into separate Error and Output fields
on BuildHostResult so the cause (e.g. "build timed out after 30m0s")
is always visible in logs and CLI output.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-12 13:24:04 +01:00
c13914bf5a fix(builder): truncate large error output to prevent log overflow
Build errors from nix can be very large (100k+ chars). This truncates
error output to the first 50 and last 50 lines when it exceeds 100
lines, preventing journal and NATS message overflow.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-02-11 00:42:13 +01:00
16 changed files with 244 additions and 32 deletions

View File

@@ -9,15 +9,15 @@ import (
"syscall" "syscall"
"time" "time"
"git.t-juice.club/torjus/homelab-deploy/internal/builder" "code.t-juice.club/torjus/homelab-deploy/internal/builder"
deploycli "git.t-juice.club/torjus/homelab-deploy/internal/cli" deploycli "code.t-juice.club/torjus/homelab-deploy/internal/cli"
"git.t-juice.club/torjus/homelab-deploy/internal/listener" "code.t-juice.club/torjus/homelab-deploy/internal/listener"
"git.t-juice.club/torjus/homelab-deploy/internal/mcp" "code.t-juice.club/torjus/homelab-deploy/internal/mcp"
"git.t-juice.club/torjus/homelab-deploy/internal/messages" "code.t-juice.club/torjus/homelab-deploy/internal/messages"
"github.com/urfave/cli/v3" "github.com/urfave/cli/v3"
) )
const version = "0.2.0" const version = "0.2.5"
func main() { func main() {
app := &cli.Command{ app := &cli.Command{
@@ -565,6 +565,14 @@ func buildCommand() *cli.Command {
result.FinalResponse.Succeeded, result.FinalResponse.Succeeded,
result.FinalResponse.Failed, result.FinalResponse.Failed,
result.FinalResponse.TotalDurationSeconds) result.FinalResponse.TotalDurationSeconds)
for _, hr := range result.FinalResponse.Results {
if !hr.Success {
fmt.Printf("\n--- %s (error: %s) ---\n", hr.Host, hr.Error)
if hr.Output != "" {
fmt.Println(hr.Output)
}
}
}
} }
if !result.AllSucceeded() { if !result.AllSucceeded() {

2
go.mod
View File

@@ -1,4 +1,4 @@
module git.t-juice.club/torjus/homelab-deploy module code.t-juice.club/torjus/homelab-deploy
go 1.25.5 go 1.25.5

View File

@@ -6,18 +6,48 @@ import (
"log/slog" "log/slog"
"regexp" "regexp"
"sort" "sort"
"strings"
"sync" "sync"
"time" "time"
"git.t-juice.club/torjus/homelab-deploy/internal/messages" "code.t-juice.club/torjus/homelab-deploy/internal/messages"
"git.t-juice.club/torjus/homelab-deploy/internal/metrics" "code.t-juice.club/torjus/homelab-deploy/internal/metrics"
"git.t-juice.club/torjus/homelab-deploy/internal/nats" "code.t-juice.club/torjus/homelab-deploy/internal/nats"
) )
// hostnameRegex validates hostnames from flake output. // hostnameRegex validates hostnames from flake output.
// Allows: alphanumeric, dashes, underscores, dots. // Allows: alphanumeric, dashes, underscores, dots.
var hostnameRegex = regexp.MustCompile(`^[a-zA-Z0-9._-]+$`) var hostnameRegex = regexp.MustCompile(`^[a-zA-Z0-9._-]+$`)
// truncateOutputLines truncates output to the first and last N lines if it exceeds 2*N lines,
// returning the result as a slice of strings.
func truncateOutputLines(output string, keepLines int) []string {
lines := strings.Split(output, "\n")
if len(lines) <= keepLines*2 {
return lines
}
head := lines[:keepLines]
tail := lines[len(lines)-keepLines:]
omitted := len(lines) - keepLines*2
result := make([]string, 0, keepLines*2+1)
result = append(result, head...)
result = append(result, fmt.Sprintf("... (%d lines omitted) ...", omitted))
result = append(result, tail...)
return result
}
// truncateOutput truncates output to the first and last N lines if it exceeds 2*N lines.
func truncateOutput(output string, keepLines int) string {
lines := strings.Split(output, "\n")
if len(lines) <= keepLines*2 {
return output
}
head := lines[:keepLines]
tail := lines[len(lines)-keepLines:]
omitted := len(lines) - keepLines*2
return strings.Join(head, "\n") + fmt.Sprintf("\n\n... (%d lines omitted) ...\n\n", omitted) + strings.Join(tail, "\n")
}
// BuilderConfig holds the configuration for the builder. // BuilderConfig holds the configuration for the builder.
type BuilderConfig struct { type BuilderConfig struct {
NATSUrl string NATSUrl string
@@ -243,6 +273,8 @@ func (b *Builder) handleBuildRequest(subject string, data []byte) {
hostStart := time.Now() hostStart := time.Now()
b.logger.Info("building host", b.logger.Info("building host",
"host", host, "host", host,
"repo", req.Repo,
"rev", branch,
"progress", fmt.Sprintf("%d/%d", i+1, len(hosts)), "progress", fmt.Sprintf("%d/%d", i+1, len(hosts)),
"command", b.executor.BuildCommand(repo.URL, branch, host), "command", b.executor.BuildCommand(repo.URL, branch, host),
) )
@@ -256,22 +288,29 @@ func (b *Builder) handleBuildRequest(subject string, data []byte) {
DurationSeconds: hostDuration, DurationSeconds: hostDuration,
} }
if !result.Success { if !result.Success {
hostResult.Error = result.Stderr if result.Error != nil {
if hostResult.Error == "" && result.Error != nil {
hostResult.Error = result.Error.Error() hostResult.Error = result.Error.Error()
} }
if result.Stderr != "" {
hostResult.Output = truncateOutput(result.Stderr, 50)
}
} }
results = append(results, hostResult) results = append(results, hostResult)
if result.Success { if result.Success {
succeeded++ succeeded++
b.logger.Info("host build succeeded", "host", host, "duration", hostDuration) b.logger.Info("host build succeeded", "host", host, "repo", req.Repo, "rev", branch, "duration", hostDuration)
if b.metrics != nil { if b.metrics != nil {
b.metrics.RecordHostBuildSuccess(req.Repo, host, hostDuration) b.metrics.RecordHostBuildSuccess(req.Repo, host, hostDuration)
} }
} else { } else {
failed++ failed++
b.logger.Error("host build failed", "host", host, "error", hostResult.Error) b.logger.Error("host build failed", "host", host, "repo", req.Repo, "rev", branch, "error", hostResult.Error)
if result.Stderr != "" {
for _, line := range truncateOutputLines(result.Stderr, 50) {
b.logger.Warn("build output", "host", host, "repo", req.Repo, "line", line)
}
}
if b.metrics != nil { if b.metrics != nil {
b.metrics.RecordHostBuildFailure(req.Repo, host, hostDuration) b.metrics.RecordHostBuildFailure(req.Repo, host, hostDuration)
} }

View File

@@ -0,0 +1,164 @@
package builder
import (
"fmt"
"strings"
"testing"
)
func TestTruncateOutput(t *testing.T) {
tests := []struct {
name string
input string
keepLines int
wantLines int
wantOmit bool
}{
{
name: "short output unchanged",
input: "line1\nline2\nline3",
keepLines: 50,
wantLines: 3,
wantOmit: false,
},
{
name: "exactly at threshold unchanged",
input: strings.Join(makeLines(100), "\n"),
keepLines: 50,
wantLines: 100,
wantOmit: false,
},
{
name: "over threshold truncated",
input: strings.Join(makeLines(150), "\n"),
keepLines: 50,
wantLines: 103, // 50 + 1 (empty) + 1 (omitted msg) + 1 (empty) + 50
wantOmit: true,
},
{
name: "large output truncated",
input: strings.Join(makeLines(1000), "\n"),
keepLines: 50,
wantLines: 103,
wantOmit: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := truncateOutput(tt.input, tt.keepLines)
gotLines := strings.Split(got, "\n")
if len(gotLines) != tt.wantLines {
t.Errorf("got %d lines, want %d", len(gotLines), tt.wantLines)
}
hasOmit := strings.Contains(got, "lines omitted")
if hasOmit != tt.wantOmit {
t.Errorf("got omit marker = %v, want %v", hasOmit, tt.wantOmit)
}
if tt.wantOmit {
// Verify first and last lines are preserved
inputLines := strings.Split(tt.input, "\n")
firstLine := inputLines[0]
lastLine := inputLines[len(inputLines)-1]
if !strings.HasPrefix(got, firstLine+"\n") {
t.Errorf("first line not preserved, got prefix %q, want %q",
gotLines[0], firstLine)
}
if !strings.HasSuffix(got, lastLine) {
t.Errorf("last line not preserved, got suffix %q, want %q",
gotLines[len(gotLines)-1], lastLine)
}
}
})
}
}
func makeLines(n int) []string {
lines := make([]string, n)
for i := range lines {
lines[i] = "line " + strings.Repeat("x", i%80)
}
return lines
}
func TestTruncateOutputLines(t *testing.T) {
t.Run("short output returns all lines", func(t *testing.T) {
input := "line1\nline2\nline3"
got := truncateOutputLines(input, 50)
if len(got) != 3 {
t.Errorf("got %d lines, want 3", len(got))
}
if got[0] != "line1" || got[1] != "line2" || got[2] != "line3" {
t.Errorf("unexpected lines: %v", got)
}
})
t.Run("over threshold returns head + marker + tail", func(t *testing.T) {
lines := makeLines(200)
input := strings.Join(lines, "\n")
got := truncateOutputLines(input, 50)
// Should be 50 head + 1 marker + 50 tail = 101
if len(got) != 101 {
t.Errorf("got %d lines, want 101", len(got))
}
// Check first and last lines preserved
if got[0] != lines[0] {
t.Errorf("first line = %q, want %q", got[0], lines[0])
}
if got[len(got)-1] != lines[len(lines)-1] {
t.Errorf("last line = %q, want %q", got[len(got)-1], lines[len(lines)-1])
}
// Check omitted marker
marker := got[50]
expected := fmt.Sprintf("... (%d lines omitted) ...", 100)
if marker != expected {
t.Errorf("marker = %q, want %q", marker, expected)
}
})
t.Run("exactly at threshold returns all lines", func(t *testing.T) {
lines := makeLines(100)
input := strings.Join(lines, "\n")
got := truncateOutputLines(input, 50)
if len(got) != 100 {
t.Errorf("got %d lines, want 100", len(got))
}
})
}
func TestTruncateOutputPreservesContent(t *testing.T) {
// Create input with distinct first and last lines
lines := make([]string, 200)
for i := range lines {
lines[i] = "middle"
}
lines[0] = "FIRST"
lines[49] = "LAST_OF_HEAD"
lines[150] = "FIRST_OF_TAIL"
lines[199] = "LAST"
input := strings.Join(lines, "\n")
got := truncateOutput(input, 50)
if !strings.Contains(got, "FIRST") {
t.Error("missing FIRST")
}
if !strings.Contains(got, "LAST_OF_HEAD") {
t.Error("missing LAST_OF_HEAD")
}
if !strings.Contains(got, "FIRST_OF_TAIL") {
t.Error("missing FIRST_OF_TAIL")
}
if !strings.Contains(got, "LAST") {
t.Error("missing LAST")
}
if !strings.Contains(got, "(100 lines omitted)") {
t.Errorf("wrong omitted count, got: %s", got)
}
}

View File

@@ -9,8 +9,8 @@ import (
"github.com/google/uuid" "github.com/google/uuid"
"git.t-juice.club/torjus/homelab-deploy/internal/messages" "code.t-juice.club/torjus/homelab-deploy/internal/messages"
"git.t-juice.club/torjus/homelab-deploy/internal/nats" "code.t-juice.club/torjus/homelab-deploy/internal/nats"
) )
// BuildConfig holds configuration for a build operation. // BuildConfig holds configuration for a build operation.

View File

@@ -8,8 +8,8 @@ import (
"github.com/google/uuid" "github.com/google/uuid"
"git.t-juice.club/torjus/homelab-deploy/internal/messages" "code.t-juice.club/torjus/homelab-deploy/internal/messages"
"git.t-juice.club/torjus/homelab-deploy/internal/nats" "code.t-juice.club/torjus/homelab-deploy/internal/nats"
) )
// DeployConfig holds configuration for a deploy operation. // DeployConfig holds configuration for a deploy operation.

View File

@@ -3,7 +3,7 @@ package cli
import ( import (
"testing" "testing"
"git.t-juice.club/torjus/homelab-deploy/internal/messages" "code.t-juice.club/torjus/homelab-deploy/internal/messages"
) )
func TestDeployResult_AllSucceeded(t *testing.T) { func TestDeployResult_AllSucceeded(t *testing.T) {

View File

@@ -7,7 +7,7 @@ import (
"os/exec" "os/exec"
"time" "time"
"git.t-juice.club/torjus/homelab-deploy/internal/messages" "code.t-juice.club/torjus/homelab-deploy/internal/messages"
) )
// Executor handles the execution of nixos-rebuild commands. // Executor handles the execution of nixos-rebuild commands.

View File

@@ -4,7 +4,7 @@ import (
"testing" "testing"
"time" "time"
"git.t-juice.club/torjus/homelab-deploy/internal/messages" "code.t-juice.club/torjus/homelab-deploy/internal/messages"
) )
func TestExecutor_BuildCommand(t *testing.T) { func TestExecutor_BuildCommand(t *testing.T) {

View File

@@ -6,10 +6,10 @@ import (
"log/slog" "log/slog"
"time" "time"
"git.t-juice.club/torjus/homelab-deploy/internal/deploy" "code.t-juice.club/torjus/homelab-deploy/internal/deploy"
"git.t-juice.club/torjus/homelab-deploy/internal/messages" "code.t-juice.club/torjus/homelab-deploy/internal/messages"
"git.t-juice.club/torjus/homelab-deploy/internal/metrics" "code.t-juice.club/torjus/homelab-deploy/internal/metrics"
"git.t-juice.club/torjus/homelab-deploy/internal/nats" "code.t-juice.club/torjus/homelab-deploy/internal/nats"
) )
// Config holds the configuration for the listener. // Config holds the configuration for the listener.

View File

@@ -7,8 +7,8 @@ import (
"github.com/mark3labs/mcp-go/mcp" "github.com/mark3labs/mcp-go/mcp"
deploycli "git.t-juice.club/torjus/homelab-deploy/internal/cli" deploycli "code.t-juice.club/torjus/homelab-deploy/internal/cli"
"git.t-juice.club/torjus/homelab-deploy/internal/messages" "code.t-juice.club/torjus/homelab-deploy/internal/messages"
) )
// BuildTool creates the build tool definition. // BuildTool creates the build tool definition.

View File

@@ -9,8 +9,8 @@ import (
"github.com/mark3labs/mcp-go/mcp" "github.com/mark3labs/mcp-go/mcp"
deploycli "git.t-juice.club/torjus/homelab-deploy/internal/cli" deploycli "code.t-juice.club/torjus/homelab-deploy/internal/cli"
"git.t-juice.club/torjus/homelab-deploy/internal/messages" "code.t-juice.club/torjus/homelab-deploy/internal/messages"
) )
// ToolConfig holds configuration for the MCP tools. // ToolConfig holds configuration for the MCP tools.

View File

@@ -82,6 +82,7 @@ type BuildHostResult struct {
Host string `json:"host"` Host string `json:"host"`
Success bool `json:"success"` Success bool `json:"success"`
Error string `json:"error,omitempty"` Error string `json:"error,omitempty"`
Output string `json:"output,omitempty"`
DurationSeconds float64 `json:"duration_seconds"` DurationSeconds float64 `json:"duration_seconds"`
} }

View File

@@ -35,7 +35,7 @@ func NewBuildCollector(reg prometheus.Registerer) *BuildCollector {
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Name: "homelab_deploy_build_duration_seconds", Name: "homelab_deploy_build_duration_seconds",
Help: "Build execution time per host", Help: "Build execution time per host",
Buckets: []float64{30, 60, 120, 300, 600, 900, 1200, 1800, 3600}, Buckets: []float64{5, 10, 30, 60, 120, 300, 600, 1800, 3600, 7200, 14400},
}, },
[]string{"repo", "host"}, []string{"repo", "host"},
), ),

View File

@@ -2,7 +2,7 @@
package metrics package metrics
import ( import (
"git.t-juice.club/torjus/homelab-deploy/internal/messages" "code.t-juice.club/torjus/homelab-deploy/internal/messages"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
) )

View File

@@ -8,7 +8,7 @@ import (
"testing" "testing"
"time" "time"
"git.t-juice.club/torjus/homelab-deploy/internal/messages" "code.t-juice.club/torjus/homelab-deploy/internal/messages"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/client_golang/prometheus/testutil"
) )