feat: MCP tools and nixpkgs indexer

- Add options.json parser with mdDoc support - Add nixpkgs indexer using nix-build - Implement all MCP tool handlers: - search_options: Full-text search with filters - get_option: Option details with children - get_file: Fetch file contents - index_revision: Build and index options - list_revisions: Show indexed versions - delete_revision: Remove indexed data - Add parser tests Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-02-03 17:34:50 +01:00
parent 93245c1439
commit 0b0ada3ccd
4 changed files with 1023 additions and 0 deletions
--- a/internal/nixos/indexer.go
+++ b/internal/nixos/indexer.go
@@ -0,0 +1,380 @@
+package nixos
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"git.t-juice.club/torjus/labmcp/internal/database"
+)
+
+// Indexer handles indexing of nixpkgs revisions.
+type Indexer struct {
+	store      database.Store
+	httpClient *http.Client
+}
+
+// NewIndexer creates a new indexer.
+func NewIndexer(store database.Store) *Indexer {
+	return &Indexer{
+		store: store,
+		httpClient: &http.Client{
+			Timeout: 5 * time.Minute,
+		},
+	}
+}
+
+// IndexResult contains the results of an indexing operation.
+type IndexResult struct {
+	Revision    *database.Revision
+	OptionCount int
+	FileCount   int
+	Duration    time.Duration
+}
+
+// IndexRevision indexes a nixpkgs revision by git hash or channel name.
+func (idx *Indexer) IndexRevision(ctx context.Context, revision string) (*IndexResult, error) {
+	start := time.Now()
+
+	// Resolve channel names to git refs
+	ref := resolveRevision(revision)
+
+	// Check if already indexed
+	existing, err := idx.store.GetRevision(ctx, ref)
+	if err != nil {
+		return nil, fmt.Errorf("failed to check existing revision: %w", err)
+	}
+	if existing != nil {
+		return &IndexResult{
+			Revision:    existing,
+			OptionCount: existing.OptionCount,
+			Duration:    time.Since(start),
+		}, nil
+	}
+
+	// Build options.json using nix
+	optionsPath, cleanup, err := idx.buildOptions(ctx, ref)
+	if err != nil {
+		return nil, fmt.Errorf("failed to build options: %w", err)
+	}
+	defer cleanup()
+
+	// Parse options.json
+	optionsFile, err := os.Open(optionsPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open options.json: %w", err)
+	}
+	defer optionsFile.Close()
+
+	options, err := ParseOptions(optionsFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse options: %w", err)
+	}
+
+	// Get commit info
+	commitDate, err := idx.getCommitDate(ctx, ref)
+	if err != nil {
+		// Non-fatal, use current time
+		commitDate = time.Now()
+	}
+
+	// Create revision record
+	rev := &database.Revision{
+		GitHash:     ref,
+		ChannelName: getChannelName(revision),
+		CommitDate:  commitDate,
+		OptionCount: len(options),
+	}
+	if err := idx.store.CreateRevision(ctx, rev); err != nil {
+		return nil, fmt.Errorf("failed to create revision: %w", err)
+	}
+
+	// Store options
+	if err := idx.storeOptions(ctx, rev.ID, options); err != nil {
+		// Cleanup on failure
+		idx.store.DeleteRevision(ctx, rev.ID)
+		return nil, fmt.Errorf("failed to store options: %w", err)
+	}
+
+	return &IndexResult{
+		Revision:    rev,
+		OptionCount: len(options),
+		Duration:    time.Since(start),
+	}, nil
+}
+
+// buildOptions builds options.json for a nixpkgs revision.
+func (idx *Indexer) buildOptions(ctx context.Context, ref string) (string, func(), error) {
+	// Create temp directory
+	tmpDir, err := os.MkdirTemp("", "nixos-options-*")
+	if err != nil {
+		return "", nil, fmt.Errorf("failed to create temp dir: %w", err)
+	}
+
+	cleanup := func() {
+		os.RemoveAll(tmpDir)
+	}
+
+	// Build options.json using nix-build
+	// This evaluates the NixOS options from the specified nixpkgs revision
+	nixExpr := fmt.Sprintf(`
+		let
+			nixpkgs = builtins.fetchTarball {
+				url = "https://github.com/NixOS/nixpkgs/archive/%s.tar.gz";
+			};
+			pkgs = import nixpkgs { config = {}; };
+			eval = import (nixpkgs + "/nixos/lib/eval-config.nix") {
+				modules = [];
+				system = "x86_64-linux";
+			};
+			opts = (pkgs.nixosOptionsDoc { options = eval.options; }).optionsJSON;
+		in opts
+	`, ref)
+
+	cmd := exec.CommandContext(ctx, "nix-build", "--no-out-link", "-E", nixExpr)
+	cmd.Dir = tmpDir
+
+	output, err := cmd.Output()
+	if err != nil {
+		cleanup()
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			return "", nil, fmt.Errorf("nix-build failed: %s", string(exitErr.Stderr))
+		}
+		return "", nil, fmt.Errorf("nix-build failed: %w", err)
+	}
+
+	// The output is the store path containing share/doc/nixos/options.json
+	storePath := strings.TrimSpace(string(output))
+	optionsPath := filepath.Join(storePath, "share", "doc", "nixos", "options.json")
+
+	if _, err := os.Stat(optionsPath); err != nil {
+		cleanup()
+		return "", nil, fmt.Errorf("options.json not found at %s", optionsPath)
+	}
+
+	return optionsPath, cleanup, nil
+}
+
+// storeOptions stores parsed options in the database.
+func (idx *Indexer) storeOptions(ctx context.Context, revisionID int64, options map[string]*ParsedOption) error {
+	// Prepare batch of options
+	opts := make([]*database.Option, 0, len(options))
+	declsByName := make(map[string][]*database.Declaration)
+
+	for name, opt := range options {
+		dbOpt := &database.Option{
+			RevisionID:   revisionID,
+			Name:         name,
+			ParentPath:   database.ParentPath(name),
+			Type:         opt.Type,
+			DefaultValue: opt.Default,
+			Example:      opt.Example,
+			Description:  opt.Description,
+			ReadOnly:     opt.ReadOnly,
+		}
+		opts = append(opts, dbOpt)
+
+		// Prepare declarations for this option
+		decls := make([]*database.Declaration, 0, len(opt.Declarations))
+		for _, path := range opt.Declarations {
+			decls = append(decls, &database.Declaration{
+				FilePath: path,
+			})
+		}
+		declsByName[name] = decls
+	}
+
+	// Store options in batches
+	batchSize := 1000
+	for i := 0; i < len(opts); i += batchSize {
+		end := i + batchSize
+		if end > len(opts) {
+			end = len(opts)
+		}
+		batch := opts[i:end]
+
+		if err := idx.store.CreateOptionsBatch(ctx, batch); err != nil {
+			return fmt.Errorf("failed to store options batch: %w", err)
+		}
+	}
+
+	// Store declarations
+	for _, opt := range opts {
+		decls := declsByName[opt.Name]
+		for _, decl := range decls {
+			decl.OptionID = opt.ID
+		}
+		if len(decls) > 0 {
+			if err := idx.store.CreateDeclarationsBatch(ctx, decls); err != nil {
+				return fmt.Errorf("failed to store declarations for %s: %w", opt.Name, err)
+			}
+		}
+	}
+
+	return nil
+}
+
+// getCommitDate gets the commit date for a git ref.
+func (idx *Indexer) getCommitDate(ctx context.Context, ref string) (time.Time, error) {
+	// Use GitHub API to get commit info
+	url := fmt.Sprintf("https://api.github.com/repos/NixOS/nixpkgs/commits/%s", ref)
+
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return time.Time{}, err
+	}
+	req.Header.Set("Accept", "application/vnd.github.v3+json")
+
+	resp, err := idx.httpClient.Do(req)
+	if err != nil {
+		return time.Time{}, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return time.Time{}, fmt.Errorf("GitHub API returned %d", resp.StatusCode)
+	}
+
+	var commit struct {
+		Commit struct {
+			Committer struct {
+				Date time.Time `json:"date"`
+			} `json:"committer"`
+		} `json:"commit"`
+	}
+
+	if err := json.NewDecoder(resp.Body).Decode(&commit); err != nil {
+		return time.Time{}, err
+	}
+
+	return commit.Commit.Committer.Date, nil
+}
+
+// resolveRevision resolves a channel name or ref to a git ref.
+func resolveRevision(revision string) string {
+	// Check if it's a known channel alias
+	if ref, ok := ChannelAliases[revision]; ok {
+		return ref
+	}
+	return revision
+}
+
+// getChannelName returns the channel name if the revision matches one.
+func getChannelName(revision string) string {
+	if _, ok := ChannelAliases[revision]; ok {
+		return revision
+	}
+	// Check if the revision is a channel ref value
+	for name, ref := range ChannelAliases {
+		if ref == revision {
+			return name
+		}
+	}
+	return ""
+}
+
+// IndexFiles indexes files from a nixpkgs tarball.
+// This is a separate operation that can be run after IndexRevision.
+func (idx *Indexer) IndexFiles(ctx context.Context, revisionID int64, ref string) (int, error) {
+	// Download nixpkgs tarball
+	url := fmt.Sprintf("https://github.com/NixOS/nixpkgs/archive/%s.tar.gz", ref)
+
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return 0, fmt.Errorf("failed to create request: %w", err)
+	}
+
+	resp, err := idx.httpClient.Do(req)
+	if err != nil {
+		return 0, fmt.Errorf("failed to download tarball: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return 0, fmt.Errorf("download failed with status %d", resp.StatusCode)
+	}
+
+	// Extract and index files
+	gz, err := gzip.NewReader(resp.Body)
+	if err != nil {
+		return 0, fmt.Errorf("failed to create gzip reader: %w", err)
+	}
+	defer gz.Close()
+
+	tr := tar.NewReader(gz)
+	count := 0
+	batch := make([]*database.File, 0, 100)
+
+	for {
+		header, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return count, fmt.Errorf("tar read error: %w", err)
+		}
+
+		// Skip directories
+		if header.Typeflag != tar.TypeReg {
+			continue
+		}
+
+		// Check file extension
+		ext := filepath.Ext(header.Name)
+		if !AllowedExtensions[ext] {
+			continue
+		}
+
+		// Skip very large files (> 1MB)
+		if header.Size > 1024*1024 {
+			continue
+		}
+
+		// Remove the top-level directory (nixpkgs-<hash>/)
+		path := header.Name
+		if idx := strings.Index(path, "/"); idx >= 0 {
+			path = path[idx+1:]
+		}
+
+		// Read content
+		content, err := io.ReadAll(tr)
+		if err != nil {
+			continue
+		}
+
+		file := &database.File{
+			RevisionID: revisionID,
+			FilePath:   path,
+			Extension:  ext,
+			Content:    string(content),
+		}
+		batch = append(batch, file)
+		count++
+
+		// Store in batches
+		if len(batch) >= 100 {
+			if err := idx.store.CreateFilesBatch(ctx, batch); err != nil {
+				return count, fmt.Errorf("failed to store files batch: %w", err)
+			}
+			batch = batch[:0]
+		}
+	}
+
+	// Store remaining files
+	if len(batch) > 0 {
+		if err := idx.store.CreateFilesBatch(ctx, batch); err != nil {
+			return count, fmt.Errorf("failed to store final files batch: %w", err)
+		}
+	}
+
+	return count, nil
+}
--- a/internal/nixos/parser.go
+++ b/internal/nixos/parser.go
@@ -0,0 +1,126 @@
+package nixos
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+)
+
+// ParseOptions parses the options.json file from nixpkgs.
+// The options.json structure is a map from option name to option definition.
+func ParseOptions(r io.Reader) (map[string]*ParsedOption, error) {
+	var raw map[string]json.RawMessage
+	if err := json.NewDecoder(r).Decode(&raw); err != nil {
+		return nil, fmt.Errorf("failed to decode options.json: %w", err)
+	}
+
+	options := make(map[string]*ParsedOption, len(raw))
+	for name, data := range raw {
+		opt, err := parseOption(name, data)
+		if err != nil {
+			// Log but don't fail - some options might have unusual formats
+			continue
+		}
+		options[name] = opt
+	}
+
+	return options, nil
+}
+
+// ParsedOption represents a parsed NixOS option with all its metadata.
+type ParsedOption struct {
+	Name         string
+	Type         string
+	Description  string
+	Default      string // JSON-encoded value
+	Example      string // JSON-encoded value
+	ReadOnly     bool
+	Declarations []string
+}
+
+// optionJSON is the internal structure for parsing options.json entries.
+type optionJSON struct {
+	Declarations []string        `json:"declarations"`
+	Default      json.RawMessage `json:"default,omitempty"`
+	Description  interface{}     `json:"description"` // Can be string or object
+	Example      json.RawMessage `json:"example,omitempty"`
+	ReadOnly     bool            `json:"readOnly"`
+	Type         string          `json:"type"`
+}
+
+// parseOption parses a single option entry.
+func parseOption(name string, data json.RawMessage) (*ParsedOption, error) {
+	var opt optionJSON
+	if err := json.Unmarshal(data, &opt); err != nil {
+		return nil, fmt.Errorf("failed to parse option %s: %w", name, err)
+	}
+
+	// Handle description which can be a string or an object with _type: "mdDoc"
+	description := extractDescription(opt.Description)
+
+	// Convert declarations to relative paths
+	declarations := make([]string, 0, len(opt.Declarations))
+	for _, d := range opt.Declarations {
+		declarations = append(declarations, normalizeDeclarationPath(d))
+	}
+
+	return &ParsedOption{
+		Name:         name,
+		Type:         opt.Type,
+		Description:  description,
+		Default:      string(opt.Default),
+		Example:      string(opt.Example),
+		ReadOnly:     opt.ReadOnly,
+		Declarations: declarations,
+	}, nil
+}
+
+// extractDescription extracts the description string from various formats.
+func extractDescription(desc interface{}) string {
+	switch v := desc.(type) {
+	case string:
+		return v
+	case map[string]interface{}:
+		// Handle mdDoc format: {"_type": "mdDoc", "text": "..."}
+		if text, ok := v["text"].(string); ok {
+			return text
+		}
+		// Try "description" key
+		if text, ok := v["description"].(string); ok {
+			return text
+		}
+	}
+	return ""
+}
+
+// normalizeDeclarationPath converts a full store path to a relative nixpkgs path.
+// Input: "/nix/store/xxx-source/nixos/modules/services/web-servers/nginx/default.nix"
+// Output: "nixos/modules/services/web-servers/nginx/default.nix"
+func normalizeDeclarationPath(path string) string {
+	// Look for common prefixes and strip them
+	markers := []string{
+		"/nixos/",
+		"/pkgs/",
+		"/lib/",
+		"/maintainers/",
+	}
+
+	for _, marker := range markers {
+		if idx := findSubstring(path, marker); idx >= 0 {
+			return path[idx+1:] // +1 to skip the leading /
+		}
+	}
+
+	// If no marker found, return as-is
+	return path
+}
+
+// findSubstring returns the index of the first occurrence of substr in s, or -1.
+func findSubstring(s, substr string) int {
+	for i := 0; i <= len(s)-len(substr); i++ {
+		if s[i:i+len(substr)] == substr {
+			return i
+		}
+	}
+	return -1
+}
--- a/internal/nixos/parser_test.go
+++ b/internal/nixos/parser_test.go
@@ -0,0 +1,154 @@
+package nixos
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestParseOptions(t *testing.T) {
+	// Sample options.json content
+	input := `{
+		"services.nginx.enable": {
+			"declarations": ["/nix/store/xxx-source/nixos/modules/services/web-servers/nginx/default.nix"],
+			"default": false,
+			"description": "Whether to enable Nginx Web Server.",
+			"readOnly": false,
+			"type": "boolean"
+		},
+		"services.nginx.package": {
+			"declarations": ["/nix/store/xxx-source/nixos/modules/services/web-servers/nginx/default.nix"],
+			"default": {},
+			"description": {"_type": "mdDoc", "text": "The nginx package to use."},
+			"readOnly": false,
+			"type": "package"
+		},
+		"services.caddy.enable": {
+			"declarations": ["/nix/store/xxx-source/nixos/modules/services/web-servers/caddy/default.nix"],
+			"default": false,
+			"description": "Enable Caddy web server",
+			"readOnly": false,
+			"type": "boolean"
+		}
+	}`
+
+	options, err := ParseOptions(strings.NewReader(input))
+	if err != nil {
+		t.Fatalf("ParseOptions failed: %v", err)
+	}
+
+	if len(options) != 3 {
+		t.Errorf("Expected 3 options, got %d", len(options))
+	}
+
+	// Check nginx.enable
+	opt := options["services.nginx.enable"]
+	if opt == nil {
+		t.Fatal("Expected services.nginx.enable option")
+	}
+	if opt.Type != "boolean" {
+		t.Errorf("Type = %q, want boolean", opt.Type)
+	}
+	if opt.Description != "Whether to enable Nginx Web Server." {
+		t.Errorf("Description = %q", opt.Description)
+	}
+	if opt.Default != "false" {
+		t.Errorf("Default = %q, want false", opt.Default)
+	}
+	if len(opt.Declarations) != 1 {
+		t.Errorf("Expected 1 declaration, got %d", len(opt.Declarations))
+	}
+	// Check declaration path normalization
+	if !strings.HasPrefix(opt.Declarations[0], "nixos/") {
+		t.Errorf("Declaration path not normalized: %q", opt.Declarations[0])
+	}
+
+	// Check nginx.package (mdDoc description)
+	opt = options["services.nginx.package"]
+	if opt == nil {
+		t.Fatal("Expected services.nginx.package option")
+	}
+	if opt.Description != "The nginx package to use." {
+		t.Errorf("Description = %q (mdDoc not extracted)", opt.Description)
+	}
+}
+
+func TestNormalizeDeclarationPath(t *testing.T) {
+	tests := []struct {
+		input  string
+		expect string
+	}{
+		{
+			"/nix/store/xxx-source/nixos/modules/services/web-servers/nginx/default.nix",
+			"nixos/modules/services/web-servers/nginx/default.nix",
+		},
+		{
+			"/nix/store/xxx/pkgs/top-level/all-packages.nix",
+			"pkgs/top-level/all-packages.nix",
+		},
+		{
+			"/nix/store/abc123/lib/types.nix",
+			"lib/types.nix",
+		},
+		{
+			"relative/path.nix",
+			"relative/path.nix",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.input, func(t *testing.T) {
+			got := normalizeDeclarationPath(tt.input)
+			if got != tt.expect {
+				t.Errorf("normalizeDeclarationPath(%q) = %q, want %q", tt.input, got, tt.expect)
+			}
+		})
+	}
+}
+
+func TestExtractDescription(t *testing.T) {
+	tests := []struct {
+		name   string
+		input  interface{}
+		expect string
+	}{
+		{
+			"string",
+			"Simple description",
+			"Simple description",
+		},
+		{
+			"mdDoc",
+			map[string]interface{}{
+				"_type": "mdDoc",
+				"text":  "Markdown description",
+			},
+			"Markdown description",
+		},
+		{
+			"description key",
+			map[string]interface{}{
+				"description": "Nested description",
+			},
+			"Nested description",
+		},
+		{
+			"nil",
+			nil,
+			"",
+		},
+		{
+			"empty map",
+			map[string]interface{}{},
+			"",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := extractDescription(tt.input)
+			if got != tt.expect {
+				t.Errorf("extractDescription = %q, want %q", got, tt.expect)
+			}
+		})
+	}
+}