feat: add nixpkgs-search binary with package search support
Add a new nixpkgs-search CLI that combines NixOS options search with Nix package search functionality. This provides two MCP servers from a single binary: - `nixpkgs-search options serve` for NixOS options - `nixpkgs-search packages serve` for Nix packages Key changes: - Add packages table to database schema (version 3) - Add Package type and search methods to database layer - Create internal/packages/ with indexer and parser for nix-env JSON - Add MCP server mode (options/packages) with separate tool sets - Add package handlers: search_packages, get_package - Create cmd/nixpkgs-search with combined indexing support - Update flake.nix with nixpkgs-search package (now default) - Bump version to 0.2.0 The index command can index both options and packages together, or use --no-packages/--no-options flags for partial indexing. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -8,12 +8,13 @@ import (
|
||||
|
||||
// Revision represents an indexed nixpkgs revision.
|
||||
type Revision struct {
|
||||
ID int64
|
||||
GitHash string
|
||||
ChannelName string
|
||||
CommitDate time.Time
|
||||
IndexedAt time.Time
|
||||
OptionCount int
|
||||
ID int64
|
||||
GitHash string
|
||||
ChannelName string
|
||||
CommitDate time.Time
|
||||
IndexedAt time.Time
|
||||
OptionCount int
|
||||
PackageCount int
|
||||
}
|
||||
|
||||
// Option represents a NixOS configuration option.
|
||||
@@ -48,6 +49,24 @@ type File struct {
|
||||
LineCount int
|
||||
}
|
||||
|
||||
// Package represents a Nix package from nixpkgs.
|
||||
type Package struct {
|
||||
ID int64
|
||||
RevisionID int64
|
||||
AttrPath string // e.g., "python312Packages.requests"
|
||||
Pname string // Package name
|
||||
Version string
|
||||
Description string
|
||||
LongDescription string
|
||||
Homepage string
|
||||
License string // JSON array
|
||||
Platforms string // JSON array
|
||||
Maintainers string // JSON array
|
||||
Broken bool
|
||||
Unfree bool
|
||||
Insecure bool
|
||||
}
|
||||
|
||||
// DeclarationWithMetadata includes declaration info plus file metadata.
|
||||
type DeclarationWithMetadata struct {
|
||||
Declaration
|
||||
@@ -79,6 +98,15 @@ type SearchFilters struct {
|
||||
Offset int
|
||||
}
|
||||
|
||||
// PackageSearchFilters contains optional filters for package search.
|
||||
type PackageSearchFilters struct {
|
||||
Broken *bool
|
||||
Unfree *bool
|
||||
Insecure *bool
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
// Store defines the interface for database operations.
|
||||
type Store interface {
|
||||
// Schema operations
|
||||
@@ -111,4 +139,11 @@ type Store interface {
|
||||
CreateFilesBatch(ctx context.Context, files []*File) error
|
||||
GetFile(ctx context.Context, revisionID int64, path string) (*File, error)
|
||||
GetFileWithRange(ctx context.Context, revisionID int64, path string, r FileRange) (*FileResult, error)
|
||||
|
||||
// Package operations
|
||||
CreatePackage(ctx context.Context, pkg *Package) error
|
||||
CreatePackagesBatch(ctx context.Context, pkgs []*Package) error
|
||||
GetPackage(ctx context.Context, revisionID int64, attrPath string) (*Package, error)
|
||||
SearchPackages(ctx context.Context, revisionID int64, query string, filters PackageSearchFilters) ([]*Package, error)
|
||||
UpdateRevisionPackageCount(ctx context.Context, id int64, count int) error
|
||||
}
|
||||
|
||||
@@ -43,6 +43,7 @@ func (s *PostgresStore) Initialize(ctx context.Context) error {
|
||||
dropStmts := []string{
|
||||
DropDeclarations,
|
||||
DropOptions,
|
||||
DropPackages,
|
||||
DropFiles,
|
||||
DropRevisions,
|
||||
DropSchemaInfo,
|
||||
@@ -64,7 +65,8 @@ func (s *PostgresStore) Initialize(ctx context.Context) error {
|
||||
channel_name TEXT,
|
||||
commit_date TIMESTAMP,
|
||||
indexed_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
option_count INTEGER NOT NULL DEFAULT 0
|
||||
option_count INTEGER NOT NULL DEFAULT 0,
|
||||
package_count INTEGER NOT NULL DEFAULT 0
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS options (
|
||||
id SERIAL PRIMARY KEY,
|
||||
@@ -92,10 +94,28 @@ func (s *PostgresStore) Initialize(ctx context.Context) error {
|
||||
byte_size INTEGER NOT NULL DEFAULT 0,
|
||||
line_count INTEGER NOT NULL DEFAULT 0
|
||||
)`,
|
||||
`CREATE TABLE IF NOT EXISTS packages (
|
||||
id SERIAL PRIMARY KEY,
|
||||
revision_id INTEGER NOT NULL REFERENCES revisions(id) ON DELETE CASCADE,
|
||||
attr_path TEXT NOT NULL,
|
||||
pname TEXT NOT NULL,
|
||||
version TEXT,
|
||||
description TEXT,
|
||||
long_description TEXT,
|
||||
homepage TEXT,
|
||||
license TEXT,
|
||||
platforms TEXT,
|
||||
maintainers TEXT,
|
||||
broken BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
unfree BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
insecure BOOLEAN NOT NULL DEFAULT FALSE
|
||||
)`,
|
||||
IndexOptionsRevisionName,
|
||||
IndexOptionsRevisionParent,
|
||||
IndexFilesRevisionPath,
|
||||
IndexDeclarationsOption,
|
||||
IndexPackagesRevisionAttr,
|
||||
IndexPackagesRevisionPname,
|
||||
}
|
||||
|
||||
for _, stmt := range createStmts {
|
||||
@@ -104,13 +124,22 @@ func (s *PostgresStore) Initialize(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Create full-text search index for PostgreSQL
|
||||
// Create full-text search index for PostgreSQL options
|
||||
_, err = s.db.ExecContext(ctx, `
|
||||
CREATE INDEX IF NOT EXISTS idx_options_fts
|
||||
ON options USING GIN(to_tsvector('english', name || ' ' || COALESCE(description, '')))
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create FTS index: %w", err)
|
||||
return fmt.Errorf("failed to create options FTS index: %w", err)
|
||||
}
|
||||
|
||||
// Create full-text search index for PostgreSQL packages
|
||||
_, err = s.db.ExecContext(ctx, `
|
||||
CREATE INDEX IF NOT EXISTS idx_packages_fts
|
||||
ON packages USING GIN(to_tsvector('english', attr_path || ' ' || pname || ' ' || COALESCE(description, '')))
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create packages FTS index: %w", err)
|
||||
}
|
||||
|
||||
// Set schema version
|
||||
@@ -133,10 +162,10 @@ func (s *PostgresStore) Close() error {
|
||||
// CreateRevision creates a new revision record.
|
||||
func (s *PostgresStore) CreateRevision(ctx context.Context, rev *Revision) error {
|
||||
err := s.db.QueryRowContext(ctx, `
|
||||
INSERT INTO revisions (git_hash, channel_name, commit_date, option_count)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
INSERT INTO revisions (git_hash, channel_name, commit_date, option_count, package_count)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
RETURNING id, indexed_at`,
|
||||
rev.GitHash, rev.ChannelName, rev.CommitDate, rev.OptionCount,
|
||||
rev.GitHash, rev.ChannelName, rev.CommitDate, rev.OptionCount, rev.PackageCount,
|
||||
).Scan(&rev.ID, &rev.IndexedAt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create revision: %w", err)
|
||||
@@ -148,9 +177,9 @@ func (s *PostgresStore) CreateRevision(ctx context.Context, rev *Revision) error
|
||||
func (s *PostgresStore) GetRevision(ctx context.Context, gitHash string) (*Revision, error) {
|
||||
rev := &Revision{}
|
||||
err := s.db.QueryRowContext(ctx, `
|
||||
SELECT id, git_hash, channel_name, commit_date, indexed_at, option_count
|
||||
SELECT id, git_hash, channel_name, commit_date, indexed_at, option_count, package_count
|
||||
FROM revisions WHERE git_hash = $1`, gitHash,
|
||||
).Scan(&rev.ID, &rev.GitHash, &rev.ChannelName, &rev.CommitDate, &rev.IndexedAt, &rev.OptionCount)
|
||||
).Scan(&rev.ID, &rev.GitHash, &rev.ChannelName, &rev.CommitDate, &rev.IndexedAt, &rev.OptionCount, &rev.PackageCount)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -164,10 +193,10 @@ func (s *PostgresStore) GetRevision(ctx context.Context, gitHash string) (*Revis
|
||||
func (s *PostgresStore) GetRevisionByChannel(ctx context.Context, channel string) (*Revision, error) {
|
||||
rev := &Revision{}
|
||||
err := s.db.QueryRowContext(ctx, `
|
||||
SELECT id, git_hash, channel_name, commit_date, indexed_at, option_count
|
||||
SELECT id, git_hash, channel_name, commit_date, indexed_at, option_count, package_count
|
||||
FROM revisions WHERE channel_name = $1
|
||||
ORDER BY indexed_at DESC LIMIT 1`, channel,
|
||||
).Scan(&rev.ID, &rev.GitHash, &rev.ChannelName, &rev.CommitDate, &rev.IndexedAt, &rev.OptionCount)
|
||||
).Scan(&rev.ID, &rev.GitHash, &rev.ChannelName, &rev.CommitDate, &rev.IndexedAt, &rev.OptionCount, &rev.PackageCount)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -180,7 +209,7 @@ func (s *PostgresStore) GetRevisionByChannel(ctx context.Context, channel string
|
||||
// ListRevisions returns all indexed revisions.
|
||||
func (s *PostgresStore) ListRevisions(ctx context.Context) ([]*Revision, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT id, git_hash, channel_name, commit_date, indexed_at, option_count
|
||||
SELECT id, git_hash, channel_name, commit_date, indexed_at, option_count, package_count
|
||||
FROM revisions ORDER BY indexed_at DESC`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list revisions: %w", err)
|
||||
@@ -190,7 +219,7 @@ func (s *PostgresStore) ListRevisions(ctx context.Context) ([]*Revision, error)
|
||||
var revisions []*Revision
|
||||
for rows.Next() {
|
||||
rev := &Revision{}
|
||||
if err := rows.Scan(&rev.ID, &rev.GitHash, &rev.ChannelName, &rev.CommitDate, &rev.IndexedAt, &rev.OptionCount); err != nil {
|
||||
if err := rows.Scan(&rev.ID, &rev.GitHash, &rev.ChannelName, &rev.CommitDate, &rev.IndexedAt, &rev.OptionCount, &rev.PackageCount); err != nil {
|
||||
return nil, fmt.Errorf("failed to scan revision: %w", err)
|
||||
}
|
||||
revisions = append(revisions, rev)
|
||||
@@ -542,3 +571,126 @@ func (s *PostgresStore) GetFileWithRange(ctx context.Context, revisionID int64,
|
||||
|
||||
return applyLineRange(file, r), nil
|
||||
}
|
||||
|
||||
// CreatePackage creates a new package record.
|
||||
func (s *PostgresStore) CreatePackage(ctx context.Context, pkg *Package) error {
|
||||
err := s.db.QueryRowContext(ctx, `
|
||||
INSERT INTO packages (revision_id, attr_path, pname, version, description, long_description, homepage, license, platforms, maintainers, broken, unfree, insecure)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
|
||||
RETURNING id`,
|
||||
pkg.RevisionID, pkg.AttrPath, pkg.Pname, pkg.Version, pkg.Description, pkg.LongDescription, pkg.Homepage, pkg.License, pkg.Platforms, pkg.Maintainers, pkg.Broken, pkg.Unfree, pkg.Insecure,
|
||||
).Scan(&pkg.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create package: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreatePackagesBatch creates multiple packages in a batch.
|
||||
func (s *PostgresStore) CreatePackagesBatch(ctx context.Context, pkgs []*Package) error {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to begin transaction: %w", err)
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after commit returns error, which is expected
|
||||
|
||||
stmt, err := tx.PrepareContext(ctx, `
|
||||
INSERT INTO packages (revision_id, attr_path, pname, version, description, long_description, homepage, license, platforms, maintainers, broken, unfree, insecure)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
|
||||
RETURNING id`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to prepare statement: %w", err)
|
||||
}
|
||||
defer stmt.Close() //nolint:errcheck // statement closed with transaction
|
||||
|
||||
for _, pkg := range pkgs {
|
||||
err := stmt.QueryRowContext(ctx,
|
||||
pkg.RevisionID, pkg.AttrPath, pkg.Pname, pkg.Version, pkg.Description, pkg.LongDescription, pkg.Homepage, pkg.License, pkg.Platforms, pkg.Maintainers, pkg.Broken, pkg.Unfree, pkg.Insecure,
|
||||
).Scan(&pkg.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to insert package %s: %w", pkg.AttrPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// GetPackage retrieves a package by revision and attr_path.
|
||||
func (s *PostgresStore) GetPackage(ctx context.Context, revisionID int64, attrPath string) (*Package, error) {
|
||||
pkg := &Package{}
|
||||
err := s.db.QueryRowContext(ctx, `
|
||||
SELECT id, revision_id, attr_path, pname, version, description, long_description, homepage, license, platforms, maintainers, broken, unfree, insecure
|
||||
FROM packages WHERE revision_id = $1 AND attr_path = $2`, revisionID, attrPath,
|
||||
).Scan(&pkg.ID, &pkg.RevisionID, &pkg.AttrPath, &pkg.Pname, &pkg.Version, &pkg.Description, &pkg.LongDescription, &pkg.Homepage, &pkg.License, &pkg.Platforms, &pkg.Maintainers, &pkg.Broken, &pkg.Unfree, &pkg.Insecure)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get package: %w", err)
|
||||
}
|
||||
return pkg, nil
|
||||
}
|
||||
|
||||
// SearchPackages searches for packages matching a query.
|
||||
func (s *PostgresStore) SearchPackages(ctx context.Context, revisionID int64, query string, filters PackageSearchFilters) ([]*Package, error) {
|
||||
baseQuery := `
|
||||
SELECT id, revision_id, attr_path, pname, version, description, long_description, homepage, license, platforms, maintainers, broken, unfree, insecure
|
||||
FROM packages
|
||||
WHERE revision_id = $1
|
||||
AND to_tsvector('english', attr_path || ' ' || pname || ' ' || COALESCE(description, '')) @@ plainto_tsquery('english', $2)`
|
||||
args := []interface{}{revisionID, query}
|
||||
argNum := 3
|
||||
|
||||
if filters.Broken != nil {
|
||||
baseQuery += fmt.Sprintf(" AND broken = $%d", argNum)
|
||||
args = append(args, *filters.Broken)
|
||||
argNum++
|
||||
}
|
||||
|
||||
if filters.Unfree != nil {
|
||||
baseQuery += fmt.Sprintf(" AND unfree = $%d", argNum)
|
||||
args = append(args, *filters.Unfree)
|
||||
argNum++
|
||||
}
|
||||
|
||||
if filters.Insecure != nil {
|
||||
baseQuery += fmt.Sprintf(" AND insecure = $%d", argNum)
|
||||
args = append(args, *filters.Insecure)
|
||||
_ = argNum // silence ineffassign - argNum tracks position but final value unused
|
||||
}
|
||||
|
||||
baseQuery += " ORDER BY attr_path"
|
||||
|
||||
if filters.Limit > 0 {
|
||||
baseQuery += fmt.Sprintf(" LIMIT %d", filters.Limit)
|
||||
}
|
||||
if filters.Offset > 0 {
|
||||
baseQuery += fmt.Sprintf(" OFFSET %d", filters.Offset)
|
||||
}
|
||||
|
||||
rows, err := s.db.QueryContext(ctx, baseQuery, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to search packages: %w", err)
|
||||
}
|
||||
defer rows.Close() //nolint:errcheck // rows.Err() checked after iteration
|
||||
|
||||
var packages []*Package
|
||||
for rows.Next() {
|
||||
pkg := &Package{}
|
||||
if err := rows.Scan(&pkg.ID, &pkg.RevisionID, &pkg.AttrPath, &pkg.Pname, &pkg.Version, &pkg.Description, &pkg.LongDescription, &pkg.Homepage, &pkg.License, &pkg.Platforms, &pkg.Maintainers, &pkg.Broken, &pkg.Unfree, &pkg.Insecure); err != nil {
|
||||
return nil, fmt.Errorf("failed to scan package: %w", err)
|
||||
}
|
||||
packages = append(packages, pkg)
|
||||
}
|
||||
return packages, rows.Err()
|
||||
}
|
||||
|
||||
// UpdateRevisionPackageCount updates the package count for a revision.
|
||||
func (s *PostgresStore) UpdateRevisionPackageCount(ctx context.Context, id int64, count int) error {
|
||||
_, err := s.db.ExecContext(ctx,
|
||||
"UPDATE revisions SET package_count = $1 WHERE id = $2", count, id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to update package count: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ package database
|
||||
|
||||
// SchemaVersion is the current database schema version.
|
||||
// When this changes, the database will be dropped and recreated.
|
||||
const SchemaVersion = 2
|
||||
const SchemaVersion = 3
|
||||
|
||||
// Common SQL statements shared between implementations.
|
||||
const (
|
||||
@@ -20,7 +20,8 @@ const (
|
||||
channel_name TEXT,
|
||||
commit_date TIMESTAMP,
|
||||
indexed_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
option_count INTEGER NOT NULL DEFAULT 0
|
||||
option_count INTEGER NOT NULL DEFAULT 0,
|
||||
package_count INTEGER NOT NULL DEFAULT 0
|
||||
)`
|
||||
|
||||
// OptionsTable creates the options table.
|
||||
@@ -57,6 +58,25 @@ const (
|
||||
byte_size INTEGER NOT NULL DEFAULT 0,
|
||||
line_count INTEGER NOT NULL DEFAULT 0
|
||||
)`
|
||||
|
||||
// PackagesTable creates the packages table.
|
||||
PackagesTable = `
|
||||
CREATE TABLE IF NOT EXISTS packages (
|
||||
id INTEGER PRIMARY KEY,
|
||||
revision_id INTEGER NOT NULL REFERENCES revisions(id) ON DELETE CASCADE,
|
||||
attr_path TEXT NOT NULL,
|
||||
pname TEXT NOT NULL,
|
||||
version TEXT,
|
||||
description TEXT,
|
||||
long_description TEXT,
|
||||
homepage TEXT,
|
||||
license TEXT,
|
||||
platforms TEXT,
|
||||
maintainers TEXT,
|
||||
broken BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
unfree BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
insecure BOOLEAN NOT NULL DEFAULT FALSE
|
||||
)`
|
||||
)
|
||||
|
||||
// Index creation statements.
|
||||
@@ -80,6 +100,16 @@ const (
|
||||
IndexDeclarationsOption = `
|
||||
CREATE INDEX IF NOT EXISTS idx_declarations_option
|
||||
ON declarations(option_id)`
|
||||
|
||||
// IndexPackagesRevisionAttr creates an index on packages(revision_id, attr_path).
|
||||
IndexPackagesRevisionAttr = `
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_packages_revision_attr
|
||||
ON packages(revision_id, attr_path)`
|
||||
|
||||
// IndexPackagesRevisionPname creates an index on packages(revision_id, pname).
|
||||
IndexPackagesRevisionPname = `
|
||||
CREATE INDEX IF NOT EXISTS idx_packages_revision_pname
|
||||
ON packages(revision_id, pname)`
|
||||
)
|
||||
|
||||
// Drop statements for schema recreation.
|
||||
@@ -87,6 +117,7 @@ const (
|
||||
DropSchemaInfo = `DROP TABLE IF EXISTS schema_info`
|
||||
DropDeclarations = `DROP TABLE IF EXISTS declarations`
|
||||
DropOptions = `DROP TABLE IF EXISTS options`
|
||||
DropPackages = `DROP TABLE IF EXISTS packages`
|
||||
DropFiles = `DROP TABLE IF EXISTS files`
|
||||
DropRevisions = `DROP TABLE IF EXISTS revisions`
|
||||
)
|
||||
|
||||
@@ -44,10 +44,12 @@ func (s *SQLiteStore) Initialize(ctx context.Context) error {
|
||||
dropStmts := []string{
|
||||
DropDeclarations,
|
||||
DropOptions,
|
||||
DropPackages,
|
||||
DropFiles,
|
||||
DropRevisions,
|
||||
DropSchemaInfo,
|
||||
"DROP TABLE IF EXISTS options_fts",
|
||||
"DROP TABLE IF EXISTS packages_fts",
|
||||
}
|
||||
for _, stmt := range dropStmts {
|
||||
if _, err := s.db.ExecContext(ctx, stmt); err != nil {
|
||||
@@ -63,10 +65,13 @@ func (s *SQLiteStore) Initialize(ctx context.Context) error {
|
||||
OptionsTable,
|
||||
DeclarationsTable,
|
||||
FilesTable,
|
||||
PackagesTable,
|
||||
IndexOptionsRevisionName,
|
||||
IndexOptionsRevisionParent,
|
||||
IndexFilesRevisionPath,
|
||||
IndexDeclarationsOption,
|
||||
IndexPackagesRevisionAttr,
|
||||
IndexPackagesRevisionPname,
|
||||
}
|
||||
|
||||
for _, stmt := range createStmts {
|
||||
@@ -88,8 +93,8 @@ func (s *SQLiteStore) Initialize(ctx context.Context) error {
|
||||
return fmt.Errorf("failed to create FTS table: %w", err)
|
||||
}
|
||||
|
||||
// Create triggers to keep FTS in sync
|
||||
triggers := []string{
|
||||
// Create triggers to keep options FTS in sync
|
||||
optionsTriggers := []string{
|
||||
`CREATE TRIGGER IF NOT EXISTS options_ai AFTER INSERT ON options BEGIN
|
||||
INSERT INTO options_fts(rowid, name, description) VALUES (new.id, new.name, new.description);
|
||||
END`,
|
||||
@@ -101,9 +106,42 @@ func (s *SQLiteStore) Initialize(ctx context.Context) error {
|
||||
INSERT INTO options_fts(rowid, name, description) VALUES (new.id, new.name, new.description);
|
||||
END`,
|
||||
}
|
||||
for _, trigger := range triggers {
|
||||
for _, trigger := range optionsTriggers {
|
||||
if _, err := s.db.ExecContext(ctx, trigger); err != nil {
|
||||
return fmt.Errorf("failed to create trigger: %w", err)
|
||||
return fmt.Errorf("failed to create options trigger: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create FTS5 virtual table for packages full-text search
|
||||
_, err = s.db.ExecContext(ctx, `
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS packages_fts USING fts5(
|
||||
attr_path,
|
||||
pname,
|
||||
description,
|
||||
content='packages',
|
||||
content_rowid='id'
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create packages FTS table: %w", err)
|
||||
}
|
||||
|
||||
// Create triggers to keep packages FTS in sync
|
||||
packagesTriggers := []string{
|
||||
`CREATE TRIGGER IF NOT EXISTS packages_ai AFTER INSERT ON packages BEGIN
|
||||
INSERT INTO packages_fts(rowid, attr_path, pname, description) VALUES (new.id, new.attr_path, new.pname, new.description);
|
||||
END`,
|
||||
`CREATE TRIGGER IF NOT EXISTS packages_ad AFTER DELETE ON packages BEGIN
|
||||
INSERT INTO packages_fts(packages_fts, rowid, attr_path, pname, description) VALUES('delete', old.id, old.attr_path, old.pname, old.description);
|
||||
END`,
|
||||
`CREATE TRIGGER IF NOT EXISTS packages_au AFTER UPDATE ON packages BEGIN
|
||||
INSERT INTO packages_fts(packages_fts, rowid, attr_path, pname, description) VALUES('delete', old.id, old.attr_path, old.pname, old.description);
|
||||
INSERT INTO packages_fts(rowid, attr_path, pname, description) VALUES (new.id, new.attr_path, new.pname, new.description);
|
||||
END`,
|
||||
}
|
||||
for _, trigger := range packagesTriggers {
|
||||
if _, err := s.db.ExecContext(ctx, trigger); err != nil {
|
||||
return fmt.Errorf("failed to create packages trigger: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -127,9 +165,9 @@ func (s *SQLiteStore) Close() error {
|
||||
// CreateRevision creates a new revision record.
|
||||
func (s *SQLiteStore) CreateRevision(ctx context.Context, rev *Revision) error {
|
||||
result, err := s.db.ExecContext(ctx, `
|
||||
INSERT INTO revisions (git_hash, channel_name, commit_date, option_count)
|
||||
VALUES (?, ?, ?, ?)`,
|
||||
rev.GitHash, rev.ChannelName, rev.CommitDate, rev.OptionCount,
|
||||
INSERT INTO revisions (git_hash, channel_name, commit_date, option_count, package_count)
|
||||
VALUES (?, ?, ?, ?, ?)`,
|
||||
rev.GitHash, rev.ChannelName, rev.CommitDate, rev.OptionCount, rev.PackageCount,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create revision: %w", err)
|
||||
@@ -155,9 +193,9 @@ func (s *SQLiteStore) CreateRevision(ctx context.Context, rev *Revision) error {
|
||||
func (s *SQLiteStore) GetRevision(ctx context.Context, gitHash string) (*Revision, error) {
|
||||
rev := &Revision{}
|
||||
err := s.db.QueryRowContext(ctx, `
|
||||
SELECT id, git_hash, channel_name, commit_date, indexed_at, option_count
|
||||
SELECT id, git_hash, channel_name, commit_date, indexed_at, option_count, package_count
|
||||
FROM revisions WHERE git_hash = ?`, gitHash,
|
||||
).Scan(&rev.ID, &rev.GitHash, &rev.ChannelName, &rev.CommitDate, &rev.IndexedAt, &rev.OptionCount)
|
||||
).Scan(&rev.ID, &rev.GitHash, &rev.ChannelName, &rev.CommitDate, &rev.IndexedAt, &rev.OptionCount, &rev.PackageCount)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -171,10 +209,10 @@ func (s *SQLiteStore) GetRevision(ctx context.Context, gitHash string) (*Revisio
|
||||
func (s *SQLiteStore) GetRevisionByChannel(ctx context.Context, channel string) (*Revision, error) {
|
||||
rev := &Revision{}
|
||||
err := s.db.QueryRowContext(ctx, `
|
||||
SELECT id, git_hash, channel_name, commit_date, indexed_at, option_count
|
||||
SELECT id, git_hash, channel_name, commit_date, indexed_at, option_count, package_count
|
||||
FROM revisions WHERE channel_name = ?
|
||||
ORDER BY indexed_at DESC LIMIT 1`, channel,
|
||||
).Scan(&rev.ID, &rev.GitHash, &rev.ChannelName, &rev.CommitDate, &rev.IndexedAt, &rev.OptionCount)
|
||||
).Scan(&rev.ID, &rev.GitHash, &rev.ChannelName, &rev.CommitDate, &rev.IndexedAt, &rev.OptionCount, &rev.PackageCount)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -187,17 +225,17 @@ func (s *SQLiteStore) GetRevisionByChannel(ctx context.Context, channel string)
|
||||
// ListRevisions returns all indexed revisions.
|
||||
func (s *SQLiteStore) ListRevisions(ctx context.Context) ([]*Revision, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT id, git_hash, channel_name, commit_date, indexed_at, option_count
|
||||
SELECT id, git_hash, channel_name, commit_date, indexed_at, option_count, package_count
|
||||
FROM revisions ORDER BY indexed_at DESC`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list revisions: %w", err)
|
||||
}
|
||||
defer rows.Close() //nolint:errcheck // rows.Err() checked after iteration //nolint:errcheck // rows.Err() checked after iteration
|
||||
defer rows.Close() //nolint:errcheck // rows.Err() checked after iteration
|
||||
|
||||
var revisions []*Revision
|
||||
for rows.Next() {
|
||||
rev := &Revision{}
|
||||
if err := rows.Scan(&rev.ID, &rev.GitHash, &rev.ChannelName, &rev.CommitDate, &rev.IndexedAt, &rev.OptionCount); err != nil {
|
||||
if err := rows.Scan(&rev.ID, &rev.GitHash, &rev.ChannelName, &rev.CommitDate, &rev.IndexedAt, &rev.OptionCount, &rev.PackageCount); err != nil {
|
||||
return nil, fmt.Errorf("failed to scan revision: %w", err)
|
||||
}
|
||||
revisions = append(revisions, rev)
|
||||
@@ -588,6 +626,138 @@ func (s *SQLiteStore) GetFileWithRange(ctx context.Context, revisionID int64, pa
|
||||
return applyLineRange(file, r), nil
|
||||
}
|
||||
|
||||
// CreatePackage creates a new package record.
|
||||
func (s *SQLiteStore) CreatePackage(ctx context.Context, pkg *Package) error {
|
||||
result, err := s.db.ExecContext(ctx, `
|
||||
INSERT INTO packages (revision_id, attr_path, pname, version, description, long_description, homepage, license, platforms, maintainers, broken, unfree, insecure)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
pkg.RevisionID, pkg.AttrPath, pkg.Pname, pkg.Version, pkg.Description, pkg.LongDescription, pkg.Homepage, pkg.License, pkg.Platforms, pkg.Maintainers, pkg.Broken, pkg.Unfree, pkg.Insecure,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create package: %w", err)
|
||||
}
|
||||
|
||||
id, err := result.LastInsertId()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get last insert id: %w", err)
|
||||
}
|
||||
pkg.ID = id
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreatePackagesBatch creates multiple packages in a batch.
|
||||
func (s *SQLiteStore) CreatePackagesBatch(ctx context.Context, pkgs []*Package) error {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to begin transaction: %w", err)
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // rollback after commit returns error, which is expected
|
||||
|
||||
stmt, err := tx.PrepareContext(ctx, `
|
||||
INSERT INTO packages (revision_id, attr_path, pname, version, description, long_description, homepage, license, platforms, maintainers, broken, unfree, insecure)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to prepare statement: %w", err)
|
||||
}
|
||||
defer stmt.Close() //nolint:errcheck // statement closed with transaction
|
||||
|
||||
for _, pkg := range pkgs {
|
||||
result, err := stmt.ExecContext(ctx,
|
||||
pkg.RevisionID, pkg.AttrPath, pkg.Pname, pkg.Version, pkg.Description, pkg.LongDescription, pkg.Homepage, pkg.License, pkg.Platforms, pkg.Maintainers, pkg.Broken, pkg.Unfree, pkg.Insecure,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to insert package %s: %w", pkg.AttrPath, err)
|
||||
}
|
||||
id, err := result.LastInsertId()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get last insert id: %w", err)
|
||||
}
|
||||
pkg.ID = id
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// GetPackage retrieves a package by revision and attr_path.
|
||||
func (s *SQLiteStore) GetPackage(ctx context.Context, revisionID int64, attrPath string) (*Package, error) {
|
||||
pkg := &Package{}
|
||||
err := s.db.QueryRowContext(ctx, `
|
||||
SELECT id, revision_id, attr_path, pname, version, description, long_description, homepage, license, platforms, maintainers, broken, unfree, insecure
|
||||
FROM packages WHERE revision_id = ? AND attr_path = ?`, revisionID, attrPath,
|
||||
).Scan(&pkg.ID, &pkg.RevisionID, &pkg.AttrPath, &pkg.Pname, &pkg.Version, &pkg.Description, &pkg.LongDescription, &pkg.Homepage, &pkg.License, &pkg.Platforms, &pkg.Maintainers, &pkg.Broken, &pkg.Unfree, &pkg.Insecure)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get package: %w", err)
|
||||
}
|
||||
return pkg, nil
|
||||
}
|
||||
|
||||
// SearchPackages searches for packages matching a query.
|
||||
func (s *SQLiteStore) SearchPackages(ctx context.Context, revisionID int64, query string, filters PackageSearchFilters) ([]*Package, error) {
|
||||
baseQuery := `
|
||||
SELECT p.id, p.revision_id, p.attr_path, p.pname, p.version, p.description, p.long_description, p.homepage, p.license, p.platforms, p.maintainers, p.broken, p.unfree, p.insecure
|
||||
FROM packages p
|
||||
INNER JOIN packages_fts fts ON p.id = fts.rowid
|
||||
WHERE p.revision_id = ?
|
||||
AND packages_fts MATCH ?`
|
||||
|
||||
// Escape the query for FTS5 by wrapping in double quotes for literal matching.
|
||||
escapedQuery := `"` + strings.ReplaceAll(query, `"`, `""`) + `"`
|
||||
args := []interface{}{revisionID, escapedQuery}
|
||||
|
||||
if filters.Broken != nil {
|
||||
baseQuery += " AND p.broken = ?"
|
||||
args = append(args, *filters.Broken)
|
||||
}
|
||||
|
||||
if filters.Unfree != nil {
|
||||
baseQuery += " AND p.unfree = ?"
|
||||
args = append(args, *filters.Unfree)
|
||||
}
|
||||
|
||||
if filters.Insecure != nil {
|
||||
baseQuery += " AND p.insecure = ?"
|
||||
args = append(args, *filters.Insecure)
|
||||
}
|
||||
|
||||
baseQuery += " ORDER BY p.attr_path"
|
||||
|
||||
if filters.Limit > 0 {
|
||||
baseQuery += fmt.Sprintf(" LIMIT %d", filters.Limit)
|
||||
}
|
||||
if filters.Offset > 0 {
|
||||
baseQuery += fmt.Sprintf(" OFFSET %d", filters.Offset)
|
||||
}
|
||||
|
||||
rows, err := s.db.QueryContext(ctx, baseQuery, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to search packages: %w", err)
|
||||
}
|
||||
defer rows.Close() //nolint:errcheck // rows.Err() checked after iteration
|
||||
|
||||
var packages []*Package
|
||||
for rows.Next() {
|
||||
pkg := &Package{}
|
||||
if err := rows.Scan(&pkg.ID, &pkg.RevisionID, &pkg.AttrPath, &pkg.Pname, &pkg.Version, &pkg.Description, &pkg.LongDescription, &pkg.Homepage, &pkg.License, &pkg.Platforms, &pkg.Maintainers, &pkg.Broken, &pkg.Unfree, &pkg.Insecure); err != nil {
|
||||
return nil, fmt.Errorf("failed to scan package: %w", err)
|
||||
}
|
||||
packages = append(packages, pkg)
|
||||
}
|
||||
return packages, rows.Err()
|
||||
}
|
||||
|
||||
// UpdateRevisionPackageCount updates the package count for a revision.
|
||||
func (s *SQLiteStore) UpdateRevisionPackageCount(ctx context.Context, id int64, count int) error {
|
||||
_, err := s.db.ExecContext(ctx,
|
||||
"UPDATE revisions SET package_count = ? WHERE id = ?", count, id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to update package count: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// countLines counts the number of lines in content.
|
||||
func countLines(content string) int {
|
||||
if content == "" {
|
||||
|
||||
@@ -10,9 +10,10 @@ import (
|
||||
|
||||
"git.t-juice.club/torjus/labmcp/internal/database"
|
||||
"git.t-juice.club/torjus/labmcp/internal/options"
|
||||
"git.t-juice.club/torjus/labmcp/internal/packages"
|
||||
)
|
||||
|
||||
// RegisterHandlers registers all tool handlers on the server.
|
||||
// RegisterHandlers registers all tool handlers on the server for options mode.
|
||||
func (s *Server) RegisterHandlers(indexer options.Indexer) {
|
||||
s.tools["search_options"] = s.handleSearchOptions
|
||||
s.tools["get_option"] = s.handleGetOption
|
||||
@@ -22,6 +23,15 @@ func (s *Server) RegisterHandlers(indexer options.Indexer) {
|
||||
s.tools["delete_revision"] = s.handleDeleteRevision
|
||||
}
|
||||
|
||||
// RegisterPackageHandlers registers all tool handlers on the server for packages mode.
|
||||
func (s *Server) RegisterPackageHandlers(pkgIndexer *packages.Indexer) {
|
||||
s.tools["search_packages"] = s.handleSearchPackages
|
||||
s.tools["get_package"] = s.handleGetPackage
|
||||
s.tools["get_file"] = s.handleGetFile
|
||||
s.tools["list_revisions"] = s.handleListRevisionsWithPackages
|
||||
s.tools["delete_revision"] = s.handleDeleteRevision
|
||||
}
|
||||
|
||||
// handleSearchOptions handles the search_options tool.
|
||||
func (s *Server) handleSearchOptions(ctx context.Context, args map[string]interface{}) (CallToolResult, error) {
|
||||
query, _ := args["query"].(string)
|
||||
@@ -420,3 +430,196 @@ func formatJSON(s string) string {
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// handleSearchPackages handles the search_packages tool.
|
||||
func (s *Server) handleSearchPackages(ctx context.Context, args map[string]interface{}) (CallToolResult, error) {
|
||||
query, _ := args["query"].(string)
|
||||
if query == "" {
|
||||
return ErrorContent(fmt.Errorf("query is required")), nil
|
||||
}
|
||||
|
||||
revision, _ := args["revision"].(string)
|
||||
rev, err := s.resolveRevision(ctx, revision)
|
||||
if err != nil {
|
||||
return ErrorContent(err), nil
|
||||
}
|
||||
if rev == nil {
|
||||
return ErrorContent(fmt.Errorf("no indexed revision available")), nil
|
||||
}
|
||||
|
||||
filters := database.PackageSearchFilters{
|
||||
Limit: 50,
|
||||
}
|
||||
|
||||
if broken, ok := args["broken"].(bool); ok {
|
||||
filters.Broken = &broken
|
||||
}
|
||||
if unfree, ok := args["unfree"].(bool); ok {
|
||||
filters.Unfree = &unfree
|
||||
}
|
||||
if limit, ok := args["limit"].(float64); ok && limit > 0 {
|
||||
filters.Limit = int(limit)
|
||||
}
|
||||
|
||||
pkgs, err := s.store.SearchPackages(ctx, rev.ID, query, filters)
|
||||
if err != nil {
|
||||
return ErrorContent(fmt.Errorf("search failed: %w", err)), nil
|
||||
}
|
||||
|
||||
// Format results
|
||||
var sb strings.Builder
|
||||
sb.WriteString(fmt.Sprintf("Found %d packages matching '%s' in revision %s:\n\n", len(pkgs), query, rev.GitHash[:8]))
|
||||
|
||||
for _, pkg := range pkgs {
|
||||
sb.WriteString(fmt.Sprintf("## %s\n", pkg.AttrPath))
|
||||
sb.WriteString(fmt.Sprintf("**Name:** %s", pkg.Pname))
|
||||
if pkg.Version != "" {
|
||||
sb.WriteString(fmt.Sprintf(" %s", pkg.Version))
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
if pkg.Description != "" {
|
||||
desc := pkg.Description
|
||||
if len(desc) > 200 {
|
||||
desc = desc[:200] + "..."
|
||||
}
|
||||
sb.WriteString(fmt.Sprintf("**Description:** %s\n", desc))
|
||||
}
|
||||
if pkg.Broken || pkg.Unfree || pkg.Insecure {
|
||||
var flags []string
|
||||
if pkg.Broken {
|
||||
flags = append(flags, "broken")
|
||||
}
|
||||
if pkg.Unfree {
|
||||
flags = append(flags, "unfree")
|
||||
}
|
||||
if pkg.Insecure {
|
||||
flags = append(flags, "insecure")
|
||||
}
|
||||
sb.WriteString(fmt.Sprintf("**Flags:** %s\n", strings.Join(flags, ", ")))
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
|
||||
return CallToolResult{
|
||||
Content: []Content{TextContent(sb.String())},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// handleGetPackage handles the get_package tool.
|
||||
func (s *Server) handleGetPackage(ctx context.Context, args map[string]interface{}) (CallToolResult, error) {
|
||||
attrPath, _ := args["attr_path"].(string)
|
||||
if attrPath == "" {
|
||||
return ErrorContent(fmt.Errorf("attr_path is required")), nil
|
||||
}
|
||||
|
||||
revision, _ := args["revision"].(string)
|
||||
rev, err := s.resolveRevision(ctx, revision)
|
||||
if err != nil {
|
||||
return ErrorContent(err), nil
|
||||
}
|
||||
if rev == nil {
|
||||
return ErrorContent(fmt.Errorf("no indexed revision available")), nil
|
||||
}
|
||||
|
||||
pkg, err := s.store.GetPackage(ctx, rev.ID, attrPath)
|
||||
if err != nil {
|
||||
return ErrorContent(fmt.Errorf("failed to get package: %w", err)), nil
|
||||
}
|
||||
if pkg == nil {
|
||||
return ErrorContent(fmt.Errorf("package '%s' not found", attrPath)), nil
|
||||
}
|
||||
|
||||
// Format result
|
||||
var sb strings.Builder
|
||||
sb.WriteString(fmt.Sprintf("# %s\n\n", pkg.AttrPath))
|
||||
sb.WriteString(fmt.Sprintf("**Package name:** %s\n", pkg.Pname))
|
||||
if pkg.Version != "" {
|
||||
sb.WriteString(fmt.Sprintf("**Version:** %s\n", pkg.Version))
|
||||
}
|
||||
|
||||
if pkg.Description != "" {
|
||||
sb.WriteString(fmt.Sprintf("\n**Description:**\n%s\n", pkg.Description))
|
||||
}
|
||||
|
||||
if pkg.LongDescription != "" {
|
||||
sb.WriteString(fmt.Sprintf("\n**Long description:**\n%s\n", pkg.LongDescription))
|
||||
}
|
||||
|
||||
if pkg.Homepage != "" {
|
||||
sb.WriteString(fmt.Sprintf("\n**Homepage:** %s\n", pkg.Homepage))
|
||||
}
|
||||
|
||||
if pkg.License != "" && pkg.License != "[]" {
|
||||
sb.WriteString(fmt.Sprintf("\n**License:** %s\n", formatJSONArray(pkg.License)))
|
||||
}
|
||||
|
||||
if pkg.Maintainers != "" && pkg.Maintainers != "[]" {
|
||||
sb.WriteString(fmt.Sprintf("\n**Maintainers:** %s\n", formatJSONArray(pkg.Maintainers)))
|
||||
}
|
||||
|
||||
if pkg.Platforms != "" && pkg.Platforms != "[]" {
|
||||
sb.WriteString(fmt.Sprintf("\n**Platforms:** %s\n", formatJSONArray(pkg.Platforms)))
|
||||
}
|
||||
|
||||
// Status flags
|
||||
if pkg.Broken || pkg.Unfree || pkg.Insecure {
|
||||
sb.WriteString("\n**Status:**\n")
|
||||
if pkg.Broken {
|
||||
sb.WriteString("- ⚠️ This package is marked as **broken**\n")
|
||||
}
|
||||
if pkg.Unfree {
|
||||
sb.WriteString("- This package has an **unfree** license\n")
|
||||
}
|
||||
if pkg.Insecure {
|
||||
sb.WriteString("- ⚠️ This package is marked as **insecure**\n")
|
||||
}
|
||||
}
|
||||
|
||||
return CallToolResult{
|
||||
Content: []Content{TextContent(sb.String())},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// handleListRevisionsWithPackages handles the list_revisions tool for packages mode.
|
||||
func (s *Server) handleListRevisionsWithPackages(ctx context.Context, args map[string]interface{}) (CallToolResult, error) {
|
||||
revisions, err := s.store.ListRevisions(ctx)
|
||||
if err != nil {
|
||||
return ErrorContent(fmt.Errorf("failed to list revisions: %w", err)), nil
|
||||
}
|
||||
|
||||
if len(revisions) == 0 {
|
||||
return CallToolResult{
|
||||
Content: []Content{TextContent("No revisions indexed. Use the nixpkgs-search CLI to index packages.")},
|
||||
}, nil
|
||||
}
|
||||
|
||||
var sb strings.Builder
|
||||
sb.WriteString(fmt.Sprintf("Indexed revisions (%d):\n\n", len(revisions)))
|
||||
|
||||
for _, rev := range revisions {
|
||||
sb.WriteString(fmt.Sprintf("- **%s**", rev.GitHash[:12]))
|
||||
if rev.ChannelName != "" {
|
||||
sb.WriteString(fmt.Sprintf(" (%s)", rev.ChannelName))
|
||||
}
|
||||
sb.WriteString(fmt.Sprintf("\n Options: %d, Packages: %d, Indexed: %s\n",
|
||||
rev.OptionCount, rev.PackageCount, rev.IndexedAt.Format("2006-01-02 15:04")))
|
||||
}
|
||||
|
||||
return CallToolResult{
|
||||
Content: []Content{TextContent(sb.String())},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// formatJSONArray formats a JSON array string as a comma-separated list.
|
||||
func formatJSONArray(s string) string {
|
||||
if s == "" || s == "[]" {
|
||||
return ""
|
||||
}
|
||||
|
||||
var arr []string
|
||||
if err := json.Unmarshal([]byte(s), &arr); err != nil {
|
||||
return s
|
||||
}
|
||||
|
||||
return strings.Join(arr, ", ")
|
||||
}
|
||||
|
||||
@@ -10,6 +10,16 @@ import (
|
||||
"git.t-juice.club/torjus/labmcp/internal/database"
|
||||
)
|
||||
|
||||
// ServerMode indicates which type of tools the server should expose.
|
||||
type ServerMode string
|
||||
|
||||
const (
|
||||
// ModeOptions exposes only option-related tools.
|
||||
ModeOptions ServerMode = "options"
|
||||
// ModePackages exposes only package-related tools.
|
||||
ModePackages ServerMode = "packages"
|
||||
)
|
||||
|
||||
// ServerConfig contains configuration for the MCP server.
|
||||
type ServerConfig struct {
|
||||
// Name is the server name reported in initialization.
|
||||
@@ -22,15 +32,18 @@ type ServerConfig struct {
|
||||
DefaultChannel string
|
||||
// SourceName is the name of the source repository (e.g., "nixpkgs", "home-manager").
|
||||
SourceName string
|
||||
// Mode specifies which tools to expose (options or packages).
|
||||
Mode ServerMode
|
||||
}
|
||||
|
||||
// DefaultNixOSConfig returns the default configuration for NixOS options server.
|
||||
func DefaultNixOSConfig() ServerConfig {
|
||||
return ServerConfig{
|
||||
Name: "nixos-options",
|
||||
Version: "0.1.2",
|
||||
Version: "0.2.0",
|
||||
DefaultChannel: "nixos-stable",
|
||||
SourceName: "nixpkgs",
|
||||
Mode: ModeOptions,
|
||||
Instructions: `NixOS Options MCP Server - Search and query NixOS configuration options.
|
||||
|
||||
If the current project contains a flake.lock file, you can index the exact nixpkgs revision used by the project:
|
||||
@@ -43,13 +56,32 @@ This ensures option documentation matches the nixpkgs version the project actual
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultNixpkgsPackagesConfig returns the default configuration for nixpkgs packages server.
|
||||
func DefaultNixpkgsPackagesConfig() ServerConfig {
|
||||
return ServerConfig{
|
||||
Name: "nixpkgs-packages",
|
||||
Version: "0.2.0",
|
||||
DefaultChannel: "nixos-stable",
|
||||
SourceName: "nixpkgs",
|
||||
Mode: ModePackages,
|
||||
Instructions: `Nixpkgs Packages MCP Server - Search and query Nix packages from nixpkgs.
|
||||
|
||||
If the current project contains a flake.lock file, you can search packages from the exact nixpkgs revision used by the project:
|
||||
1. Read the flake.lock file to find the nixpkgs "rev" field
|
||||
2. Ensure the revision is indexed (packages are indexed separately from options)
|
||||
|
||||
This ensures package information matches the nixpkgs version the project actually uses.`,
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultHomeManagerConfig returns the default configuration for Home Manager options server.
|
||||
func DefaultHomeManagerConfig() ServerConfig {
|
||||
return ServerConfig{
|
||||
Name: "hm-options",
|
||||
Version: "0.1.2",
|
||||
Version: "0.2.0",
|
||||
DefaultChannel: "hm-stable",
|
||||
SourceName: "home-manager",
|
||||
Mode: ModeOptions,
|
||||
Instructions: `Home Manager Options MCP Server - Search and query Home Manager configuration options.
|
||||
|
||||
If the current project contains a flake.lock file, you can index the exact home-manager revision used by the project:
|
||||
@@ -205,6 +237,17 @@ func (s *Server) handleToolsList(req *Request) *Response {
|
||||
|
||||
// getToolDefinitions returns the tool definitions.
|
||||
func (s *Server) getToolDefinitions() []Tool {
|
||||
// For packages mode, return package tools
|
||||
if s.config.Mode == ModePackages {
|
||||
return s.getPackageToolDefinitions()
|
||||
}
|
||||
|
||||
// Default: options mode
|
||||
return s.getOptionToolDefinitions()
|
||||
}
|
||||
|
||||
// getOptionToolDefinitions returns the tool definitions for options mode.
|
||||
func (s *Server) getOptionToolDefinitions() []Tool {
|
||||
// Determine naming based on source
|
||||
optionType := "NixOS"
|
||||
sourceRepo := "nixpkgs"
|
||||
@@ -344,6 +387,114 @@ func (s *Server) getToolDefinitions() []Tool {
|
||||
}
|
||||
}
|
||||
|
||||
// getPackageToolDefinitions returns the tool definitions for packages mode.
|
||||
func (s *Server) getPackageToolDefinitions() []Tool {
|
||||
exampleChannels := "'nixos-unstable', 'nixos-24.05'"
|
||||
exampleFilePath := "pkgs/applications/networking/browsers/firefox/default.nix"
|
||||
|
||||
return []Tool{
|
||||
{
|
||||
Name: "search_packages",
|
||||
Description: "Search for Nix packages by name or description",
|
||||
InputSchema: InputSchema{
|
||||
Type: "object",
|
||||
Properties: map[string]Property{
|
||||
"query": {
|
||||
Type: "string",
|
||||
Description: "Search query (matches package name, attr path, and description)",
|
||||
},
|
||||
"revision": {
|
||||
Type: "string",
|
||||
Description: fmt.Sprintf("Git hash or channel name (e.g., %s). Uses default if not specified.", exampleChannels),
|
||||
},
|
||||
"broken": {
|
||||
Type: "boolean",
|
||||
Description: "Filter by broken status (true = only broken, false = only working)",
|
||||
},
|
||||
"unfree": {
|
||||
Type: "boolean",
|
||||
Description: "Filter by license (true = only unfree, false = only free)",
|
||||
},
|
||||
"limit": {
|
||||
Type: "integer",
|
||||
Description: "Maximum number of results (default: 50)",
|
||||
Default: 50,
|
||||
},
|
||||
},
|
||||
Required: []string{"query"},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "get_package",
|
||||
Description: "Get full details for a specific Nix package by attribute path",
|
||||
InputSchema: InputSchema{
|
||||
Type: "object",
|
||||
Properties: map[string]Property{
|
||||
"attr_path": {
|
||||
Type: "string",
|
||||
Description: "Package attribute path (e.g., 'firefox', 'python312Packages.requests')",
|
||||
},
|
||||
"revision": {
|
||||
Type: "string",
|
||||
Description: "Git hash or channel name. Uses default if not specified.",
|
||||
},
|
||||
},
|
||||
Required: []string{"attr_path"},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "get_file",
|
||||
Description: "Fetch the contents of a file from nixpkgs",
|
||||
InputSchema: InputSchema{
|
||||
Type: "object",
|
||||
Properties: map[string]Property{
|
||||
"path": {
|
||||
Type: "string",
|
||||
Description: fmt.Sprintf("File path relative to nixpkgs root (e.g., '%s')", exampleFilePath),
|
||||
},
|
||||
"revision": {
|
||||
Type: "string",
|
||||
Description: "Git hash or channel name. Uses default if not specified.",
|
||||
},
|
||||
"offset": {
|
||||
Type: "integer",
|
||||
Description: "Line offset (0-based). Default: 0",
|
||||
Default: 0,
|
||||
},
|
||||
"limit": {
|
||||
Type: "integer",
|
||||
Description: "Maximum lines to return. Default: 250, use 0 for all lines",
|
||||
Default: 250,
|
||||
},
|
||||
},
|
||||
Required: []string{"path"},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "list_revisions",
|
||||
Description: "List all indexed nixpkgs revisions",
|
||||
InputSchema: InputSchema{
|
||||
Type: "object",
|
||||
Properties: map[string]Property{},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "delete_revision",
|
||||
Description: "Delete an indexed revision and all its data",
|
||||
InputSchema: InputSchema{
|
||||
Type: "object",
|
||||
Properties: map[string]Property{
|
||||
"revision": {
|
||||
Type: "string",
|
||||
Description: "Git hash or channel name of the revision to delete",
|
||||
},
|
||||
},
|
||||
Required: []string{"revision"},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// handleToolsCall handles a tool invocation.
|
||||
func (s *Server) handleToolsCall(ctx context.Context, req *Request) *Response {
|
||||
var params CallToolParams
|
||||
|
||||
257
internal/packages/indexer.go
Normal file
257
internal/packages/indexer.go
Normal file
@@ -0,0 +1,257 @@
|
||||
package packages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.t-juice.club/torjus/labmcp/internal/database"
|
||||
)
|
||||
|
||||
// revisionPattern validates revision strings to prevent injection attacks.
|
||||
// Allows: alphanumeric, hyphens, underscores, dots (for channel names like "nixos-24.11"
|
||||
// and git hashes). Must be 1-64 characters.
|
||||
var revisionPattern = regexp.MustCompile(`^[a-zA-Z0-9._-]{1,64}$`)
|
||||
|
||||
// Indexer handles indexing of packages from nixpkgs revisions.
|
||||
type Indexer struct {
|
||||
store database.Store
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// NewIndexer creates a new packages indexer.
|
||||
func NewIndexer(store database.Store) *Indexer {
|
||||
return &Indexer{
|
||||
store: store,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 10 * time.Minute, // Longer timeout for package evaluation
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ValidateRevision checks if a revision string is safe to use.
|
||||
// Returns an error if the revision contains potentially dangerous characters.
|
||||
func ValidateRevision(revision string) error {
|
||||
if !revisionPattern.MatchString(revision) {
|
||||
return fmt.Errorf("invalid revision format: must be 1-64 alphanumeric characters, hyphens, underscores, or dots")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// IndexPackages indexes packages for an existing revision.
|
||||
// The revision must already exist in the database (created by options indexer).
|
||||
func (idx *Indexer) IndexPackages(ctx context.Context, revisionID int64, ref string) (*IndexResult, error) {
|
||||
start := time.Now()
|
||||
|
||||
// Validate revision to prevent injection attacks
|
||||
if err := ValidateRevision(ref); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Build packages JSON using nix-env
|
||||
packagesPath, cleanup, err := idx.buildPackages(ctx, ref)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to build packages: %w", err)
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
// Parse and store packages using streaming to reduce memory usage
|
||||
packagesFile, err := os.Open(packagesPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open packages.json: %w", err)
|
||||
}
|
||||
defer packagesFile.Close() //nolint:errcheck // read-only file
|
||||
|
||||
// Store packages in batches
|
||||
batch := make([]*database.Package, 0, 1000)
|
||||
count := 0
|
||||
|
||||
_, err = ParsePackagesStream(packagesFile, func(pkg *ParsedPackage) error {
|
||||
dbPkg := &database.Package{
|
||||
RevisionID: revisionID,
|
||||
AttrPath: pkg.AttrPath,
|
||||
Pname: pkg.Pname,
|
||||
Version: pkg.Version,
|
||||
Description: pkg.Description,
|
||||
LongDescription: pkg.LongDescription,
|
||||
Homepage: pkg.Homepage,
|
||||
License: pkg.License,
|
||||
Platforms: pkg.Platforms,
|
||||
Maintainers: pkg.Maintainers,
|
||||
Broken: pkg.Broken,
|
||||
Unfree: pkg.Unfree,
|
||||
Insecure: pkg.Insecure,
|
||||
}
|
||||
batch = append(batch, dbPkg)
|
||||
count++
|
||||
|
||||
// Store in batches
|
||||
if len(batch) >= 1000 {
|
||||
if err := idx.store.CreatePackagesBatch(ctx, batch); err != nil {
|
||||
return fmt.Errorf("failed to store packages batch: %w", err)
|
||||
}
|
||||
batch = batch[:0]
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse packages: %w", err)
|
||||
}
|
||||
|
||||
// Store remaining packages
|
||||
if len(batch) > 0 {
|
||||
if err := idx.store.CreatePackagesBatch(ctx, batch); err != nil {
|
||||
return nil, fmt.Errorf("failed to store final packages batch: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Update revision package count
|
||||
if err := idx.store.UpdateRevisionPackageCount(ctx, revisionID, count); err != nil {
|
||||
return nil, fmt.Errorf("failed to update package count: %w", err)
|
||||
}
|
||||
|
||||
return &IndexResult{
|
||||
RevisionID: revisionID,
|
||||
PackageCount: count,
|
||||
Duration: time.Since(start),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// buildPackages builds a JSON file containing all packages for a nixpkgs revision.
|
||||
func (idx *Indexer) buildPackages(ctx context.Context, ref string) (string, func(), error) {
|
||||
// Create temp directory
|
||||
tmpDir, err := os.MkdirTemp("", "nixpkgs-packages-*")
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("failed to create temp dir: %w", err)
|
||||
}
|
||||
|
||||
cleanup := func() {
|
||||
_ = os.RemoveAll(tmpDir) //nolint:errcheck // best-effort temp dir cleanup
|
||||
}
|
||||
|
||||
outputPath := filepath.Join(tmpDir, "packages.json")
|
||||
|
||||
// First, fetch the nixpkgs tarball to the nix store
|
||||
// This ensures it's available for nix-env evaluation
|
||||
nixExpr := fmt.Sprintf(`
|
||||
builtins.fetchTarball {
|
||||
url = "https://github.com/NixOS/nixpkgs/archive/%s.tar.gz";
|
||||
}
|
||||
`, ref)
|
||||
|
||||
fetchCmd := exec.CommandContext(ctx, "nix-instantiate", "--eval", "-E", nixExpr)
|
||||
fetchCmd.Dir = tmpDir
|
||||
fetchOutput, err := fetchCmd.Output()
|
||||
if err != nil {
|
||||
cleanup()
|
||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||
return "", nil, fmt.Errorf("nix-instantiate fetch failed: %s", string(exitErr.Stderr))
|
||||
}
|
||||
return "", nil, fmt.Errorf("nix-instantiate fetch failed: %w", err)
|
||||
}
|
||||
|
||||
// The output is the store path in quotes, e.g., "/nix/store/xxx-source"
|
||||
nixpkgsPath := strings.Trim(strings.TrimSpace(string(fetchOutput)), "\"")
|
||||
|
||||
// Run nix-env to get all packages as JSON
|
||||
// Use --json --meta to get full metadata
|
||||
cmd := exec.CommandContext(ctx, "nix-env",
|
||||
"-f", nixpkgsPath,
|
||||
"-qaP", "--json", "--meta",
|
||||
)
|
||||
cmd.Dir = tmpDir
|
||||
|
||||
// Create output file
|
||||
outputFile, err := os.Create(outputPath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return "", nil, fmt.Errorf("failed to create output file: %w", err)
|
||||
}
|
||||
|
||||
cmd.Stdout = outputFile
|
||||
// Suppress stderr warnings about unfree/broken packages
|
||||
cmd.Stderr = nil
|
||||
|
||||
err = cmd.Run()
|
||||
outputFile.Close() //nolint:errcheck // output file, will check stat below
|
||||
if err != nil {
|
||||
cleanup()
|
||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||
return "", nil, fmt.Errorf("nix-env failed: %s", string(exitErr.Stderr))
|
||||
}
|
||||
return "", nil, fmt.Errorf("nix-env failed: %w", err)
|
||||
}
|
||||
|
||||
// Verify output file exists and has content
|
||||
stat, err := os.Stat(outputPath)
|
||||
if err != nil || stat.Size() == 0 {
|
||||
cleanup()
|
||||
return "", nil, fmt.Errorf("packages.json not found or empty")
|
||||
}
|
||||
|
||||
return outputPath, cleanup, nil
|
||||
}
|
||||
|
||||
// ResolveRevision resolves a channel name or ref to a git ref.
|
||||
func (idx *Indexer) ResolveRevision(revision string) string {
|
||||
if ref, ok := ChannelAliases[revision]; ok {
|
||||
return ref
|
||||
}
|
||||
return revision
|
||||
}
|
||||
|
||||
// GetChannelName returns the channel name if the revision matches one.
|
||||
func (idx *Indexer) GetChannelName(revision string) string {
|
||||
if _, ok := ChannelAliases[revision]; ok {
|
||||
return revision
|
||||
}
|
||||
// Check if the revision is a channel ref value
|
||||
for name, ref := range ChannelAliases {
|
||||
if ref == revision {
|
||||
return name
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// GetCommitDate gets the commit date for a git ref using GitHub API.
|
||||
func (idx *Indexer) GetCommitDate(ctx context.Context, ref string) (time.Time, error) {
|
||||
url := fmt.Sprintf("https://api.github.com/repos/NixOS/nixpkgs/commits/%s", ref)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/vnd.github.v3+json")
|
||||
|
||||
resp, err := idx.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
defer resp.Body.Close() //nolint:errcheck // response body read-only
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return time.Time{}, fmt.Errorf("GitHub API returned %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var commit struct {
|
||||
Commit struct {
|
||||
Committer struct {
|
||||
Date time.Time `json:"date"`
|
||||
} `json:"committer"`
|
||||
} `json:"commit"`
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(&commit); err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
|
||||
return commit.Commit.Committer.Date, nil
|
||||
}
|
||||
82
internal/packages/indexer_test.go
Normal file
82
internal/packages/indexer_test.go
Normal file
@@ -0,0 +1,82 @@
|
||||
package packages
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidateRevision(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
revision string
|
||||
expectErr bool
|
||||
}{
|
||||
{"valid hash", "abc123def456", false},
|
||||
{"valid channel", "nixos-unstable", false},
|
||||
{"valid version channel", "nixos-24.11", false},
|
||||
{"empty", "", true},
|
||||
{"too long", "a" + string(make([]byte, 100)), true},
|
||||
{"shell injection", "$(rm -rf /)", true},
|
||||
{"path traversal", "../../../etc/passwd", true},
|
||||
{"semicolon", "abc;rm -rf /", true},
|
||||
{"backtick", "`whoami`", true},
|
||||
{"space", "abc def", true},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := ValidateRevision(tc.revision)
|
||||
if tc.expectErr && err == nil {
|
||||
t.Error("Expected error, got nil")
|
||||
}
|
||||
if !tc.expectErr && err != nil {
|
||||
t.Errorf("Expected no error, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveRevision(t *testing.T) {
|
||||
idx := &Indexer{}
|
||||
|
||||
tests := []struct {
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{"nixos-unstable", "nixos-unstable"},
|
||||
{"nixos-stable", "nixos-24.11"},
|
||||
{"nixos-24.11", "nixos-24.11"},
|
||||
{"abc123", "abc123"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.input, func(t *testing.T) {
|
||||
result := idx.ResolveRevision(tc.input)
|
||||
if result != tc.expected {
|
||||
t.Errorf("Expected %q, got %q", tc.expected, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetChannelName(t *testing.T) {
|
||||
idx := &Indexer{}
|
||||
|
||||
tests := []struct {
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{"nixos-unstable", "nixos-unstable"},
|
||||
{"nixos-stable", "nixos-stable"},
|
||||
{"nixos-24.11", "nixos-24.11"},
|
||||
{"abc123", ""},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.input, func(t *testing.T) {
|
||||
result := idx.GetChannelName(tc.input)
|
||||
if result != tc.expected {
|
||||
t.Errorf("Expected %q, got %q", tc.expected, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
199
internal/packages/parser.go
Normal file
199
internal/packages/parser.go
Normal file
@@ -0,0 +1,199 @@
|
||||
package packages
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ParsePackages reads and parses a nix-env JSON output file.
|
||||
func ParsePackages(r io.Reader) (map[string]*ParsedPackage, error) {
|
||||
var raw PackagesFile
|
||||
if err := json.NewDecoder(r).Decode(&raw); err != nil {
|
||||
return nil, fmt.Errorf("failed to decode packages JSON: %w", err)
|
||||
}
|
||||
|
||||
packages := make(map[string]*ParsedPackage, len(raw))
|
||||
for attrPath, pkg := range raw {
|
||||
parsed := &ParsedPackage{
|
||||
AttrPath: attrPath,
|
||||
Pname: pkg.Pname,
|
||||
Version: pkg.Version,
|
||||
Description: pkg.Meta.Description,
|
||||
LongDescription: pkg.Meta.LongDescription,
|
||||
Homepage: normalizeHomepage(pkg.Meta.Homepage),
|
||||
License: normalizeLicense(pkg.Meta.License),
|
||||
Platforms: normalizePlatforms(pkg.Meta.Platforms),
|
||||
Maintainers: normalizeMaintainers(pkg.Meta.Maintainers),
|
||||
Broken: pkg.Meta.Broken,
|
||||
Unfree: pkg.Meta.Unfree,
|
||||
Insecure: pkg.Meta.Insecure,
|
||||
}
|
||||
packages[attrPath] = parsed
|
||||
}
|
||||
|
||||
return packages, nil
|
||||
}
|
||||
|
||||
// normalizeHomepage converts homepage to a string.
|
||||
func normalizeHomepage(v interface{}) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
switch hp := v.(type) {
|
||||
case string:
|
||||
return hp
|
||||
case []interface{}:
|
||||
if len(hp) > 0 {
|
||||
if s, ok := hp[0].(string); ok {
|
||||
return s
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// normalizeLicense converts license to a JSON array string.
|
||||
func normalizeLicense(v interface{}) string {
|
||||
if v == nil {
|
||||
return "[]"
|
||||
}
|
||||
|
||||
licenses := make([]string, 0)
|
||||
|
||||
switch l := v.(type) {
|
||||
case string:
|
||||
licenses = append(licenses, l)
|
||||
case map[string]interface{}:
|
||||
// Single license object
|
||||
if spdxID, ok := l["spdxId"].(string); ok {
|
||||
licenses = append(licenses, spdxID)
|
||||
} else if fullName, ok := l["fullName"].(string); ok {
|
||||
licenses = append(licenses, fullName)
|
||||
} else if shortName, ok := l["shortName"].(string); ok {
|
||||
licenses = append(licenses, shortName)
|
||||
}
|
||||
case []interface{}:
|
||||
for _, item := range l {
|
||||
switch li := item.(type) {
|
||||
case string:
|
||||
licenses = append(licenses, li)
|
||||
case map[string]interface{}:
|
||||
if spdxID, ok := li["spdxId"].(string); ok {
|
||||
licenses = append(licenses, spdxID)
|
||||
} else if fullName, ok := li["fullName"].(string); ok {
|
||||
licenses = append(licenses, fullName)
|
||||
} else if shortName, ok := li["shortName"].(string); ok {
|
||||
licenses = append(licenses, shortName)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
data, _ := json.Marshal(licenses)
|
||||
return string(data)
|
||||
}
|
||||
|
||||
// normalizePlatforms converts platforms to a JSON array string.
|
||||
func normalizePlatforms(v []interface{}) string {
|
||||
if v == nil {
|
||||
return "[]"
|
||||
}
|
||||
|
||||
platforms := make([]string, 0, len(v))
|
||||
for _, p := range v {
|
||||
switch pv := p.(type) {
|
||||
case string:
|
||||
platforms = append(platforms, pv)
|
||||
// Skip complex platform specs (objects)
|
||||
}
|
||||
}
|
||||
|
||||
data, _ := json.Marshal(platforms)
|
||||
return string(data)
|
||||
}
|
||||
|
||||
// normalizeMaintainers converts maintainers to a JSON array string.
|
||||
func normalizeMaintainers(maintainers []Maintainer) string {
|
||||
if len(maintainers) == 0 {
|
||||
return "[]"
|
||||
}
|
||||
|
||||
names := make([]string, 0, len(maintainers))
|
||||
for _, m := range maintainers {
|
||||
name := m.Name
|
||||
if name == "" && m.Github != "" {
|
||||
name = "@" + m.Github
|
||||
}
|
||||
if name != "" {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
|
||||
data, _ := json.Marshal(names)
|
||||
return string(data)
|
||||
}
|
||||
|
||||
// ParsePackagesStream parses packages from a reader using streaming to reduce memory usage.
|
||||
// It yields parsed packages through a callback function.
|
||||
func ParsePackagesStream(r io.Reader, callback func(*ParsedPackage) error) (int, error) {
|
||||
dec := json.NewDecoder(r)
|
||||
|
||||
// Read the opening brace
|
||||
t, err := dec.Token()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to read opening token: %w", err)
|
||||
}
|
||||
if delim, ok := t.(json.Delim); !ok || delim != '{' {
|
||||
return 0, fmt.Errorf("expected opening brace, got %v", t)
|
||||
}
|
||||
|
||||
count := 0
|
||||
for dec.More() {
|
||||
// Read the key (attr path)
|
||||
t, err := dec.Token()
|
||||
if err != nil {
|
||||
return count, fmt.Errorf("failed to read attr path: %w", err)
|
||||
}
|
||||
attrPath, ok := t.(string)
|
||||
if !ok {
|
||||
return count, fmt.Errorf("expected string key, got %T", t)
|
||||
}
|
||||
|
||||
// Read the value (package)
|
||||
var pkg RawPackage
|
||||
if err := dec.Decode(&pkg); err != nil {
|
||||
// Skip malformed packages
|
||||
continue
|
||||
}
|
||||
|
||||
parsed := &ParsedPackage{
|
||||
AttrPath: attrPath,
|
||||
Pname: pkg.Pname,
|
||||
Version: pkg.Version,
|
||||
Description: pkg.Meta.Description,
|
||||
LongDescription: pkg.Meta.LongDescription,
|
||||
Homepage: normalizeHomepage(pkg.Meta.Homepage),
|
||||
License: normalizeLicense(pkg.Meta.License),
|
||||
Platforms: normalizePlatforms(pkg.Meta.Platforms),
|
||||
Maintainers: normalizeMaintainers(pkg.Meta.Maintainers),
|
||||
Broken: pkg.Meta.Broken,
|
||||
Unfree: pkg.Meta.Unfree,
|
||||
Insecure: pkg.Meta.Insecure,
|
||||
}
|
||||
|
||||
if err := callback(parsed); err != nil {
|
||||
return count, fmt.Errorf("callback error for %s: %w", attrPath, err)
|
||||
}
|
||||
count++
|
||||
}
|
||||
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// SplitAttrPath splits an attribute path into its components.
|
||||
// For example, "python312Packages.requests" returns ["python312Packages", "requests"].
|
||||
func SplitAttrPath(attrPath string) []string {
|
||||
return strings.Split(attrPath, ".")
|
||||
}
|
||||
215
internal/packages/parser_test.go
Normal file
215
internal/packages/parser_test.go
Normal file
@@ -0,0 +1,215 @@
|
||||
package packages
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParsePackages(t *testing.T) {
|
||||
input := `{
|
||||
"firefox": {
|
||||
"name": "firefox-120.0",
|
||||
"pname": "firefox",
|
||||
"version": "120.0",
|
||||
"system": "x86_64-linux",
|
||||
"meta": {
|
||||
"description": "A web browser built from Firefox source tree",
|
||||
"homepage": "https://www.mozilla.org/firefox/",
|
||||
"license": {"spdxId": "MPL-2.0", "fullName": "Mozilla Public License 2.0"},
|
||||
"maintainers": [
|
||||
{"name": "John Doe", "github": "johndoe", "githubId": 12345}
|
||||
],
|
||||
"platforms": ["x86_64-linux", "aarch64-linux"]
|
||||
}
|
||||
},
|
||||
"python312Packages.requests": {
|
||||
"name": "python3.12-requests-2.31.0",
|
||||
"pname": "requests",
|
||||
"version": "2.31.0",
|
||||
"system": "x86_64-linux",
|
||||
"meta": {
|
||||
"description": "HTTP library for Python",
|
||||
"homepage": ["https://requests.readthedocs.io/"],
|
||||
"license": [{"spdxId": "Apache-2.0"}],
|
||||
"unfree": false
|
||||
}
|
||||
}
|
||||
}`
|
||||
|
||||
packages, err := ParsePackages(strings.NewReader(input))
|
||||
if err != nil {
|
||||
t.Fatalf("ParsePackages failed: %v", err)
|
||||
}
|
||||
|
||||
if len(packages) != 2 {
|
||||
t.Errorf("Expected 2 packages, got %d", len(packages))
|
||||
}
|
||||
|
||||
// Check firefox
|
||||
firefox, ok := packages["firefox"]
|
||||
if !ok {
|
||||
t.Fatal("firefox package not found")
|
||||
}
|
||||
if firefox.Pname != "firefox" {
|
||||
t.Errorf("Expected pname 'firefox', got %q", firefox.Pname)
|
||||
}
|
||||
if firefox.Version != "120.0" {
|
||||
t.Errorf("Expected version '120.0', got %q", firefox.Version)
|
||||
}
|
||||
if firefox.Homepage != "https://www.mozilla.org/firefox/" {
|
||||
t.Errorf("Expected homepage 'https://www.mozilla.org/firefox/', got %q", firefox.Homepage)
|
||||
}
|
||||
if firefox.License != `["MPL-2.0"]` {
|
||||
t.Errorf("Expected license '[\"MPL-2.0\"]', got %q", firefox.License)
|
||||
}
|
||||
|
||||
// Check python requests
|
||||
requests, ok := packages["python312Packages.requests"]
|
||||
if !ok {
|
||||
t.Fatal("python312Packages.requests package not found")
|
||||
}
|
||||
if requests.Pname != "requests" {
|
||||
t.Errorf("Expected pname 'requests', got %q", requests.Pname)
|
||||
}
|
||||
// Homepage is array, should extract first element
|
||||
if requests.Homepage != "https://requests.readthedocs.io/" {
|
||||
t.Errorf("Expected homepage 'https://requests.readthedocs.io/', got %q", requests.Homepage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePackagesStream(t *testing.T) {
|
||||
input := `{
|
||||
"hello": {
|
||||
"name": "hello-2.12",
|
||||
"pname": "hello",
|
||||
"version": "2.12",
|
||||
"system": "x86_64-linux",
|
||||
"meta": {
|
||||
"description": "A program that produces a familiar, friendly greeting"
|
||||
}
|
||||
},
|
||||
"world": {
|
||||
"name": "world-1.0",
|
||||
"pname": "world",
|
||||
"version": "1.0",
|
||||
"system": "x86_64-linux",
|
||||
"meta": {}
|
||||
}
|
||||
}`
|
||||
|
||||
var packages []*ParsedPackage
|
||||
count, err := ParsePackagesStream(strings.NewReader(input), func(pkg *ParsedPackage) error {
|
||||
packages = append(packages, pkg)
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("ParsePackagesStream failed: %v", err)
|
||||
}
|
||||
|
||||
if count != 2 {
|
||||
t.Errorf("Expected count 2, got %d", count)
|
||||
}
|
||||
|
||||
if len(packages) != 2 {
|
||||
t.Errorf("Expected 2 packages, got %d", len(packages))
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeLicense(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input interface{}
|
||||
expected string
|
||||
}{
|
||||
{"nil", nil, "[]"},
|
||||
{"string", "MIT", `["MIT"]`},
|
||||
{"object with spdxId", map[string]interface{}{"spdxId": "MIT"}, `["MIT"]`},
|
||||
{"object with fullName", map[string]interface{}{"fullName": "MIT License"}, `["MIT License"]`},
|
||||
{"array of strings", []interface{}{"MIT", "Apache-2.0"}, `["MIT","Apache-2.0"]`},
|
||||
{"array of objects", []interface{}{
|
||||
map[string]interface{}{"spdxId": "MIT"},
|
||||
map[string]interface{}{"spdxId": "Apache-2.0"},
|
||||
}, `["MIT","Apache-2.0"]`},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result := normalizeLicense(tc.input)
|
||||
if result != tc.expected {
|
||||
t.Errorf("Expected %q, got %q", tc.expected, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeHomepage(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input interface{}
|
||||
expected string
|
||||
}{
|
||||
{"nil", nil, ""},
|
||||
{"string", "https://example.com", "https://example.com"},
|
||||
{"array", []interface{}{"https://example.com", "https://docs.example.com"}, "https://example.com"},
|
||||
{"empty array", []interface{}{}, ""},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result := normalizeHomepage(tc.input)
|
||||
if result != tc.expected {
|
||||
t.Errorf("Expected %q, got %q", tc.expected, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMaintainers(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
maintainers []Maintainer
|
||||
expected string
|
||||
}{
|
||||
{"empty", nil, "[]"},
|
||||
{"with name", []Maintainer{{Name: "John Doe"}}, `["John Doe"]`},
|
||||
{"with github only", []Maintainer{{Github: "johndoe"}}, `["@johndoe"]`},
|
||||
{"multiple", []Maintainer{{Name: "Alice"}, {Name: "Bob"}}, `["Alice","Bob"]`},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result := normalizeMaintainers(tc.maintainers)
|
||||
if result != tc.expected {
|
||||
t.Errorf("Expected %q, got %q", tc.expected, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitAttrPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expected []string
|
||||
}{
|
||||
{"firefox", []string{"firefox"}},
|
||||
{"python312Packages.requests", []string{"python312Packages", "requests"}},
|
||||
{"haskellPackages.aeson.components.library", []string{"haskellPackages", "aeson", "components", "library"}},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.input, func(t *testing.T) {
|
||||
result := SplitAttrPath(tc.input)
|
||||
if len(result) != len(tc.expected) {
|
||||
t.Errorf("Expected %v, got %v", tc.expected, result)
|
||||
return
|
||||
}
|
||||
for i := range result {
|
||||
if result[i] != tc.expected[i] {
|
||||
t.Errorf("Expected %v, got %v", tc.expected, result)
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
78
internal/packages/types.go
Normal file
78
internal/packages/types.go
Normal file
@@ -0,0 +1,78 @@
|
||||
// Package packages contains types and logic for indexing Nix packages.
|
||||
package packages
|
||||
|
||||
// RawPackage represents a package as parsed from nix-env --json output.
|
||||
type RawPackage struct {
|
||||
Pname string `json:"pname"`
|
||||
Version string `json:"version"`
|
||||
System string `json:"system"`
|
||||
Meta RawPackageMeta `json:"meta"`
|
||||
Name string `json:"name"`
|
||||
OutputName string `json:"outputName,omitempty"`
|
||||
Outputs map[string]interface{} `json:"outputs,omitempty"`
|
||||
}
|
||||
|
||||
// RawPackageMeta contains package metadata.
|
||||
type RawPackageMeta struct {
|
||||
Available bool `json:"available,omitempty"`
|
||||
Broken bool `json:"broken,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Homepage interface{} `json:"homepage,omitempty"` // Can be string or []string
|
||||
Insecure bool `json:"insecure,omitempty"`
|
||||
License interface{} `json:"license,omitempty"` // Can be string, object, or []interface{}
|
||||
LongDescription string `json:"longDescription,omitempty"`
|
||||
Maintainers []Maintainer `json:"maintainers,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
OutputsToInstall []string `json:"outputsToInstall,omitempty"`
|
||||
Platforms []interface{} `json:"platforms,omitempty"` // Can be strings or objects
|
||||
Position string `json:"position,omitempty"`
|
||||
Unfree bool `json:"unfree,omitempty"`
|
||||
}
|
||||
|
||||
// Maintainer represents a package maintainer.
|
||||
type Maintainer struct {
|
||||
Email string `json:"email,omitempty"`
|
||||
Github string `json:"github,omitempty"`
|
||||
GithubID int `json:"githubId,omitempty"`
|
||||
Matrix string `json:"matrix,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// ParsedPackage represents a package ready for database storage.
|
||||
type ParsedPackage struct {
|
||||
AttrPath string
|
||||
Pname string
|
||||
Version string
|
||||
Description string
|
||||
LongDescription string
|
||||
Homepage string
|
||||
License string // JSON array
|
||||
Platforms string // JSON array
|
||||
Maintainers string // JSON array
|
||||
Broken bool
|
||||
Unfree bool
|
||||
Insecure bool
|
||||
}
|
||||
|
||||
// PackagesFile represents the top-level structure of nix-env JSON output.
|
||||
// It's a map from attr path to package definition.
|
||||
type PackagesFile map[string]RawPackage
|
||||
|
||||
// ChannelAliases maps friendly channel names to their git branch/ref patterns.
|
||||
// These are the same as NixOS options since packages come from the same repo.
|
||||
var ChannelAliases = map[string]string{
|
||||
"nixos-unstable": "nixos-unstable",
|
||||
"nixos-stable": "nixos-24.11",
|
||||
"nixos-24.11": "nixos-24.11",
|
||||
"nixos-24.05": "nixos-24.05",
|
||||
"nixos-23.11": "nixos-23.11",
|
||||
"nixos-23.05": "nixos-23.05",
|
||||
}
|
||||
|
||||
// IndexResult contains the results of a package indexing operation.
|
||||
type IndexResult struct {
|
||||
RevisionID int64
|
||||
PackageCount int
|
||||
Duration interface{} // time.Duration - kept as interface to avoid import cycle
|
||||
AlreadyIndexed bool // True if revision already has packages
|
||||
}
|
||||
Reference in New Issue
Block a user