From d9aab773c62a1b7a45bcecabc30ea41f5db35e65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torjus=20H=C3=A5kestad?= Date: Wed, 4 Feb 2026 01:30:39 +0100 Subject: [PATCH] feat(database): add file size metadata and range parameters - Add byte_size and line_count columns to files table - Increment SchemaVersion to 2 (requires re-indexing) - Add DeclarationWithMetadata, FileRange, FileResult types - Add GetDeclarationsWithMetadata method for file metadata lookup - Add GetFileWithRange method for paginated file retrieval - Implement countLines and applyLineRange helpers Co-Authored-By: Claude Opus 4.5 --- internal/database/interface.go | 26 ++++++ internal/database/postgres.go | 74 +++++++++++++++--- internal/database/schema.go | 6 +- internal/database/sqlite.go | 139 +++++++++++++++++++++++++++++++-- 4 files changed, 226 insertions(+), 19 deletions(-) diff --git a/internal/database/interface.go b/internal/database/interface.go index f9cd6dd..397c018 100644 --- a/internal/database/interface.go +++ b/internal/database/interface.go @@ -44,6 +44,30 @@ type File struct { FilePath string Extension string Content string + ByteSize int + LineCount int +} + +// DeclarationWithMetadata includes declaration info plus file metadata. +type DeclarationWithMetadata struct { + Declaration + ByteSize int // File size in bytes, 0 if file not indexed + LineCount int // Number of lines, 0 if file not indexed + HasFile bool // True if file is indexed +} + +// FileRange specifies a range of lines to return from a file. +type FileRange struct { + Offset int // Line offset (0-based) + Limit int // Maximum lines to return (0 = default 250) +} + +// FileResult contains a file with range metadata. +type FileResult struct { + *File + TotalLines int // Total lines in the file + StartLine int // First line returned (1-based) + EndLine int // Last line returned (1-based) } // SearchFilters contains optional filters for option search. @@ -80,9 +104,11 @@ type Store interface { CreateDeclaration(ctx context.Context, decl *Declaration) error CreateDeclarationsBatch(ctx context.Context, decls []*Declaration) error GetDeclarations(ctx context.Context, optionID int64) ([]*Declaration, error) + GetDeclarationsWithMetadata(ctx context.Context, revisionID, optionID int64) ([]*DeclarationWithMetadata, error) // File operations CreateFile(ctx context.Context, file *File) error CreateFilesBatch(ctx context.Context, files []*File) error GetFile(ctx context.Context, revisionID int64, path string) (*File, error) + GetFileWithRange(ctx context.Context, revisionID int64, path string, r FileRange) (*FileResult, error) } diff --git a/internal/database/postgres.go b/internal/database/postgres.go index 2cde849..765cc9f 100644 --- a/internal/database/postgres.go +++ b/internal/database/postgres.go @@ -88,7 +88,9 @@ func (s *PostgresStore) Initialize(ctx context.Context) error { revision_id INTEGER NOT NULL REFERENCES revisions(id) ON DELETE CASCADE, file_path TEXT NOT NULL, extension TEXT, - content TEXT NOT NULL + content TEXT NOT NULL, + byte_size INTEGER NOT NULL DEFAULT 0, + line_count INTEGER NOT NULL DEFAULT 0 )`, IndexOptionsRevisionName, IndexOptionsRevisionParent, @@ -432,11 +434,19 @@ func (s *PostgresStore) GetDeclarations(ctx context.Context, optionID int64) ([] // CreateFile creates a new file record. func (s *PostgresStore) CreateFile(ctx context.Context, file *File) error { + // Compute metadata if not already set + if file.ByteSize == 0 { + file.ByteSize = len(file.Content) + } + if file.LineCount == 0 { + file.LineCount = countLines(file.Content) + } + err := s.db.QueryRowContext(ctx, ` - INSERT INTO files (revision_id, file_path, extension, content) - VALUES ($1, $2, $3, $4) + INSERT INTO files (revision_id, file_path, extension, content, byte_size, line_count) + VALUES ($1, $2, $3, $4, $5, $6) RETURNING id`, - file.RevisionID, file.FilePath, file.Extension, file.Content, + file.RevisionID, file.FilePath, file.Extension, file.Content, file.ByteSize, file.LineCount, ).Scan(&file.ID) if err != nil { return fmt.Errorf("failed to create file: %w", err) @@ -453,8 +463,8 @@ func (s *PostgresStore) CreateFilesBatch(ctx context.Context, files []*File) err defer tx.Rollback() stmt, err := tx.PrepareContext(ctx, ` - INSERT INTO files (revision_id, file_path, extension, content) - VALUES ($1, $2, $3, $4) + INSERT INTO files (revision_id, file_path, extension, content, byte_size, line_count) + VALUES ($1, $2, $3, $4, $5, $6) RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) @@ -462,7 +472,15 @@ func (s *PostgresStore) CreateFilesBatch(ctx context.Context, files []*File) err defer stmt.Close() for _, file := range files { - err := stmt.QueryRowContext(ctx, file.RevisionID, file.FilePath, file.Extension, file.Content).Scan(&file.ID) + // Compute metadata if not already set + if file.ByteSize == 0 { + file.ByteSize = len(file.Content) + } + if file.LineCount == 0 { + file.LineCount = countLines(file.Content) + } + + err := stmt.QueryRowContext(ctx, file.RevisionID, file.FilePath, file.Extension, file.Content, file.ByteSize, file.LineCount).Scan(&file.ID) if err != nil { return fmt.Errorf("failed to insert file: %w", err) } @@ -475,9 +493,9 @@ func (s *PostgresStore) CreateFilesBatch(ctx context.Context, files []*File) err func (s *PostgresStore) GetFile(ctx context.Context, revisionID int64, path string) (*File, error) { file := &File{} err := s.db.QueryRowContext(ctx, ` - SELECT id, revision_id, file_path, extension, content + SELECT id, revision_id, file_path, extension, content, byte_size, line_count FROM files WHERE revision_id = $1 AND file_path = $2`, revisionID, path, - ).Scan(&file.ID, &file.RevisionID, &file.FilePath, &file.Extension, &file.Content) + ).Scan(&file.ID, &file.RevisionID, &file.FilePath, &file.Extension, &file.Content, &file.ByteSize, &file.LineCount) if err == sql.ErrNoRows { return nil, nil } @@ -486,3 +504,41 @@ func (s *PostgresStore) GetFile(ctx context.Context, revisionID int64, path stri } return file, nil } + +// GetDeclarationsWithMetadata retrieves declarations with file metadata for an option. +func (s *PostgresStore) GetDeclarationsWithMetadata(ctx context.Context, revisionID, optionID int64) ([]*DeclarationWithMetadata, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT d.id, d.option_id, d.file_path, d.line, + COALESCE(f.byte_size, 0), COALESCE(f.line_count, 0), (f.id IS NOT NULL) + FROM declarations d + LEFT JOIN files f ON f.revision_id = $1 AND f.file_path = d.file_path + WHERE d.option_id = $2`, revisionID, optionID) + if err != nil { + return nil, fmt.Errorf("failed to get declarations with metadata: %w", err) + } + defer rows.Close() + + var decls []*DeclarationWithMetadata + for rows.Next() { + decl := &DeclarationWithMetadata{} + if err := rows.Scan(&decl.ID, &decl.OptionID, &decl.FilePath, &decl.Line, + &decl.ByteSize, &decl.LineCount, &decl.HasFile); err != nil { + return nil, fmt.Errorf("failed to scan declaration: %w", err) + } + decls = append(decls, decl) + } + return decls, rows.Err() +} + +// GetFileWithRange retrieves a file with a specified line range. +func (s *PostgresStore) GetFileWithRange(ctx context.Context, revisionID int64, path string, r FileRange) (*FileResult, error) { + file, err := s.GetFile(ctx, revisionID, path) + if err != nil { + return nil, err + } + if file == nil { + return nil, nil + } + + return applyLineRange(file, r), nil +} diff --git a/internal/database/schema.go b/internal/database/schema.go index 10f6d18..aacfb3c 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -2,7 +2,7 @@ package database // SchemaVersion is the current database schema version. // When this changes, the database will be dropped and recreated. -const SchemaVersion = 1 +const SchemaVersion = 2 // Common SQL statements shared between implementations. const ( @@ -53,7 +53,9 @@ const ( revision_id INTEGER NOT NULL REFERENCES revisions(id) ON DELETE CASCADE, file_path TEXT NOT NULL, extension TEXT, - content TEXT NOT NULL + content TEXT NOT NULL, + byte_size INTEGER NOT NULL DEFAULT 0, + line_count INTEGER NOT NULL DEFAULT 0 )` ) diff --git a/internal/database/sqlite.go b/internal/database/sqlite.go index e2be387..a381d6e 100644 --- a/internal/database/sqlite.go +++ b/internal/database/sqlite.go @@ -470,10 +470,18 @@ func (s *SQLiteStore) GetDeclarations(ctx context.Context, optionID int64) ([]*D // CreateFile creates a new file record. func (s *SQLiteStore) CreateFile(ctx context.Context, file *File) error { + // Compute metadata if not already set + if file.ByteSize == 0 { + file.ByteSize = len(file.Content) + } + if file.LineCount == 0 { + file.LineCount = countLines(file.Content) + } + result, err := s.db.ExecContext(ctx, ` - INSERT INTO files (revision_id, file_path, extension, content) - VALUES (?, ?, ?, ?)`, - file.RevisionID, file.FilePath, file.Extension, file.Content, + INSERT INTO files (revision_id, file_path, extension, content, byte_size, line_count) + VALUES (?, ?, ?, ?, ?, ?)`, + file.RevisionID, file.FilePath, file.Extension, file.Content, file.ByteSize, file.LineCount, ) if err != nil { return fmt.Errorf("failed to create file: %w", err) @@ -496,15 +504,23 @@ func (s *SQLiteStore) CreateFilesBatch(ctx context.Context, files []*File) error defer tx.Rollback() stmt, err := tx.PrepareContext(ctx, ` - INSERT INTO files (revision_id, file_path, extension, content) - VALUES (?, ?, ?, ?)`) + INSERT INTO files (revision_id, file_path, extension, content, byte_size, line_count) + VALUES (?, ?, ?, ?, ?, ?)`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } defer stmt.Close() for _, file := range files { - result, err := stmt.ExecContext(ctx, file.RevisionID, file.FilePath, file.Extension, file.Content) + // Compute metadata if not already set + if file.ByteSize == 0 { + file.ByteSize = len(file.Content) + } + if file.LineCount == 0 { + file.LineCount = countLines(file.Content) + } + + result, err := stmt.ExecContext(ctx, file.RevisionID, file.FilePath, file.Extension, file.Content, file.ByteSize, file.LineCount) if err != nil { return fmt.Errorf("failed to insert file: %w", err) } @@ -522,9 +538,9 @@ func (s *SQLiteStore) CreateFilesBatch(ctx context.Context, files []*File) error func (s *SQLiteStore) GetFile(ctx context.Context, revisionID int64, path string) (*File, error) { file := &File{} err := s.db.QueryRowContext(ctx, ` - SELECT id, revision_id, file_path, extension, content + SELECT id, revision_id, file_path, extension, content, byte_size, line_count FROM files WHERE revision_id = ? AND file_path = ?`, revisionID, path, - ).Scan(&file.ID, &file.RevisionID, &file.FilePath, &file.Extension, &file.Content) + ).Scan(&file.ID, &file.RevisionID, &file.FilePath, &file.Extension, &file.Content, &file.ByteSize, &file.LineCount) if err == sql.ErrNoRows { return nil, nil } @@ -533,3 +549,110 @@ func (s *SQLiteStore) GetFile(ctx context.Context, revisionID int64, path string } return file, nil } + +// GetDeclarationsWithMetadata retrieves declarations with file metadata for an option. +func (s *SQLiteStore) GetDeclarationsWithMetadata(ctx context.Context, revisionID, optionID int64) ([]*DeclarationWithMetadata, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT d.id, d.option_id, d.file_path, d.line, + COALESCE(f.byte_size, 0), COALESCE(f.line_count, 0), (f.id IS NOT NULL) + FROM declarations d + LEFT JOIN files f ON f.revision_id = ? AND f.file_path = d.file_path + WHERE d.option_id = ?`, revisionID, optionID) + if err != nil { + return nil, fmt.Errorf("failed to get declarations with metadata: %w", err) + } + defer rows.Close() + + var decls []*DeclarationWithMetadata + for rows.Next() { + decl := &DeclarationWithMetadata{} + if err := rows.Scan(&decl.ID, &decl.OptionID, &decl.FilePath, &decl.Line, + &decl.ByteSize, &decl.LineCount, &decl.HasFile); err != nil { + return nil, fmt.Errorf("failed to scan declaration: %w", err) + } + decls = append(decls, decl) + } + return decls, rows.Err() +} + +// GetFileWithRange retrieves a file with a specified line range. +func (s *SQLiteStore) GetFileWithRange(ctx context.Context, revisionID int64, path string, r FileRange) (*FileResult, error) { + file, err := s.GetFile(ctx, revisionID, path) + if err != nil { + return nil, err + } + if file == nil { + return nil, nil + } + + return applyLineRange(file, r), nil +} + +// countLines counts the number of lines in content. +func countLines(content string) int { + if content == "" { + return 0 + } + count := 1 + for _, c := range content { + if c == '\n' { + count++ + } + } + // Don't count trailing newline as extra line + if len(content) > 0 && content[len(content)-1] == '\n' { + count-- + } + return count +} + +// applyLineRange extracts a range of lines from a file. +func applyLineRange(file *File, r FileRange) *FileResult { + lines := strings.Split(file.Content, "\n") + totalLines := len(lines) + + // Handle trailing newline + if totalLines > 0 && lines[totalLines-1] == "" { + totalLines-- + lines = lines[:totalLines] + } + + // Apply defaults + offset := r.Offset + if offset < 0 { + offset = 0 + } + limit := r.Limit + if limit <= 0 { + limit = 250 // Default limit + } + + // Calculate range + startLine := offset + 1 // 1-based + if offset >= totalLines { + // Beyond end of file + return &FileResult{ + File: &File{ID: file.ID, RevisionID: file.RevisionID, FilePath: file.FilePath, Extension: file.Extension, Content: "", ByteSize: file.ByteSize, LineCount: file.LineCount}, + TotalLines: totalLines, + StartLine: 0, + EndLine: 0, + } + } + + endIdx := offset + limit + if endIdx > totalLines { + endIdx = totalLines + } + endLine := endIdx // 1-based (last line included) + + // Extract lines + selectedLines := lines[offset:endIdx] + content := strings.Join(selectedLines, "\n") + + return &FileResult{ + File: &File{ID: file.ID, RevisionID: file.RevisionID, FilePath: file.FilePath, Extension: file.Extension, Content: content, ByteSize: file.ByteSize, LineCount: file.LineCount}, + TotalLines: totalLines, + StartLine: startLine, + EndLine: endLine, + } +}