Improve filestore test

.woodpecker.yml

@@ -4,7 +4,7 @@ pipeline:
     commands:
       - go build -o gpaste-client ./cmd/client/client.go
       - go build -o gpaste-server ./cmd/server/server.go
-      - go test -v ./...
+      - go test -cover ./...
       - go vet ./...
     when:
       branch: master

Dockerfile

@@ -10,4 +10,5 @@ RUN go build -o gpaste-server ./cmd/server/server.go
 FROM alpine:latest
 COPY --from=builder /src/gpaste /bin/gpaste
 COPY --from=builder /src/gpaste-server /bin/gpaste-server
+EXPOSE 8080
 CMD ["/bin/gpaste-server"]

README.md

@@ -0,0 +1,35 @@
+![status-badge](https://ci.t-juice.club/api/badges/torjus/gpaste/status.svg)
+
+# gpaste
+
+Simple pastebin-style webapp.
+
+## Build
+
+### gpaste-server
+
+```text
+go build -o gpaste-server cmd/server/server.go
+```
+
+### gpaste-client
+
+```text
+go build -o gpaste cmd/client/client.go
+```
+
+## Run using docker
+
+### From registry
+
+```text
+docker pull registry.t-juice.club/gpaste:latest
+docker run --rm -it -p 8080:8080 registry.t-juice.club/gpaste:latest
+```
+
+### From Dockerfile
+
+```text
+docker build -t gpaste:latest .
+docker run --rm -it p 8080:8080 gpaste:latest
+```

api/http.go

@@ -4,7 +4,9 @@ import (
 	"encoding/json"
 	"io"
 	"net/http"
+	"strconv"
 	"strings"
+	"time"
 
 	"git.t-juice.club/torjus/gpaste"
 	"git.t-juice.club/torjus/gpaste/files"
@@ -62,10 +64,6 @@ func (s *HTTPServer) HandlerIndex(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s *HTTPServer) HandlerAPIFilePost(w http.ResponseWriter, r *http.Request) {
-	f := &files.File{
-		ID:   uuid.Must(uuid.NewRandom()).String(),
-		Body: r.Body,
-	}
 	reqID := middleware.GetReqID(r.Context())
 
 	// Check if multipart form
@@ -74,6 +72,11 @@ func (s *HTTPServer) HandlerAPIFilePost(w http.ResponseWriter, r *http.Request)
 		s.processMultiPartFormUpload(w, r)
 		return
 	}
+
+	f := fileFromParams(r)
+	f.ID = uuid.NewString()
+	f.Body = r.Body
+
 	err := s.Files.Store(f)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
@@ -149,11 +152,10 @@ func (s *HTTPServer) processMultiPartFormUpload(w http.ResponseWriter, r *http.R
 			s.Logger.Warnw("Error reading file from multipart form.", "req_id", reqID, "error", err)
 			return
 		}
-		f := &files.File{
+		f := fileFromParams(r)
-			ID:               uuid.Must(uuid.NewRandom()).String(),
+		f.ID = uuid.NewString()
-			OriginalFilename: fh.Filename,
+		f.OriginalFilename = fh.Filename
-			Body:             ff,
+		f.Body = ff
-		}
 
 		if err := s.Files.Store(f); err != nil {
 			w.WriteHeader(http.StatusInternalServerError)
@@ -220,7 +222,7 @@ func (s *HTTPServer) HandlerAPIUserCreate(w http.ResponseWriter, r *http.Request
 	}
 
 	// TODO: Ensure user does not already exist
-	user := &users.User{Username: req.Username}
+	user := &users.User{Username: req.Username, Role: users.RoleUser}
 	if err := user.SetPassword(req.Password); err != nil {
 		s.Logger.Warnw("Error setting user password.", "req_id", reqID, "error", err, "remote_addr", r.RemoteAddr)
 		w.WriteHeader(http.StatusBadRequest)
@@ -235,3 +237,45 @@ func (s *HTTPServer) HandlerAPIUserCreate(w http.ResponseWriter, r *http.Request
 	w.WriteHeader(http.StatusAccepted)
 	s.Logger.Infow("Created user.", "req_id", reqID, "remote_addr", r.RemoteAddr, "username", req.Username)
 }
+
+func (s *HTTPServer) HandlerAPIUserList(w http.ResponseWriter, r *http.Request) {
+	reqID := middleware.GetReqID(r.Context())
+
+	l, err := s.Users.List()
+	if err != nil {
+		s.Logger.Warnw("Error listing users.", "req_id", reqID, "error", err)
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	encoder := json.NewEncoder(w)
+	if err := encoder.Encode(l); err != nil {
+		s.Logger.Warnw("Error encoding response.", "req_id", "error", err)
+	}
+}
+
+func fileFromParams(r *http.Request) *files.File {
+	const (
+		keyMaxViews  = "max_views"
+		keyExpiresOn = "exp"
+	)
+	var f files.File
+
+	q := r.URL.Query()
+
+	if q.Has(keyMaxViews) {
+		views, err := strconv.ParseUint(q.Get(keyMaxViews), 10, 64)
+		if err == nil {
+			f.MaxViews = uint(views)
+		}
+	}
+
+	if q.Has(keyExpiresOn) {
+		exp, err := time.Parse(time.RFC3339, q.Get(keyExpiresOn))
+		if err == nil {
+			f.ExpiresOn = exp
+		}
+	}
+
+	return &f
+}

api/http_test.go

@@ -16,20 +16,22 @@ import (
 	"git.t-juice.club/torjus/gpaste/api"
 	"git.t-juice.club/torjus/gpaste/files"
 	"git.t-juice.club/torjus/gpaste/users"
+	"github.com/google/go-cmp/cmp"
 	"github.com/google/uuid"
 )
 
 func TestHandlers(t *testing.T) {
-	cfg := &gpaste.ServerConfig{
+	//cfg := &gpaste.ServerConfig{
-		SigningSecret: "abc123",
+	//	SigningSecret: "abc123",
-		Store: &gpaste.ServerStoreConfig{
+	//	Store: &gpaste.ServerStoreConfig{
-			Type: "memory",
+	//		Type: "memory",
-		},
+	//	},
-		URL: "http://localhost:8080",
+	//	URL: "http://localhost:8080",
-	}
+	//}
-	hs := api.NewHTTPServer(cfg)
+	//hs := api.NewHTTPServer(cfg)
 
-	t.Run("HandlerIndex", func(t *testing.T) {
+	t.Run("index", func(t *testing.T) {
+		hs := newServer()
 		rr := httptest.NewRecorder()
 		req := httptest.NewRequest(http.MethodGet, "/", nil)
 
@@ -44,7 +46,11 @@ func TestHandlers(t *testing.T) {
 			t.Errorf("Body does not match expected. Got %s want %s", body, expectedBody)
 		}
 	})
-	t.Run("HandlerAPIFilePost", func(t *testing.T) {
+	t.Run("api", func(t *testing.T) {
+		t.Run("file", func(t *testing.T) {
+			// POST /api/file
+			t.Run("POST", func(t *testing.T) {
+				hs := newServer()
 				rr := httptest.NewRecorder()
 				buf := &bytes.Buffer{}
 				mw := multipart.NewWriter(buf)
@@ -58,7 +64,7 @@ func TestHandlers(t *testing.T) {
 				}
 				mw.Close()
 
-		req := httptest.NewRequest(http.MethodPost, "/api/file", buf)
+				req := httptest.NewRequest(http.MethodPost, "/api/file?max_views=99", buf)
 				req.Header.Add("Content-Type", mw.FormDataContentType())
 
 				hs.Handler.ServeHTTP(rr, req)
@@ -87,9 +93,36 @@ func TestHandlers(t *testing.T) {
 					t.Errorf("Response has empty id")
 				}
 
-		t.Run("HandlerAPIFileGet", func(t *testing.T) {
+				retrieved, err := hs.Files.Get(uploadID)
+				if err != nil {
+					t.Errorf("Error retrieving file: %s", err)
+				}
+				defer retrieved.Body.Close()
+				retBuf := new(bytes.Buffer)
+				io.Copy(retBuf, retrieved.Body)
+				if diff := cmp.Diff(retBuf.String(), expectedData); diff != "" {
+					t.Errorf("Retrieved file mismatch: %s", diff)
+				}
+
+				if retrieved.MaxViews != 99 {
+					t.Errorf("Uploaded file has wrong max_views: %d", retrieved.MaxViews)
+				}
+			})
+			// GET /api/file/id
+			t.Run("GET", func(t *testing.T) {
+				hs := newServer()
+				fileData := "abc123456"
+				sr := io.NopCloser(strings.NewReader(fileData))
+				file := &files.File{
+					ID:               uuid.NewString(),
+					OriginalFilename: "test-file.txt",
+					MaxViews:         99,
+					ExpiresOn:        time.Now().Add(90 * time.Second),
+					Body:             sr,
+				}
+				hs.Files.Store(file)
 				rr := httptest.NewRecorder()
-			url := fmt.Sprintf("/api/file/%s", uploadID)
+				url := fmt.Sprintf("/api/file/%s", file.ID)
 				req := httptest.NewRequest(http.MethodGet, url, nil)
 
 				hs.Handler.ServeHTTP(rr, req)
@@ -98,20 +131,14 @@ func TestHandlers(t *testing.T) {
 					t.Errorf("Returned unexpected status. Got %d want %d", status, http.StatusAccepted)
 					t.Logf(url)
 				}
-			if body := rr.Body.String(); body != expectedData {
+				if diff := cmp.Diff(rr.Body.String(), fileData); diff != "" {
-				t.Errorf("Returned body does not match expected.")
+					t.Errorf("Returned body does not match expected: %s", diff)
 				}
 			})
-	})
+
-	t.Run("HandlerAPIFileDelete", func(t *testing.T) {
+			// DELETE /api/file/id
-		cfg := &gpaste.ServerConfig{
+			t.Run("DELETE", func(t *testing.T) {
-			SigningSecret: "abc123",
+				hs := newServer()
-			Store: &gpaste.ServerStoreConfig{
-				Type: "memory",
-			},
-			URL: "http://localhost:8080",
-		}
-		hs := api.NewHTTPServer(cfg)
 				fileBody := io.NopCloser(strings.NewReader("roflcopter"))
 				file := &files.File{
 					ID:               uuid.NewString(),
@@ -137,9 +164,67 @@ func TestHandlers(t *testing.T) {
 				if _, err := hs.Files.Get(file.ID); err == nil {
 					t.Errorf("Getting after delete returned no error")
 				}
-
 			})
-	t.Run("HandlerAPILogin", func(t *testing.T) {
+		})
+		// /api/user
+
+		t.Run("user", func(t *testing.T) {
+			t.Run("POST", func(t *testing.T) {
+				hs := newServer()
+				adminPw := "admin"
+				admin := &users.User{
+					Username: "admin",
+					Role:     users.RoleAdmin,
+				}
+				_ = admin.SetPassword(adminPw)
+				_ = hs.Users.Store(admin)
+
+				token, err := hs.Auth.Login(admin.Username, adminPw)
+				if err != nil {
+					t.Fatalf("error getting admin token: %s", err)
+				}
+
+				requestData := &api.RequestAPIUserCreate{
+					Username: "test",
+					Password: "test",
+				}
+				body := new(bytes.Buffer)
+				encoder := json.NewEncoder(body)
+				if err := encoder.Encode(requestData); err != nil {
+					t.Fatalf("Error encoding data: %s", err)
+				}
+
+				rr := httptest.NewRecorder()
+				req := httptest.NewRequest(http.MethodPost, "/api/user", body)
+				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
+				hs.Handler.ServeHTTP(rr, req)
+
+				if rr.Result().StatusCode != http.StatusAccepted {
+					t.Fatalf("Create returned wrong status: %s", rr.Result().Status)
+				}
+
+				user, err := hs.Users.Get(requestData.Username)
+				if err != nil {
+					t.Fatalf("Unable to get user after create: %s", err)
+				}
+
+				expectedUser := &users.User{
+					Username: requestData.Username,
+					Role:     users.RoleUser,
+				}
+				ignorePW := cmp.FilterPath(func(p cmp.Path) bool {
+					return p.String() == "HashedPassword"
+				}, cmp.Ignore())
+
+				if diff := cmp.Diff(user, expectedUser, ignorePW); diff != "" {
+					t.Errorf("User does not match expected: %s", diff)
+				}
+			})
+		})
+
+		// /api/login
+		t.Run("Login", func(t *testing.T) {
+			hs := newServer()
 			// TODO: Add test
 			username := "admin"
 			password := "admin"
@@ -183,4 +268,17 @@ func TestHandlers(t *testing.T) {
 				t.Fatalf("Unable to validate received token: %s", err)
 			}
 		})
+	})
+
+}
+
+func newServer() *api.HTTPServer {
+	cfg := &gpaste.ServerConfig{
+		SigningSecret: "abc123",
+		Store: &gpaste.ServerStoreConfig{
+			Type: "memory",
+		},
+		URL: "http://localhost:8080",
+	}
+	return api.NewHTTPServer(cfg)
 }

api/json.go

@@ -18,3 +18,7 @@ type ResponseAPIFilePost struct {
 	ID      string `json:"id"`
 	URL     string `json:"url"`
 }
+
+type ResponseAPIUserList struct {
+	Usernames []string `json:"usernames"`
+}

cmd/server/actions/actions.go

@@ -2,6 +2,7 @@ package actions
 
 import (
 	"context"
+	"io"
 	"net/http"
 	"os"
 	"os/signal"
@@ -21,14 +22,27 @@ func ActionServe(c *cli.Context) error {
 		configPath = c.String("config")
 	}
 
-	f, err := os.Open(configPath)
+	var cfg *gpaste.ServerConfig
+	var r io.ReadCloser
+	r, err := os.Open(configPath)
+	if err != nil {
+		cfg = &gpaste.ServerConfig{
+			LogLevel:      "INFO",
+			URL:           "localhost:8080",
+			ListenAddr:    ":8080",
+			SigningSecret: "TODO: CHANGE THIS LOL",
+			Store: &gpaste.ServerStoreConfig{
+				Type: "memory",
+			},
+		}
+	} else {
+		defer r.Close()
+		cfg, err = gpaste.ServerConfigFromReader(r)
+		if err != nil {
 			if err != nil {
 				return cli.Exit(err, 1)
 			}
-	defer f.Close()
+		}
-	cfg, err := gpaste.ServerConfigFromReader(f)
-	if err != nil {
-		return cli.Exit(err, 1)
 	}
 	// Setup loggers
 	rootLogger := getRootLogger(cfg.LogLevel)

files/filestore.go

@@ -12,6 +12,8 @@ type File struct {
 	ExpiresOn        time.Time `json:"expires_on"`
 
 	Body io.ReadCloser
+
+	FileSize int64 `json:"file_size"`
 }
 
 type FileStore interface {

files/filestore_fs.go

@@ -40,10 +40,12 @@ func (s *FSFileStore) Store(f *File) error {
 	}
 	defer dst.Close()
 
-	if _, err := io.Copy(dst, f.Body); err != nil {
+	n, err := io.Copy(dst, f.Body)
+	if err != nil {
 		return err
 	}
 	s.metadata[f.ID] = metadata
+	s.metadata[f.ID].FileSize = n
 	if err := s.writeMetadata(); err != nil {
 		delete(s.metadata, f.ID)
 		return err

files/filestore_memory.go

@@ -14,6 +14,7 @@ type fileData struct {
 
 	MaxViews  uint
 	ExpiresOn time.Time
+	FileSize  int64
 }
 
 type MemoryFileStore struct {
@@ -35,9 +36,11 @@ func (s *MemoryFileStore) Store(f *File) error {
 		ExpiresOn: f.ExpiresOn,
 	}
 
-	_, err := io.Copy(&data.Body, f.Body)
+	n, err := io.Copy(&data.Body, f.Body)
 	_ = f.Body.Close()
 
+	data.FileSize = n
+
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
@@ -58,6 +61,7 @@ func (s *MemoryFileStore) Get(id string) (*File, error) {
 		MaxViews:  fd.MaxViews,
 		ExpiresOn: fd.ExpiresOn,
 		Body:      io.NopCloser(&fd.Body),
+		FileSize:  fd.FileSize,
 	}
 
 	return f, nil

files/filestore_test.go

@@ -22,8 +22,9 @@ func RunFilestoreTest(s files.FileStore, t *testing.T) {
 		body := io.NopCloser(bodyBuf)
 		f := &files.File{
 			ID:        id,
-			MaxViews: 0,
+			MaxViews:  99,
 			Body:      body,
+			ExpiresOn: time.Now().Add(99 * time.Second),
 		}
 
 		err := s.Store(f)
@@ -50,6 +51,17 @@ func RunFilestoreTest(s files.FileStore, t *testing.T) {
 		if retrievedBuf.String() != dataString {
 			t.Fatalf("Data from retrieved body mismatch. Got %s want %s", retrievedBuf.String(), dataString)
 		}
+		expected := &files.File{
+			ID:        f.ID,
+			MaxViews:  f.MaxViews,
+			ExpiresOn: f.ExpiresOn,
+			FileSize:  int64(len(dataString)),
+		}
+
+		ignoreBody := cmp.FilterPath(func(p cmp.Path) bool { return p.String() == "Body" }, cmp.Ignore())
+		if diff := cmp.Diff(retrieved, expected, ignoreBody); diff != "" {
+			t.Errorf("File comparison failed: %s", diff)
+		}
 
 		// List
 		ids, err := s.List()
@@ -92,6 +104,7 @@ func RunPersistentFilestoreTest(newStoreFunc func() files.FileStore, t *testing.
 				MaxViews:         5,
 				ExpiresOn:        time.Now().Add(10 * time.Minute),
 				Body:             io.NopCloser(strings.NewReader("cocks!")),
+				FileSize:         6,
 			},
 			ExpectedData: "cocks!",
 		},
@@ -102,6 +115,7 @@ func RunPersistentFilestoreTest(newStoreFunc func() files.FileStore, t *testing.
 				MaxViews:         5,
 				ExpiresOn:        time.Now().Add(10 * time.Minute),
 				Body:             io.NopCloser(strings.NewReader("derps!")),
+				FileSize:         6,
 			},
 			ExpectedData: "derps!",
 		},

users/user.go

@@ -1,6 +1,10 @@
 package users
 
-import "golang.org/x/crypto/bcrypt"
+import (
+	"fmt"
+
+	"golang.org/x/crypto/bcrypt"
+)
 
 type Role string
 
@@ -16,10 +20,13 @@ type User struct {
 	Role           Role   `json:"role"`
 }
 
+var ErrNoSuchUser = fmt.Errorf("no such user")
+
 type UserStore interface {
 	Get(username string) (*User, error)
 	Store(user *User) error
 	Delete(username string) error
+	List() ([]string, error)
 }
 
 func (u *User) ValidatePassword(password string) error {

users/userstore_bolt.go

@@ -6,6 +6,8 @@ import (
 	"go.etcd.io/bbolt"
 )
 
+var _ UserStore = &BoltUserStore{}
+
 var keyUsers = []byte("users")
 
 type BoltUserStore struct {
@@ -67,3 +69,22 @@ func (s *BoltUserStore) Delete(username string) error {
 		return bkt.Delete([]byte(username))
 	})
 }
+func (s *BoltUserStore) List() ([]string, error) {
+	var ids []string
+
+	err := s.db.View(func(tx *bbolt.Tx) error {
+		bkt := tx.Bucket(keyUsers)
+
+		c := bkt.Cursor()
+
+		for k, _ := c.First(); k != nil; k, _ = c.Next() {
+			ids = append(ids, string(k))
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return ids, nil
+}

users/userstore_memory.go

@@ -5,6 +5,8 @@ import (
 	"sync"
 )
 
+var _ UserStore = &MemoryUserStore{}
+
 type MemoryUserStore struct {
 	users map[string]*User
 	lock  sync.Mutex
@@ -37,3 +39,14 @@ func (s *MemoryUserStore) Delete(username string) error {
 	delete(s.users, username)
 	return nil
 }
+func (s *MemoryUserStore) List() ([]string, error) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	var ids []string
+	for k := range s.users {
+		ids = append(ids, k)
+	}
+
+	return ids, nil
+}

users/userstore_test.go

@@ -47,5 +47,27 @@ func RunUserStoreTest(newFunc func() (func(), users.UserStore), t *testing.T) {
 				t.Errorf("User mismatch: %s", cmp.Diff(user, userMap[k]))
 			}
 		}
+
+		l, err := s.List()
+		if err != nil {
+			t.Fatalf("Error listing users: %s", err)
+		}
+		if len(l) != len(userMap) {
+			t.Errorf("List wrong amount of users.")
+		}
+
+		for _, username := range l {
+			if err := s.Delete(username); err != nil {
+				t.Fatalf("Error deleting user: %s", err)
+			}
+		}
+
+		l, err = s.List()
+		if err != nil {
+			t.Fatalf("Error listing after delete: %s", err)
+		}
+		if len(l) != 0 {
+			t.Fatalf("List is not empty after deleting all")
+		}
 	})
 }