Add fs filestore

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Debug server",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${workspaceFolder}/cmd/server/server.go",
+            "cwd": "${workspaceFolder}"
+        }
+    ]
+}

cmd/client/client.go

@@ -1,7 +1,107 @@
 package main
 
-import "fmt"
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	version = "dev"
+	commit  = "none"
+	date    = "unknown"
+)
 
 func main() {
-	fmt.Println("Starting gpaste client")
+	cli.VersionFlag = &cli.BoolFlag{Name: "version"}
+
+	app := cli.App{
+		Name:    "gpaste",
+		Version: fmt.Sprintf("gpaste %s-%s (%s)", version, commit, date),
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:  "config",
+				Usage: "Path to config-file.",
+			},
+		},
+		Commands: []*cli.Command{
+			{
+				Name:      "upload",
+				Usage:     "Upload file(s)",
+				ArgsUsage: "FILE [FILE]...",
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:  "url",
+						Usage: "Base url of gpaste server",
+					},
+				},
+				Action: ActionUpload,
+			},
+		},
+	}
+
+	app.Run(os.Args)
+}
+
+func ActionUpload(c *cli.Context) error {
+	url := fmt.Sprintf("%s/api/file", c.String("url"))
+	client := &http.Client{}
+	// TODO: Change timeout
+	ctx, cancel := context.WithTimeout(c.Context, 10*time.Minute)
+	defer cancel()
+
+	buf := &bytes.Buffer{}
+	mw := multipart.NewWriter(buf)
+
+	for _, arg := range c.Args().Slice() {
+		f, err := os.Open(arg)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		fw, err := mw.CreateFormFile(uuid.Must(uuid.NewRandom()).String(), arg)
+		if err != nil {
+			return err
+		}
+		if _, err := io.Copy(fw, f); err != nil {
+			return err
+		}
+	}
+	mw.Close()
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, buf)
+	if err != nil {
+		return err
+	}
+	req.Header.Add("Content-Type", mw.FormDataContentType())
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	var expectedResp []struct {
+		Message string `json:"message"`
+		ID      string `json:"id"`
+		URL     string `json:"url"`
+	}
+
+	decoder := json.NewDecoder(resp.Body)
+	if err := decoder.Decode(&expectedResp); err != nil {
+		return fmt.Errorf("error decoding response: %w", err)
+	}
+
+	for _, r := range expectedResp {
+		fmt.Printf("Uploaded file %s\n", r.ID)
+	}
+	return nil
 }

cmd/server/server.go

@@ -69,6 +69,7 @@ func ActionServe(c *cli.Context) error {
 	go func() {
 		srv := gpaste.NewHTTPServer(cfg)
 		srv.Addr = cfg.ListenAddr
+		srv.Logger = serverLogger
 
 		// Wait for cancel
 		go func() {

config.go

@@ -3,6 +3,8 @@ package gpaste
 import (
 	"fmt"
 	"io"
+	"os"
+	"strings"
 
 	"github.com/pelletier/go-toml"
 )
@@ -15,15 +17,48 @@ type ServerConfig struct {
 }
 
 type ServerStoreConfig struct {
-	Type string `toml:"Type"`
+	Type string                    `toml:"Type"`
+	FS   *ServerStoreFSStoreConfig `toml:"FS"`
+}
+
+type ServerStoreFSStoreConfig struct {
+	Dir string `toml:"Dir"`
 }
 
 func ServerConfigFromReader(r io.Reader) (*ServerConfig, error) {
 	decoder := toml.NewDecoder(r)
-	var c ServerConfig
+	c := ServerConfig{
+		Store: &ServerStoreConfig{
+			FS: &ServerStoreFSStoreConfig{},
+		},
+	}
 	if err := decoder.Decode(&c); err != nil {
 		return nil, fmt.Errorf("error decoding server config: %w", err)
 	}
 
+	c.updateFromEnv()
+
 	return &c, nil
 }
+
+func (sc *ServerConfig) updateFromEnv() {
+	if value, ok := os.LookupEnv("GPASTE_LOGLEVEL"); ok {
+		sc.LogLevel = strings.ToUpper(value)
+	}
+
+	if value, ok := os.LookupEnv("GPASTE_URL"); ok {
+		sc.URL = value
+	}
+
+	if value, ok := os.LookupEnv("GPASTE_LISTENADDR"); ok {
+		sc.ListenAddr = value
+	}
+
+	if value, ok := os.LookupEnv("GPASTE_STORE_TYPE"); ok {
+		sc.Store.Type = value
+	}
+
+	if value, ok := os.LookupEnv("GPASTE_STORE_FS_DIR"); ok {
+		sc.Store.FS.Dir = value
+	}
+}

config_test.go

@@ -0,0 +1,93 @@
+package gpaste_test
+
+import (
+	"os"
+	"strings"
+	"testing"
+
+	"git.t-juice.club/torjus/gpaste"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestServerConfig(t *testing.T) {
+	t.Run("FromReader", func(t *testing.T) {
+		clearEnv()
+		simpleConfig := `
+LogLevel = "INFO"
+URL = "http://paste.example.org"
+ListenAddr = ":8080"
+
+[Store]
+Type = "fs"
+[Store.FS]
+Dir = "/tmp"
+`
+		expected := &gpaste.ServerConfig{
+			LogLevel:   "INFO",
+			URL:        "http://paste.example.org",
+			ListenAddr: ":8080",
+			Store: &gpaste.ServerStoreConfig{
+				Type: "fs",
+				FS: &gpaste.ServerStoreFSStoreConfig{
+					Dir: "/tmp",
+				},
+			},
+		}
+		sr := strings.NewReader(simpleConfig)
+
+		c, err := gpaste.ServerConfigFromReader(sr)
+		if err != nil {
+			t.Fatalf("Error parsing config: %s", err)
+		}
+
+		if !cmp.Equal(c, expected) {
+			t.Errorf("Result does not match: %s", cmp.Diff(c, expected))
+		}
+	})
+	t.Run("FromEnv", func(t *testing.T) {
+		clearEnv()
+
+		var envMap map[string]string = map[string]string{
+			"GPASTE_LOGLEVEL":     "DEBUG",
+			"GPASTE_URL":          "http://gpaste.example.org",
+			"GPASTE_STORE_TYPE":   "fs",
+			"GPASTE_LISTENADDR":   ":8000",
+			"GPASTE_STORE_FS_DIR": "/tmp",
+		}
+		expected := &gpaste.ServerConfig{
+			LogLevel:   "DEBUG",
+			URL:        "http://gpaste.example.org",
+			ListenAddr: ":8000",
+			Store: &gpaste.ServerStoreConfig{
+				Type: "fs",
+				FS: &gpaste.ServerStoreFSStoreConfig{
+					Dir: "/tmp",
+				},
+			},
+		}
+
+		for k, v := range envMap {
+			os.Setenv(k, v)
+		}
+
+		sr := strings.NewReader("")
+		c, err := gpaste.ServerConfigFromReader(sr)
+		if err != nil {
+			t.Fatalf("Error parsing empty config")
+		}
+
+		if !cmp.Equal(c, expected) {
+			t.Errorf("Result does not match: %s", cmp.Diff(c, expected))
+		}
+	})
+}
+
+func clearEnv() {
+	for _, env := range os.Environ() {
+		result := strings.Split(env, "=")
+		value := result[0]
+		if strings.Contains(value, "GPASTE_") {
+			os.Unsetenv(value)
+		}
+	}
+}

filestore.go

@@ -6,11 +6,12 @@ import (
 )
 
 type File struct {
-	ID   string
+	ID               string    `json:"id"`
-	Body io.ReadCloser
+	OriginalFilename string    `json:"original_filename"`
+	MaxViews         uint      `json:"max_views"`
+	ExpiresOn        time.Time `json:"expires_on"`
 
-	MaxViews  uint
+	Body io.ReadCloser
-	ExpiresOn time.Time
 }
 
 type FileStore interface {

filestore_fs.go

@@ -0,0 +1,115 @@
+package gpaste
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+type FSFileStore struct {
+	dir      string
+	metadata map[string]*File
+}
+
+func NewFSFileStore(dir string) (*FSFileStore, error) {
+	s := &FSFileStore{
+		dir:      dir,
+		metadata: make(map[string]*File),
+	}
+
+	err := s.readMetadata()
+
+	return s, err
+}
+func (s *FSFileStore) Store(f *File) error {
+	defer f.Body.Close()
+
+	metadata := &File{
+		ID:               f.ID,
+		OriginalFilename: f.OriginalFilename,
+		MaxViews:         f.MaxViews,
+		ExpiresOn:        f.ExpiresOn,
+	}
+
+	path := filepath.Join(s.dir, f.ID)
+	dst, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer dst.Close()
+
+	if _, err := io.Copy(dst, f.Body); err != nil {
+		return err
+	}
+	s.metadata[f.ID] = metadata
+	if err := s.writeMetadata(); err != nil {
+		delete(s.metadata, f.ID)
+		return err
+	}
+	return nil
+}
+
+func (s *FSFileStore) Get(id string) (*File, error) {
+	metadata, ok := s.metadata[id]
+	if !ok {
+		return nil, fmt.Errorf("no such item")
+	}
+
+	path := filepath.Join(s.dir, id)
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	metadata.Body = f
+	return metadata, nil
+}
+
+func (s *FSFileStore) Delete(id string) error {
+	path := filepath.Join(s.dir, id)
+	if err := os.Remove(path); err != nil {
+		return err
+	}
+	delete(s.metadata, id)
+	return nil
+}
+
+func (s *FSFileStore) List() ([]string, error) {
+	var results []string
+	for k := range s.metadata {
+		results = append(results, k)
+	}
+	return results, nil
+}
+
+func (s *FSFileStore) writeMetadata() error {
+	path := filepath.Join(s.dir, "metadata.json")
+	f, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	encoder := json.NewEncoder(f)
+	if err := encoder.Encode(s.metadata); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *FSFileStore) readMetadata() error {
+	path := filepath.Join(s.dir, "metadata.json")
+	f, err := os.Open(path)
+	if err != nil {
+		// TODO: Handle errors better
+		return nil
+	}
+	defer f.Close()
+
+	decoder := json.NewDecoder(f)
+	if err := decoder.Decode(&s.metadata); err != nil {
+		return err
+	}
+	return nil
+}

filestore_fs_test.go

@@ -0,0 +1,17 @@
+package gpaste_test
+
+import (
+	"testing"
+
+	"git.t-juice.club/torjus/gpaste"
+)
+
+func TestFSFileStore(t *testing.T) {
+	dir := t.TempDir()
+	s, err := gpaste.NewFSFileStore(dir)
+	if err != nil {
+		t.Fatalf("Error creating store: %s", err)
+	}
+
+	RunFilestoreTest(s, t)
+}

go.mod

@@ -7,6 +7,7 @@ require github.com/google/uuid v1.3.0
 require github.com/go-chi/chi/v5 v5.0.7
 
 require (
+	github.com/google/go-cmp v0.5.6
 	github.com/pelletier/go-toml v1.9.4
 	github.com/urfave/cli/v2 v2.3.0
 	go.uber.org/zap v1.20.0

go.sum

@@ -9,6 +9,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8=
 github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -65,6 +67,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

http.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"io"
 	"net/http"
+	"strings"
 
 	"github.com/go-chi/chi/v5"
 	"github.com/google/uuid"
@@ -43,6 +44,12 @@ func (s *HTTPServer) HandlerAPIFilePost(w http.ResponseWriter, r *http.Request)
 		Body: r.Body,
 	}
 
+	// Check if multipart form
+	ct := r.Header.Get("Content-Type")
+	if strings.Contains(ct, "multipart/form-data") {
+		s.processMultiPartFormUpload(w, r)
+		return
+	}
 	err := s.store.Store(f)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
@@ -85,3 +92,46 @@ func (s *HTTPServer) HandlerAPIFileGet(w http.ResponseWriter, r *http.Request) {
 		s.Logger.Warnw("Error writing file to client.", "error", err, "remote_addr", r.RemoteAddr)
 	}
 }
+
+func (s *HTTPServer) processMultiPartFormUpload(w http.ResponseWriter, r *http.Request) {
+	s.Logger.Debugw("Processing multipart form.")
+	type resp struct {
+		Message string `json:"message"`
+		ID      string `json:"id"`
+		URL     string `json:"url"`
+	}
+
+	var responses []resp
+
+	if err := r.ParseMultipartForm(1024 * 1024 * 10); err != nil {
+		s.Logger.Warnw("Error parsing multipart form.", "err", err)
+	}
+	for k := range r.MultipartForm.File {
+		ff, fh, err := r.FormFile(k)
+		if err != nil {
+			s.Logger.Warnw("Error reading file from multipart form.", "error", err)
+			return
+		}
+		f := &File{
+			ID:               uuid.Must(uuid.NewRandom()).String(),
+			OriginalFilename: fh.Filename,
+			Body:             ff,
+		}
+
+		if err := s.store.Store(f); err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			s.Logger.Warnw("Error storing file.", "erorr", err, "id", f.ID, "remote_addr", r.RemoteAddr)
+			return
+		}
+		s.Logger.Infow("Stored file.", "id", f.ID, "filename", f.OriginalFilename, "remote_addr", r.RemoteAddr)
+
+		responses = append(responses, resp{Message: "OK", ID: f.ID, URL: "TODO"})
+
+	}
+
+	w.WriteHeader(http.StatusAccepted)
+	encoder := json.NewEncoder(w)
+	if err := encoder.Encode(&responses); err != nil {
+		s.Logger.Warnw("Error encoding response to client.", "error", err, "remote_addr", r.RemoteAddr)
+	}
+}

http_test.go

@@ -0,0 +1,99 @@
+package gpaste_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"git.t-juice.club/torjus/gpaste"
+)
+
+func TestHandlers(t *testing.T) {
+	cfg := &gpaste.ServerConfig{
+		Store: &gpaste.ServerStoreConfig{
+			Type: "memory",
+		},
+		URL: "http://localhost:8080",
+	}
+	hs := gpaste.NewHTTPServer(cfg)
+
+	t.Run("HandlerIndex", func(t *testing.T) {
+		rr := httptest.NewRecorder()
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+
+		hs.Handler.ServeHTTP(rr, req)
+
+		if status := rr.Code; status != http.StatusOK {
+			t.Errorf("Returned unexpected status")
+		}
+
+		expectedBody := "index"
+		if body := rr.Body.String(); body != expectedBody {
+			t.Errorf("Body does not match expected. Got %s want %s", body, expectedBody)
+		}
+	})
+	t.Run("HandlerAPIFilePost", func(t *testing.T) {
+		rr := httptest.NewRecorder()
+		buf := &bytes.Buffer{}
+		mw := multipart.NewWriter(buf)
+		fw, err := mw.CreateFormFile("test", "test.txt")
+		if err != nil {
+			t.Fatalf("Unable to create form file: %s", err)
+		}
+		expectedData := "Test OMEGALUL PLS."
+		if _, err := io.WriteString(fw, expectedData); err != nil {
+			t.Fatalf("Unable to write body to buffer: %s", err)
+		}
+		mw.Close()
+
+		req := httptest.NewRequest(http.MethodPost, "/api/file", buf)
+		req.Header.Add("Content-Type", mw.FormDataContentType())
+
+		hs.Handler.ServeHTTP(rr, req)
+
+		if status := rr.Code; status != http.StatusAccepted {
+			t.Errorf("Returned unexpected status. Got %d want %d", status, http.StatusAccepted)
+		}
+
+		var expectedResp []struct {
+			Message string `json:"message"`
+			ID      string `json:"id"`
+			URL     string `json:"url"`
+		}
+
+		decoder := json.NewDecoder(rr.Result().Body)
+		if err := decoder.Decode(&expectedResp); err != nil {
+			t.Fatalf("error decoding response: %s", err)
+		}
+
+		if l := len(expectedResp); l != 1 {
+			t.Errorf("Response has wrong length. Got %d want %d", l, 1)
+		}
+
+		uploadID := expectedResp[0].ID
+		if uploadID == "" {
+			t.Errorf("Response has empty id")
+		}
+
+		t.Run("HandlerAPIFileGet", func(t *testing.T) {
+			rr := httptest.NewRecorder()
+			url := fmt.Sprintf("/api/file/%s", uploadID)
+			req := httptest.NewRequest(http.MethodGet, url, nil)
+
+			hs.Handler.ServeHTTP(rr, req)
+
+			if status := rr.Code; status != http.StatusOK {
+				t.Errorf("Returned unexpected status. Got %d want %d", status, http.StatusAccepted)
+				t.Logf(url)
+			}
+			if body := rr.Body.String(); body != expectedData {
+				t.Errorf("Returned body does not match expected.")
+			}
+		})
+	})
+}