Compare commits

..

10 Commits

Author SHA1 Message Date
94bcb0087f Fix typo in pipeline
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2022-02-01 06:53:23 +01:00
12e9c72463 More detailed coverage in pipeline
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
2022-02-01 06:52:20 +01:00
cc4835f224 Remove wsl linter
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2022-01-27 11:48:49 +01:00
9d9bb3f0ac Tidy go.mod
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
2022-01-27 11:47:43 +01:00
daa6ad4278 Update dependencies
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
2022-01-27 11:46:32 +01:00
cc4e61f981 Add gofumpt linter
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
2022-01-27 11:36:16 +01:00
c6eb147e2c Use configured store
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2022-01-25 01:57:28 +01:00
150ffc3400 Fix error in memorystore
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2022-01-24 23:18:13 +01:00
ee761d4006 Change access logger
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2022-01-24 22:53:46 +01:00
8ae5ee64bb Fix scoop url
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2022-01-24 20:50:42 +01:00
18 changed files with 277 additions and 112 deletions

View File

@ -24,10 +24,10 @@ linters:
- ireturn
- gocritic
- whitespace
- wsl
- stylecheck
- exportloopref
- godot
- gofumpt
linters-settings:
gomnd:

View File

@ -49,8 +49,7 @@ changelog:
- '^test:'
scoop:
# Default for gitea is "https://gitea.com/<repo_owner>/<repo_name>/releases/download/{{ .Tag }}/{{ .ArtifactName }}"
url_template: "https://git.t-juice.club/torjus/scoop-tjuice/releases/download/{{ .Tag }}/{{ .ArtifactName }}"
url_template: "https://git.t-juice.club/torjus/gpaste/releases/download/{{ .Tag }}/{{ .ArtifactName }}"
bucket:
owner: torjus
name: scoop-tjuice

View File

@ -4,7 +4,8 @@ pipeline:
commands:
- go build -o gpaste-client ./cmd/client/client.go
- go build -o gpaste-server ./cmd/server/server.go
- go test -cover ./...
- go test -cover -coverprofile="/tmp/cover.out" ./...
- go tool cover -func="/tmp/cover.out"
- go vet ./...
when:
branch: master

View File

@ -35,16 +35,11 @@ func NewHTTPServer(cfg *gpaste.ServerConfig) *HTTPServer {
config: cfg,
Logger: zap.NewNop().Sugar(),
AccessLogger: zap.NewNop().Sugar(),
Files: files.NewMemoryFileStore(),
Users: users.NewMemoryUserStore(),
}
srv.Files = files.NewMemoryFileStore()
srv.Users = users.NewMemoryUserStore()
srv.Auth = gpaste.NewAuthService(srv.Users, []byte(srv.config.SigningSecret))
// Create initial user
// TODO: Do properly
user := &users.User{Username: "admin", Role: users.RoleAdmin}
_ = user.SetPassword("admin")
_ = srv.Users.Store(user)
signingSecret, _ := uuid.Must(uuid.NewRandom()).MarshalBinary()
srv.Auth = gpaste.NewAuthService(srv.Users, signingSecret)
r := chi.NewRouter()
r.Use(middleware.RealIP)

View File

@ -13,6 +13,7 @@ type RequestAPILogin struct {
type ResponseAPILogin struct {
Token string `json:"token"`
}
type ResponseAPIFilePost struct {
Message string `json:"message"`
Files []ResponseAPIFilePostFiles `json:"files"`

View File

@ -10,6 +10,7 @@ import (
"git.t-juice.club/torjus/gpaste"
"git.t-juice.club/torjus/gpaste/users"
"github.com/go-chi/chi/v5/middleware"
"go.uber.org/zap"
)
type authCtxKey int
@ -27,14 +28,38 @@ func (s *HTTPServer) MiddlewareAccessLogger(next http.Handler) http.Handler {
reqID := middleware.GetReqID(r.Context())
// TODO: Maybe desugar in HTTPServer to avoid doing for all requests
logger := s.AccessLogger.Desugar()
defer func() {
s.AccessLogger.Infow(r.Method,
"path", r.URL.Path,
"status", ww.Status(),
"written", ww.BytesWritten(),
"remote_addr", r.RemoteAddr,
"processing_time_ms", time.Since(t1).Milliseconds(),
"req_id", reqID)
// DEBUG level
if ce := logger.Check(zap.DebugLevel, r.Method); ce != nil {
ct := r.Header.Get("Content-Type")
ce.Write(
zap.String("req_id", reqID),
zap.String("path", r.URL.Path),
zap.Int("status", ww.Status()),
zap.String("remote_addr", r.RemoteAddr),
zap.Int("bytes_written", ww.BytesWritten()),
zap.Duration("processing_time", time.Since(t1)),
zap.String("content_type", ct),
zap.Any("headers", r.Header),
)
} else {
// INFO level
if ce := logger.Check(zap.InfoLevel, r.Method); ce != nil {
ce.Write(
zap.String("req_id", reqID),
zap.String("path", r.URL.Path),
zap.Int("status", ww.Status()),
zap.String("remote_addr", r.RemoteAddr),
zap.Int("bytes_written", ww.BytesWritten()),
zap.Duration("processing_time", time.Since(t1)),
)
}
}
_ = logger.Sync()
}()
next.ServeHTTP(ww, r)

View File

@ -58,7 +58,6 @@ func (as *AuthService) ValidateToken(rawToken string) (*Claims, error) {
token, err := jwt.ParseWithClaims(rawToken, claims, func(t *jwt.Token) (interface{}, error) {
return as.hmacSecret, nil
})
if err != nil {
return nil, err
}

View File

@ -31,6 +31,7 @@ func (c *Client) WriteConfigToWriter(w io.Writer) error {
encoder := json.NewEncoder(w)
return encoder.Encode(c)
}
func (c *Client) WriteConfig() error {
dir := configdir.LocalConfig("gpaste")
// Ensure dir exists

View File

@ -6,11 +6,15 @@ import (
"net/http"
"os"
"os/signal"
"path/filepath"
"strings"
"time"
"git.t-juice.club/torjus/gpaste"
"git.t-juice.club/torjus/gpaste/api"
"git.t-juice.club/torjus/gpaste/files"
"git.t-juice.club/torjus/gpaste/users"
"github.com/google/uuid"
"github.com/urfave/cli/v2"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
@ -62,11 +66,49 @@ func ActionServe(c *cli.Context) error {
httpShutdownCtx, httpShutdownCancel := context.WithCancel(context.Background())
defer httpShutdownCancel()
// Setup stores
// Files
fileStore, fileClose, err := getFileStore(cfg)
if err != nil {
return err
}
defer fileClose() // nolint: errcheck
// Users
userStore, userClose, err := getUserStore(cfg)
if err != nil {
return err
}
defer userClose() // nolint: errcheck
if userList, err := userStore.List(); err != nil {
serverLogger.Panicw("Error checking userstore for users.", "error", err)
} else if len(userList) < 1 {
admin := users.User{
Username: "admin",
Role: users.RoleAdmin,
}
password := uuid.NewString()
if err := admin.SetPassword(password); err != nil {
serverLogger.DPanic("Error setting admin-user password.", "error", err)
}
serverLogger.Warnw("Created admin-user.", "username", admin.Username, "password", password)
}
// Auth
auth := gpaste.NewAuthService(userStore, []byte(cfg.SigningSecret))
go func() {
srv := api.NewHTTPServer(cfg)
srv.Users = userStore
srv.Files = fileStore
srv.Addr = cfg.ListenAddr
srv.Logger = serverLogger
srv.AccessLogger = accessLogger
srv.Auth = auth
// Wait for cancel
go func() {
@ -126,3 +168,48 @@ func getRootLogger(level string) *zap.SugaredLogger {
return rootLogger.Sugar()
}
// nolint: ireturn
func getUserStore(cfg *gpaste.ServerConfig) (users.UserStore, func() error, error) {
closer := func() error { return nil }
switch cfg.Store.Type {
case "memory":
return users.NewMemoryUserStore(), closer, nil
case "fs":
path := filepath.Join(cfg.Store.FS.Dir, "gpaste-users.db")
bs, err := users.NewBoltUserStore(path)
if err != nil {
return nil, closer, cli.Exit("error setting up user store", 1)
}
return bs, bs.Close, nil
default:
return nil, closer, cli.Exit("no userstore configured", 1)
}
}
// nolint: ireturn
func getFileStore(cfg *gpaste.ServerConfig) (files.FileStore, func() error, error) {
closer := func() error { return nil }
switch cfg.Store.Type {
case "memory":
return files.NewMemoryFileStore(), closer, nil
case "fs":
var err error
s, err := files.NewFSFileStore(cfg.Store.FS.Dir)
if err != nil {
return nil, closer, cli.Exit("error setting up filestore", 1)
}
return s, closer, nil
default:
return nil, closer, cli.Exit("No store configured", 1)
}
}

View File

@ -23,6 +23,7 @@ func NewFSFileStore(dir string) (*FSFileStore, error) {
return s, err
}
func (s *FSFileStore) Store(f *File) error {
defer f.Body.Close()

View File

@ -7,20 +7,23 @@ import (
)
func TestFSFileStore(t *testing.T) {
dir := t.TempDir()
s, err := files.NewFSFileStore(dir)
if err != nil {
t.Fatalf("Error creating store: %s", err)
newFunc := func() files.FileStore {
dir := t.TempDir()
s, err := files.NewFSFileStore(dir)
if err != nil {
t.Fatalf("Error creating store: %s", err)
}
return s
}
RunFilestoreTest(s, t)
RunFilestoreTest(newFunc, t)
persistentDir := t.TempDir()
newFunc := func() files.FileStore {
persistentFunc := func() files.FileStore {
s, err := files.NewFSFileStore(persistentDir)
if err != nil {
t.Fatalf("Error creating store: %s", err)
}
return s
}
RunPersistentFilestoreTest(newFunc, t)
RunPersistentFilestoreTest(persistentFunc, t)
}

View File

@ -57,11 +57,16 @@ func (s *MemoryFileStore) Get(id string) (*File, error) {
return nil, fmt.Errorf("no such item")
}
body := new(bytes.Buffer)
if _, err := body.Write(fd.Body.Bytes()); err != nil {
return nil, err
}
f := &File{
ID: fd.ID,
MaxViews: fd.MaxViews,
ExpiresOn: fd.ExpiresOn,
Body: io.NopCloser(&fd.Body),
Body: io.NopCloser(body),
FileSize: fd.FileSize,
}

View File

@ -7,7 +7,9 @@ import (
)
func TestMemoryFileStore(t *testing.T) {
s := files.NewMemoryFileStore()
newFunc := func() files.FileStore {
return files.NewMemoryFileStore()
}
RunFilestoreTest(s, t)
RunFilestoreTest(newFunc, t)
}

View File

@ -12,8 +12,11 @@ import (
"github.com/google/uuid"
)
func RunFilestoreTest(s files.FileStore, t *testing.T) {
var ignoreBody = cmp.FilterPath(func(p cmp.Path) bool { return p.String() == "Body" }, cmp.Ignore())
func RunFilestoreTest(newStoreFunc func() files.FileStore, t *testing.T) {
t.Run("Basic", func(t *testing.T) {
s := newStoreFunc()
// Create
dataString := "TEST_LOL_OMG"
id := uuid.Must(uuid.NewRandom()).String()
@ -58,7 +61,6 @@ func RunFilestoreTest(s files.FileStore, t *testing.T) {
FileSize: int64(len(dataString)),
}
ignoreBody := cmp.FilterPath(func(p cmp.Path) bool { return p.String() == "Body" }, cmp.Ignore())
if diff := cmp.Diff(retrieved, expected, ignoreBody); diff != "" {
t.Errorf("File comparison failed: %s", diff)
}
@ -88,84 +90,128 @@ func RunFilestoreTest(s files.FileStore, t *testing.T) {
t.Fatalf("List after delete has wrong length: %d", len(ids))
}
})
t.Run("MultipleGet", func(t *testing.T) {
s := newStoreFunc()
fileContents := "multiple get test !"
body := io.NopCloser(strings.NewReader(fileContents))
file := &files.File{
ID: uuid.NewString(),
OriginalFilename: "multiple.txt",
MaxViews: 999,
ExpiresOn: time.Now().Add(1 * time.Hour),
Body: body,
FileSize: int64(len(fileContents)),
}
if err := s.Store(file); err != nil {
t.Fatalf("Error storing file: %s", err)
}
first, err := s.Get(file.ID)
if err != nil {
t.Errorf("Error retrieving first file: %s", err)
}
firstBody := new(bytes.Buffer)
io.Copy(firstBody, first.Body)
first.Body.Close()
if diff := cmp.Diff(firstBody.String(), fileContents); diff != "" {
t.Fatalf("File contents mismatch: %s", diff)
}
second, err := s.Get(file.ID)
if err != nil {
t.Errorf("Error retrieving first file: %s", err)
}
secondBody := new(bytes.Buffer)
io.Copy(secondBody, second.Body)
first.Body.Close()
if diff := cmp.Diff(secondBody.String(), fileContents); diff != "" {
t.Fatalf("File contents mismatch: %s", diff)
}
})
}
func RunPersistentFilestoreTest(newStoreFunc func() files.FileStore, t *testing.T) {
s := newStoreFunc()
t.Run("Basics", func(t *testing.T) {
s := newStoreFunc()
files := []struct {
File *files.File
ExpectedData string
}{
{
File: &files.File{
ID: uuid.NewString(),
OriginalFilename: "testfile.txt",
MaxViews: 5,
ExpiresOn: time.Now().Add(10 * time.Minute),
Body: io.NopCloser(strings.NewReader("cocks!")),
FileSize: 6,
files := []struct {
File *files.File
ExpectedData string
}{
{
File: &files.File{
ID: uuid.NewString(),
OriginalFilename: "testfile.txt",
MaxViews: 5,
ExpiresOn: time.Now().Add(10 * time.Minute),
Body: io.NopCloser(strings.NewReader("cocks!")),
FileSize: 6,
},
ExpectedData: "cocks!",
},
ExpectedData: "cocks!",
},
{
File: &files.File{
ID: uuid.NewString(),
OriginalFilename: "testfile2.txt",
MaxViews: 5,
ExpiresOn: time.Now().Add(10 * time.Minute),
Body: io.NopCloser(strings.NewReader("derps!")),
FileSize: 6,
{
File: &files.File{
ID: uuid.NewString(),
OriginalFilename: "testfile2.txt",
MaxViews: 5,
ExpiresOn: time.Now().Add(10 * time.Minute),
Body: io.NopCloser(strings.NewReader("derps!")),
FileSize: 6,
},
ExpectedData: "derps!",
},
ExpectedData: "derps!",
},
}
for _, f := range files {
err := s.Store(f.File)
if err != nil {
t.Fatalf("Error storing file: %s", err)
}
}
for _, f := range files {
retrieved, err := s.Get(f.File.ID)
if err != nil {
t.Fatalf("Unable to retrieve file: %s", err)
}
ignoreBody := cmp.FilterPath(func(p cmp.Path) bool { return p.String() == "Body" }, cmp.Ignore())
if !cmp.Equal(retrieved, f.File, ignoreBody) {
t.Errorf("Mismatch: %s", cmp.Diff(retrieved, f.File))
for _, f := range files {
err := s.Store(f.File)
if err != nil {
t.Fatalf("Error storing file: %s", err)
}
}
buf := new(strings.Builder)
if _, err := io.Copy(buf, retrieved.Body); err != nil {
t.Fatalf("Error reading from body: %s", err)
}
retrieved.Body.Close()
if buf.String() != f.ExpectedData {
t.Fatalf("Data does not match. %s", cmp.Diff(buf.String(), f.ExpectedData))
}
}
for _, f := range files {
retrieved, err := s.Get(f.File.ID)
if err != nil {
t.Fatalf("Unable to retrieve file: %s", err)
}
// Reopen store, and fetch again
s = newStoreFunc()
for _, f := range files {
retrieved, err := s.Get(f.File.ID)
if err != nil {
t.Fatalf("Unable to retrieve file: %s", err)
if !cmp.Equal(retrieved, f.File, ignoreBody) {
t.Errorf("Mismatch: %s", cmp.Diff(retrieved, f.File))
}
buf := new(strings.Builder)
if _, err := io.Copy(buf, retrieved.Body); err != nil {
t.Fatalf("Error reading from body: %s", err)
}
retrieved.Body.Close()
if buf.String() != f.ExpectedData {
t.Fatalf("Data does not match. %s", cmp.Diff(buf.String(), f.ExpectedData))
}
}
ignoreBody := cmp.FilterPath(func(p cmp.Path) bool { return p.String() == "Body" }, cmp.Ignore())
if !cmp.Equal(retrieved, f.File, ignoreBody) {
t.Errorf("Mismatch: %s", cmp.Diff(retrieved, f.File))
// Reopen store, and fetch again
s = newStoreFunc()
for _, f := range files {
retrieved, err := s.Get(f.File.ID)
if err != nil {
t.Fatalf("Unable to retrieve file: %s", err)
}
if !cmp.Equal(retrieved, f.File, ignoreBody) {
t.Errorf("Mismatch: %s", cmp.Diff(retrieved, f.File))
}
buf := new(strings.Builder)
if _, err := io.Copy(buf, retrieved.Body); err != nil {
t.Fatalf("Error reading from body: %s", err)
}
retrieved.Body.Close()
if buf.String() != f.ExpectedData {
t.Fatalf("Data does not match. %s", cmp.Diff(buf.String(), f.ExpectedData))
}
}
buf := new(strings.Builder)
if _, err := io.Copy(buf, retrieved.Body); err != nil {
t.Fatalf("Error reading from body: %s", err)
}
retrieved.Body.Close()
if buf.String() != f.ExpectedData {
t.Fatalf("Data does not match. %s", cmp.Diff(buf.String(), f.ExpectedData))
}
}
})
}

10
go.mod
View File

@ -2,19 +2,17 @@ module git.t-juice.club/torjus/gpaste
go 1.17
require github.com/google/uuid v1.3.0
require github.com/go-chi/chi/v5 v5.0.7
require (
github.com/go-chi/chi/v5 v5.0.7
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/google/go-cmp v0.5.6
github.com/google/go-cmp v0.5.7
github.com/google/uuid v1.3.0
github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f
github.com/pelletier/go-toml v1.9.4
github.com/urfave/cli/v2 v2.3.0
go.etcd.io/bbolt v1.3.6
go.uber.org/zap v1.20.0
golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce
golang.org/x/crypto v0.0.0-20220126234351-aa10faf2a1f8
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
)

8
go.sum
View File

@ -11,8 +11,8 @@ github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8=
github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f h1:dKccXx7xA56UNqOcFIbuqFjAWPVtP688j5QMgmo6OHU=
@ -51,8 +51,8 @@ go.uber.org/zap v1.20.0 h1:N4oPlghZwYG55MlU6LXk/Zp00FVNE9X9wrYO8CEs4lc=
go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce h1:Roh6XWxHFKrPgC/EQhVubSAGQ6Ozk6IdxHSzt1mR0EI=
golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220126234351-aa10faf2a1f8 h1:kACShD3qhmr/3rLmg1yXyt+N4HcwutKyPRB93s54TIU=
golang.org/x/crypto v0.0.0-20220126234351-aa10faf2a1f8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=

View File

@ -15,7 +15,7 @@ type BoltUserStore struct {
}
func NewBoltUserStore(path string) (*BoltUserStore, error) {
db, err := bbolt.Open(path, 0666, nil) // nolint: gomnd
db, err := bbolt.Open(path, 0o666, nil) // nolint: gomnd
if err != nil {
return nil, err
}
@ -45,7 +45,6 @@ func (s *BoltUserStore) Get(username string) (*User, error) {
}
return nil
})
if err != nil {
return nil, err
}
@ -72,6 +71,7 @@ func (s *BoltUserStore) Delete(username string) error {
return bkt.Delete([]byte(username))
})
}
func (s *BoltUserStore) List() ([]string, error) {
var ids []string

View File

@ -15,6 +15,7 @@ type MemoryUserStore struct {
func NewMemoryUserStore() *MemoryUserStore {
return &MemoryUserStore{users: make(map[string]*User)}
}
func (s *MemoryUserStore) Get(username string) (*User, error) {
s.lock.Lock()
defer s.lock.Unlock()
@ -45,6 +46,7 @@ func (s *MemoryUserStore) Delete(username string) error {
return nil
}
func (s *MemoryUserStore) List() ([]string, error) {
s.lock.Lock()
defer s.lock.Unlock()