From bf9f8d80cda23e42a8ed631a19b10aba694e8e66 Mon Sep 17 00:00:00 2001 From: = Date: Sat, 4 Dec 2021 09:58:16 +0100 Subject: [PATCH] Add bolt store --- cmd/ezshare.go | 12 ++++-- config/config.go | 45 +++++++++++++++++--- ezshare.example.toml | 7 +++- go.mod | 1 + go.sum | 3 ++ store/bolt.go | 90 ++++++++++++++++++++++++++++++++++++++++ store/bolt_test.go | 18 ++++++++ store/filesystem_test.go | 10 +---- 8 files changed, 166 insertions(+), 20 deletions(-) create mode 100644 store/bolt.go create mode 100644 store/bolt_test.go diff --git a/cmd/ezshare.go b/cmd/ezshare.go index 65c3614..db8f4d1 100644 --- a/cmd/ezshare.go +++ b/cmd/ezshare.go @@ -18,7 +18,6 @@ import ( "gitea.benny.dog/torjus/ezshare/config" "gitea.benny.dog/torjus/ezshare/pb" "gitea.benny.dog/torjus/ezshare/server" - "gitea.benny.dog/torjus/ezshare/store" "github.com/urfave/cli/v2" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -137,7 +136,12 @@ func ActionServe(c *cli.Context) error { return err } - fileStore := store.NewMemoryFileStore() + // Setup store + s, closeFunc, err := cfg.Server.StoreConfig.GetStore() + if err != nil { + return fmt.Errorf("unable to initialize store: %w", err) + } + defer closeFunc() // Setup shutdown-handling rootCtx, rootCancel := signal.NotifyContext(context.Background(), os.Interrupt) defer rootCancel() @@ -157,7 +161,7 @@ func ActionServe(c *cli.Context) error { grpcAddr = c.String("grpc-addr") } - grpcFileServer := server.NewGRPCFileServiceServer(fileStore) + grpcFileServer := server.NewGRPCFileServiceServer(s) if c.IsSet("hostname") { grpcFileServer.Hostname = c.String("hostname") } @@ -215,7 +219,7 @@ func ActionServe(c *cli.Context) error { if c.IsSet("http-addr") { httpAddr = c.String("http-addr") } - httpServer := server.NewHTTPSever(fileStore) + httpServer := server.NewHTTPSever(s) httpServer.Addr = httpAddr // wait for cancel diff --git a/config/config.go b/config/config.go index 097db91..ffe32e6 100644 --- a/config/config.go +++ b/config/config.go @@ -10,7 +10,9 @@ import ( "io/ioutil" "os" "path/filepath" + "strings" + "gitea.benny.dog/torjus/ezshare/store" "github.com/pelletier/go-toml" "google.golang.org/grpc/credentials" ) @@ -26,23 +28,33 @@ type CertificatePaths struct { } type ServerConfig struct { - LogLevel string `toml:"LogLevel"` - Hostname string `toml:"Hostname"` - GRPC *ServerGRPCConfig `toml:"GRPC"` - HTTP *ServerHTTPConfig `toml:"HTTP"` + LogLevel string `toml:"LogLevel"` + Hostname string `toml:"Hostname"` + StoreConfig *ServerStoreConfig `toml:"Store"` + GRPC *ServerGRPCConfig `toml:"GRPC"` + HTTP *ServerHTTPConfig `toml:"HTTP"` } + type ServerStoreConfig struct { - Type string `toml:"Type"` - FSStoreConfig *FSStoreConfig `toml:"Filesystem"` + Type string `toml:"Type"` + FSStoreConfig *FSStoreConfig `toml:"Filesystem"` + BoltStoreConfig *BoltStoreConfig `toml:"Bolt"` } + +type BoltStoreConfig struct { + Path string `toml:"Path"` +} + type FSStoreConfig struct { Dir string `toml:"Dir"` } + type ServerGRPCConfig struct { ListenAddr string `toml:"ListenAddr"` CACerts *CertificatePaths `toml:"CACerts"` Certs *CertificatePaths `toml:"Certs"` } + type ServerHTTPConfig struct { ListenAddr string `toml:"ListenAddr"` } @@ -213,3 +225,24 @@ func (c *Config) ToDefaultFile() error { } return fmt.Errorf("config-file already exists") } + +func (sc *ServerStoreConfig) GetStore() (store.FileStore, func() error, error) { + nopCloseFunc := func() error { return nil } + if strings.EqualFold(sc.Type, "bolt") { + s, err := store.NewBoltStore(sc.BoltStoreConfig.Path) + if err != nil { + return nil, nil, err + } + return s, s.Close, err + + } + if strings.EqualFold(sc.Type, "filesystem") { + s := store.NewFileSystemStore(sc.FSStoreConfig.Dir) + return s, nopCloseFunc, nil + } + if strings.EqualFold(sc.Type, "memory") { + return store.NewMemoryFileStore(), nopCloseFunc, nil + } + + return nil, nil, fmt.Errorf("invalid store config") +} diff --git a/ezshare.example.toml b/ezshare.example.toml index 81d0d1c..0b9769d 100644 --- a/ezshare.example.toml +++ b/ezshare.example.toml @@ -15,10 +15,15 @@ Hostname = "localhost" # Storage configuration [Server.Store] # How server stores file -# Must be one of: filesystem, memory +# Must be one of: filesystem, memory, bolt # Required Type = "filesystem" +[Server.Store.Bolt] +# Where the bolt-db is stored +# Required if store-type is bolt +Path = "" + [Server.Store.Filesystem] # Where files are stored # Required if store-type is filesystem diff --git a/go.mod b/go.mod index 5e6c6de..5312724 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/google/uuid v1.3.0 github.com/pelletier/go-toml v1.9.4 github.com/urfave/cli/v2 v2.3.0 + go.etcd.io/bbolt v1.3.6 google.golang.org/grpc v1.42.0 google.golang.org/protobuf v1.27.1 ) diff --git a/go.sum b/go.sum index 36bcde1..ae37d31 100644 --- a/go.sum +++ b/go.sum @@ -70,6 +70,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -97,6 +99,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/store/bolt.go b/store/bolt.go new file mode 100644 index 0000000..cad319c --- /dev/null +++ b/store/bolt.go @@ -0,0 +1,90 @@ +package store + +import ( + "fmt" + + "gitea.benny.dog/torjus/ezshare/pb" + "github.com/google/uuid" + bolt "go.etcd.io/bbolt" + "google.golang.org/protobuf/proto" +) + +type BoltStore struct { + db *bolt.DB +} + +var bktKey = []byte("files") + +func NewBoltStore(path string) (*BoltStore, error) { + s := &BoltStore{} + db, err := bolt.Open(path, 0660, &bolt.Options{}) + if err != nil { + return nil, fmt.Errorf("unable to open store: %w", err) + } + s.db = db + err = db.Update(func(t *bolt.Tx) error { + if _, err := t.CreateBucketIfNotExists(bktKey); err != nil { + return err + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("unable to create bucket: %w", err) + } + return s, nil +} + +func (s *BoltStore) Close() error { + return s.db.Close() +} + +func (s *BoltStore) GetFile(id string) (*pb.File, error) { + var file pb.File + var data []byte + if err := s.db.View(func(t *bolt.Tx) error { + bkt := t.Bucket(bktKey) + data = bkt.Get([]byte(id)) + return nil + }); err != nil { + return nil, fmt.Errorf("error getting file: %w", err) + } + if data == nil { + return nil, ErrNoSuchFile + } + + err := proto.Unmarshal(data, &file) + if err != nil { + return nil, fmt.Errorf("unable to unmarshal file: %w", err) + } + + return &file, nil +} + +func (s *BoltStore) StoreFile(file *pb.File) (string, error) { + file.FileId = uuid.Must(uuid.NewRandom()).String() + data, err := proto.Marshal(file) + if err != nil { + return "", fmt.Errorf("unable to marshal file: %w", err) + } + + err = s.db.Update(func(t *bolt.Tx) error { + bkt := t.Bucket(bktKey) + return bkt.Put([]byte(file.FileId), data) + }) + if err != nil { + return "", fmt.Errorf("unable to store file: %w", err) + } + return file.FileId, nil +} + +func (s *BoltStore) DeleteFile(id string) error { + return s.db.Update(func(t *bolt.Tx) error { + bkt := t.Bucket(bktKey) + data := bkt.Get([]byte(id)) + if data == nil { + return ErrNoSuchFile + } + return bkt.Delete([]byte(id)) + }) +} diff --git a/store/bolt_test.go b/store/bolt_test.go new file mode 100644 index 0000000..4502bab --- /dev/null +++ b/store/bolt_test.go @@ -0,0 +1,18 @@ +package store_test + +import ( + "path/filepath" + "testing" + + "gitea.benny.dog/torjus/ezshare/store" +) + +func TestBoltStore(t *testing.T) { + path := filepath.Join(t.TempDir(), "boltstore.db") + s, err := store.NewBoltStore(path) + if err != nil { + t.Fatalf("Error opening store: %s", err) + } + doFileStoreTest(s, t) + s.Close() +} diff --git a/store/filesystem_test.go b/store/filesystem_test.go index ed109e6..929fff4 100644 --- a/store/filesystem_test.go +++ b/store/filesystem_test.go @@ -1,21 +1,13 @@ package store_test import ( - "io/ioutil" - "os" "testing" "gitea.benny.dog/torjus/ezshare/store" ) func TestFileStore(t *testing.T) { - dir, err := ioutil.TempDir(os.TempDir(), "ezshare-test") - if err != nil { - t.Fatalf("unable to create temp-dir") - } - defer func() { - _ = os.RemoveAll(dir) - }() + dir := t.TempDir() s := store.NewFileSystemStore(dir) doFileStoreTest(s, t)