From 6db99dd2bbeeb53db453780587cb0d3374c68d78 Mon Sep 17 00:00:00 2001 From: Lennart Date: Fri, 30 Jul 2021 14:09:58 +0200 Subject: [PATCH] replace web/file by web/file2 --- web/{file2 => file}/createfs.go | 4 +- web/file/error.go | 8 -- web/file/file.go | 12 --- web/{file2 => file}/fileinfo.go | 0 web/{file2 => file}/fs.go | 0 web/{file2 => file}/fs/file.go | 0 web/{file2 => file}/fs/fileinfo.go | 0 web/{file2 => file}/fs/fs.go | 0 web/{file2 => file}/fs/fs_test.go | 4 +- web/file/fs/main.go | 63 ------------ web/file/fs/main_test.go | 92 ----------------- .../a.txt | 1 - .../d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/data | 0 .../d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/name | 0 .../d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/type | 0 web/{file2 => file}/fs_test.go | 2 +- web/file/main_test.go | 98 ------------------- web/file/manager.go | 20 ---- web/file/s3/address.go | 57 ----------- web/{file2 => file}/s3/file.go | 0 web/{file2 => file}/s3/fileinfo.go | 0 web/{file2 => file}/s3/fs.go | 0 web/{file2 => file}/s3/fs_test.go | 3 +- web/file/service.go | 69 ------------- 24 files changed, 7 insertions(+), 426 deletions(-) rename web/{file2 => file}/createfs.go (92%) delete mode 100644 web/file/error.go delete mode 100644 web/file/file.go rename web/{file2 => file}/fileinfo.go (100%) rename web/{file2 => file}/fs.go (100%) rename web/{file2 => file}/fs/file.go (100%) rename web/{file2 => file}/fs/fileinfo.go (100%) rename web/{file2 => file}/fs/fs.go (100%) rename web/{file2 => file}/fs/fs_test.go (92%) delete mode 100644 web/file/fs/main.go delete mode 100644 web/file/fs/main_test.go delete mode 100644 web/file/fs/test/00000000-0000-0000-0000-000000000000/a.txt rename web/{file2 => file}/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/data (100%) rename web/{file2 => file}/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/name (100%) rename web/{file2 => file}/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/type (100%) rename web/{file2 => file}/fs_test.go (98%) delete mode 100644 web/file/main_test.go delete mode 100644 web/file/manager.go delete mode 100644 web/file/s3/address.go rename web/{file2 => file}/s3/file.go (100%) rename web/{file2 => file}/s3/fileinfo.go (100%) rename web/{file2 => file}/s3/fs.go (100%) rename web/{file2 => file}/s3/fs_test.go (83%) delete mode 100644 web/file/service.go diff --git a/web/file2/createfs.go b/web/file/createfs.go similarity index 92% rename from web/file2/createfs.go rename to web/file/createfs.go index b605614..ebb9802 100644 --- a/web/file2/createfs.go +++ b/web/file/createfs.go @@ -3,8 +3,8 @@ package file import ( "errors" - "dev.sum7.eu/genofire/golang-lib/web/file2/fs" - "dev.sum7.eu/genofire/golang-lib/web/file2/s3" + "dev.sum7.eu/genofire/golang-lib/web/file/fs" + "dev.sum7.eu/genofire/golang-lib/web/file/s3" ) // fsType represents a type of file store. diff --git a/web/file/error.go b/web/file/error.go deleted file mode 100644 index 8a58381..0000000 --- a/web/file/error.go +++ /dev/null @@ -1,8 +0,0 @@ -package file - -import "errors" - -// Error Messages -var ( - ErrUnsupportedStorageType = errors.New("storage type invalid") -) diff --git a/web/file/file.go b/web/file/file.go deleted file mode 100644 index dee76cb..0000000 --- a/web/file/file.go +++ /dev/null @@ -1,12 +0,0 @@ -package file - -import "github.com/google/uuid" - -// File to store information in database -type File struct { - ID uuid.UUID `json:"id" gorm:"type:uuid;default:gen_random_uuid()" example:"32466d63-efa4-4f27-9f2b-a1f06c8e2e1d"` - StorageType string `json:"storage_type,omitempty"` - Path string `json:"path,omitempty"` - Filename string `json:"filename"` - ContentType string `json:"content-type"` -} diff --git a/web/file2/fileinfo.go b/web/file/fileinfo.go similarity index 100% rename from web/file2/fileinfo.go rename to web/file/fileinfo.go diff --git a/web/file2/fs.go b/web/file/fs.go similarity index 100% rename from web/file2/fs.go rename to web/file/fs.go diff --git a/web/file2/fs/file.go b/web/file/fs/file.go similarity index 100% rename from web/file2/fs/file.go rename to web/file/fs/file.go diff --git a/web/file2/fs/fileinfo.go b/web/file/fs/fileinfo.go similarity index 100% rename from web/file2/fs/fileinfo.go rename to web/file/fs/fileinfo.go diff --git a/web/file2/fs/fs.go b/web/file/fs/fs.go similarity index 100% rename from web/file2/fs/fs.go rename to web/file/fs/fs.go diff --git a/web/file2/fs/fs_test.go b/web/file/fs/fs_test.go similarity index 92% rename from web/file2/fs/fs_test.go rename to web/file/fs/fs_test.go index fd58242..3a236db 100644 --- a/web/file2/fs/fs_test.go +++ b/web/file/fs/fs_test.go @@ -9,8 +9,8 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/assert" - "dev.sum7.eu/genofire/golang-lib/web/file2" - "dev.sum7.eu/genofire/golang-lib/web/file2/fs" + "dev.sum7.eu/genofire/golang-lib/web/file" + "dev.sum7.eu/genofire/golang-lib/web/file/fs" ) func TestOpenStat(t *testing.T) { diff --git a/web/file/fs/main.go b/web/file/fs/main.go deleted file mode 100644 index b096c26..0000000 --- a/web/file/fs/main.go +++ /dev/null @@ -1,63 +0,0 @@ -package fs - -import ( - "errors" - "io" - "mime/multipart" - "os" - "path" - - "github.com/google/uuid" - - "dev.sum7.eu/genofire/golang-lib/web/file" -) - -// consts for filemanager -const ( - StorageTypeFS = "fs" -) - -// error messages -var ( - ErrPathNotExistsOrNoDirectory = errors.New("path invalid: not exists or not an directory") -) - -// FileManager to handle data on disk -type FileManager struct { -} - -// Check if filemanager could be used -func (m *FileManager) Check(s *file.Service) error { - info, err := os.Stat(s.Path) - if os.IsNotExist(err) || !info.IsDir() { - return ErrPathNotExistsOrNoDirectory - } - return nil -} - -// Save a file on disk and update file db -func (m *FileManager) Save(s *file.Service, file *file.File, src multipart.File) error { - file.ID = uuid.New() - file.Path = path.Join(file.ID.String(), file.Filename) - - directory := path.Join(s.Path, file.ID.String()) - os.Mkdir(directory, 0750) - - out, err := os.Create(path.Join(s.Path, file.Path)) - if err != nil { - return err - } - defer out.Close() - - _, err = io.Copy(out, src) - return err -} - -// Read get an reader of an file -func (m *FileManager) Read(s *file.Service, file *file.File) (io.Reader, error) { - return os.Open(path.Join(s.Path, file.Path)) -} - -func init() { - file.AddManager(StorageTypeFS, &FileManager{}) -} diff --git a/web/file/fs/main_test.go b/web/file/fs/main_test.go deleted file mode 100644 index 7f7963b..0000000 --- a/web/file/fs/main_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package fs - -import ( - "io" - "os" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - - "dev.sum7.eu/genofire/golang-lib/web" - "dev.sum7.eu/genofire/golang-lib/web/file" -) - -func TestCheck(t *testing.T) { - assert := assert.New(t) - - service := file.Service{ - StorageType: StorageTypeFS, - Path: "./test", - } - - assert.NoError(service.Check()) - - service.StorageType = "s3" - assert.ErrorIs(file.ErrUnsupportedStorageType, service.Check()) - - service.StorageType = StorageTypeFS - service.Path = "./main_test.go" - assert.ErrorIs(ErrPathNotExistsOrNoDirectory, service.Check()) - - /* TODO no write permission - service.Path = "/dev" - assert.ErrorIs(ErrPathNotExistsOrNoDirectory, service.Check()) - */ -} - -func TestSave(t *testing.T) { - assert := assert.New(t) - - service := file.Service{ - StorageType: "s3", - Path: "./test", - } - - _, err := service.Upload(nil) - assert.ErrorIs(file.ErrUnsupportedStorageType, err) - - service.StorageType = StorageTypeFS - req, err := web.NewRequestWithFile("localhost", "./test/00000000-0000-0000-0000-000000000000/a.txt") - assert.NoError(err) - assert.NotNil(req) - - _, err = service.Upload(req) - assert.NoError(err) - - service.Path = "/dev" - _, err = service.Upload(req) - assert.True(os.IsNotExist(err)) - //assert.True(os.IsPermission(err)) - - // TODO no write permission -} - -func TestRead(t *testing.T) { - assert := assert.New(t) - - service := file.Service{ - StorageType: "s3", - Path: "./test", - } - - _, err := service.Read(nil) - assert.ErrorIs(file.ErrUnsupportedStorageType, err) - - service.StorageType = StorageTypeFS - - file := &file.File{ - Path: "00000000-0000-0000-0000-000000000000/a.txt", - } - r, err := service.Read(file) - assert.NoError(err) - buf := &strings.Builder{} - _, err = io.Copy(buf, r) - assert.Equal("Hello world\n", buf.String()) - - service.Path = "/dev" - _, err = service.Read(file) - assert.True(os.IsNotExist(err)) - - // TODO no write permission -} diff --git a/web/file/fs/test/00000000-0000-0000-0000-000000000000/a.txt b/web/file/fs/test/00000000-0000-0000-0000-000000000000/a.txt deleted file mode 100644 index 802992c..0000000 --- a/web/file/fs/test/00000000-0000-0000-0000-000000000000/a.txt +++ /dev/null @@ -1 +0,0 @@ -Hello world diff --git a/web/file2/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/data b/web/file/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/data similarity index 100% rename from web/file2/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/data rename to web/file/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/data diff --git a/web/file2/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/name b/web/file/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/name similarity index 100% rename from web/file2/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/name rename to web/file/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/name diff --git a/web/file2/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/type b/web/file/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/type similarity index 100% rename from web/file2/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/type rename to web/file/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/type diff --git a/web/file2/fs_test.go b/web/file/fs_test.go similarity index 98% rename from web/file2/fs_test.go rename to web/file/fs_test.go index 98ba2f3..0f9c1d1 100644 --- a/web/file2/fs_test.go +++ b/web/file/fs_test.go @@ -15,7 +15,7 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/assert" - "dev.sum7.eu/genofire/golang-lib/web/file2" + "dev.sum7.eu/genofire/golang-lib/web/file" ) type TestFS struct { diff --git a/web/file/main_test.go b/web/file/main_test.go deleted file mode 100644 index 7e49f36..0000000 --- a/web/file/main_test.go +++ /dev/null @@ -1,98 +0,0 @@ -package file - -import ( - "bytes" - "errors" - "io" - "mime/multipart" - "net/http" - "strings" - "testing" - - "dev.sum7.eu/genofire/golang-lib/web" - "github.com/gin-gonic/gin" - "github.com/stretchr/testify/assert" -) - -const storageTypeDummy = "dummy" - -type dummyManager struct { -} - -func (m *dummyManager) Check(s *Service) error { - return nil -} -func (m *dummyManager) Save(s *Service, file *File, src multipart.File) error { - if src == nil { - return errors.New("nothing to fill") - } - return nil -} -func (m *dummyManager) Read(s *Service, file *File) (io.Reader, error) { - b := bytes.Buffer{} - b.WriteString("Hello world\n") - return &b, nil -} - -func init() { - AddManager(storageTypeDummy, &dummyManager{}) -} - -func TestCheck(t *testing.T) { - assert := assert.New(t) - - service := Service{ - StorageType: storageTypeDummy, - Path: "./fs/test", - } - assert.NoError(service.Check()) - - service.StorageType = "s3" - assert.ErrorIs(ErrUnsupportedStorageType, service.Check()) -} - -func TestSave(t *testing.T) { - assert := assert.New(t) - - service := Service{ - StorageType: "fs", - Path: "./fs/test", - } - - _, err := service.Upload(nil) - assert.ErrorIs(ErrUnsupportedStorageType, err) - - service.StorageType = storageTypeDummy - _, err = service.GINUpload(&gin.Context{Request: &http.Request{}}) - assert.ErrorIs(err, http.ErrNotMultipart) - - req, err := web.NewRequestWithFile("http://localhost/upload", "./fs/test/00000000-0000-0000-0000-000000000000/a.txt") - assert.NoError(err) - assert.NotNil(req) - - _, err = service.Upload(req) - assert.NoError(err) -} - -func TestRead(t *testing.T) { - assert := assert.New(t) - - service := Service{ - StorageType: "fs", - Path: "./fs/test", - } - - _, err := service.Read(nil) - assert.ErrorIs(ErrUnsupportedStorageType, err) - - service.StorageType = "dummy" - - file := &File{ - Path: "00000000-0000-0000-0000-000000000000/a.txt", - } - r, err := service.Read(file) - assert.NoError(err) - buf := &strings.Builder{} - _, err = io.Copy(buf, r) - assert.Equal("Hello world\n", buf.String()) -} diff --git a/web/file/manager.go b/web/file/manager.go deleted file mode 100644 index a3760fd..0000000 --- a/web/file/manager.go +++ /dev/null @@ -1,20 +0,0 @@ -package file - -import ( - "io" - "mime/multipart" -) - -type FileManager interface { - Check(s *Service) error - Save(s *Service, fileObj *File, file multipart.File) error - Read(s *Service, fileObj *File) (io.Reader, error) -} - -var ( - managers = make(map[string]FileManager) -) - -func AddManager(typ string, m FileManager) { - managers[typ] = m -} diff --git a/web/file/s3/address.go b/web/file/s3/address.go deleted file mode 100644 index f255bdf..0000000 --- a/web/file/s3/address.go +++ /dev/null @@ -1,57 +0,0 @@ -package s3 - -import ( - "context" - "errors" - "net/url" - - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" -) - -// Error Messages during connect -var ( - ErrNoPassword = errors.New("no secret access key found") -) - -// Connect try to use a path to setup a connection to s3 server -func Connect(path string) (*minio.Client, string, error) { - u, err := url.Parse(path) - if err != nil { - return nil, "", err - } - - tls := u.Scheme[len(u.Scheme)-1] == 's' - accessKeyID := u.User.Username() - secretAccessKey, ok := u.User.Password() - if !ok { - return nil, "", ErrNoPassword - } - query := u.Query() - bucketName := query.Get("bucket") - location := query.Get("location") - - u.User = nil - u.RawQuery = "" - - ctx := context.Background() - - // Initialize minio client object. - minioClient, err := minio.New(u.String(), &minio.Options{ - Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), - Secure: tls, - }) - if err != nil { - return nil, "", err - } - - // create and check for bucket - err = minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: location}) - if err != nil { - if exists, err := minioClient.BucketExists(ctx, bucketName); err != nil || !exists { - return nil, "", err - } - } - - return minioClient, bucketName, err -} diff --git a/web/file2/s3/file.go b/web/file/s3/file.go similarity index 100% rename from web/file2/s3/file.go rename to web/file/s3/file.go diff --git a/web/file2/s3/fileinfo.go b/web/file/s3/fileinfo.go similarity index 100% rename from web/file2/s3/fileinfo.go rename to web/file/s3/fileinfo.go diff --git a/web/file2/s3/fs.go b/web/file/s3/fs.go similarity index 100% rename from web/file2/s3/fs.go rename to web/file/s3/fs.go diff --git a/web/file2/s3/fs_test.go b/web/file/s3/fs_test.go similarity index 83% rename from web/file2/s3/fs_test.go rename to web/file/s3/fs_test.go index 84e8c83..cb4ea04 100644 --- a/web/file2/s3/fs_test.go +++ b/web/file/s3/fs_test.go @@ -5,7 +5,8 @@ import ( "github.com/stretchr/testify/assert" - "dev.sum7.eu/genofire/golang-lib/web/file2" + "dev.sum7.eu/genofire/golang-lib/web/file" + "dev.sum7.eu/genofire/golang-lib/web/file/s3" ) // TODO: actually test, either using little dummies or using sth like play.min.io diff --git a/web/file/service.go b/web/file/service.go deleted file mode 100644 index 84162a4..0000000 --- a/web/file/service.go +++ /dev/null @@ -1,69 +0,0 @@ -package file - -import ( - "io" - "net/http" - "path/filepath" - - "github.com/gin-gonic/gin" -) - -// A Service to handle file-uploads in golang -type Service struct { - StorageType string `toml:"storage_type"` - Path string `toml:"path"` -} - -// Check if Service is configurated and useable -func (s *Service) Check() error { - mgmt, ok := managers[s.StorageType] - if !ok { - return ErrUnsupportedStorageType - } - return mgmt.Check(s) -} - -// Upload a file to storage -func (s *Service) Upload(request *http.Request) (*File, error) { - mgmt, ok := managers[s.StorageType] - if !ok { - return nil, ErrUnsupportedStorageType - } - file, fileRequest, err := request.FormFile("file") - if err != nil { - return nil, err - } - fileObj := File{ - Filename: filepath.Base(fileRequest.Filename), - } - - // detect contenttype - buffer := make([]byte, 512) - n, err := file.Read(buffer) - if err != nil && err != io.EOF { - return nil, err - } - fileObj.ContentType = http.DetectContentType(buffer[:n]) - - // Reset the read pointer - file.Seek(0, io.SeekStart) - if err := mgmt.Save(s, &fileObj, file); err != nil { - return nil, err - } - return &fileObj, nil - -} - -// GINUpload a file to storage using gin-gonic -func (s *Service) GINUpload(c *gin.Context) (*File, error) { - return s.Upload(c.Request) -} - -// Read a file to storage -func (s *Service) Read(file *File) (io.Reader, error) { - mgmt, ok := managers[s.StorageType] - if !ok { - return nil, ErrUnsupportedStorageType - } - return mgmt.Read(s, file) -}