From 689bb03277c64e549cee106a13374fa306b8aecb Mon Sep 17 00:00:00 2001 From: Geno Date: Wed, 21 Jul 2021 15:35:22 +0200 Subject: [PATCH] first try to setup file upload --- web/file/error.go | 8 ++ web/file/file.go | 12 +++ web/file/fs/main.go | 63 ++++++++++++ web/file/fs/main_test.go | 92 +++++++++++++++++ .../a.txt | 1 + web/file/main_test.go | 98 +++++++++++++++++++ web/file/manager.go | 20 ++++ web/file/s3/address.go | 28 ++++++ web/file/service.go | 69 +++++++++++++ web/request.go | 42 ++++++++ 10 files changed, 433 insertions(+) create mode 100644 web/file/error.go create mode 100644 web/file/file.go create mode 100644 web/file/fs/main.go create mode 100644 web/file/fs/main_test.go create mode 100644 web/file/fs/test/00000000-0000-0000-0000-000000000000/a.txt create mode 100644 web/file/main_test.go create mode 100644 web/file/manager.go create mode 100644 web/file/s3/address.go create mode 100644 web/file/service.go diff --git a/web/file/error.go b/web/file/error.go new file mode 100644 index 0000000..8a58381 --- /dev/null +++ b/web/file/error.go @@ -0,0 +1,8 @@ +package file + +import "errors" + +// Error Messages +var ( + ErrUnsupportedStorageType = errors.New("storage type invalid") +) diff --git a/web/file/file.go b/web/file/file.go new file mode 100644 index 0000000..dee76cb --- /dev/null +++ b/web/file/file.go @@ -0,0 +1,12 @@ +package file + +import "github.com/google/uuid" + +// File to store information in database +type File struct { + ID uuid.UUID `json:"id" gorm:"type:uuid;default:gen_random_uuid()" example:"32466d63-efa4-4f27-9f2b-a1f06c8e2e1d"` + StorageType string `json:"storage_type,omitempty"` + Path string `json:"path,omitempty"` + Filename string `json:"filename"` + ContentType string `json:"content-type"` +} diff --git a/web/file/fs/main.go b/web/file/fs/main.go new file mode 100644 index 0000000..b096c26 --- /dev/null +++ b/web/file/fs/main.go @@ -0,0 +1,63 @@ +package fs + +import ( + "errors" + "io" + "mime/multipart" + "os" + "path" + + "github.com/google/uuid" + + "dev.sum7.eu/genofire/golang-lib/web/file" +) + +// consts for filemanager +const ( + StorageTypeFS = "fs" +) + +// error messages +var ( + ErrPathNotExistsOrNoDirectory = errors.New("path invalid: not exists or not an directory") +) + +// FileManager to handle data on disk +type FileManager struct { +} + +// Check if filemanager could be used +func (m *FileManager) Check(s *file.Service) error { + info, err := os.Stat(s.Path) + if os.IsNotExist(err) || !info.IsDir() { + return ErrPathNotExistsOrNoDirectory + } + return nil +} + +// Save a file on disk and update file db +func (m *FileManager) Save(s *file.Service, file *file.File, src multipart.File) error { + file.ID = uuid.New() + file.Path = path.Join(file.ID.String(), file.Filename) + + directory := path.Join(s.Path, file.ID.String()) + os.Mkdir(directory, 0750) + + out, err := os.Create(path.Join(s.Path, file.Path)) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, src) + return err +} + +// Read get an reader of an file +func (m *FileManager) Read(s *file.Service, file *file.File) (io.Reader, error) { + return os.Open(path.Join(s.Path, file.Path)) +} + +func init() { + file.AddManager(StorageTypeFS, &FileManager{}) +} diff --git a/web/file/fs/main_test.go b/web/file/fs/main_test.go new file mode 100644 index 0000000..7f7963b --- /dev/null +++ b/web/file/fs/main_test.go @@ -0,0 +1,92 @@ +package fs + +import ( + "io" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "dev.sum7.eu/genofire/golang-lib/web" + "dev.sum7.eu/genofire/golang-lib/web/file" +) + +func TestCheck(t *testing.T) { + assert := assert.New(t) + + service := file.Service{ + StorageType: StorageTypeFS, + Path: "./test", + } + + assert.NoError(service.Check()) + + service.StorageType = "s3" + assert.ErrorIs(file.ErrUnsupportedStorageType, service.Check()) + + service.StorageType = StorageTypeFS + service.Path = "./main_test.go" + assert.ErrorIs(ErrPathNotExistsOrNoDirectory, service.Check()) + + /* TODO no write permission + service.Path = "/dev" + assert.ErrorIs(ErrPathNotExistsOrNoDirectory, service.Check()) + */ +} + +func TestSave(t *testing.T) { + assert := assert.New(t) + + service := file.Service{ + StorageType: "s3", + Path: "./test", + } + + _, err := service.Upload(nil) + assert.ErrorIs(file.ErrUnsupportedStorageType, err) + + service.StorageType = StorageTypeFS + req, err := web.NewRequestWithFile("localhost", "./test/00000000-0000-0000-0000-000000000000/a.txt") + assert.NoError(err) + assert.NotNil(req) + + _, err = service.Upload(req) + assert.NoError(err) + + service.Path = "/dev" + _, err = service.Upload(req) + assert.True(os.IsNotExist(err)) + //assert.True(os.IsPermission(err)) + + // TODO no write permission +} + +func TestRead(t *testing.T) { + assert := assert.New(t) + + service := file.Service{ + StorageType: "s3", + Path: "./test", + } + + _, err := service.Read(nil) + assert.ErrorIs(file.ErrUnsupportedStorageType, err) + + service.StorageType = StorageTypeFS + + file := &file.File{ + Path: "00000000-0000-0000-0000-000000000000/a.txt", + } + r, err := service.Read(file) + assert.NoError(err) + buf := &strings.Builder{} + _, err = io.Copy(buf, r) + assert.Equal("Hello world\n", buf.String()) + + service.Path = "/dev" + _, err = service.Read(file) + assert.True(os.IsNotExist(err)) + + // TODO no write permission +} diff --git a/web/file/fs/test/00000000-0000-0000-0000-000000000000/a.txt b/web/file/fs/test/00000000-0000-0000-0000-000000000000/a.txt new file mode 100644 index 0000000..802992c --- /dev/null +++ b/web/file/fs/test/00000000-0000-0000-0000-000000000000/a.txt @@ -0,0 +1 @@ +Hello world diff --git a/web/file/main_test.go b/web/file/main_test.go new file mode 100644 index 0000000..7e49f36 --- /dev/null +++ b/web/file/main_test.go @@ -0,0 +1,98 @@ +package file + +import ( + "bytes" + "errors" + "io" + "mime/multipart" + "net/http" + "strings" + "testing" + + "dev.sum7.eu/genofire/golang-lib/web" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" +) + +const storageTypeDummy = "dummy" + +type dummyManager struct { +} + +func (m *dummyManager) Check(s *Service) error { + return nil +} +func (m *dummyManager) Save(s *Service, file *File, src multipart.File) error { + if src == nil { + return errors.New("nothing to fill") + } + return nil +} +func (m *dummyManager) Read(s *Service, file *File) (io.Reader, error) { + b := bytes.Buffer{} + b.WriteString("Hello world\n") + return &b, nil +} + +func init() { + AddManager(storageTypeDummy, &dummyManager{}) +} + +func TestCheck(t *testing.T) { + assert := assert.New(t) + + service := Service{ + StorageType: storageTypeDummy, + Path: "./fs/test", + } + assert.NoError(service.Check()) + + service.StorageType = "s3" + assert.ErrorIs(ErrUnsupportedStorageType, service.Check()) +} + +func TestSave(t *testing.T) { + assert := assert.New(t) + + service := Service{ + StorageType: "fs", + Path: "./fs/test", + } + + _, err := service.Upload(nil) + assert.ErrorIs(ErrUnsupportedStorageType, err) + + service.StorageType = storageTypeDummy + _, err = service.GINUpload(&gin.Context{Request: &http.Request{}}) + assert.ErrorIs(err, http.ErrNotMultipart) + + req, err := web.NewRequestWithFile("http://localhost/upload", "./fs/test/00000000-0000-0000-0000-000000000000/a.txt") + assert.NoError(err) + assert.NotNil(req) + + _, err = service.Upload(req) + assert.NoError(err) +} + +func TestRead(t *testing.T) { + assert := assert.New(t) + + service := Service{ + StorageType: "fs", + Path: "./fs/test", + } + + _, err := service.Read(nil) + assert.ErrorIs(ErrUnsupportedStorageType, err) + + service.StorageType = "dummy" + + file := &File{ + Path: "00000000-0000-0000-0000-000000000000/a.txt", + } + r, err := service.Read(file) + assert.NoError(err) + buf := &strings.Builder{} + _, err = io.Copy(buf, r) + assert.Equal("Hello world\n", buf.String()) +} diff --git a/web/file/manager.go b/web/file/manager.go new file mode 100644 index 0000000..a3760fd --- /dev/null +++ b/web/file/manager.go @@ -0,0 +1,20 @@ +package file + +import ( + "io" + "mime/multipart" +) + +type FileManager interface { + Check(s *Service) error + Save(s *Service, fileObj *File, file multipart.File) error + Read(s *Service, fileObj *File) (io.Reader, error) +} + +var ( + managers = make(map[string]FileManager) +) + +func AddManager(typ string, m FileManager) { + managers[typ] = m +} diff --git a/web/file/s3/address.go b/web/file/s3/address.go new file mode 100644 index 0000000..76f16d9 --- /dev/null +++ b/web/file/s3/address.go @@ -0,0 +1,28 @@ +package s3 + +import ( + "net/url" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +// Connect try to use a path to setup a connection to s3 server +func Connect(path string) (client, string, error) { + url, err := url.Parse(path) + if err != nil { + return nil, "", err + } + + secretAccessKey, err := url.Userinfo.Password() + if err != nil { + return nil, "", err + } + + // Initialize minio client object. + minioClient, err := minio.New(url.Host, &minio.Options{ + Creds: credentials.NewStaticV4(url.Userinfo.Username(), secretAccessKey, ""), + Secure: url.Schema[-1] == "s", + }) + return minioClient, url.Path, err +} diff --git a/web/file/service.go b/web/file/service.go new file mode 100644 index 0000000..84162a4 --- /dev/null +++ b/web/file/service.go @@ -0,0 +1,69 @@ +package file + +import ( + "io" + "net/http" + "path/filepath" + + "github.com/gin-gonic/gin" +) + +// A Service to handle file-uploads in golang +type Service struct { + StorageType string `toml:"storage_type"` + Path string `toml:"path"` +} + +// Check if Service is configurated and useable +func (s *Service) Check() error { + mgmt, ok := managers[s.StorageType] + if !ok { + return ErrUnsupportedStorageType + } + return mgmt.Check(s) +} + +// Upload a file to storage +func (s *Service) Upload(request *http.Request) (*File, error) { + mgmt, ok := managers[s.StorageType] + if !ok { + return nil, ErrUnsupportedStorageType + } + file, fileRequest, err := request.FormFile("file") + if err != nil { + return nil, err + } + fileObj := File{ + Filename: filepath.Base(fileRequest.Filename), + } + + // detect contenttype + buffer := make([]byte, 512) + n, err := file.Read(buffer) + if err != nil && err != io.EOF { + return nil, err + } + fileObj.ContentType = http.DetectContentType(buffer[:n]) + + // Reset the read pointer + file.Seek(0, io.SeekStart) + if err := mgmt.Save(s, &fileObj, file); err != nil { + return nil, err + } + return &fileObj, nil + +} + +// GINUpload a file to storage using gin-gonic +func (s *Service) GINUpload(c *gin.Context) (*File, error) { + return s.Upload(c.Request) +} + +// Read a file to storage +func (s *Service) Read(file *File) (io.Reader, error) { + mgmt, ok := managers[s.StorageType] + if !ok { + return nil, ErrUnsupportedStorageType + } + return mgmt.Read(s, file) +} diff --git a/web/request.go b/web/request.go index 6d45ddb..d833277 100644 --- a/web/request.go +++ b/web/request.go @@ -1,8 +1,13 @@ package web import ( + "bytes" "encoding/json" + "io" + "mime/multipart" "net/http" + "os" + "path/filepath" "time" ) @@ -26,3 +31,40 @@ func JSONRequest(url string, value interface{}) error { } return nil } + +// NewRequestWithFile Create a Request with file as body +func NewRequestWithFile(url, filename string) (*http.Request, error) { + buf := bytes.NewBuffer(nil) + bodyWriter := multipart.NewWriter(buf) + + // We need to truncate the input filename, as the server might be stupid and take the input + // filename verbatim. Then, he will have directory parts which do not exist on the server. + fileWriter, err := bodyWriter.CreateFormFile("file", filepath.Base(filename)) + if err != nil { + return nil, err + } + + file, err := os.Open(filename) + if err != nil { + return nil, err + } + defer file.Close() + + if _, err := io.Copy(fileWriter, file); err != nil { + return nil, err + } + + // We have all the data written to the bodyWriter. + // Now we can infer the content type + contentType := bodyWriter.FormDataContentType() + + // This is mandatory as it flushes the buffer. + bodyWriter.Close() + + req, err := http.NewRequest(http.MethodPost, url, buf) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", contentType) + return req, nil +}