test(web/file/fs): check Size on fileinfo

web/file/createfs.go

@@ -0,0 +1,57 @@
+package file
+
+import (
+	"dev.sum7.eu/genofire/golang-lib/web/file/fs"
+	"dev.sum7.eu/genofire/golang-lib/web/file/s3"
+)
+
+// fsType represents a type of file store.
+type fsType int
+
+const (
+	typeFS fsType = iota
+	typeS3
+)
+
+var stringToType = map[string]fsType{
+	"fs": typeFS,
+	"s3": typeS3,
+}
+
+func (t *fsType) UnmarshalText(input []byte) error {
+	val, ok := stringToType[string(input)]
+	if !ok {
+		return ErrInvalidFSType
+	}
+	*t = val
+	return nil
+}
+
+// FSInfo is a TOML structure storing access information about a file store.
+type FSInfo struct {
+	FSType fsType `toml:"type"`
+	// file system
+	Root string `toml:",omitempty"`
+	// s3
+	Endpoint string `toml:",omitempty"`
+	Secure   bool   `toml:",omitempty"`
+	ID       string `toml:",omitempty"`
+	Secret   string `toml:",omitempty"`
+	Bucket   string `toml:",omitempty"`
+	Location string `toml:",omitempty"`
+}
+
+// Create creates a file store from the information provided.
+func (i *FSInfo) Create() (FS, error) {
+	switch i.FSType {
+	case typeFS:
+		if len(i.Root) == 0 {
+			return nil, ErrNoFSRoot
+		}
+		return &fs.FS{Root: i.Root}, nil
+	case typeS3:
+		return s3.New(i.Endpoint, i.Secure, i.ID, i.Secret, i.Bucket, i.Location)
+	default:
+		return nil, ErrNotImplementedFSType
+	}
+}

web/file/createfs_test.go

@@ -0,0 +1,62 @@
+package file_test
+
+import (
+	"testing"
+
+	fsfile "dev.sum7.eu/genofire/golang-lib/file"
+	"dev.sum7.eu/genofire/golang-lib/web/file"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCreateFSOK(t *testing.T) {
+	assert := assert.New(t)
+
+	config := file.FSInfo{}
+	err := fsfile.ReadTOML("testdata/createfs_fs.toml", &config)
+	assert.NoError(err)
+
+	fs, err := config.Create()
+	assert.NoError(err)
+	assert.NoError(fs.Check())
+}
+
+func TestCreateS3(t *testing.T) {
+	assert := assert.New(t)
+
+	config := file.FSInfo{}
+	err := fsfile.ReadTOML("testdata/createfs_s3.toml", &config)
+	assert.NoError(err)
+
+	fs, err := config.Create()
+	assert.NoError(err)
+	assert.NoError(fs.Check())
+}
+
+func TestCreateFSNotOK(t *testing.T) {
+	assert := assert.New(t)
+
+	config := file.FSInfo{}
+	err := fsfile.ReadTOML("testdata/createfs_fsnone.toml", &config)
+	assert.NoError(err)
+
+	_, err = config.Create()
+	assert.ErrorIs(err, file.ErrNoFSRoot)
+}
+
+func TestCreateFSNone(t *testing.T) {
+	assert := assert.New(t)
+
+	config := file.FSInfo{}
+	err := fsfile.ReadTOML("testdata/createfs_none.toml", &config)
+
+	// https://github.com/naoina/toml/pull/51
+	assert.Contains(err.Error(), file.ErrInvalidFSType.Error())
+}
+
+func TestCreateFSInvalid(t *testing.T) {
+	assert := assert.New(t)
+
+	config := file.FSInfo{}
+	_, err := config.Create()
+	assert.ErrorIs(err, file.ErrNoFSRoot)
+}

web/file/error.go

@@ -2,7 +2,9 @@ package file
 
 import "errors"
 
-// Error Messages
+// errors
 var (
-	ErrUnsupportedStorageType = errors.New("storage type invalid")
+	ErrInvalidFSType        = errors.New("invalid file store type")
+	ErrNoFSRoot             = errors.New("no file store root")
+	ErrNotImplementedFSType = errors.New("FSInfo.Create not implemented for provided file store type")
 )

web/file/file.go

@@ -1,12 +0,0 @@
-package file
-
-import "github.com/google/uuid"
-
-// File to store information in database
-type File struct {
-	ID          uuid.UUID `json:"id" gorm:"type:uuid;default:gen_random_uuid()" example:"32466d63-efa4-4f27-9f2b-a1f06c8e2e1d"`
-	StorageType string    `json:"storage_type,omitempty"`
-	Path        string    `json:"path,omitempty"`
-	Filename    string    `json:"filename"`
-	ContentType string    `json:"content-type"`
-}

web/file/fileinfo.go

@@ -0,0 +1,13 @@
+package file
+
+import (
+	"io/fs"
+
+	"github.com/google/uuid"
+)
+
+type FileInfo interface {
+	fs.FileInfo
+	ID() uuid.UUID
+	ContentType() string
+}

web/file/fs.go

@@ -0,0 +1,69 @@
+/*
+Package file abstracts non-hierarchical file stores. Each file consists of a
+name, a MIME type, a UUID, and data. File names may be duplicate.
+
+TODO: think about name vs. UUID again—should the ``name'' really be the filename
+and not the UUID?
+*/
+package file
+
+import (
+	"fmt"
+	"io"
+	"io/fs"
+	"net/http"
+	"strings"
+
+	"github.com/google/uuid"
+)
+
+// An FS is a file store.
+type FS interface {
+	// Store stores a new file with the given UUID, name, and MIME type.
+	// Its data is taken from the provided Reader. If it encounters an
+	// error, it does nothing. Any existing file with the same UUID is
+	// overwritten.
+	Store(id uuid.UUID, name, contentType string, data io.Reader) error
+	// RemoveUUID deletes a file.
+	RemoveUUID(id uuid.UUID) error
+	// Open opens a file by its name. If multiple files have the same name,
+	// it is unspecified which one is opened. This may very well be very
+	// slow. This is bad. Go away.
+	Open(name string) (fs.File, error)
+	// OpenUUID opens a file by its UUID.
+	OpenUUID(id uuid.UUID) (fs.File, error)
+	// Check checks the health of the file store. If the file store is not
+	// healthy, it returns a descriptive error. Otherwise, the file store
+	// should be usable.
+	Check() error
+}
+
+// StoreFromHTTP stores the first file with given form key from an HTTP
+// multipart/form-data request. Its Content-Type header is ignored; the type is
+// detected. The file name is the last part of the provided file name not
+// containing any slashes or backslashes.
+//
+// TODO: store all files with the given key instead of just the first one
+func StoreFromHTTP(fs FS, r *http.Request, key string) error {
+	file, fileHeader, err := r.FormFile(key)
+	if err != nil {
+		return fmt.Errorf("get file from request: %w", err)
+	}
+	defer file.Close()
+
+	buf := make([]byte, 512)
+	n, err := file.Read(buf)
+	if err != nil && err != io.EOF {
+		return fmt.Errorf("read from file: %w", err)
+	}
+	contentType := http.DetectContentType(buf[:n])
+	_, err = file.Seek(0, io.SeekStart)
+	if err != nil {
+		return fmt.Errorf("seek in file: %w", err)
+	}
+
+	i := strings.LastIndexAny(fileHeader.Filename, "/\\")
+	// if i == -1 { i = -1 }
+
+	return fs.Store(uuid.New(), fileHeader.Filename[i+1:], contentType, file)
+}

web/file/fs/file.go

@@ -0,0 +1,37 @@
+package fs
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"path"
+
+	"github.com/google/uuid"
+)
+
+// File is a file handle with its associated ID.
+type File struct {
+	*os.File
+	fs *FS
+	id uuid.UUID
+}
+
+func (f File) Stat() (fs.FileInfo, error) {
+	fi := FileInfo{id: f.id}
+	var err error
+	fi.FileInfo, err = f.File.Stat()
+	if err != nil {
+		return nil, fmt.Errorf("os stat: %w", err)
+	}
+	name, err := os.ReadFile(path.Join(f.fs.Root, f.id.String(), "name"))
+	if err != nil {
+		return nil, fmt.Errorf("reading name: %w", err)
+	}
+	fi.name = string(name)
+	contentType, err := os.ReadFile(path.Join(f.fs.Root, f.id.String(), "type"))
+	if err != nil {
+		return nil, fmt.Errorf("reading type: %w", err)
+	}
+	fi.contentType = string(contentType)
+	return fi, nil
+}

web/file/fs/fileinfo.go

@@ -0,0 +1,19 @@
+package fs
+
+import (
+	"os"
+
+	"github.com/google/uuid"
+)
+
+type FileInfo struct {
+	id          uuid.UUID
+	name        string
+	contentType string
+	os.FileInfo
+}
+
+func (fi FileInfo) ID() uuid.UUID       { return fi.id }
+func (fi FileInfo) ContentType() string { return fi.contentType }
+func (fi FileInfo) Name() string        { return fi.name }
+func (fi FileInfo) Sys() interface{}    { return fi.FileInfo }

web/file/fs/fs.go

@@ -0,0 +1,123 @@
+/*
+Package fs implements a non-hierarchical file store using the underlying (disk)
+file system.
+
+A file store contains directories named after UUIDs, each containing the
+following files:
+    name    the file name (``basename'')
+    type    the file's MIME type
+    data    the file data
+
+This is not meant for anything for which the word ``scale'' plays any role at
+all, ever, anywhere.
+
+TODO: should io/fs ever support writable file systems, use one of those instead
+of the ``disk'' (os.Open & Co.) (see https://github.com/golang/go/issues/45757)
+*/
+package fs
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path"
+
+	"github.com/google/uuid"
+)
+
+// FS is an on-disk file store.
+type FS struct {
+	// Root is the path of the file store, relative to the process's working
+	// directory or absolute.
+	Root string
+}
+
+func (fs *FS) Store(id uuid.UUID, name, contentType string, data io.Reader) error {
+	p := path.Join(fs.Root, id.String())
+
+	_, err := os.Stat(p)
+	if err == nil {
+		err = os.RemoveAll(p)
+		if err != nil {
+			return fmt.Errorf("remove old %v: %w", id, err)
+		}
+	}
+
+	err = os.Mkdir(p, 0o750)
+	if err != nil {
+		return fmt.Errorf("mkdir %s: %w", id.String(), err)
+	}
+
+	defer func() {
+		if err != nil {
+			os.RemoveAll(p)
+		}
+	}()
+
+	err = os.WriteFile(path.Join(p, "name"), []byte(name), 0o640)
+	if err != nil {
+		return fmt.Errorf("write name: %w", err)
+	}
+	err = os.WriteFile(path.Join(p, "type"), []byte(contentType), 0o640)
+	if err != nil {
+		return fmt.Errorf("write type: %w", err)
+	}
+	f, err := os.Create(path.Join(p, "data"))
+	if err != nil {
+		return fmt.Errorf("create data file: %w", err)
+	}
+	_, err = io.Copy(f, data)
+	if err != nil {
+		return fmt.Errorf("write data: %w", err)
+	}
+	return nil
+}
+
+func (fs *FS) RemoveUUID(id uuid.UUID) error {
+	return os.RemoveAll(path.Join(fs.Root, id.String()))
+}
+
+// OpenUUID opens the file with the given UUID.
+func (fs *FS) OpenUUID(id uuid.UUID) (fs.File, error) {
+	f, err := os.Open(path.Join(fs.Root, id.String(), "data"))
+	if err != nil {
+		return nil, fmt.Errorf("open data: %w", err)
+	}
+	return File{File: f, fs: fs, id: id}, nil
+}
+
+// Open searches for and opens the file with the given name.
+func (fs *FS) Open(name string) (fs.File, error) {
+	files, err := os.ReadDir(fs.Root)
+	if err != nil {
+		return nil, err
+	}
+	for _, v := range files {
+		id := v.Name()
+		entryName, err := os.ReadFile(path.Join(fs.Root, id, "name"))
+		if err != nil || string(entryName) != name {
+			continue
+		}
+		f, err := os.Open(path.Join(fs.Root, id, "data"))
+		if err != nil {
+			return nil, fmt.Errorf("open data: %w", err)
+		}
+		return File{File: f, fs: fs, id: uuid.MustParse(id)}, nil
+	}
+	return nil, errors.New("no such file")
+}
+
+// Check checks whether the file store is operable, ie. whether fs.Root exists
+// and is a directory.
+func (fs *FS) Check() error {
+	fi, err := os.Stat(fs.Root)
+	if err != nil {
+		return err
+	}
+	if !fi.IsDir() {
+		return errors.New("file store is not a directory")
+	}
+	return nil
+}

web/file/fs/fs_test.go

@@ -0,0 +1,52 @@
+package fs_test
+
+import (
+	"io"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+
+	"dev.sum7.eu/genofire/golang-lib/web/file"
+	"dev.sum7.eu/genofire/golang-lib/web/file/fs"
+)
+
+func TestOpenStat(t *testing.T) {
+	assert := assert.New(t)
+	var fs file.FS = &fs.FS{Root: "./testdata"}
+	assert.NoError(fs.Check())
+
+	f, err := fs.Open("glenda")
+	assert.NoError(err)
+	assert.NotNil(f)
+
+	fi, err := f.Stat()
+	assert.NoError(err)
+	assert.NotNil(fi)
+
+	assert.Equal(uuid.MustParse("d2750ced-4bdc-41d0-8c2f-5b9de44b84ef"), fi.(file.FileInfo).ID())
+	assert.Equal("text/plain", fi.(file.FileInfo).ContentType())
+	assert.Equal("glenda", fi.Name())
+	assert.Equal(int64(99), fi.Size())
+}
+
+func TestCreateOpenUUIDRead(t *testing.T) {
+	assert := assert.New(t)
+	var fs file.FS = &fs.FS{Root: "./testdata"}
+	assert.NoError(fs.Check())
+
+	err := fs.Store(uuid.MustParse("f9375ccb-ee09-4ecf-917e-b88725efcb68"), "$name", "text/plain", strings.NewReader("hello, world\n"))
+	assert.NoError(err)
+
+	f, err := fs.OpenUUID(uuid.MustParse("f9375ccb-ee09-4ecf-917e-b88725efcb68"))
+	assert.NoError(err)
+	assert.NotNil(f)
+
+	buf, err := io.ReadAll(f)
+	assert.NoError(err)
+	assert.Equal("hello, world\n", string(buf))
+
+	os.RemoveAll("./testdata/f9375ccb-ee09-4ecf-917e-b88725efcb68")
+}

web/file/fs/main.go

@@ -1,63 +0,0 @@
-package fs
-
-import (
-	"errors"
-	"io"
-	"mime/multipart"
-	"os"
-	"path"
-
-	"github.com/google/uuid"
-
-	"dev.sum7.eu/genofire/golang-lib/web/file"
-)
-
-// consts for filemanager
-const (
-	StorageTypeFS = "fs"
-)
-
-// error messages
-var (
-	ErrPathNotExistsOrNoDirectory = errors.New("path invalid: not exists or not an directory")
-)
-
-// FileManager to handle data on disk
-type FileManager struct {
-}
-
-// Check if filemanager could be used
-func (m *FileManager) Check(s *file.Service) error {
-	info, err := os.Stat(s.Path)
-	if os.IsNotExist(err) || !info.IsDir() {
-		return ErrPathNotExistsOrNoDirectory
-	}
-	return nil
-}
-
-// Save a file on disk and update file db
-func (m *FileManager) Save(s *file.Service, file *file.File, src multipart.File) error {
-	file.ID = uuid.New()
-	file.Path = path.Join(file.ID.String(), file.Filename)
-
-	directory := path.Join(s.Path, file.ID.String())
-	os.Mkdir(directory, 0750)
-
-	out, err := os.Create(path.Join(s.Path, file.Path))
-	if err != nil {
-		return err
-	}
-	defer out.Close()
-
-	_, err = io.Copy(out, src)
-	return err
-}
-
-// Read get an reader of an file
-func (m *FileManager) Read(s *file.Service, file *file.File) (io.Reader, error) {
-	return os.Open(path.Join(s.Path, file.Path))
-}
-
-func init() {
-	file.AddManager(StorageTypeFS, &FileManager{})
-}

web/file/fs/main_test.go

@@ -1,92 +0,0 @@
-package fs
-
-import (
-	"io"
-	"os"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	"dev.sum7.eu/genofire/golang-lib/web"
-	"dev.sum7.eu/genofire/golang-lib/web/file"
-)
-
-func TestCheck(t *testing.T) {
-	assert := assert.New(t)
-
-	service := file.Service{
-		StorageType: StorageTypeFS,
-		Path:        "./test",
-	}
-
-	assert.NoError(service.Check())
-
-	service.StorageType = "s3"
-	assert.ErrorIs(file.ErrUnsupportedStorageType, service.Check())
-
-	service.StorageType = StorageTypeFS
-	service.Path = "./main_test.go"
-	assert.ErrorIs(ErrPathNotExistsOrNoDirectory, service.Check())
-
-	/* TODO no write permission
-	service.Path = "/dev"
-	assert.ErrorIs(ErrPathNotExistsOrNoDirectory, service.Check())
-	*/
-}
-
-func TestSave(t *testing.T) {
-	assert := assert.New(t)
-
-	service := file.Service{
-		StorageType: "s3",
-		Path:        "./test",
-	}
-
-	_, err := service.Upload(nil)
-	assert.ErrorIs(file.ErrUnsupportedStorageType, err)
-
-	service.StorageType = StorageTypeFS
-	req, err := web.NewRequestWithFile("localhost", "./test/00000000-0000-0000-0000-000000000000/a.txt")
-	assert.NoError(err)
-	assert.NotNil(req)
-
-	_, err = service.Upload(req)
-	assert.NoError(err)
-
-	service.Path = "/dev"
-	_, err = service.Upload(req)
-	assert.True(os.IsNotExist(err))
-	//assert.True(os.IsPermission(err))
-
-	// TODO no write permission
-}
-
-func TestRead(t *testing.T) {
-	assert := assert.New(t)
-
-	service := file.Service{
-		StorageType: "s3",
-		Path:        "./test",
-	}
-
-	_, err := service.Read(nil)
-	assert.ErrorIs(file.ErrUnsupportedStorageType, err)
-
-	service.StorageType = StorageTypeFS
-
-	file := &file.File{
-		Path: "00000000-0000-0000-0000-000000000000/a.txt",
-	}
-	r, err := service.Read(file)
-	assert.NoError(err)
-	buf := &strings.Builder{}
-	_, err = io.Copy(buf, r)
-	assert.Equal("Hello world\n", buf.String())
-
-	service.Path = "/dev"
-	_, err = service.Read(file)
-	assert.True(os.IsNotExist(err))
-
-	// TODO no write permission
-}

web/file/fs/test/00000000-0000-0000-0000-000000000000/a.txt

@@ -1 +0,0 @@
-Hello world

web/file/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/data

@@ -0,0 +1,8 @@
+Small ‘hand-drwawn’ Glenda by ems:
+
+  (\(\
+  ¸". ..
+  (  . .)
+  |   ° ¡
+  ¿     ;
+ c?".UJ"

web/file/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/name

@@ -0,0 +1 @@
+glenda

web/file/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/type

@@ -0,0 +1 @@
+text/plain

web/file/fs_test.go

@@ -0,0 +1,111 @@
+package file_test
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"mime"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+
+	"dev.sum7.eu/genofire/golang-lib/web/file"
+)
+
+type TestFS struct {
+	assert      *assert.Assertions
+	filename    string
+	data        string
+	contentType string
+}
+
+func (f TestFS) Store(id uuid.UUID, name, contentType string, data io.Reader) error {
+	f.assert.Equal(f.filename, name)
+	dat, err := io.ReadAll(data)
+	f.assert.NoError(err)
+	f.assert.Equal(f.data, string(dat))
+	contentType, _, err = mime.ParseMediaType(contentType)
+	f.assert.NoError(err)
+	f.assert.Equal(f.contentType, contentType)
+	return nil
+}
+
+func (f TestFS) RemoveUUID(id uuid.UUID) error {
+	return errors.New("TestFS.RemoveUUID called")
+}
+
+func (f TestFS) Open(name string) (fs.File, error) {
+	return nil, errors.New("TestFS.Open called")
+}
+
+func (f TestFS) OpenUUID(uuid.UUID) (fs.File, error) {
+	return nil, errors.New("TestFS.OpenUUID called")
+}
+
+func (f TestFS) Check() error { return nil }
+
+func TestStoreFromHTTP(t *testing.T) {
+	assert := assert.New(t)
+
+	testfs := TestFS{
+		assert:      assert,
+		filename:    "cute-cat.png",
+		data:        "content\nof file",
+		contentType: "text/plain",
+	}
+
+	r, w := io.Pipe()
+	m := multipart.NewWriter(w)
+	rq := httptest.NewRequest("PUT", "/", r)
+	rq.Header.Set("Content-Type", m.FormDataContentType())
+	go func() {
+		f, err := m.CreateFormFile("file", testfs.filename)
+		assert.NoError(err)
+		_, err = f.Write([]byte(testfs.data))
+		assert.NoError(err)
+		m.Close()
+	}()
+
+	assert.NoError(file.StoreFromHTTP(testfs, rq, "file"))
+}
+
+var fstore file.FS
+
+func ExampleFS() {
+	// generate the UUID for the new file
+	id := uuid.New()
+
+	// store a file
+	{
+		f, _ := os.Open("glenda.png")
+		fstore.Store(id, "glenda.png", "image/png", f)
+		f.Close()
+	}
+
+	// copy back to a local file
+	{
+		r, _ := fstore.OpenUUID(id)
+		w, _ := os.Create("glenda.png")
+		io.Copy(w, r)
+		r.Close()
+		w.Close()
+	}
+}
+
+func ExampleStoreFromHTTP() {
+	http.HandleFunc("/upload", func(w http.ResponseWriter, r *http.Request) {
+		if err := file.StoreFromHTTP(fstore, r, "file"); err != nil {
+			w.Header().Set("content-type", "application/json")
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(fmt.Sprintf(`{"message":"%v"}`, err)))
+		} else {
+			w.WriteHeader(http.StatusOK)
+		}
+	})
+}

web/file/main_test.go

@@ -1,98 +0,0 @@
-package file
-
-import (
-	"bytes"
-	"errors"
-	"io"
-	"mime/multipart"
-	"net/http"
-	"strings"
-	"testing"
-
-	"dev.sum7.eu/genofire/golang-lib/web"
-	"github.com/gin-gonic/gin"
-	"github.com/stretchr/testify/assert"
-)
-
-const storageTypeDummy = "dummy"
-
-type dummyManager struct {
-}
-
-func (m *dummyManager) Check(s *Service) error {
-	return nil
-}
-func (m *dummyManager) Save(s *Service, file *File, src multipart.File) error {
-	if src == nil {
-		return errors.New("nothing to fill")
-	}
-	return nil
-}
-func (m *dummyManager) Read(s *Service, file *File) (io.Reader, error) {
-	b := bytes.Buffer{}
-	b.WriteString("Hello world\n")
-	return &b, nil
-}
-
-func init() {
-	AddManager(storageTypeDummy, &dummyManager{})
-}
-
-func TestCheck(t *testing.T) {
-	assert := assert.New(t)
-
-	service := Service{
-		StorageType: storageTypeDummy,
-		Path:        "./fs/test",
-	}
-	assert.NoError(service.Check())
-
-	service.StorageType = "s3"
-	assert.ErrorIs(ErrUnsupportedStorageType, service.Check())
-}
-
-func TestSave(t *testing.T) {
-	assert := assert.New(t)
-
-	service := Service{
-		StorageType: "fs",
-		Path:        "./fs/test",
-	}
-
-	_, err := service.Upload(nil)
-	assert.ErrorIs(ErrUnsupportedStorageType, err)
-
-	service.StorageType = storageTypeDummy
-	_, err = service.GINUpload(&gin.Context{Request: &http.Request{}})
-	assert.ErrorIs(err, http.ErrNotMultipart)
-
-	req, err := web.NewRequestWithFile("http://localhost/upload", "./fs/test/00000000-0000-0000-0000-000000000000/a.txt")
-	assert.NoError(err)
-	assert.NotNil(req)
-
-	_, err = service.Upload(req)
-	assert.NoError(err)
-}
-
-func TestRead(t *testing.T) {
-	assert := assert.New(t)
-
-	service := Service{
-		StorageType: "fs",
-		Path:        "./fs/test",
-	}
-
-	_, err := service.Read(nil)
-	assert.ErrorIs(ErrUnsupportedStorageType, err)
-
-	service.StorageType = "dummy"
-
-	file := &File{
-		Path: "00000000-0000-0000-0000-000000000000/a.txt",
-	}
-	r, err := service.Read(file)
-	assert.NoError(err)
-	buf := &strings.Builder{}
-	_, err = io.Copy(buf, r)
-	assert.Equal("Hello world\n", buf.String())
-}

web/file/manager.go

@@ -1,20 +0,0 @@
-package file
-
-import (
-	"io"
-	"mime/multipart"
-)
-
-type FileManager interface {
-	Check(s *Service) error
-	Save(s *Service, fileObj *File, file multipart.File) error
-	Read(s *Service, fileObj *File) (io.Reader, error)
-}
-
-var (
-	managers = make(map[string]FileManager)
-)
-
-func AddManager(typ string, m FileManager) {
-	managers[typ] = m
-}

web/file/s3/address.go

@@ -1,57 +0,0 @@
-package s3
-
-import (
-	"context"
-	"errors"
-	"net/url"
-
-	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
-)
-
-// Error Messages during connect
-var (
-	ErrNoPassword = errors.New("no secret access key found")
-)
-
-// Connect try to use a path to setup a connection to s3 server
-func Connect(path string) (*minio.Client, string, error) {
-	u, err := url.Parse(path)
-	if err != nil {
-		return nil, "", err
-	}
-
-	tls := u.Scheme[len(u.Scheme)-1] == 's'
-	accessKeyID := u.User.Username()
-	secretAccessKey, ok := u.User.Password()
-	if !ok {
-		return nil, "", ErrNoPassword
-	}
-	query := u.Query()
-	bucketName := query.Get("bucket")
-	location := query.Get("location")
-
-	u.User = nil
-	u.RawQuery = ""
-
-	ctx := context.Background()
-
-	// Initialize minio client object.
-	minioClient, err := minio.New(u.String(), &minio.Options{
-		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
-		Secure: tls,
-	})
-	if err != nil {
-		return nil, "", err
-	}
-
-	// create and check for bucket
-	err = minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: location})
-	if err != nil {
-		if exists, err := minioClient.BucketExists(ctx, bucketName); err != nil || !exists {
-			return nil, "", err
-		}
-	}
-
-	return minioClient, bucketName, err
-}

web/file/s3/file.go

@@ -0,0 +1,21 @@
+package s3
+
+import (
+	"io/fs"
+
+	"github.com/minio/minio-go/v7"
+)
+
+type File struct {
+	*minio.Object
+}
+
+func (f File) Stat() (fs.FileInfo, error) {
+	var fi FileInfo
+	var err error
+	fi.ObjectInfo, err = f.Object.Stat()
+	if err != nil {
+		return nil, err
+	}
+	return fi, nil
+}

web/file/s3/fileinfo.go

@@ -0,0 +1,24 @@
+package s3
+
+import (
+	"io/fs"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/minio/minio-go/v7"
+)
+
+type FileInfo struct {
+	minio.ObjectInfo
+}
+
+func (fi FileInfo) Name() string { return fi.UserMetadata["filename"] }
+func (fi FileInfo) Size() int64  { return fi.ObjectInfo.Size }
+
+// TODO: try to map s3 permissions to these, somehow
+func (fi FileInfo) Mode() fs.FileMode   { return 0o640 }
+func (fi FileInfo) ModTime() time.Time  { return fi.LastModified }
+func (fi FileInfo) IsDir() bool         { return false }
+func (fi FileInfo) Sys() interface{}    { return fi.ObjectInfo }
+func (fi FileInfo) ID() uuid.UUID       { return uuid.MustParse(fi.Key) }
+func (fi FileInfo) ContentType() string { return fi.ObjectInfo.ContentType }

web/file/s3/fs.go

@@ -0,0 +1,87 @@
+/*
+Package s3 implements a non-hierarchical file store using Amazon s3. A file
+store uses a single bucket. Each file is an object with its UUID as the name.
+The file name is stored in the user-defined object metadata x-amz-meta-filename.
+*/
+package s3
+
+import (
+	"context"
+	"errors"
+	"io"
+	"io/fs"
+
+	"github.com/google/uuid"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+type FS struct {
+	client *minio.Client
+	bucket string
+}
+
+// New ``connects'' to an s3 endpoint and creates a file store using the
+// specified bucket. The bucket is created if it doesn't exist.
+func New(endpoint string, secure bool, id, secret, bucket, location string) (*FS, error) {
+	var fs FS
+	var err error
+	fs.client, err = minio.New(endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(id, secret, ""),
+		Secure: secure,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	ctx := context.TODO()
+
+	err = fs.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{
+		Region: location,
+	})
+	if err != nil {
+		if exists, err := fs.client.BucketExists(ctx, bucket); err != nil || !exists {
+			return nil, err
+		}
+	}
+	fs.bucket = bucket
+
+	return &fs, nil
+}
+
+func (fs *FS) Store(id uuid.UUID, name, contentType string, data io.Reader) error {
+	ctx := context.TODO()
+	_, err := fs.client.PutObject(ctx, fs.bucket, id.String(), data, -1, minio.PutObjectOptions{
+		UserMetadata: map[string]string{
+			"filename": name,
+		},
+		ContentType: contentType,
+	})
+	return err
+}
+
+func (fs *FS) RemoveUUID(id uuid.UUID) error {
+	ctx := context.TODO()
+	return fs.client.RemoveObject(ctx, fs.bucket, id.String(), minio.RemoveObjectOptions{})
+}
+
+// TODO: implement
+func (fs *FS) Open(name string) (fs.File, error) {
+	return nil, errors.New("not implemented")
+}
+
+func (fs *FS) OpenUUID(id uuid.UUID) (fs.File, error) {
+	ctx := context.TODO()
+
+	object, err := fs.client.GetObject(ctx, fs.bucket, id.String(), minio.GetObjectOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	return File{Object: object}, nil
+}
+
+// TODO: do some checking
+func (fs *FS) Check() error {
+	return nil
+}

web/file/s3/fs_test.go

@@ -0,0 +1,25 @@
+package s3_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"dev.sum7.eu/genofire/golang-lib/web/file"
+	"dev.sum7.eu/genofire/golang-lib/web/file/s3"
+)
+
+// TODO: actually test, either using little dummies or using sth like play.min.io
+
+func TestTypes(t *testing.T) {
+	assert := assert.New(t)
+
+	var fstore file.FS
+	fstore, err := s3.New("127.0.0.1", false, "", "", "", "")
+	_ = fstore
+	assert.Error(err)
+}
+
+func ExampleNew() {
+	s3.New("play.min.io", true, "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", "file-store", "")
+}

web/file/service.go

@@ -1,69 +0,0 @@
-package file
-
-import (
-	"io"
-	"net/http"
-	"path/filepath"
-
-	"github.com/gin-gonic/gin"
-)
-
-// A Service to handle file-uploads in golang
-type Service struct {
-	StorageType string `toml:"storage_type"`
-	Path        string `toml:"path"`
-}
-
-// Check if Service is configurated and useable
-func (s *Service) Check() error {
-	mgmt, ok := managers[s.StorageType]
-	if !ok {
-		return ErrUnsupportedStorageType
-	}
-	return mgmt.Check(s)
-}
-
-// Upload a file to storage
-func (s *Service) Upload(request *http.Request) (*File, error) {
-	mgmt, ok := managers[s.StorageType]
-	if !ok {
-		return nil, ErrUnsupportedStorageType
-	}
-	file, fileRequest, err := request.FormFile("file")
-	if err != nil {
-		return nil, err
-	}
-	fileObj := File{
-		Filename: filepath.Base(fileRequest.Filename),
-	}
-
-	// detect contenttype
-	buffer := make([]byte, 512)
-	n, err := file.Read(buffer)
-	if err != nil && err != io.EOF {
-		return nil, err
-	}
-	fileObj.ContentType = http.DetectContentType(buffer[:n])
-
-	// Reset the read pointer
-	file.Seek(0, io.SeekStart)
-	if err := mgmt.Save(s, &fileObj, file); err != nil {
-		return nil, err
-	}
-	return &fileObj, nil
-
-}
-
-// GINUpload a file to storage using gin-gonic
-func (s *Service) GINUpload(c *gin.Context) (*File, error) {
-	return s.Upload(c.Request)
-}
-
-// Read a file to storage
-func (s *Service) Read(file *File) (io.Reader, error) {
-	mgmt, ok := managers[s.StorageType]
-	if !ok {
-		return nil, ErrUnsupportedStorageType
-	}
-	return mgmt.Read(s, file)
-}

web/file/testdata/createfs_fs.toml

@@ -0,0 +1,2 @@
+type = "fs"
+root = "fs/testdata"

web/file/testdata/createfs_fsnone.toml

@@ -0,0 +1,2 @@
+type = "fs"
+root = ""

web/file/testdata/createfs_none.toml

@@ -0,0 +1 @@
+type = "notsupported"

web/file/testdata/createfs_s3.toml

@@ -0,0 +1,7 @@
+type = "s3"
+endpoint = "play.min.io"
+secure = true
+id = "Q3AM3UQ867SPQQA43P2F"
+secret = "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
+bucket = "file-store"
+location = ""