From b5a323aeb4006a10af90b39b43dbcefc1e30cafc Mon Sep 17 00:00:00 2001 From: Lennart Date: Mon, 26 Jul 2021 17:48:20 +0200 Subject: [PATCH] add web/file2 --- web/file2/fileinfo.go | 13 ++ web/file2/fs.go | 65 +++++++++ web/file2/fs/file.go | 37 ++++++ web/file2/fs/fileinfo.go | 19 +++ web/file2/fs/fs.go | 123 ++++++++++++++++++ web/file2/fs/fs_test.go | 50 +++++++ .../d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/data | 8 ++ .../d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/name | 1 + .../d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/type | 1 + web/file2/fs_test.go | 47 +++++++ web/file2/s3/file.go | 21 +++ web/file2/s3/fileinfo.go | 24 ++++ web/file2/s3/fs.go | 87 +++++++++++++ web/file2/s3/fs_test.go | 24 ++++ 14 files changed, 520 insertions(+) create mode 100644 web/file2/fileinfo.go create mode 100644 web/file2/fs.go create mode 100644 web/file2/fs/file.go create mode 100644 web/file2/fs/fileinfo.go create mode 100644 web/file2/fs/fs.go create mode 100644 web/file2/fs/fs_test.go create mode 100644 web/file2/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/data create mode 100644 web/file2/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/name create mode 100644 web/file2/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/type create mode 100644 web/file2/fs_test.go create mode 100644 web/file2/s3/file.go create mode 100644 web/file2/s3/fileinfo.go create mode 100644 web/file2/s3/fs.go create mode 100644 web/file2/s3/fs_test.go diff --git a/web/file2/fileinfo.go b/web/file2/fileinfo.go new file mode 100644 index 0000000..0cce29f --- /dev/null +++ b/web/file2/fileinfo.go @@ -0,0 +1,13 @@ +package file + +import ( + "io/fs" + + "github.com/google/uuid" +) + +type FileInfo interface { + fs.FileInfo + ID() uuid.UUID + ContentType() string +} diff --git a/web/file2/fs.go b/web/file2/fs.go new file mode 100644 index 0000000..aeb0370 --- /dev/null +++ b/web/file2/fs.go @@ -0,0 +1,65 @@ +/* +Package file abstracts non-hierarchical file stores. Each file consists of a +name, a MIME type, a UUID, and data. File names may be duplicate. + +TODO: think about name vs. UUID again—should the ``name'' really be the filename +and not the UUID? +*/ +package file + +import ( + "fmt" + "io" + "io/fs" + "net/http" + "strings" + + "github.com/google/uuid" +) + +// An FS is a file store. +type FS interface { + // Store stores a new file with the given UUID, name, and MIME type. + // Its data is taken from the provided Reader. If it encounters an + // error, it does nothing. Any existing file with the same UUID is + // overwritten. + Store(id uuid.UUID, name, contentType string, data io.Reader) error + // RemoveUUID deletes a file. + RemoveUUID(id uuid.UUID) error + // Open opens a file by its name. If multiple files have the same name, + // it is unspecified which one is opened. This may very well be very + // slow. This is bad. Go away. + Open(name string) (fs.File, error) + // OpenUUID opens a file by its UUID. + OpenUUID(id uuid.UUID) (fs.File, error) + // Check checks the health of the file store. If the file store is not + // healthy, it returns a descriptive error. Otherwise, the file store + // should be usable. + Check() error +} + +// StoreFromHTTP stores the first file with given form key from an HTTP +// multipart/form-data request. Its Content-Type header is ignored; the type is +// detected. The file name is the last part of the provided file name not +// containing any slashes or backslashes. +// +// TODO: store all files with the given key instead of just the first one +func StoreFromHTTP(fs FS, r *http.Request, key string) error { + file, fileHeader, err := r.FormFile(key) + if err != nil { + return fmt.Errorf("get file from request: %w", err) + } + defer file.Close() + + buf := make([]byte, 512) + n, err := file.Read(buf) + if err != nil && err != io.EOF { + return err + } + contentType := http.DetectContentType(buf[:n]) + + i := strings.LastIndexAny(fileHeader.Filename, "/\\") + // if i == -1 { i = -1 } + + return fs.Store(uuid.New(), fileHeader.Filename[i+1:], contentType, file) +} diff --git a/web/file2/fs/file.go b/web/file2/fs/file.go new file mode 100644 index 0000000..6165d89 --- /dev/null +++ b/web/file2/fs/file.go @@ -0,0 +1,37 @@ +package fs + +import ( + "fmt" + "io/fs" + "os" + "path" + + "github.com/google/uuid" +) + +// File is a file handle with its associated ID. +type File struct { + *os.File + fs *FS + id uuid.UUID +} + +func (f File) Stat() (fs.FileInfo, error) { + fi := FileInfo{id: f.id} + var err error + fi.FileInfo, err = f.File.Stat() + if err != nil { + return nil, fmt.Errorf("os stat: %w", err) + } + name, err := os.ReadFile(path.Join(f.fs.Root, f.id.String(), "name")) + if err != nil { + return nil, fmt.Errorf("reading name: %w", err) + } + fi.name = string(name) + contentType, err := os.ReadFile(path.Join(f.fs.Root, f.id.String(), "type")) + if err != nil { + return nil, fmt.Errorf("reading type: %w", err) + } + fi.contentType = string(contentType) + return fi, nil +} diff --git a/web/file2/fs/fileinfo.go b/web/file2/fs/fileinfo.go new file mode 100644 index 0000000..041085c --- /dev/null +++ b/web/file2/fs/fileinfo.go @@ -0,0 +1,19 @@ +package fs + +import ( + "os" + + "github.com/google/uuid" +) + +type FileInfo struct { + id uuid.UUID + name string + contentType string + os.FileInfo +} + +func (fi FileInfo) ID() uuid.UUID { return fi.id } +func (fi FileInfo) ContentType() string { return fi.contentType } +func (fi FileInfo) Name() string { return fi.name } +func (fi FileInfo) Sys() interface{} { return fi.FileInfo } diff --git a/web/file2/fs/fs.go b/web/file2/fs/fs.go new file mode 100644 index 0000000..858e38e --- /dev/null +++ b/web/file2/fs/fs.go @@ -0,0 +1,123 @@ +/* +Package fs implements a non-hierarchical file store using the underlying (disk) +file system. + +A file store contains directories named after UUIDs, each containing the +following files: + name the file name (``basename'') + type the file's MIME type + data the file data + +This is not meant for anything for which the word ``scale'' plays any role at +all, ever, anywhere. + +TODO: should io/fs ever support writable file systems, use one of those instead +of the ``disk'' (os.Open & Co.) (see https://github.com/golang/go/issues/45757) +*/ +package fs + +import ( + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + + "github.com/google/uuid" +) + +// FS is an on-disk file store. +type FS struct { + // Root is the path of the file store, relative to the process's working + // directory or absolute. + Root string +} + +func (fs *FS) Store(id uuid.UUID, name, contentType string, data io.Reader) error { + p := path.Join(fs.Root, id.String()) + + _, err := os.Stat(p) + if err == nil { + err = os.RemoveAll(p) + if err != nil { + return fmt.Errorf("remove old %v: %w", id, err) + } + } + + err = os.Mkdir(p, 0o750) + if err != nil { + return fmt.Errorf("mkdir %s: %w", id.String(), err) + } + + defer func() { + if err != nil { + os.RemoveAll(p) + } + }() + + err = os.WriteFile(path.Join(p, "name"), []byte(name), 0o640) + if err != nil { + return fmt.Errorf("write name: %w", err) + } + err = os.WriteFile(path.Join(p, "type"), []byte(contentType), 0o640) + if err != nil { + return fmt.Errorf("write type: %w", err) + } + f, err := os.Create(path.Join(p, "data")) + if err != nil { + return fmt.Errorf("create data file: %w", err) + } + _, err = io.Copy(f, data) + if err != nil { + return fmt.Errorf("write data: %w", err) + } + return nil +} + +func (fs *FS) RemoveUUID(id uuid.UUID) error { + return os.RemoveAll(path.Join(fs.Root, id.String())) +} + +// OpenUUID opens the file with the given UUID. +func (fs *FS) OpenUUID(id uuid.UUID) (fs.File, error) { + f, err := os.Open(path.Join(fs.Root, id.String(), "data")) + if err != nil { + return nil, fmt.Errorf("open data: %w", err) + } + return File{File: f, fs: fs, id: uuid.MustParse(id.String())}, nil +} + +// Open searches for and opens the file with the given name. +func (fs *FS) Open(name string) (fs.File, error) { + files, err := os.ReadDir(fs.Root) + if err != nil { + return nil, err + } + for _, v := range files { + id := v.Name() + entryName, err := os.ReadFile(path.Join(fs.Root, id, "name")) + if err != nil || string(entryName) != name { + continue + } + f, err := os.Open(path.Join(fs.Root, id, "data")) + if err != nil { + return nil, fmt.Errorf("open data: %w", err) + } + return File{File: f, fs: fs, id: uuid.MustParse(id)}, nil + } + return nil, errors.New("no such file") +} + +// Check checks whether the file store is operable, ie. whether fs.Root exists +// and is a directory. +func (fs *FS) Check() error { + fi, err := os.Stat(fs.Root) + if err != nil { + return err + } + if !fi.IsDir() { + return errors.New("file store is not a directory") + } + return nil +} diff --git a/web/file2/fs/fs_test.go b/web/file2/fs/fs_test.go new file mode 100644 index 0000000..9a7c628 --- /dev/null +++ b/web/file2/fs/fs_test.go @@ -0,0 +1,50 @@ +package fs + +import ( + "io" + "os" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + + "dev.sum7.eu/genofire/golang-lib/web/file2" +) + +func TestOpenStat(t *testing.T) { + assert := assert.New(t) + var fs file.FS = &FS{Root: "./testdata"} + assert.NoError(fs.Check()) + + f, err := fs.Open("glenda") + assert.NoError(err) + assert.NotNil(f) + + fi, err := f.Stat() + assert.NoError(err) + assert.NotNil(fi) + + assert.Equal(uuid.MustParse("d2750ced-4bdc-41d0-8c2f-5b9de44b84ef"), fi.(file.FileInfo).ID()) + assert.Equal("text/plain", fi.(file.FileInfo).ContentType()) + assert.Equal("glenda", fi.Name()) +} + +func TestCreateOpenUUIDRead(t *testing.T) { + assert := assert.New(t) + var fs file.FS = &FS{Root: "./testdata"} + assert.NoError(fs.Check()) + + err := fs.Store(uuid.MustParse("f9375ccb-ee09-4ecf-917e-b88725efcb68"), "$name", "text/plain", strings.NewReader("hello, world\n")) + assert.NoError(err) + + f, err := fs.OpenUUID(uuid.MustParse("f9375ccb-ee09-4ecf-917e-b88725efcb68")) + assert.NoError(err) + assert.NotNil(f) + + buf, err := io.ReadAll(f) + assert.NoError(err) + assert.Equal("hello, world\n", string(buf)) + + os.RemoveAll("./testdata/f9375ccb-ee09-4ecf-917e-b88725efcb68") +} diff --git a/web/file2/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/data b/web/file2/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/data new file mode 100644 index 0000000..1b2f1ce --- /dev/null +++ b/web/file2/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/data @@ -0,0 +1,8 @@ +Small ‘hand-drwawn’ Glenda by ems: + + (\(\ + ¸". .. + ( . .) + | ° ¡ + ¿ ; + c?".UJ" diff --git a/web/file2/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/name b/web/file2/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/name new file mode 100644 index 0000000..c096c9b --- /dev/null +++ b/web/file2/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/name @@ -0,0 +1 @@ +glenda \ No newline at end of file diff --git a/web/file2/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/type b/web/file2/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/type new file mode 100644 index 0000000..f1148cc --- /dev/null +++ b/web/file2/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/type @@ -0,0 +1 @@ +text/plain \ No newline at end of file diff --git a/web/file2/fs_test.go b/web/file2/fs_test.go new file mode 100644 index 0000000..852a198 --- /dev/null +++ b/web/file2/fs_test.go @@ -0,0 +1,47 @@ +package file_test + +import ( + "fmt" + "io" + "net/http" + "os" + + "github.com/google/uuid" + + "dev.sum7.eu/genofire/golang-lib/web/file2" +) + +var fs file.FS + +func ExampleFS() { + // generate the UUID for the new file + id := uuid.New() + + // store a file + { + f, _ := os.Open("glenda.png") + fs.Store(id, "glenda.png", "image/png", f) + f.Close() + } + + // copy back to a local file + { + r, _ := fs.OpenUUID(id) + w, _ := os.Create("glenda.png") + io.Copy(w, r) + r.Close() + w.Close() + } +} + +func ExampleStoreFromHTTP() { + http.HandleFunc("/upload", func(w http.ResponseWriter, r *http.Request) { + if err := file.StoreFromHTTP(fs, r, "file"); err != nil { + w.Header().Set("content-type", "application/json") + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(fmt.Sprintf(`{"message":"%v"}`, err))) + } else { + w.WriteHeader(http.StatusOK) + } + }) +} diff --git a/web/file2/s3/file.go b/web/file2/s3/file.go new file mode 100644 index 0000000..6f066d5 --- /dev/null +++ b/web/file2/s3/file.go @@ -0,0 +1,21 @@ +package s3 + +import ( + "io/fs" + + "github.com/minio/minio-go/v7" +) + +type File struct { + *minio.Object +} + +func (f File) Stat() (fs.FileInfo, error) { + var fi FileInfo + var err error + fi.ObjectInfo, err = f.Object.Stat() + if err != nil { + return nil, err + } + return fi, nil +} diff --git a/web/file2/s3/fileinfo.go b/web/file2/s3/fileinfo.go new file mode 100644 index 0000000..6b8f867 --- /dev/null +++ b/web/file2/s3/fileinfo.go @@ -0,0 +1,24 @@ +package s3 + +import ( + "io/fs" + "time" + + "github.com/google/uuid" + "github.com/minio/minio-go/v7" +) + +type FileInfo struct { + minio.ObjectInfo +} + +func (fi FileInfo) Name() string { return fi.UserMetadata["filename"] } +func (fi FileInfo) Size() int64 { return fi.ObjectInfo.Size } + +// TODO: try to map s3 permissions to these, somehow +func (fi FileInfo) Mode() fs.FileMode { return 0o640 } +func (fi FileInfo) ModTime() time.Time { return fi.LastModified } +func (fi FileInfo) IsDir() bool { return false } +func (fi FileInfo) Sys() interface{} { return fi.ObjectInfo } +func (fi FileInfo) ID() uuid.UUID { return uuid.MustParse(fi.Key) } +func (fi FileInfo) ContentType() string { return fi.ObjectInfo.ContentType } diff --git a/web/file2/s3/fs.go b/web/file2/s3/fs.go new file mode 100644 index 0000000..1563071 --- /dev/null +++ b/web/file2/s3/fs.go @@ -0,0 +1,87 @@ +/* +Package s3 implements a non-hierarchical file store using Amazon s3. A file +store uses a single bucket. Each file is an object with its UUID as the name. +The file name is stored in the user-defined object metadata x-amz-meta-filename. +*/ +package s3 + +import ( + "context" + "errors" + "io" + "io/fs" + + "github.com/google/uuid" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type FS struct { + client *minio.Client + bucket string +} + +// New ``connects'' to an s3 endpoint and creates a file store using the +// specified bucket. The bucket is created if it doesn't exist. +func New(endpoint string, secure bool, id, secret, bucket, location string) (*FS, error) { + var fs FS + var err error + fs.client, err = minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(id, secret, ""), + Secure: secure, + }) + if err != nil { + return nil, err + } + + ctx := context.TODO() + + err = fs.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{ + Region: location, + }) + if err != nil { + if exists, err := fs.client.BucketExists(ctx, bucket); err != nil || !exists { + return nil, err + } + } + fs.bucket = bucket + + return &fs, nil +} + +func (fs *FS) Store(id uuid.UUID, name, contentType string, data io.Reader) error { + ctx := context.TODO() + _, err := fs.client.PutObject(ctx, fs.bucket, id.String(), data, -1, minio.PutObjectOptions{ + UserMetadata: map[string]string{ + "filename": name, + }, + ContentType: contentType, + }) + return err +} + +func (fs *FS) RemoveUUID(id uuid.UUID) error { + ctx := context.TODO() + return fs.client.RemoveObject(ctx, fs.bucket, id.String(), minio.RemoveObjectOptions{}) +} + +// TODO: implement +func (fs *FS) Open(name string) (fs.File, error) { + return nil, errors.New("not implemented") +} + +func (fs *FS) OpenUUID(id uuid.UUID) (fs.File, error) { + ctx := context.TODO() + + object, err := fs.client.GetObject(ctx, fs.bucket, id.String(), minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + + return File{Object: object}, nil +} + +// TODO: do some checking +func (fs *FS) Check() error { + return nil +} diff --git a/web/file2/s3/fs_test.go b/web/file2/s3/fs_test.go new file mode 100644 index 0000000..ccd4ee1 --- /dev/null +++ b/web/file2/s3/fs_test.go @@ -0,0 +1,24 @@ +package s3 + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "dev.sum7.eu/genofire/golang-lib/web/file2" +) + +// TODO: actually test, either using little dummies or using sth like play.min.io + +func TestTypes(t *testing.T) { + assert := assert.New(t) + + var fs file.FS + fs, err := New("127.0.0.1", false, "", "", "", "") + _ = fs + assert.Error(err) +} + +func ExampleNew() { + New("play.min.io", true, "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", "file-store", "") +}