add web/file2
This commit is contained in:
Lennart
Martin
parent
32f0d84427
commit
b5a323aeb4
|
|
@ -0,0 +1,13 @@
|
|||
package file
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type FileInfo interface {
|
||||
fs.FileInfo
|
||||
ID() uuid.UUID
|
||||
ContentType() string
|
||||
}
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
/*
|
||||
Package file abstracts non-hierarchical file stores. Each file consists of a
|
||||
name, a MIME type, a UUID, and data. File names may be duplicate.
|
||||
|
||||
TODO: think about name vs. UUID again—should the ``name'' really be the filename
|
||||
and not the UUID?
|
||||
*/
|
||||
package file
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// An FS is a file store.
|
||||
type FS interface {
|
||||
// Store stores a new file with the given UUID, name, and MIME type.
|
||||
// Its data is taken from the provided Reader. If it encounters an
|
||||
// error, it does nothing. Any existing file with the same UUID is
|
||||
// overwritten.
|
||||
Store(id uuid.UUID, name, contentType string, data io.Reader) error
|
||||
// RemoveUUID deletes a file.
|
||||
RemoveUUID(id uuid.UUID) error
|
||||
// Open opens a file by its name. If multiple files have the same name,
|
||||
// it is unspecified which one is opened. This may very well be very
|
||||
// slow. This is bad. Go away.
|
||||
Open(name string) (fs.File, error)
|
||||
// OpenUUID opens a file by its UUID.
|
||||
OpenUUID(id uuid.UUID) (fs.File, error)
|
||||
// Check checks the health of the file store. If the file store is not
|
||||
// healthy, it returns a descriptive error. Otherwise, the file store
|
||||
// should be usable.
|
||||
Check() error
|
||||
}
|
||||
|
||||
// StoreFromHTTP stores the first file with given form key from an HTTP
|
||||
// multipart/form-data request. Its Content-Type header is ignored; the type is
|
||||
// detected. The file name is the last part of the provided file name not
|
||||
// containing any slashes or backslashes.
|
||||
//
|
||||
// TODO: store all files with the given key instead of just the first one
|
||||
func StoreFromHTTP(fs FS, r *http.Request, key string) error {
|
||||
file, fileHeader, err := r.FormFile(key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get file from request: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
buf := make([]byte, 512)
|
||||
n, err := file.Read(buf)
|
||||
if err != nil && err != io.EOF {
|
||||
return err
|
||||
}
|
||||
contentType := http.DetectContentType(buf[:n])
|
||||
|
||||
i := strings.LastIndexAny(fileHeader.Filename, "/\\")
|
||||
// if i == -1 { i = -1 }
|
||||
|
||||
return fs.Store(uuid.New(), fileHeader.Filename[i+1:], contentType, file)
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
package fs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// File is a file handle with its associated ID.
|
||||
type File struct {
|
||||
*os.File
|
||||
fs *FS
|
||||
id uuid.UUID
|
||||
}
|
||||
|
||||
func (f File) Stat() (fs.FileInfo, error) {
|
||||
fi := FileInfo{id: f.id}
|
||||
var err error
|
||||
fi.FileInfo, err = f.File.Stat()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("os stat: %w", err)
|
||||
}
|
||||
name, err := os.ReadFile(path.Join(f.fs.Root, f.id.String(), "name"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading name: %w", err)
|
||||
}
|
||||
fi.name = string(name)
|
||||
contentType, err := os.ReadFile(path.Join(f.fs.Root, f.id.String(), "type"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading type: %w", err)
|
||||
}
|
||||
fi.contentType = string(contentType)
|
||||
return fi, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package fs
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type FileInfo struct {
|
||||
id uuid.UUID
|
||||
name string
|
||||
contentType string
|
||||
os.FileInfo
|
||||
}
|
||||
|
||||
func (fi FileInfo) ID() uuid.UUID { return fi.id }
|
||||
func (fi FileInfo) ContentType() string { return fi.contentType }
|
||||
func (fi FileInfo) Name() string { return fi.name }
|
||||
func (fi FileInfo) Sys() interface{} { return fi.FileInfo }
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
/*
|
||||
Package fs implements a non-hierarchical file store using the underlying (disk)
|
||||
file system.
|
||||
|
||||
A file store contains directories named after UUIDs, each containing the
|
||||
following files:
|
||||
name the file name (``basename'')
|
||||
type the file's MIME type
|
||||
data the file data
|
||||
|
||||
This is not meant for anything for which the word ``scale'' plays any role at
|
||||
all, ever, anywhere.
|
||||
|
||||
TODO: should io/fs ever support writable file systems, use one of those instead
|
||||
of the ``disk'' (os.Open & Co.) (see https://github.com/golang/go/issues/45757)
|
||||
*/
|
||||
package fs
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// FS is an on-disk file store.
|
||||
type FS struct {
|
||||
// Root is the path of the file store, relative to the process's working
|
||||
// directory or absolute.
|
||||
Root string
|
||||
}
|
||||
|
||||
func (fs *FS) Store(id uuid.UUID, name, contentType string, data io.Reader) error {
|
||||
p := path.Join(fs.Root, id.String())
|
||||
|
||||
_, err := os.Stat(p)
|
||||
if err == nil {
|
||||
err = os.RemoveAll(p)
|
||||
if err != nil {
|
||||
return fmt.Errorf("remove old %v: %w", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
err = os.Mkdir(p, 0o750)
|
||||
if err != nil {
|
||||
return fmt.Errorf("mkdir %s: %w", id.String(), err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err != nil {
|
||||
os.RemoveAll(p)
|
||||
}
|
||||
}()
|
||||
|
||||
err = os.WriteFile(path.Join(p, "name"), []byte(name), 0o640)
|
||||
if err != nil {
|
||||
return fmt.Errorf("write name: %w", err)
|
||||
}
|
||||
err = os.WriteFile(path.Join(p, "type"), []byte(contentType), 0o640)
|
||||
if err != nil {
|
||||
return fmt.Errorf("write type: %w", err)
|
||||
}
|
||||
f, err := os.Create(path.Join(p, "data"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("create data file: %w", err)
|
||||
}
|
||||
_, err = io.Copy(f, data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("write data: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fs *FS) RemoveUUID(id uuid.UUID) error {
|
||||
return os.RemoveAll(path.Join(fs.Root, id.String()))
|
||||
}
|
||||
|
||||
// OpenUUID opens the file with the given UUID.
|
||||
func (fs *FS) OpenUUID(id uuid.UUID) (fs.File, error) {
|
||||
f, err := os.Open(path.Join(fs.Root, id.String(), "data"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open data: %w", err)
|
||||
}
|
||||
return File{File: f, fs: fs, id: uuid.MustParse(id.String())}, nil
|
||||
}
|
||||
|
||||
// Open searches for and opens the file with the given name.
|
||||
func (fs *FS) Open(name string) (fs.File, error) {
|
||||
files, err := os.ReadDir(fs.Root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, v := range files {
|
||||
id := v.Name()
|
||||
entryName, err := os.ReadFile(path.Join(fs.Root, id, "name"))
|
||||
if err != nil || string(entryName) != name {
|
||||
continue
|
||||
}
|
||||
f, err := os.Open(path.Join(fs.Root, id, "data"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open data: %w", err)
|
||||
}
|
||||
return File{File: f, fs: fs, id: uuid.MustParse(id)}, nil
|
||||
}
|
||||
return nil, errors.New("no such file")
|
||||
}
|
||||
|
||||
// Check checks whether the file store is operable, ie. whether fs.Root exists
|
||||
// and is a directory.
|
||||
func (fs *FS) Check() error {
|
||||
fi, err := os.Stat(fs.Root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !fi.IsDir() {
|
||||
return errors.New("file store is not a directory")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
package fs
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"dev.sum7.eu/genofire/golang-lib/web/file2"
|
||||
)
|
||||
|
||||
func TestOpenStat(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
var fs file.FS = &FS{Root: "./testdata"}
|
||||
assert.NoError(fs.Check())
|
||||
|
||||
f, err := fs.Open("glenda")
|
||||
assert.NoError(err)
|
||||
assert.NotNil(f)
|
||||
|
||||
fi, err := f.Stat()
|
||||
assert.NoError(err)
|
||||
assert.NotNil(fi)
|
||||
|
||||
assert.Equal(uuid.MustParse("d2750ced-4bdc-41d0-8c2f-5b9de44b84ef"), fi.(file.FileInfo).ID())
|
||||
assert.Equal("text/plain", fi.(file.FileInfo).ContentType())
|
||||
assert.Equal("glenda", fi.Name())
|
||||
}
|
||||
|
||||
func TestCreateOpenUUIDRead(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
var fs file.FS = &FS{Root: "./testdata"}
|
||||
assert.NoError(fs.Check())
|
||||
|
||||
err := fs.Store(uuid.MustParse("f9375ccb-ee09-4ecf-917e-b88725efcb68"), "$name", "text/plain", strings.NewReader("hello, world\n"))
|
||||
assert.NoError(err)
|
||||
|
||||
f, err := fs.OpenUUID(uuid.MustParse("f9375ccb-ee09-4ecf-917e-b88725efcb68"))
|
||||
assert.NoError(err)
|
||||
assert.NotNil(f)
|
||||
|
||||
buf, err := io.ReadAll(f)
|
||||
assert.NoError(err)
|
||||
assert.Equal("hello, world\n", string(buf))
|
||||
|
||||
os.RemoveAll("./testdata/f9375ccb-ee09-4ecf-917e-b88725efcb68")
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
Small ‘hand-drwawn’ Glenda by ems:
|
||||
|
||||
(\(\
|
||||
¸". ..
|
||||
( . .)
|
||||
| ° ¡
|
||||
¿ ;
|
||||
c?".UJ"
|
||||
|
|
@ -0,0 +1 @@
|
|||
glenda
|
||||
|
|
@ -0,0 +1 @@
|
|||
text/plain
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
package file_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"dev.sum7.eu/genofire/golang-lib/web/file2"
|
||||
)
|
||||
|
||||
var fs file.FS
|
||||
|
||||
func ExampleFS() {
|
||||
// generate the UUID for the new file
|
||||
id := uuid.New()
|
||||
|
||||
// store a file
|
||||
{
|
||||
f, _ := os.Open("glenda.png")
|
||||
fs.Store(id, "glenda.png", "image/png", f)
|
||||
f.Close()
|
||||
}
|
||||
|
||||
// copy back to a local file
|
||||
{
|
||||
r, _ := fs.OpenUUID(id)
|
||||
w, _ := os.Create("glenda.png")
|
||||
io.Copy(w, r)
|
||||
r.Close()
|
||||
w.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func ExampleStoreFromHTTP() {
|
||||
http.HandleFunc("/upload", func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := file.StoreFromHTTP(fs, r, "file"); err != nil {
|
||||
w.Header().Set("content-type", "application/json")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
w.Write([]byte(fmt.Sprintf(`{"message":"%v"}`, err)))
|
||||
} else {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
package s3
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
)
|
||||
|
||||
type File struct {
|
||||
*minio.Object
|
||||
}
|
||||
|
||||
func (f File) Stat() (fs.FileInfo, error) {
|
||||
var fi FileInfo
|
||||
var err error
|
||||
fi.ObjectInfo, err = f.Object.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fi, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
package s3
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/minio/minio-go/v7"
|
||||
)
|
||||
|
||||
type FileInfo struct {
|
||||
minio.ObjectInfo
|
||||
}
|
||||
|
||||
func (fi FileInfo) Name() string { return fi.UserMetadata["filename"] }
|
||||
func (fi FileInfo) Size() int64 { return fi.ObjectInfo.Size }
|
||||
|
||||
// TODO: try to map s3 permissions to these, somehow
|
||||
func (fi FileInfo) Mode() fs.FileMode { return 0o640 }
|
||||
func (fi FileInfo) ModTime() time.Time { return fi.LastModified }
|
||||
func (fi FileInfo) IsDir() bool { return false }
|
||||
func (fi FileInfo) Sys() interface{} { return fi.ObjectInfo }
|
||||
func (fi FileInfo) ID() uuid.UUID { return uuid.MustParse(fi.Key) }
|
||||
func (fi FileInfo) ContentType() string { return fi.ObjectInfo.ContentType }
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
/*
|
||||
Package s3 implements a non-hierarchical file store using Amazon s3. A file
|
||||
store uses a single bucket. Each file is an object with its UUID as the name.
|
||||
The file name is stored in the user-defined object metadata x-amz-meta-filename.
|
||||
*/
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"io/fs"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
type FS struct {
|
||||
client *minio.Client
|
||||
bucket string
|
||||
}
|
||||
|
||||
// New ``connects'' to an s3 endpoint and creates a file store using the
|
||||
// specified bucket. The bucket is created if it doesn't exist.
|
||||
func New(endpoint string, secure bool, id, secret, bucket, location string) (*FS, error) {
|
||||
var fs FS
|
||||
var err error
|
||||
fs.client, err = minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(id, secret, ""),
|
||||
Secure: secure,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ctx := context.TODO()
|
||||
|
||||
err = fs.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{
|
||||
Region: location,
|
||||
})
|
||||
if err != nil {
|
||||
if exists, err := fs.client.BucketExists(ctx, bucket); err != nil || !exists {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
fs.bucket = bucket
|
||||
|
||||
return &fs, nil
|
||||
}
|
||||
|
||||
func (fs *FS) Store(id uuid.UUID, name, contentType string, data io.Reader) error {
|
||||
ctx := context.TODO()
|
||||
_, err := fs.client.PutObject(ctx, fs.bucket, id.String(), data, -1, minio.PutObjectOptions{
|
||||
UserMetadata: map[string]string{
|
||||
"filename": name,
|
||||
},
|
||||
ContentType: contentType,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (fs *FS) RemoveUUID(id uuid.UUID) error {
|
||||
ctx := context.TODO()
|
||||
return fs.client.RemoveObject(ctx, fs.bucket, id.String(), minio.RemoveObjectOptions{})
|
||||
}
|
||||
|
||||
// TODO: implement
|
||||
func (fs *FS) Open(name string) (fs.File, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (fs *FS) OpenUUID(id uuid.UUID) (fs.File, error) {
|
||||
ctx := context.TODO()
|
||||
|
||||
object, err := fs.client.GetObject(ctx, fs.bucket, id.String(), minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return File{Object: object}, nil
|
||||
}
|
||||
|
||||
// TODO: do some checking
|
||||
func (fs *FS) Check() error {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
package s3
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"dev.sum7.eu/genofire/golang-lib/web/file2"
|
||||
)
|
||||
|
||||
// TODO: actually test, either using little dummies or using sth like play.min.io
|
||||
|
||||
func TestTypes(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
var fs file.FS
|
||||
fs, err := New("127.0.0.1", false, "", "", "", "")
|
||||
_ = fs
|
||||
assert.Error(err)
|
||||
}
|
||||
|
||||
func ExampleNew() {
|
||||
New("play.min.io", true, "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", "file-store", "")
|
||||
}
|
||||
Loading…
Reference in New Issue