add web/file2

web/file2/fileinfo.go

@@ -0,0 +1,13 @@
+package file
+
+import (
+	"io/fs"
+
+	"github.com/google/uuid"
+)
+
+type FileInfo interface {
+	fs.FileInfo
+	ID() uuid.UUID
+	ContentType() string
+}

web/file2/fs.go

@@ -0,0 +1,65 @@
+/*
+Package file abstracts non-hierarchical file stores. Each file consists of a
+name, a MIME type, a UUID, and data. File names may be duplicate.
+
+TODO: think about name vs. UUID again—should the ``name'' really be the filename
+and not the UUID?
+*/
+package file
+
+import (
+	"fmt"
+	"io"
+	"io/fs"
+	"net/http"
+	"strings"
+
+	"github.com/google/uuid"
+)
+
+// An FS is a file store.
+type FS interface {
+	// Store stores a new file with the given UUID, name, and MIME type.
+	// Its data is taken from the provided Reader. If it encounters an
+	// error, it does nothing. Any existing file with the same UUID is
+	// overwritten.
+	Store(id uuid.UUID, name, contentType string, data io.Reader) error
+	// RemoveUUID deletes a file.
+	RemoveUUID(id uuid.UUID) error
+	// Open opens a file by its name. If multiple files have the same name,
+	// it is unspecified which one is opened. This may very well be very
+	// slow. This is bad. Go away.
+	Open(name string) (fs.File, error)
+	// OpenUUID opens a file by its UUID.
+	OpenUUID(id uuid.UUID) (fs.File, error)
+	// Check checks the health of the file store. If the file store is not
+	// healthy, it returns a descriptive error. Otherwise, the file store
+	// should be usable.
+	Check() error
+}
+
+// StoreFromHTTP stores the first file with given form key from an HTTP
+// multipart/form-data request. Its Content-Type header is ignored; the type is
+// detected. The file name is the last part of the provided file name not
+// containing any slashes or backslashes.
+//
+// TODO: store all files with the given key instead of just the first one
+func StoreFromHTTP(fs FS, r *http.Request, key string) error {
+	file, fileHeader, err := r.FormFile(key)
+	if err != nil {
+		return fmt.Errorf("get file from request: %w", err)
+	}
+	defer file.Close()
+
+	buf := make([]byte, 512)
+	n, err := file.Read(buf)
+	if err != nil && err != io.EOF {
+		return err
+	}
+	contentType := http.DetectContentType(buf[:n])
+
+	i := strings.LastIndexAny(fileHeader.Filename, "/\\")
+	// if i == -1 { i = -1 }
+
+	return fs.Store(uuid.New(), fileHeader.Filename[i+1:], contentType, file)
+}

web/file2/fs/file.go

@@ -0,0 +1,37 @@
+package fs
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"path"
+
+	"github.com/google/uuid"
+)
+
+// File is a file handle with its associated ID.
+type File struct {
+	*os.File
+	fs *FS
+	id uuid.UUID
+}
+
+func (f File) Stat() (fs.FileInfo, error) {
+	fi := FileInfo{id: f.id}
+	var err error
+	fi.FileInfo, err = f.File.Stat()
+	if err != nil {
+		return nil, fmt.Errorf("os stat: %w", err)
+	}
+	name, err := os.ReadFile(path.Join(f.fs.Root, f.id.String(), "name"))
+	if err != nil {
+		return nil, fmt.Errorf("reading name: %w", err)
+	}
+	fi.name = string(name)
+	contentType, err := os.ReadFile(path.Join(f.fs.Root, f.id.String(), "type"))
+	if err != nil {
+		return nil, fmt.Errorf("reading type: %w", err)
+	}
+	fi.contentType = string(contentType)
+	return fi, nil
+}

web/file2/fs/fileinfo.go

@@ -0,0 +1,19 @@
+package fs
+
+import (
+	"os"
+
+	"github.com/google/uuid"
+)
+
+type FileInfo struct {
+	id          uuid.UUID
+	name        string
+	contentType string
+	os.FileInfo
+}
+
+func (fi FileInfo) ID() uuid.UUID       { return fi.id }
+func (fi FileInfo) ContentType() string { return fi.contentType }
+func (fi FileInfo) Name() string        { return fi.name }
+func (fi FileInfo) Sys() interface{}    { return fi.FileInfo }

web/file2/fs/fs.go

@@ -0,0 +1,123 @@
+/*
+Package fs implements a non-hierarchical file store using the underlying (disk)
+file system.
+
+A file store contains directories named after UUIDs, each containing the
+following files:
+    name    the file name (``basename'')
+    type    the file's MIME type
+    data    the file data
+
+This is not meant for anything for which the word ``scale'' plays any role at
+all, ever, anywhere.
+
+TODO: should io/fs ever support writable file systems, use one of those instead
+of the ``disk'' (os.Open & Co.) (see https://github.com/golang/go/issues/45757)
+*/
+package fs
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path"
+
+	"github.com/google/uuid"
+)
+
+// FS is an on-disk file store.
+type FS struct {
+	// Root is the path of the file store, relative to the process's working
+	// directory or absolute.
+	Root string
+}
+
+func (fs *FS) Store(id uuid.UUID, name, contentType string, data io.Reader) error {
+	p := path.Join(fs.Root, id.String())
+
+	_, err := os.Stat(p)
+	if err == nil {
+		err = os.RemoveAll(p)
+		if err != nil {
+			return fmt.Errorf("remove old %v: %w", id, err)
+		}
+	}
+
+	err = os.Mkdir(p, 0o750)
+	if err != nil {
+		return fmt.Errorf("mkdir %s: %w", id.String(), err)
+	}
+
+	defer func() {
+		if err != nil {
+			os.RemoveAll(p)
+		}
+	}()
+
+	err = os.WriteFile(path.Join(p, "name"), []byte(name), 0o640)
+	if err != nil {
+		return fmt.Errorf("write name: %w", err)
+	}
+	err = os.WriteFile(path.Join(p, "type"), []byte(contentType), 0o640)
+	if err != nil {
+		return fmt.Errorf("write type: %w", err)
+	}
+	f, err := os.Create(path.Join(p, "data"))
+	if err != nil {
+		return fmt.Errorf("create data file: %w", err)
+	}
+	_, err = io.Copy(f, data)
+	if err != nil {
+		return fmt.Errorf("write data: %w", err)
+	}
+	return nil
+}
+
+func (fs *FS) RemoveUUID(id uuid.UUID) error {
+	return os.RemoveAll(path.Join(fs.Root, id.String()))
+}
+
+// OpenUUID opens the file with the given UUID.
+func (fs *FS) OpenUUID(id uuid.UUID) (fs.File, error) {
+	f, err := os.Open(path.Join(fs.Root, id.String(), "data"))
+	if err != nil {
+		return nil, fmt.Errorf("open data: %w", err)
+	}
+	return File{File: f, fs: fs, id: uuid.MustParse(id.String())}, nil
+}
+
+// Open searches for and opens the file with the given name.
+func (fs *FS) Open(name string) (fs.File, error) {
+	files, err := os.ReadDir(fs.Root)
+	if err != nil {
+		return nil, err
+	}
+	for _, v := range files {
+		id := v.Name()
+		entryName, err := os.ReadFile(path.Join(fs.Root, id, "name"))
+		if err != nil || string(entryName) != name {
+			continue
+		}
+		f, err := os.Open(path.Join(fs.Root, id, "data"))
+		if err != nil {
+			return nil, fmt.Errorf("open data: %w", err)
+		}
+		return File{File: f, fs: fs, id: uuid.MustParse(id)}, nil
+	}
+	return nil, errors.New("no such file")
+}
+
+// Check checks whether the file store is operable, ie. whether fs.Root exists
+// and is a directory.
+func (fs *FS) Check() error {
+	fi, err := os.Stat(fs.Root)
+	if err != nil {
+		return err
+	}
+	if !fi.IsDir() {
+		return errors.New("file store is not a directory")
+	}
+	return nil
+}

web/file2/fs/fs_test.go

@@ -0,0 +1,50 @@
+package fs
+
+import (
+	"io"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+
+	"dev.sum7.eu/genofire/golang-lib/web/file2"
+)
+
+func TestOpenStat(t *testing.T) {
+	assert := assert.New(t)
+	var fs file.FS = &FS{Root: "./testdata"}
+	assert.NoError(fs.Check())
+
+	f, err := fs.Open("glenda")
+	assert.NoError(err)
+	assert.NotNil(f)
+
+	fi, err := f.Stat()
+	assert.NoError(err)
+	assert.NotNil(fi)
+
+	assert.Equal(uuid.MustParse("d2750ced-4bdc-41d0-8c2f-5b9de44b84ef"), fi.(file.FileInfo).ID())
+	assert.Equal("text/plain", fi.(file.FileInfo).ContentType())
+	assert.Equal("glenda", fi.Name())
+}
+
+func TestCreateOpenUUIDRead(t *testing.T) {
+	assert := assert.New(t)
+	var fs file.FS = &FS{Root: "./testdata"}
+	assert.NoError(fs.Check())
+
+	err := fs.Store(uuid.MustParse("f9375ccb-ee09-4ecf-917e-b88725efcb68"), "$name", "text/plain", strings.NewReader("hello, world\n"))
+	assert.NoError(err)
+
+	f, err := fs.OpenUUID(uuid.MustParse("f9375ccb-ee09-4ecf-917e-b88725efcb68"))
+	assert.NoError(err)
+	assert.NotNil(f)
+
+	buf, err := io.ReadAll(f)
+	assert.NoError(err)
+	assert.Equal("hello, world\n", string(buf))
+
+	os.RemoveAll("./testdata/f9375ccb-ee09-4ecf-917e-b88725efcb68")
+}

web/file2/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/data

@@ -0,0 +1,8 @@
+Small ‘hand-drwawn’ Glenda by ems:
+
+  (\(\
+  ¸". ..
+  (  . .)
+  |   ° ¡
+  ¿     ;
+ c?".UJ"

web/file2/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/name

@@ -0,0 +1 @@
+glenda

web/file2/fs/testdata/d2750ced-4bdc-41d0-8c2f-5b9de44b84ef/type

@@ -0,0 +1 @@
+text/plain

web/file2/fs_test.go

@@ -0,0 +1,47 @@
+package file_test
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+
+	"github.com/google/uuid"
+
+	"dev.sum7.eu/genofire/golang-lib/web/file2"
+)
+
+var fs file.FS
+
+func ExampleFS() {
+	// generate the UUID for the new file
+	id := uuid.New()
+
+	// store a file
+	{
+		f, _ := os.Open("glenda.png")
+		fs.Store(id, "glenda.png", "image/png", f)
+		f.Close()
+	}
+
+	// copy back to a local file
+	{
+		r, _ := fs.OpenUUID(id)
+		w, _ := os.Create("glenda.png")
+		io.Copy(w, r)
+		r.Close()
+		w.Close()
+	}
+}
+
+func ExampleStoreFromHTTP() {
+	http.HandleFunc("/upload", func(w http.ResponseWriter, r *http.Request) {
+		if err := file.StoreFromHTTP(fs, r, "file"); err != nil {
+			w.Header().Set("content-type", "application/json")
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(fmt.Sprintf(`{"message":"%v"}`, err)))
+		} else {
+			w.WriteHeader(http.StatusOK)
+		}
+	})
+}

web/file2/s3/file.go

@@ -0,0 +1,21 @@
+package s3
+
+import (
+	"io/fs"
+
+	"github.com/minio/minio-go/v7"
+)
+
+type File struct {
+	*minio.Object
+}
+
+func (f File) Stat() (fs.FileInfo, error) {
+	var fi FileInfo
+	var err error
+	fi.ObjectInfo, err = f.Object.Stat()
+	if err != nil {
+		return nil, err
+	}
+	return fi, nil
+}

web/file2/s3/fileinfo.go

@@ -0,0 +1,24 @@
+package s3
+
+import (
+	"io/fs"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/minio/minio-go/v7"
+)
+
+type FileInfo struct {
+	minio.ObjectInfo
+}
+
+func (fi FileInfo) Name() string { return fi.UserMetadata["filename"] }
+func (fi FileInfo) Size() int64  { return fi.ObjectInfo.Size }
+
+// TODO: try to map s3 permissions to these, somehow
+func (fi FileInfo) Mode() fs.FileMode   { return 0o640 }
+func (fi FileInfo) ModTime() time.Time  { return fi.LastModified }
+func (fi FileInfo) IsDir() bool         { return false }
+func (fi FileInfo) Sys() interface{}    { return fi.ObjectInfo }
+func (fi FileInfo) ID() uuid.UUID       { return uuid.MustParse(fi.Key) }
+func (fi FileInfo) ContentType() string { return fi.ObjectInfo.ContentType }

web/file2/s3/fs.go

@@ -0,0 +1,87 @@
+/*
+Package s3 implements a non-hierarchical file store using Amazon s3. A file
+store uses a single bucket. Each file is an object with its UUID as the name.
+The file name is stored in the user-defined object metadata x-amz-meta-filename.
+*/
+package s3
+
+import (
+	"context"
+	"errors"
+	"io"
+	"io/fs"
+
+	"github.com/google/uuid"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+type FS struct {
+	client *minio.Client
+	bucket string
+}
+
+// New ``connects'' to an s3 endpoint and creates a file store using the
+// specified bucket. The bucket is created if it doesn't exist.
+func New(endpoint string, secure bool, id, secret, bucket, location string) (*FS, error) {
+	var fs FS
+	var err error
+	fs.client, err = minio.New(endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(id, secret, ""),
+		Secure: secure,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	ctx := context.TODO()
+
+	err = fs.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{
+		Region: location,
+	})
+	if err != nil {
+		if exists, err := fs.client.BucketExists(ctx, bucket); err != nil || !exists {
+			return nil, err
+		}
+	}
+	fs.bucket = bucket
+
+	return &fs, nil
+}
+
+func (fs *FS) Store(id uuid.UUID, name, contentType string, data io.Reader) error {
+	ctx := context.TODO()
+	_, err := fs.client.PutObject(ctx, fs.bucket, id.String(), data, -1, minio.PutObjectOptions{
+		UserMetadata: map[string]string{
+			"filename": name,
+		},
+		ContentType: contentType,
+	})
+	return err
+}
+
+func (fs *FS) RemoveUUID(id uuid.UUID) error {
+	ctx := context.TODO()
+	return fs.client.RemoveObject(ctx, fs.bucket, id.String(), minio.RemoveObjectOptions{})
+}
+
+// TODO: implement
+func (fs *FS) Open(name string) (fs.File, error) {
+	return nil, errors.New("not implemented")
+}
+
+func (fs *FS) OpenUUID(id uuid.UUID) (fs.File, error) {
+	ctx := context.TODO()
+
+	object, err := fs.client.GetObject(ctx, fs.bucket, id.String(), minio.GetObjectOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	return File{Object: object}, nil
+}
+
+// TODO: do some checking
+func (fs *FS) Check() error {
+	return nil
+}

web/file2/s3/fs_test.go

@@ -0,0 +1,24 @@
+package s3
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"dev.sum7.eu/genofire/golang-lib/web/file2"
+)
+
+// TODO: actually test, either using little dummies or using sth like play.min.io
+
+func TestTypes(t *testing.T) {
+	assert := assert.New(t)
+
+	var fs file.FS
+	fs, err := New("127.0.0.1", false, "", "", "", "")
+	_ = fs
+	assert.Error(err)
+}
+
+func ExampleNew() {
+	New("play.min.io", true, "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", "file-store", "")
+}