commit a71a4faa67326fad7b69ad0dad2e769b20acad6c Author: Martin/Geno Date: Thu Mar 21 17:03:50 2019 +0100 add webdav2archive diff --git a/.ci/check-gofmt b/.ci/check-gofmt new file mode 100755 index 0000000..4a1c0b2 --- /dev/null +++ b/.ci/check-gofmt @@ -0,0 +1,8 @@ +#!/bin/bash + +result="$(gofmt -s -l . | grep -v '^vendor/' )" +if [ -n "$result" ]; then + echo "Go code is not formatted, run 'gofmt -s -w .'" >&2 + echo "$result" + exit 1 +fi diff --git a/.ci/check-testfiles b/.ci/check-testfiles new file mode 100755 index 0000000..132ff73 --- /dev/null +++ b/.ci/check-testfiles @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# checks if every desired package has test files + +import os +import re +import sys + +source_re = re.compile(".*\.go") +test_re = re.compile(".*_test\.go") +missing = False + +for root, dirs, files in os.walk("."): + # ignore some paths + if root == "." or root.startswith("./vendor") or root.startswith("./."): + continue + + # source files but not test files? + if len(filter(source_re.match, files)) > 0 and len(filter(test_re.match, files)) == 0: + print("no test files for {}".format(root)) + missing = True + +if missing: + sys.exit(1) +else: + print("every package has test files") diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..26a012f --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,37 @@ +image: golang:latest +stages: + - build + - test + +before_script: + - mkdir -p "/go/src/dev.sum7.eu/$CI_PROJECT_NAMESPACE/" + - cp -R "/builds/$CI_PROJECT_PATH" "/go/src/dev.sum7.eu/$CI_PROJECT_NAMESPACE/" + - cd "/go/src/dev.sum7.eu/$CI_PROJECT_PATH" + - go get -d -t ./... + +build-my-project: + stage: build + script: + - go install "dev.sum7.eu/$CI_PROJECT_PATH/..." + - mkdir "/builds/$CI_PROJECT_PATH/bin/" + - mv /go/bin/* "/builds/$CI_PROJECT_PATH/bin/" + artifacts: + paths: + - "bin/" + +test-my-project: + stage: test + script: + - go get github.com/client9/misspell/cmd/misspell + - misspell -error . + - ./.ci/check-gofmt + - ./.ci/check-testfiles + - go test $(go list ./... | grep -v /vendor/) -v -coverprofile .testCoverage.txt + artifacts: + paths: + - .testCoverage.txt + +test-race-my-project: + stage: test + script: + - go test -race ./... diff --git a/README.md b/README.md new file mode 100644 index 0000000..2c08fc7 --- /dev/null +++ b/README.md @@ -0,0 +1,25 @@ +# Golang Snippts +[![Build Status](https://dev.sum7.eu/genofire/golang-snippets/badges/master/build.svg)](https://dev.sum7.eu/genofire/golang-snippets/pipelines) +[![Go Report Card](https://goreportcard.com/badge/dev.sum7.eu/genofire/golang-snippets)](https://goreportcard.com/report/dev.sum7.eu/genofire/golang-snippets) +[![GoDoc](https://godoc.org/dev.sum7.eu/genofire/golang-snippets?status.svg)](https://godoc.org/dev.sum7.eu/genofire/golang-snippets) + + +## webdav2archive +Download files from webdav server and put them in order structur by mtime + +Get Latest Version: [Download](https://dev.sum7.eu/genofire/golang-snippets/-/jobs/artifacts/master/raw/bin/webdav2archive?inline=false&job=build-my-project) + +``` +Usage of webdav2archive: + -e string + export path on local (default "download") + -f delete on download + -h string + address of webdav server (default "https://cloud.sum7.eu/remote.php/webdav/") + -i string + import path from webdave (default "AutoUpload") + -p string + password to auth at webdav server (use environment variable WEBDAV_PASSWORD for security reasen) + -u string + username to auth at webdav server (default "genofire") +``` diff --git a/webdav2archive/main.go b/webdav2archive/main.go new file mode 100644 index 0000000..606156c --- /dev/null +++ b/webdav2archive/main.go @@ -0,0 +1,94 @@ +package main + +import ( + "flag" + "fmt" + "io" + "os" + "path" + "strconv" + "sync" + + "github.com/bdlm/log" + "github.com/studio-b12/gowebdav" +) + +var ( + root = "https://cloud.sum7.eu/remote.php/webdav/" + user = "genofire" + password = os.Getenv("WEBDAV_PASSWORD") + + fromDir = "AutoUpload" + exportDir = "download" + deleteOnDownload = false + + client *gowebdav.Client + wg = &sync.WaitGroup{} +) + +func handleFile(file os.FileInfo) { + defer wg.Done() + + filepath := path.Join(fromDir, file.Name()) + logger := log.WithField("from", filepath) + + mtime := file.ModTime() + + exportpathDir := path.Join(exportDir, strconv.Itoa(mtime.Year()), fmt.Sprintf("%.2d", mtime.Month())) + os.MkdirAll(exportpathDir, os.ModePerm) + exportpath := path.Join(exportpathDir, file.Name()) + logger = logger.WithField("to", exportpath) + + reader, _ := client.ReadStream(filepath) + + f, err := os.Create(exportpath) + if err != nil { + logger.Warnf("unable to create dest: %s", err) + return + } + defer f.Close() + + if _, err = io.Copy(f, reader); err != nil { + logger.Warnf("unable to download: %s", err) + return + } + + if deleteOnDownload { + if err = client.Remove(filepath); err != nil { + logger.Warnf("unable to remove: %s", err) + return + } + logger = logger.WithField("deleted", true) + } + + logger.Info("downloaded") +} + +func main() { + flag.StringVar(&user, "u", user, "username to auth at webdav server") + flag.StringVar(&password, "p", password, "password to auth at webdav server (use environment variable WEBDAV_PASSWORD for security reasen)") + flag.StringVar(&root, "h", root, "address of webdav server") + + flag.StringVar(&fromDir, "i", fromDir, "import path from webdave") + flag.StringVar(&exportDir, "e", exportDir, "export path on local") + flag.BoolVar(&deleteOnDownload, "f", deleteOnDownload, "delete on download") + flag.Parse() + + client = gowebdav.NewClient(root, user, password) + + if client == nil { + log.Panic("cloud not connect to webdav client") + } + + files, _ := client.ReadDir(fromDir) + + for _, file := range files { + if file.IsDir() { + continue + } + wg.Add(1) + go handleFile(file) + } + wg.Wait() + log.Info("done") +} diff --git a/webdav2archive/main_test.go b/webdav2archive/main_test.go new file mode 100644 index 0000000..06ab7d0 --- /dev/null +++ b/webdav2archive/main_test.go @@ -0,0 +1 @@ +package main diff --git a/webdav2archive/test/.gitkeep b/webdav2archive/test/.gitkeep new file mode 100644 index 0000000..e69de29