diff --git a/.test-coverage b/.test-coverage new file mode 100755 index 0000000..da2df9e --- /dev/null +++ b/.test-coverage @@ -0,0 +1,28 @@ +#!/bin/bash +# Issue: https://github.com/mattn/goveralls/issues/20 +# Source: https://github.com/uber/go-torch/blob/63da5d33a225c195fea84610e2456d5f722f3963/.test-cover.sh +CI=$1 +echo "run for $CI" + +echo "mode: count" > profile.cov +FAIL=0 + +# Standard go tooling behavior is to ignore dirs with leading underscors +for dir in $(find . -maxdepth 10 -not -path './vendor/*' -not -path './.git*' -not -path '*/_*' -type d); +do + if ls $dir/*.go &> /dev/null; then + go test -v -covermode=count -coverprofile=profile.tmp $dir || FAIL=$? + if [ -f profile.tmp ] + then + tail -n +2 < profile.tmp >> profile.cov + rm profile.tmp + fi + fi +done + +# Failures have incomplete results, so don't send +if [ "$FAIL" -eq 0 ]; then + goveralls -v -coverprofile=profile.cov -service=$CI -repotoken=$COVERALLS_REPO_TOKEN +fi + +exit $FAIL diff --git a/api/recieve/main.go b/api/recieve/main.go new file mode 100644 index 0000000..72940ab --- /dev/null +++ b/api/recieve/main.go @@ -0,0 +1,72 @@ +package recieve + +import ( + "encoding/json" + "net/http" + + "github.com/genofire/logmania/database" + "github.com/genofire/logmania/log" + "github.com/gorilla/websocket" +) + +type Handler struct { + upgrader websocket.Upgrader +} + +func NewHandler() *Handler { + return &Handler{ + upgrader: websocket.Upgrader{}, + } +} + +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + logEntry := log.HTTP(r) + c, err := h.upgrader.Upgrade(w, r, nil) + if err != nil { + logEntry.Warn("no webservice upgrade:", err) + return + } + token := "" + defer c.Close() + for { + if token == "" { + var maybeToken string + msgType, msg, err := c.ReadMessage() + if err != nil { + logEntry.Error("recieving token", err) + break + } + if msgType != websocket.TextMessage { + logEntry.Warn("recieve no token") + break + } + maybeToken = string(msg) + logEntry.AddField("token", maybeToken) + if !database.IsTokenValid(maybeToken) { + logEntry.Warn("recieve wrong token") + break + } else { + token = maybeToken + logEntry.Info("recieve valid token") + } + continue + } + var entry log.Entry + msgType, msg, err := c.ReadMessage() + if msgType == -1 { + c.Close() + logEntry.Info("connecting closed") + break + } + if err != nil { + logEntry.Error("recieving log entry:", err) + break + } + err = json.Unmarshal(msg, &entry) + if err != nil { + logEntry.Error("umarshal log entry:", err) + break + } + database.InsertEntry(token, &entry) + } +} diff --git a/api/recieve/main_test.go b/api/recieve/main_test.go new file mode 100644 index 0000000..77c89d3 --- /dev/null +++ b/api/recieve/main_test.go @@ -0,0 +1 @@ +package recieve diff --git a/circle.yml b/circle.yml new file mode 100644 index 0000000..10c145e --- /dev/null +++ b/circle.yml @@ -0,0 +1,35 @@ +notify: + webhooks: + - url: https://hook2xmpp.pub.warehost.de/circleci + +machine: + environment: + GOROOT: "" + PATH: "/usr/local/go/bin:/usr/local/go_workspace/bin:~/.go_workspace/bin:${PATH}" + GOPATH: "${HOME}/.go_workspace" + +dependencies: + override: + - mkdir -p ~/.go_workspace/src/github.com/${CIRCLE_PROJECT_USERNAME} + - ln -s ${HOME}/${CIRCLE_PROJECT_REPONAME} ${HOME}/.go_workspace/src/github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME} + - go get -t -d -v ./... + - go install github.com/genofire/logmania/cmd/logmania + post: + - cp ~/.go_workspace/bin/logmania logmania.bin + - tar -cvzf logmania-builded.tar.gz logmania.bin logmania_example.conf + - mv logmania-builded.tar.gz $CIRCLE_ARTIFACTS + + + +test: + pre: + - go get github.com/mattn/goveralls + - go get golang.org/x/tools/cmd/cover + override: + - ./.test-coverage circle-ci + +deployment: + staging: + branch: master + commands: + - ./deploy.sh $HOST_FOR_STAGING $PORT_FOR_STAGING diff --git a/cmd/logmania/main.go b/cmd/logmania/main.go index 1b1383b..31ce7f1 100644 --- a/cmd/logmania/main.go +++ b/cmd/logmania/main.go @@ -6,24 +6,48 @@ import ( "os/signal" "syscall" + "github.com/genofire/logmania/api/recieve" + "github.com/genofire/logmania/database" "github.com/genofire/logmania/lib" "github.com/genofire/logmania/log" - _ "github.com/genofire/logmania/log/hook/output" + logOutput "github.com/genofire/logmania/log/hook/output" ) var ( configPath string config *lib.Config + api *lib.HTTPServer + apiNoPanic *bool + debug bool ) func main() { flag.StringVar(&configPath, "config", "logmania.conf", "config file") + flag.BoolVar(&debug, "debug", false, "enable debuging") + flag.Parse() + logger := NewSelfLogger() + + if debug { + logger.AboveLevel = log.DebugLevel + logOutput.AboveLevel = log.DebugLevel + } + log.Info("starting logmania") + config, err := lib.ReadConfig(configPath) if config == nil || err != nil { log.Panicf("Could not load '%s' for configuration.", configPath) } + database.Connect(config.Database.Type, config.Database.Connect) + log.AddLogger(logger) + + api = &lib.HTTPServer{ + Addr: config.API.Bind, + Handler: recieve.NewHandler(), + } + api.Start() + // Wait for system signal sigchan := make(chan os.Signal, 1) signal.Notify(sigchan, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGUSR1) @@ -51,7 +75,17 @@ func reload() { log.Info("reload config file") config, err := lib.ReadConfig(configPath) if config == nil || err != nil { - log.Errorf("Could not load '%s' for new configuration. Skip reload.", configPath) + log.Errorf("reload: could not load '%s' for new configuration. Skip reload.", configPath) return } + if config.API.Bind != api.Addr { + api.ErrorNoPanic = true + api.Close() + api.Addr = config.API.Bind + api.Start() + log.Info("reload: new api bind") + } + if database.ReplaceConnect(config.Database.Type, config.Database.Connect) { + log.Info("reload: new database connection establish") + } } diff --git a/cmd/logmania/selflog.go b/cmd/logmania/selflog.go new file mode 100644 index 0000000..36f09b7 --- /dev/null +++ b/cmd/logmania/selflog.go @@ -0,0 +1,40 @@ +package main + +import ( + "github.com/genofire/logmania/database" + "github.com/genofire/logmania/log" +) + +type SelfLogger struct { + log.Logger + AboveLevel log.LogLevel + lastMsg string + lastTime int +} + +func NewSelfLogger() *SelfLogger { + return &SelfLogger{ + AboveLevel: log.InfoLevel, + } +} + +func (l *SelfLogger) Hook(e *log.Entry) { + if e.Level >= l.AboveLevel { + return + } + // TODO strange logger + if l.lastTime > 15 { + panic("selflogger same log to oftern") + } + if l.lastMsg == e.Text{ + l.lastTime += 1 + } else { + l.lastMsg = e.Text + l.lastTime = 1 + } + database.InsertEntry("",e) +} + + +func (l *SelfLogger) Close() { +} diff --git a/database/app.go b/database/app.go new file mode 100644 index 0000000..abf7518 --- /dev/null +++ b/database/app.go @@ -0,0 +1,13 @@ +package database + +type Application struct { + ID int `json:"id"` + Name string `json:"name"` + Token string `json:"token"` + //Entries []*Entry `json:"entries" gorm:"-"` +} + +func IsTokenValid(token string) bool { + result := db.Where("token = ?", token).First(&Application{}) + return !result.RecordNotFound() +} diff --git a/database/bootstrap.go b/database/bootstrap.go new file mode 100644 index 0000000..887167d --- /dev/null +++ b/database/bootstrap.go @@ -0,0 +1,66 @@ +package database + +import ( + "github.com/jinzhu/gorm" + _ "github.com/jinzhu/gorm/dialects/postgres" + _ "github.com/jinzhu/gorm/dialects/sqlite" + + "github.com/genofire/logmania/log" +) + +var ( + dbType, connect string + db *gorm.DB +) + +func Connect(initDBType, initConnect string) { + var err error + db, err = gorm.Open(initDBType, initConnect) + if err != nil { + log.Panic("failed to connect to database", err) + } + bootstrap() + dbType = initDBType + connect = initConnect +} + +func ReplaceConnect(initDBType, initConnect string) bool { + if dbType == initDBType && connect == initConnect { + return false + } + dbTemp, err := gorm.Open(initDBType, initConnect) + if err != nil { + log.Error("failed to setup new database connection", err) + return false + } + + err = db.Close() + if err != nil { + log.Error("failed to close old database connection", err) + return false + } + db = dbTemp + bootstrap() + dbType = initDBType + connect = initConnect + return true +} + +func bootstrap() { + + var user User + var app Application + db.AutoMigrate(&user) + db.AutoMigrate(&app) + db.AutoMigrate(&Entry{}) + if resultUser := db.First(&user); resultUser.RecordNotFound() { + user.Name = "root" + if resultApp := db.First(app); resultApp.RecordNotFound() { + app.Name = "TestSoftware" + app.Token = "example" + db.Create(&app) + user.Permissions = []Application{app} + } + db.Create(&user) + } +} diff --git a/database/bootstrap_test.go b/database/bootstrap_test.go new file mode 100644 index 0000000..636bab8 --- /dev/null +++ b/database/bootstrap_test.go @@ -0,0 +1 @@ +package database diff --git a/database/entry.go b/database/entry.go new file mode 100644 index 0000000..889cdb4 --- /dev/null +++ b/database/entry.go @@ -0,0 +1,41 @@ +package database + +import ( + "encoding/json" + "time" + + "github.com/genofire/logmania/log" +) + +type Entry struct { + ID int `json:"id"` + Time time.Time + ApplicationID int + Fields string `sql:"type:json"` + Text string + Level int +} + +func transformToDB(dbEntry *log.Entry) *Entry { + jsonData, err := json.Marshal(dbEntry.Fields) + if err != nil { + return nil + } + return &Entry{ + Level: int(dbEntry.Level), + Text: dbEntry.Text, + Fields: string(jsonData), + } +} + +func InsertEntry(token string, entryLog *log.Entry) { + app := Application{} + db.Where("token = ?", token).First(&app) + entry := transformToDB(entryLog) + entry.Time = time.Now() + entry.ApplicationID = app.ID + result := db.Create(&entry) + if result.Error != nil { + log.Error("saving log entry to database", result.Error) + } +} diff --git a/database/user.go b/database/user.go new file mode 100644 index 0000000..aeabb8e --- /dev/null +++ b/database/user.go @@ -0,0 +1,14 @@ +package database + +import "github.com/genofire/logmania/log" + +type User struct { + ID int `json:"id"` + Name string + Mail string + XMPP string + NotifyMail bool + NotifyXMPP bool + NotifyAfterLoglevel log.LogLevel + Permissions []Application `gorm:"many2many:user_permissions;"` +} diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..a79bef8 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,14 @@ +#!/bin/bash +host=$1 +port=$2 +remote="circleci@${host}" +echo "deploying..." +ssh -p $port $remote sudo systemctl stop logmania; +RETVAL=$? +[ $RETVAL -ne 0 ] && exit 1 +scp -q -P $port ~/.go_workspace/bin/logmania $remote:~/bin/logmania; +RETVAL=$? +ssh -p $port $remote sudo systemctl start logmania; +[ $RETVAL -eq 0 ] && RETVAL=$? +[ $RETVAL -ne 0 ] && exit 1 +echo "deployed" diff --git a/examples/with/main.go b/examples/with/main.go index 1b4bbe8..0e49fab 100644 --- a/examples/with/main.go +++ b/examples/with/main.go @@ -9,7 +9,7 @@ import ( ) func main() { - logClient.Init("ws://localhost:8081/blub", "example") + logClient.Init("ws://localhost:8081", "example", log.DebugLevel) log.Info("startup") log.New().AddField("answer", 42).AddFields(map[string]interface{}{"answer": 3, "foo": "bar"}).Warn("Some spezial") log.Debug("Never shown up") diff --git a/lib/config_test.go b/lib/config_test.go new file mode 100644 index 0000000..55c21f8 --- /dev/null +++ b/lib/config_test.go @@ -0,0 +1 @@ +package lib diff --git a/lib/http.go b/lib/http.go new file mode 100644 index 0000000..19ccac3 --- /dev/null +++ b/lib/http.go @@ -0,0 +1,35 @@ +package lib + +import ( + "net/http" + + "github.com/genofire/logmania/log" +) + +type HTTPServer struct { + srv *http.Server + ErrorNoPanic bool + Addr string + Handler http.Handler +} + +func (hs *HTTPServer) Start() { + hs.srv = &http.Server{ + Addr: hs.Addr, + Handler: hs.Handler, + } + go func() { + log.Debug("startup of http listener") + if err := hs.srv.ListenAndServe(); err != nil { + if hs.ErrorNoPanic { + log.Debug("httpserver shutdown without panic") + return + } + log.Panic(err) + } + }() +} +func (hs *HTTPServer) Close() { + log.Debug("startup of http listener") + hs.srv.Close() +} diff --git a/log/hook.go b/log/hook.go deleted file mode 100644 index d13505a..0000000 --- a/log/hook.go +++ /dev/null @@ -1,15 +0,0 @@ -package log - -type Hook func(e *Entry) - -var hooks = make([]Hook, 0) - -func AddHook(hook Hook) { - hooks = append(hooks, hook) -} - -func save(e *Entry) { - for _, hook := range hooks { - hook(e) - } -} diff --git a/log/hook/client/main.go b/log/hook/client/main.go index b91f75e..8dfeabe 100644 --- a/log/hook/client/main.go +++ b/log/hook/client/main.go @@ -9,38 +9,65 @@ import ( ) type Logger struct { + log.Logger AboveLevel log.LogLevel conn *websocket.Conn + closed bool } -func (l *Logger) hook(e *log.Entry) { +var CurrentLogger *Logger + +func NewLogger(url, token string, AboveLevel log.LogLevel) *Logger { + c, _, err := websocket.DefaultDialer.Dial(fmt.Sprint(url, "/logger"), nil) + if err != nil { + log.Error("[logmania] error on connect: ", err) + return nil + } + err = c.WriteMessage(websocket.TextMessage, []byte(token)) + if err != nil { + log.Error("[logmania] could not send token:", err) + return nil + } + return &Logger{ + AboveLevel: AboveLevel, + conn: c, + } +} + +func (l *Logger) Hook(e *log.Entry) { + if l.closed { + return + } if e.Level < l.AboveLevel { return } err := l.conn.WriteJSON(e) if err != nil { - log.Panic("[logmania] could not send token") + log.Error("[logmania] could not send log entry:", err) + l.Close() + } +} +func (l *Logger) Listen() { + for { + msgType, _, err := l.conn.ReadMessage() + if msgType == -1 { + l.closed = true + l.conn.Close() + return + } + if err != nil { + log.Warn("[logmania] close listener:", err) + } } } func (l *Logger) Close() { - l.conn.Close() + l.conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + l.closed = true } -func Init(url, token string) *Logger { - logger := &Logger{ - AboveLevel: log.InfoLevel, - } - c, _, err := websocket.DefaultDialer.Dial(fmt.Sprint(url, "/logger"), nil) - if err != nil { - log.Panic("[logmania] error on connect") - return nil - } - err = c.WriteJSON(token) - if err != nil { - log.Panic("[logmania] could not send token") - return nil - } - logger.conn = c - log.AddHook(logger.hook) - return logger +func Init(url, token string, AboveLevel log.LogLevel) *Logger { + CurrentLogger = NewLogger(url, token, AboveLevel) + go CurrentLogger.Listen() + log.AddLogger(CurrentLogger) + return CurrentLogger } diff --git a/log/hook/client/main_test.go b/log/hook/client/main_test.go new file mode 100644 index 0000000..da13c8e --- /dev/null +++ b/log/hook/client/main_test.go @@ -0,0 +1 @@ +package client diff --git a/log/hook/output/main.go b/log/hook/output/main.go index c83eb23..d3e30cb 100644 --- a/log/hook/output/main.go +++ b/log/hook/output/main.go @@ -5,6 +5,8 @@ import ( "os" "time" + "github.com/bclicn/color" + "github.com/genofire/logmania/log" ) @@ -14,7 +16,24 @@ var ( AboveLevel = log.InfoLevel ) -func hook(e *log.Entry) { +type Logger struct { + log.Logger + TimeFormat string + ShowTime bool + AboveLevel log.LogLevel +} + +var CurrentLogger *Logger + +func NewLogger() *Logger { + return &Logger{ + TimeFormat: "2006-01-02 15:04:05", + ShowTime: true, + AboveLevel: log.InfoLevel, + } +} + +func (l *Logger) Hook(e *log.Entry) { if e.Level < AboveLevel { return } @@ -23,13 +42,26 @@ func hook(e *log.Entry) { if ShowTime { format = "%s [%s] %s" - v = append(v, time.Now().Format(TimeFormat)) + v = append(v, color.LightBlue(time.Now().Format(TimeFormat))) + } + lvl := e.Level.String() + switch e.Level { + case log.DebugLevel: + lvl = color.DarkGray(lvl) + case log.InfoLevel: + lvl = color.Green(lvl) + case log.WarnLevel: + lvl = color.Yellow(lvl) + case log.ErrorLevel: + lvl = color.Red(lvl) + case log.PanicLevel: + lvl = color.BRed(lvl) } - v = append(v, e.Level.String(), e.Text) + v = append(v, lvl, e.Text) if len(e.Fields) > 0 { - v = append(v, e.FieldString()) + v = append(v, color.Purple(e.FieldString())) format = fmt.Sprintf("%s (%%s)\n", format) } else { format = fmt.Sprintf("%s\n", format) @@ -37,15 +69,17 @@ func hook(e *log.Entry) { text := fmt.Sprintf(format, v...) - if e.Level == log.PanicLevel { - panic(text) - } else if e.Level > log.WarnLevel { + if e.Level > log.WarnLevel { os.Stderr.WriteString(text) } else { os.Stdout.WriteString(text) } } -func init() { - log.AddHook(hook) +func (l *Logger) Close() { +} + +func init() { + CurrentLogger = NewLogger() + log.AddLogger(CurrentLogger) } diff --git a/log/hook/output/main_test.go b/log/hook/output/main_test.go new file mode 100644 index 0000000..ad89311 --- /dev/null +++ b/log/hook/output/main_test.go @@ -0,0 +1 @@ +package output diff --git a/log/init.go b/log/init.go new file mode 100644 index 0000000..6065935 --- /dev/null +++ b/log/init.go @@ -0,0 +1,39 @@ +package log + +import ( + "net/http" + + wsGozilla "github.com/gorilla/websocket" + "golang.org/x/net/websocket" +) + +func getIP(r *http.Request) string { + ip := r.Header.Get("X-Forwarded-For") + if ip == "" { + ip = r.RemoteAddr + } + return ip +} + +func HTTP(r *http.Request) *Entry { + return New().AddFields(map[string]interface{}{ + "remote": getIP(r), + "method": r.Method, + "url": r.URL.RequestURI(), + }) +} + +func WebsocketX(ws *websocket.Conn) *Entry { + r := ws.Request() + return New().AddFields(map[string]interface{}{ + "remote": getIP(r), + "websocket": true, + "url": r.URL.RequestURI(), + }) +} +func WebsocketGozilla(ws *wsGozilla.Conn) *Entry { + return New().AddFields(map[string]interface{}{ + "remote": ws.RemoteAddr().String(), + "websocket": true, + }) +} diff --git a/log/logger.go b/log/logger.go new file mode 100644 index 0000000..b27939f --- /dev/null +++ b/log/logger.go @@ -0,0 +1,24 @@ +package log + +type Logger interface { + Hook(*Entry) + Close() +} + +var loggers = make([]Logger, 0) + +func AddLogger(logger Logger) { + loggers = append(loggers, logger) +} + +func save(e *Entry) { + for _, logger := range loggers { + logger.Hook(e) + } + if e.Level == PanicLevel { + for _, logger := range loggers { + logger.Close() + } + panic("panic see last log in logmania") + } +} diff --git a/log/main_test.go b/log/main_test.go new file mode 100644 index 0000000..7330d54 --- /dev/null +++ b/log/main_test.go @@ -0,0 +1 @@ +package log diff --git a/logmania_example.conf b/logmania_example.conf index 95cdb67..c34868d 100644 --- a/logmania_example.conf +++ b/logmania_example.conf @@ -1,6 +1,9 @@ [api] +bind = ":8081" [database] +type = "sqlite3" +connect = "test.db" [webserver] enable = true