From 3b98fb36cbec0b59a49a49ebc8e62e1bbe10cd7e Mon Sep 17 00:00:00 2001 From: Martin/Geno Date: Fri, 8 Mar 2019 15:53:45 +0100 Subject: [PATCH] database --- capture/socket_msg.go | 14 ++++---- cmd/controller.go | 11 +++--- config_example.conf | 7 +++- controller/handler.go | 20 +++++++++-- controller/learn.go | 57 +++++++++++++++++++++++++++++ controller/main.go | 14 +++++--- data/ap.go | 20 +++++++++++ data/client.go | 26 ++++++++++++++ data/hwaddr.go | 17 ++++++--- database/ap.go | 22 ------------ database/client.go | 84 ------------------------------------------- database/main.go | 47 ------------------------ database/main_test.go | 1 - web/webserver.go | 23 ++++++++++++ 14 files changed, 185 insertions(+), 178 deletions(-) create mode 100644 controller/learn.go create mode 100644 data/ap.go create mode 100644 data/client.go delete mode 100644 database/ap.go delete mode 100644 database/client.go delete mode 100644 database/main.go delete mode 100644 database/main_test.go diff --git a/capture/socket_msg.go b/capture/socket_msg.go index eeb2a58..d199a92 100644 --- a/capture/socket_msg.go +++ b/capture/socket_msg.go @@ -51,17 +51,17 @@ func (msg *SocketMSG) Marshal() ([]byte, error) { pos += 4 if msg.Types.Is(SocketMSGTypeClient) { - obj[pos] = msg.Client.Addr.HardwareAddr[0] + obj[pos] = msg.Client.Addr[0] pos++ - obj[pos] = msg.Client.Addr.HardwareAddr[1] + obj[pos] = msg.Client.Addr[1] pos++ - obj[pos] = msg.Client.Addr.HardwareAddr[2] + obj[pos] = msg.Client.Addr[2] pos++ - obj[pos] = msg.Client.Addr.HardwareAddr[3] + obj[pos] = msg.Client.Addr[3] pos++ - obj[pos] = msg.Client.Addr.HardwareAddr[4] + obj[pos] = msg.Client.Addr[4] pos++ - obj[pos] = msg.Client.Addr.HardwareAddr[5] + obj[pos] = msg.Client.Addr[5] pos++ binary.BigEndian.PutUint32(obj[pos:(pos+4)], uint32(msg.Client.Time.Unix())) pos += 4 @@ -100,7 +100,7 @@ func (msg *SocketMSG) Unmarshal(obj []byte) error { if msg.Types.Is(SocketMSGTypeClient) { msg.Client = &WifiClient{ - Addr: data.HardwareAddr{HardwareAddr: obj[pos:(pos + 6)]}, + Addr: data.HardwareAddr(obj[pos:(pos + 6)]), Time: time.Unix(int64(binary.BigEndian.Uint32(obj[(pos+6):(pos+10)])), 0), TryProbe: binary.BigEndian.Uint16(obj[(pos + 10):(pos + 12)]), TryAuth: binary.BigEndian.Uint16(obj[(pos + 12):(pos + 14)]), diff --git a/cmd/controller.go b/cmd/controller.go index 6015940..513b58f 100644 --- a/cmd/controller.go +++ b/cmd/controller.go @@ -6,18 +6,18 @@ import ( "os/signal" "syscall" + "dev.sum7.eu/genofire/golang-lib/database" "dev.sum7.eu/genofire/golang-lib/file" "github.com/bdlm/log" "github.com/spf13/cobra" "dev.sum7.eu/genofire/wifictld-analyzer/capture" "dev.sum7.eu/genofire/wifictld-analyzer/controller" - "dev.sum7.eu/genofire/wifictld-analyzer/database" "dev.sum7.eu/genofire/wifictld-analyzer/web" ) type ControllerConfig struct { - Database string `toml:"database"` + Database database.Config `toml:"database"` Answer bool `toml:"answer"` Webserver *web.Config `toml:"webserver"` Interfaces []*capture.IFaceConfig `toml:"interfaces"` @@ -34,9 +34,12 @@ var controllerCmd = &cobra.Command{ file.ReadTOML(args[0], config) - db := database.NewDB(config.Database) + if err := database.Open(config.Database); err != nil { + log.Panicf("no database connection: %s", err) + } + defer database.Close() - ctr := controller.NewController(db) + ctr := controller.NewController() defer ctr.Close() var handlers []capture.Handler diff --git a/config_example.conf b/config_example.conf index 683193d..94cdea6 100644 --- a/config_example.conf +++ b/config_example.conf @@ -1,6 +1,11 @@ -database = "/tmp/wifictld.json" answer = false +[database] +type = "sqlite3" +logging = true +connection = "file:/tmp/wifictld.db" +# For Master-Slave cluster +# read_connection = "" [webserver] enable = true diff --git a/controller/handler.go b/controller/handler.go index 781b4ba..b930f39 100644 --- a/controller/handler.go +++ b/controller/handler.go @@ -2,24 +2,38 @@ package controller import ( "net" + "time" // "github.com/bdlm/log" + "dev.sum7.eu/genofire/golang-lib/database" "dev.sum7.eu/genofire/wifictld-analyzer/capture" + "dev.sum7.eu/genofire/wifictld-analyzer/data" ) func (c *Controller) Handler(addr *net.UDPAddr, msg *capture.SocketMSG) (*capture.SocketMSG, error) { ignore := false if msg.Types.Is(capture.SocketMSGTypeClient) && msg.Client != nil { - ignore = c.db.LearnClient(addr.IP, msg.Client) + ignore = c.LearnClient(addr.IP, msg.Client) } if !msg.Types.Is(capture.SocketMSGTypeRequest) { return nil, nil } + client := &data.Client{Addr: msg.Client.Addr} + + if result := database.Read.Select([]string{"try_probe", "try_auth"}).First(client); result.Error != nil { + return nil, result.Error + } + msg = &capture.SocketMSG{ - Types: (capture.SocketMSGTypeResponse | capture.SocketMSGTypeClient), - Client: c.db.GetClient(msg.Client.Addr), + Types: (capture.SocketMSGTypeResponse | capture.SocketMSGTypeClient), + Client: &capture.WifiClient{ + Addr: msg.Client.Addr, + Time: time.Now(), + TryProbe: client.TryProbe, + TryAuth: client.TryAuth, + }, } if !ignore { diff --git a/controller/learn.go b/controller/learn.go new file mode 100644 index 0000000..a1384f5 --- /dev/null +++ b/controller/learn.go @@ -0,0 +1,57 @@ +package controller + +import ( + "net" + "time" + + // "github.com/bdlm/log" + "dev.sum7.eu/genofire/golang-lib/database" + + "dev.sum7.eu/genofire/wifictld-analyzer/capture" + "dev.sum7.eu/genofire/wifictld-analyzer/data" +) + +func (c *Controller) LearnClient(apIP net.IP, clientWifictl *capture.WifiClient) bool { + ret := false + + // learn ap + ap := &data.AP{ + IP: apIP, + Lastseen: time.Now(), + } + result := database.Read.First(ap) + if result.RowsAffected > 0 { + database.Write.Save(ap) + } else { + database.Write.Create(ap) + } + + // learn client + client := &data.Client{ + Addr: clientWifictl.Addr, + Lastseen: time.Now(), + APAddr: apIP, + Connected: clientWifictl.Connected, + SignalLowFreq: clientWifictl.SignalLowFreq, + SignalHighFreq: clientWifictl.SignalHighFreq, + } + + database.Write.FirstOrCreate(client) + + if clientWifictl.TryAuth > client.TryAuth { + client.TryAuth = clientWifictl.TryAuth + } + if clientWifictl.TryProbe > client.TryProbe { + client.TryProbe = clientWifictl.TryProbe + } + + if client.FreqHighest < clientWifictl.FreqHighest { + ret = (client.FreqHighest != 0) + client.FreqHighest = clientWifictl.FreqHighest + } + if clientWifictl.Authed { + client.Authed = clientWifictl.Authed + } + database.Write.Save(client) + return ret +} diff --git a/controller/main.go b/controller/main.go index dba5826..80a7281 100644 --- a/controller/main.go +++ b/controller/main.go @@ -4,23 +4,22 @@ import ( "net" "time" + "dev.sum7.eu/genofire/golang-lib/database" "github.com/bdlm/log" "dev.sum7.eu/genofire/wifictld-analyzer/capture" - "dev.sum7.eu/genofire/wifictld-analyzer/database" + "dev.sum7.eu/genofire/wifictld-analyzer/data" ) type Controller struct { SendTo func(addr *net.UDPAddr, msg *capture.SocketMSG) Send func(msg *capture.SocketMSG) - db *database.DB ticker *time.Ticker } -func NewController(db *database.DB) *Controller { +func NewController() *Controller { ctl := &Controller{ ticker: time.NewTicker(time.Minute), - db: db, } go ctl.Repeated() return ctl @@ -31,7 +30,12 @@ func (c *Controller) Close() { } func (c *Controller) Repeated() { + aps := 0 + clients := 0 + for range c.ticker.C { - log.Debugf("lerned: %d APs, %d Clients", len(c.db.APs), len(c.db.Clients)) + database.Read.Model(&data.AP{}).Count(&aps) + database.Read.Model(&data.Client{}).Count(&clients) + log.Debugf("learned: %d APs, %d Clients", aps, clients) } } diff --git a/data/ap.go b/data/ap.go new file mode 100644 index 0000000..91ebd3f --- /dev/null +++ b/data/ap.go @@ -0,0 +1,20 @@ +package data + +import ( + "net" + + "time" + + "dev.sum7.eu/genofire/golang-lib/database" +) + +type AP struct { + IP net.IP `json:"ip" gorm:"PRIMARY_KEY"` + Lastseen time.Time `json:"lastseen"` + Clients []Client `gorm:"foreignkey:APAddr" json:"-"` +} + +// Function to initialize the database +func init() { + database.AddModel(&AP{}) +} diff --git a/data/client.go b/data/client.go new file mode 100644 index 0000000..41d70ee --- /dev/null +++ b/data/client.go @@ -0,0 +1,26 @@ +package data + +import ( + "net" + "time" + + "dev.sum7.eu/genofire/golang-lib/database" +) + +type Client struct { + Addr HardwareAddr `gorm:"PRIMARY_KEY" json:"addr"` + APAddr net.IP `gorm:"column:ap" json:"ap"` + TryProbe uint16 `json:"try_probe"` + TryAuth uint16 `json:"try_auth"` + Connected bool `json:"connected"` + Authed bool `json:"authed"` + FreqHighest uint16 `json:"freq_highest"` + SignalLowFreq int16 `json:"signal_low_freq"` + SignalHighFreq int16 `json:"signal_high_freq"` + Lastseen time.Time `json:"lastseen"` +} + +// Function to initialize the database +func init() { + database.AddModel(&Client{}) +} diff --git a/data/hwaddr.go b/data/hwaddr.go index 11ffa78..f05e721 100644 --- a/data/hwaddr.go +++ b/data/hwaddr.go @@ -2,7 +2,12 @@ package data import "net" -type HardwareAddr struct{ net.HardwareAddr } +type HardwareAddr net.HardwareAddr + +//MarshalJSON to bytearray +func (a HardwareAddr) String() string { + return net.HardwareAddr(a).String() +} //MarshalJSON to bytearray func (a HardwareAddr) MarshalText() ([]byte, error) { @@ -10,7 +15,11 @@ func (a HardwareAddr) MarshalText() ([]byte, error) { } // UnmarshalJSON from bytearray -func (a HardwareAddr) UnmarshalText(data []byte) (err error) { - a.HardwareAddr, err = net.ParseMAC(string(data)) - return +func (a HardwareAddr) UnmarshalText(data []byte) error { + b, err := net.ParseMAC(string(data)) + if err != nil { + return err + } + a = HardwareAddr(b) + return nil } diff --git a/database/ap.go b/database/ap.go deleted file mode 100644 index d9ab148..0000000 --- a/database/ap.go +++ /dev/null @@ -1,22 +0,0 @@ -package database - -import ( - "net" - - "github.com/FreifunkBremen/yanic/lib/jsontime" -) - -type AP struct { - IP *net.IP `json:"ip"` - Lastseen jsontime.Time `json:"lastseen"` -} - -func (db *DB) GetClients(ap *AP) []*Client { - var clients []*Client - for _, client := range db.Clients { - if client.AP == ap { - clients = append(clients, client) - } - } - return clients -} diff --git a/database/client.go b/database/client.go deleted file mode 100644 index 1c8857f..0000000 --- a/database/client.go +++ /dev/null @@ -1,84 +0,0 @@ -package database - -import ( - "net" - "time" - - // "github.com/bdlm/log" - "github.com/FreifunkBremen/yanic/lib/jsontime" - - "dev.sum7.eu/genofire/wifictld-analyzer/capture" - "dev.sum7.eu/genofire/wifictld-analyzer/data" -) - -type Client struct { - AP *AP `json:"-"` - APAddr string `json:"ap"` - Addr data.HardwareAddr `json:"-"` - TryProbe uint16 `json:"try_probe"` - TryAuth uint16 `json:"try_auth"` - Connected bool `json:"connected"` - Authed bool `json:"authed"` - FreqHighest uint16 `json:"freq_highest"` - SignalLowFreq int16 `json:"signal_low_freq"` - SignalHighFreq int16 `json:"signal_high_freq"` - Lastseen jsontime.Time `json:"lastseen"` -} - -func (db *DB) LearnClient(apIP net.IP, clientWifictl *capture.WifiClient) bool { - ret := false - - // learn ap - apAddr := apIP.String() - ap, ok := db.APs[apAddr] - if !ok { - ap = &AP{} - db.APs[apAddr] = ap - } - ap.IP = &apIP - ap.Lastseen = jsontime.Now() - - // learn client - clientAddr := clientWifictl.Addr.String() - client, ok := db.Clients[clientAddr] - if !ok { - client = &Client{ - Addr: clientWifictl.Addr, - } - db.Clients[clientAddr] = client - } - client.Lastseen = jsontime.Now() - client.AP = ap - client.APAddr = apAddr - client.Connected = clientWifictl.Connected - client.SignalLowFreq = clientWifictl.SignalLowFreq - client.SignalHighFreq = clientWifictl.SignalHighFreq - - if clientWifictl.TryAuth > client.TryAuth { - client.TryAuth = clientWifictl.TryAuth - } - if clientWifictl.TryProbe > client.TryProbe { - client.TryProbe = clientWifictl.TryProbe - } - - if client.FreqHighest < clientWifictl.FreqHighest { - ret = (client.FreqHighest != 0) - client.FreqHighest = clientWifictl.FreqHighest - } - if clientWifictl.Authed { - client.Authed = clientWifictl.Authed - } - return ret -} - -func (db *DB) GetClient(addr data.HardwareAddr) *capture.WifiClient { - client, ok := db.Clients[addr.String()] - wClient := &capture.WifiClient{ - Addr: addr, - Time: time.Now(), - } - if ok { - wClient.TryProbe = client.TryProbe - } - return wClient -} diff --git a/database/main.go b/database/main.go deleted file mode 100644 index cf2ab84..0000000 --- a/database/main.go +++ /dev/null @@ -1,47 +0,0 @@ -package database - -import ( - "time" - - "github.com/bdlm/log" - - "dev.sum7.eu/genofire/golang-lib/file" - "dev.sum7.eu/genofire/golang-lib/worker" -) - -type DB struct { - Clients map[string]*Client `json:"client"` - APs map[string]*AP `json:"ap"` - worker *worker.Worker -} - -func NewDB(path string) *DB { - db := &DB{ - Clients: make(map[string]*Client), - APs: make(map[string]*AP), - } - - file.ReadJSON(path, db) - - for addr, client := range db.Clients { - client.Addr.UnmarshalText([]byte(addr)) - if ap, ok := db.APs[client.APAddr]; ok { - client.AP = ap - } - } - - db.worker = worker.NewWorker(time.Minute, func() { - file.SaveJSON(path, db) - log.Debug("save db state") - }) - - db.worker.Start() - - return db -} - -func (db *DB) Close() { - if db.worker != nil { - db.worker.Close() - } -} diff --git a/database/main_test.go b/database/main_test.go deleted file mode 100644 index 636bab8..0000000 --- a/database/main_test.go +++ /dev/null @@ -1 +0,0 @@ -package database diff --git a/web/webserver.go b/web/webserver.go index fef33d7..b1f301e 100644 --- a/web/webserver.go +++ b/web/webserver.go @@ -7,9 +7,12 @@ import ( "github.com/NYTimes/gziphandler" "github.com/bdlm/log" + "dev.sum7.eu/genofire/golang-lib/database" + lib "dev.sum7.eu/genofire/golang-lib/http" "dev.sum7.eu/genofire/golang-lib/websocket" "dev.sum7.eu/genofire/wifictld-analyzer/capture" + "dev.sum7.eu/genofire/wifictld-analyzer/data" ) type Server struct { @@ -22,6 +25,26 @@ func New(config *Config) *Server { ws := websocket.NewWebsocketHandlerService() ws.Listen("/ws") + http.HandleFunc("/data.json", func(w http.ResponseWriter, r *http.Request) { + type dataResponse struct { + APs []data.AP `json:"aps"` + Clients []data.Client `json:"clients"` + } + data := &dataResponse{} + if result := database.Read.Find(&data.APs); result.Error != nil { + log.WithField("error", result.Error.Error()).Warn("not possible to read APs") + http.Error(w, "not possible to read APs", http.StatusNotFound) + return + } + if result := database.Read.Find(&data.Clients); result.Error != nil { + log.WithField("error", result.Error.Error()).Warn("not possible to read Clients") + http.Error(w, "not possible to read Clients", http.StatusNotFound) + return + } + lib.Write(w, data) + log.Info("fetch data") + }) + http.Handle("/", gziphandler.GzipHandler(http.FileServer(http.Dir(config.Webroot)))) return &Server{ web: &http.Server{