From cbe2577aa9ce138bd8f134932dd92f1a9896a625 Mon Sep 17 00:00:00 2001 From: Julian Kornberger Date: Sat, 8 Oct 2016 10:50:41 +0200 Subject: [PATCH 1/3] Expire nodes after n days of inactivity closes #4 --- config_example.yml | 12 +++++++-- jsontime/jsontime.go | 6 ++++- models/config.go | 3 ++- models/nodes.go | 60 +++++++++++++++++++++++++++++++------------- 4 files changed, 60 insertions(+), 21 deletions(-) diff --git a/config_example.yml b/config_example.yml index b28551c..2aae894 100644 --- a/config_example.yml +++ b/config_example.yml @@ -2,7 +2,9 @@ respondd: enable: true interface: eth0 - collectinterval: 15 + + # Collected data every n seconds + collectinterval: 60 webserver: enable: false port: 8080 @@ -16,9 +18,15 @@ nodes: nodes_path: /var/www/html/meshviewer/data/nodes_all.json nodesmini_path: /var/www/html/meshviewer/data/nodes.json graphs_path: /var/www/html/meshviewer/data/graph.json - saveinterval: 5 aliases_enable: false aliases_path: /var/www/html/meshviewer/data/aliases.json + + # Export nodes and graph every n seconds + saveinterval: 5 + + # Expire offline nodes after n days + max_age: 7 + influxdb: enable: false host: http://localhost:8086 diff --git a/jsontime/jsontime.go b/jsontime/jsontime.go index 399f78a..fb97f5c 100644 --- a/jsontime/jsontime.go +++ b/jsontime/jsontime.go @@ -29,7 +29,7 @@ func (t *Time) UnmarshalJSON(data []byte) (err error) { } return } -func (t Time) GetTime() time.Time{ +func (t Time) GetTime() time.Time { return t.time } func (t Time) Unix() int64 { @@ -46,3 +46,7 @@ func (t Time) Add(d time.Duration) Time { func (t Time) After(u Time) bool { return t.time.After(u.GetTime()) } + +func (t Time) Before(u Time) bool { + return t.time.Before(u.GetTime()) +} diff --git a/models/config.go b/models/config.go index 8e04e0c..fdd20e6 100644 --- a/models/config.go +++ b/models/config.go @@ -31,7 +31,8 @@ type Config struct { NodesMiniPath string `yaml:"nodesmini_path"` GraphsPath string `yaml:"graphs_path"` AliasesPath string `yaml:"aliases_path"` - SaveInterval int `yaml:"saveinterval"` + SaveInterval int `yaml:"saveinterval"` // Save nodes every n seconds + MaxAge int `yaml:"max_age"` // Remove nodes after n days of inactivity } `yaml:"nodes"` Influxdb struct { Enable bool `yaml:"enable"` diff --git a/models/nodes.go b/models/nodes.go index 9eaa516..d0e2a18 100644 --- a/models/nodes.go +++ b/models/nodes.go @@ -145,27 +145,53 @@ func (nodes *Nodes) worker() { c := time.Tick(time.Second * time.Duration(nodes.config.Nodes.SaveInterval)) for range c { - log.Println("saving", len(nodes.List), "nodes") - nodes.Timestamp = jsontime.Now() - nodes.Lock() - // - // set node as offline (without statistics) - for _, node := range nodes.List { - if node.Statistics != nil && nodes.Timestamp.After(node.Lastseen.Add(time.Second*time.Duration(10*nodes.config.Respondd.CollectInterval))) { - if node.Flags != nil { - node.Flags.Online = false - } + nodes.expire() + nodes.save() + } +} + +// Expires nodes and set nodes offline +func (nodes *Nodes) expire() { + nodes.Timestamp = jsontime.Now() + + // Nodes last seen before expireTime will be removed + maxAge := nodes.config.Nodes.MaxAge + if maxAge <= 0 { + maxAge = 7 // our default + } + expireTime := nodes.Timestamp.Add(-time.Duration(maxAge) * time.Hour * 24) + + // Nodes last seen before offlineTime are changed to 'offline' + offlineTime := nodes.Timestamp.Add(-time.Minute * 10) + + // Locking foo + nodes.Lock() + defer nodes.Unlock() + + for id, node := range nodes.List { + if node.Lastseen.Before(expireTime) { + // expire + delete(nodes.List, id) + } else if node.Lastseen.Before(offlineTime) { + // set to offline + if node.Flags != nil { + node.Flags.Online = false } } - // serialize nodes - save(nodes, nodes.config.Nodes.NodesPath) - save(nodes.GetNodesMini(), nodes.config.Nodes.NodesMiniPath) + } +} - if path := nodes.config.Nodes.GraphsPath; path != "" { - save(nodes.BuildGraph(), path) - } +func (nodes *Nodes) save() { + // Locking foo + nodes.RLock() + defer nodes.RUnlock() - nodes.Unlock() + // serialize nodes + save(nodes, nodes.config.Nodes.NodesPath) + save(nodes.GetNodesMini(), nodes.config.Nodes.NodesMiniPath) + + if path := nodes.config.Nodes.GraphsPath; path != "" { + save(nodes.BuildGraph(), path) } } From e9420828a24f8356ec97dfc7b206ee123d9734d7 Mon Sep 17 00:00:00 2001 From: Julian Kornberger Date: Sat, 8 Oct 2016 11:41:08 +0200 Subject: [PATCH 2/3] Remove indirection of Flags Simplifies the code --- meshviewer/meshviewer.go | 2 +- models/graph.go | 2 +- models/graph_test.go | 6 +++++- models/node.go | 12 ++++++------ models/nodes.go | 13 ++----------- 5 files changed, 15 insertions(+), 20 deletions(-) diff --git a/meshviewer/meshviewer.go b/meshviewer/meshviewer.go index 663f661..e2a8723 100644 --- a/meshviewer/meshviewer.go +++ b/meshviewer/meshviewer.go @@ -11,7 +11,7 @@ import ( type Node struct { Firstseen jsontime.Time `json:"firstseen"` Lastseen jsontime.Time `json:"lastseen"` - Flags *Flags `json:"flags,omitempty"` + Flags Flags `json:"flags"` Statistics *Statistics `json:"statistics"` Nodeinfo *data.NodeInfo `json:"nodeinfo"` Neighbours *data.Neighbours `json:"-"` diff --git a/models/graph.go b/models/graph.go index f182be3..3fb35f2 100644 --- a/models/graph.go +++ b/models/graph.go @@ -78,7 +78,7 @@ func (builder *GraphBuilder) readNodes(nodes map[string]*Node) { // Add links for sourceID, node := range nodes { - if flag := node.Flags; flag == nil || flag.Online { + if node.Flags.Online { if neighbours := node.Neighbours; neighbours != nil { // Batman neighbours for _, batadvNeighbours := range neighbours.Batadv { diff --git a/models/graph_test.go b/models/graph_test.go index 816e211..023996d 100644 --- a/models/graph_test.go +++ b/models/graph_test.go @@ -35,7 +35,11 @@ func testGetNodesByFile(files ...string) *Nodes { } for _, file := range files { - nodes.List[file] = testGetNodeByFile(file) + node := testGetNodeByFile(file) + nodes.Update(file, &data.ResponseData{ + NodeInfo: node.Nodeinfo, + Neighbours: node.Neighbours, + }) } return nodes diff --git a/models/node.go b/models/node.go index d871a62..5359ac6 100644 --- a/models/node.go +++ b/models/node.go @@ -11,12 +11,12 @@ import ( // Node struct type Node struct { - Firstseen jsontime.Time `json:"firstseen"` - Lastseen jsontime.Time `json:"lastseen"` - Flags *meshviewer.Flags `json:"flags,omitempty"` - Statistics *data.Statistics `json:"statistics"` - Nodeinfo *data.NodeInfo `json:"nodeinfo"` - Neighbours *data.Neighbours `json:"-"` + Firstseen jsontime.Time `json:"firstseen"` + Lastseen jsontime.Time `json:"lastseen"` + Flags meshviewer.Flags `json:"flags"` + Statistics *data.Statistics `json:"statistics"` + Nodeinfo *data.NodeInfo `json:"nodeinfo"` + Neighbours *data.Neighbours `json:"-"` } // Returns tags and fields for InfluxDB diff --git a/models/nodes.go b/models/nodes.go index d0e2a18..7488022 100644 --- a/models/nodes.go +++ b/models/nodes.go @@ -56,20 +56,13 @@ func (nodes *Nodes) Update(nodeID string, res *data.ResponseData) *Node { if node == nil { node = &Node{ Firstseen: now, - Flags: &meshviewer.Flags{ - Online: true, - Gateway: false, - }, } nodes.List[nodeID] = node } nodes.Unlock() node.Lastseen = now - - if node.Flags != nil { - node.Flags.Online = true - } + node.Flags.Online = true // Update neighbours if val := res.Neighbours; val != nil { @@ -174,9 +167,7 @@ func (nodes *Nodes) expire() { delete(nodes.List, id) } else if node.Lastseen.Before(offlineTime) { // set to offline - if node.Flags != nil { - node.Flags.Online = false - } + node.Flags.Online = false } } } From 997735ffce4f4cfa121b2794d495fd508f59f7d0 Mon Sep 17 00:00:00 2001 From: Julian Kornberger Date: Sat, 8 Oct 2016 11:52:11 +0200 Subject: [PATCH 3/3] Add test for node expiration --- models/nodes_test.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/models/nodes_test.go b/models/nodes_test.go index 50006e2..970a110 100644 --- a/models/nodes_test.go +++ b/models/nodes_test.go @@ -4,11 +4,45 @@ import ( "io/ioutil" "os" "testing" + "time" "github.com/FreifunkBremen/respond-collector/data" "github.com/stretchr/testify/assert" ) +func TestExpire(t *testing.T) { + assert := assert.New(t) + config := &Config{} + config.Nodes.MaxAge = 6 + nodes := &Nodes{ + config: config, + List: make(map[string]*Node), + } + + nodes.Update("expire", &data.ResponseData{}) // should expire + nodes.Update("offline", &data.ResponseData{}) // should become offline + nodes.Update("online", &data.ResponseData{}) // should stay online + + expire := nodes.List["expire"] + expire.Lastseen = expire.Lastseen.Add((-6 * time.Hour * 24) - time.Minute) + offline := nodes.List["offline"] + offline.Lastseen = offline.Lastseen.Add((-6 * time.Hour * 24) + time.Minute) + + nodes.expire() + + // one expired? + assert.Equal(2, len(nodes.List)) + assert.Nil(nodes.List["expire"]) + + // one offline? + assert.NotNil(nodes.List["offline"]) + assert.False(nodes.List["offline"].Flags.Online) + + // one online? + assert.NotNil(nodes.List["online"]) + assert.True(nodes.List["online"].Flags.Online) +} + func TestLoadAndSave(t *testing.T) { assert := assert.New(t)