Merge pull request #5 from FreifunkBremen/nodes_expiration

Expire nodes after n days of inactivity
(Ist okay so -> gleich mal ein Test auf meiner Maschine ...)
This commit is contained in:
Geno 2016-10-09 18:49:31 +02:00 committed by GitHub
commit 38f32952c2
9 changed files with 108 additions and 40 deletions

View File

@ -2,7 +2,9 @@
respondd: respondd:
enable: true enable: true
interface: eth0 interface: eth0
collectinterval: 15
# Collected data every n seconds
collectinterval: 60
webserver: webserver:
enable: false enable: false
port: 8080 port: 8080
@ -16,9 +18,15 @@ nodes:
nodes_path: /var/www/html/meshviewer/data/nodes_all.json nodes_path: /var/www/html/meshviewer/data/nodes_all.json
nodesmini_path: /var/www/html/meshviewer/data/nodes.json nodesmini_path: /var/www/html/meshviewer/data/nodes.json
graphs_path: /var/www/html/meshviewer/data/graph.json graphs_path: /var/www/html/meshviewer/data/graph.json
saveinterval: 5
aliases_enable: false aliases_enable: false
aliases_path: /var/www/html/meshviewer/data/aliases.json aliases_path: /var/www/html/meshviewer/data/aliases.json
# Export nodes and graph every n seconds
saveinterval: 5
# Expire offline nodes after n days
max_age: 7
influxdb: influxdb:
enable: false enable: false
host: http://localhost:8086 host: http://localhost:8086

View File

@ -46,3 +46,7 @@ func (t Time) Add(d time.Duration) Time {
func (t Time) After(u Time) bool { func (t Time) After(u Time) bool {
return t.time.After(u.GetTime()) return t.time.After(u.GetTime())
} }
func (t Time) Before(u Time) bool {
return t.time.Before(u.GetTime())
}

View File

@ -11,7 +11,7 @@ import (
type Node struct { type Node struct {
Firstseen jsontime.Time `json:"firstseen"` Firstseen jsontime.Time `json:"firstseen"`
Lastseen jsontime.Time `json:"lastseen"` Lastseen jsontime.Time `json:"lastseen"`
Flags *Flags `json:"flags,omitempty"` Flags Flags `json:"flags"`
Statistics *Statistics `json:"statistics"` Statistics *Statistics `json:"statistics"`
Nodeinfo *data.NodeInfo `json:"nodeinfo"` Nodeinfo *data.NodeInfo `json:"nodeinfo"`
Neighbours *data.Neighbours `json:"-"` Neighbours *data.Neighbours `json:"-"`

View File

@ -31,7 +31,8 @@ type Config struct {
NodesMiniPath string `yaml:"nodesmini_path"` NodesMiniPath string `yaml:"nodesmini_path"`
GraphsPath string `yaml:"graphs_path"` GraphsPath string `yaml:"graphs_path"`
AliasesPath string `yaml:"aliases_path"` AliasesPath string `yaml:"aliases_path"`
SaveInterval int `yaml:"saveinterval"` SaveInterval int `yaml:"saveinterval"` // Save nodes every n seconds
MaxAge int `yaml:"max_age"` // Remove nodes after n days of inactivity
} `yaml:"nodes"` } `yaml:"nodes"`
Influxdb struct { Influxdb struct {
Enable bool `yaml:"enable"` Enable bool `yaml:"enable"`

View File

@ -78,7 +78,7 @@ func (builder *GraphBuilder) readNodes(nodes map[string]*Node) {
// Add links // Add links
for sourceID, node := range nodes { for sourceID, node := range nodes {
if flag := node.Flags; flag == nil || flag.Online { if node.Flags.Online {
if neighbours := node.Neighbours; neighbours != nil { if neighbours := node.Neighbours; neighbours != nil {
// Batman neighbours // Batman neighbours
for _, batadvNeighbours := range neighbours.Batadv { for _, batadvNeighbours := range neighbours.Batadv {

View File

@ -35,7 +35,11 @@ func testGetNodesByFile(files ...string) *Nodes {
} }
for _, file := range files { for _, file := range files {
nodes.List[file] = testGetNodeByFile(file) node := testGetNodeByFile(file)
nodes.Update(file, &data.ResponseData{
NodeInfo: node.Nodeinfo,
Neighbours: node.Neighbours,
})
} }
return nodes return nodes

View File

@ -13,7 +13,7 @@ import (
type Node struct { type Node struct {
Firstseen jsontime.Time `json:"firstseen"` Firstseen jsontime.Time `json:"firstseen"`
Lastseen jsontime.Time `json:"lastseen"` Lastseen jsontime.Time `json:"lastseen"`
Flags *meshviewer.Flags `json:"flags,omitempty"` Flags meshviewer.Flags `json:"flags"`
Statistics *data.Statistics `json:"statistics"` Statistics *data.Statistics `json:"statistics"`
Nodeinfo *data.NodeInfo `json:"nodeinfo"` Nodeinfo *data.NodeInfo `json:"nodeinfo"`
Neighbours *data.Neighbours `json:"-"` Neighbours *data.Neighbours `json:"-"`

View File

@ -56,20 +56,13 @@ func (nodes *Nodes) Update(nodeID string, res *data.ResponseData) *Node {
if node == nil { if node == nil {
node = &Node{ node = &Node{
Firstseen: now, Firstseen: now,
Flags: &meshviewer.Flags{
Online: true,
Gateway: false,
},
} }
nodes.List[nodeID] = node nodes.List[nodeID] = node
} }
nodes.Unlock() nodes.Unlock()
node.Lastseen = now node.Lastseen = now
if node.Flags != nil {
node.Flags.Online = true node.Flags.Online = true
}
// Update neighbours // Update neighbours
if val := res.Neighbours; val != nil { if val := res.Neighbours; val != nil {
@ -145,18 +138,45 @@ func (nodes *Nodes) worker() {
c := time.Tick(time.Second * time.Duration(nodes.config.Nodes.SaveInterval)) c := time.Tick(time.Second * time.Duration(nodes.config.Nodes.SaveInterval))
for range c { for range c {
log.Println("saving", len(nodes.List), "nodes") nodes.expire()
nodes.save()
}
}
// Expires nodes and set nodes offline
func (nodes *Nodes) expire() {
nodes.Timestamp = jsontime.Now() nodes.Timestamp = jsontime.Now()
// Nodes last seen before expireTime will be removed
maxAge := nodes.config.Nodes.MaxAge
if maxAge <= 0 {
maxAge = 7 // our default
}
expireTime := nodes.Timestamp.Add(-time.Duration(maxAge) * time.Hour * 24)
// Nodes last seen before offlineTime are changed to 'offline'
offlineTime := nodes.Timestamp.Add(-time.Minute * 10)
// Locking foo
nodes.Lock() nodes.Lock()
// defer nodes.Unlock()
// set node as offline (without statistics)
for _, node := range nodes.List { for id, node := range nodes.List {
if node.Statistics != nil && nodes.Timestamp.After(node.Lastseen.Add(time.Second*time.Duration(10*nodes.config.Respondd.CollectInterval))) { if node.Lastseen.Before(expireTime) {
if node.Flags != nil { // expire
delete(nodes.List, id)
} else if node.Lastseen.Before(offlineTime) {
// set to offline
node.Flags.Online = false node.Flags.Online = false
} }
} }
} }
func (nodes *Nodes) save() {
// Locking foo
nodes.RLock()
defer nodes.RUnlock()
// serialize nodes // serialize nodes
save(nodes, nodes.config.Nodes.NodesPath) save(nodes, nodes.config.Nodes.NodesPath)
save(nodes.GetNodesMini(), nodes.config.Nodes.NodesMiniPath) save(nodes.GetNodesMini(), nodes.config.Nodes.NodesMiniPath)
@ -164,9 +184,6 @@ func (nodes *Nodes) worker() {
if path := nodes.config.Nodes.GraphsPath; path != "" { if path := nodes.config.Nodes.GraphsPath; path != "" {
save(nodes.BuildGraph(), path) save(nodes.BuildGraph(), path)
} }
nodes.Unlock()
}
} }
// Returns global statistics for InfluxDB // Returns global statistics for InfluxDB

View File

@ -4,11 +4,45 @@ import (
"io/ioutil" "io/ioutil"
"os" "os"
"testing" "testing"
"time"
"github.com/FreifunkBremen/respond-collector/data" "github.com/FreifunkBremen/respond-collector/data"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
func TestExpire(t *testing.T) {
assert := assert.New(t)
config := &Config{}
config.Nodes.MaxAge = 6
nodes := &Nodes{
config: config,
List: make(map[string]*Node),
}
nodes.Update("expire", &data.ResponseData{}) // should expire
nodes.Update("offline", &data.ResponseData{}) // should become offline
nodes.Update("online", &data.ResponseData{}) // should stay online
expire := nodes.List["expire"]
expire.Lastseen = expire.Lastseen.Add((-6 * time.Hour * 24) - time.Minute)
offline := nodes.List["offline"]
offline.Lastseen = offline.Lastseen.Add((-6 * time.Hour * 24) + time.Minute)
nodes.expire()
// one expired?
assert.Equal(2, len(nodes.List))
assert.Nil(nodes.List["expire"])
// one offline?
assert.NotNil(nodes.List["offline"])
assert.False(nodes.List["offline"].Flags.Online)
// one online?
assert.NotNil(nodes.List["online"])
assert.True(nodes.List["online"].Flags.Online)
}
func TestLoadAndSave(t *testing.T) { func TestLoadAndSave(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)