Merge pull request #5 from FreifunkBremen/nodes_expiration
Expire nodes after n days of inactivity (Ist okay so -> gleich mal ein Test auf meiner Maschine ...)
This commit is contained in:
commit
38f32952c2
|
@ -2,7 +2,9 @@
|
||||||
respondd:
|
respondd:
|
||||||
enable: true
|
enable: true
|
||||||
interface: eth0
|
interface: eth0
|
||||||
collectinterval: 15
|
|
||||||
|
# Collected data every n seconds
|
||||||
|
collectinterval: 60
|
||||||
webserver:
|
webserver:
|
||||||
enable: false
|
enable: false
|
||||||
port: 8080
|
port: 8080
|
||||||
|
@ -16,9 +18,15 @@ nodes:
|
||||||
nodes_path: /var/www/html/meshviewer/data/nodes_all.json
|
nodes_path: /var/www/html/meshviewer/data/nodes_all.json
|
||||||
nodesmini_path: /var/www/html/meshviewer/data/nodes.json
|
nodesmini_path: /var/www/html/meshviewer/data/nodes.json
|
||||||
graphs_path: /var/www/html/meshviewer/data/graph.json
|
graphs_path: /var/www/html/meshviewer/data/graph.json
|
||||||
saveinterval: 5
|
|
||||||
aliases_enable: false
|
aliases_enable: false
|
||||||
aliases_path: /var/www/html/meshviewer/data/aliases.json
|
aliases_path: /var/www/html/meshviewer/data/aliases.json
|
||||||
|
|
||||||
|
# Export nodes and graph every n seconds
|
||||||
|
saveinterval: 5
|
||||||
|
|
||||||
|
# Expire offline nodes after n days
|
||||||
|
max_age: 7
|
||||||
|
|
||||||
influxdb:
|
influxdb:
|
||||||
enable: false
|
enable: false
|
||||||
host: http://localhost:8086
|
host: http://localhost:8086
|
||||||
|
|
|
@ -29,7 +29,7 @@ func (t *Time) UnmarshalJSON(data []byte) (err error) {
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
func (t Time) GetTime() time.Time{
|
func (t Time) GetTime() time.Time {
|
||||||
return t.time
|
return t.time
|
||||||
}
|
}
|
||||||
func (t Time) Unix() int64 {
|
func (t Time) Unix() int64 {
|
||||||
|
@ -46,3 +46,7 @@ func (t Time) Add(d time.Duration) Time {
|
||||||
func (t Time) After(u Time) bool {
|
func (t Time) After(u Time) bool {
|
||||||
return t.time.After(u.GetTime())
|
return t.time.After(u.GetTime())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (t Time) Before(u Time) bool {
|
||||||
|
return t.time.Before(u.GetTime())
|
||||||
|
}
|
||||||
|
|
|
@ -11,7 +11,7 @@ import (
|
||||||
type Node struct {
|
type Node struct {
|
||||||
Firstseen jsontime.Time `json:"firstseen"`
|
Firstseen jsontime.Time `json:"firstseen"`
|
||||||
Lastseen jsontime.Time `json:"lastseen"`
|
Lastseen jsontime.Time `json:"lastseen"`
|
||||||
Flags *Flags `json:"flags,omitempty"`
|
Flags Flags `json:"flags"`
|
||||||
Statistics *Statistics `json:"statistics"`
|
Statistics *Statistics `json:"statistics"`
|
||||||
Nodeinfo *data.NodeInfo `json:"nodeinfo"`
|
Nodeinfo *data.NodeInfo `json:"nodeinfo"`
|
||||||
Neighbours *data.Neighbours `json:"-"`
|
Neighbours *data.Neighbours `json:"-"`
|
||||||
|
|
|
@ -31,7 +31,8 @@ type Config struct {
|
||||||
NodesMiniPath string `yaml:"nodesmini_path"`
|
NodesMiniPath string `yaml:"nodesmini_path"`
|
||||||
GraphsPath string `yaml:"graphs_path"`
|
GraphsPath string `yaml:"graphs_path"`
|
||||||
AliasesPath string `yaml:"aliases_path"`
|
AliasesPath string `yaml:"aliases_path"`
|
||||||
SaveInterval int `yaml:"saveinterval"`
|
SaveInterval int `yaml:"saveinterval"` // Save nodes every n seconds
|
||||||
|
MaxAge int `yaml:"max_age"` // Remove nodes after n days of inactivity
|
||||||
} `yaml:"nodes"`
|
} `yaml:"nodes"`
|
||||||
Influxdb struct {
|
Influxdb struct {
|
||||||
Enable bool `yaml:"enable"`
|
Enable bool `yaml:"enable"`
|
||||||
|
|
|
@ -78,7 +78,7 @@ func (builder *GraphBuilder) readNodes(nodes map[string]*Node) {
|
||||||
|
|
||||||
// Add links
|
// Add links
|
||||||
for sourceID, node := range nodes {
|
for sourceID, node := range nodes {
|
||||||
if flag := node.Flags; flag == nil || flag.Online {
|
if node.Flags.Online {
|
||||||
if neighbours := node.Neighbours; neighbours != nil {
|
if neighbours := node.Neighbours; neighbours != nil {
|
||||||
// Batman neighbours
|
// Batman neighbours
|
||||||
for _, batadvNeighbours := range neighbours.Batadv {
|
for _, batadvNeighbours := range neighbours.Batadv {
|
||||||
|
|
|
@ -35,7 +35,11 @@ func testGetNodesByFile(files ...string) *Nodes {
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, file := range files {
|
for _, file := range files {
|
||||||
nodes.List[file] = testGetNodeByFile(file)
|
node := testGetNodeByFile(file)
|
||||||
|
nodes.Update(file, &data.ResponseData{
|
||||||
|
NodeInfo: node.Nodeinfo,
|
||||||
|
Neighbours: node.Neighbours,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
return nodes
|
return nodes
|
||||||
|
|
|
@ -13,7 +13,7 @@ import (
|
||||||
type Node struct {
|
type Node struct {
|
||||||
Firstseen jsontime.Time `json:"firstseen"`
|
Firstseen jsontime.Time `json:"firstseen"`
|
||||||
Lastseen jsontime.Time `json:"lastseen"`
|
Lastseen jsontime.Time `json:"lastseen"`
|
||||||
Flags *meshviewer.Flags `json:"flags,omitempty"`
|
Flags meshviewer.Flags `json:"flags"`
|
||||||
Statistics *data.Statistics `json:"statistics"`
|
Statistics *data.Statistics `json:"statistics"`
|
||||||
Nodeinfo *data.NodeInfo `json:"nodeinfo"`
|
Nodeinfo *data.NodeInfo `json:"nodeinfo"`
|
||||||
Neighbours *data.Neighbours `json:"-"`
|
Neighbours *data.Neighbours `json:"-"`
|
||||||
|
|
|
@ -56,20 +56,13 @@ func (nodes *Nodes) Update(nodeID string, res *data.ResponseData) *Node {
|
||||||
if node == nil {
|
if node == nil {
|
||||||
node = &Node{
|
node = &Node{
|
||||||
Firstseen: now,
|
Firstseen: now,
|
||||||
Flags: &meshviewer.Flags{
|
|
||||||
Online: true,
|
|
||||||
Gateway: false,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
nodes.List[nodeID] = node
|
nodes.List[nodeID] = node
|
||||||
}
|
}
|
||||||
nodes.Unlock()
|
nodes.Unlock()
|
||||||
|
|
||||||
node.Lastseen = now
|
node.Lastseen = now
|
||||||
|
|
||||||
if node.Flags != nil {
|
|
||||||
node.Flags.Online = true
|
node.Flags.Online = true
|
||||||
}
|
|
||||||
|
|
||||||
// Update neighbours
|
// Update neighbours
|
||||||
if val := res.Neighbours; val != nil {
|
if val := res.Neighbours; val != nil {
|
||||||
|
@ -145,18 +138,45 @@ func (nodes *Nodes) worker() {
|
||||||
c := time.Tick(time.Second * time.Duration(nodes.config.Nodes.SaveInterval))
|
c := time.Tick(time.Second * time.Duration(nodes.config.Nodes.SaveInterval))
|
||||||
|
|
||||||
for range c {
|
for range c {
|
||||||
log.Println("saving", len(nodes.List), "nodes")
|
nodes.expire()
|
||||||
|
nodes.save()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Expires nodes and set nodes offline
|
||||||
|
func (nodes *Nodes) expire() {
|
||||||
nodes.Timestamp = jsontime.Now()
|
nodes.Timestamp = jsontime.Now()
|
||||||
|
|
||||||
|
// Nodes last seen before expireTime will be removed
|
||||||
|
maxAge := nodes.config.Nodes.MaxAge
|
||||||
|
if maxAge <= 0 {
|
||||||
|
maxAge = 7 // our default
|
||||||
|
}
|
||||||
|
expireTime := nodes.Timestamp.Add(-time.Duration(maxAge) * time.Hour * 24)
|
||||||
|
|
||||||
|
// Nodes last seen before offlineTime are changed to 'offline'
|
||||||
|
offlineTime := nodes.Timestamp.Add(-time.Minute * 10)
|
||||||
|
|
||||||
|
// Locking foo
|
||||||
nodes.Lock()
|
nodes.Lock()
|
||||||
//
|
defer nodes.Unlock()
|
||||||
// set node as offline (without statistics)
|
|
||||||
for _, node := range nodes.List {
|
for id, node := range nodes.List {
|
||||||
if node.Statistics != nil && nodes.Timestamp.After(node.Lastseen.Add(time.Second*time.Duration(10*nodes.config.Respondd.CollectInterval))) {
|
if node.Lastseen.Before(expireTime) {
|
||||||
if node.Flags != nil {
|
// expire
|
||||||
|
delete(nodes.List, id)
|
||||||
|
} else if node.Lastseen.Before(offlineTime) {
|
||||||
|
// set to offline
|
||||||
node.Flags.Online = false
|
node.Flags.Online = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (nodes *Nodes) save() {
|
||||||
|
// Locking foo
|
||||||
|
nodes.RLock()
|
||||||
|
defer nodes.RUnlock()
|
||||||
|
|
||||||
// serialize nodes
|
// serialize nodes
|
||||||
save(nodes, nodes.config.Nodes.NodesPath)
|
save(nodes, nodes.config.Nodes.NodesPath)
|
||||||
save(nodes.GetNodesMini(), nodes.config.Nodes.NodesMiniPath)
|
save(nodes.GetNodesMini(), nodes.config.Nodes.NodesMiniPath)
|
||||||
|
@ -164,9 +184,6 @@ func (nodes *Nodes) worker() {
|
||||||
if path := nodes.config.Nodes.GraphsPath; path != "" {
|
if path := nodes.config.Nodes.GraphsPath; path != "" {
|
||||||
save(nodes.BuildGraph(), path)
|
save(nodes.BuildGraph(), path)
|
||||||
}
|
}
|
||||||
|
|
||||||
nodes.Unlock()
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Returns global statistics for InfluxDB
|
// Returns global statistics for InfluxDB
|
||||||
|
|
|
@ -4,11 +4,45 @@ import (
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"os"
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/FreifunkBremen/respond-collector/data"
|
"github.com/FreifunkBremen/respond-collector/data"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestExpire(t *testing.T) {
|
||||||
|
assert := assert.New(t)
|
||||||
|
config := &Config{}
|
||||||
|
config.Nodes.MaxAge = 6
|
||||||
|
nodes := &Nodes{
|
||||||
|
config: config,
|
||||||
|
List: make(map[string]*Node),
|
||||||
|
}
|
||||||
|
|
||||||
|
nodes.Update("expire", &data.ResponseData{}) // should expire
|
||||||
|
nodes.Update("offline", &data.ResponseData{}) // should become offline
|
||||||
|
nodes.Update("online", &data.ResponseData{}) // should stay online
|
||||||
|
|
||||||
|
expire := nodes.List["expire"]
|
||||||
|
expire.Lastseen = expire.Lastseen.Add((-6 * time.Hour * 24) - time.Minute)
|
||||||
|
offline := nodes.List["offline"]
|
||||||
|
offline.Lastseen = offline.Lastseen.Add((-6 * time.Hour * 24) + time.Minute)
|
||||||
|
|
||||||
|
nodes.expire()
|
||||||
|
|
||||||
|
// one expired?
|
||||||
|
assert.Equal(2, len(nodes.List))
|
||||||
|
assert.Nil(nodes.List["expire"])
|
||||||
|
|
||||||
|
// one offline?
|
||||||
|
assert.NotNil(nodes.List["offline"])
|
||||||
|
assert.False(nodes.List["offline"].Flags.Online)
|
||||||
|
|
||||||
|
// one online?
|
||||||
|
assert.NotNil(nodes.List["online"])
|
||||||
|
assert.True(nodes.List["online"].Flags.Online)
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadAndSave(t *testing.T) {
|
func TestLoadAndSave(t *testing.T) {
|
||||||
assert := assert.New(t)
|
assert := assert.New(t)
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue