From ac5858259cb407686c81e355368dcfc5b93d0a2f Mon Sep 17 00:00:00 2001 From: Julian Kornberger Date: Thu, 15 Dec 2016 11:10:09 +0100 Subject: [PATCH] Add measurements for models and firmwares closes #12 --- database/database.go | 25 +++++++++++++-- models/node_test.go | 31 ------------------ models/nodes.go | 38 ---------------------- models/nodes_test.go | 39 +++++++++++------------ models/stats.go | 66 ++++++++++++++++++++++++++++++++++++++ models/stats_test.go | 76 ++++++++++++++++++++++++++++++++++++++++++++ respond/collector.go | 30 ++++++++++++++--- 7 files changed, 207 insertions(+), 98 deletions(-) delete mode 100644 models/node_test.go create mode 100644 models/stats.go create mode 100644 models/stats_test.go diff --git a/database/database.go b/database/database.go index 0e604c2..01b9724 100644 --- a/database/database.go +++ b/database/database.go @@ -12,9 +12,11 @@ import ( ) const ( - MeasurementNode = "node" // Measurement for per-node statistics - MeasurementGlobal = "global" // Measurement for summarized global statistics - batchMaxSize = 500 + MeasurementNode = "node" // Measurement for per-node statistics + MeasurementGlobal = "global" // Measurement for summarized global statistics + MeasurementFirmware = "firmware" // Measurement for firmware statistics + MeasurementModel = "model" // Measurement for model statistics + batchMaxSize = 500 ) type DB struct { @@ -65,6 +67,23 @@ func (db *DB) AddPoint(name string, tags imodels.Tags, fields imodels.Fields, ti db.points <- point } +// Saves the values of a CounterMap in the database. +// The key are used as 'value' tag. +// The value is used as 'counter' field. +func (db *DB) AddCounterMap(name string, m models.CounterMap) { + now := time.Now() + for key, count := range m { + db.AddPoint( + name, + imodels.Tags{ + imodels.Tag{Key: []byte("value"), Value: []byte(key)}, + }, + imodels.Fields{"count": count}, + now, + ) + } +} + // Add data for a single node func (db *DB) Add(nodeId string, node *models.Node) { tags, fields := node.ToInflux() diff --git a/models/node_test.go b/models/node_test.go deleted file mode 100644 index 098e0a5..0000000 --- a/models/node_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package models - -import ( - "testing" - - "github.com/FreifunkBremen/respond-collector/data" - "github.com/stretchr/testify/assert" -) - -func TestToInflux(t *testing.T) { - assert := assert.New(t) - - node := Node{ - Statistics: &data.Statistics{ - NodeId: "foobar", - LoadAverage: 0.5, - }, - Nodeinfo: &data.NodeInfo{ - Owner: &data.Owner{ - Contact: "nobody", - }, - }, - Neighbours: &data.Neighbours{}, - } - - tags, fields := node.ToInflux() - - assert.Equal("foobar", tags.GetString("nodeid")) - assert.Equal("nobody", tags.GetString("owner")) - assert.Equal(0.5, fields["load"]) -} diff --git a/models/nodes.go b/models/nodes.go index 27f2c7b..939c3bb 100644 --- a/models/nodes.go +++ b/models/nodes.go @@ -21,14 +21,6 @@ type Nodes struct { sync.RWMutex } -type GlobalStats struct { - Nodes uint32 - Clients uint32 - ClientsWifi uint32 - ClientsWifi24 uint32 - ClientsWifi5 uint32 -} - // NewNodes create Nodes structs func NewNodes(config *Config) *Nodes { nodes := &Nodes{ @@ -207,36 +199,6 @@ func (nodes *Nodes) save() { } } -// Returns global statistics for InfluxDB -func (nodes *Nodes) GlobalStats() (result *GlobalStats) { - result = &GlobalStats{} - nodes.Lock() - for _, node := range nodes.List { - if node.Flags.Online { - result.Nodes += 1 - if stats := node.Statistics; stats != nil { - result.Clients += stats.Clients.Total - result.ClientsWifi24 += stats.Clients.Wifi24 - result.ClientsWifi5 += stats.Clients.Wifi5 - result.ClientsWifi += stats.Clients.Wifi - } - } - } - nodes.Unlock() - return -} - -// Returns fields for InfluxDB -func (stats *GlobalStats) Fields() map[string]interface{} { - return map[string]interface{}{ - "nodes": stats.Nodes, - "clients.total": stats.Clients, - "clients.wifi": stats.ClientsWifi, - "clients.wifi24": stats.ClientsWifi24, - "clients.wifi5": stats.ClientsWifi5, - } -} - // Marshals the input and writes it into the given file func save(input interface{}, outputFile string) { tmpFile := outputFile + ".tmp" diff --git a/models/nodes_test.go b/models/nodes_test.go index 0335ffd..b5874e4 100644 --- a/models/nodes_test.go +++ b/models/nodes_test.go @@ -74,28 +74,25 @@ func TestUpdateNodes(t *testing.T) { assert.Equal(1, len(nodes.List)) } -func TestGlobalStats(t *testing.T) { - stats := createTestNodes().GlobalStats() - +func TestToInflux(t *testing.T) { assert := assert.New(t) - assert.EqualValues(uint32(1), stats.Nodes) - assert.EqualValues(uint32(23), stats.Clients) -} -func TestNodesMini(t *testing.T) { - mini := createTestNodes().GetNodesMini() - - assert := assert.New(t) - assert.Equal(1, len(mini.List)) -} - -func createTestNodes() *Nodes { - nodes := NewNodes(&Config{}) - - res := &data.ResponseData{ - Statistics: &data.Statistics{}, + node := Node{ + Statistics: &data.Statistics{ + NodeId: "foobar", + LoadAverage: 0.5, + }, + Nodeinfo: &data.NodeInfo{ + Owner: &data.Owner{ + Contact: "nobody", + }, + }, + Neighbours: &data.Neighbours{}, } - res.Statistics.Clients.Total = 23 - nodes.Update("abcdef012345", res) - return nodes + + tags, fields := node.ToInflux() + + assert.Equal("foobar", tags.GetString("nodeid")) + assert.Equal("nobody", tags.GetString("owner")) + assert.Equal(0.5, fields["load"]) } diff --git a/models/stats.go b/models/stats.go new file mode 100644 index 0000000..8fb9166 --- /dev/null +++ b/models/stats.go @@ -0,0 +1,66 @@ +package models + +type CounterMap map[string]uint32 + +type GlobalStats struct { + Clients uint32 + ClientsWifi uint32 + ClientsWifi24 uint32 + ClientsWifi5 uint32 + Gateways uint32 + Nodes uint32 + + Firmwares CounterMap + Models CounterMap +} + +// Returns global statistics for InfluxDB +func NewGlobalStats(nodes *Nodes) (result *GlobalStats) { + result = &GlobalStats{ + Firmwares: make(CounterMap), + Models: make(CounterMap), + } + + nodes.Lock() + for _, node := range nodes.List { + if node.Flags.Online { + result.Nodes += 1 + if stats := node.Statistics; stats != nil { + result.Clients += stats.Clients.Total + result.ClientsWifi24 += stats.Clients.Wifi24 + result.ClientsWifi5 += stats.Clients.Wifi5 + result.ClientsWifi += stats.Clients.Wifi + } + if node.Flags.Gateway { + result.Gateways += 1 + } + if info := node.Nodeinfo; info != nil { + result.Models.Increment(info.Hardware.Model) + result.Firmwares.Increment(info.Software.Firmware.Release) + } + } + } + nodes.Unlock() + return +} + +// Increment counter in the map by one +// if the value is not empty +func (m CounterMap) Increment(key string) { + if key != "" { + val := m[key] + m[key] = val + 1 + } +} + +// Returns fields for InfluxDB +func (stats *GlobalStats) Fields() map[string]interface{} { + return map[string]interface{}{ + "nodes": stats.Nodes, + "gateways": stats.Gateways, + "clients.total": stats.Clients, + "clients.wifi": stats.ClientsWifi, + "clients.wifi24": stats.ClientsWifi24, + "clients.wifi5": stats.ClientsWifi5, + } +} diff --git a/models/stats_test.go b/models/stats_test.go new file mode 100644 index 0000000..9283585 --- /dev/null +++ b/models/stats_test.go @@ -0,0 +1,76 @@ +package models + +import ( + "testing" + + "github.com/FreifunkBremen/respond-collector/data" + "github.com/stretchr/testify/assert" +) + +func TestGlobalStats(t *testing.T) { + stats := NewGlobalStats(createTestNodes()) + + assert := assert.New(t) + assert.EqualValues(1, stats.Gateways) + assert.EqualValues(3, stats.Nodes) + assert.EqualValues(25, stats.Clients) + + // check models + assert.EqualValues(2, len(stats.Models)) + assert.EqualValues(2, stats.Models["TP-Link 841"]) + assert.EqualValues(1, stats.Models["Xeon Multi-Core"]) + + // check firmwares + assert.EqualValues(1, len(stats.Firmwares)) + assert.EqualValues(1, stats.Firmwares["2016.1.6+entenhausen1"]) +} + +func TestNodesMini(t *testing.T) { + mini := createTestNodes().GetNodesMini() + + assert := assert.New(t) + assert.Equal(2, len(mini.List)) +} + +func createTestNodes() *Nodes { + nodes := NewNodes(&Config{}) + + nodeData := &data.ResponseData{ + Statistics: &data.Statistics{ + Clients: data.Clients{ + Total: 23, + }, + }, + NodeInfo: &data.NodeInfo{ + Hardware: data.Hardware{ + Model: "TP-Link 841", + }, + }, + } + nodeData.NodeInfo.Software.Firmware.Release = "2016.1.6+entenhausen1" + nodes.Update("abcdef012345", nodeData) + + nodes.Update("112233445566", &data.ResponseData{ + Statistics: &data.Statistics{ + Clients: data.Clients{ + Total: 2, + }, + }, + NodeInfo: &data.NodeInfo{ + Hardware: data.Hardware{ + Model: "TP-Link 841", + }, + }, + }) + + nodes.Update("0xdeadbeef0x", &data.ResponseData{ + NodeInfo: &data.NodeInfo{ + VPN: true, + Hardware: data.Hardware{ + Model: "Xeon Multi-Core", + }, + }, + }) + + return nodes +} diff --git a/respond/collector.go b/respond/collector.go index 3b2ffac..d206f46 100644 --- a/respond/collector.go +++ b/respond/collector.go @@ -56,6 +56,10 @@ func NewCollector(db *database.DB, nodes *models.Nodes, interval time.Duration, go collector.receiver() go collector.parser() + if collector.db != nil { + go collector.globalStatsWorker() + } + // Run senders go func() { collector.sendOnce() // immediately @@ -97,11 +101,6 @@ func (coll *Collector) sender() { case <-coll.stop: return case <-coll.ticker.C: - // save global statistics - if coll.db != nil { - coll.db.AddPoint(database.MeasurementGlobal, nil, coll.nodes.GlobalStats().Fields(), time.Now()) - } - // send the multicast packet to request per-node statistics coll.sendOnce() } @@ -172,3 +171,24 @@ func (coll *Collector) receiver() { } } } + +func (coll *Collector) globalStatsWorker() { + ticker := time.NewTicker(time.Minute) + for { + select { + case <-coll.stop: + return + case <-ticker.C: + coll.saveGlobalStats() + } + } +} + +// saves global statistics +func (coll *Collector) saveGlobalStats() { + stats := models.NewGlobalStats(coll.nodes) + + coll.db.AddPoint(database.MeasurementGlobal, nil, stats.Fields(), time.Now()) + coll.db.AddCounterMap(database.MeasurementFirmware, stats.Firmwares) + coll.db.AddCounterMap(database.MeasurementModel, stats.Models) +}