Merge pull request #14 from FreifunkBremen/measurements

Add measurements for models and firmwares

database/database.go

@@ -12,9 +12,11 @@ import (
 )
 
 const (
-	MeasurementNode   = "node"   // Measurement for per-node statistics
+	MeasurementNode     = "node"     // Measurement for per-node statistics
-	MeasurementGlobal = "global" // Measurement for summarized global statistics
+	MeasurementGlobal   = "global"   // Measurement for summarized global statistics
-	batchMaxSize      = 500
+	MeasurementFirmware = "firmware" // Measurement for firmware statistics
+	MeasurementModel    = "model"    // Measurement for model statistics
+	batchMaxSize        = 500
 )
 
 type DB struct {
@@ -65,6 +67,23 @@ func (db *DB) AddPoint(name string, tags imodels.Tags, fields imodels.Fields, ti
 	db.points <- point
 }
 
+// Saves the values of a CounterMap in the database.
+// The key are used as 'value' tag.
+// The value is used as 'counter' field.
+func (db *DB) AddCounterMap(name string, m models.CounterMap) {
+	now := time.Now()
+	for key, count := range m {
+		db.AddPoint(
+			name,
+			imodels.Tags{
+				imodels.Tag{Key: []byte("value"), Value: []byte(key)},
+			},
+			imodels.Fields{"count": count},
+			now,
+		)
+	}
+}
+
 // Add data for a single node
 func (db *DB) Add(nodeId string, node *models.Node) {
 	tags, fields := node.ToInflux()

models/node_test.go

@@ -1,31 +0,0 @@
-package models
-
-import (
-	"testing"
-
-	"github.com/FreifunkBremen/respond-collector/data"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestToInflux(t *testing.T) {
-	assert := assert.New(t)
-
-	node := Node{
-		Statistics: &data.Statistics{
-			NodeId:      "foobar",
-			LoadAverage: 0.5,
-		},
-		Nodeinfo: &data.NodeInfo{
-			Owner: &data.Owner{
-				Contact: "nobody",
-			},
-		},
-		Neighbours: &data.Neighbours{},
-	}
-
-	tags, fields := node.ToInflux()
-
-	assert.Equal("foobar", tags.GetString("nodeid"))
-	assert.Equal("nobody", tags.GetString("owner"))
-	assert.Equal(0.5, fields["load"])
-}

models/nodes.go

@@ -21,14 +21,6 @@ type Nodes struct {
 	sync.RWMutex
 }
 
-type GlobalStats struct {
-	Nodes         uint32
-	Clients       uint32
-	ClientsWifi   uint32
-	ClientsWifi24 uint32
-	ClientsWifi5  uint32
-}
-
 // NewNodes create Nodes structs
 func NewNodes(config *Config) *Nodes {
 	nodes := &Nodes{
@@ -207,36 +199,6 @@ func (nodes *Nodes) save() {
 	}
 }
 
-// Returns global statistics for InfluxDB
-func (nodes *Nodes) GlobalStats() (result *GlobalStats) {
-	result = &GlobalStats{}
-	nodes.Lock()
-	for _, node := range nodes.List {
-		if node.Flags.Online {
-			result.Nodes += 1
-			if stats := node.Statistics; stats != nil {
-				result.Clients += stats.Clients.Total
-				result.ClientsWifi24 += stats.Clients.Wifi24
-				result.ClientsWifi5 += stats.Clients.Wifi5
-				result.ClientsWifi += stats.Clients.Wifi
-			}
-		}
-	}
-	nodes.Unlock()
-	return
-}
-
-// Returns fields for InfluxDB
-func (stats *GlobalStats) Fields() map[string]interface{} {
-	return map[string]interface{}{
-		"nodes":          stats.Nodes,
-		"clients.total":  stats.Clients,
-		"clients.wifi":   stats.ClientsWifi,
-		"clients.wifi24": stats.ClientsWifi24,
-		"clients.wifi5":  stats.ClientsWifi5,
-	}
-}
-
 // Marshals the input and writes it into the given file
 func save(input interface{}, outputFile string) {
 	tmpFile := outputFile + ".tmp"

models/nodes_test.go

@@ -74,28 +74,25 @@ func TestUpdateNodes(t *testing.T) {
 	assert.Equal(1, len(nodes.List))
 }
 
-func TestGlobalStats(t *testing.T) {
+func TestToInflux(t *testing.T) {
-	stats := createTestNodes().GlobalStats()
-
 	assert := assert.New(t)
-	assert.EqualValues(uint32(1), stats.Nodes)
-	assert.EqualValues(uint32(23), stats.Clients)
-}
 
-func TestNodesMini(t *testing.T) {
+	node := Node{
-	mini := createTestNodes().GetNodesMini()
+		Statistics: &data.Statistics{
-
+			NodeId:      "foobar",
-	assert := assert.New(t)
+			LoadAverage: 0.5,
-	assert.Equal(1, len(mini.List))
+		},
-}
+		Nodeinfo: &data.NodeInfo{
-
+			Owner: &data.Owner{
-func createTestNodes() *Nodes {
+				Contact: "nobody",
-	nodes := NewNodes(&Config{})
+			},
-
+		},
-	res := &data.ResponseData{
+		Neighbours: &data.Neighbours{},
-		Statistics: &data.Statistics{},
 	}
-	res.Statistics.Clients.Total = 23
+
-	nodes.Update("abcdef012345", res)
+	tags, fields := node.ToInflux()
-	return nodes
+
+	assert.Equal("foobar", tags.GetString("nodeid"))
+	assert.Equal("nobody", tags.GetString("owner"))
+	assert.Equal(0.5, fields["load"])
 }

models/stats.go

@@ -0,0 +1,66 @@
+package models
+
+type CounterMap map[string]uint32
+
+type GlobalStats struct {
+	Clients       uint32
+	ClientsWifi   uint32
+	ClientsWifi24 uint32
+	ClientsWifi5  uint32
+	Gateways      uint32
+	Nodes         uint32
+
+	Firmwares CounterMap
+	Models    CounterMap
+}
+
+// Returns global statistics for InfluxDB
+func NewGlobalStats(nodes *Nodes) (result *GlobalStats) {
+	result = &GlobalStats{
+		Firmwares: make(CounterMap),
+		Models:    make(CounterMap),
+	}
+
+	nodes.Lock()
+	for _, node := range nodes.List {
+		if node.Flags.Online {
+			result.Nodes += 1
+			if stats := node.Statistics; stats != nil {
+				result.Clients += stats.Clients.Total
+				result.ClientsWifi24 += stats.Clients.Wifi24
+				result.ClientsWifi5 += stats.Clients.Wifi5
+				result.ClientsWifi += stats.Clients.Wifi
+			}
+			if node.Flags.Gateway {
+				result.Gateways += 1
+			}
+			if info := node.Nodeinfo; info != nil {
+				result.Models.Increment(info.Hardware.Model)
+				result.Firmwares.Increment(info.Software.Firmware.Release)
+			}
+		}
+	}
+	nodes.Unlock()
+	return
+}
+
+// Increment counter in the map by one
+// if the value is not empty
+func (m CounterMap) Increment(key string) {
+	if key != "" {
+		val := m[key]
+		m[key] = val + 1
+	}
+}
+
+// Returns fields for InfluxDB
+func (stats *GlobalStats) Fields() map[string]interface{} {
+	return map[string]interface{}{
+		"nodes":          stats.Nodes,
+		"gateways":       stats.Gateways,
+		"clients.total":  stats.Clients,
+		"clients.wifi":   stats.ClientsWifi,
+		"clients.wifi24": stats.ClientsWifi24,
+		"clients.wifi5":  stats.ClientsWifi5,
+	}
+}

models/stats_test.go

@@ -0,0 +1,76 @@
+package models
+
+import (
+	"testing"
+
+	"github.com/FreifunkBremen/respond-collector/data"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGlobalStats(t *testing.T) {
+	stats := NewGlobalStats(createTestNodes())
+
+	assert := assert.New(t)
+	assert.EqualValues(1, stats.Gateways)
+	assert.EqualValues(3, stats.Nodes)
+	assert.EqualValues(25, stats.Clients)
+
+	// check models
+	assert.EqualValues(2, len(stats.Models))
+	assert.EqualValues(2, stats.Models["TP-Link 841"])
+	assert.EqualValues(1, stats.Models["Xeon Multi-Core"])
+
+	// check firmwares
+	assert.EqualValues(1, len(stats.Firmwares))
+	assert.EqualValues(1, stats.Firmwares["2016.1.6+entenhausen1"])
+}
+
+func TestNodesMini(t *testing.T) {
+	mini := createTestNodes().GetNodesMini()
+
+	assert := assert.New(t)
+	assert.Equal(2, len(mini.List))
+}
+
+func createTestNodes() *Nodes {
+	nodes := NewNodes(&Config{})
+
+	nodeData := &data.ResponseData{
+		Statistics: &data.Statistics{
+			Clients: data.Clients{
+				Total: 23,
+			},
+		},
+		NodeInfo: &data.NodeInfo{
+			Hardware: data.Hardware{
+				Model: "TP-Link 841",
+			},
+		},
+	}
+	nodeData.NodeInfo.Software.Firmware.Release = "2016.1.6+entenhausen1"
+	nodes.Update("abcdef012345", nodeData)
+
+	nodes.Update("112233445566", &data.ResponseData{
+		Statistics: &data.Statistics{
+			Clients: data.Clients{
+				Total: 2,
+			},
+		},
+		NodeInfo: &data.NodeInfo{
+			Hardware: data.Hardware{
+				Model: "TP-Link 841",
+			},
+		},
+	})
+
+	nodes.Update("0xdeadbeef0x", &data.ResponseData{
+		NodeInfo: &data.NodeInfo{
+			VPN: true,
+			Hardware: data.Hardware{
+				Model: "Xeon Multi-Core",
+			},
+		},
+	})
+
+	return nodes
+}

respond/collector.go

@@ -56,6 +56,10 @@ func NewCollector(db *database.DB, nodes *models.Nodes, interval time.Duration,
 	go collector.receiver()
 	go collector.parser()
 
+	if collector.db != nil {
+		go collector.globalStatsWorker()
+	}
+
 	// Run senders
 	go func() {
 		collector.sendOnce() // immediately
@@ -97,11 +101,6 @@ func (coll *Collector) sender() {
 		case <-coll.stop:
 			return
 		case <-coll.ticker.C:
-			// save global statistics
-			if coll.db != nil {
-				coll.db.AddPoint(database.MeasurementGlobal, nil, coll.nodes.GlobalStats().Fields(), time.Now())
-			}
-
 			// send the multicast packet to request per-node statistics
 			coll.sendOnce()
 		}
@@ -172,3 +171,24 @@ func (coll *Collector) receiver() {
 		}
 	}
 }
+
+func (coll *Collector) globalStatsWorker() {
+	ticker := time.NewTicker(time.Minute)
+	for {
+		select {
+		case <-coll.stop:
+			return
+		case <-ticker.C:
+			coll.saveGlobalStats()
+		}
+	}
+}
+
+// saves global statistics
+func (coll *Collector) saveGlobalStats() {
+	stats := models.NewGlobalStats(coll.nodes)
+
+	coll.db.AddPoint(database.MeasurementGlobal, nil, stats.Fields(), time.Now())
+	coll.db.AddCounterMap(database.MeasurementFirmware, stats.Firmwares)
+	coll.db.AddCounterMap(database.MeasurementModel, stats.Models)
+}