From 7324567f91f64db58333138b52b5faae97303f69 Mon Sep 17 00:00:00 2001 From: Julian Labus Date: Tue, 21 Nov 2017 15:12:06 +0100 Subject: [PATCH] [TASK] add global statistics for individual sites --- cmd/import.go | 8 ++-- cmd/query.go | 2 +- cmd/serve.go | 2 +- config_example.toml | 2 + database/all/internal.go | 4 +- database/database.go | 2 +- database/graphite/global.go | 32 ++++++++++------ database/influxdb/global.go | 31 +++++++++++++--- database/influxdb/global_test.go | 18 +++++++-- database/logging/file.go | 4 +- respond/collector.go | 10 +++-- respond/collector_test.go | 4 +- runtime/config.go | 1 + runtime/stats.go | 64 ++++++++++++++++++++++---------- runtime/stats_test.go | 51 +++++++++++++++++++------ 15 files changed, 171 insertions(+), 64 deletions(-) diff --git a/cmd/import.go b/cmd/import.go index 93959b3..8b78ac5 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -12,12 +12,13 @@ import ( // importCmd represents the import command var importCmd = &cobra.Command{ - Use: "import ", + Use: "import ", Short: "Imports global statistics from the given RRD files, requires InfluxDB", - Example: "yanic import --config /etc/yanic.toml olddata.rrd", - Args: cobra.ExactArgs(1), + Example: "yanic import --config /etc/yanic.toml olddata.rrd global", + Args: cobra.ExactArgs(2), Run: func(cmd *cobra.Command, args []string) { path := args[0] + site := args[1] config := loadConfig() connections, err := all.Connect(config.Database.Connection) @@ -36,6 +37,7 @@ var importCmd = &cobra.Command{ Clients: uint32(ds.Clients), }, ds.Time, + site, ) } }, diff --git a/cmd/query.go b/cmd/query.go index 0f20a1e..8e7a73c 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -26,7 +26,7 @@ var queryCmd = &cobra.Command{ nodes := runtime.NewNodes(&runtime.Config{}) - collector := respond.NewCollector(nil, nodes, []string{iface}, 0) + collector := respond.NewCollector(nil, nodes, []string{}, []string{iface}, 0) defer collector.Close() collector.SendPacket(dstAddress) diff --git a/cmd/serve.go b/cmd/serve.go index 60d33a9..1872c1b 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -58,7 +58,7 @@ var serveCmd = &cobra.Command{ time.Sleep(delay) } - collector = respond.NewCollector(connections, nodes, config.Respondd.Interfaces, config.Respondd.Port) + collector = respond.NewCollector(connections, nodes, config.Respondd.Sites, config.Respondd.Interfaces, config.Respondd.Port) collector.Start(config.Respondd.CollectInterval.Duration) defer collector.Close() } diff --git a/config_example.toml b/config_example.toml index 10fb1c9..b681c4f 100644 --- a/config_example.toml +++ b/config_example.toml @@ -11,6 +11,8 @@ synchronize = "1m" collect_interval = "1m" # interface that has an IP in your mesh network interfaces = ["br-ffhb"] +# list of sites to save stats for (empty for global only) +sites = [] # define a port to listen # if not set or set to 0 the kernel will use a random free port at its own #port = 10001 diff --git a/database/all/internal.go b/database/all/internal.go index 721e09e..f1121d0 100644 --- a/database/all/internal.go +++ b/database/all/internal.go @@ -43,9 +43,9 @@ func (conn *Connection) InsertLink(link *runtime.Link, time time.Time) { } } -func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time) { +func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) { for _, item := range conn.list { - item.InsertGlobals(stats, time) + item.InsertGlobals(stats, time, site) } } diff --git a/database/database.go b/database/database.go index 3a40edc..e4b1381 100644 --- a/database/database.go +++ b/database/database.go @@ -15,7 +15,7 @@ type Connection interface { InsertLink(*runtime.Link, time.Time) // InsertGlobals stores global statistics - InsertGlobals(*runtime.GlobalStats, time.Time) + InsertGlobals(*runtime.GlobalStats, time.Time, string) // PruneNodes prunes historical per-node data PruneNodes(deleteAfter time.Duration) diff --git a/database/graphite/global.go b/database/graphite/global.go index 648d5c7..2c2bedb 100644 --- a/database/graphite/global.go +++ b/database/graphite/global.go @@ -7,20 +7,30 @@ import ( "github.com/fgrosse/graphigo" ) -func (c *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time) { - c.addPoint(GlobalStatsFields(stats)) - c.addCounterMap(CounterMeasurementModel, stats.Models, time) - c.addCounterMap(CounterMeasurementFirmware, stats.Firmwares, time) +func (c *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) { + measurementGlobal := MeasurementGlobal + counterMeasurementModel := CounterMeasurementModel + counterMeasurementFirmware := CounterMeasurementFirmware + + if site != runtime.GLOBAL_SITE { + measurementGlobal += "_" + site + counterMeasurementModel += "_" + site + counterMeasurementFirmware += "_" + site + } + + c.addPoint(GlobalStatsFields(measurementGlobal, stats)) + c.addCounterMap(counterMeasurementModel, stats.Models, time) + c.addCounterMap(counterMeasurementFirmware, stats.Firmwares, time) } -func GlobalStatsFields(stats *runtime.GlobalStats) []graphigo.Metric { +func GlobalStatsFields(name string, stats *runtime.GlobalStats) []graphigo.Metric { return []graphigo.Metric{ - {Name: MeasurementGlobal + ".nodes", Value: stats.Nodes}, - {Name: MeasurementGlobal + ".gateways", Value: stats.Gateways}, - {Name: MeasurementGlobal + ".clients.total", Value: stats.Clients}, - {Name: MeasurementGlobal + ".clients.wifi", Value: stats.ClientsWifi}, - {Name: MeasurementGlobal + ".clients.wifi24", Value: stats.ClientsWifi24}, - {Name: MeasurementGlobal + ".clients.wifi5", Value: stats.ClientsWifi5}, + {Name: name + ".nodes", Value: stats.Nodes}, + {Name: name + ".gateways", Value: stats.Gateways}, + {Name: name + ".clients.total", Value: stats.Clients}, + {Name: name + ".clients.wifi", Value: stats.ClientsWifi}, + {Name: name + ".clients.wifi24", Value: stats.ClientsWifi24}, + {Name: name + ".clients.wifi5", Value: stats.ClientsWifi5}, } } diff --git a/database/influxdb/global.go b/database/influxdb/global.go index 0a302d5..f49bc6e 100644 --- a/database/influxdb/global.go +++ b/database/influxdb/global.go @@ -8,11 +8,29 @@ import ( ) // InsertGlobals implementation of database -func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time) { - conn.addPoint(MeasurementGlobal, nil, GlobalStatsFields(stats), time) - conn.addCounterMap(CounterMeasurementModel, stats.Models, time) - conn.addCounterMap(CounterMeasurementFirmware, stats.Firmwares, time) - conn.addCounterMap(CounterMeasurementAutoupdater, stats.Autoupdater, time) +func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) { + var tags models.Tags + + measurementGlobal := MeasurementGlobal + counterMeasurementModel := CounterMeasurementModel + counterMeasurementFirmware := CounterMeasurementFirmware + counterMeasurementAutoupdater := CounterMeasurementAutoupdater + + if site != runtime.GLOBAL_SITE { + tags = models.Tags{ + models.Tag{Key: []byte("site"), Value: []byte(site)}, + } + + measurementGlobal += "_site" + counterMeasurementModel += "_site" + counterMeasurementFirmware += "_site" + counterMeasurementAutoupdater += "_site" + } + + conn.addPoint(measurementGlobal, tags, GlobalStatsFields(stats), time) + conn.addCounterMap(counterMeasurementModel, stats.Models, time, site) + conn.addCounterMap(counterMeasurementFirmware, stats.Firmwares, time, site) + conn.addCounterMap(counterMeasurementAutoupdater, stats.Autoupdater, time, site) } // GlobalStatsFields returns fields for InfluxDB @@ -30,12 +48,13 @@ func GlobalStatsFields(stats *runtime.GlobalStats) map[string]interface{} { // Saves the values of a CounterMap in the database. // The key are used as 'value' tag. // The value is used as 'counter' field. -func (conn *Connection) addCounterMap(name string, m runtime.CounterMap, t time.Time) { +func (conn *Connection) addCounterMap(name string, m runtime.CounterMap, t time.Time, site string) { for key, count := range m { conn.addPoint( name, models.Tags{ models.Tag{Key: []byte("value"), Value: []byte(key)}, + models.Tag{Key: []byte("site"), Value: []byte(site)}, }, models.Fields{"count": count}, t, diff --git a/database/influxdb/global_test.go b/database/influxdb/global_test.go index 5f43eb5..2bd0d2e 100644 --- a/database/influxdb/global_test.go +++ b/database/influxdb/global_test.go @@ -9,14 +9,20 @@ import ( "github.com/FreifunkBremen/yanic/runtime" ) +const TEST_SITE = "ffxx" + func TestGlobalStats(t *testing.T) { - stats := runtime.NewGlobalStats(createTestNodes()) + stats := runtime.NewGlobalStats(createTestNodes(), []string{TEST_SITE}) assert := assert.New(t) - fields := GlobalStatsFields(stats) - // check fields + // check SITE_GLOBAL fields + fields := GlobalStatsFields(stats[runtime.GLOBAL_SITE]) assert.EqualValues(3, fields["nodes"]) + + // check TEST_SITE fields + fields = GlobalStatsFields(stats[TEST_SITE]) + assert.EqualValues(2, fields["nodes"]) } func createTestNodes() *runtime.Nodes { @@ -34,6 +40,9 @@ func createTestNodes() *runtime.Nodes { Hardware: data.Hardware{ Model: "TP-Link 841", }, + System: data.System{ + SiteCode: TEST_SITE, + }, }, } nodeData.Nodeinfo.Software.Firmware.Release = "2016.1.6+entenhausen1" @@ -64,6 +73,9 @@ func createTestNodes() *runtime.Nodes { Hardware: data.Hardware{ Model: "Xeon Multi-Core", }, + System: data.System{ + SiteCode: TEST_SITE, + }, }, }) diff --git a/database/logging/file.go b/database/logging/file.go index 97908b1..c383578 100644 --- a/database/logging/file.go +++ b/database/logging/file.go @@ -56,8 +56,8 @@ func (conn *Connection) InsertLink(link *runtime.Link, time time.Time) { conn.log("InsertLink: ", link) } -func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time) { - conn.log("InsertGlobals: [", time.String(), "] nodes: ", stats.Nodes, ", clients: ", stats.Clients, " models: ", len(stats.Models)) +func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) { + conn.log("InsertGlobals: [", time.String(), "] site: ", site, ", nodes: ", stats.Nodes, ", clients: ", stats.Clients, " models: ", len(stats.Models)) } func (conn *Connection) PruneNodes(deleteAfter time.Duration) { diff --git a/respond/collector.go b/respond/collector.go index 8f7c6eb..7642f95 100644 --- a/respond/collector.go +++ b/respond/collector.go @@ -24,16 +24,18 @@ type Collector struct { queue chan *Response // received responses db database.Connection nodes *runtime.Nodes + sites []string interval time.Duration // Interval for multicast packets stop chan interface{} } // NewCollector creates a Collector struct -func NewCollector(db database.Connection, nodes *runtime.Nodes, ifaces []string, port int) *Collector { +func NewCollector(db database.Connection, nodes *runtime.Nodes, sites []string , ifaces []string, port int) *Collector { coll := &Collector{ db: db, nodes: nodes, + sites: sites, port: port, queue: make(chan *Response, 400), stop: make(chan interface{}), @@ -300,7 +302,9 @@ func (coll *Collector) globalStatsWorker() { // saves global statistics func (coll *Collector) saveGlobalStats() { - stats := runtime.NewGlobalStats(coll.nodes) + stats := runtime.NewGlobalStats(coll.nodes, coll.sites) - coll.db.InsertGlobals(stats, time.Now()) + for site, stat := range stats { + coll.db.InsertGlobals(stat, time.Now(), site) + } } diff --git a/respond/collector_test.go b/respond/collector_test.go index e73b835..ba1ff0a 100644 --- a/respond/collector_test.go +++ b/respond/collector_test.go @@ -9,10 +9,12 @@ import ( "github.com/stretchr/testify/assert" ) +const SITE_TEST = "ffxx" + func TestCollector(t *testing.T) { nodes := runtime.NewNodes(&runtime.Config{}) - collector := NewCollector(nil, nodes, []string{}, 10001) + collector := NewCollector(nil, nodes, []string{SITE_TEST}, []string{}, 10001) collector.Start(time.Millisecond) time.Sleep(time.Millisecond * 10) collector.Close() diff --git a/runtime/config.go b/runtime/config.go index f9f88d8..23e9325 100644 --- a/runtime/config.go +++ b/runtime/config.go @@ -12,6 +12,7 @@ type Config struct { Enable bool `toml:"enable"` Synchronize Duration `toml:"synchronize"` Interfaces []string `toml:"interfaces"` + Sites []string `toml:"sites"` Port int `toml:"port"` CollectInterval Duration `toml:"collect_interval"` } diff --git a/runtime/stats.go b/runtime/stats.go index 29e6562..a17975d 100644 --- a/runtime/stats.go +++ b/runtime/stats.go @@ -1,6 +1,9 @@ package runtime -const DISABLED_AUTOUPDATER = "disabled" +const ( + DISABLED_AUTOUPDATER = "disabled" + GLOBAL_SITE = "global" +) // CounterMap to manage multiple values type CounterMap map[string]uint32 @@ -20,33 +23,32 @@ type GlobalStats struct { } //NewGlobalStats returns global statistics for InfluxDB -func NewGlobalStats(nodes *Nodes) (result *GlobalStats) { - result = &GlobalStats{ +func NewGlobalStats(nodes *Nodes, sites []string) (result map[string]*GlobalStats) { + result = make(map[string]*GlobalStats) + + result[GLOBAL_SITE] = &GlobalStats{ Firmwares: make(CounterMap), Models: make(CounterMap), Autoupdater: make(CounterMap), } + for _, site := range sites { + result[site] = &GlobalStats{ + Firmwares: make(CounterMap), + Models: make(CounterMap), + Autoupdater: make(CounterMap), + } + } + nodes.RLock() for _, node := range nodes.List { if node.Online { - result.Nodes++ - if stats := node.Statistics; stats != nil { - result.Clients += stats.Clients.Total - result.ClientsWifi24 += stats.Clients.Wifi24 - result.ClientsWifi5 += stats.Clients.Wifi5 - result.ClientsWifi += stats.Clients.Wifi - } - if node.IsGateway() { - result.Gateways++ - } + result[GLOBAL_SITE].Add(node) + if info := node.Nodeinfo; info != nil { - result.Models.Increment(info.Hardware.Model) - result.Firmwares.Increment(info.Software.Firmware.Release) - if info.Software.Autoupdater.Enabled { - result.Autoupdater.Increment(info.Software.Autoupdater.Branch) - } else { - result.Autoupdater.Increment(DISABLED_AUTOUPDATER) + site := info.System.SiteCode + if _, exist := result[site]; exist { + result[site].Add(node) } } } @@ -55,6 +57,30 @@ func NewGlobalStats(nodes *Nodes) (result *GlobalStats) { return } +// Add values to GlobalStats +// if node is online +func (s *GlobalStats) Add(node *Node) { + s.Nodes++ + if stats := node.Statistics; stats != nil { + s.Clients += stats.Clients.Total + s.ClientsWifi24 += stats.Clients.Wifi24 + s.ClientsWifi5 += stats.Clients.Wifi5 + s.ClientsWifi += stats.Clients.Wifi + } + if node.IsGateway() { + s.Gateways++ + } + if info := node.Nodeinfo; info != nil { + s.Models.Increment(info.Hardware.Model) + s.Firmwares.Increment(info.Software.Firmware.Release) + if info.Software.Autoupdater.Enabled { + s.Autoupdater.Increment(info.Software.Autoupdater.Branch) + } else { + s.Autoupdater.Increment(DISABLED_AUTOUPDATER) + } + } +} + // Increment counter in the map by one // if the value is not empty func (m CounterMap) Increment(key string) { diff --git a/runtime/stats_test.go b/runtime/stats_test.go index c2bbecd..59e8378 100644 --- a/runtime/stats_test.go +++ b/runtime/stats_test.go @@ -8,26 +8,49 @@ import ( "github.com/FreifunkBremen/yanic/data" ) +const TEST_SITE = "ffxx" + func TestGlobalStats(t *testing.T) { - stats := NewGlobalStats(createTestNodes()) + stats := NewGlobalStats(createTestNodes(), []string{TEST_SITE}) assert := assert.New(t) - assert.EqualValues(1, stats.Gateways) - assert.EqualValues(3, stats.Nodes) - assert.EqualValues(25, stats.Clients) + assert.Len(stats, 2) + + //check GLOBAL_SITE stats + assert.EqualValues(1, stats[GLOBAL_SITE].Gateways) + assert.EqualValues(3, stats[GLOBAL_SITE].Nodes) + assert.EqualValues(25, stats[GLOBAL_SITE].Clients) // check models - assert.Len(stats.Models, 2) - assert.EqualValues(2, stats.Models["TP-Link 841"]) - assert.EqualValues(1, stats.Models["Xeon Multi-Core"]) + assert.Len(stats[GLOBAL_SITE].Models, 2) + assert.EqualValues(2, stats[GLOBAL_SITE].Models["TP-Link 841"]) + assert.EqualValues(1, stats[GLOBAL_SITE].Models["Xeon Multi-Core"]) // check firmwares - assert.Len(stats.Firmwares, 1) - assert.EqualValues(1, stats.Firmwares["2016.1.6+entenhausen1"]) + assert.Len(stats[GLOBAL_SITE].Firmwares, 1) + assert.EqualValues(1, stats[GLOBAL_SITE].Firmwares["2016.1.6+entenhausen1"]) // check autoupdater - assert.Len(stats.Autoupdater, 2) - assert.EqualValues(1, stats.Autoupdater["stable"]) + assert.Len(stats[GLOBAL_SITE].Autoupdater, 2) + assert.EqualValues(1, stats[GLOBAL_SITE].Autoupdater["stable"]) + + // check TEST_SITE stats + assert.EqualValues(1, stats[TEST_SITE].Gateways) + assert.EqualValues(2, stats[TEST_SITE].Nodes) + assert.EqualValues(23, stats[TEST_SITE].Clients) + + // check models + assert.Len(stats[TEST_SITE].Models, 2) + assert.EqualValues(1, stats[TEST_SITE].Models["TP-Link 841"]) + assert.EqualValues(1, stats[TEST_SITE].Models["Xeon Multi-Core"]) + + // check firmwares + assert.Len(stats[TEST_SITE].Firmwares, 1) + assert.EqualValues(1, stats[TEST_SITE].Firmwares["2016.1.6+entenhausen1"]) + + // check autoupdater + assert.Len(stats[TEST_SITE].Autoupdater, 1) + assert.EqualValues(0, stats[TEST_SITE].Autoupdater["stable"]) } func createTestNodes() *Nodes { @@ -45,6 +68,9 @@ func createTestNodes() *Nodes { Hardware: data.Hardware{ Model: "TP-Link 841", }, + System: data.System{ + SiteCode: TEST_SITE, + }, }, } nodeData.Nodeinfo.Software.Firmware.Release = "2016.1.6+entenhausen1" @@ -82,6 +108,9 @@ func createTestNodes() *Nodes { Hardware: data.Hardware{ Model: "Xeon Multi-Core", }, + System: data.System{ + SiteCode: TEST_SITE, + }, }, })