ffhb/yanic
ffhb
/
yanic
mirror of https://github.com/FreifunkBremen/yanic.git
1
0
Fork 0
Code Issues Releases Wiki Activity

[TASK] handle domain_code (with filters) (#119)

This commit is contained in:
Martin Geno 2018-01-17 20:20:35 +01:00 committed by Geno
parent 092eafb086
commit 950ad8457f
29 changed files with 493 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/config_test.go
View File

@ -20,6 +20,10 @@ func TestReadConfig(t *testing.T) {
assert.Equal(time.Hour*24*7, config.Nodes.PruneAfter.Duration)
assert.Equal(time.Hour*24*7, config.Database.DeleteAfter.Duration)
assert.Len(config.Respondd.Sites, 1)
assert.Contains(config.Respondd.Sites, "ffhb")
assert.Contains(config.Respondd.Sites["ffhb"].Domains, "city")
// Test output plugins
assert.Len(config.Nodes.Output, 3)
outputs := config.Nodes.Output["meshviewer"].([]interface{})

8
cmd/import.go
View File

@ -11,13 +11,14 @@ import (
// importCmd represents the import command
var importCmd = &cobra.Command{
Use: "import <file.rrd> <site>",
Use: "import <file.rrd> <site> <domain>",
Short: "Imports global statistics from the given RRD files, requires InfluxDB",
Example: "yanic import --config /etc/yanic.toml olddata.rrd global",
Args: cobra.ExactArgs(2),
Example: "yanic import --config /etc/yanic.toml olddata.rrd global global",
Args: cobra.ExactArgs(3),
Run: func(cmd *cobra.Command, args []string) {
path := args[0]
site := args[1]
domain := args[2]
config := loadConfig()
err := allDatabase.Start(config.Database)
@ -36,6 +37,7 @@ var importCmd = &cobra.Command{
},
ds.Time,
site,
domain,
)
}
},

4
cmd/query.go
View File

@ -26,7 +26,9 @@ var queryCmd = &cobra.Command{
nodes := runtime.NewNodes(&runtime.NodesConfig{})
collector := respond.NewCollector(nil, nodes, []string{}, []string{iface}, 0)
sitesDomains := make(map[string][]string)
collector := respond.NewCollector(nil, nodes, sitesDomains, []string{iface}, 0)
defer collector.Close()
collector.SendPacket(dstAddress)

2
cmd/serve.go
View File

@ -54,7 +54,7 @@ var serveCmd = &cobra.Command{
time.Sleep(delay)
}
collector = respond.NewCollector(allDatabase.Conn, nodes, config.Respondd.Sites, config.Respondd.Interfaces, config.Respondd.Port)
collector = respond.NewCollector(allDatabase.Conn, nodes, config.Respondd.SitesDomains(), config.Respondd.Interfaces, config.Respondd.Port)
collector.Start(config.Respondd.CollectInterval.Duration)
defer collector.Close()
}

17
config_example.toml
View File

@ -11,12 +11,17 @@ synchronize = "1m"
collect_interval = "1m"
# interface that has an IP in your mesh network
interfaces = ["br-ffhb"]
# list of sites to save stats for (empty for global only)
sites = []
# define a port to listen
# if not set or set to 0 the kernel will use a random free port at its own
#port = 10001
# table of a site to save stats for (not exists for global only)
#[respondd.sites.example]
## list of domains on this site to save stats for (empty for global only)
#domains = []
## example
[respondd.sites.ffhb]
domains = ["city"]
# A little build-in webserver, which statically serves a directory.
# This is useful for testing purposes or for a little standalone installation.
@ -56,6 +61,14 @@ offline_after = "10m"
# List of site_codes of nodes that should be included in the output
#sites = ["ffhb"]
#
# replace the site_code with the domain_code in this output
# e.g. site_code='ffhb',domain_code='city' => site_code='city', domain_code=''
#domain_as_site = true
#
# append on the site_code the domain_code with a '.' in this output
# e.g. site_code='ffhb',domain_code='city' => site_code='ffhb.city', domain_code=''
#domain_append_site = true
#
# set has_location to true if you want to include only nodes that have geo-coordinates set
# (setting this to false has no sensible effect, unless you'd want to hide nodes that have coordinates)
#has_location = true

1
data/nodeinfo.go
View File

@ -44,6 +44,7 @@ type Owner struct {
// System struct
type System struct {
SiteCode string `json:"site_code,omitempty"`
DomainCode string `json:"domain_code,omitempty"`
}
// Location struct

4
database/all/connection.go
View File

@ -60,9 +60,9 @@ func (conn *Connection) InsertLink(link *runtime.Link, time time.Time) {
}
}
func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) {
func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string, domain string) {
for _, item := range conn.list {
item.InsertGlobals(stats, time, site)
item.InsertGlobals(stats, time, site, domain)
}
}

2
database/database.go
View File

@ -15,7 +15,7 @@ type Connection interface {
InsertLink(*runtime.Link, time.Time)
// InsertGlobals stores global statistics
InsertGlobals(*runtime.GlobalStats, time.Time, string)
InsertGlobals(*runtime.GlobalStats, time.Time, string, string)
// PruneNodes prunes historical per-node data
PruneNodes(deleteAfter time.Duration)

9
database/graphite/global.go
View File

@ -7,7 +7,7 @@ import (
"github.com/fgrosse/graphigo"
)
func (c *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) {
func (c *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string, domain string) {
measurementGlobal := MeasurementGlobal
counterMeasurementModel := CounterMeasurementModel
counterMeasurementFirmware := CounterMeasurementFirmware
@ -20,6 +20,13 @@ func (c *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, s
counterMeasurementAutoupdater += "_" + site
}
if domain != runtime.GLOBAL_DOMAIN {
measurementGlobal += "_" + domain
counterMeasurementModel += "_" + domain
counterMeasurementFirmware += "_" + domain
counterMeasurementAutoupdater += "_" + domain
}
c.addPoint(GlobalStatsFields(measurementGlobal, stats))
c.addCounterMap(counterMeasurementModel, stats.Models, time)
c.addCounterMap(counterMeasurementFirmware, stats.Firmwares, time)

25
database/influxdb/global.go
View File

@ -8,8 +8,8 @@ import (
)
// InsertGlobals implementation of database
func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) {
var tags models.Tags
func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string, domain string) {
tags := models.Tags{}
measurementGlobal := MeasurementGlobal
counterMeasurementModel := CounterMeasurementModel
@ -17,20 +17,26 @@ func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time
counterMeasurementAutoupdater := CounterMeasurementAutoupdater
if site != runtime.GLOBAL_SITE {
tags = models.Tags{
models.Tag{Key: []byte("site"), Value: []byte(site)},
}
tags.Set([]byte("site"), []byte(site))
measurementGlobal += "_site"
counterMeasurementModel += "_site"
counterMeasurementFirmware += "_site"
counterMeasurementAutoupdater += "_site"
}
if domain != runtime.GLOBAL_DOMAIN {
tags.Set([]byte("domain"), []byte(domain))
measurementGlobal += "_domain"
counterMeasurementModel += "_domain"
counterMeasurementFirmware += "_domain"
counterMeasurementAutoupdater += "_domain"
}
conn.addPoint(measurementGlobal, tags, GlobalStatsFields(stats), time)
conn.addCounterMap(counterMeasurementModel, stats.Models, time, site)
conn.addCounterMap(counterMeasurementFirmware, stats.Firmwares, time, site)
conn.addCounterMap(counterMeasurementAutoupdater, stats.Autoupdater, time, site)
conn.addCounterMap(counterMeasurementModel, stats.Models, time, site, domain)
conn.addCounterMap(counterMeasurementFirmware, stats.Firmwares, time, site, domain)
conn.addCounterMap(counterMeasurementAutoupdater, stats.Autoupdater, time, site, domain)
}
// GlobalStatsFields returns fields for InfluxDB
@ -48,13 +54,14 @@ func GlobalStatsFields(stats *runtime.GlobalStats) map[string]interface{} {
// Saves the values of a CounterMap in the database.
// The key are used as 'value' tag.
// The value is used as 'counter' field.
func (conn *Connection) addCounterMap(name string, m runtime.CounterMap, t time.Time, site string) {
func (conn *Connection) addCounterMap(name string, m runtime.CounterMap, t time.Time, site string, domain string) {
for key, count := range m {
conn.addPoint(
name,
models.Tags{
models.Tag{Key: []byte("value"), Value: []byte(key)},
models.Tag{Key: []byte("site"), Value: []byte(site)},
models.Tag{Key: []byte("domain"), Value: []byte(domain)},
},
models.Fields{"count": count},
t,

68
database/influxdb/global_test.go
View File

@ -12,18 +12,24 @@ import (
"github.com/FreifunkBremen/yanic/runtime"
)
const TEST_SITE = "ffxx"
const (
TEST_SITE = "ffhb"
TEST_DOMAIN = "city"
)
func TestGlobalStats(t *testing.T) {
stats := runtime.NewGlobalStats(createTestNodes(), []string{TEST_SITE})
stats := runtime.NewGlobalStats(createTestNodes(), map[string][]string{TEST_SITE: {TEST_DOMAIN}})
assert := assert.New(t)
// check SITE_GLOBAL fields
fields := GlobalStatsFields(stats[runtime.GLOBAL_SITE])
fields := GlobalStatsFields(stats[runtime.GLOBAL_SITE][runtime.GLOBAL_DOMAIN])
assert.EqualValues(3, fields["nodes"])
fields = GlobalStatsFields(stats[TEST_SITE])
fields = GlobalStatsFields(stats[TEST_SITE][runtime.GLOBAL_DOMAIN])
assert.EqualValues(2, fields["nodes"])
fields = GlobalStatsFields(stats[TEST_SITE][TEST_DOMAIN])
assert.EqualValues(1, fields["nodes"])
conn := &Connection{
@ -32,59 +38,80 @@ func TestGlobalStats(t *testing.T) {
global := 0
globalSite := 0
globalDomain := 0
model := 0
modelSite := 0
modelDomain := 0
firmware := 0
firmwareSite := 0
firmwareDomain := 0
autoupdater := 0
autoupdaterSite := 0
autoupdaterDomain := 0
wg := sync.WaitGroup{}
wg.Add(9)
wg.Add(15)
go func() {
for p := range conn.points {
switch p.Name() {
case MeasurementGlobal:
global++
break
case "global_site":
globalSite++
break
case "global_site_domain":
globalDomain++
case CounterMeasurementModel:
model++
break
case "model_site":
modelSite++
break
case "model_site_domain":
modelDomain++
case CounterMeasurementFirmware:
firmware++
break
case "firmware_site":
firmwareSite++
break
case "firmware_site_domain":
firmwareDomain++
case CounterMeasurementAutoupdater:
autoupdater++
break
case "autoupdater_site":
autoupdaterSite++
break
case "autoupdater_site_domain":
autoupdaterDomain++
default:
assert.Equal("invalid p.Name found", p.Name())
}
wg.Done()
}
}()
for site, stat := range stats {
conn.InsertGlobals(stat, time.Now(), site)
for site, domains := range stats {
for domain, stat := range domains {
conn.InsertGlobals(stat, time.Now(), site, domain)
}
}
wg.Wait()
assert.Equal(1, global)
assert.Equal(1, globalSite)
assert.Equal(1, globalDomain)
assert.Equal(2, model)
assert.Equal(1, modelSite)
assert.Equal(2, modelSite)
assert.Equal(1, modelDomain)
assert.Equal(1, firmware)
assert.Equal(0, firmwareSite)
assert.Equal(1, firmwareSite)
assert.Equal(0, firmwareDomain)
assert.Equal(2, autoupdater)
assert.Equal(1, autoupdaterSite)
assert.Equal(2, autoupdaterSite)
assert.Equal(1, autoupdaterDomain)
}
func createTestNodes() *runtime.Nodes {
@ -102,7 +129,9 @@ func createTestNodes() *runtime.Nodes {
Hardware: data.Hardware{
Model: "TP-Link 841",
},
System: data.System{},
System: data.System{
SiteCode: TEST_SITE,
},
},
}
nodeData.Nodeinfo.Software.Firmware.Release = "2016.1.6+entenhausen1"
@ -135,6 +164,7 @@ func createTestNodes() *runtime.Nodes {
},
System: data.System{
SiteCode: TEST_SITE,
DomainCode: TEST_DOMAIN,
},
},
})

3
database/influxdb/node.go
View File

@ -52,6 +52,9 @@ func (conn *Connection) InsertNode(node *runtime.Node) {
if nodeinfo.System.SiteCode != "" {
tags.SetString("site", nodeinfo.System.SiteCode)
}
if nodeinfo.System.DomainCode != "" {
tags.SetString("domain", nodeinfo.System.DomainCode)
}
if owner := nodeinfo.Owner; owner != nil {
tags.SetString("owner", owner.Contact)
}

6
database/influxdb/node_test.go
View File

@ -52,7 +52,8 @@ func TestToInflux(t *testing.T) {
Contact: "nobody",
},
System: data.System{
SiteCode: "ffxx",
SiteCode: "ffhb",
DomainCode: "city",
},
Wireless: &data.Wireless{
TxPower24: 3,
@ -133,7 +134,8 @@ func TestToInflux(t *testing.T) {
assert.EqualValues("deadbeef", tags["nodeid"])
assert.EqualValues("nobody", tags["owner"])
assert.EqualValues("testing", tags["autoupdater"])
assert.EqualValues("ffxx", tags["site"])
assert.EqualValues("ffhb", tags["site"])
assert.EqualValues("city", tags["domain"])
assert.EqualValues(0.5, fields["load"])
assert.EqualValues(0, fields["neighbours.lldp"])
assert.EqualValues(1, fields["neighbours.batadv"])

4
database/logging/file.go
View File

@ -50,8 +50,8 @@ func (conn *Connection) InsertLink(link *runtime.Link, time time.Time) {
conn.log("InsertLink: ", link)
}
func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) {
conn.log("InsertGlobals: [", time.String(), "] site: ", site, ", nodes: ", stats.Nodes, ", clients: ", stats.Clients, " models: ", len(stats.Models))
func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string, domain string) {
conn.log("InsertGlobals: [", time.String(), "] site: ", site, " domain: ", domain, ", nodes: ", stats.Nodes, ", clients: ", stats.Clients, " models: ", len(stats.Models))
}
func (conn *Connection) PruneNodes(deleteAfter time.Duration) {

2
database/logging/file_test.go
View File

@ -43,7 +43,7 @@ func TestStart(t *testing.T) {
assert.Contains(string(dat), "InsertLink")
assert.NotContains(string(dat), "InsertGlobals")
conn.InsertGlobals(&runtime.GlobalStats{}, time.Now(), runtime.GLOBAL_SITE)
conn.InsertGlobals(&runtime.GlobalStats{}, time.Now(), runtime.GLOBAL_SITE, runtime.GLOBAL_DOMAIN)
dat, _ = ioutil.ReadFile(path)
assert.Contains(string(dat), "InsertGlobals")

2
database/respondd/main.go
View File

@ -85,7 +85,7 @@ func (conn *Connection) InsertNode(node *runtime.Node) {
func (conn *Connection) InsertLink(link *runtime.Link, time time.Time) {
}
func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) {
func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string, domain string) {
}
func (conn *Connection) PruneNodes(deleteAfter time.Duration) {

63
docs/docs_configuration.md
View File

@ -15,8 +15,9 @@ enable = true
# synchronize = "1m"
collect_interval = "1m"
interfaces = ["br-ffhb"]
sites = ["ffhb"]
#port = 10001
#[respondd.sites.example]
#domains = ["city"]
```
{% endmethod %}
@ -64,16 +65,6 @@ interfaces = ["br-ffhb"]
{% endmethod %}
### sites
{% method %}
List of sites to save stats for (empty for global only)
{% sample lang="toml" %}
```toml
sites = ["ffhb"]
```
{% endmethod %}
### port
{% method %}
Define a port to listen and send the respondd packages.
@ -84,6 +75,25 @@ port = 10001
```
{% endmethod %}
### [respondd.sites.example]
{% method %}
Tables of sites to save stats for (not exists for global only).
Here is the site _ffhb_.
{% sample lang="toml" %}
```toml
[respondd.sites.ffhb]
domains = ["city"]
```
{% endmethod %}
#### domains
{% method %}
list of domains on this site to save stats for (empty for global only)
{% sample lang="toml" %}
```toml
domains = ["city"]
```
{% endmethod %}
## [webserver]
@ -197,6 +207,8 @@ enable = true
no_owner = true
blacklist = ["00112233445566", "1337f0badead"]
sites = ["ffhb"]
domain_as_site = true
domain_append_site = true
has_location = true
[nodes.output.example.filter.in_area]
latitude_min = 34.30
@ -258,6 +270,35 @@ blacklist = ["00112233445566", "1337f0badead"]
{% endmethod %}
### sites
{% method %}
List of site_codes of nodes that should be included in output
{% sample lang="toml" %}
```toml
sites = ["ffhb"]
```
{% endmethod %}
### domain_as_site
{% method %}
Replace the `site_code` with the `domain_code` in this output.
e.g. `site_code='ffhb',domain_code='city'` becomes `site_code='city', domain_code=''`
{% sample lang="toml" %}
```toml
domain_as_site = true
```
{% endmethod %}
### domain_append_site
{% method %}
Append on the `site_code` the `domain_code` with a `.` in this output.
e.g. `site_code='ffhb',domain_code='city'` becomes `site_code='ffhb.city', domain_code=''`
{% sample lang="toml" %}
```toml
domain_append_site = true
```
{% endmethod %}
### sites
{% method %}
List of site_codes of nodes that should be included in output

2
output/all/filter.go
View File

@ -2,6 +2,8 @@ package all
import (
_ "github.com/FreifunkBremen/yanic/output/filter/blacklist"
_ "github.com/FreifunkBremen/yanic/output/filter/domainappendsite"
_ "github.com/FreifunkBremen/yanic/output/filter/domainassite"
_ "github.com/FreifunkBremen/yanic/output/filter/haslocation"
_ "github.com/FreifunkBremen/yanic/output/filter/inarea"
_ "github.com/FreifunkBremen/yanic/output/filter/noowner"

50
output/filter/domainappendsite/domainappendsite.go Normal file
View File

@ -0,0 +1,50 @@
package domainappendsite
import (
"errors"
"github.com/FreifunkBremen/yanic/data"
"github.com/FreifunkBremen/yanic/output/filter"
"github.com/FreifunkBremen/yanic/runtime"
)
type domainAppendSite struct{ set bool }
func init() {
filter.Register("domain_append_site", build)
}
func build(config interface{}) (filter.Filter, error) {
if value, ok := config.(bool); ok {
return &domainAppendSite{set: value}, nil
}
return nil, errors.New("invalid configuration, boolean expected")
}
func (config *domainAppendSite) Apply(node *runtime.Node) *runtime.Node {
if nodeinfo := node.Nodeinfo; nodeinfo != nil && config.set && nodeinfo.System.DomainCode != "" {
node = &runtime.Node{
Address: node.Address,
Firstseen: node.Firstseen,
Lastseen: node.Lastseen,
Online: node.Online,
Statistics: node.Statistics,
Nodeinfo: &data.NodeInfo{
NodeID: nodeinfo.NodeID,
Network: nodeinfo.Network,
System: data.System{
SiteCode: nodeinfo.System.SiteCode + "." + nodeinfo.System.DomainCode,
},
Owner: nodeinfo.Owner,
Hostname: nodeinfo.Hostname,
Location: nodeinfo.Location,
Software: nodeinfo.Software,
Hardware: nodeinfo.Hardware,
VPN: nodeinfo.VPN,
Wireless: nodeinfo.Wireless,
},
Neighbours: node.Neighbours,
}
}
return node
}

43
output/filter/domainappendsite/domainappendsite_test.go Normal file
View File

@ -0,0 +1,43 @@
package domainappendsite
import (
"testing"
"github.com/FreifunkBremen/yanic/data"
"github.com/FreifunkBremen/yanic/runtime"
"github.com/stretchr/testify/assert"
)
func TestFilter(t *testing.T) {
assert := assert.New(t)
// invalid config
filter, err := build("nope")
assert.Error(err)
// delete owner by configuration
filter, _ = build(true)
n := filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{
System: data.System{
SiteCode: "ffhb",
DomainCode: "city",
},
}})
assert.NotNil(n)
assert.Equal("ffhb.city", n.Nodeinfo.System.SiteCode)
assert.Equal("", n.Nodeinfo.System.DomainCode)
// keep owner configuration
filter, _ = build(false)
n = filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{
System: data.System{
SiteCode: "ffhb",
DomainCode: "city",
},
}})
assert.NotNil(n)
assert.Equal("ffhb", n.Nodeinfo.System.SiteCode)
assert.Equal("city", n.Nodeinfo.System.DomainCode)
}

50
output/filter/domainassite/domainassite.go Normal file
View File

@ -0,0 +1,50 @@
package domainassite
import (
"errors"
"github.com/FreifunkBremen/yanic/data"
"github.com/FreifunkBremen/yanic/output/filter"
"github.com/FreifunkBremen/yanic/runtime"
)
type domainAsSite struct{ set bool }
func init() {
filter.Register("domain_as_site", build)
}
func build(config interface{}) (filter.Filter, error) {
if value, ok := config.(bool); ok {
return &domainAsSite{set: value}, nil
}
return nil, errors.New("invalid configuration, boolean expected")
}
func (config *domainAsSite) Apply(node *runtime.Node) *runtime.Node {
if nodeinfo := node.Nodeinfo; nodeinfo != nil && config.set && nodeinfo.System.DomainCode != "" {
node = &runtime.Node{
Address: node.Address,
Firstseen: node.Firstseen,
Lastseen: node.Lastseen,
Online: node.Online,
Statistics: node.Statistics,
Nodeinfo: &data.NodeInfo{
NodeID: nodeinfo.NodeID,
Network: nodeinfo.Network,
System: data.System{
SiteCode: nodeinfo.System.DomainCode,
},
Owner: nodeinfo.Owner,
Hostname: nodeinfo.Hostname,
Location: nodeinfo.Location,
Software: nodeinfo.Software,
Hardware: nodeinfo.Hardware,
VPN: nodeinfo.VPN,
Wireless: nodeinfo.Wireless,
},
Neighbours: node.Neighbours,
}
}
return node
}

43
output/filter/domainassite/domainassite_test.go Normal file
View File

@ -0,0 +1,43 @@
package domainassite
import (
"testing"
"github.com/FreifunkBremen/yanic/data"
"github.com/FreifunkBremen/yanic/runtime"
"github.com/stretchr/testify/assert"
)
func TestFilter(t *testing.T) {
assert := assert.New(t)
// invalid config
filter, err := build("nope")
assert.Error(err)
// delete owner by configuration
filter, _ = build(true)
n := filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{
System: data.System{
SiteCode: "ffhb",
DomainCode: "city",
},
}})
assert.NotNil(n)
assert.Equal("city", n.Nodeinfo.System.SiteCode)
assert.Equal("", n.Nodeinfo.System.DomainCode)
// keep owner configuration
filter, _ = build(false)
n = filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{
System: data.System{
SiteCode: "ffhb",
DomainCode: "city",
},
}})
assert.NotNil(n)
assert.Equal("ffhb", n.Nodeinfo.System.SiteCode)
assert.Equal("city", n.Nodeinfo.System.DomainCode)
}

2
output/meshviewer-ffrgb/struct.go
View File

@ -33,6 +33,7 @@ type Node struct {
MAC string `json:"mac"`
Addresses []string `json:"addresses"`
SiteCode string `json:"site_code,omitempty"`
DomainCode string `json:"-"`
Hostname string `json:"hostname"`
Owner string `json:"owner,omitempty"`
Location *Location `json:"location,omitempty"`
@ -85,6 +86,7 @@ func NewNode(nodes *runtime.Nodes, n *runtime.Node) *Node {
node.MAC = nodeinfo.Network.Mac
node.Addresses = nodeinfo.Network.Addresses
node.SiteCode = nodeinfo.System.SiteCode
node.DomainCode = nodeinfo.System.DomainCode
node.Hostname = nodeinfo.Hostname
if owner := nodeinfo.Owner; owner != nil {
node.Owner = owner.Contact

14
respond/collector.go
View File

@ -24,18 +24,18 @@ type Collector struct {
queue chan *Response // received responses
db database.Connection
nodes *runtime.Nodes
sites []string
sitesDomains map[string][]string
interval time.Duration // Interval for multicast packets
stop chan interface{}
}
// NewCollector creates a Collector struct
func NewCollector(db database.Connection, nodes *runtime.Nodes, sites []string, ifaces []string, port int) *Collector {
func NewCollector(db database.Connection, nodes *runtime.Nodes, sitesDomains map[string][]string, ifaces []string, port int) *Collector {
coll := &Collector{
db: db,
nodes: nodes,
sites: sites,
sitesDomains: sitesDomains,
port: port,
queue: make(chan *Response, 400),
stop: make(chan interface{}),
@ -302,9 +302,11 @@ func (coll *Collector) globalStatsWorker() {
// saves global statistics
func (coll *Collector) saveGlobalStats() {
stats := runtime.NewGlobalStats(coll.nodes, coll.sites)
stats := runtime.NewGlobalStats(coll.nodes, coll.sitesDomains)
for site, stat := range stats {
coll.db.InsertGlobals(stat, time.Now(), site)
for site, domains := range stats {
for domain, stat := range domains {
coll.db.InsertGlobals(stat, time.Now(), site, domain)
}
}
}

7
respond/collector_test.go
View File

@ -9,12 +9,15 @@ import (
"github.com/stretchr/testify/assert"
)
const SITE_TEST = "ffxx"
const (
SITE_TEST = "ffhb"
DOMAIN_TEST = "city"
)
func TestCollector(t *testing.T) {
nodes := runtime.NewNodes(&runtime.NodesConfig{})
collector := NewCollector(nil, nodes, []string{SITE_TEST}, []string{}, 10001)
collector := NewCollector(nil, nodes, map[string][]string{SITE_TEST: {DOMAIN_TEST}}, []string{}, 10001)
collector.Start(time.Millisecond)
time.Sleep(time.Millisecond * 10)
collector.Close()

14
respond/config.go
View File

@ -6,7 +6,19 @@ type Config struct {
Enable bool `toml:"enable"`
Synchronize duration.Duration `toml:"synchronize"`
Interfaces []string `toml:"interfaces"`
Sites []string `toml:"sites"`
Sites map[string]SiteConfig `toml:"sites"`
Port int `toml:"port"`
CollectInterval duration.Duration `toml:"collect_interval"`
}
func (c *Config) SitesDomains() (result map[string][]string) {
result = make(map[string][]string)
for site, siteConfig := range c.Sites {
result[site] = siteConfig.Domains
}
return
}
type SiteConfig struct {
Domains []string `toml:"domains"`
}

24
respond/config_test.go Normal file
View File

@ -0,0 +1,24 @@
package respond
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestSitesDomainsConfigTransform(t *testing.T) {
assert := assert.New(t)
c := Config{
Sites: map[string]SiteConfig{
"ffhb": {Domains: []string{"city"}},
},
}
result := c.SitesDomains()
assert.Len(result, 1)
assert.Contains(result, "ffhb")
domains := result["ffhb"]
assert.Len(domains, 1)
assert.Equal("city", domains[0])
}

30
runtime/stats.go
View File

@ -3,6 +3,7 @@ package runtime
const (
DISABLED_AUTOUPDATER = "disabled"
GLOBAL_SITE = "global"
GLOBAL_DOMAIN = "global"
)
// CounterMap to manage multiple values
@ -23,32 +24,45 @@ type GlobalStats struct {
}
//NewGlobalStats returns global statistics for InfluxDB
func NewGlobalStats(nodes *Nodes, sites []string) (result map[string]*GlobalStats) {
result = make(map[string]*GlobalStats)
func NewGlobalStats(nodes *Nodes, sitesDomains map[string][]string) (result map[string]map[string]*GlobalStats) {
result = make(map[string]map[string]*GlobalStats)
result[GLOBAL_SITE] = &GlobalStats{
result[GLOBAL_SITE] = make(map[string]*GlobalStats)
result[GLOBAL_SITE][GLOBAL_DOMAIN] = &GlobalStats{
Firmwares: make(CounterMap),
Models: make(CounterMap),
Autoupdater: make(CounterMap),
}
for _, site := range sites {
result[site] = &GlobalStats{
for site, domains := range sitesDomains {
result[site] = make(map[string]*GlobalStats)
result[site][GLOBAL_DOMAIN] = &GlobalStats{
Firmwares: make(CounterMap),
Models: make(CounterMap),
Autoupdater: make(CounterMap),
}
for _, domain := range domains {
result[site][domain] = &GlobalStats{
Firmwares: make(CounterMap),
Models: make(CounterMap),
Autoupdater: make(CounterMap),
}
}
}
nodes.RLock()
for _, node := range nodes.List {
if node.Online {
result[GLOBAL_SITE].Add(node)
result[GLOBAL_SITE][GLOBAL_DOMAIN].Add(node)
if info := node.Nodeinfo; info != nil {
site := info.System.SiteCode
if _, exist := result[site]; exist {
result[site].Add(node)
domain := info.System.DomainCode
if _, ok := result[site]; ok {
result[site][GLOBAL_DOMAIN].Add(node)
if _, ok := result[site][domain]; ok {
result[site][domain].Add(node)
}
}
}
}

66
runtime/stats_test.go
View File

@ -8,49 +8,70 @@ import (
"github.com/FreifunkBremen/yanic/data"
)
const TEST_SITE = "ffxx"
const (
TEST_SITE = "ffhb"
TEST_DOMAIN = "city"
)
func TestGlobalStats(t *testing.T) {
stats := NewGlobalStats(createTestNodes(), []string{TEST_SITE})
stats := NewGlobalStats(createTestNodes(), map[string][]string{TEST_SITE: {TEST_DOMAIN}})
assert := assert.New(t)
assert.Len(stats, 2)
//check GLOBAL_SITE stats
assert.EqualValues(1, stats[GLOBAL_SITE].Gateways)
assert.EqualValues(3, stats[GLOBAL_SITE].Nodes)
assert.EqualValues(25, stats[GLOBAL_SITE].Clients)
assert.EqualValues(1, stats[GLOBAL_SITE][GLOBAL_DOMAIN].Gateways)
assert.EqualValues(3, stats[GLOBAL_SITE][GLOBAL_DOMAIN].Nodes)
assert.EqualValues(25, stats[GLOBAL_SITE][GLOBAL_DOMAIN].Clients)
// check models
assert.Len(stats[GLOBAL_SITE].Models, 2)
assert.EqualValues(2, stats[GLOBAL_SITE].Models["TP-Link 841"])
assert.EqualValues(1, stats[GLOBAL_SITE].Models["Xeon Multi-Core"])
assert.Len(stats[GLOBAL_SITE][GLOBAL_DOMAIN].Models, 2)
assert.EqualValues(2, stats[GLOBAL_SITE][GLOBAL_DOMAIN].Models["TP-Link 841"])
assert.EqualValues(1, stats[GLOBAL_SITE][GLOBAL_DOMAIN].Models["Xeon Multi-Core"])
// check firmwares
assert.Len(stats[GLOBAL_SITE].Firmwares, 1)
assert.EqualValues(1, stats[GLOBAL_SITE].Firmwares["2016.1.6+entenhausen1"])
assert.Len(stats[GLOBAL_SITE][GLOBAL_DOMAIN].Firmwares, 1)
assert.EqualValues(1, stats[GLOBAL_SITE][GLOBAL_DOMAIN].Firmwares["2016.1.6+entenhausen1"])
// check autoupdater
assert.Len(stats[GLOBAL_SITE].Autoupdater, 2)
assert.EqualValues(1, stats[GLOBAL_SITE].Autoupdater["stable"])
assert.Len(stats[GLOBAL_SITE][GLOBAL_DOMAIN].Autoupdater, 2)
assert.EqualValues(1, stats[GLOBAL_SITE][GLOBAL_DOMAIN].Autoupdater["stable"])
// check TEST_SITE stats
assert.EqualValues(1, stats[TEST_SITE].Gateways)
assert.EqualValues(2, stats[TEST_SITE].Nodes)
assert.EqualValues(23, stats[TEST_SITE].Clients)
assert.EqualValues(1, stats[TEST_SITE][GLOBAL_DOMAIN].Gateways)
assert.EqualValues(2, stats[TEST_SITE][GLOBAL_DOMAIN].Nodes)
assert.EqualValues(23, stats[TEST_SITE][GLOBAL_DOMAIN].Clients)
// check models
assert.Len(stats[TEST_SITE].Models, 2)
assert.EqualValues(1, stats[TEST_SITE].Models["TP-Link 841"])
assert.EqualValues(1, stats[TEST_SITE].Models["Xeon Multi-Core"])
assert.Len(stats[TEST_SITE][GLOBAL_DOMAIN].Models, 2)
assert.EqualValues(1, stats[TEST_SITE][GLOBAL_DOMAIN].Models["TP-Link 841"])
assert.EqualValues(1, stats[TEST_SITE][GLOBAL_DOMAIN].Models["Xeon Multi-Core"])
// check firmwares
assert.Len(stats[TEST_SITE].Firmwares, 1)
assert.EqualValues(1, stats[TEST_SITE].Firmwares["2016.1.6+entenhausen1"])
assert.Len(stats[TEST_SITE][GLOBAL_DOMAIN].Firmwares, 1)
assert.EqualValues(1, stats[TEST_SITE][GLOBAL_DOMAIN].Firmwares["2016.1.6+entenhausen1"])
// check autoupdater
assert.Len(stats[TEST_SITE].Autoupdater, 1)
assert.EqualValues(0, stats[TEST_SITE].Autoupdater["stable"])
assert.Len(stats[TEST_SITE][GLOBAL_DOMAIN].Autoupdater, 1)
assert.EqualValues(0, stats[TEST_SITE][GLOBAL_DOMAIN].Autoupdater["stable"])
// check TEST_DOMAIN stats
assert.EqualValues(1, stats[TEST_SITE][TEST_DOMAIN].Gateways)
assert.EqualValues(1, stats[TEST_SITE][TEST_DOMAIN].Nodes)
assert.EqualValues(0, stats[TEST_SITE][TEST_DOMAIN].Clients)
// check models
assert.Len(stats[TEST_SITE][TEST_DOMAIN].Models, 1)
assert.EqualValues(0, stats[TEST_SITE][TEST_DOMAIN].Models["TP-Link 841"])
assert.EqualValues(1, stats[TEST_SITE][TEST_DOMAIN].Models["Xeon Multi-Core"])
// check firmwares
assert.Len(stats[TEST_SITE][TEST_DOMAIN].Firmwares, 0)
assert.EqualValues(0, stats[TEST_SITE][TEST_DOMAIN].Firmwares["2016.1.6+entenhausen1"])
// check autoupdater
assert.Len(stats[TEST_SITE][TEST_DOMAIN].Autoupdater, 1)
assert.EqualValues(0, stats[TEST_SITE][TEST_DOMAIN].Autoupdater["stable"])
}
func createTestNodes() *Nodes {
@ -110,6 +131,7 @@ func createTestNodes() *Nodes {
},
System: data.System{
SiteCode: TEST_SITE,
DomainCode: TEST_DOMAIN,
},
},
})