From 950ad8457f07aa738e7cb09d3c27689067e7cb42 Mon Sep 17 00:00:00 2001 From: Martin Geno Date: Wed, 17 Jan 2018 20:20:35 +0100 Subject: [PATCH] [TASK] handle domain_code (with filters) (#119) --- cmd/config_test.go | 4 ++ cmd/import.go | 8 ++- cmd/query.go | 4 +- cmd/serve.go | 2 +- config_example.toml | 17 ++++- data/nodeinfo.go | 3 +- database/all/connection.go | 4 +- database/database.go | 2 +- database/graphite/global.go | 9 ++- database/influxdb/global.go | 25 ++++--- database/influxdb/global_test.go | 70 +++++++++++++------ database/influxdb/node.go | 3 + database/influxdb/node_test.go | 6 +- database/logging/file.go | 4 +- database/logging/file_test.go | 2 +- database/respondd/main.go | 2 +- docs/docs_configuration.md | 63 ++++++++++++++--- output/all/filter.go | 2 + .../domainappendsite/domainappendsite.go | 50 +++++++++++++ .../domainappendsite/domainappendsite_test.go | 43 ++++++++++++ output/filter/domainassite/domainassite.go | 50 +++++++++++++ .../filter/domainassite/domainassite_test.go | 43 ++++++++++++ output/meshviewer-ffrgb/struct.go | 2 + respond/collector.go | 36 +++++----- respond/collector_test.go | 7 +- respond/config.go | 24 +++++-- respond/config_test.go | 24 +++++++ runtime/stats.go | 30 +++++--- runtime/stats_test.go | 68 ++++++++++++------ 29 files changed, 493 insertions(+), 114 deletions(-) create mode 100644 output/filter/domainappendsite/domainappendsite.go create mode 100644 output/filter/domainappendsite/domainappendsite_test.go create mode 100644 output/filter/domainassite/domainassite.go create mode 100644 output/filter/domainassite/domainassite_test.go create mode 100644 respond/config_test.go diff --git a/cmd/config_test.go b/cmd/config_test.go index 6a85403..eaa95f1 100644 --- a/cmd/config_test.go +++ b/cmd/config_test.go @@ -20,6 +20,10 @@ func TestReadConfig(t *testing.T) { assert.Equal(time.Hour*24*7, config.Nodes.PruneAfter.Duration) assert.Equal(time.Hour*24*7, config.Database.DeleteAfter.Duration) + assert.Len(config.Respondd.Sites, 1) + assert.Contains(config.Respondd.Sites, "ffhb") + assert.Contains(config.Respondd.Sites["ffhb"].Domains, "city") + // Test output plugins assert.Len(config.Nodes.Output, 3) outputs := config.Nodes.Output["meshviewer"].([]interface{}) diff --git a/cmd/import.go b/cmd/import.go index 643fa18..a110376 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -11,13 +11,14 @@ import ( // importCmd represents the import command var importCmd = &cobra.Command{ - Use: "import ", + Use: "import ", Short: "Imports global statistics from the given RRD files, requires InfluxDB", - Example: "yanic import --config /etc/yanic.toml olddata.rrd global", - Args: cobra.ExactArgs(2), + Example: "yanic import --config /etc/yanic.toml olddata.rrd global global", + Args: cobra.ExactArgs(3), Run: func(cmd *cobra.Command, args []string) { path := args[0] site := args[1] + domain := args[2] config := loadConfig() err := allDatabase.Start(config.Database) @@ -36,6 +37,7 @@ var importCmd = &cobra.Command{ }, ds.Time, site, + domain, ) } }, diff --git a/cmd/query.go b/cmd/query.go index ea81708..9ab3785 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -26,7 +26,9 @@ var queryCmd = &cobra.Command{ nodes := runtime.NewNodes(&runtime.NodesConfig{}) - collector := respond.NewCollector(nil, nodes, []string{}, []string{iface}, 0) + sitesDomains := make(map[string][]string) + + collector := respond.NewCollector(nil, nodes, sitesDomains, []string{iface}, 0) defer collector.Close() collector.SendPacket(dstAddress) diff --git a/cmd/serve.go b/cmd/serve.go index 8fe5b5f..36edaf8 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -54,7 +54,7 @@ var serveCmd = &cobra.Command{ time.Sleep(delay) } - collector = respond.NewCollector(allDatabase.Conn, nodes, config.Respondd.Sites, config.Respondd.Interfaces, config.Respondd.Port) + collector = respond.NewCollector(allDatabase.Conn, nodes, config.Respondd.SitesDomains(), config.Respondd.Interfaces, config.Respondd.Port) collector.Start(config.Respondd.CollectInterval.Duration) defer collector.Close() } diff --git a/config_example.toml b/config_example.toml index 993f9fc..bc76630 100644 --- a/config_example.toml +++ b/config_example.toml @@ -11,12 +11,17 @@ synchronize = "1m" collect_interval = "1m" # interface that has an IP in your mesh network interfaces = ["br-ffhb"] -# list of sites to save stats for (empty for global only) -sites = [] # define a port to listen # if not set or set to 0 the kernel will use a random free port at its own #port = 10001 +# table of a site to save stats for (not exists for global only) +#[respondd.sites.example] +## list of domains on this site to save stats for (empty for global only) +#domains = [] +## example +[respondd.sites.ffhb] +domains = ["city"] # A little build-in webserver, which statically serves a directory. # This is useful for testing purposes or for a little standalone installation. @@ -56,6 +61,14 @@ offline_after = "10m" # List of site_codes of nodes that should be included in the output #sites = ["ffhb"] # +# replace the site_code with the domain_code in this output +# e.g. site_code='ffhb',domain_code='city' => site_code='city', domain_code='' +#domain_as_site = true +# +# append on the site_code the domain_code with a '.' in this output +# e.g. site_code='ffhb',domain_code='city' => site_code='ffhb.city', domain_code='' +#domain_append_site = true +# # set has_location to true if you want to include only nodes that have geo-coordinates set # (setting this to false has no sensible effect, unless you'd want to hide nodes that have coordinates) #has_location = true diff --git a/data/nodeinfo.go b/data/nodeinfo.go index 6a402a2..3f9fb29 100644 --- a/data/nodeinfo.go +++ b/data/nodeinfo.go @@ -43,7 +43,8 @@ type Owner struct { // System struct type System struct { - SiteCode string `json:"site_code,omitempty"` + SiteCode string `json:"site_code,omitempty"` + DomainCode string `json:"domain_code,omitempty"` } // Location struct diff --git a/database/all/connection.go b/database/all/connection.go index fcdc3a2..2c69828 100644 --- a/database/all/connection.go +++ b/database/all/connection.go @@ -60,9 +60,9 @@ func (conn *Connection) InsertLink(link *runtime.Link, time time.Time) { } } -func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) { +func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string, domain string) { for _, item := range conn.list { - item.InsertGlobals(stats, time, site) + item.InsertGlobals(stats, time, site, domain) } } diff --git a/database/database.go b/database/database.go index c40768d..096a2b5 100644 --- a/database/database.go +++ b/database/database.go @@ -15,7 +15,7 @@ type Connection interface { InsertLink(*runtime.Link, time.Time) // InsertGlobals stores global statistics - InsertGlobals(*runtime.GlobalStats, time.Time, string) + InsertGlobals(*runtime.GlobalStats, time.Time, string, string) // PruneNodes prunes historical per-node data PruneNodes(deleteAfter time.Duration) diff --git a/database/graphite/global.go b/database/graphite/global.go index acdd056..62fff66 100644 --- a/database/graphite/global.go +++ b/database/graphite/global.go @@ -7,7 +7,7 @@ import ( "github.com/fgrosse/graphigo" ) -func (c *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) { +func (c *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string, domain string) { measurementGlobal := MeasurementGlobal counterMeasurementModel := CounterMeasurementModel counterMeasurementFirmware := CounterMeasurementFirmware @@ -20,6 +20,13 @@ func (c *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, s counterMeasurementAutoupdater += "_" + site } + if domain != runtime.GLOBAL_DOMAIN { + measurementGlobal += "_" + domain + counterMeasurementModel += "_" + domain + counterMeasurementFirmware += "_" + domain + counterMeasurementAutoupdater += "_" + domain + } + c.addPoint(GlobalStatsFields(measurementGlobal, stats)) c.addCounterMap(counterMeasurementModel, stats.Models, time) c.addCounterMap(counterMeasurementFirmware, stats.Firmwares, time) diff --git a/database/influxdb/global.go b/database/influxdb/global.go index f49bc6e..c99e699 100644 --- a/database/influxdb/global.go +++ b/database/influxdb/global.go @@ -8,8 +8,8 @@ import ( ) // InsertGlobals implementation of database -func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) { - var tags models.Tags +func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string, domain string) { + tags := models.Tags{} measurementGlobal := MeasurementGlobal counterMeasurementModel := CounterMeasurementModel @@ -17,20 +17,26 @@ func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time counterMeasurementAutoupdater := CounterMeasurementAutoupdater if site != runtime.GLOBAL_SITE { - tags = models.Tags{ - models.Tag{Key: []byte("site"), Value: []byte(site)}, - } + tags.Set([]byte("site"), []byte(site)) measurementGlobal += "_site" counterMeasurementModel += "_site" counterMeasurementFirmware += "_site" counterMeasurementAutoupdater += "_site" } + if domain != runtime.GLOBAL_DOMAIN { + tags.Set([]byte("domain"), []byte(domain)) + + measurementGlobal += "_domain" + counterMeasurementModel += "_domain" + counterMeasurementFirmware += "_domain" + counterMeasurementAutoupdater += "_domain" + } conn.addPoint(measurementGlobal, tags, GlobalStatsFields(stats), time) - conn.addCounterMap(counterMeasurementModel, stats.Models, time, site) - conn.addCounterMap(counterMeasurementFirmware, stats.Firmwares, time, site) - conn.addCounterMap(counterMeasurementAutoupdater, stats.Autoupdater, time, site) + conn.addCounterMap(counterMeasurementModel, stats.Models, time, site, domain) + conn.addCounterMap(counterMeasurementFirmware, stats.Firmwares, time, site, domain) + conn.addCounterMap(counterMeasurementAutoupdater, stats.Autoupdater, time, site, domain) } // GlobalStatsFields returns fields for InfluxDB @@ -48,13 +54,14 @@ func GlobalStatsFields(stats *runtime.GlobalStats) map[string]interface{} { // Saves the values of a CounterMap in the database. // The key are used as 'value' tag. // The value is used as 'counter' field. -func (conn *Connection) addCounterMap(name string, m runtime.CounterMap, t time.Time, site string) { +func (conn *Connection) addCounterMap(name string, m runtime.CounterMap, t time.Time, site string, domain string) { for key, count := range m { conn.addPoint( name, models.Tags{ models.Tag{Key: []byte("value"), Value: []byte(key)}, models.Tag{Key: []byte("site"), Value: []byte(site)}, + models.Tag{Key: []byte("domain"), Value: []byte(domain)}, }, models.Fields{"count": count}, t, diff --git a/database/influxdb/global_test.go b/database/influxdb/global_test.go index f990b5e..65c3320 100644 --- a/database/influxdb/global_test.go +++ b/database/influxdb/global_test.go @@ -12,18 +12,24 @@ import ( "github.com/FreifunkBremen/yanic/runtime" ) -const TEST_SITE = "ffxx" +const ( + TEST_SITE = "ffhb" + TEST_DOMAIN = "city" +) func TestGlobalStats(t *testing.T) { - stats := runtime.NewGlobalStats(createTestNodes(), []string{TEST_SITE}) + stats := runtime.NewGlobalStats(createTestNodes(), map[string][]string{TEST_SITE: {TEST_DOMAIN}}) assert := assert.New(t) // check SITE_GLOBAL fields - fields := GlobalStatsFields(stats[runtime.GLOBAL_SITE]) + fields := GlobalStatsFields(stats[runtime.GLOBAL_SITE][runtime.GLOBAL_DOMAIN]) assert.EqualValues(3, fields["nodes"]) - fields = GlobalStatsFields(stats[TEST_SITE]) + fields = GlobalStatsFields(stats[TEST_SITE][runtime.GLOBAL_DOMAIN]) + assert.EqualValues(2, fields["nodes"]) + fields = GlobalStatsFields(stats[TEST_SITE][TEST_DOMAIN]) + assert.EqualValues(1, fields["nodes"]) conn := &Connection{ @@ -32,59 +38,80 @@ func TestGlobalStats(t *testing.T) { global := 0 globalSite := 0 + globalDomain := 0 + model := 0 modelSite := 0 + modelDomain := 0 + firmware := 0 firmwareSite := 0 + firmwareDomain := 0 + autoupdater := 0 autoupdaterSite := 0 + autoupdaterDomain := 0 + wg := sync.WaitGroup{} - wg.Add(9) + wg.Add(15) go func() { for p := range conn.points { switch p.Name() { case MeasurementGlobal: global++ - break case "global_site": globalSite++ - break + case "global_site_domain": + globalDomain++ + case CounterMeasurementModel: model++ - break case "model_site": modelSite++ - break + case "model_site_domain": + modelDomain++ + case CounterMeasurementFirmware: firmware++ - break case "firmware_site": firmwareSite++ - break + case "firmware_site_domain": + firmwareDomain++ + case CounterMeasurementAutoupdater: autoupdater++ - break case "autoupdater_site": autoupdaterSite++ - break + case "autoupdater_site_domain": + autoupdaterDomain++ + default: assert.Equal("invalid p.Name found", p.Name()) } wg.Done() } }() - for site, stat := range stats { - conn.InsertGlobals(stat, time.Now(), site) + for site, domains := range stats { + for domain, stat := range domains { + conn.InsertGlobals(stat, time.Now(), site, domain) + } } wg.Wait() assert.Equal(1, global) assert.Equal(1, globalSite) + assert.Equal(1, globalDomain) + assert.Equal(2, model) - assert.Equal(1, modelSite) + assert.Equal(2, modelSite) + assert.Equal(1, modelDomain) + assert.Equal(1, firmware) - assert.Equal(0, firmwareSite) + assert.Equal(1, firmwareSite) + assert.Equal(0, firmwareDomain) + assert.Equal(2, autoupdater) - assert.Equal(1, autoupdaterSite) + assert.Equal(2, autoupdaterSite) + assert.Equal(1, autoupdaterDomain) } func createTestNodes() *runtime.Nodes { @@ -102,7 +129,9 @@ func createTestNodes() *runtime.Nodes { Hardware: data.Hardware{ Model: "TP-Link 841", }, - System: data.System{}, + System: data.System{ + SiteCode: TEST_SITE, + }, }, } nodeData.Nodeinfo.Software.Firmware.Release = "2016.1.6+entenhausen1" @@ -134,7 +163,8 @@ func createTestNodes() *runtime.Nodes { Model: "Xeon Multi-Core", }, System: data.System{ - SiteCode: TEST_SITE, + SiteCode: TEST_SITE, + DomainCode: TEST_DOMAIN, }, }, }) diff --git a/database/influxdb/node.go b/database/influxdb/node.go index f54db7d..17acc13 100644 --- a/database/influxdb/node.go +++ b/database/influxdb/node.go @@ -52,6 +52,9 @@ func (conn *Connection) InsertNode(node *runtime.Node) { if nodeinfo.System.SiteCode != "" { tags.SetString("site", nodeinfo.System.SiteCode) } + if nodeinfo.System.DomainCode != "" { + tags.SetString("domain", nodeinfo.System.DomainCode) + } if owner := nodeinfo.Owner; owner != nil { tags.SetString("owner", owner.Contact) } diff --git a/database/influxdb/node_test.go b/database/influxdb/node_test.go index 70d310a..47cb5fc 100644 --- a/database/influxdb/node_test.go +++ b/database/influxdb/node_test.go @@ -52,7 +52,8 @@ func TestToInflux(t *testing.T) { Contact: "nobody", }, System: data.System{ - SiteCode: "ffxx", + SiteCode: "ffhb", + DomainCode: "city", }, Wireless: &data.Wireless{ TxPower24: 3, @@ -133,7 +134,8 @@ func TestToInflux(t *testing.T) { assert.EqualValues("deadbeef", tags["nodeid"]) assert.EqualValues("nobody", tags["owner"]) assert.EqualValues("testing", tags["autoupdater"]) - assert.EqualValues("ffxx", tags["site"]) + assert.EqualValues("ffhb", tags["site"]) + assert.EqualValues("city", tags["domain"]) assert.EqualValues(0.5, fields["load"]) assert.EqualValues(0, fields["neighbours.lldp"]) assert.EqualValues(1, fields["neighbours.batadv"]) diff --git a/database/logging/file.go b/database/logging/file.go index 328e307..e086d0f 100644 --- a/database/logging/file.go +++ b/database/logging/file.go @@ -50,8 +50,8 @@ func (conn *Connection) InsertLink(link *runtime.Link, time time.Time) { conn.log("InsertLink: ", link) } -func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) { - conn.log("InsertGlobals: [", time.String(), "] site: ", site, ", nodes: ", stats.Nodes, ", clients: ", stats.Clients, " models: ", len(stats.Models)) +func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string, domain string) { + conn.log("InsertGlobals: [", time.String(), "] site: ", site, " domain: ", domain, ", nodes: ", stats.Nodes, ", clients: ", stats.Clients, " models: ", len(stats.Models)) } func (conn *Connection) PruneNodes(deleteAfter time.Duration) { diff --git a/database/logging/file_test.go b/database/logging/file_test.go index 50b59ac..7c01be2 100644 --- a/database/logging/file_test.go +++ b/database/logging/file_test.go @@ -43,7 +43,7 @@ func TestStart(t *testing.T) { assert.Contains(string(dat), "InsertLink") assert.NotContains(string(dat), "InsertGlobals") - conn.InsertGlobals(&runtime.GlobalStats{}, time.Now(), runtime.GLOBAL_SITE) + conn.InsertGlobals(&runtime.GlobalStats{}, time.Now(), runtime.GLOBAL_SITE, runtime.GLOBAL_DOMAIN) dat, _ = ioutil.ReadFile(path) assert.Contains(string(dat), "InsertGlobals") diff --git a/database/respondd/main.go b/database/respondd/main.go index 998eec7..34f1115 100644 --- a/database/respondd/main.go +++ b/database/respondd/main.go @@ -85,7 +85,7 @@ func (conn *Connection) InsertNode(node *runtime.Node) { func (conn *Connection) InsertLink(link *runtime.Link, time time.Time) { } -func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) { +func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string, domain string) { } func (conn *Connection) PruneNodes(deleteAfter time.Duration) { diff --git a/docs/docs_configuration.md b/docs/docs_configuration.md index 5c67c16..fce86d1 100644 --- a/docs/docs_configuration.md +++ b/docs/docs_configuration.md @@ -15,8 +15,9 @@ enable = true # synchronize = "1m" collect_interval = "1m" interfaces = ["br-ffhb"] -sites = ["ffhb"] #port = 10001 +#[respondd.sites.example] +#domains = ["city"] ``` {% endmethod %} @@ -64,16 +65,6 @@ interfaces = ["br-ffhb"] {% endmethod %} -### sites -{% method %} -List of sites to save stats for (empty for global only) -{% sample lang="toml" %} -```toml -sites = ["ffhb"] -``` -{% endmethod %} - - ### port {% method %} Define a port to listen and send the respondd packages. @@ -84,6 +75,25 @@ port = 10001 ``` {% endmethod %} +### [respondd.sites.example] +{% method %} +Tables of sites to save stats for (not exists for global only). +Here is the site _ffhb_. +{% sample lang="toml" %} +```toml +[respondd.sites.ffhb] +domains = ["city"] +``` +{% endmethod %} +#### domains +{% method %} +list of domains on this site to save stats for (empty for global only) +{% sample lang="toml" %} +```toml +domains = ["city"] +``` +{% endmethod %} + ## [webserver] @@ -197,6 +207,8 @@ enable = true no_owner = true blacklist = ["00112233445566", "1337f0badead"] sites = ["ffhb"] +domain_as_site = true +domain_append_site = true has_location = true [nodes.output.example.filter.in_area] latitude_min = 34.30 @@ -258,6 +270,35 @@ blacklist = ["00112233445566", "1337f0badead"] {% endmethod %} +### sites +{% method %} +List of site_codes of nodes that should be included in output +{% sample lang="toml" %} +```toml +sites = ["ffhb"] +``` +{% endmethod %} + +### domain_as_site +{% method %} +Replace the `site_code` with the `domain_code` in this output. +e.g. `site_code='ffhb',domain_code='city'` becomes `site_code='city', domain_code=''` +{% sample lang="toml" %} +```toml +domain_as_site = true +``` +{% endmethod %} + +### domain_append_site +{% method %} +Append on the `site_code` the `domain_code` with a `.` in this output. +e.g. `site_code='ffhb',domain_code='city'` becomes `site_code='ffhb.city', domain_code=''` +{% sample lang="toml" %} +```toml +domain_append_site = true +``` +{% endmethod %} + ### sites {% method %} List of site_codes of nodes that should be included in output diff --git a/output/all/filter.go b/output/all/filter.go index 1c95ef5..72b114c 100644 --- a/output/all/filter.go +++ b/output/all/filter.go @@ -2,6 +2,8 @@ package all import ( _ "github.com/FreifunkBremen/yanic/output/filter/blacklist" + _ "github.com/FreifunkBremen/yanic/output/filter/domainappendsite" + _ "github.com/FreifunkBremen/yanic/output/filter/domainassite" _ "github.com/FreifunkBremen/yanic/output/filter/haslocation" _ "github.com/FreifunkBremen/yanic/output/filter/inarea" _ "github.com/FreifunkBremen/yanic/output/filter/noowner" diff --git a/output/filter/domainappendsite/domainappendsite.go b/output/filter/domainappendsite/domainappendsite.go new file mode 100644 index 0000000..17d3e89 --- /dev/null +++ b/output/filter/domainappendsite/domainappendsite.go @@ -0,0 +1,50 @@ +package domainappendsite + +import ( + "errors" + + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/output/filter" + "github.com/FreifunkBremen/yanic/runtime" +) + +type domainAppendSite struct{ set bool } + +func init() { + filter.Register("domain_append_site", build) +} + +func build(config interface{}) (filter.Filter, error) { + if value, ok := config.(bool); ok { + return &domainAppendSite{set: value}, nil + } + return nil, errors.New("invalid configuration, boolean expected") +} + +func (config *domainAppendSite) Apply(node *runtime.Node) *runtime.Node { + if nodeinfo := node.Nodeinfo; nodeinfo != nil && config.set && nodeinfo.System.DomainCode != "" { + node = &runtime.Node{ + Address: node.Address, + Firstseen: node.Firstseen, + Lastseen: node.Lastseen, + Online: node.Online, + Statistics: node.Statistics, + Nodeinfo: &data.NodeInfo{ + NodeID: nodeinfo.NodeID, + Network: nodeinfo.Network, + System: data.System{ + SiteCode: nodeinfo.System.SiteCode + "." + nodeinfo.System.DomainCode, + }, + Owner: nodeinfo.Owner, + Hostname: nodeinfo.Hostname, + Location: nodeinfo.Location, + Software: nodeinfo.Software, + Hardware: nodeinfo.Hardware, + VPN: nodeinfo.VPN, + Wireless: nodeinfo.Wireless, + }, + Neighbours: node.Neighbours, + } + } + return node +} diff --git a/output/filter/domainappendsite/domainappendsite_test.go b/output/filter/domainappendsite/domainappendsite_test.go new file mode 100644 index 0000000..78ca86e --- /dev/null +++ b/output/filter/domainappendsite/domainappendsite_test.go @@ -0,0 +1,43 @@ +package domainappendsite + +import ( + "testing" + + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/runtime" + "github.com/stretchr/testify/assert" +) + +func TestFilter(t *testing.T) { + assert := assert.New(t) + + // invalid config + filter, err := build("nope") + assert.Error(err) + + // delete owner by configuration + filter, _ = build(true) + n := filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{ + System: data.System{ + SiteCode: "ffhb", + DomainCode: "city", + }, + }}) + + assert.NotNil(n) + assert.Equal("ffhb.city", n.Nodeinfo.System.SiteCode) + assert.Equal("", n.Nodeinfo.System.DomainCode) + + // keep owner configuration + filter, _ = build(false) + n = filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{ + System: data.System{ + SiteCode: "ffhb", + DomainCode: "city", + }, + }}) + + assert.NotNil(n) + assert.Equal("ffhb", n.Nodeinfo.System.SiteCode) + assert.Equal("city", n.Nodeinfo.System.DomainCode) +} diff --git a/output/filter/domainassite/domainassite.go b/output/filter/domainassite/domainassite.go new file mode 100644 index 0000000..7146d19 --- /dev/null +++ b/output/filter/domainassite/domainassite.go @@ -0,0 +1,50 @@ +package domainassite + +import ( + "errors" + + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/output/filter" + "github.com/FreifunkBremen/yanic/runtime" +) + +type domainAsSite struct{ set bool } + +func init() { + filter.Register("domain_as_site", build) +} + +func build(config interface{}) (filter.Filter, error) { + if value, ok := config.(bool); ok { + return &domainAsSite{set: value}, nil + } + return nil, errors.New("invalid configuration, boolean expected") +} + +func (config *domainAsSite) Apply(node *runtime.Node) *runtime.Node { + if nodeinfo := node.Nodeinfo; nodeinfo != nil && config.set && nodeinfo.System.DomainCode != "" { + node = &runtime.Node{ + Address: node.Address, + Firstseen: node.Firstseen, + Lastseen: node.Lastseen, + Online: node.Online, + Statistics: node.Statistics, + Nodeinfo: &data.NodeInfo{ + NodeID: nodeinfo.NodeID, + Network: nodeinfo.Network, + System: data.System{ + SiteCode: nodeinfo.System.DomainCode, + }, + Owner: nodeinfo.Owner, + Hostname: nodeinfo.Hostname, + Location: nodeinfo.Location, + Software: nodeinfo.Software, + Hardware: nodeinfo.Hardware, + VPN: nodeinfo.VPN, + Wireless: nodeinfo.Wireless, + }, + Neighbours: node.Neighbours, + } + } + return node +} diff --git a/output/filter/domainassite/domainassite_test.go b/output/filter/domainassite/domainassite_test.go new file mode 100644 index 0000000..504500c --- /dev/null +++ b/output/filter/domainassite/domainassite_test.go @@ -0,0 +1,43 @@ +package domainassite + +import ( + "testing" + + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/runtime" + "github.com/stretchr/testify/assert" +) + +func TestFilter(t *testing.T) { + assert := assert.New(t) + + // invalid config + filter, err := build("nope") + assert.Error(err) + + // delete owner by configuration + filter, _ = build(true) + n := filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{ + System: data.System{ + SiteCode: "ffhb", + DomainCode: "city", + }, + }}) + + assert.NotNil(n) + assert.Equal("city", n.Nodeinfo.System.SiteCode) + assert.Equal("", n.Nodeinfo.System.DomainCode) + + // keep owner configuration + filter, _ = build(false) + n = filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{ + System: data.System{ + SiteCode: "ffhb", + DomainCode: "city", + }, + }}) + + assert.NotNil(n) + assert.Equal("ffhb", n.Nodeinfo.System.SiteCode) + assert.Equal("city", n.Nodeinfo.System.DomainCode) +} diff --git a/output/meshviewer-ffrgb/struct.go b/output/meshviewer-ffrgb/struct.go index 3c3fb82..e39e775 100644 --- a/output/meshviewer-ffrgb/struct.go +++ b/output/meshviewer-ffrgb/struct.go @@ -33,6 +33,7 @@ type Node struct { MAC string `json:"mac"` Addresses []string `json:"addresses"` SiteCode string `json:"site_code,omitempty"` + DomainCode string `json:"-"` Hostname string `json:"hostname"` Owner string `json:"owner,omitempty"` Location *Location `json:"location,omitempty"` @@ -85,6 +86,7 @@ func NewNode(nodes *runtime.Nodes, n *runtime.Node) *Node { node.MAC = nodeinfo.Network.Mac node.Addresses = nodeinfo.Network.Addresses node.SiteCode = nodeinfo.System.SiteCode + node.DomainCode = nodeinfo.System.DomainCode node.Hostname = nodeinfo.Hostname if owner := nodeinfo.Owner; owner != nil { node.Owner = owner.Contact diff --git a/respond/collector.go b/respond/collector.go index 5fed200..b258158 100644 --- a/respond/collector.go +++ b/respond/collector.go @@ -21,25 +21,25 @@ type Collector struct { ifaceToConn map[string]*net.UDPConn // map from interface name to UDP socket port int - queue chan *Response // received responses - db database.Connection - nodes *runtime.Nodes - sites []string - interval time.Duration // Interval for multicast packets - stop chan interface{} + queue chan *Response // received responses + db database.Connection + nodes *runtime.Nodes + sitesDomains map[string][]string + interval time.Duration // Interval for multicast packets + stop chan interface{} } // NewCollector creates a Collector struct -func NewCollector(db database.Connection, nodes *runtime.Nodes, sites []string, ifaces []string, port int) *Collector { +func NewCollector(db database.Connection, nodes *runtime.Nodes, sitesDomains map[string][]string, ifaces []string, port int) *Collector { coll := &Collector{ - db: db, - nodes: nodes, - sites: sites, - port: port, - queue: make(chan *Response, 400), - stop: make(chan interface{}), - ifaceToConn: make(map[string]*net.UDPConn), + db: db, + nodes: nodes, + sitesDomains: sitesDomains, + port: port, + queue: make(chan *Response, 400), + stop: make(chan interface{}), + ifaceToConn: make(map[string]*net.UDPConn), } for _, iface := range ifaces { @@ -302,9 +302,11 @@ func (coll *Collector) globalStatsWorker() { // saves global statistics func (coll *Collector) saveGlobalStats() { - stats := runtime.NewGlobalStats(coll.nodes, coll.sites) + stats := runtime.NewGlobalStats(coll.nodes, coll.sitesDomains) - for site, stat := range stats { - coll.db.InsertGlobals(stat, time.Now(), site) + for site, domains := range stats { + for domain, stat := range domains { + coll.db.InsertGlobals(stat, time.Now(), site, domain) + } } } diff --git a/respond/collector_test.go b/respond/collector_test.go index d6e5bd9..82dca23 100644 --- a/respond/collector_test.go +++ b/respond/collector_test.go @@ -9,12 +9,15 @@ import ( "github.com/stretchr/testify/assert" ) -const SITE_TEST = "ffxx" +const ( + SITE_TEST = "ffhb" + DOMAIN_TEST = "city" +) func TestCollector(t *testing.T) { nodes := runtime.NewNodes(&runtime.NodesConfig{}) - collector := NewCollector(nil, nodes, []string{SITE_TEST}, []string{}, 10001) + collector := NewCollector(nil, nodes, map[string][]string{SITE_TEST: {DOMAIN_TEST}}, []string{}, 10001) collector.Start(time.Millisecond) time.Sleep(time.Millisecond * 10) collector.Close() diff --git a/respond/config.go b/respond/config.go index b221fa1..1a52313 100644 --- a/respond/config.go +++ b/respond/config.go @@ -3,10 +3,22 @@ package respond import "github.com/FreifunkBremen/yanic/lib/duration" type Config struct { - Enable bool `toml:"enable"` - Synchronize duration.Duration `toml:"synchronize"` - Interfaces []string `toml:"interfaces"` - Sites []string `toml:"sites"` - Port int `toml:"port"` - CollectInterval duration.Duration `toml:"collect_interval"` + Enable bool `toml:"enable"` + Synchronize duration.Duration `toml:"synchronize"` + Interfaces []string `toml:"interfaces"` + Sites map[string]SiteConfig `toml:"sites"` + Port int `toml:"port"` + CollectInterval duration.Duration `toml:"collect_interval"` +} + +func (c *Config) SitesDomains() (result map[string][]string) { + result = make(map[string][]string) + for site, siteConfig := range c.Sites { + result[site] = siteConfig.Domains + } + return +} + +type SiteConfig struct { + Domains []string `toml:"domains"` } diff --git a/respond/config_test.go b/respond/config_test.go new file mode 100644 index 0000000..69e4fc9 --- /dev/null +++ b/respond/config_test.go @@ -0,0 +1,24 @@ +package respond + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSitesDomainsConfigTransform(t *testing.T) { + assert := assert.New(t) + c := Config{ + Sites: map[string]SiteConfig{ + "ffhb": {Domains: []string{"city"}}, + }, + } + result := c.SitesDomains() + assert.Len(result, 1) + assert.Contains(result, "ffhb") + + domains := result["ffhb"] + + assert.Len(domains, 1) + assert.Equal("city", domains[0]) +} diff --git a/runtime/stats.go b/runtime/stats.go index 506a0f9..145d8bb 100644 --- a/runtime/stats.go +++ b/runtime/stats.go @@ -3,6 +3,7 @@ package runtime const ( DISABLED_AUTOUPDATER = "disabled" GLOBAL_SITE = "global" + GLOBAL_DOMAIN = "global" ) // CounterMap to manage multiple values @@ -23,32 +24,45 @@ type GlobalStats struct { } //NewGlobalStats returns global statistics for InfluxDB -func NewGlobalStats(nodes *Nodes, sites []string) (result map[string]*GlobalStats) { - result = make(map[string]*GlobalStats) +func NewGlobalStats(nodes *Nodes, sitesDomains map[string][]string) (result map[string]map[string]*GlobalStats) { + result = make(map[string]map[string]*GlobalStats) - result[GLOBAL_SITE] = &GlobalStats{ + result[GLOBAL_SITE] = make(map[string]*GlobalStats) + result[GLOBAL_SITE][GLOBAL_DOMAIN] = &GlobalStats{ Firmwares: make(CounterMap), Models: make(CounterMap), Autoupdater: make(CounterMap), } - for _, site := range sites { - result[site] = &GlobalStats{ + for site, domains := range sitesDomains { + result[site] = make(map[string]*GlobalStats) + result[site][GLOBAL_DOMAIN] = &GlobalStats{ Firmwares: make(CounterMap), Models: make(CounterMap), Autoupdater: make(CounterMap), } + for _, domain := range domains { + result[site][domain] = &GlobalStats{ + Firmwares: make(CounterMap), + Models: make(CounterMap), + Autoupdater: make(CounterMap), + } + } } nodes.RLock() for _, node := range nodes.List { if node.Online { - result[GLOBAL_SITE].Add(node) + result[GLOBAL_SITE][GLOBAL_DOMAIN].Add(node) if info := node.Nodeinfo; info != nil { site := info.System.SiteCode - if _, exist := result[site]; exist { - result[site].Add(node) + domain := info.System.DomainCode + if _, ok := result[site]; ok { + result[site][GLOBAL_DOMAIN].Add(node) + if _, ok := result[site][domain]; ok { + result[site][domain].Add(node) + } } } } diff --git a/runtime/stats_test.go b/runtime/stats_test.go index e25ac62..93363e8 100644 --- a/runtime/stats_test.go +++ b/runtime/stats_test.go @@ -8,49 +8,70 @@ import ( "github.com/FreifunkBremen/yanic/data" ) -const TEST_SITE = "ffxx" +const ( + TEST_SITE = "ffhb" + TEST_DOMAIN = "city" +) func TestGlobalStats(t *testing.T) { - stats := NewGlobalStats(createTestNodes(), []string{TEST_SITE}) + stats := NewGlobalStats(createTestNodes(), map[string][]string{TEST_SITE: {TEST_DOMAIN}}) assert := assert.New(t) assert.Len(stats, 2) //check GLOBAL_SITE stats - assert.EqualValues(1, stats[GLOBAL_SITE].Gateways) - assert.EqualValues(3, stats[GLOBAL_SITE].Nodes) - assert.EqualValues(25, stats[GLOBAL_SITE].Clients) + assert.EqualValues(1, stats[GLOBAL_SITE][GLOBAL_DOMAIN].Gateways) + assert.EqualValues(3, stats[GLOBAL_SITE][GLOBAL_DOMAIN].Nodes) + assert.EqualValues(25, stats[GLOBAL_SITE][GLOBAL_DOMAIN].Clients) // check models - assert.Len(stats[GLOBAL_SITE].Models, 2) - assert.EqualValues(2, stats[GLOBAL_SITE].Models["TP-Link 841"]) - assert.EqualValues(1, stats[GLOBAL_SITE].Models["Xeon Multi-Core"]) + assert.Len(stats[GLOBAL_SITE][GLOBAL_DOMAIN].Models, 2) + assert.EqualValues(2, stats[GLOBAL_SITE][GLOBAL_DOMAIN].Models["TP-Link 841"]) + assert.EqualValues(1, stats[GLOBAL_SITE][GLOBAL_DOMAIN].Models["Xeon Multi-Core"]) // check firmwares - assert.Len(stats[GLOBAL_SITE].Firmwares, 1) - assert.EqualValues(1, stats[GLOBAL_SITE].Firmwares["2016.1.6+entenhausen1"]) + assert.Len(stats[GLOBAL_SITE][GLOBAL_DOMAIN].Firmwares, 1) + assert.EqualValues(1, stats[GLOBAL_SITE][GLOBAL_DOMAIN].Firmwares["2016.1.6+entenhausen1"]) // check autoupdater - assert.Len(stats[GLOBAL_SITE].Autoupdater, 2) - assert.EqualValues(1, stats[GLOBAL_SITE].Autoupdater["stable"]) + assert.Len(stats[GLOBAL_SITE][GLOBAL_DOMAIN].Autoupdater, 2) + assert.EqualValues(1, stats[GLOBAL_SITE][GLOBAL_DOMAIN].Autoupdater["stable"]) // check TEST_SITE stats - assert.EqualValues(1, stats[TEST_SITE].Gateways) - assert.EqualValues(2, stats[TEST_SITE].Nodes) - assert.EqualValues(23, stats[TEST_SITE].Clients) + assert.EqualValues(1, stats[TEST_SITE][GLOBAL_DOMAIN].Gateways) + assert.EqualValues(2, stats[TEST_SITE][GLOBAL_DOMAIN].Nodes) + assert.EqualValues(23, stats[TEST_SITE][GLOBAL_DOMAIN].Clients) // check models - assert.Len(stats[TEST_SITE].Models, 2) - assert.EqualValues(1, stats[TEST_SITE].Models["TP-Link 841"]) - assert.EqualValues(1, stats[TEST_SITE].Models["Xeon Multi-Core"]) + assert.Len(stats[TEST_SITE][GLOBAL_DOMAIN].Models, 2) + assert.EqualValues(1, stats[TEST_SITE][GLOBAL_DOMAIN].Models["TP-Link 841"]) + assert.EqualValues(1, stats[TEST_SITE][GLOBAL_DOMAIN].Models["Xeon Multi-Core"]) // check firmwares - assert.Len(stats[TEST_SITE].Firmwares, 1) - assert.EqualValues(1, stats[TEST_SITE].Firmwares["2016.1.6+entenhausen1"]) + assert.Len(stats[TEST_SITE][GLOBAL_DOMAIN].Firmwares, 1) + assert.EqualValues(1, stats[TEST_SITE][GLOBAL_DOMAIN].Firmwares["2016.1.6+entenhausen1"]) // check autoupdater - assert.Len(stats[TEST_SITE].Autoupdater, 1) - assert.EqualValues(0, stats[TEST_SITE].Autoupdater["stable"]) + assert.Len(stats[TEST_SITE][GLOBAL_DOMAIN].Autoupdater, 1) + assert.EqualValues(0, stats[TEST_SITE][GLOBAL_DOMAIN].Autoupdater["stable"]) + + // check TEST_DOMAIN stats + assert.EqualValues(1, stats[TEST_SITE][TEST_DOMAIN].Gateways) + assert.EqualValues(1, stats[TEST_SITE][TEST_DOMAIN].Nodes) + assert.EqualValues(0, stats[TEST_SITE][TEST_DOMAIN].Clients) + + // check models + assert.Len(stats[TEST_SITE][TEST_DOMAIN].Models, 1) + assert.EqualValues(0, stats[TEST_SITE][TEST_DOMAIN].Models["TP-Link 841"]) + assert.EqualValues(1, stats[TEST_SITE][TEST_DOMAIN].Models["Xeon Multi-Core"]) + + // check firmwares + assert.Len(stats[TEST_SITE][TEST_DOMAIN].Firmwares, 0) + assert.EqualValues(0, stats[TEST_SITE][TEST_DOMAIN].Firmwares["2016.1.6+entenhausen1"]) + + // check autoupdater + assert.Len(stats[TEST_SITE][TEST_DOMAIN].Autoupdater, 1) + assert.EqualValues(0, stats[TEST_SITE][TEST_DOMAIN].Autoupdater["stable"]) } func createTestNodes() *Nodes { @@ -109,7 +130,8 @@ func createTestNodes() *Nodes { Model: "Xeon Multi-Core", }, System: data.System{ - SiteCode: TEST_SITE, + SiteCode: TEST_SITE, + DomainCode: TEST_DOMAIN, }, }, })