From 6b522c629ce03c47ed9fe716af3d3d23666ca33f Mon Sep 17 00:00:00 2001 From: Julian Kornberger Date: Sat, 13 Jan 2018 19:08:46 +0100 Subject: [PATCH] [TASK] refactoring of filters (#114) --- cmd/config_test.go | 3 + config_example.toml | 22 +++-- docs/docs_configuration.md | 16 ++++ output/all/filter.go | 45 +-------- output/all/filter_blacklist.go | 24 ----- output/all/filter_blacklist_test.go | 34 ------- output/all/filter_haslocation.go | 26 ----- output/all/filter_haslocation_test.go | 50 ---------- output/all/filter_inarea.go | 42 -------- output/all/filter_inarea_test.go | 65 ------------- output/all/filter_noowner.go | 37 ------- output/all/filter_noowner_test.go | 49 ---------- output/all/filter_test.go | 41 -------- output/all/output.go | 30 +++--- output/all/output_test.go | 30 +++++- output/filter/blacklist/blacklist.go | 36 +++++++ output/filter/blacklist/blacklist_test.go | 38 ++++++++ output/filter/filter.go | 73 ++++++++++++++ output/filter/filter_test.go | 96 +++++++++++++++++++ output/filter/haslocation/haslocation.go | 40 ++++++++ output/filter/haslocation/haslocation_test.go | 53 ++++++++++ output/filter/inarea/inarea.go | 65 +++++++++++++ output/filter/inarea/inarea_test.go | 86 +++++++++++++++++ output/filter/noowner/noowner.go | 48 ++++++++++ output/filter/noowner/noowner_test.go | 39 ++++++++ output/filter/site/site.go | 36 +++++++ output/filter/site/site_test.go | 32 +++++++ 27 files changed, 724 insertions(+), 432 deletions(-) delete mode 100644 output/all/filter_blacklist.go delete mode 100644 output/all/filter_blacklist_test.go delete mode 100644 output/all/filter_haslocation.go delete mode 100644 output/all/filter_haslocation_test.go delete mode 100644 output/all/filter_inarea.go delete mode 100644 output/all/filter_inarea_test.go delete mode 100644 output/all/filter_noowner.go delete mode 100644 output/all/filter_noowner_test.go delete mode 100644 output/all/filter_test.go create mode 100644 output/filter/blacklist/blacklist.go create mode 100644 output/filter/blacklist/blacklist_test.go create mode 100644 output/filter/filter.go create mode 100644 output/filter/filter_test.go create mode 100644 output/filter/haslocation/haslocation.go create mode 100644 output/filter/haslocation/haslocation_test.go create mode 100644 output/filter/inarea/inarea.go create mode 100644 output/filter/inarea/inarea_test.go create mode 100644 output/filter/noowner/noowner.go create mode 100644 output/filter/noowner/noowner_test.go create mode 100644 output/filter/site/site.go create mode 100644 output/filter/site/site_test.go diff --git a/cmd/config_test.go b/cmd/config_test.go index 960e9ca..6a85403 100644 --- a/cmd/config_test.go +++ b/cmd/config_test.go @@ -31,6 +31,9 @@ func TestReadConfig(t *testing.T) { "enable": false, "nodes_path": "/var/www/html/meshviewer/data/nodes.json", "graph_path": "/var/www/html/meshviewer/data/graph.json", + "filter": map[string]interface{}{ + "no_owner": true, + }, }, meshviewer) _, err = ReadConfigFile("testdata/config_invalid.toml") diff --git a/config_example.toml b/config_example.toml index 27de896..bcceb1c 100644 --- a/config_example.toml +++ b/config_example.toml @@ -46,12 +46,16 @@ offline_after = "10m" # For each output format there can be set different filters #[nodes.output.example.filter] # -# Set to false, if you want the json files to contain the owner information +# WARNING: if it is not set, it will publish contact information of other persons +# Set to true, if you did not want the json files to contain the owner information #no_owner = true # # List of nodeids of nodes that should be filtered out, so they won't appear in output #blacklist = ["00112233445566", "1337f0badead"] # +# List of site_codes of nodes that should be included in the output +#sites = ["ffhb"] +# # set has_location to true if you want to include only nodes that have geo-coordinates set # (setting this to false has no sensible effect, unless you'd want to hide nodes that have coordinates) #has_location = true @@ -69,9 +73,11 @@ offline_after = "10m" enable = true path = "/var/www/html/meshviewer/data/meshviewer.json" -#[nodes.output.meshviewer-ffrgb.filter] -#no_owner = false +[nodes.output.meshviewer-ffrgb.filter] +# WARNING: if it is not set, it will publish contact information of other persons +no_owner = false #blacklist = ["00112233445566", "1337f0badead"] +#sites = ["ffhb"] #has_location = true #[nodes.output.meshviewer-ffrgb.filter.in_area] @@ -96,8 +102,9 @@ nodes_path = "/var/www/html/meshviewer/data/nodes.json" # path where to store graph.json graph_path = "/var/www/html/meshviewer/data/graph.json" -#[nodes.output.meshviewer.filter] -#no_owner = false +[nodes.output.meshviewer.filter] +# WARNING: if it is not set, it will publish contact information of other persons +no_owner = true # definition for nodelist.json @@ -105,8 +112,9 @@ graph_path = "/var/www/html/meshviewer/data/graph.json" enable = true path = "/var/www/html/meshviewer/data/nodelist.json" -#[nodes.output.nodelist.filter] -#no_owner = false +[nodes.output.nodelist.filter] +# WARNING: if it is not set, it will publish contact information of other persons +no_owner = true diff --git a/docs/docs_configuration.md b/docs/docs_configuration.md index 3af7b64..27b77ad 100644 --- a/docs/docs_configuration.md +++ b/docs/docs_configuration.md @@ -196,6 +196,7 @@ enable = true [nodes.output.example.filter] no_owner = true blacklist = ["00112233445566", "1337f0badead"] +sites = ["ffhb"] has_location = true [nodes.output.example.filter.in_area] latitude_min = 34.30 @@ -222,6 +223,7 @@ For each output format there can be set different filters [nodes.output.example.filter] no_owner = true blacklist = ["00112233445566", "1337f0badead"] +sites = ["ffhb"] has_location = true [nodes.output.example.filter.in_area] latitude_min = 34.30 @@ -235,6 +237,10 @@ longitude_max = 39.72 ### no_owner {% method %} Set to false, if you want the json files to contain the owner information + + +**WARNING: if it is not set, it will publish contact information of other persons.** + {% sample lang="toml" %} ```toml no_owner = true @@ -252,6 +258,16 @@ blacklist = ["00112233445566", "1337f0badead"] {% endmethod %} +### sites +{% method %} +List of site_codes of nodes that should be included in output +{% sample lang="toml" %} +```toml +sites = ["ffhb"] +``` +{% endmethod %} + + ### has_location {% method %} set has_location to true if you want to include only nodes that have geo-coordinates set diff --git a/output/all/filter.go b/output/all/filter.go index ce9d0c2..1c95ef5 100644 --- a/output/all/filter.go +++ b/output/all/filter.go @@ -1,44 +1,9 @@ package all import ( - "github.com/FreifunkBremen/yanic/runtime" + _ "github.com/FreifunkBremen/yanic/output/filter/blacklist" + _ "github.com/FreifunkBremen/yanic/output/filter/haslocation" + _ "github.com/FreifunkBremen/yanic/output/filter/inarea" + _ "github.com/FreifunkBremen/yanic/output/filter/noowner" + _ "github.com/FreifunkBremen/yanic/output/filter/site" ) - -// Config Filter -type filterConfig map[string]interface{} - -type filterFunc func(*runtime.Node) *runtime.Node - -func noFilter(node *runtime.Node) *runtime.Node { - return node -} - -// Create Filter -func (f filterConfig) filtering(nodesOrigin *runtime.Nodes) *runtime.Nodes { - nodes := runtime.NewNodes(&runtime.NodesConfig{}) - filterfuncs := []filterFunc{ - f.HasLocation(), - f.Blacklist(), - f.InArea(), - f.NoOwner(), - } - - nodesOrigin.Lock() - defer nodesOrigin.Unlock() - - for _, nodeOrigin := range nodesOrigin.List { - //maybe cloning of this object is better? - node := nodeOrigin - for _, f := range filterfuncs { - node = f(node) - if node == nil { - break - } - } - - if node != nil { - nodes.AddNode(node) - } - } - return nodes -} diff --git a/output/all/filter_blacklist.go b/output/all/filter_blacklist.go deleted file mode 100644 index cd1c0b7..0000000 --- a/output/all/filter_blacklist.go +++ /dev/null @@ -1,24 +0,0 @@ -package all - -import "github.com/FreifunkBremen/yanic/runtime" - -func (f filterConfig) Blacklist() filterFunc { - v, ok := f["blacklist"] - if !ok { - return noFilter - } - - list := make(map[string]interface{}) - for _, nodeid := range v.([]interface{}) { - list[nodeid.(string)] = true - } - - return func(node *runtime.Node) *runtime.Node { - if nodeinfo := node.Nodeinfo; nodeinfo != nil { - if _, ok := list[nodeinfo.NodeID]; ok { - return nil - } - } - return node - } -} diff --git a/output/all/filter_blacklist_test.go b/output/all/filter_blacklist_test.go deleted file mode 100644 index c7c73d9..0000000 --- a/output/all/filter_blacklist_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package all - -import ( - "testing" - - "github.com/FreifunkBremen/yanic/data" - "github.com/FreifunkBremen/yanic/runtime" - "github.com/stretchr/testify/assert" -) - -func TestFilterBlacklist(t *testing.T) { - assert := assert.New(t) - var config filterConfig - - config = map[string]interface{}{} - - filterBlacklist := config.Blacklist() - - n := filterBlacklist(&runtime.Node{Nodeinfo: &data.NodeInfo{}}) - assert.NotNil(n) - - config["blacklist"] = []interface{}{"a", "c"} - filterBlacklist = config.Blacklist() - - n = filterBlacklist(&runtime.Node{Nodeinfo: &data.NodeInfo{NodeID: "a"}}) - assert.Nil(n) - - n = filterBlacklist(&runtime.Node{Nodeinfo: &data.NodeInfo{}}) - assert.NotNil(n) - - n = filterBlacklist(&runtime.Node{}) - assert.NotNil(n) - -} diff --git a/output/all/filter_haslocation.go b/output/all/filter_haslocation.go deleted file mode 100644 index 2f3580c..0000000 --- a/output/all/filter_haslocation.go +++ /dev/null @@ -1,26 +0,0 @@ -package all - -import "github.com/FreifunkBremen/yanic/runtime" - -func (f filterConfig) HasLocation() filterFunc { - withLocation, ok := f["has_location"].(bool) - if !ok { - return noFilter - } - return func(node *runtime.Node) *runtime.Node { - if nodeinfo := node.Nodeinfo; nodeinfo != nil { - if withLocation { - if location := nodeinfo.Location; location != nil { - return node - } - } else { - if location := nodeinfo.Location; location == nil { - return node - } - } - } else if !withLocation { - return node - } - return nil - } -} diff --git a/output/all/filter_haslocation_test.go b/output/all/filter_haslocation_test.go deleted file mode 100644 index 61e6b44..0000000 --- a/output/all/filter_haslocation_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package all - -import ( - "testing" - - "github.com/FreifunkBremen/yanic/data" - "github.com/FreifunkBremen/yanic/runtime" - "github.com/stretchr/testify/assert" -) - -func TestFilterHasLocation(t *testing.T) { - assert := assert.New(t) - var config filterConfig - - config = map[string]interface{}{} - - filterHasLocation := config.HasLocation() - n := filterHasLocation(&runtime.Node{Nodeinfo: &data.NodeInfo{ - Location: &data.Location{}, - }}) - assert.NotNil(n) - - config["has_location"] = true - filterHasLocation = config.HasLocation() - - n = filterHasLocation(&runtime.Node{Nodeinfo: &data.NodeInfo{ - Location: &data.Location{}, - }}) - assert.NotNil(n) - - n = filterHasLocation(&runtime.Node{Nodeinfo: &data.NodeInfo{}}) - assert.Nil(n) - - n = filterHasLocation(&runtime.Node{}) - assert.Nil(n) - - config["has_location"] = false - filterHasLocation = config.HasLocation() - - n = filterHasLocation(&runtime.Node{Nodeinfo: &data.NodeInfo{ - Location: &data.Location{}, - }}) - assert.Nil(n) - - n = filterHasLocation(&runtime.Node{Nodeinfo: &data.NodeInfo{}}) - assert.NotNil(n) - - n = filterHasLocation(&runtime.Node{}) - assert.NotNil(n) -} diff --git a/output/all/filter_inarea.go b/output/all/filter_inarea.go deleted file mode 100644 index 6de44f0..0000000 --- a/output/all/filter_inarea.go +++ /dev/null @@ -1,42 +0,0 @@ -package all - -import "github.com/FreifunkBremen/yanic/runtime" - -type area struct { - latitudeMin float64 - latitudeMax float64 - longitudeMin float64 - longitudeMax float64 -} - -func (f filterConfig) InArea() filterFunc { - if areaConfigInt, ok := f["in_area"]; ok { - areaConfig := areaConfigInt.(map[string]interface{}) - a := area{} - if v, ok := areaConfig["latitude_min"]; ok { - a.latitudeMin = v.(float64) - } - if v, ok := areaConfig["latitude_max"]; ok { - a.latitudeMax = v.(float64) - } - if v, ok := areaConfig["longitude_min"]; ok { - a.longitudeMin = v.(float64) - } - if v, ok := areaConfig["longitude_max"]; ok { - a.longitudeMax = v.(float64) - } - return func(node *runtime.Node) *runtime.Node { - if nodeinfo := node.Nodeinfo; nodeinfo != nil { - location := nodeinfo.Location - if location == nil { - return node - } - if location.Latitude >= a.latitudeMin && location.Latitude <= a.latitudeMax && location.Longitude >= a.longitudeMin && location.Longitude <= a.longitudeMax { - return node - } - } - return nil - } - } - return noFilter -} diff --git a/output/all/filter_inarea_test.go b/output/all/filter_inarea_test.go deleted file mode 100644 index b19bf33..0000000 --- a/output/all/filter_inarea_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package all - -import ( - "testing" - - "github.com/FreifunkBremen/yanic/data" - "github.com/FreifunkBremen/yanic/runtime" - "github.com/stretchr/testify/assert" -) - -func TestFilterInArea(t *testing.T) { - assert := assert.New(t) - var config filterConfig - areaConfig := map[string]interface{}{ - "latitude_min": 3.0, - "latitude_max": 5.0, - "longitude_min": 10.0, - "longitude_max": 12.0, - } - config = map[string]interface{}{} - - filterLocationInArea := config.InArea() - n := filterLocationInArea(&runtime.Node{Nodeinfo: &data.NodeInfo{ - Location: &data.Location{Latitude: 4.0, Longitude: 11.0}, - }}) - assert.NotNil(n) - - config["in_area"] = areaConfig - filterLocationInArea = config.InArea() - - // drop area without nodeinfo - n = filterLocationInArea(&runtime.Node{}) - assert.Nil(n) - - // keep without location - n = filterLocationInArea(&runtime.Node{Nodeinfo: &data.NodeInfo{}}) - assert.NotNil(n) - - // zeros not in area (0, 0) - n = filterLocationInArea(&runtime.Node{Nodeinfo: &data.NodeInfo{ - Location: &data.Location{}, - }}) - assert.Nil(n) - - // in area - n = filterLocationInArea(&runtime.Node{Nodeinfo: &data.NodeInfo{ - Location: &data.Location{Latitude: 4.0, Longitude: 11.0}, - }}) - assert.NotNil(n) - - n = filterLocationInArea(&runtime.Node{Nodeinfo: &data.NodeInfo{ - Location: &data.Location{Latitude: 4.0, Longitude: 13.0}, - }}) - assert.Nil(n) - - n = filterLocationInArea(&runtime.Node{Nodeinfo: &data.NodeInfo{ - Location: &data.Location{Latitude: 6.0, Longitude: 11.0}, - }}) - assert.Nil(n) - - n = filterLocationInArea(&runtime.Node{Nodeinfo: &data.NodeInfo{ - Location: &data.Location{Latitude: 1.0, Longitude: 2.0}, - }}) - assert.Nil(n) -} diff --git a/output/all/filter_noowner.go b/output/all/filter_noowner.go deleted file mode 100644 index 76bc47a..0000000 --- a/output/all/filter_noowner.go +++ /dev/null @@ -1,37 +0,0 @@ -package all - -import ( - "github.com/FreifunkBremen/yanic/data" - "github.com/FreifunkBremen/yanic/runtime" -) - -func (f filterConfig) NoOwner() filterFunc { - if v, ok := f["no_owner"]; ok && v.(bool) == false { - return noFilter - } - return func(node *runtime.Node) *runtime.Node { - if nodeinfo := node.Nodeinfo; nodeinfo != nil { - return &runtime.Node{ - Address: node.Address, - Firstseen: node.Firstseen, - Lastseen: node.Lastseen, - Online: node.Online, - Statistics: node.Statistics, - Nodeinfo: &data.NodeInfo{ - NodeID: nodeinfo.NodeID, - Network: nodeinfo.Network, - System: nodeinfo.System, - Owner: nil, - Hostname: nodeinfo.Hostname, - Location: nodeinfo.Location, - Software: nodeinfo.Software, - Hardware: nodeinfo.Hardware, - VPN: nodeinfo.VPN, - Wireless: nodeinfo.Wireless, - }, - Neighbours: node.Neighbours, - } - } - return node - } -} diff --git a/output/all/filter_noowner_test.go b/output/all/filter_noowner_test.go deleted file mode 100644 index 1aa496f..0000000 --- a/output/all/filter_noowner_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package all - -import ( - "testing" - - "github.com/FreifunkBremen/yanic/data" - "github.com/FreifunkBremen/yanic/runtime" - "github.com/stretchr/testify/assert" -) - -func TestFilterNoOwner(t *testing.T) { - assert := assert.New(t) - var config filterConfig - - config = map[string]interface{}{} - - filterNoOwner := config.NoOwner() - n := filterNoOwner(&runtime.Node{Nodeinfo: &data.NodeInfo{ - Owner: &data.Owner{ - Contact: "blub", - }, - }}) - assert.NotNil(n) - assert.Nil(n.Nodeinfo.Owner) - - n = filterNoOwner(&runtime.Node{}) - assert.NotNil(n) - - config["no_owner"] = true - filterNoOwner = config.NoOwner() - n = filterNoOwner(&runtime.Node{Nodeinfo: &data.NodeInfo{ - Owner: &data.Owner{ - Contact: "blub", - }, - }}) - assert.NotNil(n) - assert.Nil(n.Nodeinfo.Owner) - - config["no_owner"] = false - filterNoOwner = config.NoOwner() - - n = filterNoOwner(&runtime.Node{Nodeinfo: &data.NodeInfo{ - Owner: &data.Owner{ - Contact: "blub", - }, - }}) - assert.NotNil(n) - assert.NotNil(n.Nodeinfo.Owner) -} diff --git a/output/all/filter_test.go b/output/all/filter_test.go deleted file mode 100644 index d94c2b2..0000000 --- a/output/all/filter_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package all - -import ( - "testing" - - "github.com/FreifunkBremen/yanic/data" - "github.com/FreifunkBremen/yanic/runtime" - "github.com/stretchr/testify/assert" -) - -func TestFilter(t *testing.T) { - assert := assert.New(t) - - // filtered - do not run all - nodes := &runtime.Nodes{ - List: map[string]*runtime.Node{ - "a": { - Nodeinfo: &data.NodeInfo{NodeID: "a"}, - }, - }, - } - config := filterConfig{ - "has_location": true, - } - nodes = config.filtering(nodes) - assert.Len(nodes.List, 0) - - // run to end - nodes = &runtime.Nodes{ - List: map[string]*runtime.Node{ - "a": { - Nodeinfo: &data.NodeInfo{NodeID: "a"}, - }, - }, - } - config = filterConfig{ - "has_location": false, - } - nodes = config.filtering(nodes) - assert.Len(nodes.List, 1) -} diff --git a/output/all/output.go b/output/all/output.go index ae64c08..2977d4b 100644 --- a/output/all/output.go +++ b/output/all/output.go @@ -5,18 +5,19 @@ import ( "log" "github.com/FreifunkBremen/yanic/output" + "github.com/FreifunkBremen/yanic/output/filter" "github.com/FreifunkBremen/yanic/runtime" ) type Output struct { output.Output - list map[int]output.Output - filter map[int]filterConfig + list map[int]output.Output + outputFilter map[int]filter.Set } func Register(configuration map[string]interface{}) (output.Output, error) { list := make(map[int]output.Output) - filter := make(map[int]filterConfig) + outputFilter := make(map[int]filter.Set) i := 1 allOutputs := configuration for outputType, outputRegister := range output.Adapters { @@ -44,25 +45,26 @@ func Register(configuration map[string]interface{}) (output.Output, error) { if output == nil { continue } - list[i] = output + var errs []error + var filterSet filter.Set if c := config["filter"]; c != nil { - filter[i] = config["filter"].(map[string]interface{}) + if filterConf, ok := c.(map[string]interface{}); ok { + filterSet, errs = filter.New(filterConf) + } + if len(errs) > 0 { + return nil, fmt.Errorf("filter configuration errors: %v", errs) + } + outputFilter[i] = filterSet } + list[i] = output i++ } } - return &Output{list: list, filter: filter}, nil + return &Output{list: list, outputFilter: outputFilter}, nil } func (o *Output) Save(nodes *runtime.Nodes) { for i, item := range o.list { - var filteredNodes *runtime.Nodes - if config := o.filter[i]; config != nil { - filteredNodes = config.filtering(nodes) - } else { - filteredNodes = filterConfig{}.filtering(nodes) - } - - item.Save(filteredNodes) + item.Save(o.outputFilter[i].Apply(nodes)) } } diff --git a/output/all/output_test.go b/output/all/output_test.go index 75513bc..f30ceb0 100644 --- a/output/all/output_test.go +++ b/output/all/output_test.go @@ -82,16 +82,40 @@ func TestStart(t *testing.T) { allOutput.Save(nodes) assert.Equal(3, globalOutput.Get()) + // wrong format - map _, err = Register(map[string]interface{}{ - "e": []map[string]interface{}{ - {}, + "e": []interface{}{ + false, }, }) assert.Error(err) - // wrong format + // wrong format - array _, err = Register(map[string]interface{}{ "e": true, }) assert.Error(err) + + // output error + _, err = Register(map[string]interface{}{ + "e": []interface{}{ + map[string]interface{}{ + "enable": true, + }, + }, + }) + assert.Error(err) + + // output error invalid config of filter + _, err = Register(map[string]interface{}{ + "a": []interface{}{ + map[string]interface{}{ + "enable": true, + "filter": map[string]interface{}{ + "blacklist": true, + }, + }, + }, + }) + assert.Error(err) } diff --git a/output/filter/blacklist/blacklist.go b/output/filter/blacklist/blacklist.go new file mode 100644 index 0000000..77c7cdc --- /dev/null +++ b/output/filter/blacklist/blacklist.go @@ -0,0 +1,36 @@ +package blacklist + +import ( + "errors" + + "github.com/FreifunkBremen/yanic/output/filter" + "github.com/FreifunkBremen/yanic/runtime" +) + +type blacklist map[string]interface{} + +func init() { + filter.Register("blacklist", build) +} + +func build(config interface{}) (filter.Filter, error) { + values, ok := config.([]string) + if !ok { + return nil, errors.New("invalid configuration, array of strings expected") + } + + list := make(blacklist) + for _, nodeid := range values { + list[nodeid] = struct{}{} + } + return &list, nil +} + +func (list blacklist) Apply(node *runtime.Node) *runtime.Node { + if nodeinfo := node.Nodeinfo; nodeinfo != nil { + if _, ok := list[nodeinfo.NodeID]; ok { + return nil + } + } + return node +} diff --git a/output/filter/blacklist/blacklist_test.go b/output/filter/blacklist/blacklist_test.go new file mode 100644 index 0000000..a225389 --- /dev/null +++ b/output/filter/blacklist/blacklist_test.go @@ -0,0 +1,38 @@ +package blacklist + +import ( + "testing" + + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/runtime" + "github.com/stretchr/testify/assert" +) + +func TestFilterBlacklist(t *testing.T) { + assert := assert.New(t) + + // invalid config + filter, err := build(3) + assert.Error(err) + + // tests with empty list + filter, err = build([]string{}) + + // keep node without nodeid + n := filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{}}) + assert.NotNil(n) + + // tests with blacklist + filter, _ = build([]string{"a", "c"}) + + // blacklist contains node with nodeid -> drop it + n = filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{NodeID: "a"}}) + assert.Nil(n) + + // blacklist does not contains node without nodeid -> keep it + n = filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{}}) + assert.NotNil(n) + + n = filter.Apply(&runtime.Node{}) + assert.NotNil(n) +} diff --git a/output/filter/filter.go b/output/filter/filter.go new file mode 100644 index 0000000..137d2ba --- /dev/null +++ b/output/filter/filter.go @@ -0,0 +1,73 @@ +package filter + +import ( + "fmt" + + "github.com/FreifunkBremen/yanic/runtime" + "github.com/pkg/errors" +) + +// factory function for building a filter +// may return nil if the filter never applies +type factory func(interface{}) (Filter, error) + +// Filter is a filter instance +type Filter interface { + Apply(*runtime.Node) *runtime.Node +} + +// Set is a list of configured filters +type Set []Filter + +var filters = make(map[string]factory) + +// Register registers a new filter +func Register(name string, f factory) { + if _, ok := filters[name]; ok { + panic("already registered: " + name) + } + filters[name] = f +} + +// New returns and initializes a set of filters +func New(configs map[string]interface{}) (set Set, errs []error) { + for name, config := range configs { + if config == nil { + return + } + + f, _ := filters[name] + if f == nil { + errs = append(errs, fmt.Errorf("unknown filter: %s", name)) + } else if filter, err := f(config); err != nil { + errs = append(errs, errors.Wrapf(err, "unable to initialize filter %s", name)) + } else if filter != nil { + set = append(set, filter) + } + } + return +} + +// Apply applies the filter set to the given node list and returns a new node list +func (set Set) Apply(nodesOrigin *runtime.Nodes) *runtime.Nodes { + nodes := runtime.NewNodes(&runtime.NodesConfig{}) + + nodesOrigin.Lock() + defer nodesOrigin.Unlock() + + for _, nodeOrigin := range nodesOrigin.List { + //maybe cloning of this object is better? + node := nodeOrigin + for _, filter := range set { + node = filter.Apply(node) + if node == nil { + break + } + } + + if node != nil { + nodes.AddNode(node) + } + } + return nodes +} diff --git a/output/filter/filter_test.go b/output/filter/filter_test.go new file mode 100644 index 0000000..c9f5ca0 --- /dev/null +++ b/output/filter/filter_test.go @@ -0,0 +1,96 @@ +package filter + +import ( + "errors" + "testing" + + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/runtime" + "github.com/stretchr/testify/assert" +) + +type filterBool struct{ bool } + +func (f filterBool) Apply(node *runtime.Node) *runtime.Node { + if f.bool { + return node + } + return nil +} + +func build(v interface{}) (Filter, error) { + if config, ok := v.(bool); ok { + return &filterBool{config}, nil + } + return nil, nil +} + +func buildNil(v interface{}) (Filter, error) { + return nil, nil +} + +func buildError(v interface{}) (Filter, error) { + if v != nil { + return nil, errors.New("some errors") + } + return nil, nil +} + +func TestFilter(t *testing.T) { + assert := assert.New(t) + + Register("test_nil", buildNil) + Register("test_err", buildError) + Register("test", build) + + // filter still exists + filter, err := New(map[string]interface{}{ + "adsa": true, + }) + assert.Len(err, 1) + assert.Nil(filter) + + // no filter + filter, err = New(map[string]interface{}{ + "test_nil": 3, + }) + assert.Len(err, 0) + assert.Len(filter, 0) + + // filter error + filter, err = New(map[string]interface{}{ + "test_err": false, + }) + assert.Len(err, 1) + assert.Nil(filter) + + // filter a node + nodes := &runtime.Nodes{ + List: map[string]*runtime.Node{ + "a": { + Nodeinfo: &data.NodeInfo{NodeID: "a"}, + }, + }, + } + filter, err = New(map[string]interface{}{ + "test": false, + }) + assert.Len(err, 0) + nodes = filter.Apply(nodes) + assert.Len(nodes.List, 0) + + // keep a node + nodes = &runtime.Nodes{ + List: map[string]*runtime.Node{ + "a": { + Nodeinfo: &data.NodeInfo{NodeID: "a"}, + }, + }, + } + filter, err = New(map[string]interface{}{ + "test": true, + }) + assert.Len(err, 0) + nodes = filter.Apply(nodes) + assert.Len(nodes.List, 1) +} diff --git a/output/filter/haslocation/haslocation.go b/output/filter/haslocation/haslocation.go new file mode 100644 index 0000000..210d9e1 --- /dev/null +++ b/output/filter/haslocation/haslocation.go @@ -0,0 +1,40 @@ +package haslocation + +import ( + "errors" + + "github.com/FreifunkBremen/yanic/output/filter" + "github.com/FreifunkBremen/yanic/runtime" +) + +type haslocation struct { + has bool +} + +func init() { + filter.Register("has_location", build) +} + +func build(config interface{}) (filter.Filter, error) { + if value, ok := config.(bool); ok { + return &haslocation{has: value}, nil + } + return nil, errors.New("invalid configuration, bool expected") +} + +func (h *haslocation) Apply(node *runtime.Node) *runtime.Node { + if nodeinfo := node.Nodeinfo; nodeinfo != nil { + if h.has { + if location := nodeinfo.Location; location != nil { + return node + } + } else { + if location := nodeinfo.Location; location == nil { + return node + } + } + } else if !h.has { + return node + } + return nil +} diff --git a/output/filter/haslocation/haslocation_test.go b/output/filter/haslocation/haslocation_test.go new file mode 100644 index 0000000..34a7bfd --- /dev/null +++ b/output/filter/haslocation/haslocation_test.go @@ -0,0 +1,53 @@ +package haslocation + +import ( + "testing" + + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/runtime" + "github.com/stretchr/testify/assert" +) + +func TestFilterHasLocation(t *testing.T) { + assert := assert.New(t) + + // invalid config + filter, err := build(3) + assert.Error(err) + + // test to drop nodes without location + filter, err = build(true) + assert.NoError(err) + + // node has location (with 0,0) -> keep it + n := filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Location: &data.Location{}, + }}) + assert.NotNil(n) + + // node without location has no location -> drop it + n = filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{}}) + assert.Nil(n) + + // node without nodeinfo has no location -> drop it + n = filter.Apply(&runtime.Node{}) + assert.Nil(n) + + // test to drop nodes without location + filter, err = build(false) + assert.NoError(err) + + // node has location (with 0,0) -> drop it + n = filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Location: &data.Location{}, + }}) + assert.Nil(n) + + // node without location has no location -> keep it + n = filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{}}) + assert.NotNil(n) + + // node without nodeinfo has no location -> keep it + n = filter.Apply(&runtime.Node{}) + assert.NotNil(n) +} diff --git a/output/filter/inarea/inarea.go b/output/filter/inarea/inarea.go new file mode 100644 index 0000000..e6e9a06 --- /dev/null +++ b/output/filter/inarea/inarea.go @@ -0,0 +1,65 @@ +package inarea + +import ( + "errors" + + "github.com/FreifunkBremen/yanic/output/filter" + "github.com/FreifunkBremen/yanic/runtime" +) + +type area struct { + latitudeMin float64 + latitudeMax float64 + longitudeMin float64 + longitudeMax float64 +} + +func init() { + filter.Register("in_area", build) +} + +func build(config interface{}) (filter.Filter, error) { + values, ok := config.(map[string]interface{}) + if !ok { + return nil, errors.New("invalid configuration, map expected") + } + + a := area{} + if v, ok := values["latitude_min"]; ok { + a.latitudeMin = v.(float64) + } + if v, ok := values["latitude_max"]; ok { + a.latitudeMax = v.(float64) + } + if v, ok := values["longitude_min"]; ok { + a.longitudeMin = v.(float64) + } + if v, ok := values["longitude_max"]; ok { + a.longitudeMax = v.(float64) + } + + if a.latitudeMin >= a.latitudeMax { + return nil, errors.New("invalid latitude: max is bigger then min") + } + if a.longitudeMin >= a.longitudeMax { + return nil, errors.New("invalid longitude: max is bigger then min") + } + + // TODO bessere Fehlerbehandlung! + + return &a, nil +} + +func (a *area) Apply(node *runtime.Node) *runtime.Node { + if nodeinfo := node.Nodeinfo; nodeinfo != nil { + location := nodeinfo.Location + if location == nil { + return node + } + if location.Latitude >= a.latitudeMin && location.Latitude <= a.latitudeMax && location.Longitude >= a.longitudeMin && location.Longitude <= a.longitudeMax { + return node + } + return nil + } + return node +} diff --git a/output/filter/inarea/inarea_test.go b/output/filter/inarea/inarea_test.go new file mode 100644 index 0000000..bcac400 --- /dev/null +++ b/output/filter/inarea/inarea_test.go @@ -0,0 +1,86 @@ +package inarea + +import ( + "testing" + + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/runtime" + "github.com/stretchr/testify/assert" +) + +func TestFilterInArea(t *testing.T) { + assert := assert.New(t) + + filter, _ := build(map[string]interface{}{ + "latitude_min": 3.0, + "latitude_max": 5.0, + "longitude_min": 10.0, + "longitude_max": 12.0, + }) + + n := filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Location: &data.Location{Latitude: 4.0, Longitude: 11.0}, + }}) + assert.NotNil(n) + + // keep without nodeinfo -> use has_location for it + n = filter.Apply(&runtime.Node{}) + assert.NotNil(n) + + // keep without location -> use has_location for it + n = filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{}}) + assert.NotNil(n) + + // zeros not in area (0, 0) + n = filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Location: &data.Location{}, + }}) + assert.Nil(n) + + // in area + n = filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Location: &data.Location{Latitude: 4.0, Longitude: 11.0}, + }}) + assert.NotNil(n) + + // over max longitude -> dropped + n = filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Location: &data.Location{Latitude: 4.0, Longitude: 13.0}, + }}) + assert.Nil(n) + + // over max latitude -> dropped + n = filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Location: &data.Location{Latitude: 6.0, Longitude: 11.0}, + }}) + assert.Nil(n) + + // lower then mix latitde -> dropped + n = filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Location: &data.Location{Latitude: 1.0, Longitude: 2.0}, + }}) + assert.Nil(n) + + // invalid config format + _, err := build(true) + assert.Error(err) + + // invalid config latitude switched max and min + _, err = build(map[string]interface{}{ + "latitude_min": 5.0, + "latitude_max": 3.0, + "longitude_min": 10.0, + "longitude_max": 12.0, + }) + assert.Error(err) + + // invalid config longitude switched max and min + _, err = build(map[string]interface{}{ + "latitude_min": 3.0, + "latitude_max": 5.0, + "longitude_min": 15.0, + "longitude_max": 10.0, + }) + assert.Error(err) + +} diff --git a/output/filter/noowner/noowner.go b/output/filter/noowner/noowner.go new file mode 100644 index 0000000..b8fe69c --- /dev/null +++ b/output/filter/noowner/noowner.go @@ -0,0 +1,48 @@ +package noowner + +import ( + "errors" + + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/output/filter" + "github.com/FreifunkBremen/yanic/runtime" +) + +type noowner struct{ has bool } + +func init() { + filter.Register("noowner", build) +} + +func build(config interface{}) (filter.Filter, error) { + if value, ok := config.(bool); ok { + return &noowner{has: value}, nil + } + return nil, errors.New("invalid configuration, boolean expected") +} + +func (no *noowner) Apply(node *runtime.Node) *runtime.Node { + if nodeinfo := node.Nodeinfo; nodeinfo != nil && no.has { + node = &runtime.Node{ + Address: node.Address, + Firstseen: node.Firstseen, + Lastseen: node.Lastseen, + Online: node.Online, + Statistics: node.Statistics, + Nodeinfo: &data.NodeInfo{ + NodeID: nodeinfo.NodeID, + Network: nodeinfo.Network, + System: nodeinfo.System, + Owner: nil, + Hostname: nodeinfo.Hostname, + Location: nodeinfo.Location, + Software: nodeinfo.Software, + Hardware: nodeinfo.Hardware, + VPN: nodeinfo.VPN, + Wireless: nodeinfo.Wireless, + }, + Neighbours: node.Neighbours, + } + } + return node +} diff --git a/output/filter/noowner/noowner_test.go b/output/filter/noowner/noowner_test.go new file mode 100644 index 0000000..96be019 --- /dev/null +++ b/output/filter/noowner/noowner_test.go @@ -0,0 +1,39 @@ +package noowner + +import ( + "testing" + + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/runtime" + "github.com/stretchr/testify/assert" +) + +func TestFilter(t *testing.T) { + assert := assert.New(t) + + // invalid config + filter, err := build("nope") + assert.Error(err) + + // delete owner by configuration + filter, _ = build(true) + n := filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Owner: &data.Owner{ + Contact: "blub", + }, + }}) + + assert.NotNil(n) + assert.Nil(n.Nodeinfo.Owner) + + // keep owner configuration + filter, _ = build(false) + n = filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{ + Owner: &data.Owner{ + Contact: "blub", + }, + }}) + + assert.NotNil(n) + assert.NotNil(n.Nodeinfo.Owner) +} diff --git a/output/filter/site/site.go b/output/filter/site/site.go new file mode 100644 index 0000000..51a35e6 --- /dev/null +++ b/output/filter/site/site.go @@ -0,0 +1,36 @@ +package site + +import ( + "errors" + + "github.com/FreifunkBremen/yanic/output/filter" + "github.com/FreifunkBremen/yanic/runtime" +) + +type sites map[string]interface{} + +func init() { + filter.Register("sites", build) +} + +func build(config interface{}) (filter.Filter, error) { + values, ok := config.([]string) + if !ok { + return nil, errors.New("invalid configuration, array of strings expected") + } + + list := make(sites) + for _, nodeid := range values { + list[nodeid] = struct{}{} + } + return &list, nil +} + +func (list sites) Apply(node *runtime.Node) *runtime.Node { + if nodeinfo := node.Nodeinfo; nodeinfo != nil { + if _, ok := list[nodeinfo.System.SiteCode]; ok { + return node + } + } + return nil +} diff --git a/output/filter/site/site_test.go b/output/filter/site/site_test.go new file mode 100644 index 0000000..c442498 --- /dev/null +++ b/output/filter/site/site_test.go @@ -0,0 +1,32 @@ +package site + +import ( + "testing" + + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/runtime" + "github.com/stretchr/testify/assert" +) + +func TestFilterSite(t *testing.T) { + assert := assert.New(t) + + // invalid config + filter, err := build("ffhb") + assert.Error(err) + + filter, err = build([]string{"ffhb"}) + assert.NoError(err) + + // wronge node + n := filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{System: data.System{SiteCode: "ffxx"}}}) + assert.Nil(n) + + // right node + n = filter.Apply(&runtime.Node{Nodeinfo: &data.NodeInfo{System: data.System{SiteCode: "ffhb"}}}) + assert.NotNil(n) + + // node without data -> wrong node + n = filter.Apply(&runtime.Node{}) + assert.Nil(n) +}