[TASK] improve output (filtering, fix links type, new outputs nodelist for freifunkapi and meshviewer-ffrgb v10 NextGen) - #63

INSTALL.md

@@ -59,7 +59,7 @@ accessible under `/var/www/html/meshviewer`.
 #### With webserver (Apache, nginx)
 The meshviewer needs the output files like `nodes_path` and `graph_path` inside
 the same directory as the `dataPath`. Change the path in the section
-`[meshviewer]` accordingly.
+`[[nodes.output.meshviewer]]` accordingly.
 
 ### Service
 ```bash

cmd/serve.go

@@ -8,8 +8,9 @@ import (
 	"time"
 
 	"github.com/FreifunkBremen/yanic/database"
-	"github.com/FreifunkBremen/yanic/database/all"
+	allDatabase "github.com/FreifunkBremen/yanic/database/all"
-	"github.com/FreifunkBremen/yanic/meshviewer"
+	"github.com/FreifunkBremen/yanic/output"
+	allOutput "github.com/FreifunkBremen/yanic/output/all"
 	"github.com/FreifunkBremen/yanic/respond"
 	"github.com/FreifunkBremen/yanic/runtime"
 	"github.com/FreifunkBremen/yanic/webserver"
@@ -24,7 +25,7 @@ var serveCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		config := loadConfig()
 
-		connections, err := all.Connect(config.Database.Connection)
+		connections, err := allDatabase.Connect(config.Database.Connection)
 		if err != nil {
 			panic(err)
 		}
@@ -33,7 +34,13 @@ var serveCmd = &cobra.Command{
 
 		nodes = runtime.NewNodes(config)
 		nodes.Start()
-		meshviewer.Start(config, nodes)
+
+		outputs, err := allOutput.Register(config.Nodes.Output)
+		if err != nil {
+			panic(err)
+		}
+		output.Start(outputs, nodes, config)
+		defer output.Close()
 
 		if config.Webserver.Enable {
 			log.Println("starting webserver on", config.Webserver.Bind)

config_example.toml

@@ -34,7 +34,17 @@ save_interval = "5s"
 offline_after = "10m"
 
 
-[meshviewer]
+## [[nodes.output.-]]
+# every output:
+#  needs to be enabled just adding:
+#    enable = true
+#  could filter the nodes by using a there filter entry (see output meshviewer)
+#   [nodes.output.-.filter]
+#  could be used multiple times (suggested by the "[[...]]" instatt of "[...]")
+#  it is useful for e.g. filter by different array and use multiple meshviewers
+
+[[nodes.output.meshviewer]]
+enable = true
 # The structure version of the output which should be generated (i.e. nodes.json)
 # version 1 is accepted by the legacy meshviewer (which is the master branch)
 #   i.e. https://github.com/ffnord/meshviewer/tree/master
@@ -48,6 +58,26 @@ nodes_path = "/var/www/html/meshviewer/data/nodes.json"
 graph_path = "/var/www/html/meshviewer/data/graph.json"
 
 
+[nodes.output.meshviewer.filter]
+# no_owner = true
+has_location = true
+blacklist = ["vpnid"]
+
+[nodes.output.meshviewer.filter.in_area]
+latitude_min = 34.30
+latitude_max = 71.85
+longitude_min = -24.96
+longitude_max = 39.72
+
+[[nodes.output.nodelist]]
+enable   = true
+path = "/var/www/html/meshviewer/data/nodelist.json"
+
+[[nodes.output.meshviewer-ffrgb]]
+enable   = true
+path = "/var/www/html/meshviewer/data/meshviewer.json"
+
+
 [database]
 # this will send delete commands to the database to prune data
 # which is older than:
@@ -55,6 +85,13 @@ delete_after    = "7d"
 # how often run the cleaning
 delete_interval = "1h"
 
+## [[database.connection.-]]
+# every output:
+#  needs to be enabled just adding:
+#    enable = true
+#  could be used multiple times (suggested by the "[[...]]" instatt of "[...]")
+#  it is useful for e.g. save into a database before and behind a firewall
+
 # Save collected data to InfluxDB.
 # There are the following measurments:
 #   node: store node specific data i.e. clients memory, airtime

data/nodeinfo.go

@@ -4,7 +4,7 @@ package data
 type NodeInfo struct {
 	NodeID   string    `json:"node_id"`
 	Network  Network   `json:"network"`
-	Owner    *Owner    `json:"-"` // Removed for privacy reasons
+	Owner    *Owner    `json:"owner"`
 	System   System    `json:"system"`
 	Hostname string    `json:"hostname"`
 	Location *Location `json:"location,omitempty"`

database/influxdb/global_test.go

@@ -22,37 +22,43 @@ func TestGlobalStats(t *testing.T) {
 func createTestNodes() *runtime.Nodes {
 	nodes := runtime.NewNodes(&runtime.Config{})
 
-	nodeData := &data.ResponseData{
+	nodeData := &runtime.Node{
+		Online: true,
 		Statistics: &data.Statistics{
 			Clients: data.Clients{
 				Total: 23,
 			},
 		},
-		NodeInfo: &data.NodeInfo{
+		Nodeinfo: &data.NodeInfo{
+			NodeID: "abcdef012345",
 			Hardware: data.Hardware{
 				Model: "TP-Link 841",
 			},
 		},
 	}
-	nodeData.NodeInfo.Software.Firmware.Release = "2016.1.6+entenhausen1"
+	nodeData.Nodeinfo.Software.Firmware.Release = "2016.1.6+entenhausen1"
-	nodes.Update("abcdef012345", nodeData)
+	nodes.AddNode(nodeData)
 
-	nodes.Update("112233445566", &data.ResponseData{
+	nodes.AddNode(&runtime.Node{
+		Online: true,
 		Statistics: &data.Statistics{
 			Clients: data.Clients{
 				Total: 2,
 			},
 		},
-		NodeInfo: &data.NodeInfo{
+		Nodeinfo: &data.NodeInfo{
+			NodeID: "112233445566",
 			Hardware: data.Hardware{
 				Model: "TP-Link 841",
 			},
 		},
 	})
 
-	nodes.Update("0xdeadbeef0x", &data.ResponseData{
+	nodes.AddNode(&runtime.Node{
-		NodeInfo: &data.NodeInfo{
+		Online: true,
-			VPN: true,
+		Nodeinfo: &data.NodeInfo{
+			NodeID: "0xdeadbeef0x",
+			VPN:    true,
 			Hardware: data.Hardware{
 				Model: "Xeon Multi-Core",
 			},

database/influxdb/node_test.go

@@ -144,7 +144,7 @@ func testPoints(nodes ...*runtime.Node) (points []*client.Point) {
 	}
 
 	for _, node := range nodes {
-		nodesList.Update(node.Nodeinfo.NodeID, &data.ResponseData{NodeInfo: node.Nodeinfo})
+		nodesList.AddNode(node)
 	}
 
 	// Process data

meshviewer/nodes.go

@@ -1,50 +0,0 @@
-package meshviewer
-
-import (
-	"log"
-	"time"
-
-	"github.com/FreifunkBremen/yanic/runtime"
-)
-
-type nodeBuilder func(*runtime.Nodes) interface{}
-
-var nodeFormats = map[int]nodeBuilder{
-	1: BuildNodesV1,
-	2: BuildNodesV2,
-}
-
-// Start all services to manage Nodes
-func Start(config *runtime.Config, nodes *runtime.Nodes) {
-	go worker(config, nodes)
-}
-
-// Periodically saves the cached DB to json file
-func worker(config *runtime.Config, nodes *runtime.Nodes) {
-	c := time.Tick(config.Nodes.SaveInterval.Duration)
-
-	for range c {
-		saveMeshviewer(config, nodes)
-	}
-}
-
-func saveMeshviewer(config *runtime.Config, nodes *runtime.Nodes) {
-	// Locking foo
-	nodes.RLock()
-	defer nodes.RUnlock()
-	if path := config.Meshviewer.NodesPath; path != "" {
-		version := config.Meshviewer.Version
-		builder := nodeFormats[version]
-
-		if builder != nil {
-			runtime.SaveJSON(builder(nodes), path)
-		} else {
-			log.Panicf("invalid nodes version: %d", version)
-		}
-
-	}
-
-	if path := config.Meshviewer.GraphPath; path != "" {
-		runtime.SaveJSON(BuildGraph(nodes), path)
-	}
-}

output/all/filter.go

@@ -0,0 +1,39 @@
+package all
+
+import "github.com/FreifunkBremen/yanic/runtime"
+
+// Config Filter
+type filterConfig map[string]interface{}
+
+type filterFunc func(*runtime.Node) *runtime.Node
+
+func noFilter(node *runtime.Node) *runtime.Node {
+	return node
+}
+
+// Create Filter
+func (f filterConfig) filtering(nodesOrigin *runtime.Nodes) *runtime.Nodes {
+	nodes := runtime.NewNodes(&runtime.Config{})
+	filterfuncs := []filterFunc{
+		f.HasLocation(),
+		f.Blacklist(),
+		f.InArea(),
+		f.NoOwner(),
+	}
+
+	for _, nodeOrigin := range nodesOrigin.List {
+		//maybe cloning of this object is better?
+		node := nodeOrigin
+		for _, f := range filterfuncs {
+			node = f(node)
+			if node == nil {
+				break
+			}
+		}
+
+		if node != nil {
+			nodes.AddNode(node)
+		}
+	}
+	return nodes
+}

output/all/filter_blacklist.go

@@ -0,0 +1,24 @@
+package all
+
+import "github.com/FreifunkBremen/yanic/runtime"
+
+func (f filterConfig) Blacklist() filterFunc {
+	v, ok := f["blacklist"]
+	if !ok {
+		return noFilter
+	}
+
+	list := make(map[string]interface{})
+	for _, nodeid := range v.([]interface{}) {
+		list[nodeid.(string)] = true
+	}
+
+	return func(node *runtime.Node) *runtime.Node {
+		if nodeinfo := node.Nodeinfo; nodeinfo != nil {
+			if _, ok := list[nodeinfo.NodeID]; ok {
+				return nil
+			}
+		}
+		return node
+	}
+}

output/all/filter_blacklist_test.go

@@ -0,0 +1,34 @@
+package all
+
+import (
+	"testing"
+
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFilterBlacklist(t *testing.T) {
+	assert := assert.New(t)
+	var config filterConfig
+
+	config = map[string]interface{}{}
+
+	filterBlacklist := config.Blacklist()
+
+	n := filterBlacklist(&runtime.Node{Nodeinfo: &data.NodeInfo{}})
+	assert.NotNil(n)
+
+	config["blacklist"] = []interface{}{"a", "c"}
+	filterBlacklist = config.Blacklist()
+
+	n = filterBlacklist(&runtime.Node{Nodeinfo: &data.NodeInfo{NodeID: "a"}})
+	assert.Nil(n)
+
+	n = filterBlacklist(&runtime.Node{Nodeinfo: &data.NodeInfo{}})
+	assert.NotNil(n)
+
+	n = filterBlacklist(&runtime.Node{})
+	assert.NotNil(n)
+
+}

output/all/filter_haslocation.go

@@ -0,0 +1,26 @@
+package all
+
+import "github.com/FreifunkBremen/yanic/runtime"
+
+func (f filterConfig) HasLocation() filterFunc {
+	withLocation, ok := f["has_location"].(bool)
+	if !ok {
+		return noFilter
+	}
+	return func(node *runtime.Node) *runtime.Node {
+		if nodeinfo := node.Nodeinfo; nodeinfo != nil {
+			if withLocation {
+				if location := nodeinfo.Location; location != nil {
+					return node
+				}
+			} else {
+				if location := nodeinfo.Location; location == nil {
+					return node
+				}
+			}
+		} else if !withLocation {
+			return node
+		}
+		return nil
+	}
+}

output/all/filter_haslocation_test.go

@@ -0,0 +1,50 @@
+package all
+
+import (
+	"testing"
+
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFilterHasLocation(t *testing.T) {
+	assert := assert.New(t)
+	var config filterConfig
+
+	config = map[string]interface{}{}
+
+	filterHasLocation := config.HasLocation()
+	n := filterHasLocation(&runtime.Node{Nodeinfo: &data.NodeInfo{
+		Location: &data.Location{},
+	}})
+	assert.NotNil(n)
+
+	config["has_location"] = true
+	filterHasLocation = config.HasLocation()
+
+	n = filterHasLocation(&runtime.Node{Nodeinfo: &data.NodeInfo{
+		Location: &data.Location{},
+	}})
+	assert.NotNil(n)
+
+	n = filterHasLocation(&runtime.Node{Nodeinfo: &data.NodeInfo{}})
+	assert.Nil(n)
+
+	n = filterHasLocation(&runtime.Node{})
+	assert.Nil(n)
+
+	config["has_location"] = false
+	filterHasLocation = config.HasLocation()
+
+	n = filterHasLocation(&runtime.Node{Nodeinfo: &data.NodeInfo{
+		Location: &data.Location{},
+	}})
+	assert.Nil(n)
+
+	n = filterHasLocation(&runtime.Node{Nodeinfo: &data.NodeInfo{}})
+	assert.NotNil(n)
+
+	n = filterHasLocation(&runtime.Node{})
+	assert.NotNil(n)
+}

output/all/filter_inarea.go

@@ -0,0 +1,42 @@
+package all
+
+import "github.com/FreifunkBremen/yanic/runtime"
+
+type area struct {
+	latitudeMin  float64
+	latitudeMax  float64
+	longitudeMin float64
+	longitudeMax float64
+}
+
+func (f filterConfig) InArea() filterFunc {
+	if areaConfigInt, ok := f["in_area"]; ok {
+		areaConfig := areaConfigInt.(map[string]interface{})
+		a := area{}
+		if v, ok := areaConfig["latitude_min"]; ok {
+			a.latitudeMin = v.(float64)
+		}
+		if v, ok := areaConfig["latitude_max"]; ok {
+			a.latitudeMax = v.(float64)
+		}
+		if v, ok := areaConfig["longitude_min"]; ok {
+			a.longitudeMin = v.(float64)
+		}
+		if v, ok := areaConfig["longitude_max"]; ok {
+			a.longitudeMax = v.(float64)
+		}
+		return func(node *runtime.Node) *runtime.Node {
+			if nodeinfo := node.Nodeinfo; nodeinfo != nil {
+				location := nodeinfo.Location
+				if location == nil {
+					return node
+				}
+				if location.Latitude >= a.latitudeMin && location.Latitude <= a.latitudeMax && location.Longtitude >= a.longitudeMin && location.Longtitude <= a.longitudeMax {
+					return node
+				}
+			}
+			return nil
+		}
+	}
+	return noFilter
+}

output/all/filter_inarea_test.go

@@ -0,0 +1,65 @@
+package all
+
+import (
+	"testing"
+
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFilterInArea(t *testing.T) {
+	assert := assert.New(t)
+	var config filterConfig
+	areaConfig := map[string]interface{}{
+		"latitude_min":  3.0,
+		"latitude_max":  5.0,
+		"longitude_min": 10.0,
+		"longitude_max": 12.0,
+	}
+	config = map[string]interface{}{}
+
+	filterLocationInArea := config.InArea()
+	n := filterLocationInArea(&runtime.Node{Nodeinfo: &data.NodeInfo{
+		Location: &data.Location{Latitude: 4.0, Longtitude: 11.0},
+	}})
+	assert.NotNil(n)
+
+	config["in_area"] = areaConfig
+	filterLocationInArea = config.InArea()
+
+	// drop area without nodeinfo
+	n = filterLocationInArea(&runtime.Node{})
+	assert.Nil(n)
+
+	// keep without location
+	n = filterLocationInArea(&runtime.Node{Nodeinfo: &data.NodeInfo{}})
+	assert.NotNil(n)
+
+	// zeros not in area (0, 0)
+	n = filterLocationInArea(&runtime.Node{Nodeinfo: &data.NodeInfo{
+		Location: &data.Location{},
+	}})
+	assert.Nil(n)
+
+	// in area
+	n = filterLocationInArea(&runtime.Node{Nodeinfo: &data.NodeInfo{
+		Location: &data.Location{Latitude: 4.0, Longtitude: 11.0},
+	}})
+	assert.NotNil(n)
+
+	n = filterLocationInArea(&runtime.Node{Nodeinfo: &data.NodeInfo{
+		Location: &data.Location{Latitude: 4.0, Longtitude: 13.0},
+	}})
+	assert.Nil(n)
+
+	n = filterLocationInArea(&runtime.Node{Nodeinfo: &data.NodeInfo{
+		Location: &data.Location{Latitude: 6.0, Longtitude: 11.0},
+	}})
+	assert.Nil(n)
+
+	n = filterLocationInArea(&runtime.Node{Nodeinfo: &data.NodeInfo{
+		Location: &data.Location{Latitude: 1.0, Longtitude: 2.0},
+	}})
+	assert.Nil(n)
+}

output/all/filter_noowner.go

@@ -0,0 +1,37 @@
+package all
+
+import (
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+func (f filterConfig) NoOwner() filterFunc {
+	if v, ok := f["no_owner"]; ok && v.(bool) == false {
+		return noFilter
+	}
+	return func(node *runtime.Node) *runtime.Node {
+		if nodeinfo := node.Nodeinfo; nodeinfo != nil {
+			return &runtime.Node{
+				Address:    node.Address,
+				Firstseen:  node.Firstseen,
+				Lastseen:   node.Lastseen,
+				Online:     node.Online,
+				Statistics: node.Statistics,
+				Nodeinfo: &data.NodeInfo{
+					NodeID:   nodeinfo.NodeID,
+					Network:  nodeinfo.Network,
+					System:   nodeinfo.System,
+					Owner:    nil,
+					Hostname: nodeinfo.Hostname,
+					Location: nodeinfo.Location,
+					Software: nodeinfo.Software,
+					Hardware: nodeinfo.Hardware,
+					VPN:      nodeinfo.VPN,
+					Wireless: nodeinfo.Wireless,
+				},
+				Neighbours: node.Neighbours,
+			}
+		}
+		return node
+	}
+}

output/all/filter_noowner_test.go

@@ -0,0 +1,49 @@
+package all
+
+import (
+	"testing"
+
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFilterNoOwner(t *testing.T) {
+	assert := assert.New(t)
+	var config filterConfig
+
+	config = map[string]interface{}{}
+
+	filterNoOwner := config.NoOwner()
+	n := filterNoOwner(&runtime.Node{Nodeinfo: &data.NodeInfo{
+		Owner: &data.Owner{
+			Contact: "blub",
+		},
+	}})
+	assert.NotNil(n)
+	assert.Nil(n.Nodeinfo.Owner)
+
+	n = filterNoOwner(&runtime.Node{})
+	assert.NotNil(n)
+
+	config["no_owner"] = true
+	filterNoOwner = config.NoOwner()
+	n = filterNoOwner(&runtime.Node{Nodeinfo: &data.NodeInfo{
+		Owner: &data.Owner{
+			Contact: "blub",
+		},
+	}})
+	assert.NotNil(n)
+	assert.Nil(n.Nodeinfo.Owner)
+
+	config["no_owner"] = false
+	filterNoOwner = config.NoOwner()
+
+	n = filterNoOwner(&runtime.Node{Nodeinfo: &data.NodeInfo{
+		Owner: &data.Owner{
+			Contact: "blub",
+		},
+	}})
+	assert.NotNil(n)
+	assert.NotNil(n.Nodeinfo.Owner)
+}

output/all/filter_test.go

@@ -0,0 +1,41 @@
+package all
+
+import (
+	"testing"
+
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFilter(t *testing.T) {
+	assert := assert.New(t)
+
+	// filtered - do not run all
+	nodes := &runtime.Nodes{
+		List: map[string]*runtime.Node{
+			"a": &runtime.Node{
+				Nodeinfo: &data.NodeInfo{NodeID: "a"},
+			},
+		},
+	}
+	config := filterConfig{
+		"has_location": true,
+	}
+	nodes = config.filtering(nodes)
+	assert.Len(nodes.List, 0)
+
+	// run to end
+	nodes = &runtime.Nodes{
+		List: map[string]*runtime.Node{
+			"a": &runtime.Node{
+				Nodeinfo: &data.NodeInfo{NodeID: "a"},
+			},
+		},
+	}
+	config = filterConfig{
+		"has_location": false,
+	}
+	nodes = config.filtering(nodes)
+	assert.Len(nodes.List, 1)
+}

output/all/internal.go

@@ -0,0 +1,63 @@
+package all
+
+import (
+	"log"
+
+	"github.com/FreifunkBremen/yanic/output"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+type Output struct {
+	output.Output
+	list   map[int]output.Output
+	filter map[int]filterConfig
+}
+
+func Register(configuration map[string]interface{}) (output.Output, error) {
+	list := make(map[int]output.Output)
+	filter := make(map[int]filterConfig)
+	i := 1
+	allOutputs := configuration
+	for outputType, outputRegister := range output.Adapters {
+		configForOutput := allOutputs[outputType]
+		if configForOutput == nil {
+			log.Printf("the output type '%s' has no configuration\n", outputType)
+			continue
+		}
+		outputConfigs, ok := configForOutput.([]map[string]interface{})
+		if !ok {
+			log.Panicf("the output type '%s' has the wrong format\n", outputType)
+		}
+		for _, config := range outputConfigs {
+			if c, ok := config["enable"].(bool); ok && !c {
+				continue
+			}
+			output, err := outputRegister(config)
+			if err != nil {
+				return nil, err
+			}
+			if output == nil {
+				continue
+			}
+			list[i] = output
+			if c := config["filter"]; c != nil {
+				filter[i] = config["filter"].(map[string]interface{})
+			}
+			i++
+		}
+	}
+	return &Output{list: list, filter: filter}, nil
+}
+
+func (o *Output) Save(nodes *runtime.Nodes) {
+	for i, item := range o.list {
+		var filteredNodes *runtime.Nodes
+		if config := o.filter[i]; config != nil {
+			filteredNodes = config.filtering(nodes)
+		} else {
+			filteredNodes = filterConfig{}.filtering(nodes)
+		}
+
+		item.Save(filteredNodes)
+	}
+}

output/all/internal_test.go

@@ -0,0 +1,89 @@
+package all
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/FreifunkBremen/yanic/output"
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/stretchr/testify/assert"
+)
+
+type testOutput struct {
+	output.Output
+	CountSave int
+}
+
+func (c *testOutput) Save(nodes *runtime.Nodes) {
+	c.CountSave++
+}
+
+func TestStart(t *testing.T) {
+	assert := assert.New(t)
+
+	nodes := &runtime.Nodes{}
+
+	globalOutput := &testOutput{}
+	output.RegisterAdapter("a", func(config map[string]interface{}) (output.Output, error) {
+		return globalOutput, nil
+	})
+	output.RegisterAdapter("b", func(config map[string]interface{}) (output.Output, error) {
+		return globalOutput, nil
+	})
+	output.RegisterAdapter("c", func(config map[string]interface{}) (output.Output, error) {
+		return globalOutput, nil
+	})
+	output.RegisterAdapter("d", func(config map[string]interface{}) (output.Output, error) {
+		return nil, nil
+	})
+	output.RegisterAdapter("e", func(config map[string]interface{}) (output.Output, error) {
+		return nil, errors.New("blub")
+	})
+	allOutput, err := Register(map[string]interface{}{
+		"a": []map[string]interface{}{
+			map[string]interface{}{
+				"enable": false,
+				"path":   "a1",
+			},
+			map[string]interface{}{
+				"path": "a2",
+			},
+			map[string]interface{}{
+				"enable": true,
+				"path":   "a3",
+			},
+		},
+		"b": nil,
+		"c": []map[string]interface{}{
+			map[string]interface{}{
+				"path":   "c1",
+				"filter": map[string]interface{}{},
+			},
+		},
+		// fetch continue command in Connect
+		"d": []map[string]interface{}{
+			map[string]interface{}{
+				"path": "d0",
+			},
+		},
+	})
+	assert.NoError(err)
+
+	assert.Equal(0, globalOutput.CountSave)
+	allOutput.Save(nodes)
+	assert.Equal(3, globalOutput.CountSave)
+
+	_, err = Register(map[string]interface{}{
+		"e": []map[string]interface{}{
+			map[string]interface{}{},
+		},
+	})
+	assert.Error(err)
+
+	// wrong format -> the only panic in Register
+	assert.Panics(func() {
+		Register(map[string]interface{}{
+			"e": true,
+		})
+	})
+}

output/all/main.go

@@ -0,0 +1,7 @@
+package all
+
+import (
+	_ "github.com/FreifunkBremen/yanic/output/meshviewer"
+	_ "github.com/FreifunkBremen/yanic/output/meshviewer-ffrgb"
+	_ "github.com/FreifunkBremen/yanic/output/nodelist"
+)

output/internal.go

@@ -0,0 +1,40 @@
+package output
+
+import (
+	"sync"
+	"time"
+
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+var quit chan struct{}
+var wg = sync.WaitGroup{}
+
+// Start workers of database
+// WARNING: Do not override this function
+//  you should use New()
+func Start(output Output, nodes *runtime.Nodes, config *runtime.Config) {
+	quit = make(chan struct{})
+	wg.Add(1)
+	go saveWorker(output, nodes, config.Nodes.SaveInterval.Duration)
+}
+
+func Close() {
+	close(quit)
+	wg.Wait()
+}
+
+// save periodically to output
+func saveWorker(output Output, nodes *runtime.Nodes, saveInterval time.Duration) {
+	ticker := time.NewTicker(saveInterval)
+	for {
+		select {
+		case <-ticker.C:
+			output.Save(nodes)
+		case <-quit:
+			wg.Done()
+			ticker.Stop()
+			return
+		}
+	}
+}

output/internal_test.go

@@ -0,0 +1,49 @@
+package output
+
+import (
+	"testing"
+	"time"
+
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/stretchr/testify/assert"
+)
+
+type testConn struct {
+	Output
+	CountSave int
+}
+
+func (c *testConn) Save(nodes *runtime.Nodes) {
+	c.CountSave++
+}
+
+func TestStart(t *testing.T) {
+	assert := assert.New(t)
+
+	conn := &testConn{}
+	config := &runtime.Config{
+		Nodes: struct {
+			Enable       bool             `toml:"enable"`
+			StatePath    string           `toml:"state_path"`
+			SaveInterval runtime.Duration `toml:"save_interval"`
+			OfflineAfter runtime.Duration `toml:"offline_after"`
+			PruneAfter   runtime.Duration `toml:"prune_after"`
+			Output       map[string]interface{}
+		}{
+			SaveInterval: runtime.Duration{Duration: time.Millisecond * 10},
+		},
+	}
+	assert.Nil(quit)
+
+	Start(conn, nil, config)
+	assert.NotNil(quit)
+
+	assert.Equal(0, conn.CountSave)
+	time.Sleep(time.Millisecond * 12)
+	assert.Equal(1, conn.CountSave)
+
+	time.Sleep(time.Millisecond * 12)
+	Close()
+	assert.Equal(2, conn.CountSave)
+
+}

output/meshviewer-ffrgb/meshviewer.go

@@ -0,0 +1,86 @@
+package meshviewerFFRGB
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/FreifunkBremen/yanic/jsontime"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+func transform(nodes *runtime.Nodes) *Meshviewer {
+
+	meshviewer := &Meshviewer{
+		Timestamp: jsontime.Now(),
+		Nodes:     make([]*Node, 0),
+		Links:     make([]*Link, 0),
+	}
+
+	links := make(map[string]*Link)
+
+	nodes.RLock()
+	defer nodes.RUnlock()
+
+	for _, nodeOrigin := range nodes.List {
+		node := NewNode(nodes, nodeOrigin)
+		meshviewer.Nodes = append(meshviewer.Nodes, node)
+
+		typeList := make(map[string]string)
+
+		if nodeinfo := nodeOrigin.Nodeinfo; nodeinfo != nil {
+			if meshes := nodeinfo.Network.Mesh; meshes != nil {
+				for _, mesh := range meshes {
+					for _, mac := range mesh.Interfaces.Wireless {
+						typeList[mac] = "wifi"
+					}
+					for _, mac := range mesh.Interfaces.Tunnel {
+						typeList[mac] = "vpn"
+					}
+				}
+			}
+		}
+
+		for _, linkOrigin := range nodes.NodeLinks(nodeOrigin) {
+			var key string
+			// keep source and target in the same order
+			switchSourceTarget := strings.Compare(linkOrigin.SourceMAC, linkOrigin.TargetMAC) > 0
+			if switchSourceTarget {
+				key = fmt.Sprintf("%s-%s", linkOrigin.SourceMAC, linkOrigin.TargetMAC)
+			} else {
+				key = fmt.Sprintf("%s-%s", linkOrigin.TargetMAC, linkOrigin.SourceMAC)
+			}
+			if link := links[key]; link != nil {
+				if switchSourceTarget {
+					link.TargetTQ = float32(linkOrigin.TQ) / 255.0
+				} else {
+					link.SourceTQ = float32(linkOrigin.TQ) / 255.0
+				}
+				continue
+			}
+			linkType := typeList[linkOrigin.SourceMAC]
+			if linkType == "" {
+				linkType = "other"
+			}
+			tq := float32(linkOrigin.TQ) / 255.0
+			link := &Link{
+				Type:      linkType,
+				Source:    linkOrigin.SourceID,
+				SourceMAC: linkOrigin.SourceMAC,
+				Target:    linkOrigin.TargetID,
+				TargetMAC: linkOrigin.TargetMAC,
+				SourceTQ:  tq,
+				TargetTQ:  tq,
+			}
+			if switchSourceTarget {
+				link.Source = linkOrigin.TargetID
+				link.SourceMAC = linkOrigin.TargetMAC
+				link.Target = linkOrigin.SourceID
+				link.TargetMAC = linkOrigin.SourceMAC
+			}
+			links[key] = link
+			meshviewer.Links = append(meshviewer.Links, link)
+		}
+	}
+
+	return meshviewer
+}

output/meshviewer-ffrgb/meshviewer_test.go

@@ -0,0 +1,144 @@
+package meshviewerFFRGB
+
+import (
+	"testing"
+
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTransform(t *testing.T) {
+	assert := assert.New(t)
+
+	nodes := runtime.NewNodes(&runtime.Config{})
+	nodes.AddNode(&runtime.Node{
+		Nodeinfo: &data.NodeInfo{
+			NodeID: "node_a",
+			Network: data.Network{
+				Mac: "node:a:mac",
+				Mesh: map[string]*data.BatInterface{
+					"bat0": &data.BatInterface{
+						Interfaces: struct {
+							Wireless []string `json:"wireless,omitempty"`
+							Other    []string `json:"other,omitempty"`
+							Tunnel   []string `json:"tunnel,omitempty"`
+						}{
+							Wireless: []string{"node:a:mac:wifi"},
+							Tunnel:   []string{"node:a:mac:vpn"},
+							Other:    []string{"node:a:mac:lan"},
+						},
+					},
+				},
+			},
+		},
+		Neighbours: &data.Neighbours{
+			NodeID: "node_a",
+			Batadv: map[string]data.BatadvNeighbours{
+				"node:a:mac:wifi": data.BatadvNeighbours{
+					Neighbours: map[string]data.BatmanLink{
+						"node:b:mac:wifi": data.BatmanLink{Tq: 153},
+					},
+				},
+				"node:a:mac:lan": data.BatadvNeighbours{
+					Neighbours: map[string]data.BatmanLink{
+						"node:b:mac:lan": data.BatmanLink{Tq: 51},
+					},
+				},
+			},
+		},
+	})
+	nodes.AddNode(&runtime.Node{
+		Nodeinfo: &data.NodeInfo{
+			NodeID: "node_c",
+			Network: data.Network{
+				Mac: "node:c:mac",
+				Mesh: map[string]*data.BatInterface{
+					"bat0": &data.BatInterface{
+						Interfaces: struct {
+							Wireless []string `json:"wireless,omitempty"`
+							Other    []string `json:"other,omitempty"`
+							Tunnel   []string `json:"tunnel,omitempty"`
+						}{
+							Other: []string{"node:c:mac:lan"},
+						},
+					},
+				},
+			},
+		},
+		Neighbours: &data.Neighbours{
+			NodeID: "node_b",
+			Batadv: map[string]data.BatadvNeighbours{
+				"node:c:mac:lan": data.BatadvNeighbours{
+					Neighbours: map[string]data.BatmanLink{
+						"node:b:mac:lan": data.BatmanLink{Tq: 102},
+					},
+				},
+			},
+		},
+	})
+	nodes.AddNode(&runtime.Node{
+		Nodeinfo: &data.NodeInfo{
+			NodeID: "node_b",
+			Network: data.Network{
+				Mac: "node:b:mac",
+				Mesh: map[string]*data.BatInterface{
+					"bat0": &data.BatInterface{
+						Interfaces: struct {
+							Wireless []string `json:"wireless,omitempty"`
+							Other    []string `json:"other,omitempty"`
+							Tunnel   []string `json:"tunnel,omitempty"`
+						}{
+							Wireless: []string{"node:b:mac:wifi"},
+							Other:    []string{"node:b:mac:lan"},
+						},
+					},
+				},
+			},
+		},
+		Neighbours: &data.Neighbours{
+			NodeID: "node_b",
+			Batadv: map[string]data.BatadvNeighbours{
+				"node:b:mac:lan": data.BatadvNeighbours{
+					Neighbours: map[string]data.BatmanLink{
+						"node:c:mac:lan": data.BatmanLink{Tq: 204},
+					},
+				},
+				"node:b:mac:wifi": data.BatadvNeighbours{
+					Neighbours: map[string]data.BatmanLink{
+						"node:a:mac:wifi": data.BatmanLink{Tq: 204},
+					},
+				},
+			},
+		},
+	})
+
+	meshviewer := transform(nodes)
+	assert.NotNil(meshviewer)
+	assert.Len(meshviewer.Nodes, 3)
+	links := meshviewer.Links
+	assert.Len(links, 3)
+
+	for _, link := range links {
+		switch link.SourceMAC {
+		case "node:a:mac:lan":
+			assert.Equal("other", link.Type)
+			assert.Equal("node:b:mac:lan", link.TargetMAC)
+			assert.Equal(float32(0.2), link.SourceTQ)
+			assert.Equal(float32(0.2), link.TargetTQ)
+			break
+
+		case "node:a:mac:wifi":
+			assert.Equal("wifi", link.Type)
+			assert.Equal("node:b:mac:wifi", link.TargetMAC)
+			assert.Equal(float32(0.6), link.SourceTQ)
+			assert.Equal(float32(0.8), link.TargetTQ)
+		default:
+			assert.Equal("other", link.Type)
+			assert.Equal("node:c:mac:lan", link.TargetMAC)
+			assert.Equal(float32(0.8), link.SourceTQ)
+			assert.Equal(float32(0.4), link.TargetTQ)
+			break
+		}
+	}
+}

output/meshviewer-ffrgb/output.go

@@ -0,0 +1,43 @@
+package meshviewerFFRGB
+
+import (
+	"errors"
+
+	"github.com/FreifunkBremen/yanic/output"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+type Output struct {
+	output.Output
+	path string
+}
+
+type Config map[string]interface{}
+
+func (c Config) Path() string {
+	if path, ok := c["path"]; ok {
+		return path.(string)
+	}
+	return ""
+}
+
+func init() {
+	output.RegisterAdapter("meshviewer-ffrgb", Register)
+}
+
+func Register(configuration map[string]interface{}) (output.Output, error) {
+	var config Config
+	config = configuration
+
+	if path := config.Path(); path != "" {
+		return &Output{
+			path: path,
+		}, nil
+	}
+	return nil, errors.New("no path given")
+
+}
+
+func (o *Output) Save(nodes *runtime.Nodes) {
+	runtime.SaveJSON(transform(nodes), o.path)
+}

output/meshviewer-ffrgb/output_test.go

@@ -0,0 +1,27 @@
+package meshviewerFFRGB
+
+import (
+	"os"
+	"testing"
+
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestOutput(t *testing.T) {
+	assert := assert.New(t)
+	out, err := Register(map[string]interface{}{})
+	assert.Error(err)
+	assert.Nil(out)
+
+	out, err = Register(map[string]interface{}{
+		"path": "/tmp/meshviewer.json",
+	})
+	os.Remove("/tmp/meshviewer.json")
+	assert.NoError(err)
+	assert.NotNil(out)
+
+	out.Save(&runtime.Nodes{})
+	_, err = os.Stat("/tmp/meshviewer.json")
+	assert.NoError(err)
+}

output/meshviewer-ffrgb/struct.go

@@ -0,0 +1,153 @@
+package meshviewerFFRGB
+
+import (
+	"time"
+
+	"github.com/FreifunkBremen/yanic/jsontime"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+type Meshviewer struct {
+	Timestamp jsontime.Time `json:"timestamp"`
+	Nodes     []*Node       `json:"nodes"`
+	Links     []*Link       `json:"links"`
+}
+
+type Node struct {
+	Firstseen      jsontime.Time `json:"firstseen"`
+	Lastseen       jsontime.Time `json:"lastseen"`
+	IsOnline       bool          `json:"is_online"`
+	IsGateway      bool          `json:"is_gateway"`
+	Clients        uint32        `json:"clients"`
+	ClientsWifi24  uint32        `json:"clients_wifi24"`
+	ClientsWifi5   uint32        `json:"clients_wifi5"`
+	ClientsOthers  uint32        `json:"clients_other"`
+	RootFSUsage    float64       `json:"rootfs_usage,omitempty"`
+	LoadAverage    float64       `json:"loadavg,omitempty"`
+	MemoryUsage    *float64      `json:"memory_usage,omitempty"`
+	Uptime         jsontime.Time `json:"uptime,omitempty"`
+	GatewayNexthop string        `json:"gateway_nexthop,omitempty"`
+	GatewayIPv4    string        `json:"gateway,omitempty"`
+	GatewayIPv6    string        `json:"gateway6,omitempty"`
+	NodeID         string        `json:"node_id"`
+	Network        Network       `json:"network"`
+	SiteCode       string        `json:"site_code,omitempty"`
+	Hostname       string        `json:"hostname"`
+	Location       *Location     `json:"location,omitempty"`
+	Firmware       Firmware      `json:"firmware,omitempty"`
+	Autoupdater    Autoupdater   `json:"autoupdater"`
+	Nproc          int           `json:"nproc"`
+	Model          string        `json:"model,omitempty"`
+	VPN            bool          `json:"vpn"`
+}
+
+// Firmware out of software
+type Firmware struct {
+	Base    string `json:"base,omitempty"`
+	Release string `json:"release,omitempty"`
+}
+
+// Autoupdater
+type Autoupdater struct {
+	Enabled bool   `json:"enabled"`
+	Branch  string `json:"branch,omitempty"`
+}
+
+// Network struct
+type Network struct {
+	MAC       string   `json:"mac"`
+	Addresses []string `json:"addresses"`
+}
+
+// Location struct
+type Location struct {
+	Longtitude float64 `json:"longitude,omitempty"`
+	Latitude   float64 `json:"latitude,omitempty"`
+}
+
+// Link
+type Link struct {
+	Type     string  `json:"type"`
+	Source   string  `json:"source"`
+	Target   string  `json:"target"`
+	SourceTQ float32 `json:"source_tq"`
+	TargetTQ float32 `json:"target_tq"`
+	// keep the logic for maybe later implementation
+	SourceMAC string `json:"-"`
+	TargetMAC string `json:"-"`
+}
+
+func NewNode(nodes *runtime.Nodes, n *runtime.Node) *Node {
+	node := &Node{
+		Firstseen: n.Firstseen,
+		Lastseen:  n.Lastseen,
+		IsOnline:  n.Online,
+		IsGateway: n.IsGateway(),
+	}
+
+	if nodeinfo := n.Nodeinfo; nodeinfo != nil {
+		node.NodeID = nodeinfo.NodeID
+		node.Network = Network{
+			MAC:       nodeinfo.Network.Mac,
+			Addresses: nodeinfo.Network.Addresses,
+		}
+		node.SiteCode = nodeinfo.System.SiteCode
+		node.Hostname = nodeinfo.Hostname
+		if location := nodeinfo.Location; location != nil {
+			node.Location = &Location{
+				Longtitude: location.Longtitude,
+				Latitude:   location.Latitude,
+			}
+		}
+		node.Firmware = nodeinfo.Software.Firmware
+		node.Autoupdater = Autoupdater{
+			Enabled: nodeinfo.Software.Autoupdater.Enabled,
+			Branch:  nodeinfo.Software.Autoupdater.Branch,
+		}
+		node.Nproc = nodeinfo.Hardware.Nproc
+		node.Model = nodeinfo.Hardware.Model
+		node.VPN = nodeinfo.VPN
+	}
+	if statistic := n.Statistics; statistic != nil {
+		node.Clients = statistic.Clients.Total
+		if node.Clients == 0 {
+			node.Clients = statistic.Clients.Wifi24 + statistic.Clients.Wifi5
+		}
+		node.ClientsWifi24 = statistic.Clients.Wifi24
+		node.ClientsWifi5 = statistic.Clients.Wifi5
+
+		wifi := node.ClientsWifi24 - node.ClientsWifi5
+		if node.Clients >= wifi {
+			node.ClientsOthers = node.Clients - wifi
+		}
+
+		node.RootFSUsage = statistic.RootFsUsage
+		node.LoadAverage = statistic.LoadAverage
+
+		/* The Meshviewer could not handle absolute memory output
+		 * calc the used memory as a float which 100% equal 1.0
+		 * calc is coppied from node statuspage (look discussion:
+		 * https://github.com/FreifunkBremen/yanic/issues/35)
+		 */
+		if statistic.Memory.Total > 0 {
+			usage := 1 - (float64(statistic.Memory.Free)+float64(statistic.Memory.Buffers)+float64(statistic.Memory.Cached))/float64(statistic.Memory.Total)
+			node.MemoryUsage = &usage
+		}
+
+		node.Uptime = jsontime.Now().Add(time.Duration(statistic.Uptime) * -time.Second)
+		node.GatewayNexthop = nodes.GetNodeIDbyMAC(statistic.GatewayNexthop)
+		if node.GatewayNexthop == "" {
+			node.GatewayNexthop = statistic.GatewayNexthop
+		}
+		node.GatewayIPv4 = nodes.GetNodeIDbyMAC(statistic.GatewayIPv4)
+		if node.GatewayIPv4 == "" {
+			node.GatewayIPv4 = statistic.GatewayIPv4
+		}
+		node.GatewayIPv6 = nodes.GetNodeIDbyMAC(statistic.GatewayIPv6)
+		if node.GatewayIPv6 == "" {
+			node.GatewayIPv6 = statistic.GatewayIPv6
+		}
+	}
+
+	return node
+}

output/meshviewer-ffrgb/struct_test.go

@@ -0,0 +1,50 @@
+package meshviewerFFRGB
+
+import (
+	"testing"
+
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRegister(t *testing.T) {
+	assert := assert.New(t)
+	nodes := runtime.NewNodes(&runtime.Config{})
+	node := NewNode(nodes, &runtime.Node{
+		Nodeinfo: &data.NodeInfo{
+			Network: data.Network{
+				Mac: "blub",
+			},
+			Location: &data.Location{
+				Longtitude: 13.3,
+				Latitude:   8.7,
+			},
+		},
+		Statistics: &data.Statistics{
+			Memory: data.Memory{
+				Free:  13,
+				Total: 50,
+			},
+			Wireless: []*data.WirelessAirtime{
+				&data.WirelessAirtime{
+					ChanUtil:  0.3,
+					Frequency: 2512,
+				},
+				&data.WirelessAirtime{
+					ChanUtil:  0.4,
+					Frequency: 2612,
+				},
+				&data.WirelessAirtime{
+					ChanUtil:  0.5,
+					Frequency: 5200,
+				},
+			},
+		},
+	})
+	assert.NotNil(node)
+	assert.Equal("blub", node.Network.MAC)
+	assert.Equal(13.3, node.Location.Longtitude)
+	assert.Equal(8.7, node.Location.Latitude)
+	assert.Equal(0.74, *node.MemoryUsage)
+}

output/meshviewer/graph.go

@@ -35,10 +35,9 @@ type GraphLink struct {
 
 // GraphBuilder a temporaty struct during fill the graph from the node neighbours
 type graphBuilder struct {
-	macToID map[string]string      // mapping from MAC address to node id
+	macToID map[string]string     // mapping from MAC address to node id
-	idToMac map[string]string      // mapping from node id to one MAC address
+	idToMac map[string]string     // mapping from node id to one MAC address
-	links   map[string]*GraphLink  // mapping from $idA-$idB to existing link
+	links   map[string]*GraphLink // mapping from $idA-$idB to existing link
-	vpn     map[string]interface{} // IDs/addresses of VPN servers
 }
 
 // BuildGraph transform from nodes (Neighbours) to Graph
@@ -47,7 +46,6 @@ func BuildGraph(nodes *runtime.Nodes) *Graph {
 		macToID: make(map[string]string),
 		idToMac: make(map[string]string),
 		links:   make(map[string]*GraphLink),
-		vpn:     make(map[string]interface{}),
 	}
 
 	builder.readNodes(nodes.List)
@@ -62,12 +60,8 @@ func (builder *graphBuilder) readNodes(nodes map[string]*runtime.Node) {
 	// Fill mac->id map
 	for sourceID, node := range nodes {
 		if nodeinfo := node.Nodeinfo; nodeinfo != nil {
-			// is VPN address?
-			if nodeinfo.VPN {
-				builder.vpn[sourceID] = nil
-			}
 
-			if len(nodeinfo.Network.Mac) > 0 {
+			if nodeinfo.Network.Mac != "" {
 				builder.idToMac[sourceID] = nodeinfo.Network.Mac
 			}
 
@@ -92,12 +86,21 @@ func (builder *graphBuilder) readNodes(nodes map[string]*runtime.Node) {
 	// Add links
 	for sourceID, node := range nodes {
 		if node.Online {
+			vpnInterface := make(map[string]interface{})
+			if nodeinfo := node.Nodeinfo; nodeinfo != nil {
+				for _, batinterface := range nodeinfo.Network.Mesh {
+					for _, vpn := range batinterface.Interfaces.Tunnel {
+						vpnInterface[vpn] = nil
+					}
+				}
+			}
 			if neighbours := node.Neighbours; neighbours != nil {
 				// Batman neighbours
-				for _, batadvNeighbours := range neighbours.Batadv {
+				for sourceMAC, batadvNeighbours := range neighbours.Batadv {
 					for targetAddress, link := range batadvNeighbours.Neighbours {
 						if targetID, found := builder.macToID[targetAddress]; found {
-							builder.addLink(targetID, sourceID, link.Tq)
+							_, vpn := vpnInterface[sourceMAC]
+							builder.addLink(targetID, sourceID, link.Tq, vpn)
 						}
 					}
 				}
@@ -105,7 +108,7 @@ func (builder *graphBuilder) readNodes(nodes map[string]*runtime.Node) {
 				for _, neighbours := range neighbours.LLDP {
 					for targetAddress := range neighbours {
 						if targetID, found := builder.macToID[targetAddress]; found {
-							builder.addLink(targetID, sourceID, 255)
+							builder.addLink(targetID, sourceID, 255, false)
 						}
 					}
 				}
@@ -159,16 +162,7 @@ func (builder *graphBuilder) extract() ([]*GraphNode, []*GraphLink) {
 	return cache.Nodes, links
 }
 
-func (builder *graphBuilder) isVPN(ids ...string) bool {
+func (builder *graphBuilder) addLink(targetID string, sourceID string, linkTq int, vpn bool) {
-	for _, id := range ids {
-		if _, found := builder.vpn[id]; found {
-			return true
-		}
-	}
-	return false
-}
-
-func (builder *graphBuilder) addLink(targetID string, sourceID string, linkTq int) {
 	// Sort IDs to generate the key
 	var key string
 	if strings.Compare(sourceID, targetID) > 0 {
@@ -184,7 +178,7 @@ func (builder *graphBuilder) addLink(targetID string, sourceID string, linkTq in
 
 	if link, ok := builder.links[key]; !ok {
 		builder.links[key] = &GraphLink{
-			VPN: builder.isVPN(sourceID, targetID),
+			VPN: vpn,
 			TQ:  tq,
 		}
 	} else {

output/meshviewer/graph_test.go

@@ -56,7 +56,7 @@ func testGetNodeByFile(filename string) *runtime.Node {
 }
 
 func testfile(name string, obj interface{}) {
-	file, err := ioutil.ReadFile("../runtime/testdata/" + name)
+	file, err := ioutil.ReadFile("../../runtime/testdata/" + name)
 	if err != nil {
 		panic(err)
 	}

output/meshviewer/nodes_test.go

@@ -25,37 +25,45 @@ func TestNodesV2(t *testing.T) {
 func createTestNodes() *runtime.Nodes {
 	nodes := runtime.NewNodes(&runtime.Config{})
 
-	nodeData := &data.ResponseData{
+	nodeData := &runtime.Node{
 		Statistics: &data.Statistics{
 			Clients: data.Clients{
 				Total: 23,
 			},
 		},
-		NodeInfo: &data.NodeInfo{
+		Nodeinfo: &data.NodeInfo{
+			NodeID: "abcdef012345",
 			Hardware: data.Hardware{
 				Model: "TP-Link 841",
 			},
 		},
 	}
-	nodeData.NodeInfo.Software.Firmware.Release = "2016.1.6+entenhausen1"
+	nodeData.Nodeinfo.Software.Firmware.Release = "2016.1.6+entenhausen1"
-	nodes.Update("abcdef012345", nodeData)
+	nodes.AddNode(nodeData)
 
-	nodes.Update("112233445566", &data.ResponseData{
+	nodes.AddNode(&runtime.Node{
 		Statistics: &data.Statistics{
 			Clients: data.Clients{
-				Total: 2,
+				Wifi24: 2,
+				Wifi5:  3,
+			},
+			Memory: data.Memory{
+				Total: 32,
+				Free:  8,
 			},
 		},
-		NodeInfo: &data.NodeInfo{
+		Nodeinfo: &data.NodeInfo{
+			NodeID: "112233445566",
 			Hardware: data.Hardware{
 				Model: "TP-Link 841",
 			},
 		},
 	})
 
-	nodes.Update("0xdeadbeef0x", &data.ResponseData{
+	nodes.AddNode(&runtime.Node{
-		NodeInfo: &data.NodeInfo{
+		Nodeinfo: &data.NodeInfo{
-			VPN: true,
+			NodeID: "0xdeadbeef0x",
+			VPN:    true,
 			Hardware: data.Hardware{
 				Model: "Xeon Multi-Core",
 			},

output/meshviewer/nodes_v1.go

@@ -21,9 +21,7 @@ func BuildNodesV1(nodes *runtime.Nodes) interface{} {
 		Timestamp: jsontime.Now(),
 	}
 
-	for nodeID := range nodes.List {
+	for nodeID, nodeOrigin := range nodes.List {
-		nodeOrigin := nodes.List[nodeID]
-
 		if nodeOrigin.Statistics == nil {
 			continue
 		}

output/meshviewer/nodes_v2.go

@@ -20,8 +20,7 @@ func BuildNodesV2(nodes *runtime.Nodes) interface{} {
 		Timestamp: jsontime.Now(),
 	}
 
-	for nodeID := range nodes.List {
+	for _, nodeOrigin := range nodes.List {
-		nodeOrigin := nodes.List[nodeID]
 		if nodeOrigin.Statistics == nil {
 			continue
 		}

output/meshviewer/output.go

@@ -0,0 +1,72 @@
+package meshviewer
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/FreifunkBremen/yanic/output"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+type Output struct {
+	output.Output
+	config  Config
+	builder nodeBuilder
+}
+
+type Config map[string]interface{}
+
+func (c Config) Version() int64 {
+	if v := c["version"]; v != nil {
+		return v.(int64)
+	}
+	return -1
+}
+func (c Config) NodesPath() string {
+	if c["nodes_path"] == nil {
+		log.Panic("in configuration of [[nodes.output.meshviewer]] was no nodes_path defined:\n", c)
+	}
+	return c["nodes_path"].(string)
+}
+func (c Config) GraphPath() string {
+	return c["graph_path"].(string)
+}
+
+type nodeBuilder func(*runtime.Nodes) interface{}
+
+var nodeFormats = map[int64]nodeBuilder{
+	1: BuildNodesV1,
+	2: BuildNodesV2,
+}
+
+func init() {
+	output.RegisterAdapter("meshviewer", Register)
+}
+
+func Register(configuration map[string]interface{}) (output.Output, error) {
+	var config Config
+	config = configuration
+
+	builder := nodeFormats[config.Version()]
+	if builder == nil {
+		return nil, fmt.Errorf("invalid nodes version: %d", config.Version())
+	}
+
+	return &Output{
+		config:  config,
+		builder: builder,
+	}, nil
+}
+
+func (o *Output) Save(nodes *runtime.Nodes) {
+	nodes.RLock()
+	defer nodes.RUnlock()
+
+	if path := o.config.NodesPath(); path != "" {
+		runtime.SaveJSON(o.builder(nodes), path)
+	}
+
+	if path := o.config.GraphPath(); path != "" {
+		runtime.SaveJSON(BuildGraph(nodes), path)
+	}
+}

output/meshviewer/output_test.go

@@ -0,0 +1,44 @@
+package meshviewer
+
+import (
+	"os"
+	"testing"
+
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestOutput(t *testing.T) {
+	assert := assert.New(t)
+
+	// no version defined
+	out, err := Register(map[string]interface{}{})
+	assert.Error(err)
+	assert.Nil(out)
+
+	// no nodes path defined
+	out, err = Register(map[string]interface{}{
+		"version": int64(1),
+	})
+	assert.NoError(err)
+	assert.NotNil(out)
+	assert.Panics(func() {
+		out.Save(&runtime.Nodes{})
+	})
+
+	out, err = Register(map[string]interface{}{
+		"version":    int64(2),
+		"nodes_path": "/tmp/nodes.json",
+		"graph_path": "/tmp/graph.json",
+	})
+	os.Remove("/tmp/nodes.json")
+	os.Remove("/tmp/graph.json")
+	assert.NoError(err)
+	assert.NotNil(out)
+
+	out.Save(&runtime.Nodes{})
+	_, err = os.Stat("/tmp/nodes.json")
+	assert.NoError(err)
+	_, err = os.Stat("/tmp/graph.json")
+	assert.NoError(err)
+}

output/nodelist/nodelist.go

@@ -0,0 +1,63 @@
+package nodelist
+
+import (
+	"github.com/FreifunkBremen/yanic/jsontime"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+// NodeList rewritten after: https://github.com/ffnord/ffmap-backend/blob/c33ebf62f013e18bf71b5a38bd058847340db6b7/lib/nodelist.py
+type NodeList struct {
+	Version   string        `json:"version"`
+	Timestamp jsontime.Time `json:"updated_at"` // Timestamp of the generation
+	List      []*Node       `json:"nodes"`
+}
+
+type Node struct {
+	ID       string    `json:"id"`
+	Name     string    `json:"name"`
+	Position *Position `json:"position,omitempty"`
+	Status   struct {
+		Online      bool          `json:"online"`
+		LastContact jsontime.Time `json:"lastcontact"`
+		Clients     uint32        `json:"clients"`
+	} `json:"status"`
+}
+
+type Position struct {
+	Lat  float64 `json:"lat"`
+	Long float64 `json:"long"`
+}
+
+func NewNode(n *runtime.Node) (node *Node) {
+	if nodeinfo := n.Nodeinfo; nodeinfo != nil {
+		node = &Node{
+			ID:   nodeinfo.NodeID,
+			Name: nodeinfo.Hostname,
+		}
+		if location := nodeinfo.Location; location != nil {
+			node.Position = &Position{Lat: location.Latitude, Long: location.Longtitude}
+		}
+
+		node.Status.Online = n.Online
+		node.Status.LastContact = n.Lastseen
+		if statistics := n.Statistics; statistics != nil {
+			node.Status.Clients = statistics.Clients.Total
+		}
+	}
+	return
+}
+
+func transform(nodes *runtime.Nodes) *NodeList {
+	nodelist := &NodeList{
+		Version:   "1.0.1",
+		Timestamp: jsontime.Now(),
+	}
+
+	for _, nodeOrigin := range nodes.List {
+		node := NewNode(nodeOrigin)
+		if node != nil {
+			nodelist.List = append(nodelist.List, node)
+		}
+	}
+	return nodelist
+}

output/nodelist/nodelist_test.go

@@ -0,0 +1,67 @@
+package nodelist
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+func TestTransform(t *testing.T) {
+	nodes := transform(createTestNodes())
+
+	assert := assert.New(t)
+	assert.Len(nodes.List, 3)
+}
+
+func createTestNodes() *runtime.Nodes {
+	nodes := runtime.NewNodes(&runtime.Config{})
+
+	nodeData := &runtime.Node{
+		Statistics: &data.Statistics{
+			Clients: data.Clients{
+				Total: 23,
+			},
+		},
+		Nodeinfo: &data.NodeInfo{
+			NodeID: "abcdef012345",
+			Hardware: data.Hardware{
+				Model: "TP-Link 841",
+			},
+		},
+	}
+	nodeData.Nodeinfo.Software.Firmware.Release = "2016.1.6+entenhausen1"
+	nodes.AddNode(nodeData)
+
+	nodes.AddNode(&runtime.Node{
+		Statistics: &data.Statistics{
+			Clients: data.Clients{
+				Total: 2,
+			},
+		},
+		Nodeinfo: &data.NodeInfo{
+			NodeID: "112233445566",
+			Hardware: data.Hardware{
+				Model: "TP-Link 841",
+			},
+			Location: &data.Location{
+				Latitude:   23,
+				Longtitude: 2,
+			},
+		},
+	})
+
+	nodes.AddNode(&runtime.Node{
+		Nodeinfo: &data.NodeInfo{
+			NodeID: "0xdeadbeef0x",
+			VPN:    true,
+			Hardware: data.Hardware{
+				Model: "Xeon Multi-Core",
+			},
+		},
+	})
+
+	return nodes
+}

output/nodelist/output.go

@@ -0,0 +1,46 @@
+package nodelist
+
+import (
+	"errors"
+
+	"github.com/FreifunkBremen/yanic/output"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+type Output struct {
+	output.Output
+	path string
+}
+
+type Config map[string]interface{}
+
+func (c Config) Path() string {
+	if path, ok := c["path"]; ok {
+		return path.(string)
+	}
+	return ""
+}
+
+func init() {
+	output.RegisterAdapter("nodelist", Register)
+}
+
+func Register(configuration map[string]interface{}) (output.Output, error) {
+	var config Config
+	config = configuration
+
+	if path := config.Path(); path != "" {
+		return &Output{
+			path: path,
+		}, nil
+	}
+	return nil, errors.New("no path given")
+
+}
+
+func (o *Output) Save(nodes *runtime.Nodes) {
+	nodes.RLock()
+	defer nodes.RUnlock()
+
+	runtime.SaveJSON(transform(nodes), o.path)
+}

output/nodelist/output_test.go

@@ -0,0 +1,28 @@
+package nodelist
+
+import (
+	"os"
+	"testing"
+
+	"github.com/FreifunkBremen/yanic/runtime"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestOutput(t *testing.T) {
+	assert := assert.New(t)
+
+	out, err := Register(map[string]interface{}{})
+	assert.Error(err)
+	assert.Nil(out)
+
+	out, err = Register(map[string]interface{}{
+		"path": "/tmp/nodelist.json",
+	})
+	os.Remove("/tmp/nodelist.json")
+	assert.NoError(err)
+	assert.NotNil(out)
+
+	out.Save(&runtime.Nodes{})
+	_, err = os.Stat("/tmp/nodelist.json")
+	assert.NoError(err)
+}

output/output.go

@@ -0,0 +1,19 @@
+package output
+
+import "github.com/FreifunkBremen/yanic/runtime"
+
+// Output interface to use for implementation in e.g. influxdb
+type Output interface {
+	// InsertNode stores statistics per node
+	Save(nodes *runtime.Nodes)
+}
+
+// Register function with config to get a output interface
+type Register func(config map[string]interface{}) (Output, error)
+
+// Adapters is the list of registered output adapters
+var Adapters = map[string]Register{}
+
+func RegisterAdapter(name string, n Register) {
+	Adapters[name] = n
+}

output/output_test.go

@@ -0,0 +1,18 @@
+package output
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRegister(t *testing.T) {
+	assert := assert.New(t)
+	assert.Len(Adapters, 0)
+
+	RegisterAdapter("blub", func(config map[string]interface{}) (Output, error) {
+		return nil, nil
+	})
+
+	assert.Len(Adapters, 1)
+}

respond/collector.go

@@ -256,9 +256,11 @@ func (coll *Collector) saveResponse(addr *net.UDPAddr, res *data.ResponseData) {
 
 		// Store link data
 		if neighbours := node.Neighbours; neighbours != nil {
+			coll.nodes.RLock()
 			for _, link := range coll.nodes.NodeLinks(node) {
 				db.InsertLink(&link, node.Lastseen.GetTime())
 			}
+			coll.nodes.RUnlock()
 		}
 	}
 }

runtime/config.go

@@ -26,6 +26,7 @@ type Config struct {
 		SaveInterval Duration `toml:"save_interval"` // Save nodes periodically
 		OfflineAfter Duration `toml:"offline_after"` // Set node to offline if not seen within this period
 		PruneAfter   Duration `toml:"prune_after"`   // Remove nodes after n days of inactivity
+		Output       map[string]interface{}
 	}
 	Meshviewer struct {
 		Version   int    `toml:"version"`

runtime/config_test.go

@@ -18,13 +18,18 @@ func TestReadConfig(t *testing.T) {
 	assert.Equal([]string{"eth0"}, config.Respondd.Interfaces)
 	assert.Equal(time.Minute, config.Respondd.CollectInterval.Duration)
 
-	assert.Equal(2, config.Meshviewer.Version)
-	assert.Equal("/var/www/html/meshviewer/data/nodes.json", config.Meshviewer.NodesPath)
-
 	assert.Equal(time.Hour*24*7, config.Nodes.PruneAfter.Duration)
 
 	assert.Equal(time.Hour*24*7, config.Database.DeleteAfter.Duration)
 
+	var meshviewer map[string]interface{}
+	var outputs []map[string]interface{}
+	outputs = config.Nodes.Output["meshviewer"].([]map[string]interface{})
+	assert.Len(outputs, 1, "more outputs are given")
+	meshviewer = outputs[0]
+	assert.Equal(int64(2), meshviewer["version"])
+	assert.Equal("/var/www/html/meshviewer/data/nodes.json", meshviewer["nodes_path"])
+
 	var influxdb map[string]interface{}
 	dbs := config.Database.Connection["influxdb"]
 	assert.Len(dbs, 1, "more influxdb are given")

runtime/nodes.go

@@ -39,6 +39,17 @@ func (nodes *Nodes) Start() {
 	go nodes.worker()
 }
 
+func (nodes *Nodes) AddNode(node *Node) {
+	nodeinfo := node.Nodeinfo
+	if nodeinfo == nil || nodeinfo.NodeID == "" {
+		return
+	}
+	nodes.Lock()
+	defer nodes.Unlock()
+	nodes.List[nodeinfo.NodeID] = node
+	nodes.readIfaces(nodeinfo)
+}
+
 // Update a Node
 func (nodes *Nodes) Update(nodeID string, res *data.ResponseData) *Node {
 	now := jsontime.Now()
@@ -52,6 +63,9 @@ func (nodes *Nodes) Update(nodeID string, res *data.ResponseData) *Node {
 		}
 		nodes.List[nodeID] = node
 	}
+	if res.NodeInfo != nil {
+		nodes.readIfaces(res.NodeInfo)
+	}
 	nodes.Unlock()
 
 	// Update wireless statistics
@@ -69,10 +83,6 @@ func (nodes *Nodes) Update(nodeID string, res *data.ResponseData) *Node {
 	node.Nodeinfo = res.NodeInfo
 	node.Statistics = res.Statistics
 
-	if node.Nodeinfo != nil {
-		nodes.readIfaces(node.Nodeinfo)
-	}
-
 	return node
 }
 
@@ -90,6 +100,10 @@ func (nodes *Nodes) Select(f func(*Node) bool) []*Node {
 	return result
 }
 
+func (nodes *Nodes) GetNodeIDbyMAC(mac string) string {
+	return nodes.ifaceToNodeID[mac]
+}
+
 // NodeLinks returns a list of links to known neighbours
 func (nodes *Nodes) NodeLinks(node *Node) (result []Link) {
 	// Store link data
@@ -98,9 +112,6 @@ func (nodes *Nodes) NodeLinks(node *Node) (result []Link) {
 		return
 	}
 
-	nodes.RLock()
-	defer nodes.RUnlock()
-
 	for sourceMAC, batadv := range neighbours.Batadv {
 		for neighbourMAC, link := range batadv.Neighbours {
 			if neighbourID := nodes.ifaceToNodeID[neighbourMAC]; neighbourID != "" {
@@ -165,8 +176,6 @@ func (nodes *Nodes) readIfaces(nodeinfo *data.NodeInfo) {
 		log.Println("nodeID missing in nodeinfo")
 		return
 	}
-	nodes.Lock()
-	defer nodes.Unlock()
 
 	addresses := []string{network.Mac}
 
@@ -191,11 +200,13 @@ func (nodes *Nodes) load() {
 		if err = json.NewDecoder(f).Decode(nodes); err == nil {
 			log.Println("loaded", len(nodes.List), "nodes")
 
+			nodes.Lock()
 			for _, node := range nodes.List {
 				if node.Nodeinfo != nil {
 					nodes.readIfaces(node.Nodeinfo)
 				}
 			}
+			nodes.Unlock()
 
 		} else {
 			log.Println("failed to unmarshal nodes:", err)

runtime/nodes_test.go

@@ -137,6 +137,20 @@ func TestSelectNodes(t *testing.T) {
 	assert.Equal(time, selectedNodes[0].Firstseen)
 }
 
+func TestAddNode(t *testing.T) {
+	assert := assert.New(t)
+	nodes := NewNodes(&Config{})
+
+	nodes.AddNode(&Node{})
+	assert.Len(nodes.List, 0)
+
+	nodes.AddNode(&Node{Nodeinfo: &data.NodeInfo{}})
+	assert.Len(nodes.List, 0)
+
+	nodes.AddNode(&Node{Nodeinfo: &data.NodeInfo{NodeID: "blub"}})
+	assert.Len(nodes.List, 1)
+}
+
 func TestLinksNodes(t *testing.T) {
 	assert := assert.New(t)
 
@@ -188,4 +202,7 @@ func TestLinksNodes(t *testing.T) {
 	assert.Equal(link.TargetID, "f4f26dd7a30a")
 	assert.Equal(link.TargetMAC, "f4:f2:6d:d7:a3:0a")
 	assert.Equal(link.TQ, 200)
+
+	nodeid := nodes.GetNodeIDbyMAC("f4:f2:6d:d7:a3:0a")
+	assert.Equal("f4f26dd7a30a", nodeid)
 }

runtime/stats.go

@@ -23,7 +23,7 @@ func NewGlobalStats(nodes *Nodes) (result *GlobalStats) {
 		Models:    make(CounterMap),
 	}
 
-	nodes.Lock()
+	nodes.RLock()
 	for _, node := range nodes.List {
 		if node.Online {
 			result.Nodes++
@@ -42,7 +42,7 @@ func NewGlobalStats(nodes *Nodes) (result *GlobalStats) {
 			}
 		}
 	}
-	nodes.Unlock()
+	nodes.RUnlock()
 	return
 }
 

runtime/stats_test.go

@@ -29,37 +29,43 @@ func TestGlobalStats(t *testing.T) {
 func createTestNodes() *Nodes {
 	nodes := NewNodes(&Config{})
 
-	nodeData := &data.ResponseData{
+	nodeData := &Node{
+		Online: true,
 		Statistics: &data.Statistics{
 			Clients: data.Clients{
 				Total: 23,
 			},
 		},
-		NodeInfo: &data.NodeInfo{
+		Nodeinfo: &data.NodeInfo{
+			NodeID: "abcdef012345",
 			Hardware: data.Hardware{
 				Model: "TP-Link 841",
 			},
 		},
 	}
-	nodeData.NodeInfo.Software.Firmware.Release = "2016.1.6+entenhausen1"
+	nodeData.Nodeinfo.Software.Firmware.Release = "2016.1.6+entenhausen1"
-	nodes.Update("abcdef012345", nodeData)
+	nodes.AddNode(nodeData)
 
-	nodes.Update("112233445566", &data.ResponseData{
+	nodes.AddNode(&Node{
+		Online: true,
 		Statistics: &data.Statistics{
 			Clients: data.Clients{
 				Total: 2,
 			},
 		},
-		NodeInfo: &data.NodeInfo{
+		Nodeinfo: &data.NodeInfo{
+			NodeID: "112233445566",
 			Hardware: data.Hardware{
 				Model: "TP-Link 841",
 			},
 		},
 	})
 
-	nodes.Update("0xdeadbeef0x", &data.ResponseData{
+	nodes.AddNode(&Node{
-		NodeInfo: &data.NodeInfo{
+		Online: true,
-			VPN: true,
+		Nodeinfo: &data.NodeInfo{
+			NodeID: "0xdeadbeef0x",
+			VPN:    true,
 			Hardware: data.Hardware{
 				Model: "Xeon Multi-Core",
 			},

runtime/testdata/node1.json

@@ -2,10 +2,12 @@
 	"nodeinfo":{
 		"node_id":"node1.json",
 		"network":{
+			"mac": "a",
 			"mesh":{
 				"bat0":{
 					"interfaces":{
-						"wireless":["a"]
+						"wireless":["a"],
+						"other":["a2"]
 					}
 				}
 			}
@@ -15,10 +17,13 @@
 		"batadv":{
 			"a":{
 					"neighbours":{
-						"b":{"tq":250,"lastseen":0.42},
+						"b":{"tq":150,"lastseen":0.42},
 						"c":{"tq":250,"lastseen":0.42}
 					}
 				}
+		},
+		"lldp":{
+			"a2": {"c2": {}}
 		}
 	}
 

runtime/testdata/node2.json

@@ -15,7 +15,7 @@
 		"batadv":{
 			"b":{
 					"neighbours":{
-						"a":{"tq":150,"lastseen":0.42}
+						"a":{"tq":250,"lastseen":0.42}
 					}
 				}
 		}

runtime/testdata/node3.json

@@ -5,7 +5,8 @@
 			"mesh":{
 				"bat0":{
 					"interfaces":{
-						"wireless":["c"]
+						"wireless":["c"],
+						"other":["c2"]
 					}
 				}
 			}
@@ -18,6 +19,9 @@
 						"a":{"tq":200,"lastseen":0.42}
 					}
 				}
+		},
+		"lldp":{
+			"c2": {"a2": {}}
 		}
 	}