first idea of a prometheus output

cmd/config_test.go

@@ -25,7 +25,7 @@ func TestReadConfig(t *testing.T) {
 	assert.Contains(config.Respondd.Sites["ffhb"].Domains, "city")
 
 	// Test output plugins
-	assert.Len(config.Nodes.Output, 4)
+	assert.Len(config.Nodes.Output, 5)
 	outputs := config.Nodes.Output["meshviewer"].([]interface{})
 	assert.Len(outputs, 1)
 	meshviewer := outputs[0]

config_example.toml

@@ -92,6 +92,10 @@ offline_after = "10m"
 #longitude_min = -24.96
 #longitude_max = 39.72
 
+# outputs of all nodes for prometheus exporter
+[[nodes.output.prometheus]]
+enable   = true
+path = "/var/www/html/meshviewer/data/prometheus.txt"
 
 # outputs all nodes as points into nodes.geojson
 [[nodes.output.geojson]]

output/all/main.go

@@ -5,4 +5,5 @@ import (
 	_ "github.com/FreifunkBremen/yanic/output/meshviewer"
 	_ "github.com/FreifunkBremen/yanic/output/meshviewer-ffrgb"
 	_ "github.com/FreifunkBremen/yanic/output/nodelist"
+	_ "github.com/FreifunkBremen/yanic/output/prometheus"
 )

output/prometheus/metric.go

@@ -0,0 +1,38 @@
+package prometheus
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+)
+
+type Metric struct {
+	Name   string
+	Value  interface{}
+	Labels map[string]interface{}
+}
+
+func (m *Metric) String() (string, error) {
+	if m.Value == nil {
+		return "", errors.New("no value of metric found")
+	}
+	output := m.Name
+	if len(m.Labels) > 0 {
+		output += "{"
+		for label, v := range m.Labels {
+			switch value := v.(type) {
+			case string:
+				output = fmt.Sprintf("%s%s=\"%s\",", output, label, strings.ReplaceAll(value, "\"", "'"))
+			case float32:
+				output = fmt.Sprintf("%s%s=\"%.4f\",", output, label, value)
+			case float64:
+				output = fmt.Sprintf("%s%s=\"%.4f\",", output, label, value)
+			default:
+				output = fmt.Sprintf("%s%s=\"%v\",", output, label, value)
+			}
+		}
+		lastChar := len(output) - 1
+		output = output[:lastChar] + "}"
+	}
+	return fmt.Sprintf("%s %v", output, m.Value), nil
+}

output/prometheus/metric_test.go

@@ -0,0 +1,62 @@
+package prometheus
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMetric(t *testing.T) {
+	assert := assert.New(t)
+
+	var tests = []struct {
+		input  Metric
+		err    string
+		output string
+	}{
+		{
+			input: Metric{Name: "test1"},
+			err:   "no value of metric found",
+		},
+		{
+			input:  Metric{Name: "test2", Value: 3},
+			output: "test2 3",
+		},
+		{
+			input: Metric{Name: "test3", Value: 3.2,
+				Labels: map[string]interface{}{
+					"site_code": "lola",
+				},
+			},
+			output: `test3{site_code="lola"} 3.2`,
+		},
+		{
+			input: Metric{Name: "test4", Value: "0",
+				Labels: map[string]interface{}{
+					"frequency": float32(3.2),
+				},
+			},
+			output: `test4{frequency="3.2000"} 0`,
+		},
+		{
+			input: Metric{Name: "test5", Value: 3,
+				Labels: map[string]interface{}{
+					"node_id": "lola",
+					"blub":    3.3423533,
+				},
+			},
+			output: `test5{node_id="lola",blub="3.3424"} 3`,
+		},
+	}
+
+	for _, test := range tests {
+		output, err := test.input.String()
+
+		if test.err == "" {
+			assert.NoError(err)
+			assert.Equal(test.output, output)
+		} else {
+			assert.EqualError(err, test.err)
+		}
+	}
+}

output/prometheus/output.go

@@ -0,0 +1,75 @@
+package prometheus
+
+import (
+	"errors"
+	"os"
+
+	"github.com/bdlm/log"
+
+	"github.com/FreifunkBremen/yanic/output"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+type Output struct {
+	output.Output
+	path string
+}
+
+type Config map[string]interface{}
+
+func (c Config) Path() string {
+	if path, ok := c["path"]; ok {
+		return path.(string)
+	}
+	return ""
+}
+
+func init() {
+	output.RegisterAdapter("prometheus", Register)
+}
+
+func Register(configuration map[string]interface{}) (output.Output, error) {
+	var config Config
+	config = configuration
+
+	if path := config.Path(); path != "" {
+		return &Output{
+			path: path,
+		}, nil
+	}
+	return nil, errors.New("no path given")
+
+}
+
+func (o *Output) Save(nodes *runtime.Nodes) {
+	nodes.RLock()
+	defer nodes.RUnlock()
+
+	tmpFile := o.path + ".tmp"
+
+	f, err := os.OpenFile(tmpFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		log.Panic(err)
+	}
+
+	for _, node := range nodes.List {
+		metrics := MetricsFromNode(nodes, node)
+		for _, m := range metrics {
+			str, err := m.String()
+			if err == nil {
+				f.WriteString(str + "\n")
+			} else {
+				logger := log.WithField("database", "prometheus")
+				if nodeinfo := node.Nodeinfo; nodeinfo != nil {
+					logger = logger.WithField("node_id", nodeinfo.NodeID)
+				}
+				logger.Warnf("not able to get metrics from node: %s", err)
+			}
+		}
+	}
+	f.Close()
+
+	if err := os.Rename(tmpFile, o.path); err != nil {
+		log.Panic(err)
+	}
+}

output/prometheus/output_test.go

@@ -0,0 +1,40 @@
+package prometheus
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+func TestOutput(t *testing.T) {
+	assert := assert.New(t)
+
+	out, err := Register(map[string]interface{}{})
+	assert.Error(err)
+	assert.Nil(out)
+
+	out, err = Register(map[string]interface{}{
+		"path": "/tmp/prometheus.txt",
+	})
+	os.Remove("/tmp/prometheus.txt")
+	assert.NoError(err)
+	assert.NotNil(out)
+
+	out.Save(&runtime.Nodes{
+		List: map[string]*runtime.Node{
+			"wasd": {
+				Online: true,
+				Nodeinfo: &data.Nodeinfo{
+					NodeID: "wasd",
+				},
+				Statistics: &data.Statistics{},
+			},
+		},
+	})
+	_, err = os.Stat("/tmp/prometheus.txt")
+	assert.NoError(err)
+}

output/prometheus/transform.go

@@ -0,0 +1,145 @@
+package prometheus
+
+import (
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+func MetricsFromNode(nodes *runtime.Nodes, node *runtime.Node) []Metric {
+	m := []Metric{}
+
+	// before node metrics to get link statics undependent of node validation
+	for _, link := range nodes.NodeLinks(node) {
+		m = append(m, Metric{
+			Labels: map[string]interface{}{
+				"source_id":   link.SourceID,
+				"source_addr": link.SourceAddress,
+				"target_id":   link.TargetID,
+				"target_addr": link.TargetAddress,
+			},
+			Name:  "yanic_link",
+			Value: link.TQ * 100,
+		})
+	}
+
+	nodeinfo := node.Nodeinfo
+	stats := node.Statistics
+
+	// validation
+	if nodeinfo == nil || stats == nil {
+		return m
+	}
+
+	labels := map[string]interface{}{
+		"node_id":  nodeinfo.NodeID,
+		"hostname": nodeinfo.Hostname,
+	}
+
+	if nodeinfo.System.SiteCode != "" {
+		labels["site_code"] = nodeinfo.System.SiteCode
+	}
+	if nodeinfo.System.DomainCode != "" {
+		labels["domain_code"] = nodeinfo.System.DomainCode
+	}
+	if owner := nodeinfo.Owner; owner != nil {
+		labels["owner"] = owner.Contact
+	}
+	// Hardware
+	labels["model"] = nodeinfo.Hardware.Model
+	labels["nproc"] = nodeinfo.Hardware.Nproc
+	labels["firmware_base"] = nodeinfo.Software.Firmware.Base
+	labels["firmware_release"] = nodeinfo.Software.Firmware.Release
+	if nodeinfo.Software.Autoupdater.Enabled {
+		labels["autoupdater"] = nodeinfo.Software.Autoupdater.Branch
+	} else {
+		labels["autoupdater"] = runtime.DISABLED_AUTOUPDATER
+	}
+
+	if location := nodeinfo.Location; location != nil {
+		labels["location_lat"] = location.Latitude
+		labels["location_long"] = location.Longitude
+	}
+
+	addMetric := func(name string, value interface{}) {
+		m = append(m, Metric{Labels: labels, Name: "yanic_" + name, Value: value})
+	}
+
+	if node.Online {
+		addMetric("node_up", 1)
+	} else {
+		addMetric("node_up", 0)
+	}
+
+	addMetric("node_load", stats.LoadAverage)
+
+	addMetric("node_time_up", stats.Uptime)
+	addMetric("node_time_idle", stats.Idletime)
+
+	addMetric("node_proc_running", stats.Processes.Running)
+
+	addMetric("node_clients_wifi", stats.Clients.Wifi)
+	addMetric("node_clients_wifi24", stats.Clients.Wifi24)
+	addMetric("node_clients_wifi5", stats.Clients.Wifi5)
+	addMetric("node_clients_total", stats.Clients.Total)
+
+	addMetric("node_memory_buffers", stats.Memory.Buffers)
+	addMetric("node_memory_cached", stats.Memory.Cached)
+	addMetric("node_memory_free", stats.Memory.Free)
+	addMetric("node_memory_total", stats.Memory.Total)
+	addMetric("node_memory_available", stats.Memory.Available)
+
+	//TODO Neighbours count after merging improvement in influxdb and graphite
+
+	if procstat := stats.ProcStats; procstat != nil {
+		addMetric("node_stat_cpu_user", procstat.CPU.User)
+		addMetric("node_stat_cpu_nice", procstat.CPU.Nice)
+		addMetric("node_stat_cpu_system", procstat.CPU.System)
+		addMetric("node_stat_cpu_idle", procstat.CPU.Idle)
+		addMetric("node_stat_cpu_iowait", procstat.CPU.IOWait)
+		addMetric("node_stat_cpu_irq", procstat.CPU.IRQ)
+		addMetric("node_stat_cpu_softirq", procstat.CPU.SoftIRQ)
+		addMetric("node_stat_intr", procstat.Intr)
+		addMetric("node_stat_ctxt", procstat.ContextSwitches)
+		addMetric("node_stat_softirq", procstat.SoftIRQ)
+		addMetric("node_stat_processes", procstat.Processes)
+	}
+
+	if t := stats.Traffic.Rx; t != nil {
+		addMetric("node_traffic_rx_bytes", t.Bytes)
+		addMetric("node_traffic_rx_packets", t.Packets)
+	}
+	if t := stats.Traffic.Tx; t != nil {
+		addMetric("node_traffic_tx_bytes", t.Bytes)
+		addMetric("node_traffic_tx_packets", t.Packets)
+		addMetric("node_traffic_tx_dropped", t.Dropped)
+	}
+	if t := stats.Traffic.Forward; t != nil {
+		addMetric("node_traffic_forward_bytes", t.Bytes)
+		addMetric("node_traffic_forward_packets", t.Packets)
+	}
+	if t := stats.Traffic.MgmtRx; t != nil {
+		addMetric("node_traffic_mgmt_rx_bytes", t.Bytes)
+		addMetric("node_traffic_mgmt_rx_packets", t.Packets)
+	}
+	if t := stats.Traffic.MgmtTx; t != nil {
+		addMetric("node_traffic_mgmt_tx_bytes", t.Bytes)
+		addMetric("node_traffic_mgmt_tx_packets", t.Packets)
+	}
+
+	for _, airtime := range stats.Wireless {
+		labels["frequency_name"] = airtime.FrequencyName()
+		addMetric("node_frequency", airtime.Frequency)
+		addMetric("node_airtime_chan_util", airtime.ChanUtil)
+		addMetric("node_airtime_rx_util", airtime.RxUtil)
+		addMetric("node_airtime_tx_util", airtime.TxUtil)
+		addMetric("node_airtime_noise", airtime.Noise)
+		if wireless := nodeinfo.Wireless; wireless != nil {
+			if airtime.Frequency < 5000 {
+				addMetric("node_wireless_txpower", wireless.TxPower24)
+			} else {
+				addMetric("node_wireless_txpower", wireless.TxPower5)
+			}
+		}
+	}
+
+	return m
+}

output/prometheus/transform_test.go

@@ -0,0 +1,124 @@
+package prometheus
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+func TestMetricsFromNode(t *testing.T) {
+	assert := assert.New(t)
+
+	m := MetricsFromNode(nil, &runtime.Node{})
+	assert.Len(m, 0)
+
+	nodes := runtime.NewNodes(&runtime.NodesConfig{})
+	nodes.AddNode(&runtime.Node{
+		Nodeinfo: &data.Nodeinfo{
+			NodeID: "lola",
+			Network: data.Network{
+				Mesh: map[string]*data.NetworkInterface{
+					"mesh1": {
+						Interfaces: struct {
+							Wireless []string `json:"wireless,omitempty"`
+							Other    []string `json:"other,omitempty"`
+							Tunnel   []string `json:"tunnel,omitempty"`
+						}{
+							Tunnel: []string{"fe80::2"},
+						},
+					},
+				},
+			},
+		},
+	})
+
+	node := &runtime.Node{
+		Online: false,
+		Nodeinfo: &data.Nodeinfo{
+			NodeID: "wasd1",
+			Network: data.Network{
+				Mesh: map[string]*data.NetworkInterface{
+					"mesh0": {
+						Interfaces: struct {
+							Wireless []string `json:"wireless,omitempty"`
+							Other    []string `json:"other,omitempty"`
+							Tunnel   []string `json:"tunnel,omitempty"`
+						}{
+							Tunnel: []string{"fe80::1"},
+						},
+					},
+				},
+			},
+			Software: data.Software{
+				Autoupdater: struct {
+					Enabled bool   `json:"enabled,omitempty"`
+					Branch  string `json:"branch,omitempty"`
+				}{
+					Enabled: true,
+					Branch:  "testing",
+				},
+			},
+		},
+		Statistics: &data.Statistics{},
+		Neighbours: &data.Neighbours{
+			NodeID: "wasd1",
+			Babel: map[string]data.BabelNeighbours{
+				"mesh0": {
+					LinkLocalAddress: "fe80::1",
+					Neighbours: map[string]data.BabelLink{
+						"fe80::2": {Cost: 20000},
+					},
+				},
+			},
+		},
+	}
+	nodes.AddNode(node)
+	m = MetricsFromNode(nodes, node)
+	assert.Len(m, 15)
+	assert.Equal(m[0].Labels["source_id"], "wasd1")
+
+	m = MetricsFromNode(nil, &runtime.Node{
+		Online: true,
+		Nodeinfo: &data.Nodeinfo{
+			NodeID: "wasd",
+			System: data.System{
+				SiteCode:   "ffhb",
+				DomainCode: "city",
+			},
+			Owner: &data.Owner{Contact: "mailto:blub@example.org"},
+			Location: &data.Location{
+				Latitude:  52.0,
+				Longitude: 4.0,
+			},
+			Wireless: &data.Wireless{
+				TxPower24: 0,
+			},
+		},
+		Statistics: &data.Statistics{
+			ProcStats: &data.ProcStats{},
+			Traffic: struct {
+				Tx      *data.Traffic `json:"tx"`
+				Rx      *data.Traffic `json:"rx"`
+				Forward *data.Traffic `json:"forward"`
+				MgmtTx  *data.Traffic `json:"mgmt_tx"`
+				MgmtRx  *data.Traffic `json:"mgmt_rx"`
+			}{
+				Tx:      &data.Traffic{},
+				Rx:      &data.Traffic{},
+				Forward: &data.Traffic{},
+				MgmtTx:  &data.Traffic{},
+				MgmtRx:  &data.Traffic{},
+			},
+			Wireless: data.WirelessStatistics{
+				&data.WirelessAirtime{Frequency: 5002},
+				&data.WirelessAirtime{Frequency: 2430},
+			},
+		},
+	})
+
+	assert.Len(m, 48)
+	assert.Equal(m[0].Labels["node_id"], "wasd")
+}