add support for custom fields

At the moment, if one has a custom respondd module which includes custom
fields, Yanic will simply ignore these fields. Communities which have custom
fields have to maintain patches on Yanic to have them available.

This commit allows to define custom fields in the configuration file, which
will cause Yanic to also save the values of these custom fields in its internal
data structures. Output modules can then decide whether they want to include
these fields. For most cases, this should avoid the need for patches in Yanic.

Gopkg.toml

@@ -45,6 +45,10 @@
   name = "github.com/spf13/cobra"
   version = "0.0.2"
 
+[[constraint]]
+  name = "github.com/tidwall/gjson"
+  version = "1.3.4"
+
 [[constraint]]
   name = "github.com/stretchr/testify"
   version = "1.2.1"

config_example.toml

@@ -10,6 +10,16 @@ synchronize      = "1m"
 # how often request per multicast
 collect_interval = "1m"
 
+# If you have custom respondd fields, you can ask Yanic to also collect these.
+# NOTE: This does not automatically include these fields in the output.
+#       The meshviewer-ffrgb output module will include them under "custom_fields",
+#       but other modules may simply ignore them.
+#[[respondd.custom_field]]
+#name = zip
+# You can use arbitrary GJSON expressions here, see https://github.com/tidwall/gjson
+# We expect this expression to return a string.
+#path = nodeinfo.location.zip
+
 # table of a site to save stats for (not exists for global only)
 #[respondd.sites.example]
 ## list of domains on this site to save stats for (empty for global only)

data/response.go

@@ -2,7 +2,8 @@ package data
 
 // ResponseData struct
 type ResponseData struct {
-	Neighbours *Neighbours `json:"neighbours"`
-	Nodeinfo   *Nodeinfo   `json:"nodeinfo"`
-	Statistics *Statistics `json:"statistics"`
+	Neighbours   *Neighbours            `json:"neighbours"`
+	Nodeinfo     *Nodeinfo              `json:"nodeinfo"`
+	Statistics   *Statistics            `json:"statistics"`
+	CustomFields map[string]interface{} `json:"-"`
 }

output/filter/noowner/noowner.go

@@ -41,7 +41,8 @@ func (no *noowner) Apply(node *runtime.Node) *runtime.Node {
 				VPN:      nodeinfo.VPN,
 				Wireless: nodeinfo.Wireless,
 			},
-			Neighbours: node.Neighbours,
+			Neighbours:   node.Neighbours,
+			CustomFields: node.CustomFields,
 		}
 	}
 	return node

respond/collector.go

@@ -5,10 +5,12 @@ import (
 	"compress/flate"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"time"
 
 	"github.com/bdlm/log"
+	"github.com/tidwall/gjson"
 
 	"github.com/FreifunkBremen/yanic/data"
 	"github.com/FreifunkBremen/yanic/database"
@@ -237,7 +239,7 @@ func (coll *Collector) sender() {
 
 func (coll *Collector) parser() {
 	for obj := range coll.queue {
-		if data, err := obj.parse(); err != nil {
+		if data, err := obj.parse(coll.config.CustomFields); err != nil {
 			log.WithField("address", obj.Address.String()).Errorf("unable to decode response %s", err)
 		} else {
 			coll.saveResponse(obj.Address, data)
@@ -245,14 +247,32 @@ func (coll *Collector) parser() {
 	}
 }
 
-func (res *Response) parse() (*data.ResponseData, error) {
+func (res *Response) parse(customFields []CustomFieldConfig) (*data.ResponseData, error) {
 	// Deflate
 	deflater := flate.NewReader(bytes.NewReader(res.Raw))
 	defer deflater.Close()
 
+	jsonData, err := ioutil.ReadAll(deflater)
+	if err != nil {
+		return nil, err
+	}
+
 	// Unmarshal
 	rdata := &data.ResponseData{}
-	err := json.NewDecoder(deflater).Decode(rdata)
+	err = json.Unmarshal(jsonData, rdata)
+
+	rdata.CustomFields = make(map[string]interface{})
+	if !gjson.Valid(string(jsonData)) {
+		log.WithField("jsonData", jsonData).Info("JSON data is invalid")
+	} else {
+		jsonParsed := gjson.Parse(string(jsonData))
+		for _, customField := range customFields {
+			field := jsonParsed.Get(customField.Path)
+			if field.Exists() {
+				rdata.CustomFields[customField.Name] = field.String()
+			}
+		}
+	}
 
 	return rdata, err
 }

respond/collector_test.go

@@ -41,10 +41,65 @@ func TestParse(t *testing.T) {
 		Raw: compressed,
 	}
 
-	data, err := res.parse()
+	data, err := res.parse([]CustomFieldConfig{})
 
 	assert.NoError(err)
 	assert.NotNil(data)
 
 	assert.Equal("f81a67a5e9c1", data.Nodeinfo.NodeID)
 }
+
+func TestParseCustomFields(t *testing.T) {
+	assert := assert.New(t)
+
+	// read testdata
+	compressed, err := ioutil.ReadFile("testdata/nodeinfo.flated")
+	assert.Nil(err)
+
+	res := &Response{
+		Raw: compressed,
+	}
+
+	customFields := []CustomFieldConfig{
+		{
+			Name: "my_custom_field",
+			Path: "nodeinfo.hostname",
+		},
+	}
+
+	data, err := res.parse(customFields)
+
+	assert.NoError(err)
+	assert.NotNil(data)
+
+	assert.Equal("Trillian", data.CustomFields["my_custom_field"])
+	assert.Equal("Trillian", data.Nodeinfo.Hostname)
+}
+
+func TestParseCustomFieldNotExistant(t *testing.T) {
+	assert := assert.New(t)
+
+	// read testdata
+	compressed, err := ioutil.ReadFile("testdata/nodeinfo.flated")
+	assert.Nil(err)
+
+	res := &Response{
+		Raw: compressed,
+	}
+
+	customFields := []CustomFieldConfig{
+		{
+			Name: "some_other_field",
+			Path: "nodeinfo.some_field_which_doesnt_exist",
+		},
+	}
+
+	data, err := res.parse(customFields)
+
+	assert.NoError(err)
+	assert.NotNil(data)
+
+	_, ok := data.CustomFields["some_other_field"]
+	assert.Equal("Trillian", data.Nodeinfo.Hostname)
+	assert.False(ok)
+}

respond/config.go

@@ -8,6 +8,7 @@ type Config struct {
 	Interfaces      []InterfaceConfig     `toml:"interfaces"`
 	Sites           map[string]SiteConfig `toml:"sites"`
 	CollectInterval duration.Duration     `toml:"collect_interval"`
+	CustomFields    []CustomFieldConfig   `toml:"custom_field"`
 }
 
 func (c *Config) SitesDomains() (result map[string][]string) {
@@ -29,3 +30,8 @@ type InterfaceConfig struct {
 	MulticastAddress string `toml:"multicast_address"`
 	Port             int    `toml:"port"`
 }
+
+type CustomFieldConfig struct {
+	Name string `toml:"name"`
+	Path string `toml:"path"`
+}

runtime/node.go

@@ -9,13 +9,14 @@ import (
 
 // Node struct
 type Node struct {
-	Address    *net.UDPAddr     `json:"-"` // the last known address
-	Firstseen  jsontime.Time    `json:"firstseen"`
-	Lastseen   jsontime.Time    `json:"lastseen"`
-	Online     bool             `json:"online"`
-	Statistics *data.Statistics `json:"statistics"`
-	Nodeinfo   *data.Nodeinfo   `json:"nodeinfo"`
-	Neighbours *data.Neighbours `json:"-"`
+	Address      *net.UDPAddr           `json:"-"` // the last known address
+	Firstseen    jsontime.Time          `json:"firstseen"`
+	Lastseen     jsontime.Time          `json:"lastseen"`
+	Online       bool                   `json:"online"`
+	Statistics   *data.Statistics       `json:"statistics"`
+	Nodeinfo     *data.Nodeinfo         `json:"nodeinfo"`
+	Neighbours   *data.Neighbours       `json:"-"`
+	CustomFields map[string]interface{} `json:"custom_fields"`
 }
 
 // Link represents a link between two nodes

runtime/nodes.go

@@ -83,6 +83,7 @@ func (nodes *Nodes) Update(nodeID string, res *data.ResponseData) *Node {
 	node.Neighbours = res.Neighbours
 	node.Nodeinfo = res.Nodeinfo
 	node.Statistics = res.Statistics
+	node.CustomFields = res.CustomFields
 
 	return node
 }