Browse Source

add support for custom fields

At the moment, if one has a custom respondd module which includes custom
fields, Yanic will simply ignore these fields. Communities which have custom
fields have to maintain patches on Yanic to have them available.

This commit allows to define custom fields in the configuration file, which
will cause Yanic to also save the values of these custom fields in its internal
data structures. Output modules can then decide whether they want to include
these fields. For most cases, this should avoid the need for patches in Yanic.
keep-around/5eb75ff9bbaddad7f7e1cb91c2f9624f569c1ff0
nrbffs 1 year ago
committed by genofire
parent
commit
1a1163aaa1
9 changed files with 114 additions and 15 deletions
  1. + 4
    - 0
      Gopkg.toml
  2. + 10
    - 0
      config_example.toml
  3. + 4
    - 3
      data/response.go
  4. + 2
    - 1
      output/filter/noowner/noowner.go
  5. + 23
    - 3
      respond/collector.go
  6. + 56
    - 1
      respond/collector_test.go
  7. + 6
    - 0
      respond/config.go
  8. + 8
    - 7
      runtime/node.go
  9. + 1
    - 0
      runtime/nodes.go

+ 4
- 0
Gopkg.toml

@ -45,6 +45,10 @@
name = "github.com/spf13/cobra"
version = "0.0.2"
[[constraint]]
name = "github.com/tidwall/gjson"
version = "1.3.4"
[[constraint]]
name = "github.com/stretchr/testify"
version = "1.2.1"

+ 10
- 0
config_example.toml

@ -10,6 +10,16 @@ synchronize = "1m"
# how often request per multicast
collect_interval = "1m"
# If you have custom respondd fields, you can ask Yanic to also collect these.
# NOTE: This does not automatically include these fields in the output.
# The meshviewer-ffrgb output module will include them under "custom_fields",
# but other modules may simply ignore them.
#[[respondd.custom_field]]
#name = zip
# You can use arbitrary GJSON expressions here, see https://github.com/tidwall/gjson
# We expect this expression to return a string.
#path = nodeinfo.location.zip
# table of a site to save stats for (not exists for global only)
#[respondd.sites.example]
## list of domains on this site to save stats for (empty for global only)

+ 4
- 3
data/response.go

@ -2,7 +2,8 @@ package data
// ResponseData struct
type ResponseData struct {
Neighbours *Neighbours `json:"neighbours"`
Nodeinfo *Nodeinfo `json:"nodeinfo"`
Statistics *Statistics `json:"statistics"`
Neighbours *Neighbours `json:"neighbours"`
Nodeinfo *Nodeinfo `json:"nodeinfo"`
Statistics *Statistics `json:"statistics"`
CustomFields map[string]interface{} `json:"-"`
}

+ 2
- 1
output/filter/noowner/noowner.go

@ -41,7 +41,8 @@ func (no *noowner) Apply(node *runtime.Node) *runtime.Node {
VPN: nodeinfo.VPN,
Wireless: nodeinfo.Wireless,
},
Neighbours: node.Neighbours,
Neighbours: node.Neighbours,
CustomFields: node.CustomFields,
}
}
return node

+ 23
- 3
respond/collector.go

@ -5,10 +5,12 @@ import (
"compress/flate"
"encoding/json"
"fmt"
"io/ioutil"
"net"
"time"
"github.com/bdlm/log"
"github.com/tidwall/gjson"
"github.com/FreifunkBremen/yanic/data"
"github.com/FreifunkBremen/yanic/database"
@ -237,7 +239,7 @@ func (coll *Collector) sender() {
func (coll *Collector) parser() {
for obj := range coll.queue {
if data, err := obj.parse(); err != nil {
if data, err := obj.parse(coll.config.CustomFields); err != nil {
log.WithField("address", obj.Address.String()).Errorf("unable to decode response %s", err)
} else {
coll.saveResponse(obj.Address, data)
@ -245,14 +247,32 @@ func (coll *Collector) parser() {
}
}
func (res *Response) parse() (*data.ResponseData, error) {
func (res *Response) parse(customFields []CustomFieldConfig) (*data.ResponseData, error) {
// Deflate
deflater := flate.NewReader(bytes.NewReader(res.Raw))
defer deflater.Close()
jsonData, err := ioutil.ReadAll(deflater)
if err != nil {
return nil, err
}
// Unmarshal
rdata := &data.ResponseData{}
err := json.NewDecoder(deflater).Decode(rdata)
err = json.Unmarshal(jsonData, rdata)
rdata.CustomFields = make(map[string]interface{})
if !gjson.Valid(string(jsonData)) {
log.WithField("jsonData", jsonData).Info("JSON data is invalid")
} else {
jsonParsed := gjson.Parse(string(jsonData))
for _, customField := range customFields {
field := jsonParsed.Get(customField.Path)
if field.Exists() {
rdata.CustomFields[customField.Name] = field.String()
}
}
}
return rdata, err
}

+ 56
- 1
respond/collector_test.go

@ -41,10 +41,65 @@ func TestParse(t *testing.T) {
Raw: compressed,
}
data, err := res.parse()
data, err := res.parse([]CustomFieldConfig{})
assert.NoError(err)
assert.NotNil(data)
assert.Equal("f81a67a5e9c1", data.Nodeinfo.NodeID)
}
func TestParseCustomFields(t *testing.T) {
assert := assert.New(t)
// read testdata
compressed, err := ioutil.ReadFile("testdata/nodeinfo.flated")
assert.Nil(err)
res := &Response{
Raw: compressed,
}
customFields := []CustomFieldConfig{
{
Name: "my_custom_field",
Path: "nodeinfo.hostname",
},
}
data, err := res.parse(customFields)
assert.NoError(err)
assert.NotNil(data)
assert.Equal("Trillian", data.CustomFields["my_custom_field"])
assert.Equal("Trillian", data.Nodeinfo.Hostname)
}
func TestParseCustomFieldNotExistant(t *testing.T) {
assert := assert.New(t)
// read testdata
compressed, err := ioutil.ReadFile("testdata/nodeinfo.flated")
assert.Nil(err)
res := &Response{
Raw: compressed,
}
customFields := []CustomFieldConfig{
{
Name: "some_other_field",
Path: "nodeinfo.some_field_which_doesnt_exist",
},
}
data, err := res.parse(customFields)
assert.NoError(err)
assert.NotNil(data)
_, ok := data.CustomFields["some_other_field"]
assert.Equal("Trillian", data.Nodeinfo.Hostname)
assert.False(ok)
}

+ 6
- 0
respond/config.go

@ -8,6 +8,7 @@ type Config struct {
Interfaces []InterfaceConfig `toml:"interfaces"`
Sites map[string]SiteConfig `toml:"sites"`
CollectInterval duration.Duration `toml:"collect_interval"`
CustomFields []CustomFieldConfig `toml:"custom_field"`
}
func (c *Config) SitesDomains() (result map[string][]string) {
@ -29,3 +30,8 @@ type InterfaceConfig struct {
MulticastAddress string `toml:"multicast_address"`
Port int `toml:"port"`
}
type CustomFieldConfig struct {
Name string `toml:"name"`
Path string `toml:"path"`
}

+ 8
- 7
runtime/node.go

@ -9,13 +9,14 @@ import (
// Node struct
type Node struct {
Address *net.UDPAddr `json:"-"` // the last known address
Firstseen jsontime.Time `json:"firstseen"`
Lastseen jsontime.Time `json:"lastseen"`
Online bool `json:"online"`
Statistics *data.Statistics `json:"statistics"`
Nodeinfo *data.Nodeinfo `json:"nodeinfo"`
Neighbours *data.Neighbours `json:"-"`
Address *net.UDPAddr `json:"-"` // the last known address
Firstseen jsontime.Time `json:"firstseen"`
Lastseen jsontime.Time `json:"lastseen"`
Online bool `json:"online"`
Statistics *data.Statistics `json:"statistics"`
Nodeinfo *data.Nodeinfo `json:"nodeinfo"`
Neighbours *data.Neighbours `json:"-"`
CustomFields map[string]interface{} `json:"custom_fields"`
}
// Link represents a link between two nodes

+ 1
- 0
runtime/nodes.go

@ -83,6 +83,7 @@ func (nodes *Nodes) Update(nodeID string, res *data.ResponseData) *Node {
node.Neighbours = res.Neighbours
node.Nodeinfo = res.Nodeinfo
node.Statistics = res.Statistics
node.CustomFields = res.CustomFields
return node
}

Loading…
Cancel
Save