diff --git a/Gopkg.toml b/Gopkg.toml index f504025..9e83e01 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -45,6 +45,10 @@ name = "github.com/spf13/cobra" version = "0.0.2" +[[constraint]] + name = "github.com/tidwall/gjson" + version = "1.3.4" + [[constraint]] name = "github.com/stretchr/testify" version = "1.2.1" diff --git a/config_example.toml b/config_example.toml index 2f18e45..11ad342 100644 --- a/config_example.toml +++ b/config_example.toml @@ -10,6 +10,16 @@ synchronize = "1m" # how often request per multicast collect_interval = "1m" +# If you have custom respondd fields, you can ask Yanic to also collect these. +# NOTE: This does not automatically include these fields in the output. +# The meshviewer-ffrgb output module will include them under "custom_fields", +# but other modules may simply ignore them. +#[[respondd.custom_field]] +#name = zip +# You can use arbitrary GJSON expressions here, see https://github.com/tidwall/gjson +# We expect this expression to return a string. +#path = nodeinfo.location.zip + # table of a site to save stats for (not exists for global only) #[respondd.sites.example] ## list of domains on this site to save stats for (empty for global only) diff --git a/data/response.go b/data/response.go index b52874d..0c08a8c 100644 --- a/data/response.go +++ b/data/response.go @@ -2,7 +2,8 @@ package data // ResponseData struct type ResponseData struct { - Neighbours *Neighbours `json:"neighbours"` - Nodeinfo *Nodeinfo `json:"nodeinfo"` - Statistics *Statistics `json:"statistics"` + Neighbours *Neighbours `json:"neighbours"` + Nodeinfo *Nodeinfo `json:"nodeinfo"` + Statistics *Statistics `json:"statistics"` + CustomFields map[string]interface{} `json:"-"` } diff --git a/output/filter/noowner/noowner.go b/output/filter/noowner/noowner.go index 4116d62..008f94e 100644 --- a/output/filter/noowner/noowner.go +++ b/output/filter/noowner/noowner.go @@ -41,7 +41,8 @@ func (no *noowner) Apply(node *runtime.Node) *runtime.Node { VPN: nodeinfo.VPN, Wireless: nodeinfo.Wireless, }, - Neighbours: node.Neighbours, + Neighbours: node.Neighbours, + CustomFields: node.CustomFields, } } return node diff --git a/respond/collector.go b/respond/collector.go index 911b555..6d2156b 100644 --- a/respond/collector.go +++ b/respond/collector.go @@ -5,10 +5,12 @@ import ( "compress/flate" "encoding/json" "fmt" + "io/ioutil" "net" "time" "github.com/bdlm/log" + "github.com/tidwall/gjson" "github.com/FreifunkBremen/yanic/data" "github.com/FreifunkBremen/yanic/database" @@ -237,7 +239,7 @@ func (coll *Collector) sender() { func (coll *Collector) parser() { for obj := range coll.queue { - if data, err := obj.parse(); err != nil { + if data, err := obj.parse(coll.config.CustomFields); err != nil { log.WithField("address", obj.Address.String()).Errorf("unable to decode response %s", err) } else { coll.saveResponse(obj.Address, data) @@ -245,14 +247,32 @@ func (coll *Collector) parser() { } } -func (res *Response) parse() (*data.ResponseData, error) { +func (res *Response) parse(customFields []CustomFieldConfig) (*data.ResponseData, error) { // Deflate deflater := flate.NewReader(bytes.NewReader(res.Raw)) defer deflater.Close() + jsonData, err := ioutil.ReadAll(deflater) + if err != nil { + return nil, err + } + // Unmarshal rdata := &data.ResponseData{} - err := json.NewDecoder(deflater).Decode(rdata) + err = json.Unmarshal(jsonData, rdata) + + rdata.CustomFields = make(map[string]interface{}) + if !gjson.Valid(string(jsonData)) { + log.WithField("jsonData", jsonData).Info("JSON data is invalid") + } else { + jsonParsed := gjson.Parse(string(jsonData)) + for _, customField := range customFields { + field := jsonParsed.Get(customField.Path) + if field.Exists() { + rdata.CustomFields[customField.Name] = field.String() + } + } + } return rdata, err } diff --git a/respond/collector_test.go b/respond/collector_test.go index 6ee8bd4..de1bd9e 100644 --- a/respond/collector_test.go +++ b/respond/collector_test.go @@ -41,10 +41,65 @@ func TestParse(t *testing.T) { Raw: compressed, } - data, err := res.parse() + data, err := res.parse([]CustomFieldConfig{}) assert.NoError(err) assert.NotNil(data) assert.Equal("f81a67a5e9c1", data.Nodeinfo.NodeID) } + +func TestParseCustomFields(t *testing.T) { + assert := assert.New(t) + + // read testdata + compressed, err := ioutil.ReadFile("testdata/nodeinfo.flated") + assert.Nil(err) + + res := &Response{ + Raw: compressed, + } + + customFields := []CustomFieldConfig{ + { + Name: "my_custom_field", + Path: "nodeinfo.hostname", + }, + } + + data, err := res.parse(customFields) + + assert.NoError(err) + assert.NotNil(data) + + assert.Equal("Trillian", data.CustomFields["my_custom_field"]) + assert.Equal("Trillian", data.Nodeinfo.Hostname) +} + +func TestParseCustomFieldNotExistant(t *testing.T) { + assert := assert.New(t) + + // read testdata + compressed, err := ioutil.ReadFile("testdata/nodeinfo.flated") + assert.Nil(err) + + res := &Response{ + Raw: compressed, + } + + customFields := []CustomFieldConfig{ + { + Name: "some_other_field", + Path: "nodeinfo.some_field_which_doesnt_exist", + }, + } + + data, err := res.parse(customFields) + + assert.NoError(err) + assert.NotNil(data) + + _, ok := data.CustomFields["some_other_field"] + assert.Equal("Trillian", data.Nodeinfo.Hostname) + assert.False(ok) +} diff --git a/respond/config.go b/respond/config.go index 383ad9f..2377f28 100644 --- a/respond/config.go +++ b/respond/config.go @@ -8,6 +8,7 @@ type Config struct { Interfaces []InterfaceConfig `toml:"interfaces"` Sites map[string]SiteConfig `toml:"sites"` CollectInterval duration.Duration `toml:"collect_interval"` + CustomFields []CustomFieldConfig `toml:"custom_field"` } func (c *Config) SitesDomains() (result map[string][]string) { @@ -29,3 +30,8 @@ type InterfaceConfig struct { MulticastAddress string `toml:"multicast_address"` Port int `toml:"port"` } + +type CustomFieldConfig struct { + Name string `toml:"name"` + Path string `toml:"path"` +} diff --git a/runtime/node.go b/runtime/node.go index b10be10..cb55738 100644 --- a/runtime/node.go +++ b/runtime/node.go @@ -9,13 +9,14 @@ import ( // Node struct type Node struct { - Address *net.UDPAddr `json:"-"` // the last known address - Firstseen jsontime.Time `json:"firstseen"` - Lastseen jsontime.Time `json:"lastseen"` - Online bool `json:"online"` - Statistics *data.Statistics `json:"statistics"` - Nodeinfo *data.Nodeinfo `json:"nodeinfo"` - Neighbours *data.Neighbours `json:"-"` + Address *net.UDPAddr `json:"-"` // the last known address + Firstseen jsontime.Time `json:"firstseen"` + Lastseen jsontime.Time `json:"lastseen"` + Online bool `json:"online"` + Statistics *data.Statistics `json:"statistics"` + Nodeinfo *data.Nodeinfo `json:"nodeinfo"` + Neighbours *data.Neighbours `json:"-"` + CustomFields map[string]interface{} `json:"custom_fields"` } // Link represents a link between two nodes diff --git a/runtime/nodes.go b/runtime/nodes.go index a4f0b08..1ba4c24 100644 --- a/runtime/nodes.go +++ b/runtime/nodes.go @@ -83,6 +83,7 @@ func (nodes *Nodes) Update(nodeID string, res *data.ResponseData) *Node { node.Neighbours = res.Neighbours node.Nodeinfo = res.Nodeinfo node.Statistics = res.Statistics + node.CustomFields = res.CustomFields return node }