From a76df9b9acc4a8a3971b3fd7c73798f3e8f664cb Mon Sep 17 00:00:00 2001 From: lemoer Date: Mon, 29 Mar 2021 16:12:26 +0200 Subject: [PATCH] [TASK] add output raw-jsonl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR at github: #199 This output takes the respondd response as sent by the node and includes it in a Line-Delimited JSON (JSONL) document. In this format each line can be interpreted as separate JSON element, which is useful for json streaming. The first line is json object holding the timestamp and version of the file. Then there follows one line for each node, each containing a json object. An example output looks like this: {"version":1,"updated_at":"2021-03-27T21:58:48+0100","format":"raw-nodes-jsonl"} {"firstseen": ..., "lastseen": ..., "online":true, "statistics": {...}, "nodeinfo": {...}, "neighbours":null, "custom_fields":null} {"firstseen": ..., "lastseen": ..., "online":true, "statistics": {...}, "nodeinfo": {...}, "neighbours":null, "custom_fields":null} {"firstseen": ..., "lastseen": ..., "online":true, "statistics": {...}, "nodeinfo": {...}, "neighbours":null, "custom_fields":null} {"firstseen": ..., "lastseen": ..., "online":true, "statistics": {...}, "nodeinfo": {...}, "neighbours":null, "custom_fields":null} ... Signed-off-by: Leonardo Mörlein --- docs/docs_configuration.md | 25 ++++++ output/all/main.go | 1 + output/raw-jsonl/output.go | 46 ++++++++++ output/raw-jsonl/output_test.go | 27 ++++++ output/raw-jsonl/raw_jsonl.go | 50 +++++++++++ output/raw-jsonl/raw_jsonl_test.go | 130 +++++++++++++++++++++++++++++ runtime/node.go | 10 +-- runtime/nodes.go | 22 +++++ 8 files changed, 306 insertions(+), 5 deletions(-) create mode 100644 output/raw-jsonl/output.go create mode 100644 output/raw-jsonl/output_test.go create mode 100644 output/raw-jsonl/raw_jsonl.go create mode 100644 output/raw-jsonl/raw_jsonl_test.go diff --git a/docs/docs_configuration.md b/docs/docs_configuration.md index 2703449..16c3560 100644 --- a/docs/docs_configuration.md +++ b/docs/docs_configuration.md @@ -570,6 +570,31 @@ path = "/var/www/html/meshviewer/data/raw.json" +## [[nodes.output.raw-jsonl]] +{% method %} +This output takes the respondd response as sent by the node and inserts it into a line-separated JSON document (JSONL). In this format, each line can be interpreted as a separate JSON element, which is useful for json streaming. The first line is a json object containing the timestamp and version of the file. This is followed by a line for each node, each containing a json object. +{% sample lang="toml" %} +```toml +[[nodes.output.raw-jsonl]] +enable = false +path = "/var/www/html/meshviewer/data/raw.jsonl" +#[nodes.output.raw.filter] +#no_owner = false +``` +{% endmethod %} + + +### path +{% method %} +The path, where to store raw.jsonl +{% sample lang="toml" %} +```toml +path = "/var/www/html/meshviewer/data/raw.jsonl" +``` +{% endmethod %} + + + ## [database] {% method %} The database organize all database types. diff --git a/output/all/main.go b/output/all/main.go index 0934cdc..263f078 100644 --- a/output/all/main.go +++ b/output/all/main.go @@ -6,4 +6,5 @@ import ( _ "github.com/FreifunkBremen/yanic/output/meshviewer-ffrgb" _ "github.com/FreifunkBremen/yanic/output/nodelist" _ "github.com/FreifunkBremen/yanic/output/raw" + _ "github.com/FreifunkBremen/yanic/output/raw-jsonl" ) diff --git a/output/raw-jsonl/output.go b/output/raw-jsonl/output.go new file mode 100644 index 0000000..9334da1 --- /dev/null +++ b/output/raw-jsonl/output.go @@ -0,0 +1,46 @@ +package jsonlines + +import ( + "errors" + + "github.com/FreifunkBremen/yanic/output" + "github.com/FreifunkBremen/yanic/runtime" +) + +type Output struct { + output.Output + path string +} + +type Config map[string]interface{} + +func (c Config) Path() string { + if path, ok := c["path"]; ok { + return path.(string) + } + return "" +} + +func init() { + output.RegisterAdapter("raw-jsonl", Register) +} + +func Register(configuration map[string]interface{}) (output.Output, error) { + var config Config + config = configuration + + if path := config.Path(); path != "" { + return &Output{ + path: path, + }, nil + } + return nil, errors.New("no path given") + +} + +func (o *Output) Save(nodes *runtime.Nodes) { + nodes.RLock() + defer nodes.RUnlock() + + runtime.SaveJSONL(transform(nodes), o.path) +} diff --git a/output/raw-jsonl/output_test.go b/output/raw-jsonl/output_test.go new file mode 100644 index 0000000..70c783d --- /dev/null +++ b/output/raw-jsonl/output_test.go @@ -0,0 +1,27 @@ +package jsonlines + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestOutput(t *testing.T) { + assert := assert.New(t) + + out, err := Register(map[string]interface{}{}) + assert.Error(err) + assert.Nil(out) + + out, err = Register(map[string]interface{}{ + "path": "/tmp/raw.jsonl", + }) + os.Remove("/tmp/raw.jsonl") + assert.NoError(err) + assert.NotNil(out) + + out.Save(createTestNodes()) + _, err = os.Stat("/tmp/raw.jsonl") + assert.NoError(err) +} diff --git a/output/raw-jsonl/raw_jsonl.go b/output/raw-jsonl/raw_jsonl.go new file mode 100644 index 0000000..c9f122f --- /dev/null +++ b/output/raw-jsonl/raw_jsonl.go @@ -0,0 +1,50 @@ +package jsonlines + +import ( + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/lib/jsontime" + "github.com/FreifunkBremen/yanic/runtime" +) + +// RawNode struct +type RawNode struct { + Firstseen jsontime.Time `json:"firstseen"` + Lastseen jsontime.Time `json:"lastseen"` + Online bool `json:"online"` + Statistics *data.Statistics `json:"statistics"` + Nodeinfo *data.Nodeinfo `json:"nodeinfo"` + Neighbours *data.Neighbours `json:"neighbours"` + CustomFields map[string]interface{} `json:"custom_fields"` +} + +type FileInfo struct { + Version int `json:"version"` + Timestamp jsontime.Time `json:"updated_at"` // Timestamp of the generation + Format string `json:"format"` +} + +func transform(nodes *runtime.Nodes) []interface{} { + var result []interface{} + + result = append(result, FileInfo{ + Version: 1, + Timestamp: jsontime.Now(), + Format: "raw-nodes-jsonl", + }) + + for _, nodeOrigin := range nodes.List { + if nodeOrigin != nil { + node := &RawNode{ + Firstseen: nodeOrigin.Firstseen, + Lastseen: nodeOrigin.Lastseen, + Online: nodeOrigin.Online, + Statistics: nodeOrigin.Statistics, + Nodeinfo: nodeOrigin.Nodeinfo, + Neighbours: nodeOrigin.Neighbours, + CustomFields: nodeOrigin.CustomFields, + } + result = append(result, node) + } + } + return result +} diff --git a/output/raw-jsonl/raw_jsonl_test.go b/output/raw-jsonl/raw_jsonl_test.go new file mode 100644 index 0000000..2ea6aed --- /dev/null +++ b/output/raw-jsonl/raw_jsonl_test.go @@ -0,0 +1,130 @@ +package jsonlines + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/runtime" +) + +const ( + testNodeDescription string = "Online\nClients: 42\nModel: TP-Link 841\n" + + "Site: mysite\nDomain: domain_42\n" +) + +func TestTransform(t *testing.T) { + testNodes := createTestNodes() + result := transform(testNodes) + + assert := assert.New(t) + assert.Len(testNodes.List, 4) + assert.Len(result, 5) + + fi, ok := result[0].(FileInfo) + assert.True(ok) + assert.Equal(1, fi.Version) + assert.Equal("raw-nodes-jsonl", fi.Format) + + foundNodeIDs := make(map[string]int) + + for _, element := range result[1:] { + node, ok := element.(*RawNode) + assert.True(ok) + foundNodeIDs[node.Nodeinfo.NodeID] += 1 + } + + assert.Equal(1, foundNodeIDs["abcdef012425"]) + assert.Equal(1, foundNodeIDs["abcdef012345"]) + assert.Equal(1, foundNodeIDs["112233445566"]) + assert.Equal(1, foundNodeIDs["0xdeadbeef0x"]) + assert.Equal(0, foundNodeIDs["NONEXISTING"]) +} + +func createTestNodes() *runtime.Nodes { + nodes := runtime.NewNodes(&runtime.NodesConfig{}) + + nodes.AddNode(&runtime.Node{ + Online: true, + Statistics: &data.Statistics{ + Clients: data.Clients{ + Total: 42, + }, + }, + Nodeinfo: &data.Nodeinfo{ + NodeID: "abcdef012425", + Hardware: data.Hardware{ + Model: "TP-Link 841", + }, + Location: &data.Location{ + Latitude: 24, + Longitude: 2, + }, + System: data.System{ + SiteCode: "mysite", + DomainCode: "domain_42", + }, + }, + }) + + nodeData := &runtime.Node{ + Online: true, + Statistics: &data.Statistics{ + Clients: data.Clients{ + Total: 23, + }, + }, + Nodeinfo: &data.Nodeinfo{ + NodeID: "abcdef012345", + Hardware: data.Hardware{ + Model: "TP-Link 842", + }, + System: data.System{ + SiteCode: "mysite", + DomainCode: "domain_42", + }, + }, + } + nodeData.Nodeinfo.Software.Firmware = &struct { + Base string `json:"base,omitempty"` + Release string `json:"release,omitempty"` + }{ + Release: "2019.1~exp42", + } + nodes.AddNode(nodeData) + + nodes.AddNode(&runtime.Node{ + Statistics: &data.Statistics{ + Clients: data.Clients{ + Total: 2, + }, + }, + Nodeinfo: &data.Nodeinfo{ + NodeID: "112233445566", + Hardware: data.Hardware{ + Model: "TP-Link 843", + }, + Location: &data.Location{ + Latitude: 23, + Longitude: 2, + }, + }, + }) + + nodes.AddNode(&runtime.Node{ + Nodeinfo: &data.Nodeinfo{ + NodeID: "0xdeadbeef0x", + VPN: true, + Hardware: data.Hardware{ + Model: "Xeon Multi-Core", + }, + Location: &data.Location{ + Latitude: 23, + Longitude: 22, + }, + }, + }) + + return nodes +} diff --git a/runtime/node.go b/runtime/node.go index d048639..cf00377 100644 --- a/runtime/node.go +++ b/runtime/node.go @@ -21,13 +21,13 @@ type Node struct { // Link represents a link between two nodes type Link struct { - SourceID string + SourceID string SourceHostname string - SourceAddress string - TargetID string - TargetAddress string + SourceAddress string + TargetID string + TargetAddress string TargetHostname string - TQ float32 + TQ float32 } // IsGateway returns whether the node is a gateway diff --git a/runtime/nodes.go b/runtime/nodes.go index 069e804..f163435 100644 --- a/runtime/nodes.go +++ b/runtime/nodes.go @@ -273,3 +273,25 @@ func SaveJSON(input interface{}, outputFile string) { log.Panic(err) } } + +// Save a slice of json objects as line-encoded JSON (JSONL) to a path. +func SaveJSONL(input []interface{}, outputFile string) { + tmpFile := outputFile + ".tmp" + + f, err := os.OpenFile(tmpFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + log.Panic(err) + } + + for _, element := range input { + err = json.NewEncoder(f).Encode(element) + if err != nil { + log.Panic(err) + } + } + + f.Close() + if err := os.Rename(tmpFile, outputFile); err != nil { + log.Panic(err) + } +}