diff --git a/docs/docs_configuration.md b/docs/docs_configuration.md index 2703449..16c3560 100644 --- a/docs/docs_configuration.md +++ b/docs/docs_configuration.md @@ -570,6 +570,31 @@ path = "/var/www/html/meshviewer/data/raw.json" +## [[nodes.output.raw-jsonl]] +{% method %} +This output takes the respondd response as sent by the node and inserts it into a line-separated JSON document (JSONL). In this format, each line can be interpreted as a separate JSON element, which is useful for json streaming. The first line is a json object containing the timestamp and version of the file. This is followed by a line for each node, each containing a json object. +{% sample lang="toml" %} +```toml +[[nodes.output.raw-jsonl]] +enable = false +path = "/var/www/html/meshviewer/data/raw.jsonl" +#[nodes.output.raw.filter] +#no_owner = false +``` +{% endmethod %} + + +### path +{% method %} +The path, where to store raw.jsonl +{% sample lang="toml" %} +```toml +path = "/var/www/html/meshviewer/data/raw.jsonl" +``` +{% endmethod %} + + + ## [database] {% method %} The database organize all database types. diff --git a/output/all/main.go b/output/all/main.go index 0934cdc..263f078 100644 --- a/output/all/main.go +++ b/output/all/main.go @@ -6,4 +6,5 @@ import ( _ "github.com/FreifunkBremen/yanic/output/meshviewer-ffrgb" _ "github.com/FreifunkBremen/yanic/output/nodelist" _ "github.com/FreifunkBremen/yanic/output/raw" + _ "github.com/FreifunkBremen/yanic/output/raw-jsonl" ) diff --git a/output/raw-jsonl/output.go b/output/raw-jsonl/output.go new file mode 100644 index 0000000..9334da1 --- /dev/null +++ b/output/raw-jsonl/output.go @@ -0,0 +1,46 @@ +package jsonlines + +import ( + "errors" + + "github.com/FreifunkBremen/yanic/output" + "github.com/FreifunkBremen/yanic/runtime" +) + +type Output struct { + output.Output + path string +} + +type Config map[string]interface{} + +func (c Config) Path() string { + if path, ok := c["path"]; ok { + return path.(string) + } + return "" +} + +func init() { + output.RegisterAdapter("raw-jsonl", Register) +} + +func Register(configuration map[string]interface{}) (output.Output, error) { + var config Config + config = configuration + + if path := config.Path(); path != "" { + return &Output{ + path: path, + }, nil + } + return nil, errors.New("no path given") + +} + +func (o *Output) Save(nodes *runtime.Nodes) { + nodes.RLock() + defer nodes.RUnlock() + + runtime.SaveJSONL(transform(nodes), o.path) +} diff --git a/output/raw-jsonl/output_test.go b/output/raw-jsonl/output_test.go new file mode 100644 index 0000000..70c783d --- /dev/null +++ b/output/raw-jsonl/output_test.go @@ -0,0 +1,27 @@ +package jsonlines + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestOutput(t *testing.T) { + assert := assert.New(t) + + out, err := Register(map[string]interface{}{}) + assert.Error(err) + assert.Nil(out) + + out, err = Register(map[string]interface{}{ + "path": "/tmp/raw.jsonl", + }) + os.Remove("/tmp/raw.jsonl") + assert.NoError(err) + assert.NotNil(out) + + out.Save(createTestNodes()) + _, err = os.Stat("/tmp/raw.jsonl") + assert.NoError(err) +} diff --git a/output/raw-jsonl/raw_jsonl.go b/output/raw-jsonl/raw_jsonl.go new file mode 100644 index 0000000..c9f122f --- /dev/null +++ b/output/raw-jsonl/raw_jsonl.go @@ -0,0 +1,50 @@ +package jsonlines + +import ( + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/lib/jsontime" + "github.com/FreifunkBremen/yanic/runtime" +) + +// RawNode struct +type RawNode struct { + Firstseen jsontime.Time `json:"firstseen"` + Lastseen jsontime.Time `json:"lastseen"` + Online bool `json:"online"` + Statistics *data.Statistics `json:"statistics"` + Nodeinfo *data.Nodeinfo `json:"nodeinfo"` + Neighbours *data.Neighbours `json:"neighbours"` + CustomFields map[string]interface{} `json:"custom_fields"` +} + +type FileInfo struct { + Version int `json:"version"` + Timestamp jsontime.Time `json:"updated_at"` // Timestamp of the generation + Format string `json:"format"` +} + +func transform(nodes *runtime.Nodes) []interface{} { + var result []interface{} + + result = append(result, FileInfo{ + Version: 1, + Timestamp: jsontime.Now(), + Format: "raw-nodes-jsonl", + }) + + for _, nodeOrigin := range nodes.List { + if nodeOrigin != nil { + node := &RawNode{ + Firstseen: nodeOrigin.Firstseen, + Lastseen: nodeOrigin.Lastseen, + Online: nodeOrigin.Online, + Statistics: nodeOrigin.Statistics, + Nodeinfo: nodeOrigin.Nodeinfo, + Neighbours: nodeOrigin.Neighbours, + CustomFields: nodeOrigin.CustomFields, + } + result = append(result, node) + } + } + return result +} diff --git a/output/raw-jsonl/raw_jsonl_test.go b/output/raw-jsonl/raw_jsonl_test.go new file mode 100644 index 0000000..2ea6aed --- /dev/null +++ b/output/raw-jsonl/raw_jsonl_test.go @@ -0,0 +1,130 @@ +package jsonlines + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/FreifunkBremen/yanic/data" + "github.com/FreifunkBremen/yanic/runtime" +) + +const ( + testNodeDescription string = "Online\nClients: 42\nModel: TP-Link 841\n" + + "Site: mysite\nDomain: domain_42\n" +) + +func TestTransform(t *testing.T) { + testNodes := createTestNodes() + result := transform(testNodes) + + assert := assert.New(t) + assert.Len(testNodes.List, 4) + assert.Len(result, 5) + + fi, ok := result[0].(FileInfo) + assert.True(ok) + assert.Equal(1, fi.Version) + assert.Equal("raw-nodes-jsonl", fi.Format) + + foundNodeIDs := make(map[string]int) + + for _, element := range result[1:] { + node, ok := element.(*RawNode) + assert.True(ok) + foundNodeIDs[node.Nodeinfo.NodeID] += 1 + } + + assert.Equal(1, foundNodeIDs["abcdef012425"]) + assert.Equal(1, foundNodeIDs["abcdef012345"]) + assert.Equal(1, foundNodeIDs["112233445566"]) + assert.Equal(1, foundNodeIDs["0xdeadbeef0x"]) + assert.Equal(0, foundNodeIDs["NONEXISTING"]) +} + +func createTestNodes() *runtime.Nodes { + nodes := runtime.NewNodes(&runtime.NodesConfig{}) + + nodes.AddNode(&runtime.Node{ + Online: true, + Statistics: &data.Statistics{ + Clients: data.Clients{ + Total: 42, + }, + }, + Nodeinfo: &data.Nodeinfo{ + NodeID: "abcdef012425", + Hardware: data.Hardware{ + Model: "TP-Link 841", + }, + Location: &data.Location{ + Latitude: 24, + Longitude: 2, + }, + System: data.System{ + SiteCode: "mysite", + DomainCode: "domain_42", + }, + }, + }) + + nodeData := &runtime.Node{ + Online: true, + Statistics: &data.Statistics{ + Clients: data.Clients{ + Total: 23, + }, + }, + Nodeinfo: &data.Nodeinfo{ + NodeID: "abcdef012345", + Hardware: data.Hardware{ + Model: "TP-Link 842", + }, + System: data.System{ + SiteCode: "mysite", + DomainCode: "domain_42", + }, + }, + } + nodeData.Nodeinfo.Software.Firmware = &struct { + Base string `json:"base,omitempty"` + Release string `json:"release,omitempty"` + }{ + Release: "2019.1~exp42", + } + nodes.AddNode(nodeData) + + nodes.AddNode(&runtime.Node{ + Statistics: &data.Statistics{ + Clients: data.Clients{ + Total: 2, + }, + }, + Nodeinfo: &data.Nodeinfo{ + NodeID: "112233445566", + Hardware: data.Hardware{ + Model: "TP-Link 843", + }, + Location: &data.Location{ + Latitude: 23, + Longitude: 2, + }, + }, + }) + + nodes.AddNode(&runtime.Node{ + Nodeinfo: &data.Nodeinfo{ + NodeID: "0xdeadbeef0x", + VPN: true, + Hardware: data.Hardware{ + Model: "Xeon Multi-Core", + }, + Location: &data.Location{ + Latitude: 23, + Longitude: 22, + }, + }, + }) + + return nodes +} diff --git a/runtime/node.go b/runtime/node.go index d048639..cf00377 100644 --- a/runtime/node.go +++ b/runtime/node.go @@ -21,13 +21,13 @@ type Node struct { // Link represents a link between two nodes type Link struct { - SourceID string + SourceID string SourceHostname string - SourceAddress string - TargetID string - TargetAddress string + SourceAddress string + TargetID string + TargetAddress string TargetHostname string - TQ float32 + TQ float32 } // IsGateway returns whether the node is a gateway diff --git a/runtime/nodes.go b/runtime/nodes.go index 069e804..f163435 100644 --- a/runtime/nodes.go +++ b/runtime/nodes.go @@ -273,3 +273,25 @@ func SaveJSON(input interface{}, outputFile string) { log.Panic(err) } } + +// Save a slice of json objects as line-encoded JSON (JSONL) to a path. +func SaveJSONL(input []interface{}, outputFile string) { + tmpFile := outputFile + ".tmp" + + f, err := os.OpenFile(tmpFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + log.Panic(err) + } + + for _, element := range input { + err = json.NewEncoder(f).Encode(element) + if err != nil { + log.Panic(err) + } + } + + f.Close() + if err := os.Rename(tmpFile, outputFile); err != nil { + log.Panic(err) + } +}