[TASK] add output raw-jsonl

PR at github: #199

This output takes the respondd response as sent by the node and includes
it in a Line-Delimited JSON (JSONL) document. In this format each line
can be interpreted as separate JSON element, which is useful for json
streaming. The first line is json object holding the timestamp and
version of the file. Then there follows one line for each node, each
containing a json object.

An example output looks like this:

    {"version":1,"updated_at":"2021-03-27T21:58:48+0100","format":"raw-nodes-jsonl"}
    {"firstseen": ..., "lastseen": ..., "online":true, "statistics": {...}, "nodeinfo": {...}, "neighbours":null, "custom_fields":null}
    {"firstseen": ..., "lastseen": ..., "online":true, "statistics": {...}, "nodeinfo": {...}, "neighbours":null, "custom_fields":null}
    {"firstseen": ..., "lastseen": ..., "online":true, "statistics": {...}, "nodeinfo": {...}, "neighbours":null, "custom_fields":null}
    {"firstseen": ..., "lastseen": ..., "online":true, "statistics": {...}, "nodeinfo": {...}, "neighbours":null, "custom_fields":null}
    ...

Signed-off-by: Leonardo Mörlein <git@irrelefant.net>

docs/docs_configuration.md

@@ -570,6 +570,31 @@ path     = "/var/www/html/meshviewer/data/raw.json"
 
 
 
+## [[nodes.output.raw-jsonl]]
+{% method %}
+This output takes the respondd response as sent by the node and inserts it into a line-separated JSON document (JSONL). In this format, each line can be interpreted as a separate JSON element, which is useful for json streaming. The first line is a json object containing the timestamp and version of the file. This is followed by a line for each node, each containing a json object.
+{% sample lang="toml" %}
+```toml
+[[nodes.output.raw-jsonl]]
+enable   = false
+path     = "/var/www/html/meshviewer/data/raw.jsonl"
+#[nodes.output.raw.filter]
+#no_owner = false
+```
+{% endmethod %}
+
+
+### path
+{% method %}
+The path, where to store raw.jsonl
+{% sample lang="toml" %}
+```toml
+path     = "/var/www/html/meshviewer/data/raw.jsonl"
+```
+{% endmethod %}
+
+
+
 ## [database]
 {% method %}
 The database organize all database types.

output/all/main.go

@@ -6,4 +6,5 @@ import (
 	_ "github.com/FreifunkBremen/yanic/output/meshviewer-ffrgb"
 	_ "github.com/FreifunkBremen/yanic/output/nodelist"
 	_ "github.com/FreifunkBremen/yanic/output/raw"
+	_ "github.com/FreifunkBremen/yanic/output/raw-jsonl"
 )

output/raw-jsonl/output.go

@@ -0,0 +1,46 @@
+package jsonlines
+
+import (
+	"errors"
+
+	"github.com/FreifunkBremen/yanic/output"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+type Output struct {
+	output.Output
+	path string
+}
+
+type Config map[string]interface{}
+
+func (c Config) Path() string {
+	if path, ok := c["path"]; ok {
+		return path.(string)
+	}
+	return ""
+}
+
+func init() {
+	output.RegisterAdapter("raw-jsonl", Register)
+}
+
+func Register(configuration map[string]interface{}) (output.Output, error) {
+	var config Config
+	config = configuration
+
+	if path := config.Path(); path != "" {
+		return &Output{
+			path: path,
+		}, nil
+	}
+	return nil, errors.New("no path given")
+
+}
+
+func (o *Output) Save(nodes *runtime.Nodes) {
+	nodes.RLock()
+	defer nodes.RUnlock()
+
+	runtime.SaveJSONL(transform(nodes), o.path)
+}

output/raw-jsonl/output_test.go

@@ -0,0 +1,27 @@
+package jsonlines
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestOutput(t *testing.T) {
+	assert := assert.New(t)
+
+	out, err := Register(map[string]interface{}{})
+	assert.Error(err)
+	assert.Nil(out)
+
+	out, err = Register(map[string]interface{}{
+		"path": "/tmp/raw.jsonl",
+	})
+	os.Remove("/tmp/raw.jsonl")
+	assert.NoError(err)
+	assert.NotNil(out)
+
+	out.Save(createTestNodes())
+	_, err = os.Stat("/tmp/raw.jsonl")
+	assert.NoError(err)
+}

output/raw-jsonl/raw_jsonl.go

@@ -0,0 +1,50 @@
+package jsonlines
+
+import (
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/lib/jsontime"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+// RawNode struct
+type RawNode struct {
+	Firstseen    jsontime.Time          `json:"firstseen"`
+	Lastseen     jsontime.Time          `json:"lastseen"`
+	Online       bool                   `json:"online"`
+	Statistics   *data.Statistics       `json:"statistics"`
+	Nodeinfo     *data.Nodeinfo         `json:"nodeinfo"`
+	Neighbours   *data.Neighbours       `json:"neighbours"`
+	CustomFields map[string]interface{} `json:"custom_fields"`
+}
+
+type FileInfo struct {
+	Version   int           `json:"version"`
+	Timestamp jsontime.Time `json:"updated_at"` // Timestamp of the generation
+	Format    string        `json:"format"`
+}
+
+func transform(nodes *runtime.Nodes) []interface{} {
+	var result []interface{}
+
+	result = append(result, FileInfo{
+		Version:   1,
+		Timestamp: jsontime.Now(),
+		Format:    "raw-nodes-jsonl",
+	})
+
+	for _, nodeOrigin := range nodes.List {
+		if nodeOrigin != nil {
+			node := &RawNode{
+				Firstseen:    nodeOrigin.Firstseen,
+				Lastseen:     nodeOrigin.Lastseen,
+				Online:       nodeOrigin.Online,
+				Statistics:   nodeOrigin.Statistics,
+				Nodeinfo:     nodeOrigin.Nodeinfo,
+				Neighbours:   nodeOrigin.Neighbours,
+				CustomFields: nodeOrigin.CustomFields,
+			}
+			result = append(result, node)
+		}
+	}
+	return result
+}

output/raw-jsonl/raw_jsonl_test.go

@@ -0,0 +1,130 @@
+package jsonlines
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/FreifunkBremen/yanic/data"
+	"github.com/FreifunkBremen/yanic/runtime"
+)
+
+const (
+	testNodeDescription string = "Online\nClients: 42\nModel: TP-Link 841\n" +
+		"Site: mysite\nDomain: domain_42\n"
+)
+
+func TestTransform(t *testing.T) {
+	testNodes := createTestNodes()
+	result := transform(testNodes)
+
+	assert := assert.New(t)
+	assert.Len(testNodes.List, 4)
+	assert.Len(result, 5)
+
+	fi, ok := result[0].(FileInfo)
+	assert.True(ok)
+	assert.Equal(1, fi.Version)
+	assert.Equal("raw-nodes-jsonl", fi.Format)
+
+	foundNodeIDs := make(map[string]int)
+
+	for _, element := range result[1:] {
+		node, ok := element.(*RawNode)
+		assert.True(ok)
+		foundNodeIDs[node.Nodeinfo.NodeID] += 1
+	}
+
+	assert.Equal(1, foundNodeIDs["abcdef012425"])
+	assert.Equal(1, foundNodeIDs["abcdef012345"])
+	assert.Equal(1, foundNodeIDs["112233445566"])
+	assert.Equal(1, foundNodeIDs["0xdeadbeef0x"])
+	assert.Equal(0, foundNodeIDs["NONEXISTING"])
+}
+
+func createTestNodes() *runtime.Nodes {
+	nodes := runtime.NewNodes(&runtime.NodesConfig{})
+
+	nodes.AddNode(&runtime.Node{
+		Online: true,
+		Statistics: &data.Statistics{
+			Clients: data.Clients{
+				Total: 42,
+			},
+		},
+		Nodeinfo: &data.Nodeinfo{
+			NodeID: "abcdef012425",
+			Hardware: data.Hardware{
+				Model: "TP-Link 841",
+			},
+			Location: &data.Location{
+				Latitude:  24,
+				Longitude: 2,
+			},
+			System: data.System{
+				SiteCode:   "mysite",
+				DomainCode: "domain_42",
+			},
+		},
+	})
+
+	nodeData := &runtime.Node{
+		Online: true,
+		Statistics: &data.Statistics{
+			Clients: data.Clients{
+				Total: 23,
+			},
+		},
+		Nodeinfo: &data.Nodeinfo{
+			NodeID: "abcdef012345",
+			Hardware: data.Hardware{
+				Model: "TP-Link 842",
+			},
+			System: data.System{
+				SiteCode:   "mysite",
+				DomainCode: "domain_42",
+			},
+		},
+	}
+	nodeData.Nodeinfo.Software.Firmware = &struct {
+		Base    string `json:"base,omitempty"`
+		Release string `json:"release,omitempty"`
+	}{
+		Release: "2019.1~exp42",
+	}
+	nodes.AddNode(nodeData)
+
+	nodes.AddNode(&runtime.Node{
+		Statistics: &data.Statistics{
+			Clients: data.Clients{
+				Total: 2,
+			},
+		},
+		Nodeinfo: &data.Nodeinfo{
+			NodeID: "112233445566",
+			Hardware: data.Hardware{
+				Model: "TP-Link 843",
+			},
+			Location: &data.Location{
+				Latitude:  23,
+				Longitude: 2,
+			},
+		},
+	})
+
+	nodes.AddNode(&runtime.Node{
+		Nodeinfo: &data.Nodeinfo{
+			NodeID: "0xdeadbeef0x",
+			VPN:    true,
+			Hardware: data.Hardware{
+				Model: "Xeon Multi-Core",
+			},
+			Location: &data.Location{
+				Latitude:  23,
+				Longitude: 22,
+			},
+		},
+	})
+
+	return nodes
+}

runtime/node.go

@@ -21,13 +21,13 @@ type Node struct {
 
 // Link represents a link between two nodes
 type Link struct {
-	SourceID      string
+	SourceID       string
 	SourceHostname string
-	SourceAddress string
+	SourceAddress  string
-	TargetID      string
+	TargetID       string
-	TargetAddress string
+	TargetAddress  string
 	TargetHostname string
-	TQ            float32
+	TQ             float32
 }
 
 // IsGateway returns whether the node is a gateway

runtime/nodes.go

@@ -273,3 +273,25 @@ func SaveJSON(input interface{}, outputFile string) {
 		log.Panic(err)
 	}
 }
+
+// Save a slice of json objects as line-encoded JSON (JSONL) to a path.
+func SaveJSONL(input []interface{}, outputFile string) {
+	tmpFile := outputFile + ".tmp"
+
+	f, err := os.OpenFile(tmpFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		log.Panic(err)
+	}
+
+	for _, element := range input {
+		err = json.NewEncoder(f).Encode(element)
+		if err != nil {
+			log.Panic(err)
+		}
+	}
+
+	f.Close()
+	if err := os.Rename(tmpFile, outputFile); err != nil {
+		log.Panic(err)
+	}
+}