init

.test-coverage

@@ -0,0 +1,32 @@
+#!/bin/bash
+# Issue:  https://github.com/mattn/goveralls/issues/20
+# Source: https://github.com/uber/go-torch/blob/63da5d33a225c195fea84610e2456d5f722f3963/.test-cover.sh
+CI=$1
+echo "run for $CI"
+
+if [ "$CI" == "circle-ci" ]; then
+  cd ${GOPATH}/src/github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}
+fi
+
+echo "mode: count" > profile.cov
+FAIL=0
+
+# Standard go tooling behavior is to ignore dirs with leading underscors
+for dir in $(find . -maxdepth 10 -not -path './vendor/*' -not -path './.git*' -not -path '*/_*' -type d);
+do
+  if ls $dir/*.go &> /dev/null; then
+    go test -v -covermode=count -coverprofile=profile.tmp $dir || FAIL=$?
+    if [ -f profile.tmp ]
+    then
+      tail -n +2 < profile.tmp >> profile.cov
+      rm profile.tmp
+    fi
+  fi
+done
+
+# Failures have incomplete results, so don't send
+if [ "$FAIL" -eq 0 ]; then
+  goveralls -v -coverprofile=profile.cov -service=$CI -repotoken=$COVERALLS_REPO_TOKEN
+fi
+
+exit $FAIL

circle.yml

@@ -0,0 +1,36 @@
+machine:
+  environment:
+    GOROOT: ""
+    PATH: "/usr/local/go/bin:/usr/local/go_workspace/bin:~/.go_workspace/bin:${PATH}"
+    GOPATH: "${HOME}/.go_workspace"
+
+dependencies:
+  override:
+    - mkdir -p ~/.go_workspace/src/github.com/${CIRCLE_PROJECT_USERNAME}
+    - ln -s ${HOME}/${CIRCLE_PROJECT_REPONAME} ${HOME}/.go_workspace/src/github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}
+    - go get -t -d -v ./...
+    - go install github.com/genofire/nodelistdaemon/cmd/nodelistdaemon
+  post:
+    - cp ~/.go_workspace/bin/nodelistdaemon nodelistdaemon.bin
+    - tar -cvzf nodelistdaemon-builded.tar.gz nodelistdaemon.bin
+    - mv logmania-builded.tar.gz $CIRCLE_ARTIFACTS
+
+
+
+test:
+  pre:
+    - go get github.com/mattn/goveralls
+    - go get golang.org/x/tools/cmd/cover
+  override:
+    - ./.test-coverage circle-ci
+
+deployment:
+  staging:
+    branch: master
+    commands:
+      - ./deploy.sh $HOST_FOR_STAGING $PORT_FOR_STAGING
+
+notify:
+  webhooks:
+    - url: https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN
+    - url: https://hook2xmpp.pub.warehost.de/circleci

cmd/nodelistdaemon/main.go

@@ -0,0 +1,21 @@
+package main
+
+import (
+	"flag"
+	"time"
+
+	"github.com/genofire/nodelistdaemon/runtime"
+)
+
+const DIRECTORY_URL = "https://raw.githubusercontent.com/freifunk/directory.api.freifunk.net/master/directory.json"
+
+var url string
+var filePath string
+
+func main() {
+	flag.StringVar(&url, "url", DIRECTORY_URL, "api directory")
+	flag.StringVar(&filePath, "json", "/tmp/nodelist.json", "where to save json file")
+	flag.Parse()
+	f := runtime.NewFetcher(url, time.Minute, filePath)
+	f.Start()
+}

deploy.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+host=$1
+port=$2
+remote="circleci@${host}"
+echo "deploying..."
+ssh -p $port $remote sudo systemctl stop nodelistdaemon;
+RETVAL=$?
+[ $RETVAL -ne 0 ] && exit 1
+scp -q -P $port ~/.go_workspace/bin/nodelistdaemon $remote:~/bin/nodelistdaemon;
+RETVAL=$?
+ssh -p $port $remote sudo systemctl start nodelistdaemon;
+[ $RETVAL -eq 0 ] && RETVAL=$?
+[ $RETVAL -ne 0 ] && exit 1
+echo "deployed"

runtime/directory.go

@@ -0,0 +1,12 @@
+package runtime
+
+type DirectoryAPI struct {
+	Name     string     `json:"name"`
+	NodeMaps []*NodeMap `json:"nodeMaps"`
+}
+type NodeMap struct {
+	URL           string `json:"url"`
+	Interval      string `json:"interval"`
+	TechnicalType string `json:"technicalType"`
+	MapType       string `json:"mapType`
+}

runtime/fetch.go

@@ -0,0 +1,129 @@
+package runtime
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+)
+
+type Fetcher struct {
+	url           string
+	filePath      string
+	repeat        time.Duration
+	directoryList map[string]string
+	List          map[string]*Node `json:"nodes"`
+	nodesMutex    sync.Mutex
+	closed        bool
+}
+
+func NewFetcher(url string, repeat time.Duration, filePath string) *Fetcher {
+	return &Fetcher{
+		url:        url,
+		repeat:     repeat,
+		filePath:   filePath,
+		nodesMutex: sync.Mutex{},
+		List:       make(map[string]*Node),
+	}
+}
+
+func (f *Fetcher) Start() {
+	timer := time.NewTimer(f.repeat)
+	for !f.closed {
+		select {
+		case <-timer.C:
+			f.work()
+			f.save()
+			timer.Reset(f.repeat)
+		}
+	}
+	timer.Stop()
+}
+
+func (f *Fetcher) Stop() {
+	f.closed = true
+}
+
+func (f *Fetcher) AddNode(n *Node) {
+	if n != nil {
+		f.nodesMutex.Lock()
+		f.List[n.NodeID] = n
+		f.nodesMutex.Unlock()
+	}
+}
+
+func (f *Fetcher) work() {
+	err := JSONRequest(f.url, &f.directoryList)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	wgDirectory := sync.WaitGroup{}
+	wgNodes := sync.WaitGroup{}
+	count := 0
+
+	for community, url := range f.directoryList {
+		wgDirectory.Add(1)
+		go func(community, url string) {
+			defer wgDirectory.Done()
+			var directory DirectoryAPI
+
+			err := JSONRequest(url, &directory)
+			if err != nil {
+				return
+			}
+			for _, mapEntry := range directory.NodeMaps {
+				if mapEntry.TechnicalType == "nodelist" || mapEntry.MapType == "list/status" {
+					wgNodes.Add(1)
+					count++
+					go f.updateNodes(&wgNodes, community, mapEntry.URL)
+				}
+			}
+		}(community, url)
+	}
+
+	log.Println("found", len(f.directoryList), "communities")
+	wgDirectory.Wait()
+	log.Println("wait for", count, "request for nodes")
+	wgNodes.Wait()
+	log.Println("found", len(f.List), "nodes")
+}
+
+func (f *Fetcher) updateNodes(wg *sync.WaitGroup, community string, url string) {
+	defer wg.Done()
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	transform(body, community, f)
+}
+
+func (f *Fetcher) save() {
+	f.nodesMutex.Lock()
+	defer f.nodesMutex.Unlock()
+
+	if f.filePath != "" {
+		tmpFile := f.filePath + ".tmp"
+
+		file, err := os.OpenFile(tmpFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+		if err != nil {
+			log.Panic(err)
+		}
+
+		err = json.NewEncoder(file).Encode(f)
+		if err != nil {
+			log.Panic(err)
+		}
+
+		file.Close()
+		if err := os.Rename(tmpFile, f.filePath); err != nil {
+			log.Panic(err)
+		}
+	}
+
+}

runtime/helper.go

@@ -0,0 +1,19 @@
+package runtime
+
+import (
+	"encoding/json"
+	"net/http"
+)
+
+func JSONRequest(url string, value interface{}) error {
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+
+	err = json.NewDecoder(resp.Body).Decode(&value)
+	if err != nil {
+		return err
+	}
+	return nil
+}

runtime/nodes.go

@@ -0,0 +1,19 @@
+package runtime
+
+import "github.com/FreifunkBremen/yanic/jsontime"
+
+type Node struct {
+	Firstseen jsontime.Time `json:"firstseen"`
+	Lastseen  jsontime.Time `json:"lastseen"`
+	IsOnline  bool          `json:"is_online"`
+	NodeID    string        `json:"node_id"`
+	Hostname  string        `json:"hostname"`
+	SiteCode  string        `json:"community"`
+	Location  *Location     `json:"location"`
+	Clients   uint32        `json:"clients"`
+}
+
+type Location struct {
+	Lon float64 `json:"long"`
+	Lat float64 `json:"lat"`
+}

runtime/tranformer.go

@@ -0,0 +1,14 @@
+package runtime
+
+var transformers []func([]byte, string, *Fetcher) error = []func([]byte, string, *Fetcher) error{
+	transformNodelist, transformMeshviewerV2, transformMeshviewerV1,
+}
+
+func transform(body []byte, site_code string, f *Fetcher) {
+	for _, trans := range transformers {
+		err := trans(body, site_code, f)
+		if err == nil {
+			return
+		}
+	}
+}

runtime/transform_list.go

@@ -0,0 +1,61 @@
+package runtime
+
+import (
+	"encoding/json"
+
+	yanicMeshviewer "github.com/FreifunkBremen/yanic/output/meshviewer"
+	yanicNodelist "github.com/FreifunkBremen/yanic/output/nodelist"
+	yanicRuntime "github.com/FreifunkBremen/yanic/runtime"
+)
+
+func transformNodelist(body []byte, sitecode string, f *Fetcher) error {
+	var nodes yanicNodelist.NodeList
+	err := json.Unmarshal(body, &nodes)
+	if err == nil {
+		for _, node := range nodes.List {
+			nodeEntry := transformNodelistNode(node, sitecode)
+			f.AddNode(nodeEntry)
+		}
+		return nil
+	}
+	return err
+}
+
+func transformMeshviewerV2(body []byte, sitecode string, f *Fetcher) error {
+	var nodes yanicMeshviewer.NodesV2
+	err := json.Unmarshal(body, &nodes)
+	if err == nil {
+		for _, node := range nodes.List {
+			nodeEntry := transformMeshviewerNode(node, sitecode)
+			f.AddNode(nodeEntry)
+		}
+		return nil
+	}
+	return err
+}
+
+func transformMeshviewerV1(body []byte, sitecode string, f *Fetcher) error {
+	var nodes yanicMeshviewer.NodesV1
+	err := json.Unmarshal(body, &nodes)
+	if err == nil {
+		for _, node := range nodes.List {
+			nodeEntry := transformMeshviewerNode(node, sitecode)
+			f.AddNode(nodeEntry)
+		}
+		return nil
+	}
+	return err
+}
+
+func transformYanic(body []byte, sitecode string, f *Fetcher) error {
+	var nodes yanicRuntime.Nodes
+	err := json.Unmarshal(body, &nodes)
+	if err == nil {
+		for _, node := range nodes.List {
+			nodeEntry := transformYanicNode(node, sitecode)
+			f.AddNode(nodeEntry)
+		}
+		return nil
+	}
+	return err
+}

runtime/transform_node.go

@@ -0,0 +1,72 @@
+package runtime
+
+import (
+	"github.com/FreifunkBremen/yanic/jsontime"
+
+	yanicMeshviewer "github.com/FreifunkBremen/yanic/output/meshviewer"
+	yanicNodelist "github.com/FreifunkBremen/yanic/output/nodelist"
+	yanicRuntime "github.com/FreifunkBremen/yanic/runtime"
+)
+
+func transformNodelistNode(n *yanicNodelist.Node, sitecode string) *Node {
+	node := &Node{
+		NodeID:    n.ID,
+		Hostname:  n.Name,
+		SiteCode:  sitecode,
+		Firstseen: jsontime.Now(),
+		Lastseen:  n.Status.LastContact,
+		IsOnline:  n.Status.Online,
+		Clients:   n.Status.Clients,
+	}
+	if pos := n.Position; pos != nil {
+		node.Location = &Location{
+			Lat: pos.Lat,
+			Lon: pos.Long,
+		}
+	}
+	return node
+}
+
+func transformMeshviewerNode(n *yanicMeshviewer.Node, sitecode string) *Node {
+	if nodeinfo := n.Nodeinfo; nodeinfo != nil {
+		node := &Node{
+			NodeID:    nodeinfo.NodeID,
+			Hostname:  nodeinfo.Hostname,
+			SiteCode:  sitecode,
+			Firstseen: n.Firstseen,
+			Lastseen:  n.Lastseen,
+			IsOnline:  n.Flags.Online,
+			Clients:   n.Statistics.Clients,
+		}
+		if pos := nodeinfo.Location; pos != nil {
+			node.Location = &Location{
+				Lat: pos.Latitude,
+				Lon: pos.Longtitude,
+			}
+		}
+		return node
+	}
+	return nil
+}
+
+func transformYanicNode(n *yanicRuntime.Node, sitecode string) *Node {
+	if nodeinfo := n.Nodeinfo; nodeinfo != nil {
+		node := &Node{
+			NodeID:    nodeinfo.NodeID,
+			Hostname:  nodeinfo.Hostname,
+			SiteCode:  sitecode,
+			Firstseen: n.Firstseen,
+			Lastseen:  n.Lastseen,
+			IsOnline:  n.Online,
+			Clients:   n.Statistics.Clients.Total,
+		}
+		if pos := nodeinfo.Location; pos != nil {
+			node.Location = &Location{
+				Lat: pos.Latitude,
+				Lon: pos.Longtitude,
+			}
+		}
+		return node
+	}
+	return nil
+}

vendor/github.com/FreifunkBremen/yanic

@@ -0,0 +1 @@
+Subproject commit ef726a5bece41af6180177c44a8343c312a922a8