From a72edc4f62f560d48fc0d8f1db2068fca17aed14 Mon Sep 17 00:00:00 2001 From: Martin Geno Date: Wed, 13 Sep 2017 20:41:29 +0200 Subject: [PATCH] init --- .test-coverage | 32 ++++++ circle.yml | 36 +++++++ cmd/nodelistdaemon/main.go | 21 ++++ deploy.sh | 14 +++ runtime/directory.go | 12 +++ runtime/fetch.go | 129 +++++++++++++++++++++++++ runtime/helper.go | 19 ++++ runtime/nodes.go | 19 ++++ runtime/tranformer.go | 14 +++ runtime/transform_list.go | 61 ++++++++++++ runtime/transform_node.go | 72 ++++++++++++++ vendor/github.com/FreifunkBremen/yanic | 1 + 12 files changed, 430 insertions(+) create mode 100644 .test-coverage create mode 100644 circle.yml create mode 100644 cmd/nodelistdaemon/main.go create mode 100644 deploy.sh create mode 100644 runtime/directory.go create mode 100644 runtime/fetch.go create mode 100644 runtime/helper.go create mode 100644 runtime/nodes.go create mode 100644 runtime/tranformer.go create mode 100644 runtime/transform_list.go create mode 100644 runtime/transform_node.go create mode 160000 vendor/github.com/FreifunkBremen/yanic diff --git a/.test-coverage b/.test-coverage new file mode 100644 index 0000000..cd46eda --- /dev/null +++ b/.test-coverage @@ -0,0 +1,32 @@ +#!/bin/bash +# Issue: https://github.com/mattn/goveralls/issues/20 +# Source: https://github.com/uber/go-torch/blob/63da5d33a225c195fea84610e2456d5f722f3963/.test-cover.sh +CI=$1 +echo "run for $CI" + +if [ "$CI" == "circle-ci" ]; then + cd ${GOPATH}/src/github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME} +fi + +echo "mode: count" > profile.cov +FAIL=0 + +# Standard go tooling behavior is to ignore dirs with leading underscors +for dir in $(find . -maxdepth 10 -not -path './vendor/*' -not -path './.git*' -not -path '*/_*' -type d); +do + if ls $dir/*.go &> /dev/null; then + go test -v -covermode=count -coverprofile=profile.tmp $dir || FAIL=$? + if [ -f profile.tmp ] + then + tail -n +2 < profile.tmp >> profile.cov + rm profile.tmp + fi + fi +done + +# Failures have incomplete results, so don't send +if [ "$FAIL" -eq 0 ]; then + goveralls -v -coverprofile=profile.cov -service=$CI -repotoken=$COVERALLS_REPO_TOKEN +fi + +exit $FAIL diff --git a/circle.yml b/circle.yml new file mode 100644 index 0000000..f7e6cef --- /dev/null +++ b/circle.yml @@ -0,0 +1,36 @@ +machine: + environment: + GOROOT: "" + PATH: "/usr/local/go/bin:/usr/local/go_workspace/bin:~/.go_workspace/bin:${PATH}" + GOPATH: "${HOME}/.go_workspace" + +dependencies: + override: + - mkdir -p ~/.go_workspace/src/github.com/${CIRCLE_PROJECT_USERNAME} + - ln -s ${HOME}/${CIRCLE_PROJECT_REPONAME} ${HOME}/.go_workspace/src/github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME} + - go get -t -d -v ./... + - go install github.com/genofire/nodelistdaemon/cmd/nodelistdaemon + post: + - cp ~/.go_workspace/bin/nodelistdaemon nodelistdaemon.bin + - tar -cvzf nodelistdaemon-builded.tar.gz nodelistdaemon.bin + - mv logmania-builded.tar.gz $CIRCLE_ARTIFACTS + + + +test: + pre: + - go get github.com/mattn/goveralls + - go get golang.org/x/tools/cmd/cover + override: + - ./.test-coverage circle-ci + +deployment: + staging: + branch: master + commands: + - ./deploy.sh $HOST_FOR_STAGING $PORT_FOR_STAGING + +notify: + webhooks: + - url: https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN + - url: https://hook2xmpp.pub.warehost.de/circleci diff --git a/cmd/nodelistdaemon/main.go b/cmd/nodelistdaemon/main.go new file mode 100644 index 0000000..8bd04c0 --- /dev/null +++ b/cmd/nodelistdaemon/main.go @@ -0,0 +1,21 @@ +package main + +import ( + "flag" + "time" + + "github.com/genofire/nodelistdaemon/runtime" +) + +const DIRECTORY_URL = "https://raw.githubusercontent.com/freifunk/directory.api.freifunk.net/master/directory.json" + +var url string +var filePath string + +func main() { + flag.StringVar(&url, "url", DIRECTORY_URL, "api directory") + flag.StringVar(&filePath, "json", "/tmp/nodelist.json", "where to save json file") + flag.Parse() + f := runtime.NewFetcher(url, time.Minute, filePath) + f.Start() +} diff --git a/deploy.sh b/deploy.sh new file mode 100644 index 0000000..39df234 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,14 @@ +#!/bin/bash +host=$1 +port=$2 +remote="circleci@${host}" +echo "deploying..." +ssh -p $port $remote sudo systemctl stop nodelistdaemon; +RETVAL=$? +[ $RETVAL -ne 0 ] && exit 1 +scp -q -P $port ~/.go_workspace/bin/nodelistdaemon $remote:~/bin/nodelistdaemon; +RETVAL=$? +ssh -p $port $remote sudo systemctl start nodelistdaemon; +[ $RETVAL -eq 0 ] && RETVAL=$? +[ $RETVAL -ne 0 ] && exit 1 +echo "deployed" diff --git a/runtime/directory.go b/runtime/directory.go new file mode 100644 index 0000000..a37ea1d --- /dev/null +++ b/runtime/directory.go @@ -0,0 +1,12 @@ +package runtime + +type DirectoryAPI struct { + Name string `json:"name"` + NodeMaps []*NodeMap `json:"nodeMaps"` +} +type NodeMap struct { + URL string `json:"url"` + Interval string `json:"interval"` + TechnicalType string `json:"technicalType"` + MapType string `json:"mapType` +} diff --git a/runtime/fetch.go b/runtime/fetch.go new file mode 100644 index 0000000..41b5972 --- /dev/null +++ b/runtime/fetch.go @@ -0,0 +1,129 @@ +package runtime + +import ( + "encoding/json" + "io/ioutil" + "log" + "net/http" + "os" + "sync" + "time" +) + +type Fetcher struct { + url string + filePath string + repeat time.Duration + directoryList map[string]string + List map[string]*Node `json:"nodes"` + nodesMutex sync.Mutex + closed bool +} + +func NewFetcher(url string, repeat time.Duration, filePath string) *Fetcher { + return &Fetcher{ + url: url, + repeat: repeat, + filePath: filePath, + nodesMutex: sync.Mutex{}, + List: make(map[string]*Node), + } +} + +func (f *Fetcher) Start() { + timer := time.NewTimer(f.repeat) + for !f.closed { + select { + case <-timer.C: + f.work() + f.save() + timer.Reset(f.repeat) + } + } + timer.Stop() +} + +func (f *Fetcher) Stop() { + f.closed = true +} + +func (f *Fetcher) AddNode(n *Node) { + if n != nil { + f.nodesMutex.Lock() + f.List[n.NodeID] = n + f.nodesMutex.Unlock() + } +} + +func (f *Fetcher) work() { + err := JSONRequest(f.url, &f.directoryList) + if err != nil { + log.Fatal(err) + } + + wgDirectory := sync.WaitGroup{} + wgNodes := sync.WaitGroup{} + count := 0 + + for community, url := range f.directoryList { + wgDirectory.Add(1) + go func(community, url string) { + defer wgDirectory.Done() + var directory DirectoryAPI + + err := JSONRequest(url, &directory) + if err != nil { + return + } + for _, mapEntry := range directory.NodeMaps { + if mapEntry.TechnicalType == "nodelist" || mapEntry.MapType == "list/status" { + wgNodes.Add(1) + count++ + go f.updateNodes(&wgNodes, community, mapEntry.URL) + } + } + }(community, url) + } + + log.Println("found", len(f.directoryList), "communities") + wgDirectory.Wait() + log.Println("wait for", count, "request for nodes") + wgNodes.Wait() + log.Println("found", len(f.List), "nodes") +} + +func (f *Fetcher) updateNodes(wg *sync.WaitGroup, community string, url string) { + defer wg.Done() + + resp, err := http.Get(url) + if err != nil { + return + } + body, err := ioutil.ReadAll(resp.Body) + transform(body, community, f) +} + +func (f *Fetcher) save() { + f.nodesMutex.Lock() + defer f.nodesMutex.Unlock() + + if f.filePath != "" { + tmpFile := f.filePath + ".tmp" + + file, err := os.OpenFile(tmpFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + log.Panic(err) + } + + err = json.NewEncoder(file).Encode(f) + if err != nil { + log.Panic(err) + } + + file.Close() + if err := os.Rename(tmpFile, f.filePath); err != nil { + log.Panic(err) + } + } + +} diff --git a/runtime/helper.go b/runtime/helper.go new file mode 100644 index 0000000..94e1f6e --- /dev/null +++ b/runtime/helper.go @@ -0,0 +1,19 @@ +package runtime + +import ( + "encoding/json" + "net/http" +) + +func JSONRequest(url string, value interface{}) error { + resp, err := http.Get(url) + if err != nil { + return err + } + + err = json.NewDecoder(resp.Body).Decode(&value) + if err != nil { + return err + } + return nil +} diff --git a/runtime/nodes.go b/runtime/nodes.go new file mode 100644 index 0000000..e695208 --- /dev/null +++ b/runtime/nodes.go @@ -0,0 +1,19 @@ +package runtime + +import "github.com/FreifunkBremen/yanic/jsontime" + +type Node struct { + Firstseen jsontime.Time `json:"firstseen"` + Lastseen jsontime.Time `json:"lastseen"` + IsOnline bool `json:"is_online"` + NodeID string `json:"node_id"` + Hostname string `json:"hostname"` + SiteCode string `json:"community"` + Location *Location `json:"location"` + Clients uint32 `json:"clients"` +} + +type Location struct { + Lon float64 `json:"long"` + Lat float64 `json:"lat"` +} diff --git a/runtime/tranformer.go b/runtime/tranformer.go new file mode 100644 index 0000000..a9879d7 --- /dev/null +++ b/runtime/tranformer.go @@ -0,0 +1,14 @@ +package runtime + +var transformers []func([]byte, string, *Fetcher) error = []func([]byte, string, *Fetcher) error{ + transformNodelist, transformMeshviewerV2, transformMeshviewerV1, +} + +func transform(body []byte, site_code string, f *Fetcher) { + for _, trans := range transformers { + err := trans(body, site_code, f) + if err == nil { + return + } + } +} diff --git a/runtime/transform_list.go b/runtime/transform_list.go new file mode 100644 index 0000000..ea9dd6c --- /dev/null +++ b/runtime/transform_list.go @@ -0,0 +1,61 @@ +package runtime + +import ( + "encoding/json" + + yanicMeshviewer "github.com/FreifunkBremen/yanic/output/meshviewer" + yanicNodelist "github.com/FreifunkBremen/yanic/output/nodelist" + yanicRuntime "github.com/FreifunkBremen/yanic/runtime" +) + +func transformNodelist(body []byte, sitecode string, f *Fetcher) error { + var nodes yanicNodelist.NodeList + err := json.Unmarshal(body, &nodes) + if err == nil { + for _, node := range nodes.List { + nodeEntry := transformNodelistNode(node, sitecode) + f.AddNode(nodeEntry) + } + return nil + } + return err +} + +func transformMeshviewerV2(body []byte, sitecode string, f *Fetcher) error { + var nodes yanicMeshviewer.NodesV2 + err := json.Unmarshal(body, &nodes) + if err == nil { + for _, node := range nodes.List { + nodeEntry := transformMeshviewerNode(node, sitecode) + f.AddNode(nodeEntry) + } + return nil + } + return err +} + +func transformMeshviewerV1(body []byte, sitecode string, f *Fetcher) error { + var nodes yanicMeshviewer.NodesV1 + err := json.Unmarshal(body, &nodes) + if err == nil { + for _, node := range nodes.List { + nodeEntry := transformMeshviewerNode(node, sitecode) + f.AddNode(nodeEntry) + } + return nil + } + return err +} + +func transformYanic(body []byte, sitecode string, f *Fetcher) error { + var nodes yanicRuntime.Nodes + err := json.Unmarshal(body, &nodes) + if err == nil { + for _, node := range nodes.List { + nodeEntry := transformYanicNode(node, sitecode) + f.AddNode(nodeEntry) + } + return nil + } + return err +} diff --git a/runtime/transform_node.go b/runtime/transform_node.go new file mode 100644 index 0000000..6a07aef --- /dev/null +++ b/runtime/transform_node.go @@ -0,0 +1,72 @@ +package runtime + +import ( + "github.com/FreifunkBremen/yanic/jsontime" + + yanicMeshviewer "github.com/FreifunkBremen/yanic/output/meshviewer" + yanicNodelist "github.com/FreifunkBremen/yanic/output/nodelist" + yanicRuntime "github.com/FreifunkBremen/yanic/runtime" +) + +func transformNodelistNode(n *yanicNodelist.Node, sitecode string) *Node { + node := &Node{ + NodeID: n.ID, + Hostname: n.Name, + SiteCode: sitecode, + Firstseen: jsontime.Now(), + Lastseen: n.Status.LastContact, + IsOnline: n.Status.Online, + Clients: n.Status.Clients, + } + if pos := n.Position; pos != nil { + node.Location = &Location{ + Lat: pos.Lat, + Lon: pos.Long, + } + } + return node +} + +func transformMeshviewerNode(n *yanicMeshviewer.Node, sitecode string) *Node { + if nodeinfo := n.Nodeinfo; nodeinfo != nil { + node := &Node{ + NodeID: nodeinfo.NodeID, + Hostname: nodeinfo.Hostname, + SiteCode: sitecode, + Firstseen: n.Firstseen, + Lastseen: n.Lastseen, + IsOnline: n.Flags.Online, + Clients: n.Statistics.Clients, + } + if pos := nodeinfo.Location; pos != nil { + node.Location = &Location{ + Lat: pos.Latitude, + Lon: pos.Longtitude, + } + } + return node + } + return nil +} + +func transformYanicNode(n *yanicRuntime.Node, sitecode string) *Node { + if nodeinfo := n.Nodeinfo; nodeinfo != nil { + node := &Node{ + NodeID: nodeinfo.NodeID, + Hostname: nodeinfo.Hostname, + SiteCode: sitecode, + Firstseen: n.Firstseen, + Lastseen: n.Lastseen, + IsOnline: n.Online, + Clients: n.Statistics.Clients.Total, + } + if pos := nodeinfo.Location; pos != nil { + node.Location = &Location{ + Lat: pos.Latitude, + Lon: pos.Longtitude, + } + } + return node + } + return nil +} diff --git a/vendor/github.com/FreifunkBremen/yanic b/vendor/github.com/FreifunkBremen/yanic new file mode 160000 index 0000000..ef726a5 --- /dev/null +++ b/vendor/github.com/FreifunkBremen/yanic @@ -0,0 +1 @@ +Subproject commit ef726a5bece41af6180177c44a8343c312a922a8