[TASK] add ssh manager, config and log

.gitignore

@@ -0,0 +1,131 @@
+## Core latex/pdflatex auxiliary files:
+*.aux
+*.lof
+*.log
+*.lot
+*.fls
+*.out
+*.toc
+
+## Intermediate documents:
+*.dvi
+*-converted-to.*
+# these rules might exclude image files for figures etc.
+*.ps
+*.eps
+*.pdf
+
+## Bibliography auxiliary files (bibtex/biblatex/biber):
+*.bbl
+*.bcf
+*.blg
+*-blx.aux
+*-blx.bib
+*.brf
+*.run.xml
+
+## Build tool auxiliary files:
+*.fdb_latexmk
+*.synctex
+*.synctex.gz
+*.synctex.gz(busy)
+*.pdfsync
+
+## Auxiliary and intermediate files from other packages:
+
+# algorithms
+*.alg
+*.loa
+
+# achemso
+acs-*.bib
+
+# amsthm
+*.thm
+
+# beamer
+*.nav
+*.snm
+*.vrb
+
+#(e)ledmac/(e)ledpar
+*.end
+*.[1-9]
+*.[1-9][0-9]
+*.[1-9][0-9][0-9]
+*.[1-9]R
+*.[1-9][0-9]R
+*.[1-9][0-9][0-9]R
+*.eledsec[1-9]
+*.eledsec[1-9]R
+*.eledsec[1-9][0-9]
+*.eledsec[1-9][0-9]R
+*.eledsec[1-9][0-9][0-9]
+*.eledsec[1-9][0-9][0-9]R
+
+# glossaries
+*.acn
+*.acr
+*.glg
+*.glo
+*.gls
+
+# hyperref
+*.brf
+
+# knitr
+*-concordance.tex
+*.tikz
+*-tikzDictionary
+
+# listings
+*.lol
+
+# makeidx
+*.idx
+*.ilg
+*.ind
+*.ist
+
+# minitoc
+*.maf
+*.mtc
+*.mtc0
+
+# minted
+_minted*
+*.pyg
+
+# morewrites
+*.mw
+
+# nomencl
+*.nlo
+
+# sagetex
+*.sagetex.sage
+*.sagetex.py
+*.sagetex.scmd
+
+# sympy
+*.sout
+*.sympy
+sympy-plots-for-*.tex/
+
+# todonotes
+*.tdo
+
+# xindy
+*.xdy
+
+
+__pycache__
+
+# IDE's go
+.idea/
+
+
+# go project
+profile.cov
+config.conf
+cmd/stock/config.conf

.test-coverage

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Issue:  https://github.com/mattn/goveralls/issues/20
+# Source: https://github.com/uber/go-torch/blob/63da5d33a225c195fea84610e2456d5f722f3963/.test-cover.sh
+
+echo "mode: count" > profile.cov
+FAIL=0
+
+# Standard go tooling behavior is to ignore dirs with leading underscors
+for dir in $(find . -maxdepth 10 -not -path './.git*' -not -path '*/_*' -type d);
+do
+  if ls $dir/*.go &> /dev/null; then
+    go test -p 1 -v -covermode=count -coverprofile=profile.tmp $dir || FAIL=$?
+    if [ -f profile.tmp ]
+    then
+      tail -n +2 < profile.tmp >> profile.cov
+      rm profile.tmp
+    fi
+  fi
+done
+
+# Failures have incomplete results, so don't send
+if [ "$FAIL" -eq 0 ]; then
+  goveralls -service=travis-ci -v -coverprofile=profile.cov
+fi
+
+exit $FAIL

.travis.yml

@@ -0,0 +1,10 @@
+language: go
+go:
+  - tip
+install:
+  - go get -t github.com/FreifunkBremen/freifunkmanager/...
+  - go get github.com/mattn/goveralls
+  - go get "golang.org/x/tools/cmd/cover"
+script:
+  - ./.test-coverage
+  - go install github.com/FreifunkBremen/freifunkmanager/cmd/freifunkmanager

cmd/freifunkmanager/main.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"flag"
+	"net/http"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/NYTimes/gziphandler"
+	goji "goji.io"
+	"goji.io/pat"
+
+	configPackage "github.com/FreifunkBremen/freifunkmanager/config"
+	"github.com/FreifunkBremen/freifunkmanager/lib/log"
+	"github.com/FreifunkBremen/freifunkmanager/ssh"
+)
+
+var (
+	configFile string
+	config     *configPackage.Config
+)
+
+func main() {
+	flag.StringVar(&configFile, "config", "config.conf", "path of configuration file (default:config.conf)")
+	flag.Parse()
+
+	config = configPackage.ReadConfigFile(configFile)
+
+	log.Log.Info("starting...")
+
+	sshmanager := ssh.NewManager(config.SSHPrivateKey)
+
+	// Startwebserver
+	router := goji.NewMux()
+
+	router.Handle(pat.New("/*"), gziphandler.GzipHandler(http.FileServer(http.Dir(config.Webroot))))
+
+	srv := &http.Server{
+		Addr:    config.WebserverBind,
+		Handler: router,
+	}
+	go func() {
+		if err := srv.ListenAndServe(); err != nil {
+			panic(err)
+		}
+	}()
+
+	log.Log.Info("started")
+
+	// Wait for system signal
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+	sig := <-sigs
+
+	// Stop services
+	srv.Close()
+	sshmanager.Close()
+
+	log.Log.Info("stop recieve:", sig)
+}

config/main.go

@@ -0,0 +1,38 @@
+package config
+
+import (
+	"io/ioutil"
+
+	"github.com/BurntSushi/toml"
+
+	"github.com/FreifunkBremen/freifunkmanager/lib/log"
+)
+
+//config file of this daemon (for more the config_example.conf in git repository)
+type Config struct {
+	// address on which the api and static content webserver runs
+	WebserverBind string `toml:"webserver_bind"`
+
+	// path to deliver static content
+	Webroot string `toml:"webroot"`
+	// yanic socket
+	YanicSocket string `toml:"yanic_socket"`
+
+	// SSH private key
+	SSHPrivateKey string `toml:"ssh_key"`
+}
+
+//reads a config model from path of a yml file
+func ReadConfigFile(path string) *Config {
+	config := &Config{}
+	file, err := ioutil.ReadFile(path)
+	if err != nil {
+		log.Log.Panic(err)
+	}
+
+	if err := toml.Unmarshal(file, config); err != nil {
+		log.Log.Panic(err)
+	}
+
+	return config
+}

config/main_test.go

@@ -0,0 +1,24 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReadConfig(t *testing.T) {
+	assert := assert.New(t)
+
+	config := ReadConfigFile("../config_example.conf")
+	assert.NotNil(config)
+
+	assert.Equal(":8080", config.WebserverBind)
+
+	assert.Panics(func() {
+		ReadConfigFile("../config_example.co")
+	}, "wrong file")
+
+	assert.Panics(func() {
+		ReadConfigFile("testdata/config_panic.conf")
+	}, "wrong toml")
+}

config/testdata/config_panic.conf

@@ -0,0 +1 @@
+not unmarshalable

config_example.conf

@@ -0,0 +1,4 @@
+webserver_bind = ":8080"
+webroot = "webroot"
+yanic_socket = ""
+ssh_key = "/etc/a"

lib/log/log.go

@@ -0,0 +1,33 @@
+// Package log provides the
+// functionality to start und initialize to logger
+package log
+
+import (
+	"log"
+	"net/http"
+
+	logger "github.com/Sirupsen/logrus"
+)
+
+// current logger with configuration
+var Log *logger.Logger
+
+// Function to initiate a new logger
+func init() {
+	Log = logger.New()
+	log.SetOutput(Log.Writer()) // Enable fallback if core logger
+}
+
+// Function to add the information of a http request to the log
+// Input: pointer to the http request r
+func HTTP(r *http.Request) *logger.Entry {
+	ip := r.Header.Get("X-Forwarded-For")
+	if len(ip) <= 1 {
+		ip = r.RemoteAddr
+	}
+	return Log.WithFields(logger.Fields{
+		"remote": ip,
+		"method": r.Method,
+		"url":    r.URL.RequestURI(),
+	})
+}

lib/log/log_test.go

@@ -0,0 +1,24 @@
+// Package log provides the
+// functionality to start und initialize to logger
+package log
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// Function to test the logging
+// Input: pointer to teh testing object
+func TestLog(t *testing.T) {
+	assertion := assert.New(t)
+
+	req, _ := http.NewRequest("GET", "https://google.com/lola/duda?q=wasd", nil)
+	log := HTTP(req)
+	_, ok := log.Data["remote"]
+
+	assertion.NotNil(ok, "remote address not set in logger")
+	assertion.Equal("GET", log.Data["method"], "method not set in logger")
+	assertion.Equal("/lola/duda?q=wasd", log.Data["url"], "path not set in logger")
+}

ssh/auth.go

@@ -0,0 +1,30 @@
+package ssh
+
+import (
+	"io/ioutil"
+	"net"
+	"os"
+
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/agent"
+)
+
+func SSHAgent() ssh.AuthMethod {
+	if sshAgent, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK")); err == nil {
+		return ssh.PublicKeysCallback(agent.NewClient(sshAgent).Signers)
+	}
+	return nil
+}
+
+func PublicKeyFile(file string) ssh.AuthMethod {
+	buffer, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil
+	}
+
+	key, err := ssh.ParsePrivateKey(buffer)
+	if err != nil {
+		return nil
+	}
+	return ssh.PublicKeys(key)
+}

ssh/execute.go

@@ -0,0 +1,36 @@
+package ssh
+
+import (
+	"net"
+
+	"golang.org/x/crypto/ssh"
+
+	"github.com/FreifunkBremen/freifunkmanager/lib/log"
+)
+
+func (m *Manager) ExecuteEverywhere(cmd string) {
+	for host, client := range m.clients {
+		m.execute(host, client, cmd)
+	}
+}
+
+func (m *Manager) ExecuteOn(host net.IP, cmd string) {
+	client := m.ConnectTo(host)
+	m.execute(host.String(), client, cmd)
+}
+
+func (m *Manager) execute(host string, client *ssh.Client, cmd string) {
+	session, err := client.NewSession()
+	defer session.Close()
+
+	if err != nil {
+		log.Log.Warnf("can not create session on %s: %s", host, err)
+		delete(m.clients, host)
+		return
+	}
+	err = session.Run(cmd)
+	if err != nil {
+		log.Log.Warnf("could not run %s on %s: %s", cmd, host, err)
+		delete(m.clients, host)
+	}
+}

ssh/execute_test.go

@@ -0,0 +1,23 @@
+package ssh
+
+import (
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExecute(t *testing.T) {
+	assert := assert.New(t)
+
+	mgmt := NewManager("~/.ssh/id_rsa")
+	assert.NotNil(mgmt, "no new manager created")
+
+	mgmt.ConnectTo(net.ParseIP("2a06:8782:ffbb:1337::127"))
+
+	mgmt.ExecuteEverywhere("echo $HOSTNAME")
+	mgmt.ExecuteOn(net.ParseIP("2a06:8782:ffbb:1337::127"), "uptime")
+	mgmt.ExecuteOn(net.ParseIP("2a06:8782:ffbb:1337::127"), "echo $HOSTNAME")
+
+	mgmt.Close()
+}

ssh/manager.go

@@ -0,0 +1,66 @@
+package ssh
+
+import (
+	"net"
+	"sync"
+
+	"golang.org/x/crypto/ssh"
+
+	"github.com/FreifunkBremen/freifunkmanager/lib/log"
+)
+
+// the SSH Connection Manager for multiple connections
+type Manager struct {
+	config     *ssh.ClientConfig
+	clients    map[string]*ssh.Client
+	clientsMUX sync.Mutex
+}
+
+// create a new SSH Connection Manager by ssh file
+func NewManager(file string) *Manager {
+	var auths []ssh.AuthMethod
+	if auth := SSHAgent(); auth != nil {
+		auths = append(auths, auth)
+	}
+	if auth := PublicKeyFile(file); auth != nil {
+		auths = append(auths, auth)
+	}
+
+	sshConfig := &ssh.ClientConfig{
+		User:            "root",
+		Auth:            auths,
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+	}
+	return &Manager{
+		config:  sshConfig,
+		clients: make(map[string]*ssh.Client),
+	}
+}
+
+func (m *Manager) ConnectTo(host net.IP) *ssh.Client {
+	m.clientsMUX.Lock()
+	defer m.clientsMUX.Unlock()
+
+	if client, ok := m.clients[host.String()]; ok {
+		return client
+	}
+	addr := net.TCPAddr{
+		IP:   host,
+		Port: 22,
+	}
+	client, err := ssh.Dial("tcp", addr.String(), m.config)
+	if err != nil {
+		log.Log.Error(err)
+		return nil
+	}
+
+	m.clients[host.String()] = client
+	return client
+}
+
+func (m *Manager) Close() {
+	for host, client := range m.clients {
+		client.Close()
+		delete(m.clients, host)
+	}
+}

ssh/manager_test.go

@@ -0,0 +1,19 @@
+package ssh
+
+import (
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestManager(t *testing.T) {
+	assert := assert.New(t)
+
+	mgmt := NewManager("~/.ssh/id_rsa")
+	assert.NotNil(mgmt, "no new manager created")
+
+	mgmt.ConnectTo(net.ParseIP("2a06:8782:ffbb:1337::127"))
+
+	mgmt.Close()
+}

ssh/run.go

@@ -0,0 +1,59 @@
+package ssh
+
+import (
+	"bytes"
+	"io"
+	"net"
+
+	"golang.org/x/crypto/ssh"
+
+	"github.com/FreifunkBremen/freifunkmanager/lib/log"
+)
+
+type SSHRunResultHandler func([]byte, error)
+
+func (m *Manager) RunEverywhere(cmd string, handler SSHRunResultHandler) {
+	for host, client := range m.clients {
+		result, err := m.run(host, client, cmd)
+		handler(result, err)
+	}
+}
+
+func (m *Manager) RunOn(host net.IP, cmd string) ([]byte, error) {
+	client := m.ConnectTo(host)
+	return m.run(host.String(), client, cmd)
+}
+
+func (m *Manager) run(host string, client *ssh.Client, cmd string) ([]byte, error) {
+	session, err := client.NewSession()
+	defer session.Close()
+
+	if err != nil {
+		log.Log.Warnf("can not create session on %s: %s", host, err)
+		delete(m.clients, host)
+		return nil, err
+	}
+	stdout, err := session.StdoutPipe()
+	buffer := &bytes.Buffer{}
+	go io.Copy(buffer, stdout)
+	if err != nil {
+		log.Log.Warnf("can not create pipe for run on %s: %s", host, err)
+		delete(m.clients, host)
+		return nil, err
+	}
+	err = session.Run(cmd)
+	if err != nil {
+		log.Log.Warnf("could not run %s on %s: %s", cmd, host, err)
+		delete(m.clients, host)
+		return nil, err
+	}
+	var result []byte
+	for {
+		b, err := buffer.ReadByte()
+		if err != nil {
+			break
+		}
+		result = append(result, b)
+	}
+	return result, nil
+}

ssh/run_test.go

@@ -0,0 +1,35 @@
+package ssh
+
+import (
+	"net"
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRun(t *testing.T) {
+	assert := assert.New(t)
+
+	mgmt := NewManager("~/.ssh/id_rsa")
+	assert.NotNil(mgmt, "no new manager created")
+
+	mgmt.ConnectTo(net.ParseIP("2a06:8782:ffbb:1337::127"))
+
+	mgmt.RunEverywhere("echo 13", func(result []byte, err error) {
+		assert.NoError(err)
+
+		result = result[:len(result)-1]
+
+		assert.Equal([]byte{'1', '3'}, result)
+	})
+	result, err := mgmt.RunOn(net.ParseIP("2a06:8782:ffbb:1337::127"), "echo 16")
+	assert.NoError(err)
+
+	result = result[:len(result)-1]
+	resultInt, _ := strconv.Atoi(string(result))
+
+	assert.Equal(16, resultInt)
+
+	mgmt.Close()
+}