Make configuration more intuitive and consistent

This commit is contained in:
Julian Kornberger 2017-01-29 18:30:08 +01:00
parent 724cd8ba51
commit c66e1120d3
12 changed files with 201 additions and 94 deletions

2
.gitignore vendored
View File

@ -23,4 +23,4 @@ _testmain.go
*.test *.test
*.prof *.prof
webroot webroot
/config.yml /config.toml

View File

@ -11,7 +11,7 @@
## Usage ## Usage
``` ```
Usage of ./respond-collector: Usage of ./respond-collector:
-config path/to/config.yml -config path/to/config.toml
``` ```
## Development ## Development

View File

@ -8,7 +8,6 @@ import (
"os" "os"
"os/signal" "os/signal"
"syscall" "syscall"
"time"
"github.com/NYTimes/gziphandler" "github.com/NYTimes/gziphandler"
"github.com/julienschmidt/httprouter" "github.com/julienschmidt/httprouter"
@ -31,7 +30,7 @@ var (
func main() { func main() {
var importPath string var importPath string
flag.StringVar(&importPath, "import", "", "import global statistics from the given RRD file, requires influxdb") flag.StringVar(&importPath, "import", "", "import global statistics from the given RRD file, requires influxdb")
flag.StringVar(&configFile, "config", "config.yml", "path of configuration file (default:config.yaml)") flag.StringVar(&configFile, "config", "config.toml", "path of configuration file (default:config.yaml)")
flag.Parse() flag.Parse()
config = models.ReadConfigFile(configFile) config = models.ReadConfigFile(configFile)
@ -49,9 +48,8 @@ func main() {
nodes.Start() nodes.Start()
if config.Respondd.Enable { if config.Respondd.Enable {
collectInterval := time.Second * time.Duration(config.Respondd.CollectInterval)
collector = respond.NewCollector(db, nodes, config.Respondd.Interface) collector = respond.NewCollector(db, nodes, config.Respondd.Interface)
collector.Start(collectInterval) collector.Start(config.Respondd.CollectInterval.Duration)
defer collector.Close() defer collector.Close()
} }

31
config_example.toml Normal file
View File

@ -0,0 +1,31 @@
[respondd]
enable = true
interface = "eth0"
collect_interval = "1m"
[webserver]
enable = false
port = "8080"
address = "127.0.0.1"
webroot = "webroot"
[nodes]
enable = true
nodes_version = 2
nodes_path = "/var/www/html/meshviewer/data/nodes_all.json"
graphs_path = "/var/www/html/meshviewer/data/graph.json"
aliases_path = "/var/www/html/meshviewer/data/aliases.json"
# Export nodes and graph periodically
save_interval = "5s"
# Prune offline nodes after a time of inactivity
prune_after = "7d"
[influxdb]
enable = false
address = "http://localhost:8086"
database = "ffhb"
username = ""
password = ""

View File

@ -1,35 +0,0 @@
---
respondd:
enable: true
interface: eth0
# Collected data every n seconds
collectinterval: 60
webserver:
enable: false
port: 8080
address: 127.0.0.1
webroot: webroot
api:
newnodes: true
aliases: true
nodes:
enable: true
nodes_path: /var/www/html/meshviewer/data/nodes_all.json
nodesmini_path: /var/www/html/meshviewer/data/nodes.json
graphs_path: /var/www/html/meshviewer/data/graph.json
aliases_enable: false
aliases_path: /var/www/html/meshviewer/data/aliases.json
# Export nodes and graph every n seconds
saveinterval: 5
# Expire offline nodes after n days
max_age: 7
influxdb:
enable: false
host: http://localhost:8086
database: ffhb
username:
password:

View File

@ -30,7 +30,7 @@ type DB struct {
func New(config *models.Config) *DB { func New(config *models.Config) *DB {
// Make client // Make client
c, err := client.NewHTTPClient(client.HTTPConfig{ c, err := client.NewHTTPClient(client.HTTPConfig{
Addr: config.Influxdb.Addr, Addr: config.Influxdb.Address,
Username: config.Influxdb.Username, Username: config.Influxdb.Username,
Password: config.Influxdb.Password, Password: config.Influxdb.Password,
}) })
@ -54,8 +54,7 @@ func New(config *models.Config) *DB {
} }
func (db *DB) DeletePoints() { func (db *DB) DeletePoints() {
query := fmt.Sprintf("delete from %s where time < now() - %dm", MeasurementNode, db.config.Influxdb.DeleteTill) query := fmt.Sprintf("delete from %s where time < now() - %ds", MeasurementNode, db.config.Influxdb.DeleteAfter.Duration/time.Second)
log.Println("delete", MeasurementNode, "older than", db.config.Influxdb.DeleteTill, "minutes")
db.client.Query(client.NewQuery(query, db.config.Influxdb.Database, "m")) db.client.Query(client.NewQuery(query, db.config.Influxdb.Database, "m"))
} }
@ -100,8 +99,7 @@ func (db *DB) Close() {
// prunes node-specific data periodically // prunes node-specific data periodically
func (db *DB) deleteWorker() { func (db *DB) deleteWorker() {
duration := time.Minute * time.Duration(db.config.Influxdb.DeleteInterval) ticker := time.NewTicker(db.config.Influxdb.DeleteInterval.Duration)
ticker := time.NewTicker(duration)
for { for {
select { select {
case <-ticker.C: case <-ticker.C:
@ -123,7 +121,7 @@ func (db *DB) addWorker() {
var bp client.BatchPoints var bp client.BatchPoints
var err error var err error
var writeNow, closed bool var writeNow, closed bool
batchDuration := time.Second * time.Duration(db.config.Influxdb.SaveInterval) batchDuration := db.config.Influxdb.SaveInterval.Duration
timer := time.NewTimer(batchDuration) timer := time.NewTimer(batchDuration)
for !closed { for !closed {

View File

@ -2,58 +2,61 @@ package models
import ( import (
"io/ioutil" "io/ioutil"
"log"
"gopkg.in/yaml.v2" "github.com/influxdata/toml"
) )
//Config the config File of this daemon //Config the config File of this daemon
type Config struct { type Config struct {
Respondd struct { Respondd struct {
Enable bool `yaml:"enable"` Enable bool
Interface string `yaml:"interface"` Interface string
CollectInterval int `yaml:"collectinterval"` CollectInterval Duration
} `yaml:"respondd"` }
Webserver struct { Webserver struct {
Enable bool `yaml:"enable"` Enable bool
Port string `yaml:"port"` Port string
Address string `yaml:"address"` Address string
Webroot string `yaml:"webroot"` Webroot string
API struct { API struct {
Passphrase string `yaml:"passphrase"` Passphrase string
NewNodes bool `yaml:"newnodes"` NewNodes bool
Aliases bool `yaml:"aliases"` Aliases bool
} `yaml:"api"` }
} `yaml:"webserver"` }
Nodes struct { Nodes struct {
Enable bool `yaml:"enable"` Enable bool
NodesDynamicPath string `yaml:"nodes_path"` NodesDynamicPath string
NodesV1Path string `yaml:"nodesv1_path"` NodesPath string
NodesV2Path string `yaml:"nodesv2_path"` NodesVersion int
GraphsPath string `yaml:"graphs_path"` GraphsPath string
AliasesPath string `yaml:"aliases_path"` AliasesPath string
SaveInterval int `yaml:"saveinterval"` // Save nodes every n seconds SaveInterval Duration // Save nodes periodically
MaxAge int `yaml:"max_age"` // Remove nodes after n days of inactivity PruneAfter Duration // Remove nodes after n days of inactivity
} `yaml:"nodes"` }
Influxdb struct { Influxdb struct {
Enable bool `yaml:"enable"` Enable bool
Addr string `yaml:"host"` Address string
Database string `yaml:"database"` Database string
Username string `yaml:"username"` Username string
Password string `yaml:"password"` Password string
SaveInterval int `yaml:"saveinterval"` // Save nodes every n seconds SaveInterval Duration // Save nodes every n seconds
DeleteInterval int `yaml:"deleteinterval"` // Delete stats of nodes every n minutes DeleteInterval Duration // Delete stats of nodes every n minutes
DeleteTill int `yaml:"deletetill"` // Delete stats of nodes till now-deletetill n minutes DeleteAfter Duration // Delete stats of nodes till now-deletetill n minutes
} }
} }
// ReadConfigFile reads a config model from path of a yml file // ReadConfigFile reads a config model from path of a yml file
func ReadConfigFile(path string) *Config { func ReadConfigFile(path string) *Config {
config := &Config{} config := &Config{}
file, _ := ioutil.ReadFile(path) file, err := ioutil.ReadFile(path)
err := yaml.Unmarshal(file, &config)
if err != nil { if err != nil {
log.Fatal(err) panic(err)
} }
if err := toml.Unmarshal(file, config); err != nil {
panic(err)
}
return config return config
} }

View File

@ -2,6 +2,7 @@ package models
import ( import (
"testing" "testing"
"time"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
@ -9,6 +10,14 @@ import (
func TestReadConfig(t *testing.T) { func TestReadConfig(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)
config := ReadConfigFile("../config_example.yml") config := ReadConfigFile("../config_example.toml")
assert.NotNil(config) assert.NotNil(config)
assert.True(config.Respondd.Enable)
assert.Equal("eth0", config.Respondd.Interface)
assert.Equal(time.Minute, config.Respondd.CollectInterval.Duration)
assert.Equal(2, config.Nodes.NodesVersion)
assert.Equal("/var/www/html/meshviewer/data/nodes_all.json", config.Nodes.NodesPath)
assert.Equal(time.Hour*24*7, config.Nodes.PruneAfter.Duration)
} }

50
models/duration.go Normal file
View File

@ -0,0 +1,50 @@
package models
import (
"fmt"
"strconv"
"time"
)
// Duration is a TOML datatype
// A duration string is a possibly signed sequence of
// decimal numbers and a unit suffix,
// such as "300s", "1.5h" or "5d".
// Valid time units are "s", "m", "h", "d", "w".
type Duration struct {
time.Duration
}
// UnmarshalTOML parses a duration string.
func (d *Duration) UnmarshalTOML(data []byte) error {
// " + int + unit + "
if len(data) < 4 {
return fmt.Errorf("invalid duration: %s", data)
}
unit := data[len(data)-2]
value, err := strconv.Atoi(string(data[1 : len(data)-2]))
if err != nil {
return fmt.Errorf("unable to parse duration %s: %s", data, err)
}
switch unit {
case 's':
d.Duration = time.Duration(value) * time.Second
case 'm':
d.Duration = time.Duration(value) * time.Minute
case 'h':
d.Duration = time.Duration(value) * time.Hour
case 'd':
d.Duration = time.Duration(value) * time.Hour * 24
case 'w':
d.Duration = time.Duration(value) * time.Hour * 24 * 7
case 'y':
d.Duration = time.Duration(value) * time.Hour * 24 * 365
default:
return fmt.Errorf("invalid duration unit: %s", string(unit))
}
return nil
}

47
models/duration_test.go Normal file
View File

@ -0,0 +1,47 @@
package models
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestDuration(t *testing.T) {
assert := assert.New(t)
var tests = []struct {
input string
err string
duration time.Duration
}{
{"", "invalid duration: \"\"", 0},
{"1x", "invalid duration unit: x", 0},
{"1s", "", time.Second},
{"73s", "", time.Second * 73},
{"1m", "", time.Minute},
{"73m", "", time.Minute * 73},
{"1h", "", time.Hour},
{"43h", "", time.Hour * 43},
{"1d", "", time.Hour * 24},
{"8d", "", time.Hour * 24 * 8},
{"1w", "", time.Hour * 24 * 7},
{"52w", "", time.Hour * 24 * 7 * 52},
{"1y", "", time.Hour * 24 * 365},
{"3y", "", time.Hour * 24 * 365 * 3},
}
for _, test := range tests {
d := Duration{}
err := d.UnmarshalTOML([]byte("\"" + test.input + "\""))
duration := d.Duration
if test.err == "" {
assert.NoError(err)
assert.Equal(test.duration, duration)
} else {
assert.EqualError(err, test.err)
}
}
}

View File

@ -140,7 +140,7 @@ func (nodes *Nodes) GetNodesV2() *meshviewer.NodesV2 {
// Periodically saves the cached DB to json file // Periodically saves the cached DB to json file
func (nodes *Nodes) worker() { func (nodes *Nodes) worker() {
c := time.Tick(time.Second * time.Duration(nodes.config.Nodes.SaveInterval)) c := time.Tick(nodes.config.Nodes.SaveInterval.Duration)
for range c { for range c {
nodes.expire() nodes.expire()
@ -153,11 +153,11 @@ func (nodes *Nodes) expire() {
nodes.Timestamp = jsontime.Now() nodes.Timestamp = jsontime.Now()
// Nodes last seen before expireTime will be removed // Nodes last seen before expireTime will be removed
maxAge := nodes.config.Nodes.MaxAge pruneAfter := nodes.config.Nodes.PruneAfter.Duration
if maxAge <= 0 { if pruneAfter == 0 {
maxAge = 7 // our default pruneAfter = time.Hour * 24 * 7 // our default
} }
expireTime := nodes.Timestamp.Add(-time.Duration(maxAge) * time.Hour * 24) expireTime := nodes.Timestamp.Add(-pruneAfter)
// Nodes last seen before offlineTime are changed to 'offline' // Nodes last seen before offlineTime are changed to 'offline'
offlineTime := nodes.Timestamp.Add(-time.Minute * 10) offlineTime := nodes.Timestamp.Add(-time.Minute * 10)
@ -198,11 +198,17 @@ func (nodes *Nodes) save() {
// serialize nodes // serialize nodes
save(nodes, nodes.config.Nodes.NodesDynamicPath) save(nodes, nodes.config.Nodes.NodesDynamicPath)
if path := nodes.config.Nodes.NodesV1Path; path != "" {
save(nodes.GetNodesV1(), path) if path := nodes.config.Nodes.NodesPath; path != "" {
} version := nodes.config.Nodes.NodesVersion
if path := nodes.config.Nodes.NodesV2Path; path != "" { switch version {
save(nodes.GetNodesV2(), path) case 1:
save(nodes.GetNodesV1(), path)
case 2:
save(nodes.GetNodesV2(), path)
default:
log.Panicf("invalid nodes version: %d", version)
}
} }
if path := nodes.config.Nodes.GraphsPath; path != "" { if path := nodes.config.Nodes.GraphsPath; path != "" {

View File

@ -13,7 +13,7 @@ import (
func TestExpire(t *testing.T) { func TestExpire(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)
config := &Config{} config := &Config{}
config.Nodes.MaxAge = 6 config.Nodes.PruneAfter.Duration = time.Hour * 24 * 6
nodes := &Nodes{ nodes := &Nodes{
config: config, config: config,
List: make(map[string]*Node), List: make(map[string]*Node),