Make configuration more intuitive and consistent

.gitignore

@@ -23,4 +23,4 @@ _testmain.go
 *.test
 *.prof
 webroot
-/config.yml
+/config.toml

README.md

@@ -11,7 +11,7 @@
 ## Usage
 ```
 Usage of ./respond-collector:
-  -config path/to/config.yml
+  -config path/to/config.toml
 ```
 
 ## Development

cmd/respond-collector/main.go

@@ -8,7 +8,6 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
-	"time"
 
 	"github.com/NYTimes/gziphandler"
 	"github.com/julienschmidt/httprouter"
@@ -31,7 +30,7 @@ var (
 func main() {
 	var importPath string
 	flag.StringVar(&importPath, "import", "", "import global statistics from the given RRD file, requires influxdb")
-	flag.StringVar(&configFile, "config", "config.yml", "path of configuration file (default:config.yaml)")
+	flag.StringVar(&configFile, "config", "config.toml", "path of configuration file (default:config.yaml)")
 	flag.Parse()
 	config = models.ReadConfigFile(configFile)
 
@@ -49,9 +48,8 @@ func main() {
 	nodes.Start()
 
 	if config.Respondd.Enable {
-		collectInterval := time.Second * time.Duration(config.Respondd.CollectInterval)
 		collector = respond.NewCollector(db, nodes, config.Respondd.Interface)
-		collector.Start(collectInterval)
+		collector.Start(config.Respondd.CollectInterval.Duration)
 		defer collector.Close()
 	}
 

config_example.toml

@@ -0,0 +1,31 @@
+[respondd]
+enable           = true
+interface        = "eth0"
+collect_interval = "1m"
+
+[webserver]
+enable  = false
+port    = "8080"
+address = "127.0.0.1"
+webroot = "webroot"
+
+[nodes]
+enable         = true
+nodes_version  = 2
+nodes_path     = "/var/www/html/meshviewer/data/nodes_all.json"
+graphs_path    = "/var/www/html/meshviewer/data/graph.json"
+aliases_path   = "/var/www/html/meshviewer/data/aliases.json"
+
+# Export nodes and graph periodically
+save_interval = "5s"
+
+# Prune offline nodes after a time of inactivity
+prune_after = "7d"
+
+
+[influxdb]
+enable   = false
+address  = "http://localhost:8086"
+database = "ffhb"
+username = ""
+password = ""

config_example.yml

@@ -1,35 +0,0 @@
----
-respondd:
-  enable: true
-  interface: eth0
-
-  # Collected data every n seconds
-  collectinterval: 60
-webserver:
-  enable: false
-  port: 8080
-  address: 127.0.0.1
-  webroot: webroot
-  api:
-    newnodes: true
-    aliases: true
-nodes:
-  enable: true
-  nodes_path: /var/www/html/meshviewer/data/nodes_all.json
-  nodesmini_path: /var/www/html/meshviewer/data/nodes.json
-  graphs_path: /var/www/html/meshviewer/data/graph.json
-  aliases_enable: false
-  aliases_path: /var/www/html/meshviewer/data/aliases.json
-
-  # Export nodes and graph every n seconds
-  saveinterval: 5
-
-  # Expire offline nodes after n days
-  max_age: 7
-
-influxdb:
-  enable: false
-  host: http://localhost:8086
-  database: ffhb
-  username:
-  password:

database/database.go

@@ -30,7 +30,7 @@ type DB struct {
 func New(config *models.Config) *DB {
 	// Make client
 	c, err := client.NewHTTPClient(client.HTTPConfig{
-		Addr:     config.Influxdb.Addr,
+		Addr:     config.Influxdb.Address,
 		Username: config.Influxdb.Username,
 		Password: config.Influxdb.Password,
 	})
@@ -54,8 +54,7 @@ func New(config *models.Config) *DB {
 }
 
 func (db *DB) DeletePoints() {
-	query := fmt.Sprintf("delete from %s where time < now() - %dm", MeasurementNode, db.config.Influxdb.DeleteTill)
-	log.Println("delete", MeasurementNode, "older than", db.config.Influxdb.DeleteTill, "minutes")
+	query := fmt.Sprintf("delete from %s where time < now() - %ds", MeasurementNode, db.config.Influxdb.DeleteAfter.Duration/time.Second)
 	db.client.Query(client.NewQuery(query, db.config.Influxdb.Database, "m"))
 }
 
@@ -100,8 +99,7 @@ func (db *DB) Close() {
 
 // prunes node-specific data periodically
 func (db *DB) deleteWorker() {
-	duration := time.Minute * time.Duration(db.config.Influxdb.DeleteInterval)
-	ticker := time.NewTicker(duration)
+	ticker := time.NewTicker(db.config.Influxdb.DeleteInterval.Duration)
 	for {
 		select {
 		case <-ticker.C:
@@ -123,7 +121,7 @@ func (db *DB) addWorker() {
 	var bp client.BatchPoints
 	var err error
 	var writeNow, closed bool
-	batchDuration := time.Second * time.Duration(db.config.Influxdb.SaveInterval)
+	batchDuration := db.config.Influxdb.SaveInterval.Duration
 	timer := time.NewTimer(batchDuration)
 
 	for !closed {

models/config.go

@@ -2,58 +2,61 @@ package models
 
 import (
 	"io/ioutil"
-	"log"
 
-	"gopkg.in/yaml.v2"
+	"github.com/influxdata/toml"
 )
 
 //Config the config File of this daemon
 type Config struct {
 	Respondd struct {
-		Enable          bool   `yaml:"enable"`
-		Interface       string `yaml:"interface"`
-		CollectInterval int    `yaml:"collectinterval"`
-	} `yaml:"respondd"`
+		Enable          bool
+		Interface       string
+		CollectInterval Duration
+	}
 	Webserver struct {
-		Enable  bool   `yaml:"enable"`
-		Port    string `yaml:"port"`
-		Address string `yaml:"address"`
-		Webroot string `yaml:"webroot"`
+		Enable  bool
+		Port    string
+		Address string
+		Webroot string
 		API     struct {
-			Passphrase string `yaml:"passphrase"`
-			NewNodes   bool   `yaml:"newnodes"`
-			Aliases    bool   `yaml:"aliases"`
-		} `yaml:"api"`
-	} `yaml:"webserver"`
+			Passphrase string
+			NewNodes   bool
+			Aliases    bool
+		}
+	}
 	Nodes struct {
-		Enable           bool   `yaml:"enable"`
-		NodesDynamicPath string `yaml:"nodes_path"`
-		NodesV1Path      string `yaml:"nodesv1_path"`
-		NodesV2Path      string `yaml:"nodesv2_path"`
-		GraphsPath       string `yaml:"graphs_path"`
-		AliasesPath      string `yaml:"aliases_path"`
-		SaveInterval     int    `yaml:"saveinterval"` // Save nodes every n seconds
-		MaxAge           int    `yaml:"max_age"`      // Remove nodes after n days of inactivity
-	} `yaml:"nodes"`
+		Enable           bool
+		NodesDynamicPath string
+		NodesPath        string
+		NodesVersion     int
+		GraphsPath       string
+		AliasesPath      string
+		SaveInterval     Duration // Save nodes periodically
+		PruneAfter       Duration // Remove nodes after n days of inactivity
+	}
 	Influxdb struct {
-		Enable         bool   `yaml:"enable"`
-		Addr           string `yaml:"host"`
-		Database       string `yaml:"database"`
-		Username       string `yaml:"username"`
-		Password       string `yaml:"password"`
-		SaveInterval   int    `yaml:"saveinterval"`   // Save nodes every n seconds
-		DeleteInterval int    `yaml:"deleteinterval"` // Delete stats of nodes every n minutes
-		DeleteTill     int    `yaml:"deletetill"`     // Delete stats of nodes till now-deletetill n minutes
+		Enable         bool
+		Address        string
+		Database       string
+		Username       string
+		Password       string
+		SaveInterval   Duration // Save nodes every n seconds
+		DeleteInterval Duration // Delete stats of nodes every n minutes
+		DeleteAfter    Duration // Delete stats of nodes till now-deletetill n minutes
 	}
 }
 
 // ReadConfigFile reads a config model from path of a yml file
 func ReadConfigFile(path string) *Config {
 	config := &Config{}
-	file, _ := ioutil.ReadFile(path)
-	err := yaml.Unmarshal(file, &config)
+	file, err := ioutil.ReadFile(path)
 	if err != nil {
-		log.Fatal(err)
+		panic(err)
 	}
+
+	if err := toml.Unmarshal(file, config); err != nil {
+		panic(err)
+	}
+
 	return config
 }

models/config_test.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -9,6 +10,14 @@ import (
 func TestReadConfig(t *testing.T) {
 	assert := assert.New(t)
 
-	config := ReadConfigFile("../config_example.yml")
+	config := ReadConfigFile("../config_example.toml")
 	assert.NotNil(config)
+
+	assert.True(config.Respondd.Enable)
+	assert.Equal("eth0", config.Respondd.Interface)
+	assert.Equal(time.Minute, config.Respondd.CollectInterval.Duration)
+
+	assert.Equal(2, config.Nodes.NodesVersion)
+	assert.Equal("/var/www/html/meshviewer/data/nodes_all.json", config.Nodes.NodesPath)
+	assert.Equal(time.Hour*24*7, config.Nodes.PruneAfter.Duration)
 }

models/duration.go

@@ -0,0 +1,50 @@
+package models
+
+import (
+	"fmt"
+	"strconv"
+	"time"
+)
+
+// Duration is a TOML datatype
+// A duration string is a possibly signed sequence of
+// decimal numbers  and a unit suffix,
+// such as "300s", "1.5h" or "5d".
+// Valid time units are "s", "m", "h", "d", "w".
+type Duration struct {
+	time.Duration
+}
+
+// UnmarshalTOML parses a duration string.
+func (d *Duration) UnmarshalTOML(data []byte) error {
+
+	// " + int + unit + "
+	if len(data) < 4 {
+		return fmt.Errorf("invalid duration: %s", data)
+	}
+
+	unit := data[len(data)-2]
+	value, err := strconv.Atoi(string(data[1 : len(data)-2]))
+	if err != nil {
+		return fmt.Errorf("unable to parse duration %s: %s", data, err)
+	}
+
+	switch unit {
+	case 's':
+		d.Duration = time.Duration(value) * time.Second
+	case 'm':
+		d.Duration = time.Duration(value) * time.Minute
+	case 'h':
+		d.Duration = time.Duration(value) * time.Hour
+	case 'd':
+		d.Duration = time.Duration(value) * time.Hour * 24
+	case 'w':
+		d.Duration = time.Duration(value) * time.Hour * 24 * 7
+	case 'y':
+		d.Duration = time.Duration(value) * time.Hour * 24 * 365
+	default:
+		return fmt.Errorf("invalid duration unit: %s", string(unit))
+	}
+
+	return nil
+}

models/duration_test.go

@@ -0,0 +1,47 @@
+package models
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDuration(t *testing.T) {
+	assert := assert.New(t)
+
+	var tests = []struct {
+		input    string
+		err      string
+		duration time.Duration
+	}{
+		{"", "invalid duration: \"\"", 0},
+		{"1x", "invalid duration unit: x", 0},
+		{"1s", "", time.Second},
+		{"73s", "", time.Second * 73},
+		{"1m", "", time.Minute},
+		{"73m", "", time.Minute * 73},
+		{"1h", "", time.Hour},
+		{"43h", "", time.Hour * 43},
+		{"1d", "", time.Hour * 24},
+		{"8d", "", time.Hour * 24 * 8},
+		{"1w", "", time.Hour * 24 * 7},
+		{"52w", "", time.Hour * 24 * 7 * 52},
+		{"1y", "", time.Hour * 24 * 365},
+		{"3y", "", time.Hour * 24 * 365 * 3},
+	}
+
+	for _, test := range tests {
+
+		d := Duration{}
+		err := d.UnmarshalTOML([]byte("\"" + test.input + "\""))
+		duration := d.Duration
+
+		if test.err == "" {
+			assert.NoError(err)
+			assert.Equal(test.duration, duration)
+		} else {
+			assert.EqualError(err, test.err)
+		}
+	}
+}

models/nodes.go

@@ -140,7 +140,7 @@ func (nodes *Nodes) GetNodesV2() *meshviewer.NodesV2 {
 
 // Periodically saves the cached DB to json file
 func (nodes *Nodes) worker() {
-	c := time.Tick(time.Second * time.Duration(nodes.config.Nodes.SaveInterval))
+	c := time.Tick(nodes.config.Nodes.SaveInterval.Duration)
 
 	for range c {
 		nodes.expire()
@@ -153,11 +153,11 @@ func (nodes *Nodes) expire() {
 	nodes.Timestamp = jsontime.Now()
 
 	// Nodes last seen before expireTime will be removed
-	maxAge := nodes.config.Nodes.MaxAge
-	if maxAge <= 0 {
-		maxAge = 7 // our default
+	pruneAfter := nodes.config.Nodes.PruneAfter.Duration
+	if pruneAfter == 0 {
+		pruneAfter = time.Hour * 24 * 7 // our default
 	}
-	expireTime := nodes.Timestamp.Add(-time.Duration(maxAge) * time.Hour * 24)
+	expireTime := nodes.Timestamp.Add(-pruneAfter)
 
 	// Nodes last seen before offlineTime are changed to 'offline'
 	offlineTime := nodes.Timestamp.Add(-time.Minute * 10)
@@ -198,11 +198,17 @@ func (nodes *Nodes) save() {
 
 	// serialize nodes
 	save(nodes, nodes.config.Nodes.NodesDynamicPath)
-	if path := nodes.config.Nodes.NodesV1Path; path != "" {
-		save(nodes.GetNodesV1(), path)
-	}
-	if path := nodes.config.Nodes.NodesV2Path; path != "" {
-		save(nodes.GetNodesV2(), path)
+
+	if path := nodes.config.Nodes.NodesPath; path != "" {
+		version := nodes.config.Nodes.NodesVersion
+		switch version {
+		case 1:
+			save(nodes.GetNodesV1(), path)
+		case 2:
+			save(nodes.GetNodesV2(), path)
+		default:
+			log.Panicf("invalid nodes version: %d", version)
+		}
 	}
 
 	if path := nodes.config.Nodes.GraphsPath; path != "" {

models/nodes_test.go

@@ -13,7 +13,7 @@ import (
 func TestExpire(t *testing.T) {
 	assert := assert.New(t)
 	config := &Config{}
-	config.Nodes.MaxAge = 6
+	config.Nodes.PruneAfter.Duration = time.Hour * 24 * 6
 	nodes := &Nodes{
 		config: config,
 		List:   make(map[string]*Node),