Make configuration more intuitive and consistent
This commit is contained in:
parent
724cd8ba51
commit
c66e1120d3
|
@ -23,4 +23,4 @@ _testmain.go
|
|||
*.test
|
||||
*.prof
|
||||
webroot
|
||||
/config.yml
|
||||
/config.toml
|
||||
|
|
|
@ -11,7 +11,7 @@
|
|||
## Usage
|
||||
```
|
||||
Usage of ./respond-collector:
|
||||
-config path/to/config.yml
|
||||
-config path/to/config.toml
|
||||
```
|
||||
|
||||
## Development
|
||||
|
|
|
@ -8,7 +8,6 @@ import (
|
|||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/NYTimes/gziphandler"
|
||||
"github.com/julienschmidt/httprouter"
|
||||
|
@ -31,7 +30,7 @@ var (
|
|||
func main() {
|
||||
var importPath string
|
||||
flag.StringVar(&importPath, "import", "", "import global statistics from the given RRD file, requires influxdb")
|
||||
flag.StringVar(&configFile, "config", "config.yml", "path of configuration file (default:config.yaml)")
|
||||
flag.StringVar(&configFile, "config", "config.toml", "path of configuration file (default:config.yaml)")
|
||||
flag.Parse()
|
||||
config = models.ReadConfigFile(configFile)
|
||||
|
||||
|
@ -49,9 +48,8 @@ func main() {
|
|||
nodes.Start()
|
||||
|
||||
if config.Respondd.Enable {
|
||||
collectInterval := time.Second * time.Duration(config.Respondd.CollectInterval)
|
||||
collector = respond.NewCollector(db, nodes, config.Respondd.Interface)
|
||||
collector.Start(collectInterval)
|
||||
collector.Start(config.Respondd.CollectInterval.Duration)
|
||||
defer collector.Close()
|
||||
}
|
||||
|
||||
|
|
|
@ -0,0 +1,31 @@
|
|||
[respondd]
|
||||
enable = true
|
||||
interface = "eth0"
|
||||
collect_interval = "1m"
|
||||
|
||||
[webserver]
|
||||
enable = false
|
||||
port = "8080"
|
||||
address = "127.0.0.1"
|
||||
webroot = "webroot"
|
||||
|
||||
[nodes]
|
||||
enable = true
|
||||
nodes_version = 2
|
||||
nodes_path = "/var/www/html/meshviewer/data/nodes_all.json"
|
||||
graphs_path = "/var/www/html/meshviewer/data/graph.json"
|
||||
aliases_path = "/var/www/html/meshviewer/data/aliases.json"
|
||||
|
||||
# Export nodes and graph periodically
|
||||
save_interval = "5s"
|
||||
|
||||
# Prune offline nodes after a time of inactivity
|
||||
prune_after = "7d"
|
||||
|
||||
|
||||
[influxdb]
|
||||
enable = false
|
||||
address = "http://localhost:8086"
|
||||
database = "ffhb"
|
||||
username = ""
|
||||
password = ""
|
|
@ -1,35 +0,0 @@
|
|||
---
|
||||
respondd:
|
||||
enable: true
|
||||
interface: eth0
|
||||
|
||||
# Collected data every n seconds
|
||||
collectinterval: 60
|
||||
webserver:
|
||||
enable: false
|
||||
port: 8080
|
||||
address: 127.0.0.1
|
||||
webroot: webroot
|
||||
api:
|
||||
newnodes: true
|
||||
aliases: true
|
||||
nodes:
|
||||
enable: true
|
||||
nodes_path: /var/www/html/meshviewer/data/nodes_all.json
|
||||
nodesmini_path: /var/www/html/meshviewer/data/nodes.json
|
||||
graphs_path: /var/www/html/meshviewer/data/graph.json
|
||||
aliases_enable: false
|
||||
aliases_path: /var/www/html/meshviewer/data/aliases.json
|
||||
|
||||
# Export nodes and graph every n seconds
|
||||
saveinterval: 5
|
||||
|
||||
# Expire offline nodes after n days
|
||||
max_age: 7
|
||||
|
||||
influxdb:
|
||||
enable: false
|
||||
host: http://localhost:8086
|
||||
database: ffhb
|
||||
username:
|
||||
password:
|
|
@ -30,7 +30,7 @@ type DB struct {
|
|||
func New(config *models.Config) *DB {
|
||||
// Make client
|
||||
c, err := client.NewHTTPClient(client.HTTPConfig{
|
||||
Addr: config.Influxdb.Addr,
|
||||
Addr: config.Influxdb.Address,
|
||||
Username: config.Influxdb.Username,
|
||||
Password: config.Influxdb.Password,
|
||||
})
|
||||
|
@ -54,8 +54,7 @@ func New(config *models.Config) *DB {
|
|||
}
|
||||
|
||||
func (db *DB) DeletePoints() {
|
||||
query := fmt.Sprintf("delete from %s where time < now() - %dm", MeasurementNode, db.config.Influxdb.DeleteTill)
|
||||
log.Println("delete", MeasurementNode, "older than", db.config.Influxdb.DeleteTill, "minutes")
|
||||
query := fmt.Sprintf("delete from %s where time < now() - %ds", MeasurementNode, db.config.Influxdb.DeleteAfter.Duration/time.Second)
|
||||
db.client.Query(client.NewQuery(query, db.config.Influxdb.Database, "m"))
|
||||
}
|
||||
|
||||
|
@ -100,8 +99,7 @@ func (db *DB) Close() {
|
|||
|
||||
// prunes node-specific data periodically
|
||||
func (db *DB) deleteWorker() {
|
||||
duration := time.Minute * time.Duration(db.config.Influxdb.DeleteInterval)
|
||||
ticker := time.NewTicker(duration)
|
||||
ticker := time.NewTicker(db.config.Influxdb.DeleteInterval.Duration)
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
|
@ -123,7 +121,7 @@ func (db *DB) addWorker() {
|
|||
var bp client.BatchPoints
|
||||
var err error
|
||||
var writeNow, closed bool
|
||||
batchDuration := time.Second * time.Duration(db.config.Influxdb.SaveInterval)
|
||||
batchDuration := db.config.Influxdb.SaveInterval.Duration
|
||||
timer := time.NewTimer(batchDuration)
|
||||
|
||||
for !closed {
|
||||
|
|
|
@ -2,58 +2,61 @@ package models
|
|||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"log"
|
||||
|
||||
"gopkg.in/yaml.v2"
|
||||
"github.com/influxdata/toml"
|
||||
)
|
||||
|
||||
//Config the config File of this daemon
|
||||
type Config struct {
|
||||
Respondd struct {
|
||||
Enable bool `yaml:"enable"`
|
||||
Interface string `yaml:"interface"`
|
||||
CollectInterval int `yaml:"collectinterval"`
|
||||
} `yaml:"respondd"`
|
||||
Enable bool
|
||||
Interface string
|
||||
CollectInterval Duration
|
||||
}
|
||||
Webserver struct {
|
||||
Enable bool `yaml:"enable"`
|
||||
Port string `yaml:"port"`
|
||||
Address string `yaml:"address"`
|
||||
Webroot string `yaml:"webroot"`
|
||||
Enable bool
|
||||
Port string
|
||||
Address string
|
||||
Webroot string
|
||||
API struct {
|
||||
Passphrase string `yaml:"passphrase"`
|
||||
NewNodes bool `yaml:"newnodes"`
|
||||
Aliases bool `yaml:"aliases"`
|
||||
} `yaml:"api"`
|
||||
} `yaml:"webserver"`
|
||||
Passphrase string
|
||||
NewNodes bool
|
||||
Aliases bool
|
||||
}
|
||||
}
|
||||
Nodes struct {
|
||||
Enable bool `yaml:"enable"`
|
||||
NodesDynamicPath string `yaml:"nodes_path"`
|
||||
NodesV1Path string `yaml:"nodesv1_path"`
|
||||
NodesV2Path string `yaml:"nodesv2_path"`
|
||||
GraphsPath string `yaml:"graphs_path"`
|
||||
AliasesPath string `yaml:"aliases_path"`
|
||||
SaveInterval int `yaml:"saveinterval"` // Save nodes every n seconds
|
||||
MaxAge int `yaml:"max_age"` // Remove nodes after n days of inactivity
|
||||
} `yaml:"nodes"`
|
||||
Enable bool
|
||||
NodesDynamicPath string
|
||||
NodesPath string
|
||||
NodesVersion int
|
||||
GraphsPath string
|
||||
AliasesPath string
|
||||
SaveInterval Duration // Save nodes periodically
|
||||
PruneAfter Duration // Remove nodes after n days of inactivity
|
||||
}
|
||||
Influxdb struct {
|
||||
Enable bool `yaml:"enable"`
|
||||
Addr string `yaml:"host"`
|
||||
Database string `yaml:"database"`
|
||||
Username string `yaml:"username"`
|
||||
Password string `yaml:"password"`
|
||||
SaveInterval int `yaml:"saveinterval"` // Save nodes every n seconds
|
||||
DeleteInterval int `yaml:"deleteinterval"` // Delete stats of nodes every n minutes
|
||||
DeleteTill int `yaml:"deletetill"` // Delete stats of nodes till now-deletetill n minutes
|
||||
Enable bool
|
||||
Address string
|
||||
Database string
|
||||
Username string
|
||||
Password string
|
||||
SaveInterval Duration // Save nodes every n seconds
|
||||
DeleteInterval Duration // Delete stats of nodes every n minutes
|
||||
DeleteAfter Duration // Delete stats of nodes till now-deletetill n minutes
|
||||
}
|
||||
}
|
||||
|
||||
// ReadConfigFile reads a config model from path of a yml file
|
||||
func ReadConfigFile(path string) *Config {
|
||||
config := &Config{}
|
||||
file, _ := ioutil.ReadFile(path)
|
||||
err := yaml.Unmarshal(file, &config)
|
||||
file, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if err := toml.Unmarshal(file, config); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return config
|
||||
}
|
||||
|
|
|
@ -2,6 +2,7 @@ package models
|
|||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
@ -9,6 +10,14 @@ import (
|
|||
func TestReadConfig(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
config := ReadConfigFile("../config_example.yml")
|
||||
config := ReadConfigFile("../config_example.toml")
|
||||
assert.NotNil(config)
|
||||
|
||||
assert.True(config.Respondd.Enable)
|
||||
assert.Equal("eth0", config.Respondd.Interface)
|
||||
assert.Equal(time.Minute, config.Respondd.CollectInterval.Duration)
|
||||
|
||||
assert.Equal(2, config.Nodes.NodesVersion)
|
||||
assert.Equal("/var/www/html/meshviewer/data/nodes_all.json", config.Nodes.NodesPath)
|
||||
assert.Equal(time.Hour*24*7, config.Nodes.PruneAfter.Duration)
|
||||
}
|
||||
|
|
|
@ -0,0 +1,50 @@
|
|||
package models
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Duration is a TOML datatype
|
||||
// A duration string is a possibly signed sequence of
|
||||
// decimal numbers and a unit suffix,
|
||||
// such as "300s", "1.5h" or "5d".
|
||||
// Valid time units are "s", "m", "h", "d", "w".
|
||||
type Duration struct {
|
||||
time.Duration
|
||||
}
|
||||
|
||||
// UnmarshalTOML parses a duration string.
|
||||
func (d *Duration) UnmarshalTOML(data []byte) error {
|
||||
|
||||
// " + int + unit + "
|
||||
if len(data) < 4 {
|
||||
return fmt.Errorf("invalid duration: %s", data)
|
||||
}
|
||||
|
||||
unit := data[len(data)-2]
|
||||
value, err := strconv.Atoi(string(data[1 : len(data)-2]))
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to parse duration %s: %s", data, err)
|
||||
}
|
||||
|
||||
switch unit {
|
||||
case 's':
|
||||
d.Duration = time.Duration(value) * time.Second
|
||||
case 'm':
|
||||
d.Duration = time.Duration(value) * time.Minute
|
||||
case 'h':
|
||||
d.Duration = time.Duration(value) * time.Hour
|
||||
case 'd':
|
||||
d.Duration = time.Duration(value) * time.Hour * 24
|
||||
case 'w':
|
||||
d.Duration = time.Duration(value) * time.Hour * 24 * 7
|
||||
case 'y':
|
||||
d.Duration = time.Duration(value) * time.Hour * 24 * 365
|
||||
default:
|
||||
return fmt.Errorf("invalid duration unit: %s", string(unit))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
|
@ -0,0 +1,47 @@
|
|||
package models
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestDuration(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
var tests = []struct {
|
||||
input string
|
||||
err string
|
||||
duration time.Duration
|
||||
}{
|
||||
{"", "invalid duration: \"\"", 0},
|
||||
{"1x", "invalid duration unit: x", 0},
|
||||
{"1s", "", time.Second},
|
||||
{"73s", "", time.Second * 73},
|
||||
{"1m", "", time.Minute},
|
||||
{"73m", "", time.Minute * 73},
|
||||
{"1h", "", time.Hour},
|
||||
{"43h", "", time.Hour * 43},
|
||||
{"1d", "", time.Hour * 24},
|
||||
{"8d", "", time.Hour * 24 * 8},
|
||||
{"1w", "", time.Hour * 24 * 7},
|
||||
{"52w", "", time.Hour * 24 * 7 * 52},
|
||||
{"1y", "", time.Hour * 24 * 365},
|
||||
{"3y", "", time.Hour * 24 * 365 * 3},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
|
||||
d := Duration{}
|
||||
err := d.UnmarshalTOML([]byte("\"" + test.input + "\""))
|
||||
duration := d.Duration
|
||||
|
||||
if test.err == "" {
|
||||
assert.NoError(err)
|
||||
assert.Equal(test.duration, duration)
|
||||
} else {
|
||||
assert.EqualError(err, test.err)
|
||||
}
|
||||
}
|
||||
}
|
|
@ -140,7 +140,7 @@ func (nodes *Nodes) GetNodesV2() *meshviewer.NodesV2 {
|
|||
|
||||
// Periodically saves the cached DB to json file
|
||||
func (nodes *Nodes) worker() {
|
||||
c := time.Tick(time.Second * time.Duration(nodes.config.Nodes.SaveInterval))
|
||||
c := time.Tick(nodes.config.Nodes.SaveInterval.Duration)
|
||||
|
||||
for range c {
|
||||
nodes.expire()
|
||||
|
@ -153,11 +153,11 @@ func (nodes *Nodes) expire() {
|
|||
nodes.Timestamp = jsontime.Now()
|
||||
|
||||
// Nodes last seen before expireTime will be removed
|
||||
maxAge := nodes.config.Nodes.MaxAge
|
||||
if maxAge <= 0 {
|
||||
maxAge = 7 // our default
|
||||
pruneAfter := nodes.config.Nodes.PruneAfter.Duration
|
||||
if pruneAfter == 0 {
|
||||
pruneAfter = time.Hour * 24 * 7 // our default
|
||||
}
|
||||
expireTime := nodes.Timestamp.Add(-time.Duration(maxAge) * time.Hour * 24)
|
||||
expireTime := nodes.Timestamp.Add(-pruneAfter)
|
||||
|
||||
// Nodes last seen before offlineTime are changed to 'offline'
|
||||
offlineTime := nodes.Timestamp.Add(-time.Minute * 10)
|
||||
|
@ -198,11 +198,17 @@ func (nodes *Nodes) save() {
|
|||
|
||||
// serialize nodes
|
||||
save(nodes, nodes.config.Nodes.NodesDynamicPath)
|
||||
if path := nodes.config.Nodes.NodesV1Path; path != "" {
|
||||
|
||||
if path := nodes.config.Nodes.NodesPath; path != "" {
|
||||
version := nodes.config.Nodes.NodesVersion
|
||||
switch version {
|
||||
case 1:
|
||||
save(nodes.GetNodesV1(), path)
|
||||
}
|
||||
if path := nodes.config.Nodes.NodesV2Path; path != "" {
|
||||
case 2:
|
||||
save(nodes.GetNodesV2(), path)
|
||||
default:
|
||||
log.Panicf("invalid nodes version: %d", version)
|
||||
}
|
||||
}
|
||||
|
||||
if path := nodes.config.Nodes.GraphsPath; path != "" {
|
||||
|
|
|
@ -13,7 +13,7 @@ import (
|
|||
func TestExpire(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
config := &Config{}
|
||||
config.Nodes.MaxAge = 6
|
||||
config.Nodes.PruneAfter.Duration = time.Hour * 24 * 6
|
||||
nodes := &Nodes{
|
||||
config: config,
|
||||
List: make(map[string]*Node),
|
||||
|
|
Loading…
Reference in New Issue