Make configuration more intuitive and consistent
This commit is contained in:
parent
724cd8ba51
commit
c66e1120d3
|
@ -23,4 +23,4 @@ _testmain.go
|
||||||
*.test
|
*.test
|
||||||
*.prof
|
*.prof
|
||||||
webroot
|
webroot
|
||||||
/config.yml
|
/config.toml
|
||||||
|
|
|
@ -11,7 +11,7 @@
|
||||||
## Usage
|
## Usage
|
||||||
```
|
```
|
||||||
Usage of ./respond-collector:
|
Usage of ./respond-collector:
|
||||||
-config path/to/config.yml
|
-config path/to/config.toml
|
||||||
```
|
```
|
||||||
|
|
||||||
## Development
|
## Development
|
||||||
|
|
|
@ -8,7 +8,6 @@ import (
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/NYTimes/gziphandler"
|
"github.com/NYTimes/gziphandler"
|
||||||
"github.com/julienschmidt/httprouter"
|
"github.com/julienschmidt/httprouter"
|
||||||
|
@ -31,7 +30,7 @@ var (
|
||||||
func main() {
|
func main() {
|
||||||
var importPath string
|
var importPath string
|
||||||
flag.StringVar(&importPath, "import", "", "import global statistics from the given RRD file, requires influxdb")
|
flag.StringVar(&importPath, "import", "", "import global statistics from the given RRD file, requires influxdb")
|
||||||
flag.StringVar(&configFile, "config", "config.yml", "path of configuration file (default:config.yaml)")
|
flag.StringVar(&configFile, "config", "config.toml", "path of configuration file (default:config.yaml)")
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
config = models.ReadConfigFile(configFile)
|
config = models.ReadConfigFile(configFile)
|
||||||
|
|
||||||
|
@ -49,9 +48,8 @@ func main() {
|
||||||
nodes.Start()
|
nodes.Start()
|
||||||
|
|
||||||
if config.Respondd.Enable {
|
if config.Respondd.Enable {
|
||||||
collectInterval := time.Second * time.Duration(config.Respondd.CollectInterval)
|
|
||||||
collector = respond.NewCollector(db, nodes, config.Respondd.Interface)
|
collector = respond.NewCollector(db, nodes, config.Respondd.Interface)
|
||||||
collector.Start(collectInterval)
|
collector.Start(config.Respondd.CollectInterval.Duration)
|
||||||
defer collector.Close()
|
defer collector.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,31 @@
|
||||||
|
[respondd]
|
||||||
|
enable = true
|
||||||
|
interface = "eth0"
|
||||||
|
collect_interval = "1m"
|
||||||
|
|
||||||
|
[webserver]
|
||||||
|
enable = false
|
||||||
|
port = "8080"
|
||||||
|
address = "127.0.0.1"
|
||||||
|
webroot = "webroot"
|
||||||
|
|
||||||
|
[nodes]
|
||||||
|
enable = true
|
||||||
|
nodes_version = 2
|
||||||
|
nodes_path = "/var/www/html/meshviewer/data/nodes_all.json"
|
||||||
|
graphs_path = "/var/www/html/meshviewer/data/graph.json"
|
||||||
|
aliases_path = "/var/www/html/meshviewer/data/aliases.json"
|
||||||
|
|
||||||
|
# Export nodes and graph periodically
|
||||||
|
save_interval = "5s"
|
||||||
|
|
||||||
|
# Prune offline nodes after a time of inactivity
|
||||||
|
prune_after = "7d"
|
||||||
|
|
||||||
|
|
||||||
|
[influxdb]
|
||||||
|
enable = false
|
||||||
|
address = "http://localhost:8086"
|
||||||
|
database = "ffhb"
|
||||||
|
username = ""
|
||||||
|
password = ""
|
|
@ -1,35 +0,0 @@
|
||||||
---
|
|
||||||
respondd:
|
|
||||||
enable: true
|
|
||||||
interface: eth0
|
|
||||||
|
|
||||||
# Collected data every n seconds
|
|
||||||
collectinterval: 60
|
|
||||||
webserver:
|
|
||||||
enable: false
|
|
||||||
port: 8080
|
|
||||||
address: 127.0.0.1
|
|
||||||
webroot: webroot
|
|
||||||
api:
|
|
||||||
newnodes: true
|
|
||||||
aliases: true
|
|
||||||
nodes:
|
|
||||||
enable: true
|
|
||||||
nodes_path: /var/www/html/meshviewer/data/nodes_all.json
|
|
||||||
nodesmini_path: /var/www/html/meshviewer/data/nodes.json
|
|
||||||
graphs_path: /var/www/html/meshviewer/data/graph.json
|
|
||||||
aliases_enable: false
|
|
||||||
aliases_path: /var/www/html/meshviewer/data/aliases.json
|
|
||||||
|
|
||||||
# Export nodes and graph every n seconds
|
|
||||||
saveinterval: 5
|
|
||||||
|
|
||||||
# Expire offline nodes after n days
|
|
||||||
max_age: 7
|
|
||||||
|
|
||||||
influxdb:
|
|
||||||
enable: false
|
|
||||||
host: http://localhost:8086
|
|
||||||
database: ffhb
|
|
||||||
username:
|
|
||||||
password:
|
|
|
@ -30,7 +30,7 @@ type DB struct {
|
||||||
func New(config *models.Config) *DB {
|
func New(config *models.Config) *DB {
|
||||||
// Make client
|
// Make client
|
||||||
c, err := client.NewHTTPClient(client.HTTPConfig{
|
c, err := client.NewHTTPClient(client.HTTPConfig{
|
||||||
Addr: config.Influxdb.Addr,
|
Addr: config.Influxdb.Address,
|
||||||
Username: config.Influxdb.Username,
|
Username: config.Influxdb.Username,
|
||||||
Password: config.Influxdb.Password,
|
Password: config.Influxdb.Password,
|
||||||
})
|
})
|
||||||
|
@ -54,8 +54,7 @@ func New(config *models.Config) *DB {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *DB) DeletePoints() {
|
func (db *DB) DeletePoints() {
|
||||||
query := fmt.Sprintf("delete from %s where time < now() - %dm", MeasurementNode, db.config.Influxdb.DeleteTill)
|
query := fmt.Sprintf("delete from %s where time < now() - %ds", MeasurementNode, db.config.Influxdb.DeleteAfter.Duration/time.Second)
|
||||||
log.Println("delete", MeasurementNode, "older than", db.config.Influxdb.DeleteTill, "minutes")
|
|
||||||
db.client.Query(client.NewQuery(query, db.config.Influxdb.Database, "m"))
|
db.client.Query(client.NewQuery(query, db.config.Influxdb.Database, "m"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -100,8 +99,7 @@ func (db *DB) Close() {
|
||||||
|
|
||||||
// prunes node-specific data periodically
|
// prunes node-specific data periodically
|
||||||
func (db *DB) deleteWorker() {
|
func (db *DB) deleteWorker() {
|
||||||
duration := time.Minute * time.Duration(db.config.Influxdb.DeleteInterval)
|
ticker := time.NewTicker(db.config.Influxdb.DeleteInterval.Duration)
|
||||||
ticker := time.NewTicker(duration)
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
|
@ -123,7 +121,7 @@ func (db *DB) addWorker() {
|
||||||
var bp client.BatchPoints
|
var bp client.BatchPoints
|
||||||
var err error
|
var err error
|
||||||
var writeNow, closed bool
|
var writeNow, closed bool
|
||||||
batchDuration := time.Second * time.Duration(db.config.Influxdb.SaveInterval)
|
batchDuration := db.config.Influxdb.SaveInterval.Duration
|
||||||
timer := time.NewTimer(batchDuration)
|
timer := time.NewTimer(batchDuration)
|
||||||
|
|
||||||
for !closed {
|
for !closed {
|
||||||
|
|
|
@ -2,58 +2,61 @@ package models
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"log"
|
|
||||||
|
|
||||||
"gopkg.in/yaml.v2"
|
"github.com/influxdata/toml"
|
||||||
)
|
)
|
||||||
|
|
||||||
//Config the config File of this daemon
|
//Config the config File of this daemon
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Respondd struct {
|
Respondd struct {
|
||||||
Enable bool `yaml:"enable"`
|
Enable bool
|
||||||
Interface string `yaml:"interface"`
|
Interface string
|
||||||
CollectInterval int `yaml:"collectinterval"`
|
CollectInterval Duration
|
||||||
} `yaml:"respondd"`
|
}
|
||||||
Webserver struct {
|
Webserver struct {
|
||||||
Enable bool `yaml:"enable"`
|
Enable bool
|
||||||
Port string `yaml:"port"`
|
Port string
|
||||||
Address string `yaml:"address"`
|
Address string
|
||||||
Webroot string `yaml:"webroot"`
|
Webroot string
|
||||||
API struct {
|
API struct {
|
||||||
Passphrase string `yaml:"passphrase"`
|
Passphrase string
|
||||||
NewNodes bool `yaml:"newnodes"`
|
NewNodes bool
|
||||||
Aliases bool `yaml:"aliases"`
|
Aliases bool
|
||||||
} `yaml:"api"`
|
}
|
||||||
} `yaml:"webserver"`
|
}
|
||||||
Nodes struct {
|
Nodes struct {
|
||||||
Enable bool `yaml:"enable"`
|
Enable bool
|
||||||
NodesDynamicPath string `yaml:"nodes_path"`
|
NodesDynamicPath string
|
||||||
NodesV1Path string `yaml:"nodesv1_path"`
|
NodesPath string
|
||||||
NodesV2Path string `yaml:"nodesv2_path"`
|
NodesVersion int
|
||||||
GraphsPath string `yaml:"graphs_path"`
|
GraphsPath string
|
||||||
AliasesPath string `yaml:"aliases_path"`
|
AliasesPath string
|
||||||
SaveInterval int `yaml:"saveinterval"` // Save nodes every n seconds
|
SaveInterval Duration // Save nodes periodically
|
||||||
MaxAge int `yaml:"max_age"` // Remove nodes after n days of inactivity
|
PruneAfter Duration // Remove nodes after n days of inactivity
|
||||||
} `yaml:"nodes"`
|
}
|
||||||
Influxdb struct {
|
Influxdb struct {
|
||||||
Enable bool `yaml:"enable"`
|
Enable bool
|
||||||
Addr string `yaml:"host"`
|
Address string
|
||||||
Database string `yaml:"database"`
|
Database string
|
||||||
Username string `yaml:"username"`
|
Username string
|
||||||
Password string `yaml:"password"`
|
Password string
|
||||||
SaveInterval int `yaml:"saveinterval"` // Save nodes every n seconds
|
SaveInterval Duration // Save nodes every n seconds
|
||||||
DeleteInterval int `yaml:"deleteinterval"` // Delete stats of nodes every n minutes
|
DeleteInterval Duration // Delete stats of nodes every n minutes
|
||||||
DeleteTill int `yaml:"deletetill"` // Delete stats of nodes till now-deletetill n minutes
|
DeleteAfter Duration // Delete stats of nodes till now-deletetill n minutes
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReadConfigFile reads a config model from path of a yml file
|
// ReadConfigFile reads a config model from path of a yml file
|
||||||
func ReadConfigFile(path string) *Config {
|
func ReadConfigFile(path string) *Config {
|
||||||
config := &Config{}
|
config := &Config{}
|
||||||
file, _ := ioutil.ReadFile(path)
|
file, err := ioutil.ReadFile(path)
|
||||||
err := yaml.Unmarshal(file, &config)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err := toml.Unmarshal(file, config); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
return config
|
return config
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,6 +2,7 @@ package models
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
@ -9,6 +10,14 @@ import (
|
||||||
func TestReadConfig(t *testing.T) {
|
func TestReadConfig(t *testing.T) {
|
||||||
assert := assert.New(t)
|
assert := assert.New(t)
|
||||||
|
|
||||||
config := ReadConfigFile("../config_example.yml")
|
config := ReadConfigFile("../config_example.toml")
|
||||||
assert.NotNil(config)
|
assert.NotNil(config)
|
||||||
|
|
||||||
|
assert.True(config.Respondd.Enable)
|
||||||
|
assert.Equal("eth0", config.Respondd.Interface)
|
||||||
|
assert.Equal(time.Minute, config.Respondd.CollectInterval.Duration)
|
||||||
|
|
||||||
|
assert.Equal(2, config.Nodes.NodesVersion)
|
||||||
|
assert.Equal("/var/www/html/meshviewer/data/nodes_all.json", config.Nodes.NodesPath)
|
||||||
|
assert.Equal(time.Hour*24*7, config.Nodes.PruneAfter.Duration)
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,50 @@
|
||||||
|
package models
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Duration is a TOML datatype
|
||||||
|
// A duration string is a possibly signed sequence of
|
||||||
|
// decimal numbers and a unit suffix,
|
||||||
|
// such as "300s", "1.5h" or "5d".
|
||||||
|
// Valid time units are "s", "m", "h", "d", "w".
|
||||||
|
type Duration struct {
|
||||||
|
time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalTOML parses a duration string.
|
||||||
|
func (d *Duration) UnmarshalTOML(data []byte) error {
|
||||||
|
|
||||||
|
// " + int + unit + "
|
||||||
|
if len(data) < 4 {
|
||||||
|
return fmt.Errorf("invalid duration: %s", data)
|
||||||
|
}
|
||||||
|
|
||||||
|
unit := data[len(data)-2]
|
||||||
|
value, err := strconv.Atoi(string(data[1 : len(data)-2]))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("unable to parse duration %s: %s", data, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
switch unit {
|
||||||
|
case 's':
|
||||||
|
d.Duration = time.Duration(value) * time.Second
|
||||||
|
case 'm':
|
||||||
|
d.Duration = time.Duration(value) * time.Minute
|
||||||
|
case 'h':
|
||||||
|
d.Duration = time.Duration(value) * time.Hour
|
||||||
|
case 'd':
|
||||||
|
d.Duration = time.Duration(value) * time.Hour * 24
|
||||||
|
case 'w':
|
||||||
|
d.Duration = time.Duration(value) * time.Hour * 24 * 7
|
||||||
|
case 'y':
|
||||||
|
d.Duration = time.Duration(value) * time.Hour * 24 * 365
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("invalid duration unit: %s", string(unit))
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
|
@ -0,0 +1,47 @@
|
||||||
|
package models
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDuration(t *testing.T) {
|
||||||
|
assert := assert.New(t)
|
||||||
|
|
||||||
|
var tests = []struct {
|
||||||
|
input string
|
||||||
|
err string
|
||||||
|
duration time.Duration
|
||||||
|
}{
|
||||||
|
{"", "invalid duration: \"\"", 0},
|
||||||
|
{"1x", "invalid duration unit: x", 0},
|
||||||
|
{"1s", "", time.Second},
|
||||||
|
{"73s", "", time.Second * 73},
|
||||||
|
{"1m", "", time.Minute},
|
||||||
|
{"73m", "", time.Minute * 73},
|
||||||
|
{"1h", "", time.Hour},
|
||||||
|
{"43h", "", time.Hour * 43},
|
||||||
|
{"1d", "", time.Hour * 24},
|
||||||
|
{"8d", "", time.Hour * 24 * 8},
|
||||||
|
{"1w", "", time.Hour * 24 * 7},
|
||||||
|
{"52w", "", time.Hour * 24 * 7 * 52},
|
||||||
|
{"1y", "", time.Hour * 24 * 365},
|
||||||
|
{"3y", "", time.Hour * 24 * 365 * 3},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
|
||||||
|
d := Duration{}
|
||||||
|
err := d.UnmarshalTOML([]byte("\"" + test.input + "\""))
|
||||||
|
duration := d.Duration
|
||||||
|
|
||||||
|
if test.err == "" {
|
||||||
|
assert.NoError(err)
|
||||||
|
assert.Equal(test.duration, duration)
|
||||||
|
} else {
|
||||||
|
assert.EqualError(err, test.err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -140,7 +140,7 @@ func (nodes *Nodes) GetNodesV2() *meshviewer.NodesV2 {
|
||||||
|
|
||||||
// Periodically saves the cached DB to json file
|
// Periodically saves the cached DB to json file
|
||||||
func (nodes *Nodes) worker() {
|
func (nodes *Nodes) worker() {
|
||||||
c := time.Tick(time.Second * time.Duration(nodes.config.Nodes.SaveInterval))
|
c := time.Tick(nodes.config.Nodes.SaveInterval.Duration)
|
||||||
|
|
||||||
for range c {
|
for range c {
|
||||||
nodes.expire()
|
nodes.expire()
|
||||||
|
@ -153,11 +153,11 @@ func (nodes *Nodes) expire() {
|
||||||
nodes.Timestamp = jsontime.Now()
|
nodes.Timestamp = jsontime.Now()
|
||||||
|
|
||||||
// Nodes last seen before expireTime will be removed
|
// Nodes last seen before expireTime will be removed
|
||||||
maxAge := nodes.config.Nodes.MaxAge
|
pruneAfter := nodes.config.Nodes.PruneAfter.Duration
|
||||||
if maxAge <= 0 {
|
if pruneAfter == 0 {
|
||||||
maxAge = 7 // our default
|
pruneAfter = time.Hour * 24 * 7 // our default
|
||||||
}
|
}
|
||||||
expireTime := nodes.Timestamp.Add(-time.Duration(maxAge) * time.Hour * 24)
|
expireTime := nodes.Timestamp.Add(-pruneAfter)
|
||||||
|
|
||||||
// Nodes last seen before offlineTime are changed to 'offline'
|
// Nodes last seen before offlineTime are changed to 'offline'
|
||||||
offlineTime := nodes.Timestamp.Add(-time.Minute * 10)
|
offlineTime := nodes.Timestamp.Add(-time.Minute * 10)
|
||||||
|
@ -198,11 +198,17 @@ func (nodes *Nodes) save() {
|
||||||
|
|
||||||
// serialize nodes
|
// serialize nodes
|
||||||
save(nodes, nodes.config.Nodes.NodesDynamicPath)
|
save(nodes, nodes.config.Nodes.NodesDynamicPath)
|
||||||
if path := nodes.config.Nodes.NodesV1Path; path != "" {
|
|
||||||
save(nodes.GetNodesV1(), path)
|
if path := nodes.config.Nodes.NodesPath; path != "" {
|
||||||
}
|
version := nodes.config.Nodes.NodesVersion
|
||||||
if path := nodes.config.Nodes.NodesV2Path; path != "" {
|
switch version {
|
||||||
save(nodes.GetNodesV2(), path)
|
case 1:
|
||||||
|
save(nodes.GetNodesV1(), path)
|
||||||
|
case 2:
|
||||||
|
save(nodes.GetNodesV2(), path)
|
||||||
|
default:
|
||||||
|
log.Panicf("invalid nodes version: %d", version)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if path := nodes.config.Nodes.GraphsPath; path != "" {
|
if path := nodes.config.Nodes.GraphsPath; path != "" {
|
||||||
|
|
|
@ -13,7 +13,7 @@ import (
|
||||||
func TestExpire(t *testing.T) {
|
func TestExpire(t *testing.T) {
|
||||||
assert := assert.New(t)
|
assert := assert.New(t)
|
||||||
config := &Config{}
|
config := &Config{}
|
||||||
config.Nodes.MaxAge = 6
|
config.Nodes.PruneAfter.Duration = time.Hour * 24 * 6
|
||||||
nodes := &Nodes{
|
nodes := &Nodes{
|
||||||
config: config,
|
config: config,
|
||||||
List: make(map[string]*Node),
|
List: make(map[string]*Node),
|
||||||
|
|
Loading…
Reference in New Issue