diff --git a/.gitignore b/.gitignore index 2f00525..d7fdc53 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,4 @@ _testmain.go *.test *.prof webroot -/config.yml +/config.toml diff --git a/README.md b/README.md index 8a806c6..b5c59be 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ## Usage ``` Usage of ./respond-collector: - -config path/to/config.yml + -config path/to/config.toml ``` ## Development diff --git a/cmd/respond-collector/main.go b/cmd/respond-collector/main.go index ef0f1f5..3634731 100644 --- a/cmd/respond-collector/main.go +++ b/cmd/respond-collector/main.go @@ -8,7 +8,6 @@ import ( "os" "os/signal" "syscall" - "time" "github.com/NYTimes/gziphandler" "github.com/julienschmidt/httprouter" @@ -31,7 +30,7 @@ var ( func main() { var importPath string flag.StringVar(&importPath, "import", "", "import global statistics from the given RRD file, requires influxdb") - flag.StringVar(&configFile, "config", "config.yml", "path of configuration file (default:config.yaml)") + flag.StringVar(&configFile, "config", "config.toml", "path of configuration file (default:config.yaml)") flag.Parse() config = models.ReadConfigFile(configFile) @@ -49,9 +48,8 @@ func main() { nodes.Start() if config.Respondd.Enable { - collectInterval := time.Second * time.Duration(config.Respondd.CollectInterval) collector = respond.NewCollector(db, nodes, config.Respondd.Interface) - collector.Start(collectInterval) + collector.Start(config.Respondd.CollectInterval.Duration) defer collector.Close() } diff --git a/config_example.toml b/config_example.toml new file mode 100644 index 0000000..19576e1 --- /dev/null +++ b/config_example.toml @@ -0,0 +1,31 @@ +[respondd] +enable = true +interface = "eth0" +collect_interval = "1m" + +[webserver] +enable = false +port = "8080" +address = "127.0.0.1" +webroot = "webroot" + +[nodes] +enable = true +nodes_version = 2 +nodes_path = "/var/www/html/meshviewer/data/nodes_all.json" +graphs_path = "/var/www/html/meshviewer/data/graph.json" +aliases_path = "/var/www/html/meshviewer/data/aliases.json" + +# Export nodes and graph periodically +save_interval = "5s" + +# Prune offline nodes after a time of inactivity +prune_after = "7d" + + +[influxdb] +enable = false +address = "http://localhost:8086" +database = "ffhb" +username = "" +password = "" diff --git a/config_example.yml b/config_example.yml deleted file mode 100644 index 2aae894..0000000 --- a/config_example.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- -respondd: - enable: true - interface: eth0 - - # Collected data every n seconds - collectinterval: 60 -webserver: - enable: false - port: 8080 - address: 127.0.0.1 - webroot: webroot - api: - newnodes: true - aliases: true -nodes: - enable: true - nodes_path: /var/www/html/meshviewer/data/nodes_all.json - nodesmini_path: /var/www/html/meshviewer/data/nodes.json - graphs_path: /var/www/html/meshviewer/data/graph.json - aliases_enable: false - aliases_path: /var/www/html/meshviewer/data/aliases.json - - # Export nodes and graph every n seconds - saveinterval: 5 - - # Expire offline nodes after n days - max_age: 7 - -influxdb: - enable: false - host: http://localhost:8086 - database: ffhb - username: - password: diff --git a/database/database.go b/database/database.go index 6b87adb..8e1eeea 100644 --- a/database/database.go +++ b/database/database.go @@ -30,7 +30,7 @@ type DB struct { func New(config *models.Config) *DB { // Make client c, err := client.NewHTTPClient(client.HTTPConfig{ - Addr: config.Influxdb.Addr, + Addr: config.Influxdb.Address, Username: config.Influxdb.Username, Password: config.Influxdb.Password, }) @@ -54,8 +54,7 @@ func New(config *models.Config) *DB { } func (db *DB) DeletePoints() { - query := fmt.Sprintf("delete from %s where time < now() - %dm", MeasurementNode, db.config.Influxdb.DeleteTill) - log.Println("delete", MeasurementNode, "older than", db.config.Influxdb.DeleteTill, "minutes") + query := fmt.Sprintf("delete from %s where time < now() - %ds", MeasurementNode, db.config.Influxdb.DeleteAfter.Duration/time.Second) db.client.Query(client.NewQuery(query, db.config.Influxdb.Database, "m")) } @@ -100,8 +99,7 @@ func (db *DB) Close() { // prunes node-specific data periodically func (db *DB) deleteWorker() { - duration := time.Minute * time.Duration(db.config.Influxdb.DeleteInterval) - ticker := time.NewTicker(duration) + ticker := time.NewTicker(db.config.Influxdb.DeleteInterval.Duration) for { select { case <-ticker.C: @@ -123,7 +121,7 @@ func (db *DB) addWorker() { var bp client.BatchPoints var err error var writeNow, closed bool - batchDuration := time.Second * time.Duration(db.config.Influxdb.SaveInterval) + batchDuration := db.config.Influxdb.SaveInterval.Duration timer := time.NewTimer(batchDuration) for !closed { diff --git a/models/config.go b/models/config.go index c20efce..da89ec2 100644 --- a/models/config.go +++ b/models/config.go @@ -2,58 +2,61 @@ package models import ( "io/ioutil" - "log" - "gopkg.in/yaml.v2" + "github.com/influxdata/toml" ) //Config the config File of this daemon type Config struct { Respondd struct { - Enable bool `yaml:"enable"` - Interface string `yaml:"interface"` - CollectInterval int `yaml:"collectinterval"` - } `yaml:"respondd"` + Enable bool + Interface string + CollectInterval Duration + } Webserver struct { - Enable bool `yaml:"enable"` - Port string `yaml:"port"` - Address string `yaml:"address"` - Webroot string `yaml:"webroot"` + Enable bool + Port string + Address string + Webroot string API struct { - Passphrase string `yaml:"passphrase"` - NewNodes bool `yaml:"newnodes"` - Aliases bool `yaml:"aliases"` - } `yaml:"api"` - } `yaml:"webserver"` + Passphrase string + NewNodes bool + Aliases bool + } + } Nodes struct { - Enable bool `yaml:"enable"` - NodesDynamicPath string `yaml:"nodes_path"` - NodesV1Path string `yaml:"nodesv1_path"` - NodesV2Path string `yaml:"nodesv2_path"` - GraphsPath string `yaml:"graphs_path"` - AliasesPath string `yaml:"aliases_path"` - SaveInterval int `yaml:"saveinterval"` // Save nodes every n seconds - MaxAge int `yaml:"max_age"` // Remove nodes after n days of inactivity - } `yaml:"nodes"` + Enable bool + NodesDynamicPath string + NodesPath string + NodesVersion int + GraphsPath string + AliasesPath string + SaveInterval Duration // Save nodes periodically + PruneAfter Duration // Remove nodes after n days of inactivity + } Influxdb struct { - Enable bool `yaml:"enable"` - Addr string `yaml:"host"` - Database string `yaml:"database"` - Username string `yaml:"username"` - Password string `yaml:"password"` - SaveInterval int `yaml:"saveinterval"` // Save nodes every n seconds - DeleteInterval int `yaml:"deleteinterval"` // Delete stats of nodes every n minutes - DeleteTill int `yaml:"deletetill"` // Delete stats of nodes till now-deletetill n minutes + Enable bool + Address string + Database string + Username string + Password string + SaveInterval Duration // Save nodes every n seconds + DeleteInterval Duration // Delete stats of nodes every n minutes + DeleteAfter Duration // Delete stats of nodes till now-deletetill n minutes } } // ReadConfigFile reads a config model from path of a yml file func ReadConfigFile(path string) *Config { config := &Config{} - file, _ := ioutil.ReadFile(path) - err := yaml.Unmarshal(file, &config) + file, err := ioutil.ReadFile(path) if err != nil { - log.Fatal(err) + panic(err) } + + if err := toml.Unmarshal(file, config); err != nil { + panic(err) + } + return config } diff --git a/models/config_test.go b/models/config_test.go index c78a4ab..630b93b 100644 --- a/models/config_test.go +++ b/models/config_test.go @@ -2,6 +2,7 @@ package models import ( "testing" + "time" "github.com/stretchr/testify/assert" ) @@ -9,6 +10,14 @@ import ( func TestReadConfig(t *testing.T) { assert := assert.New(t) - config := ReadConfigFile("../config_example.yml") + config := ReadConfigFile("../config_example.toml") assert.NotNil(config) + + assert.True(config.Respondd.Enable) + assert.Equal("eth0", config.Respondd.Interface) + assert.Equal(time.Minute, config.Respondd.CollectInterval.Duration) + + assert.Equal(2, config.Nodes.NodesVersion) + assert.Equal("/var/www/html/meshviewer/data/nodes_all.json", config.Nodes.NodesPath) + assert.Equal(time.Hour*24*7, config.Nodes.PruneAfter.Duration) } diff --git a/models/duration.go b/models/duration.go new file mode 100644 index 0000000..2c6e048 --- /dev/null +++ b/models/duration.go @@ -0,0 +1,50 @@ +package models + +import ( + "fmt" + "strconv" + "time" +) + +// Duration is a TOML datatype +// A duration string is a possibly signed sequence of +// decimal numbers and a unit suffix, +// such as "300s", "1.5h" or "5d". +// Valid time units are "s", "m", "h", "d", "w". +type Duration struct { + time.Duration +} + +// UnmarshalTOML parses a duration string. +func (d *Duration) UnmarshalTOML(data []byte) error { + + // " + int + unit + " + if len(data) < 4 { + return fmt.Errorf("invalid duration: %s", data) + } + + unit := data[len(data)-2] + value, err := strconv.Atoi(string(data[1 : len(data)-2])) + if err != nil { + return fmt.Errorf("unable to parse duration %s: %s", data, err) + } + + switch unit { + case 's': + d.Duration = time.Duration(value) * time.Second + case 'm': + d.Duration = time.Duration(value) * time.Minute + case 'h': + d.Duration = time.Duration(value) * time.Hour + case 'd': + d.Duration = time.Duration(value) * time.Hour * 24 + case 'w': + d.Duration = time.Duration(value) * time.Hour * 24 * 7 + case 'y': + d.Duration = time.Duration(value) * time.Hour * 24 * 365 + default: + return fmt.Errorf("invalid duration unit: %s", string(unit)) + } + + return nil +} diff --git a/models/duration_test.go b/models/duration_test.go new file mode 100644 index 0000000..8851133 --- /dev/null +++ b/models/duration_test.go @@ -0,0 +1,47 @@ +package models + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestDuration(t *testing.T) { + assert := assert.New(t) + + var tests = []struct { + input string + err string + duration time.Duration + }{ + {"", "invalid duration: \"\"", 0}, + {"1x", "invalid duration unit: x", 0}, + {"1s", "", time.Second}, + {"73s", "", time.Second * 73}, + {"1m", "", time.Minute}, + {"73m", "", time.Minute * 73}, + {"1h", "", time.Hour}, + {"43h", "", time.Hour * 43}, + {"1d", "", time.Hour * 24}, + {"8d", "", time.Hour * 24 * 8}, + {"1w", "", time.Hour * 24 * 7}, + {"52w", "", time.Hour * 24 * 7 * 52}, + {"1y", "", time.Hour * 24 * 365}, + {"3y", "", time.Hour * 24 * 365 * 3}, + } + + for _, test := range tests { + + d := Duration{} + err := d.UnmarshalTOML([]byte("\"" + test.input + "\"")) + duration := d.Duration + + if test.err == "" { + assert.NoError(err) + assert.Equal(test.duration, duration) + } else { + assert.EqualError(err, test.err) + } + } +} diff --git a/models/nodes.go b/models/nodes.go index f09d4ba..9a47be0 100644 --- a/models/nodes.go +++ b/models/nodes.go @@ -140,7 +140,7 @@ func (nodes *Nodes) GetNodesV2() *meshviewer.NodesV2 { // Periodically saves the cached DB to json file func (nodes *Nodes) worker() { - c := time.Tick(time.Second * time.Duration(nodes.config.Nodes.SaveInterval)) + c := time.Tick(nodes.config.Nodes.SaveInterval.Duration) for range c { nodes.expire() @@ -153,11 +153,11 @@ func (nodes *Nodes) expire() { nodes.Timestamp = jsontime.Now() // Nodes last seen before expireTime will be removed - maxAge := nodes.config.Nodes.MaxAge - if maxAge <= 0 { - maxAge = 7 // our default + pruneAfter := nodes.config.Nodes.PruneAfter.Duration + if pruneAfter == 0 { + pruneAfter = time.Hour * 24 * 7 // our default } - expireTime := nodes.Timestamp.Add(-time.Duration(maxAge) * time.Hour * 24) + expireTime := nodes.Timestamp.Add(-pruneAfter) // Nodes last seen before offlineTime are changed to 'offline' offlineTime := nodes.Timestamp.Add(-time.Minute * 10) @@ -198,11 +198,17 @@ func (nodes *Nodes) save() { // serialize nodes save(nodes, nodes.config.Nodes.NodesDynamicPath) - if path := nodes.config.Nodes.NodesV1Path; path != "" { - save(nodes.GetNodesV1(), path) - } - if path := nodes.config.Nodes.NodesV2Path; path != "" { - save(nodes.GetNodesV2(), path) + + if path := nodes.config.Nodes.NodesPath; path != "" { + version := nodes.config.Nodes.NodesVersion + switch version { + case 1: + save(nodes.GetNodesV1(), path) + case 2: + save(nodes.GetNodesV2(), path) + default: + log.Panicf("invalid nodes version: %d", version) + } } if path := nodes.config.Nodes.GraphsPath; path != "" { diff --git a/models/nodes_test.go b/models/nodes_test.go index e96ca1e..64f31b2 100644 --- a/models/nodes_test.go +++ b/models/nodes_test.go @@ -13,7 +13,7 @@ import ( func TestExpire(t *testing.T) { assert := assert.New(t) config := &Config{} - config.Nodes.MaxAge = 6 + config.Nodes.PruneAfter.Duration = time.Hour * 24 * 6 nodes := &Nodes{ config: config, List: make(map[string]*Node),