diff --git a/cmd/config.go b/cmd/config.go index 35beb4f..b7b184b 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -2,13 +2,24 @@ package cmd import ( "fmt" + "io/ioutil" "os" "github.com/FreifunkBremen/yanic/database" "github.com/FreifunkBremen/yanic/respond" "github.com/FreifunkBremen/yanic/runtime" + "github.com/FreifunkBremen/yanic/webserver" + "github.com/naoina/toml" ) +// Config represents the whole configuration +type Config struct { + Respondd respond.Config + Webserver webserver.Config + Nodes runtime.NodesConfig + Database database.Config +} + var ( configPath string collector *respond.Collector @@ -16,11 +27,28 @@ var ( nodes *runtime.Nodes ) -func loadConfig() *runtime.Config { - config, err := runtime.ReadConfigFile(configPath) +func loadConfig() *Config { + config, err := ReadConfigFile(configPath) if err != nil { fmt.Fprintln(os.Stderr, "unable to load config file:", err) os.Exit(2) } return config } + +// ReadConfigFile reads a config model from path of a yml file +func ReadConfigFile(path string) (config *Config, err error) { + config = &Config{} + + file, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + + err = toml.Unmarshal(file, config) + if err != nil { + return nil, err + } + + return +} diff --git a/cmd/config_test.go b/cmd/config_test.go new file mode 100644 index 0000000..960e9ca --- /dev/null +++ b/cmd/config_test.go @@ -0,0 +1,43 @@ +package cmd + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestReadConfig(t *testing.T) { + assert := assert.New(t) + + config, err := ReadConfigFile("../config_example.toml") + assert.NoError(err) + assert.NotNil(config) + + assert.True(config.Respondd.Enable) + assert.Equal([]string{"br-ffhb"}, config.Respondd.Interfaces) + assert.Equal(time.Minute, config.Respondd.CollectInterval.Duration) + assert.Equal(time.Hour*24*7, config.Nodes.PruneAfter.Duration) + assert.Equal(time.Hour*24*7, config.Database.DeleteAfter.Duration) + + // Test output plugins + assert.Len(config.Nodes.Output, 3) + outputs := config.Nodes.Output["meshviewer"].([]interface{}) + assert.Len(outputs, 1) + meshviewer := outputs[0] + + assert.EqualValues(map[string]interface{}{ + "version": int64(2), + "enable": false, + "nodes_path": "/var/www/html/meshviewer/data/nodes.json", + "graph_path": "/var/www/html/meshviewer/data/graph.json", + }, meshviewer) + + _, err = ReadConfigFile("testdata/config_invalid.toml") + assert.Error(err, "not unmarshalable") + assert.Contains(err.Error(), "invalid TOML syntax") + + _, err = ReadConfigFile("testdata/adsa.toml") + assert.Error(err, "not found able") + assert.Contains(err.Error(), "no such file or directory") +} diff --git a/cmd/import.go b/cmd/import.go index 8b78ac5..47fee2a 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -3,8 +3,7 @@ package cmd import ( "log" - "github.com/FreifunkBremen/yanic/database" - "github.com/FreifunkBremen/yanic/database/all" + allDatabase "github.com/FreifunkBremen/yanic/database/all" "github.com/FreifunkBremen/yanic/rrd" "github.com/FreifunkBremen/yanic/runtime" "github.com/spf13/cobra" @@ -21,12 +20,11 @@ var importCmd = &cobra.Command{ site := args[1] config := loadConfig() - connections, err := all.Connect(config.Database.Connection) + err := allDatabase.Start(config.Database) if err != nil { panic(err) } - database.Start(connections, config) - defer database.Close(connections) + defer allDatabase.Close() log.Println("importing RRD from", path) diff --git a/cmd/query.go b/cmd/query.go index 8e7a73c..ea81708 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -24,7 +24,7 @@ var queryCmd = &cobra.Command{ log.Printf("Sending request address=%s iface=%s", dstAddress, iface) - nodes := runtime.NewNodes(&runtime.Config{}) + nodes := runtime.NewNodes(&runtime.NodesConfig{}) collector := respond.NewCollector(nil, nodes, []string{}, []string{iface}, 0) defer collector.Close() diff --git a/cmd/serve.go b/cmd/serve.go index 1872c1b..89df9f2 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -7,9 +7,7 @@ import ( "syscall" "time" - "github.com/FreifunkBremen/yanic/database" allDatabase "github.com/FreifunkBremen/yanic/database/all" - "github.com/FreifunkBremen/yanic/output" allOutput "github.com/FreifunkBremen/yanic/output/all" "github.com/FreifunkBremen/yanic/respond" "github.com/FreifunkBremen/yanic/runtime" @@ -25,22 +23,20 @@ var serveCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { config := loadConfig() - connections, err := allDatabase.Connect(config.Database.Connection) + err := allDatabase.Start(config.Database) if err != nil { panic(err) } - database.Start(connections, config) - defer database.Close(connections) + defer allDatabase.Close() - nodes = runtime.NewNodes(config) + nodes = runtime.NewNodes(&config.Nodes) nodes.Start() - outputs, err := allOutput.Register(config.Nodes.Output) + err = allOutput.Start(nodes, config.Nodes) if err != nil { panic(err) } - output.Start(outputs, nodes, config) - defer output.Close() + defer allOutput.Close() if config.Webserver.Enable { log.Println("starting webserver on", config.Webserver.Bind) diff --git a/cmd/testdata/config_invalid.toml b/cmd/testdata/config_invalid.toml new file mode 100644 index 0000000..323fae0 --- /dev/null +++ b/cmd/testdata/config_invalid.toml @@ -0,0 +1 @@ +foobar diff --git a/database/all/connection.go b/database/all/connection.go new file mode 100644 index 0000000..9d18706 --- /dev/null +++ b/database/all/connection.go @@ -0,0 +1,75 @@ +package all + +import ( + "fmt" + "log" + "time" + + "github.com/FreifunkBremen/yanic/database" + "github.com/FreifunkBremen/yanic/runtime" +) + +type Connection struct { + database.Connection + list []database.Connection +} + +func Connect(allConnection map[string]interface{}) (database.Connection, error) { + var list []database.Connection + for dbType, conn := range database.Adapters { + configForType := allConnection[dbType] + if configForType == nil { + log.Printf("the output type '%s' has no configuration", dbType) + continue + } + dbConfigs, ok := configForType.([]map[string]interface{}) + if !ok { + return nil, fmt.Errorf("the output type '%s' has the wrong format", dbType) + } + + for _, config := range dbConfigs { + if c, ok := config["enable"].(bool); ok && !c { + continue + } + connected, err := conn(config) + if err != nil { + return nil, err + } + if connected == nil { + continue + } + list = append(list, connected) + } + } + return &Connection{list: list}, nil +} + +func (conn *Connection) InsertNode(node *runtime.Node) { + for _, item := range conn.list { + item.InsertNode(node) + } +} + +func (conn *Connection) InsertLink(link *runtime.Link, time time.Time) { + for _, item := range conn.list { + item.InsertLink(link, time) + } +} + +func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) { + for _, item := range conn.list { + item.InsertGlobals(stats, time, site) + } +} + +func (conn *Connection) PruneNodes(deleteAfter time.Duration) { + for _, item := range conn.list { + item.PruneNodes(deleteAfter) + } +} + +func (conn *Connection) Close() { + for _, item := range conn.list { + item.Close() + } +} diff --git a/database/all/internal.go b/database/all/internal.go index 8a58b66..6731454 100644 --- a/database/all/internal.go +++ b/database/all/internal.go @@ -1,74 +1,45 @@ package all import ( - "log" + "sync" "time" "github.com/FreifunkBremen/yanic/database" - "github.com/FreifunkBremen/yanic/runtime" ) -type Connection struct { - database.Connection - list []database.Connection +var conn database.Connection +var wg = sync.WaitGroup{} +var quit chan struct{} + +func Start(config database.Config) (err error) { + conn, err = Connect(config.Connection) + if err != nil { + return + } + quit = make(chan struct{}) + wg.Add(1) + go deleteWorker(config.DeleteInterval.Duration, config.DeleteAfter.Duration) + return } -func Connect(allConnection map[string]interface{}) (database.Connection, error) { - var list []database.Connection - for dbType, conn := range database.Adapters { - configForType := allConnection[dbType] - if configForType == nil { - log.Printf("the output type '%s' has no configuration\n", dbType) - continue - } - dbConfigs, ok := configForType.([]map[string]interface{}) - if !ok { - log.Panicf("the output type '%s' has the wrong format\n", dbType) - } +func Close() { + close(quit) + wg.Wait() + conn.Close() + quit = nil +} - for _, config := range dbConfigs { - if c, ok := config["enable"].(bool); ok && !c { - continue - } - connected, err := conn(config) - if err != nil { - return nil, err - } - if connected == nil { - continue - } - list = append(list, connected) +// prunes node-specific data periodically +func deleteWorker(deleteInterval time.Duration, deleteAfter time.Duration) { + ticker := time.NewTicker(deleteInterval) + for { + select { + case <-ticker.C: + conn.PruneNodes(deleteAfter) + case <-quit: + ticker.Stop() + wg.Done() + return } } - return &Connection{list: list}, nil -} - -func (conn *Connection) InsertNode(node *runtime.Node) { - for _, item := range conn.list { - item.InsertNode(node) - } -} - -func (conn *Connection) InsertLink(link *runtime.Link, time time.Time) { - for _, item := range conn.list { - item.InsertLink(link, time) - } -} - -func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) { - for _, item := range conn.list { - item.InsertGlobals(stats, time, site) - } -} - -func (conn *Connection) PruneNodes(deleteAfter time.Duration) { - for _, item := range conn.list { - item.PruneNodes(deleteAfter) - } -} - -func (conn *Connection) Close() { - for _, item := range conn.list { - item.Close() - } } diff --git a/database/all/internel_test.go b/database/all/internel_test.go index 019c2c0..e3981db 100644 --- a/database/all/internel_test.go +++ b/database/all/internel_test.go @@ -2,144 +2,59 @@ package all import ( "errors" - "sync" "testing" "time" "github.com/FreifunkBremen/yanic/database" - "github.com/FreifunkBremen/yanic/runtime" + "github.com/FreifunkBremen/yanic/lib/duration" "github.com/stretchr/testify/assert" ) -type testConn struct { - database.Connection - countNode int - countLink int - countGlobals int - countPrune int - countClose int - sync.Mutex -} - -func (c *testConn) InsertNode(node *runtime.Node) { - c.Lock() - c.countNode++ - c.Unlock() -} -func (c *testConn) GetNode() int { - c.Lock() - defer c.Unlock() - return c.countNode -} -func (c *testConn) InsertLink(link *runtime.Link, time time.Time) { - c.Lock() - c.countLink++ - c.Unlock() -} -func (c *testConn) GetLink() int { - c.Lock() - defer c.Unlock() - return c.countLink -} -func (c *testConn) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) { - c.Lock() - c.countGlobals++ - c.Unlock() -} -func (c *testConn) GetGlobal() int { - c.Lock() - defer c.Unlock() - return c.countGlobals -} -func (c *testConn) PruneNodes(time.Duration) { - c.Lock() - c.countPrune++ - c.Unlock() -} -func (c *testConn) GetPrune() int { - c.Lock() - defer c.Unlock() - return c.countPrune -} -func (c *testConn) Close() { - c.Lock() - c.countClose++ - c.Unlock() -} -func (c *testConn) GetClose() int { - c.Lock() - defer c.Unlock() - return c.countClose -} - func TestStart(t *testing.T) { assert := assert.New(t) - globalConn := &testConn{} - database.RegisterAdapter("a", func(config map[string]interface{}) (database.Connection, error) { - return globalConn, nil - }) - database.RegisterAdapter("b", func(config map[string]interface{}) (database.Connection, error) { - return globalConn, nil - }) - database.RegisterAdapter("c", func(config map[string]interface{}) (database.Connection, error) { - return globalConn, nil - }) database.RegisterAdapter("d", func(config map[string]interface{}) (database.Connection, error) { return nil, nil }) database.RegisterAdapter("e", func(config map[string]interface{}) (database.Connection, error) { return nil, errors.New("blub") }) - allConn, err := Connect(map[string]interface{}{ - "a": []map[string]interface{}{ - map[string]interface{}{ - "enable": false, - "path": "a1", + // Test for PruneNodes (by start) + assert.Nil(quit) + err := Start(database.Config{ + DeleteInterval: duration.Duration{Duration: time.Millisecond}, + Connection: map[string]interface{}{ + "a": []map[string]interface{}{ + map[string]interface{}{ + "enable": false, + "path": "a1", + }, + map[string]interface{}{ + "path": "a2", + }, + map[string]interface{}{ + "enable": true, + "path": "a3", + }, }, - map[string]interface{}{ - "path": "a2", + "b": nil, + "c": []map[string]interface{}{ + map[string]interface{}{ + "path": "c1", + }, }, - map[string]interface{}{ - "enable": true, - "path": "a3", - }, - }, - "b": nil, - "c": []map[string]interface{}{ - map[string]interface{}{ - "path": "c1", - }, - }, - // fetch continue command in Connect - "d": []map[string]interface{}{ - map[string]interface{}{ - "path": "d0", + // fetch continue command in Connect + "d": []map[string]interface{}{ + map[string]interface{}{ + "path": "d0", + }, }, }, }) assert.NoError(err) + assert.NotNil(quit) - assert.Equal(0, globalConn.GetNode()) - allConn.InsertNode(nil) - assert.Equal(3, globalConn.GetNode()) - - assert.Equal(0, globalConn.GetLink()) - allConn.InsertLink(nil, time.Now()) - assert.Equal(3, globalConn.GetLink()) - - assert.Equal(0, globalConn.GetGlobal()) - allConn.InsertGlobals(nil, time.Now(), runtime.GLOBAL_SITE) - assert.Equal(3, globalConn.GetGlobal()) - - assert.Equal(0, globalConn.GetPrune()) - allConn.PruneNodes(time.Second) - assert.Equal(3, globalConn.GetPrune()) - - assert.Equal(0, globalConn.GetClose()) - allConn.Close() - assert.Equal(3, globalConn.GetClose()) - + // connection type not found _, err = Connect(map[string]interface{}{ "e": []map[string]interface{}{ map[string]interface{}{}, @@ -147,10 +62,14 @@ func TestStart(t *testing.T) { }) assert.Error(err) - // wrong format -> the only panic in Register - assert.Panics(func() { - Connect(map[string]interface{}{ + // test close + Close() + + // wrong format + err = Start(database.Config{ + Connection: map[string]interface{}{ "e": true, - }) + }, }) + assert.Error(err) } diff --git a/database/config.go b/database/config.go new file mode 100644 index 0000000..9844901 --- /dev/null +++ b/database/config.go @@ -0,0 +1,9 @@ +package database + +import "github.com/FreifunkBremen/yanic/lib/duration" + +type Config struct { + DeleteInterval duration.Duration `toml:"delete_interval"` // Delete stats of nodes every n minutes + DeleteAfter duration.Duration `toml:"delete_after"` // Delete stats of nodes till now-deletetill n minutes + Connection map[string]interface{} +} diff --git a/database/influxdb/global_test.go b/database/influxdb/global_test.go index 72d2dec..f990b5e 100644 --- a/database/influxdb/global_test.go +++ b/database/influxdb/global_test.go @@ -88,7 +88,7 @@ func TestGlobalStats(t *testing.T) { } func createTestNodes() *runtime.Nodes { - nodes := runtime.NewNodes(&runtime.Config{}) + nodes := runtime.NewNodes(&runtime.NodesConfig{}) nodeData := &runtime.Node{ Online: true, diff --git a/database/influxdb/node_test.go b/database/influxdb/node_test.go index 9e45d86..caf6299 100644 --- a/database/influxdb/node_test.go +++ b/database/influxdb/node_test.go @@ -177,7 +177,7 @@ func testPoints(nodes ...*runtime.Node) (points []*client.Point) { panic(err) } - nodesList := runtime.NewNodes(&runtime.Config{}) + nodesList := runtime.NewNodes(&runtime.NodesConfig{}) // Create dummy connection conn := &Connection{ diff --git a/database/internal.go b/database/internal.go deleted file mode 100644 index 5a5eb1c..0000000 --- a/database/internal.go +++ /dev/null @@ -1,40 +0,0 @@ -package database - -import ( - "time" - - "github.com/FreifunkBremen/yanic/runtime" -) - -var quit chan struct{} - -// Start workers of database -// WARNING: Do not override this function -// you should use New() -func Start(conn Connection, config *runtime.Config) { - quit = make(chan struct{}) - go deleteWorker(conn, config.Database.DeleteInterval.Duration, config.Database.DeleteAfter.Duration) -} - -func Close(conn Connection) { - if quit != nil { - close(quit) - } - if conn != nil { - conn.Close() - } -} - -// prunes node-specific data periodically -func deleteWorker(conn Connection, deleteInterval time.Duration, deleteAfter time.Duration) { - ticker := time.NewTicker(deleteInterval) - for { - select { - case <-ticker.C: - conn.PruneNodes(deleteAfter) - case <-quit: - ticker.Stop() - return - } - } -} diff --git a/database/internal_test.go b/database/internal_test.go deleted file mode 100644 index 0280fb7..0000000 --- a/database/internal_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package database - -import ( - "sync" - "testing" - "time" - - "github.com/FreifunkBremen/yanic/runtime" - "github.com/stretchr/testify/assert" -) - -type testConn struct { - Connection - countClose int - countPrune int - sync.Mutex -} - -func (c *testConn) Close() { - c.Lock() - c.countClose++ - c.Unlock() -} -func (c *testConn) GetClose() int { - c.Lock() - defer c.Unlock() - return c.countClose -} -func (c *testConn) PruneNodes(time.Duration) { - c.Lock() - c.countPrune++ - c.Unlock() -} -func (c *testConn) GetPruneNodes() int { - c.Lock() - defer c.Unlock() - return c.countPrune -} - -func TestStart(t *testing.T) { - assert := assert.New(t) - - conn := &testConn{} - config := &runtime.Config{ - Database: struct { - DeleteInterval runtime.Duration `toml:"delete_interval"` - DeleteAfter runtime.Duration `toml:"delete_after"` - Connection map[string]interface{} - }{ - DeleteInterval: runtime.Duration{Duration: time.Millisecond * 10}, - }, - } - assert.Nil(quit) - - Start(conn, config) - assert.NotNil(quit) - - assert.Equal(0, conn.GetPruneNodes()) - time.Sleep(time.Millisecond * 12) - assert.Equal(1, conn.GetPruneNodes()) - - assert.Equal(0, conn.GetClose()) - Close(conn) - assert.NotNil(quit) - assert.Equal(1, conn.GetClose()) - - time.Sleep(time.Millisecond * 12) // to reach timer.Stop() line - -} diff --git a/runtime/duration.go b/lib/duration/duration.go similarity index 81% rename from runtime/duration.go rename to lib/duration/duration.go index 255d4ca..6ea5932 100644 --- a/runtime/duration.go +++ b/lib/duration/duration.go @@ -1,4 +1,4 @@ -package runtime +package duration import ( "fmt" @@ -18,14 +18,8 @@ type Duration struct { } // UnmarshalTOML parses a duration string. -func (d *Duration) UnmarshalTOML(dataInterface interface{}) error { - var data string - switch dataInterface.(type) { - case string: - data = dataInterface.(string) - default: - return fmt.Errorf("invalid duration: \"%s\"", dataInterface) - } +func (d *Duration) UnmarshalText(data []byte) error { + // " + int + unit + " if len(data) < 2 { return fmt.Errorf("invalid duration: \"%s\"", data) diff --git a/runtime/duration_test.go b/lib/duration/duration_test.go similarity index 74% rename from runtime/duration_test.go rename to lib/duration/duration_test.go index d17f475..e318b3c 100644 --- a/runtime/duration_test.go +++ b/lib/duration/duration_test.go @@ -1,4 +1,4 @@ -package runtime +package duration import ( "testing" @@ -16,6 +16,8 @@ func TestDuration(t *testing.T) { duration time.Duration }{ {"", "invalid duration: \"\"", 0}, + {"3", "invalid duration: \"3\"", 0}, + {"am", "unable to parse duration \"am\": strconv.Atoi: parsing \"a\": invalid syntax", 0}, {"1x", "invalid duration unit \"x\"", 0}, {"1s", "", time.Second}, {"73s", "", time.Second * 73}, @@ -34,7 +36,7 @@ func TestDuration(t *testing.T) { for _, test := range tests { d := Duration{} - err := d.UnmarshalTOML(test.input) + err := d.UnmarshalText([]byte(test.input)) duration := d.Duration if test.err == "" { @@ -44,13 +46,4 @@ func TestDuration(t *testing.T) { assert.EqualError(err, test.err) } } - - d := Duration{} - err := d.UnmarshalTOML(3) - assert.Error(err) - assert.Contains(err.Error(), "invalid duration") - - err = d.UnmarshalTOML("am") - assert.Error(err) - assert.EqualError(err, "unable to parse duration \"am\": strconv.Atoi: parsing \"a\": invalid syntax") } diff --git a/jsontime/jsontime.go b/lib/jsontime/jsontime.go similarity index 100% rename from jsontime/jsontime.go rename to lib/jsontime/jsontime.go diff --git a/jsontime/jsontime_test.go b/lib/jsontime/jsontime_test.go similarity index 100% rename from jsontime/jsontime_test.go rename to lib/jsontime/jsontime_test.go diff --git a/output/all/filter.go b/output/all/filter.go index d8da125..ce9d0c2 100644 --- a/output/all/filter.go +++ b/output/all/filter.go @@ -1,6 +1,8 @@ package all -import "github.com/FreifunkBremen/yanic/runtime" +import ( + "github.com/FreifunkBremen/yanic/runtime" +) // Config Filter type filterConfig map[string]interface{} @@ -13,7 +15,7 @@ func noFilter(node *runtime.Node) *runtime.Node { // Create Filter func (f filterConfig) filtering(nodesOrigin *runtime.Nodes) *runtime.Nodes { - nodes := runtime.NewNodes(&runtime.Config{}) + nodes := runtime.NewNodes(&runtime.NodesConfig{}) filterfuncs := []filterFunc{ f.HasLocation(), f.Blacklist(), diff --git a/output/all/internal.go b/output/all/internal.go index f871990..e3c3ee5 100644 --- a/output/all/internal.go +++ b/output/all/internal.go @@ -1,63 +1,45 @@ package all import ( - "log" + "sync" + "time" "github.com/FreifunkBremen/yanic/output" "github.com/FreifunkBremen/yanic/runtime" ) -type Output struct { - output.Output - list map[int]output.Output - filter map[int]filterConfig +var quit chan struct{} +var wg = sync.WaitGroup{} +var outputA output.Output + +func Start(nodes *runtime.Nodes, config runtime.NodesConfig) (err error) { + outputA, err = Register(config.Output) + if err != nil { + return + } + quit = make(chan struct{}) + wg.Add(1) + go saveWorker(nodes, config.SaveInterval.Duration) + return } -func Register(configuration map[string]interface{}) (output.Output, error) { - list := make(map[int]output.Output) - filter := make(map[int]filterConfig) - i := 1 - allOutputs := configuration - for outputType, outputRegister := range output.Adapters { - configForOutput := allOutputs[outputType] - if configForOutput == nil { - log.Printf("the output type '%s' has no configuration\n", outputType) - continue - } - outputConfigs, ok := configForOutput.([]map[string]interface{}) - if !ok { - log.Panicf("the output type '%s' has the wrong format\n", outputType) - } - for _, config := range outputConfigs { - if c, ok := config["enable"].(bool); ok && !c { - continue - } - output, err := outputRegister(config) - if err != nil { - return nil, err - } - if output == nil { - continue - } - list[i] = output - if c := config["filter"]; c != nil { - filter[i] = config["filter"].(map[string]interface{}) - } - i++ +func Close() { + close(quit) + wg.Wait() + quit = nil +} + +// save periodically to output +func saveWorker(nodes *runtime.Nodes, saveInterval time.Duration) { + ticker := time.NewTicker(saveInterval) + for { + select { + case <-ticker.C: + outputA.Save(nodes) + case <-quit: + ticker.Stop() + wg.Done() + return } } - return &Output{list: list, filter: filter}, nil -} - -func (o *Output) Save(nodes *runtime.Nodes) { - for i, item := range o.list { - var filteredNodes *runtime.Nodes - if config := o.filter[i]; config != nil { - filteredNodes = config.filtering(nodes) - } else { - filteredNodes = filterConfig{}.filtering(nodes) - } - - item.Save(filteredNodes) - } } diff --git a/output/all/output.go b/output/all/output.go new file mode 100644 index 0000000..4861659 --- /dev/null +++ b/output/all/output.go @@ -0,0 +1,64 @@ +package all + +import ( + "fmt" + "log" + + "github.com/FreifunkBremen/yanic/output" + "github.com/FreifunkBremen/yanic/runtime" +) + +type Output struct { + output.Output + list map[int]output.Output + filter map[int]filterConfig +} + +func Register(configuration map[string]interface{}) (output.Output, error) { + list := make(map[int]output.Output) + filter := make(map[int]filterConfig) + i := 1 + allOutputs := configuration + for outputType, outputRegister := range output.Adapters { + configForOutput := allOutputs[outputType] + if configForOutput == nil { + log.Printf("the output type '%s' has no configuration\n", outputType) + continue + } + outputConfigs, ok := configForOutput.([]map[string]interface{}) + if !ok { + return nil, fmt.Errorf("the output type '%s' has the wrong format", outputType) + } + for _, config := range outputConfigs { + if c, ok := config["enable"].(bool); ok && !c { + continue + } + output, err := outputRegister(config) + if err != nil { + return nil, err + } + if output == nil { + continue + } + list[i] = output + if c := config["filter"]; c != nil { + filter[i] = config["filter"].(map[string]interface{}) + } + i++ + } + } + return &Output{list: list, filter: filter}, nil +} + +func (o *Output) Save(nodes *runtime.Nodes) { + for i, item := range o.list { + var filteredNodes *runtime.Nodes + if config := o.filter[i]; config != nil { + filteredNodes = config.filtering(nodes) + } else { + filteredNodes = filterConfig{}.filtering(nodes) + } + + item.Save(filteredNodes) + } +} diff --git a/output/all/internal_test.go b/output/all/output_test.go similarity index 94% rename from output/all/internal_test.go rename to output/all/output_test.go index ef87ba2..7f77299 100644 --- a/output/all/internal_test.go +++ b/output/all/output_test.go @@ -89,10 +89,9 @@ func TestStart(t *testing.T) { }) assert.Error(err) - // wrong format -> the only panic in Register - assert.Panics(func() { - Register(map[string]interface{}{ - "e": true, - }) + // wrong format + _, err = Register(map[string]interface{}{ + "e": true, }) + assert.Error(err) } diff --git a/output/internal.go b/output/internal.go deleted file mode 100644 index dcf462b..0000000 --- a/output/internal.go +++ /dev/null @@ -1,40 +0,0 @@ -package output - -import ( - "sync" - "time" - - "github.com/FreifunkBremen/yanic/runtime" -) - -var quit chan struct{} -var wg = sync.WaitGroup{} - -// Start workers of database -// WARNING: Do not override this function -// you should use New() -func Start(output Output, nodes *runtime.Nodes, config *runtime.Config) { - quit = make(chan struct{}) - wg.Add(1) - go saveWorker(output, nodes, config.Nodes.SaveInterval.Duration) -} - -func Close() { - close(quit) - wg.Wait() -} - -// save periodically to output -func saveWorker(output Output, nodes *runtime.Nodes, saveInterval time.Duration) { - ticker := time.NewTicker(saveInterval) - for { - select { - case <-ticker.C: - output.Save(nodes) - case <-quit: - wg.Done() - ticker.Stop() - return - } - } -} diff --git a/output/internal_test.go b/output/internal_test.go deleted file mode 100644 index 2ab095a..0000000 --- a/output/internal_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package output - -import ( - "sync" - "testing" - "time" - - "github.com/FreifunkBremen/yanic/runtime" - "github.com/stretchr/testify/assert" -) - -type testConn struct { - Output - countSave int - sync.Mutex -} - -func (c *testConn) Save(nodes *runtime.Nodes) { - c.Lock() - c.countSave++ - c.Unlock() -} -func (c *testConn) Get() int { - c.Lock() - defer c.Unlock() - return c.countSave -} - -func TestStart(t *testing.T) { - assert := assert.New(t) - - conn := &testConn{} - config := &runtime.Config{ - Nodes: struct { - StatePath string `toml:"state_path"` - SaveInterval runtime.Duration `toml:"save_interval"` - OfflineAfter runtime.Duration `toml:"offline_after"` - PruneAfter runtime.Duration `toml:"prune_after"` - Output map[string]interface{} - }{ - SaveInterval: runtime.Duration{Duration: time.Millisecond * 10}, - }, - } - assert.Nil(quit) - - Start(conn, nil, config) - assert.NotNil(quit) - - assert.Equal(0, conn.Get()) - time.Sleep(time.Millisecond * 12) - assert.Equal(1, conn.Get()) - - time.Sleep(time.Millisecond * 12) - Close() - assert.Equal(2, conn.Get()) - -} diff --git a/output/meshviewer-ffrgb/meshviewer.go b/output/meshviewer-ffrgb/meshviewer.go index 98d3b21..5e05835 100644 --- a/output/meshviewer-ffrgb/meshviewer.go +++ b/output/meshviewer-ffrgb/meshviewer.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - "github.com/FreifunkBremen/yanic/jsontime" + "github.com/FreifunkBremen/yanic/lib/jsontime" "github.com/FreifunkBremen/yanic/runtime" ) diff --git a/output/meshviewer-ffrgb/meshviewer_test.go b/output/meshviewer-ffrgb/meshviewer_test.go index efd622a..90c2ee5 100644 --- a/output/meshviewer-ffrgb/meshviewer_test.go +++ b/output/meshviewer-ffrgb/meshviewer_test.go @@ -11,7 +11,7 @@ import ( func TestTransform(t *testing.T) { assert := assert.New(t) - nodes := runtime.NewNodes(&runtime.Config{}) + nodes := runtime.NewNodes(&runtime.NodesConfig{}) nodes.AddNode(&runtime.Node{ Online: true, Nodeinfo: &data.NodeInfo{ diff --git a/output/meshviewer-ffrgb/struct.go b/output/meshviewer-ffrgb/struct.go index 233775d..3c3fb82 100644 --- a/output/meshviewer-ffrgb/struct.go +++ b/output/meshviewer-ffrgb/struct.go @@ -3,7 +3,7 @@ package meshviewerFFRGB import ( "time" - "github.com/FreifunkBremen/yanic/jsontime" + "github.com/FreifunkBremen/yanic/lib/jsontime" "github.com/FreifunkBremen/yanic/runtime" ) diff --git a/output/meshviewer-ffrgb/struct_test.go b/output/meshviewer-ffrgb/struct_test.go index 863f19f..162c2d9 100644 --- a/output/meshviewer-ffrgb/struct_test.go +++ b/output/meshviewer-ffrgb/struct_test.go @@ -10,7 +10,7 @@ import ( func TestRegister(t *testing.T) { assert := assert.New(t) - nodes := runtime.NewNodes(&runtime.Config{}) + nodes := runtime.NewNodes(&runtime.NodesConfig{}) node := NewNode(nodes, &runtime.Node{ Nodeinfo: &data.NodeInfo{ Owner: &data.Owner{ @@ -21,7 +21,7 @@ func TestRegister(t *testing.T) { }, Location: &data.Location{ Longitude: 13.3, - Latitude: 8.7, + Latitude: 8.7, }, }, Statistics: &data.Statistics{ diff --git a/output/meshviewer/graph_test.go b/output/meshviewer/graph_test.go index 6c1313f..5d113eb 100644 --- a/output/meshviewer/graph_test.go +++ b/output/meshviewer/graph_test.go @@ -33,7 +33,7 @@ func TestGenerateGraph(t *testing.T) { func testGetNodesByFile(files ...string) *runtime.Nodes { - nodes := runtime.NewNodes(&runtime.Config{}) + nodes := runtime.NewNodes(&runtime.NodesConfig{}) for _, file := range files { node := testGetNodeByFile(file) diff --git a/output/meshviewer/node.go b/output/meshviewer/node.go index 4ffeefa..eb2d0e2 100644 --- a/output/meshviewer/node.go +++ b/output/meshviewer/node.go @@ -2,7 +2,7 @@ package meshviewer import ( "github.com/FreifunkBremen/yanic/data" - "github.com/FreifunkBremen/yanic/jsontime" + "github.com/FreifunkBremen/yanic/lib/jsontime" ) // Node struct diff --git a/output/meshviewer/nodes_test.go b/output/meshviewer/nodes_test.go index 27f5385..caf59de 100644 --- a/output/meshviewer/nodes_test.go +++ b/output/meshviewer/nodes_test.go @@ -23,7 +23,7 @@ func TestNodesV2(t *testing.T) { } func createTestNodes() *runtime.Nodes { - nodes := runtime.NewNodes(&runtime.Config{}) + nodes := runtime.NewNodes(&runtime.NodesConfig{}) nodeData := &runtime.Node{ Statistics: &data.Statistics{ diff --git a/output/meshviewer/nodes_v1.go b/output/meshviewer/nodes_v1.go index 7f07415..9ebb922 100644 --- a/output/meshviewer/nodes_v1.go +++ b/output/meshviewer/nodes_v1.go @@ -1,7 +1,7 @@ package meshviewer import ( - "github.com/FreifunkBremen/yanic/jsontime" + "github.com/FreifunkBremen/yanic/lib/jsontime" "github.com/FreifunkBremen/yanic/runtime" ) diff --git a/output/meshviewer/nodes_v2.go b/output/meshviewer/nodes_v2.go index c9d5611..2f726ab 100644 --- a/output/meshviewer/nodes_v2.go +++ b/output/meshviewer/nodes_v2.go @@ -1,7 +1,7 @@ package meshviewer import ( - "github.com/FreifunkBremen/yanic/jsontime" + "github.com/FreifunkBremen/yanic/lib/jsontime" "github.com/FreifunkBremen/yanic/runtime" ) diff --git a/output/nodelist/nodelist.go b/output/nodelist/nodelist.go index c101dde..ca680a0 100644 --- a/output/nodelist/nodelist.go +++ b/output/nodelist/nodelist.go @@ -1,7 +1,7 @@ package nodelist import ( - "github.com/FreifunkBremen/yanic/jsontime" + "github.com/FreifunkBremen/yanic/lib/jsontime" "github.com/FreifunkBremen/yanic/runtime" ) diff --git a/output/nodelist/nodelist_test.go b/output/nodelist/nodelist_test.go index e7a32f1..efd1159 100644 --- a/output/nodelist/nodelist_test.go +++ b/output/nodelist/nodelist_test.go @@ -17,7 +17,7 @@ func TestTransform(t *testing.T) { } func createTestNodes() *runtime.Nodes { - nodes := runtime.NewNodes(&runtime.Config{}) + nodes := runtime.NewNodes(&runtime.NodesConfig{}) nodeData := &runtime.Node{ Statistics: &data.Statistics{ @@ -47,7 +47,7 @@ func createTestNodes() *runtime.Nodes { Model: "TP-Link 841", }, Location: &data.Location{ - Latitude: 23, + Latitude: 23, Longitude: 2, }, }, diff --git a/respond/collector.go b/respond/collector.go index 7642f95..2e1327d 100644 --- a/respond/collector.go +++ b/respond/collector.go @@ -11,7 +11,7 @@ import ( "github.com/FreifunkBremen/yanic/data" "github.com/FreifunkBremen/yanic/database" - "github.com/FreifunkBremen/yanic/jsontime" + "github.com/FreifunkBremen/yanic/lib/jsontime" "github.com/FreifunkBremen/yanic/runtime" ) @@ -30,7 +30,7 @@ type Collector struct { } // NewCollector creates a Collector struct -func NewCollector(db database.Connection, nodes *runtime.Nodes, sites []string , ifaces []string, port int) *Collector { +func NewCollector(db database.Connection, nodes *runtime.Nodes, sites []string, ifaces []string, port int) *Collector { coll := &Collector{ db: db, diff --git a/respond/collector_test.go b/respond/collector_test.go index ba1ff0a..d6e5bd9 100644 --- a/respond/collector_test.go +++ b/respond/collector_test.go @@ -12,7 +12,7 @@ import ( const SITE_TEST = "ffxx" func TestCollector(t *testing.T) { - nodes := runtime.NewNodes(&runtime.Config{}) + nodes := runtime.NewNodes(&runtime.NodesConfig{}) collector := NewCollector(nil, nodes, []string{SITE_TEST}, []string{}, 10001) collector.Start(time.Millisecond) diff --git a/respond/config.go b/respond/config.go new file mode 100644 index 0000000..b221fa1 --- /dev/null +++ b/respond/config.go @@ -0,0 +1,12 @@ +package respond + +import "github.com/FreifunkBremen/yanic/lib/duration" + +type Config struct { + Enable bool `toml:"enable"` + Synchronize duration.Duration `toml:"synchronize"` + Interfaces []string `toml:"interfaces"` + Sites []string `toml:"sites"` + Port int `toml:"port"` + CollectInterval duration.Duration `toml:"collect_interval"` +} diff --git a/runtime/config.go b/runtime/config.go deleted file mode 100644 index ec3c0d1..0000000 --- a/runtime/config.go +++ /dev/null @@ -1,58 +0,0 @@ -package runtime - -import ( - "io/ioutil" - - "github.com/BurntSushi/toml" -) - -//Config the config File of this daemon -type Config struct { - Respondd struct { - Enable bool `toml:"enable"` - Synchronize Duration `toml:"synchronize"` - Interfaces []string `toml:"interfaces"` - Sites []string `toml:"sites"` - Port int `toml:"port"` - CollectInterval Duration `toml:"collect_interval"` - } - Webserver struct { - Enable bool `toml:"enable"` - Bind string `toml:"bind"` - Webroot string `toml:"webroot"` - } - Nodes struct { - StatePath string `toml:"state_path"` - SaveInterval Duration `toml:"save_interval"` // Save nodes periodically - OfflineAfter Duration `toml:"offline_after"` // Set node to offline if not seen within this period - PruneAfter Duration `toml:"prune_after"` // Remove nodes after n days of inactivity - Output map[string]interface{} - } - Meshviewer struct { - Version int `toml:"version"` - NodesPath string `toml:"nodes_path"` - GraphPath string `toml:"graph_path"` - } - Database struct { - DeleteInterval Duration `toml:"delete_interval"` // Delete stats of nodes every n minutes - DeleteAfter Duration `toml:"delete_after"` // Delete stats of nodes till now-deletetill n minutes - Connection map[string]interface{} - } -} - -// ReadConfigFile reads a config model from path of a yml file -func ReadConfigFile(path string) (config *Config, err error) { - config = &Config{} - - file, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - - err = toml.Unmarshal(file, config) - if err != nil { - return nil, err - } - - return -} diff --git a/runtime/config_test.go b/runtime/config_test.go deleted file mode 100644 index ae6269d..0000000 --- a/runtime/config_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package runtime - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestReadConfig(t *testing.T) { - assert := assert.New(t) - - config, err := ReadConfigFile("../config_example.toml") - assert.NoError(err, "no error during reading") - assert.NotNil(config) - - assert.True(config.Respondd.Enable) - assert.Equal([]string{"br-ffhb"}, config.Respondd.Interfaces) - assert.Equal(time.Minute, config.Respondd.CollectInterval.Duration) - - assert.Equal(time.Hour*24*7, config.Nodes.PruneAfter.Duration) - - assert.Equal(time.Hour*24*7, config.Database.DeleteAfter.Duration) - - var meshviewer map[string]interface{} - var outputs []map[string]interface{} - outputs = config.Nodes.Output["meshviewer"].([]map[string]interface{}) - assert.Len(outputs, 1, "more outputs are given") - meshviewer = outputs[0] - assert.Equal(int64(2), meshviewer["version"]) - assert.Equal("/var/www/html/meshviewer/data/nodes.json", meshviewer["nodes_path"]) - - var influxdb map[string]interface{} - dbs := config.Database.Connection["influxdb"].([]map[string]interface{}) - assert.Len(dbs, 1, "more influxdb are given") - influxdb = dbs[0] - assert.Equal(influxdb["database"], "ffhb") - - var graphitedb map[string]interface{} - dbs = config.Database.Connection["graphite"].([]map[string]interface{}) - assert.Len(dbs, 1, "more graphitedb are given") - graphitedb = dbs[0] - assert.Equal(graphitedb["address"], "localhost:2003") - - _, err = ReadConfigFile("testdata/config_failed.toml") - assert.Error(err, "not unmarshalable") - assert.Contains(err.Error(), "Near line ") - - _, err = ReadConfigFile("testdata/adsa.toml") - assert.Error(err, "not found able") - assert.Contains(err.Error(), "no such file or directory") -} diff --git a/runtime/node.go b/runtime/node.go index 76c45d0..f49807b 100644 --- a/runtime/node.go +++ b/runtime/node.go @@ -4,7 +4,7 @@ import ( "net" "github.com/FreifunkBremen/yanic/data" - "github.com/FreifunkBremen/yanic/jsontime" + "github.com/FreifunkBremen/yanic/lib/jsontime" ) // Node struct diff --git a/runtime/nodes.go b/runtime/nodes.go index b963974..be5f9b3 100644 --- a/runtime/nodes.go +++ b/runtime/nodes.go @@ -8,26 +8,26 @@ import ( "time" "github.com/FreifunkBremen/yanic/data" - "github.com/FreifunkBremen/yanic/jsontime" + "github.com/FreifunkBremen/yanic/lib/jsontime" ) // Nodes struct: cache DB of Node's structs type Nodes struct { List map[string]*Node `json:"nodes"` // the current nodemap, indexed by node ID ifaceToNodeID map[string]string // mapping from MAC address to NodeID - config *Config + config *NodesConfig sync.RWMutex } // NewNodes create Nodes structs -func NewNodes(config *Config) *Nodes { +func NewNodes(config *NodesConfig) *Nodes { nodes := &Nodes{ List: make(map[string]*Node), ifaceToNodeID: make(map[string]string), config: config, } - if config.Nodes.StatePath != "" { + if config.StatePath != "" { nodes.load() } @@ -130,7 +130,7 @@ func (nodes *Nodes) NodeLinks(node *Node) (result []Link) { // Periodically saves the cached DB to json file func (nodes *Nodes) worker() { - c := time.Tick(nodes.config.Nodes.SaveInterval.Duration) + c := time.Tick(nodes.config.SaveInterval.Duration) for range c { nodes.expire() @@ -143,14 +143,14 @@ func (nodes *Nodes) expire() { now := jsontime.Now() // Nodes last seen before expireAfter will be removed - prunePeriod := nodes.config.Nodes.PruneAfter.Duration + prunePeriod := nodes.config.PruneAfter.Duration if prunePeriod == 0 { prunePeriod = time.Hour * 24 * 7 // our default } pruneAfter := now.Add(-prunePeriod) // Nodes last seen within OfflineAfter are changed to 'offline' - offlineAfter := now.Add(-nodes.config.Nodes.OfflineAfter.Duration) + offlineAfter := now.Add(-nodes.config.OfflineAfter.Duration) // Locking foo nodes.Lock() @@ -194,7 +194,7 @@ func (nodes *Nodes) readIfaces(nodeinfo *data.NodeInfo) { } func (nodes *Nodes) load() { - path := nodes.config.Nodes.StatePath + path := nodes.config.StatePath if f, err := os.Open(path); err == nil { // transform data to legacy meshviewer if err = json.NewDecoder(f).Decode(nodes); err == nil { @@ -222,7 +222,7 @@ func (nodes *Nodes) save() { defer nodes.RUnlock() // serialize nodes - SaveJSON(nodes, nodes.config.Nodes.StatePath) + SaveJSON(nodes, nodes.config.StatePath) } // SaveJSON to path diff --git a/runtime/nodes_config.go b/runtime/nodes_config.go new file mode 100644 index 0000000..6d9520c --- /dev/null +++ b/runtime/nodes_config.go @@ -0,0 +1,11 @@ +package runtime + +import "github.com/FreifunkBremen/yanic/lib/duration" + +type NodesConfig struct { + StatePath string `toml:"state_path"` + SaveInterval duration.Duration `toml:"save_interval"` // Save nodes periodically + OfflineAfter duration.Duration `toml:"offline_after"` // Set node to offline if not seen within this period + PruneAfter duration.Duration `toml:"prune_after"` // Remove nodes after n days of inactivity + Output map[string]interface{} +} diff --git a/runtime/nodes_test.go b/runtime/nodes_test.go index a42ddcd..9b88503 100644 --- a/runtime/nodes_test.go +++ b/runtime/nodes_test.go @@ -9,15 +9,15 @@ import ( "github.com/stretchr/testify/assert" "github.com/FreifunkBremen/yanic/data" - "github.com/FreifunkBremen/yanic/jsontime" + "github.com/FreifunkBremen/yanic/lib/jsontime" ) func TestExpire(t *testing.T) { assert := assert.New(t) - config := &Config{} - config.Nodes.OfflineAfter.Duration = time.Minute * 10 + config := &NodesConfig{} + config.OfflineAfter.Duration = time.Minute * 10 // to get default (100%) path of testing - // config.Nodes.PruneAfter.Duration = time.Hour * 24 * 6 + // config.PruneAfter.Duration = time.Hour * 24 * 6 nodes := &Nodes{ config: config, List: make(map[string]*Node), @@ -51,22 +51,22 @@ func TestExpire(t *testing.T) { func TestLoadAndSave(t *testing.T) { assert := assert.New(t) - config := &Config{} + config := &NodesConfig{} // not autoload without StatePath NewNodes(config) // Test unmarshalable /dev/null - autolead with StatePath - config.Nodes.StatePath = "/dev/null" + config.StatePath = "/dev/null" nodes := NewNodes(config) // Test unopen able - config.Nodes.StatePath = "/root/nodes.json" + config.StatePath = "/root/nodes.json" nodes.load() // works ;) - config.Nodes.StatePath = "testdata/nodes.json" + config.StatePath = "testdata/nodes.json" nodes.load() tmpfile, _ := ioutil.TempFile("/tmp", "nodes") - config.Nodes.StatePath = tmpfile.Name() + config.StatePath = tmpfile.Name() nodes.save() os.Remove(tmpfile.Name()) @@ -113,8 +113,8 @@ func TestUpdateNodes(t *testing.T) { func TestSelectNodes(t *testing.T) { assert := assert.New(t) - config := &Config{} - config.Nodes.StatePath = "testdata/nodes.json" + config := &NodesConfig{} + config.StatePath = "testdata/nodes.json" nodes := NewNodes(config) @@ -139,7 +139,7 @@ func TestSelectNodes(t *testing.T) { func TestAddNode(t *testing.T) { assert := assert.New(t) - nodes := NewNodes(&Config{}) + nodes := NewNodes(&NodesConfig{}) nodes.AddNode(&Node{}) assert.Len(nodes.List, 0) diff --git a/runtime/stats_test.go b/runtime/stats_test.go index 59e8378..e25ac62 100644 --- a/runtime/stats_test.go +++ b/runtime/stats_test.go @@ -54,7 +54,7 @@ func TestGlobalStats(t *testing.T) { } func createTestNodes() *Nodes { - nodes := NewNodes(&Config{}) + nodes := NewNodes(&NodesConfig{}) nodeData := &Node{ Online: true, diff --git a/runtime/testdata/config_failed.toml b/runtime/testdata/config_failed.toml deleted file mode 100644 index 0247eeb..0000000 --- a/runtime/testdata/config_failed.toml +++ /dev/null @@ -1 +0,0 @@ -asdas diff --git a/webserver/config.go b/webserver/config.go new file mode 100644 index 0000000..6edca0b --- /dev/null +++ b/webserver/config.go @@ -0,0 +1,7 @@ +package webserver + +type Config struct { + Enable bool `toml:"enable"` + Bind string `toml:"bind"` + Webroot string `toml:"webroot"` +}