From cf2d2c32096256db0f207a5d76df03e72dcaef8d Mon Sep 17 00:00:00 2001 From: Martin Geno Date: Wed, 3 Jan 2018 15:41:40 +0100 Subject: [PATCH] [BUGFIX] tests without sleep --- cmd/import.go | 8 +- cmd/serve.go | 12 +- database/all/connection.go | 75 +++++++++ database/all/internal.go | 85 +++------- database/all/internel_test.go | 151 ++++-------------- database/internal.go | 40 ----- database/internal_test.go | 69 -------- output/all/internal.go | 82 ++++------ output/all/output.go | 64 ++++++++ .../all/{internal_test.go => output_test.go} | 9 +- output/internal.go | 40 ----- output/internal_test.go | 57 ------- runtime/config.go | 28 ++-- 13 files changed, 256 insertions(+), 464 deletions(-) create mode 100644 database/all/connection.go delete mode 100644 database/internal.go delete mode 100644 database/internal_test.go create mode 100644 output/all/output.go rename output/all/{internal_test.go => output_test.go} (94%) delete mode 100644 output/internal.go delete mode 100644 output/internal_test.go diff --git a/cmd/import.go b/cmd/import.go index 8b78ac5..47fee2a 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -3,8 +3,7 @@ package cmd import ( "log" - "github.com/FreifunkBremen/yanic/database" - "github.com/FreifunkBremen/yanic/database/all" + allDatabase "github.com/FreifunkBremen/yanic/database/all" "github.com/FreifunkBremen/yanic/rrd" "github.com/FreifunkBremen/yanic/runtime" "github.com/spf13/cobra" @@ -21,12 +20,11 @@ var importCmd = &cobra.Command{ site := args[1] config := loadConfig() - connections, err := all.Connect(config.Database.Connection) + err := allDatabase.Start(config.Database) if err != nil { panic(err) } - database.Start(connections, config) - defer database.Close(connections) + defer allDatabase.Close() log.Println("importing RRD from", path) diff --git a/cmd/serve.go b/cmd/serve.go index 1872c1b..f803696 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -7,9 +7,7 @@ import ( "syscall" "time" - "github.com/FreifunkBremen/yanic/database" allDatabase "github.com/FreifunkBremen/yanic/database/all" - "github.com/FreifunkBremen/yanic/output" allOutput "github.com/FreifunkBremen/yanic/output/all" "github.com/FreifunkBremen/yanic/respond" "github.com/FreifunkBremen/yanic/runtime" @@ -25,22 +23,20 @@ var serveCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { config := loadConfig() - connections, err := allDatabase.Connect(config.Database.Connection) + err := allDatabase.Start(config.Database) if err != nil { panic(err) } - database.Start(connections, config) - defer database.Close(connections) + defer allDatabase.Close() nodes = runtime.NewNodes(config) nodes.Start() - outputs, err := allOutput.Register(config.Nodes.Output) + err = allOutput.Start(nodes, config.Nodes) if err != nil { panic(err) } - output.Start(outputs, nodes, config) - defer output.Close() + defer allOutput.Close() if config.Webserver.Enable { log.Println("starting webserver on", config.Webserver.Bind) diff --git a/database/all/connection.go b/database/all/connection.go new file mode 100644 index 0000000..9d18706 --- /dev/null +++ b/database/all/connection.go @@ -0,0 +1,75 @@ +package all + +import ( + "fmt" + "log" + "time" + + "github.com/FreifunkBremen/yanic/database" + "github.com/FreifunkBremen/yanic/runtime" +) + +type Connection struct { + database.Connection + list []database.Connection +} + +func Connect(allConnection map[string]interface{}) (database.Connection, error) { + var list []database.Connection + for dbType, conn := range database.Adapters { + configForType := allConnection[dbType] + if configForType == nil { + log.Printf("the output type '%s' has no configuration", dbType) + continue + } + dbConfigs, ok := configForType.([]map[string]interface{}) + if !ok { + return nil, fmt.Errorf("the output type '%s' has the wrong format", dbType) + } + + for _, config := range dbConfigs { + if c, ok := config["enable"].(bool); ok && !c { + continue + } + connected, err := conn(config) + if err != nil { + return nil, err + } + if connected == nil { + continue + } + list = append(list, connected) + } + } + return &Connection{list: list}, nil +} + +func (conn *Connection) InsertNode(node *runtime.Node) { + for _, item := range conn.list { + item.InsertNode(node) + } +} + +func (conn *Connection) InsertLink(link *runtime.Link, time time.Time) { + for _, item := range conn.list { + item.InsertLink(link, time) + } +} + +func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) { + for _, item := range conn.list { + item.InsertGlobals(stats, time, site) + } +} + +func (conn *Connection) PruneNodes(deleteAfter time.Duration) { + for _, item := range conn.list { + item.PruneNodes(deleteAfter) + } +} + +func (conn *Connection) Close() { + for _, item := range conn.list { + item.Close() + } +} diff --git a/database/all/internal.go b/database/all/internal.go index 8a58b66..ec94453 100644 --- a/database/all/internal.go +++ b/database/all/internal.go @@ -1,74 +1,41 @@ package all import ( - "log" "time" "github.com/FreifunkBremen/yanic/database" "github.com/FreifunkBremen/yanic/runtime" ) -type Connection struct { - database.Connection - list []database.Connection +var conn database.Connection +var quit chan struct{} + +func Start(config runtime.DatabaseConfig) (err error) { + conn, err = Connect(config.Connection) + if err != nil { + return + } + quit = make(chan struct{}) + go deleteWorker(config.DeleteInterval.Duration, config.DeleteAfter.Duration) + return } -func Connect(allConnection map[string]interface{}) (database.Connection, error) { - var list []database.Connection - for dbType, conn := range database.Adapters { - configForType := allConnection[dbType] - if configForType == nil { - log.Printf("the output type '%s' has no configuration\n", dbType) - continue - } - dbConfigs, ok := configForType.([]map[string]interface{}) - if !ok { - log.Panicf("the output type '%s' has the wrong format\n", dbType) - } +func Close() { + close(quit) + conn.Close() + quit = nil +} - for _, config := range dbConfigs { - if c, ok := config["enable"].(bool); ok && !c { - continue - } - connected, err := conn(config) - if err != nil { - return nil, err - } - if connected == nil { - continue - } - list = append(list, connected) +// prunes node-specific data periodically +func deleteWorker(deleteInterval time.Duration, deleteAfter time.Duration) { + ticker := time.NewTicker(deleteInterval) + for { + select { + case <-ticker.C: + conn.PruneNodes(deleteAfter) + case <-quit: + ticker.Stop() + return } } - return &Connection{list: list}, nil -} - -func (conn *Connection) InsertNode(node *runtime.Node) { - for _, item := range conn.list { - item.InsertNode(node) - } -} - -func (conn *Connection) InsertLink(link *runtime.Link, time time.Time) { - for _, item := range conn.list { - item.InsertLink(link, time) - } -} - -func (conn *Connection) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) { - for _, item := range conn.list { - item.InsertGlobals(stats, time, site) - } -} - -func (conn *Connection) PruneNodes(deleteAfter time.Duration) { - for _, item := range conn.list { - item.PruneNodes(deleteAfter) - } -} - -func (conn *Connection) Close() { - for _, item := range conn.list { - item.Close() - } } diff --git a/database/all/internel_test.go b/database/all/internel_test.go index 019c2c0..862284b 100644 --- a/database/all/internel_test.go +++ b/database/all/internel_test.go @@ -2,7 +2,6 @@ package all import ( "errors" - "sync" "testing" "time" @@ -11,134 +10,49 @@ import ( "github.com/stretchr/testify/assert" ) -type testConn struct { - database.Connection - countNode int - countLink int - countGlobals int - countPrune int - countClose int - sync.Mutex -} - -func (c *testConn) InsertNode(node *runtime.Node) { - c.Lock() - c.countNode++ - c.Unlock() -} -func (c *testConn) GetNode() int { - c.Lock() - defer c.Unlock() - return c.countNode -} -func (c *testConn) InsertLink(link *runtime.Link, time time.Time) { - c.Lock() - c.countLink++ - c.Unlock() -} -func (c *testConn) GetLink() int { - c.Lock() - defer c.Unlock() - return c.countLink -} -func (c *testConn) InsertGlobals(stats *runtime.GlobalStats, time time.Time, site string) { - c.Lock() - c.countGlobals++ - c.Unlock() -} -func (c *testConn) GetGlobal() int { - c.Lock() - defer c.Unlock() - return c.countGlobals -} -func (c *testConn) PruneNodes(time.Duration) { - c.Lock() - c.countPrune++ - c.Unlock() -} -func (c *testConn) GetPrune() int { - c.Lock() - defer c.Unlock() - return c.countPrune -} -func (c *testConn) Close() { - c.Lock() - c.countClose++ - c.Unlock() -} -func (c *testConn) GetClose() int { - c.Lock() - defer c.Unlock() - return c.countClose -} - func TestStart(t *testing.T) { assert := assert.New(t) - globalConn := &testConn{} - database.RegisterAdapter("a", func(config map[string]interface{}) (database.Connection, error) { - return globalConn, nil - }) - database.RegisterAdapter("b", func(config map[string]interface{}) (database.Connection, error) { - return globalConn, nil - }) - database.RegisterAdapter("c", func(config map[string]interface{}) (database.Connection, error) { - return globalConn, nil - }) database.RegisterAdapter("d", func(config map[string]interface{}) (database.Connection, error) { return nil, nil }) database.RegisterAdapter("e", func(config map[string]interface{}) (database.Connection, error) { return nil, errors.New("blub") }) - allConn, err := Connect(map[string]interface{}{ - "a": []map[string]interface{}{ - map[string]interface{}{ - "enable": false, - "path": "a1", + // Test for PruneNodes (by start) + assert.Nil(quit) + err := Start(runtime.DatabaseConfig{ + DeleteInterval: runtime.Duration{Duration: time.Millisecond}, + Connection: map[string]interface{}{ + "a": []map[string]interface{}{ + map[string]interface{}{ + "enable": false, + "path": "a1", + }, + map[string]interface{}{ + "path": "a2", + }, + map[string]interface{}{ + "enable": true, + "path": "a3", + }, }, - map[string]interface{}{ - "path": "a2", + "b": nil, + "c": []map[string]interface{}{ + map[string]interface{}{ + "path": "c1", + }, }, - map[string]interface{}{ - "enable": true, - "path": "a3", - }, - }, - "b": nil, - "c": []map[string]interface{}{ - map[string]interface{}{ - "path": "c1", - }, - }, - // fetch continue command in Connect - "d": []map[string]interface{}{ - map[string]interface{}{ - "path": "d0", + // fetch continue command in Connect + "d": []map[string]interface{}{ + map[string]interface{}{ + "path": "d0", + }, }, }, }) assert.NoError(err) - - assert.Equal(0, globalConn.GetNode()) - allConn.InsertNode(nil) - assert.Equal(3, globalConn.GetNode()) - - assert.Equal(0, globalConn.GetLink()) - allConn.InsertLink(nil, time.Now()) - assert.Equal(3, globalConn.GetLink()) - - assert.Equal(0, globalConn.GetGlobal()) - allConn.InsertGlobals(nil, time.Now(), runtime.GLOBAL_SITE) - assert.Equal(3, globalConn.GetGlobal()) - - assert.Equal(0, globalConn.GetPrune()) - allConn.PruneNodes(time.Second) - assert.Equal(3, globalConn.GetPrune()) - - assert.Equal(0, globalConn.GetClose()) - allConn.Close() - assert.Equal(3, globalConn.GetClose()) + assert.NotNil(quit) _, err = Connect(map[string]interface{}{ "e": []map[string]interface{}{ @@ -147,10 +61,9 @@ func TestStart(t *testing.T) { }) assert.Error(err) - // wrong format -> the only panic in Register - assert.Panics(func() { - Connect(map[string]interface{}{ - "e": true, - }) + // wrong format + _, err = Connect(map[string]interface{}{ + "e": true, }) + assert.Error(err) } diff --git a/database/internal.go b/database/internal.go deleted file mode 100644 index 5a5eb1c..0000000 --- a/database/internal.go +++ /dev/null @@ -1,40 +0,0 @@ -package database - -import ( - "time" - - "github.com/FreifunkBremen/yanic/runtime" -) - -var quit chan struct{} - -// Start workers of database -// WARNING: Do not override this function -// you should use New() -func Start(conn Connection, config *runtime.Config) { - quit = make(chan struct{}) - go deleteWorker(conn, config.Database.DeleteInterval.Duration, config.Database.DeleteAfter.Duration) -} - -func Close(conn Connection) { - if quit != nil { - close(quit) - } - if conn != nil { - conn.Close() - } -} - -// prunes node-specific data periodically -func deleteWorker(conn Connection, deleteInterval time.Duration, deleteAfter time.Duration) { - ticker := time.NewTicker(deleteInterval) - for { - select { - case <-ticker.C: - conn.PruneNodes(deleteAfter) - case <-quit: - ticker.Stop() - return - } - } -} diff --git a/database/internal_test.go b/database/internal_test.go deleted file mode 100644 index 0280fb7..0000000 --- a/database/internal_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package database - -import ( - "sync" - "testing" - "time" - - "github.com/FreifunkBremen/yanic/runtime" - "github.com/stretchr/testify/assert" -) - -type testConn struct { - Connection - countClose int - countPrune int - sync.Mutex -} - -func (c *testConn) Close() { - c.Lock() - c.countClose++ - c.Unlock() -} -func (c *testConn) GetClose() int { - c.Lock() - defer c.Unlock() - return c.countClose -} -func (c *testConn) PruneNodes(time.Duration) { - c.Lock() - c.countPrune++ - c.Unlock() -} -func (c *testConn) GetPruneNodes() int { - c.Lock() - defer c.Unlock() - return c.countPrune -} - -func TestStart(t *testing.T) { - assert := assert.New(t) - - conn := &testConn{} - config := &runtime.Config{ - Database: struct { - DeleteInterval runtime.Duration `toml:"delete_interval"` - DeleteAfter runtime.Duration `toml:"delete_after"` - Connection map[string]interface{} - }{ - DeleteInterval: runtime.Duration{Duration: time.Millisecond * 10}, - }, - } - assert.Nil(quit) - - Start(conn, config) - assert.NotNil(quit) - - assert.Equal(0, conn.GetPruneNodes()) - time.Sleep(time.Millisecond * 12) - assert.Equal(1, conn.GetPruneNodes()) - - assert.Equal(0, conn.GetClose()) - Close(conn) - assert.NotNil(quit) - assert.Equal(1, conn.GetClose()) - - time.Sleep(time.Millisecond * 12) // to reach timer.Stop() line - -} diff --git a/output/all/internal.go b/output/all/internal.go index f871990..e3c3ee5 100644 --- a/output/all/internal.go +++ b/output/all/internal.go @@ -1,63 +1,45 @@ package all import ( - "log" + "sync" + "time" "github.com/FreifunkBremen/yanic/output" "github.com/FreifunkBremen/yanic/runtime" ) -type Output struct { - output.Output - list map[int]output.Output - filter map[int]filterConfig +var quit chan struct{} +var wg = sync.WaitGroup{} +var outputA output.Output + +func Start(nodes *runtime.Nodes, config runtime.NodesConfig) (err error) { + outputA, err = Register(config.Output) + if err != nil { + return + } + quit = make(chan struct{}) + wg.Add(1) + go saveWorker(nodes, config.SaveInterval.Duration) + return } -func Register(configuration map[string]interface{}) (output.Output, error) { - list := make(map[int]output.Output) - filter := make(map[int]filterConfig) - i := 1 - allOutputs := configuration - for outputType, outputRegister := range output.Adapters { - configForOutput := allOutputs[outputType] - if configForOutput == nil { - log.Printf("the output type '%s' has no configuration\n", outputType) - continue - } - outputConfigs, ok := configForOutput.([]map[string]interface{}) - if !ok { - log.Panicf("the output type '%s' has the wrong format\n", outputType) - } - for _, config := range outputConfigs { - if c, ok := config["enable"].(bool); ok && !c { - continue - } - output, err := outputRegister(config) - if err != nil { - return nil, err - } - if output == nil { - continue - } - list[i] = output - if c := config["filter"]; c != nil { - filter[i] = config["filter"].(map[string]interface{}) - } - i++ +func Close() { + close(quit) + wg.Wait() + quit = nil +} + +// save periodically to output +func saveWorker(nodes *runtime.Nodes, saveInterval time.Duration) { + ticker := time.NewTicker(saveInterval) + for { + select { + case <-ticker.C: + outputA.Save(nodes) + case <-quit: + ticker.Stop() + wg.Done() + return } } - return &Output{list: list, filter: filter}, nil -} - -func (o *Output) Save(nodes *runtime.Nodes) { - for i, item := range o.list { - var filteredNodes *runtime.Nodes - if config := o.filter[i]; config != nil { - filteredNodes = config.filtering(nodes) - } else { - filteredNodes = filterConfig{}.filtering(nodes) - } - - item.Save(filteredNodes) - } } diff --git a/output/all/output.go b/output/all/output.go new file mode 100644 index 0000000..4861659 --- /dev/null +++ b/output/all/output.go @@ -0,0 +1,64 @@ +package all + +import ( + "fmt" + "log" + + "github.com/FreifunkBremen/yanic/output" + "github.com/FreifunkBremen/yanic/runtime" +) + +type Output struct { + output.Output + list map[int]output.Output + filter map[int]filterConfig +} + +func Register(configuration map[string]interface{}) (output.Output, error) { + list := make(map[int]output.Output) + filter := make(map[int]filterConfig) + i := 1 + allOutputs := configuration + for outputType, outputRegister := range output.Adapters { + configForOutput := allOutputs[outputType] + if configForOutput == nil { + log.Printf("the output type '%s' has no configuration\n", outputType) + continue + } + outputConfigs, ok := configForOutput.([]map[string]interface{}) + if !ok { + return nil, fmt.Errorf("the output type '%s' has the wrong format", outputType) + } + for _, config := range outputConfigs { + if c, ok := config["enable"].(bool); ok && !c { + continue + } + output, err := outputRegister(config) + if err != nil { + return nil, err + } + if output == nil { + continue + } + list[i] = output + if c := config["filter"]; c != nil { + filter[i] = config["filter"].(map[string]interface{}) + } + i++ + } + } + return &Output{list: list, filter: filter}, nil +} + +func (o *Output) Save(nodes *runtime.Nodes) { + for i, item := range o.list { + var filteredNodes *runtime.Nodes + if config := o.filter[i]; config != nil { + filteredNodes = config.filtering(nodes) + } else { + filteredNodes = filterConfig{}.filtering(nodes) + } + + item.Save(filteredNodes) + } +} diff --git a/output/all/internal_test.go b/output/all/output_test.go similarity index 94% rename from output/all/internal_test.go rename to output/all/output_test.go index ef87ba2..7f77299 100644 --- a/output/all/internal_test.go +++ b/output/all/output_test.go @@ -89,10 +89,9 @@ func TestStart(t *testing.T) { }) assert.Error(err) - // wrong format -> the only panic in Register - assert.Panics(func() { - Register(map[string]interface{}{ - "e": true, - }) + // wrong format + _, err = Register(map[string]interface{}{ + "e": true, }) + assert.Error(err) } diff --git a/output/internal.go b/output/internal.go deleted file mode 100644 index dcf462b..0000000 --- a/output/internal.go +++ /dev/null @@ -1,40 +0,0 @@ -package output - -import ( - "sync" - "time" - - "github.com/FreifunkBremen/yanic/runtime" -) - -var quit chan struct{} -var wg = sync.WaitGroup{} - -// Start workers of database -// WARNING: Do not override this function -// you should use New() -func Start(output Output, nodes *runtime.Nodes, config *runtime.Config) { - quit = make(chan struct{}) - wg.Add(1) - go saveWorker(output, nodes, config.Nodes.SaveInterval.Duration) -} - -func Close() { - close(quit) - wg.Wait() -} - -// save periodically to output -func saveWorker(output Output, nodes *runtime.Nodes, saveInterval time.Duration) { - ticker := time.NewTicker(saveInterval) - for { - select { - case <-ticker.C: - output.Save(nodes) - case <-quit: - wg.Done() - ticker.Stop() - return - } - } -} diff --git a/output/internal_test.go b/output/internal_test.go deleted file mode 100644 index 2ab095a..0000000 --- a/output/internal_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package output - -import ( - "sync" - "testing" - "time" - - "github.com/FreifunkBremen/yanic/runtime" - "github.com/stretchr/testify/assert" -) - -type testConn struct { - Output - countSave int - sync.Mutex -} - -func (c *testConn) Save(nodes *runtime.Nodes) { - c.Lock() - c.countSave++ - c.Unlock() -} -func (c *testConn) Get() int { - c.Lock() - defer c.Unlock() - return c.countSave -} - -func TestStart(t *testing.T) { - assert := assert.New(t) - - conn := &testConn{} - config := &runtime.Config{ - Nodes: struct { - StatePath string `toml:"state_path"` - SaveInterval runtime.Duration `toml:"save_interval"` - OfflineAfter runtime.Duration `toml:"offline_after"` - PruneAfter runtime.Duration `toml:"prune_after"` - Output map[string]interface{} - }{ - SaveInterval: runtime.Duration{Duration: time.Millisecond * 10}, - }, - } - assert.Nil(quit) - - Start(conn, nil, config) - assert.NotNil(quit) - - assert.Equal(0, conn.Get()) - time.Sleep(time.Millisecond * 12) - assert.Equal(1, conn.Get()) - - time.Sleep(time.Millisecond * 12) - Close() - assert.Equal(2, conn.Get()) - -} diff --git a/runtime/config.go b/runtime/config.go index ec3c0d1..787af8b 100644 --- a/runtime/config.go +++ b/runtime/config.go @@ -21,23 +21,27 @@ type Config struct { Bind string `toml:"bind"` Webroot string `toml:"webroot"` } - Nodes struct { - StatePath string `toml:"state_path"` - SaveInterval Duration `toml:"save_interval"` // Save nodes periodically - OfflineAfter Duration `toml:"offline_after"` // Set node to offline if not seen within this period - PruneAfter Duration `toml:"prune_after"` // Remove nodes after n days of inactivity - Output map[string]interface{} - } + Nodes NodesConfig Meshviewer struct { Version int `toml:"version"` NodesPath string `toml:"nodes_path"` GraphPath string `toml:"graph_path"` } - Database struct { - DeleteInterval Duration `toml:"delete_interval"` // Delete stats of nodes every n minutes - DeleteAfter Duration `toml:"delete_after"` // Delete stats of nodes till now-deletetill n minutes - Connection map[string]interface{} - } + Database DatabaseConfig +} + +type NodesConfig struct { + StatePath string `toml:"state_path"` + SaveInterval Duration `toml:"save_interval"` // Save nodes periodically + OfflineAfter Duration `toml:"offline_after"` // Set node to offline if not seen within this period + PruneAfter Duration `toml:"prune_after"` // Remove nodes after n days of inactivity + Output map[string]interface{} +} + +type DatabaseConfig struct { + DeleteInterval Duration `toml:"delete_interval"` // Delete stats of nodes every n minutes + DeleteAfter Duration `toml:"delete_after"` // Delete stats of nodes till now-deletetill n minutes + Connection map[string]interface{} } // ReadConfigFile reads a config model from path of a yml file