130 lines
2.4 KiB
Go
130 lines
2.4 KiB
Go
package runtime
|
|
|
|
import (
|
|
"encoding/json"
|
|
"io/ioutil"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
type Fetcher struct {
|
|
url string
|
|
filePath string
|
|
repeat time.Duration
|
|
directoryList map[string]string
|
|
List map[string]*Node `json:"nodes"`
|
|
nodesMutex sync.Mutex
|
|
closed bool
|
|
}
|
|
|
|
func NewFetcher(url string, repeat time.Duration, filePath string) *Fetcher {
|
|
return &Fetcher{
|
|
url: url,
|
|
repeat: repeat,
|
|
filePath: filePath,
|
|
nodesMutex: sync.Mutex{},
|
|
List: make(map[string]*Node),
|
|
}
|
|
}
|
|
|
|
func (f *Fetcher) Start() {
|
|
timer := time.NewTimer(f.repeat)
|
|
for !f.closed {
|
|
select {
|
|
case <-timer.C:
|
|
f.work()
|
|
f.save()
|
|
timer.Reset(f.repeat)
|
|
}
|
|
}
|
|
timer.Stop()
|
|
}
|
|
|
|
func (f *Fetcher) Stop() {
|
|
f.closed = true
|
|
}
|
|
|
|
func (f *Fetcher) AddNode(n *Node) {
|
|
if n != nil {
|
|
f.nodesMutex.Lock()
|
|
f.List[n.NodeID] = n
|
|
f.nodesMutex.Unlock()
|
|
}
|
|
}
|
|
|
|
func (f *Fetcher) work() {
|
|
err := JSONRequest(f.url, &f.directoryList)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
wgDirectory := sync.WaitGroup{}
|
|
wgNodes := sync.WaitGroup{}
|
|
count := 0
|
|
|
|
for community, url := range f.directoryList {
|
|
wgDirectory.Add(1)
|
|
go func(community, url string) {
|
|
defer wgDirectory.Done()
|
|
var directory DirectoryAPI
|
|
|
|
err := JSONRequest(url, &directory)
|
|
if err != nil {
|
|
return
|
|
}
|
|
for _, mapEntry := range directory.NodeMaps {
|
|
if mapEntry.TechnicalType == "nodelist" || mapEntry.MapType == "list/status" {
|
|
wgNodes.Add(1)
|
|
count++
|
|
go f.updateNodes(&wgNodes, community, mapEntry.URL)
|
|
}
|
|
}
|
|
}(community, url)
|
|
}
|
|
|
|
log.Println("found", len(f.directoryList), "communities")
|
|
wgDirectory.Wait()
|
|
log.Println("wait for", count, "request for nodes")
|
|
wgNodes.Wait()
|
|
log.Println("found", len(f.List), "nodes")
|
|
}
|
|
|
|
func (f *Fetcher) updateNodes(wg *sync.WaitGroup, community string, url string) {
|
|
defer wg.Done()
|
|
|
|
resp, err := http.Get(url)
|
|
if err != nil {
|
|
return
|
|
}
|
|
body, err := ioutil.ReadAll(resp.Body)
|
|
transform(body, community, f)
|
|
}
|
|
|
|
func (f *Fetcher) save() {
|
|
f.nodesMutex.Lock()
|
|
defer f.nodesMutex.Unlock()
|
|
|
|
if f.filePath != "" {
|
|
tmpFile := f.filePath + ".tmp"
|
|
|
|
file, err := os.OpenFile(tmpFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
|
|
if err != nil {
|
|
log.Panic(err)
|
|
}
|
|
|
|
err = json.NewEncoder(file).Encode(f)
|
|
if err != nil {
|
|
log.Panic(err)
|
|
}
|
|
|
|
file.Close()
|
|
if err := os.Rename(tmpFile, f.filePath); err != nil {
|
|
log.Panic(err)
|
|
}
|
|
}
|
|
|
|
}
|