From 61139dc92f1876382e21b8ec1c81604ddb85de5e Mon Sep 17 00:00:00 2001 From: Elliot Peele Date: Thu, 15 Oct 2020 22:33:40 -0500 Subject: [PATCH 1/5] Reload config on update (cherry picked from commit 6abd3b7f213f61a22c5fa520bf4d63fd37afebfd) --- cmds/houndd/main.go | 104 +++++++++++++++---------------------------- config/watcher.go | 79 ++++++++++++++++++++++++++++++++ searcher/searcher.go | 23 ++++++---- 3 files changed, 129 insertions(+), 77 deletions(-) create mode 100644 config/watcher.go diff --git a/cmds/houndd/main.go b/cmds/houndd/main.go index eb8038a2..4f725fdd 100644 --- a/cmds/houndd/main.go +++ b/cmds/houndd/main.go @@ -15,6 +15,7 @@ import ( "syscall" "github.com/blang/semver/v4" + "github.com/fsnotify/fsnotify" "github.com/hound-search/hound/api" "github.com/hound-search/hound/config" "github.com/hound-search/hound/searcher" @@ -31,30 +32,30 @@ var ( basepath = filepath.Dir(b) ) -func makeSearchers(cfg *config.Config) (map[string]*searcher.Searcher, bool, error) { +func makeSearchers(cfg *config.Config, searchers map[string]*searcher.Searcher) (bool, error) { // Ensure we have a dbpath if _, err := os.Stat(cfg.DbPath); err != nil { if err := os.MkdirAll(cfg.DbPath, os.ModePerm); err != nil { - return nil, false, err + return false, err } } - searchers, errs, err := searcher.MakeAll(cfg) + errs, err := searcher.MakeAll(cfg, searchers) if err != nil { - return nil, false, err + return false, err } if len(errs) > 0 { // NOTE: This mutates the original config so the repos // are not even seen by other code paths. - for name, _ := range errs { //nolint + for name := range errs { delete(cfg.Repos, name) } - return searchers, false, nil + return false, nil } - return searchers, true, nil + return true, nil } func handleShutdown(shutdownCh <-chan os.Signal, searchers map[string]*searcher.Searcher) { @@ -79,51 +80,6 @@ func registerShutdownSignal() <-chan os.Signal { return shutdownCh } -func makeTemplateData(cfg *config.Config) (interface{}, error) { //nolint - var data struct { - ReposAsJson string - } - - res := map[string]*config.Repo{} - for name, repo := range cfg.Repos { - res[name] = repo - } - - b, err := json.Marshal(res) - if err != nil { - return nil, err - } - - data.ReposAsJson = string(b) - return &data, nil -} - -func runHttp( //nolint - addr string, - dev bool, - cfg *config.Config, - idx map[string]*searcher.Searcher) error { - m := http.DefaultServeMux - - h, err := ui.Content(dev, cfg) - if err != nil { - return err - } - - m.Handle("/", h) - api.Setup(m, idx, cfg.ResultLimit) - return http.ListenAndServe(addr, m) -} - -// TODO: Automatically increment this when building a release -func getVersion() semver.Version { - return semver.Version{ - Major: 0, - Minor: 7, - Patch: 1, - } -} - func main() { runtime.GOMAXPROCS(runtime.NumCPU()) info_log = log.New(os.Stdout, "", log.LstdFlags) @@ -141,29 +97,39 @@ func main() { os.Exit(0) } + idx := make(map[string]*searcher.Searcher) + var cfg config.Config - if err := cfg.LoadFromFile(*flagConf); err != nil { - panic(err) + + loadConfig := func() { + if err := cfg.LoadFromFile(*flagConf); err != nil { + panic(err) + } + // It's not safe to be killed during makeSearchers, so register the + // shutdown signal here and defer processing it until we are ready. + shutdownCh := registerShutdownSignal() + ok, err := makeSearchers(&cfg, idx) + if err != nil { + log.Panic(err) + } + if !ok { + info_log.Println("Some repos failed to index, see output above") + } else { + info_log.Println("All indexes built!") + } + handleShutdown(shutdownCh, idx) } + loadConfig() + + // watch for config file changes + configWatcher := config.NewWatcher(*flagConf) + configWatcher.OnChange(func(fsnotify.Event) { + loadConfig() + }) // Start the web server on a background routine. ws := web.Start(&cfg, *flagAddr, *flagDev) - // It's not safe to be killed during makeSearchers, so register the - // shutdown signal here and defer processing it until we are ready. - shutdownCh := registerShutdownSignal() - idx, ok, err := makeSearchers(&cfg) - if err != nil { - log.Panic(err) - } - if !ok { - info_log.Println("Some repos failed to index, see output above") - } else { - info_log.Println("All indexes built!") - } - - handleShutdown(shutdownCh, idx) - host := *flagAddr if strings.HasPrefix(host, ":") { //nolint host = "localhost" + host diff --git a/config/watcher.go b/config/watcher.go new file mode 100644 index 00000000..bd3a0486 --- /dev/null +++ b/config/watcher.go @@ -0,0 +1,79 @@ +package config + +import ( + "log" + "sync" + + "github.com/fsnotify/fsnotify" +) + +// WatcherListenerFunc defines the signature for listner functions +type WatcherListenerFunc func(fsnotify.Event) + +// Watcher watches for configuration updates and provides hooks for +// triggering post events +type Watcher struct { + listeners []WatcherListenerFunc +} + +// NewWatcher returns a new file watcher +func NewWatcher(cfgPath string) *Watcher { + log.Printf("setting up watcher for %s", cfgPath) + w := Watcher{} + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Panic(err) + } + defer watcher.Close() + // Event listener setup + eventWG := sync.WaitGroup{} + eventWG.Add(1) + go func() { + defer eventWG.Done() + for { + select { + case event, ok := <-watcher.Events: + if !ok { + // events channel is closed + log.Printf("error: events channel is closed\n") + return + } + // only trigger on creates and writes of the watched config file + if event.Name == cfgPath && event.Op&fsnotify.Write == fsnotify.Write { + log.Printf("change in config file (%s) detected\n", cfgPath) + for _, listener := range w.listeners { + listener(event) + } + } + case err, ok := <-watcher.Errors: + if !ok { + // errors channel is closed + log.Printf("error: errors channel is closed\n") + return + } + log.Println("error:", err) + return + } + } + }() + // add config file + if err := watcher.Add(cfgPath); err != nil { + log.Fatalf("failed to watch %s", cfgPath) + } + // setup is complete + wg.Done() + // wait for the event listener to complete before exiting + eventWG.Wait() + }() + // wait for watcher setup to complete + wg.Wait() + return &w +} + +// OnChange registers a listener function to be called if a file changes +func (w *Watcher) OnChange(listener WatcherListenerFunc) { + w.listeners = append(w.listeners, listener) +} diff --git a/searcher/searcher.go b/searcher/searcher.go index 791ce810..597cd309 100644 --- a/searcher/searcher.go +++ b/searcher/searcher.go @@ -264,7 +264,7 @@ func reportOnMemory() { // Utility function for producing a hex encoded sha1 hash for a string. func hashFor(name string) string { h := sha1.New() - h.Write([]byte(name)) //nolint + h.Write([]byte(name)) //nolint return hex.EncodeToString(h.Sum(nil)) } @@ -282,24 +282,32 @@ func init() { // occurred and no other return values are valid. If an error occurs that is specific // to a particular searcher, that searcher will not be present in the searcher map and // will have an error entry in the error map. -func MakeAll(cfg *config.Config) (map[string]*Searcher, map[string]error, error) { +func MakeAll(cfg *config.Config, searchers map[string]*Searcher) (map[string]error, error) { errs := map[string]error{} - searchers := map[string]*Searcher{} refs, err := findExistingRefs(cfg.DbPath) if err != nil { - return nil, nil, err + return nil, err } lim := makeLimiter(cfg.MaxConcurrentIndexers) - n := len(cfg.Repos) + n := 0 + for name := range cfg.Repos { + if _, ok := searchers[name]; ok { + continue + } + n++ + } // Channel to receive the results from newSearcherConcurrent function. resultCh := make(chan searcherResult, n) // Start new searchers for all repos in different go routines while // respecting cfg.MaxConcurrentIndexers. for name, repo := range cfg.Repos { + if _, ok := searchers[name]; ok { + continue + } go newSearcherConcurrent(cfg.DbPath, name, repo, refs, lim, resultCh) } @@ -315,7 +323,7 @@ func MakeAll(cfg *config.Config) (map[string]*Searcher, map[string]error, error) } if err := refs.removeUnclaimed(); err != nil { - return nil, nil, err + return nil, err } // after all the repos are in good shape, we start their polling @@ -323,7 +331,7 @@ func MakeAll(cfg *config.Config) (map[string]*Searcher, map[string]error, error) s.begin() } - return searchers, errs, nil + return errs, nil } // Creates a new Searcher that is available for searches as soon as this returns. @@ -407,7 +415,6 @@ func newSearcher( return nil, err } - rev, err := wd.PullOrClone(vcsDir, repo.Url) if err != nil { return nil, err From be179920aa745d7bcb47c29d8db1afc5d5fe59b5 Mon Sep 17 00:00:00 2001 From: Elliot Peele Date: Wed, 11 Nov 2020 00:48:18 -0500 Subject: [PATCH 2/5] claim refs that are still in use (cherry picked from commit 1e54ee455876b475acbb19060fda229cfdd1514e) --- searcher/searcher.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/searcher/searcher.go b/searcher/searcher.go index 597cd309..93c36b5d 100644 --- a/searcher/searcher.go +++ b/searcher/searcher.go @@ -52,7 +52,7 @@ type limiter chan bool */ type foundRefs struct { refs []*index.IndexRef - claimed map[*index.IndexRef]bool + claimed map[string]bool lock sync.Mutex } @@ -89,7 +89,7 @@ func (r *foundRefs) claim(ref *index.IndexRef) { r.lock.Lock() defer r.lock.Unlock() - r.claimed[ref] = true + r.claimed[ref.Dir()] = true } /** @@ -101,7 +101,7 @@ func (r *foundRefs) removeUnclaimed() error { defer r.lock.Unlock() for _, ref := range r.refs { - if r.claimed[ref] { + if r.claimed[ref.Dir()] { continue } @@ -223,7 +223,7 @@ func findExistingRefs(dbpath string) (*foundRefs, error) { return &foundRefs{ refs: refs, - claimed: map[*index.IndexRef]bool{}, + claimed: map[string]bool{}, }, nil } @@ -294,7 +294,9 @@ func MakeAll(cfg *config.Config, searchers map[string]*Searcher) (map[string]err n := 0 for name := range cfg.Repos { - if _, ok := searchers[name]; ok { + if s, ok := searchers[name]; ok { + // claim any already running searcher refs so that they don't get removed + refs.claim(s.idx.Ref) continue } n++ From 678da65a47eceb34f5e14f6264e6c28e58333730 Mon Sep 17 00:00:00 2001 From: Daniel Sullivan Date: Tue, 5 Sep 2023 15:43:13 -0500 Subject: [PATCH 3/5] fix main.go imports and deleted function, add fsnotify to go.mod --- cmds/houndd/main.go | 19 ++++++++++++------- go.mod | 3 ++- go.sum | 6 ++++++ 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/cmds/houndd/main.go b/cmds/houndd/main.go index 4f725fdd..1de8824d 100644 --- a/cmds/houndd/main.go +++ b/cmds/houndd/main.go @@ -1,11 +1,10 @@ package main import ( - "encoding/json" "flag" "fmt" + "github.com/blang/semver/v4" "log" - "net/http" "os" "os/exec" "os/signal" @@ -14,12 +13,9 @@ import ( "strings" "syscall" - "github.com/blang/semver/v4" "github.com/fsnotify/fsnotify" - "github.com/hound-search/hound/api" "github.com/hound-search/hound/config" "github.com/hound-search/hound/searcher" - "github.com/hound-search/hound/ui" "github.com/hound-search/hound/web" ) @@ -80,6 +76,15 @@ func registerShutdownSignal() <-chan os.Signal { return shutdownCh } +// TODO: Automatically increment this when building a release +func getVersion() semver.Version { + return semver.Version{ + Major: 0, + Minor: 7, + Patch: 1, + } +} + func main() { runtime.GOMAXPROCS(runtime.NumCPU()) info_log = log.New(os.Stdout, "", log.LstdFlags) @@ -141,8 +146,8 @@ func main() { webpack.Dir = basepath + "/../../" webpack.Stdout = os.Stdout webpack.Stderr = os.Stderr - err = webpack.Start() - if err != nil { + + if err := webpack.Start(); err != nil { error_log.Println(err) } } diff --git a/go.mod b/go.mod index 3d5151e6..1cf2d011 100644 --- a/go.mod +++ b/go.mod @@ -4,5 +4,6 @@ go 1.16 require ( github.com/blang/semver/v4 v4.0.0 - golang.org/x/mod v0.10.0 + github.com/fsnotify/fsnotify v1.6.0 + golang.org/x/mod v0.12.0 ) diff --git a/go.sum b/go.sum index ab72a812..1ece7cc7 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,15 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -16,6 +20,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956 h1:XeJjHH1KiLpKGb6lvMiksZ9l0fVUh+AmGcm0nOMEBOY= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 2da3beb55d40d95d66f2415f576488b5e4a22e25 Mon Sep 17 00:00:00 2001 From: Daniel Sullivan Date: Wed, 6 Sep 2023 10:55:22 -0500 Subject: [PATCH 4/5] Improved config reload handling * allowing for deleted repositories * recreation of event handling fixed * mutex for only one config at a time --- cmds/houndd/main.go | 67 +++++++++++++++++++++++++++++++++++---------- config/config.go | 4 ++- web/web.go | 12 +++++++- 3 files changed, 67 insertions(+), 16 deletions(-) diff --git a/cmds/houndd/main.go b/cmds/houndd/main.go index 1de8824d..b4673b05 100644 --- a/cmds/houndd/main.go +++ b/cmds/houndd/main.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "github.com/blang/semver/v4" + "github.com/fsnotify/fsnotify" "log" "os" "os/exec" @@ -11,16 +12,14 @@ import ( "path/filepath" "runtime" "strings" + "sync" "syscall" - "github.com/fsnotify/fsnotify" "github.com/hound-search/hound/config" "github.com/hound-search/hound/searcher" "github.com/hound-search/hound/web" ) -const gracefulShutdownSignal = syscall.SIGTERM - var ( info_log *log.Logger error_log *log.Logger @@ -70,12 +69,21 @@ func handleShutdown(shutdownCh <-chan os.Signal, searchers map[string]*searcher. }() } -func registerShutdownSignal() <-chan os.Signal { +func registerShutdownSignal() chan os.Signal { shutdownCh := make(chan os.Signal, 1) - signal.Notify(shutdownCh, gracefulShutdownSignal) + signal.Notify(shutdownCh, syscall.SIGTERM) + signal.Notify(shutdownCh, syscall.SIGINT) return shutdownCh } +func unregisterShutdownSignal(shutdownCh chan os.Signal) { + if shutdownCh == nil { + return + } + signal.Stop(shutdownCh) + shutdownCh = nil +} + // TODO: Automatically increment this when building a release func getVersion() semver.Version { return semver.Version{ @@ -106,13 +114,26 @@ func main() { var cfg config.Config - loadConfig := func() { + var shutdownCh chan os.Signal + shutdownCh = nil + var configUpdateLock sync.Mutex + + loadConfig := func(server *web.Server) { + configUpdateLock.Lock() + defer configUpdateLock.Unlock() + // store existing cfg to check if it's changed + cfgJson, _ := cfg.ToJsonString() + if err := cfg.LoadFromFile(*flagConf); err != nil { panic(err) } + // unregister shutdown signal to create a new one + unregisterShutdownSignal(shutdownCh) + // It's not safe to be killed during makeSearchers, so register the // shutdown signal here and defer processing it until we are ready. - shutdownCh := registerShutdownSignal() + shutdownCh = registerShutdownSignal() + ok, err := makeSearchers(&cfg, idx) if err != nil { log.Panic(err) @@ -122,15 +143,20 @@ func main() { } else { info_log.Println("All indexes built!") } + // handle shutdown signal now handleShutdown(shutdownCh, idx) + if server != nil { + // if server has been passed, now check if cfg json has been changed + newCfgJson, err := cfg.ToJsonString() + if (err == nil) && (cfgJson != newCfgJson) { + // cfg json changed, update the server + info_log.Println("configJson updated, reloading server") + if err := server.UpdateServeWithIndex(idx); err != nil { + error_log.Printf("updating server failed for some reason: %s", err) + } + } + } } - loadConfig() - - // watch for config file changes - configWatcher := config.NewWatcher(*flagConf) - configWatcher.OnChange(func(fsnotify.Event) { - loadConfig() - }) // Start the web server on a background routine. ws := web.Start(&cfg, *flagAddr, *flagDev) @@ -139,6 +165,11 @@ func main() { if strings.HasPrefix(host, ":") { //nolint host = "localhost" + host } + info_log.Printf("started server without indexes at http://%s\n", host) + + // Initial config load + loadConfig(nil) + info_log.Printf("loaded config") if *flagDev { info_log.Printf("[DEV] starting webpack-dev-server at localhost:8080...") @@ -152,6 +183,14 @@ func main() { } } + // watch for config file changes + configWatcher := config.NewWatcher(*flagConf) + configWatcher.OnChange( + func(fsnotify.Event) { + loadConfig(ws) + }, + ) + info_log.Printf("running server at http://%s\n", host) // Fully enable the web server now that we have indexes diff --git a/config/config.go b/config/config.go index c978c009..bbd53d49 100644 --- a/config/config.go +++ b/config/config.go @@ -194,7 +194,9 @@ func (c *Config) LoadFromFile(filename string) error { return err } defer r.Close() - + // reset Repos and VCSConfigMessages so that upon reload we clear out deleted things + c.Repos = make(map[string]*Repo) + c.VCSConfigMessages = make(map[string]*SecretMessage) if err := json.NewDecoder(r).Decode(c); err != nil { return err } diff --git a/web/web.go b/web/web.go index d32dd4e8..bd7647e9 100644 --- a/web/web.go +++ b/web/web.go @@ -70,6 +70,16 @@ func Start(cfg *config.Config, addr string, dev bool) *Server { // ServeWithIndex allow the server to start offering the search UI and the // search APIs operating on the given indexes. func (s *Server) ServeWithIndex(idx map[string]*searcher.Searcher) error { + err := s.UpdateServeWithIndex(idx) + if err != nil { + return err + } + + return <-s.ch +} + +// UpdateServeWithIndex updates the server handler without returning the error channel +func (s *Server) UpdateServeWithIndex(idx map[string]*searcher.Searcher) error { h, err := ui.Content(s.dev, s.cfg) if err != nil { return err @@ -81,5 +91,5 @@ func (s *Server) ServeWithIndex(idx map[string]*searcher.Searcher) error { s.serveWith(m) - return <-s.ch + return nil } From 48e002e982c29c901e5a72f65871165ff55c292e Mon Sep 17 00:00:00 2001 From: Daniel Sullivan Date: Wed, 6 Sep 2023 11:03:27 -0500 Subject: [PATCH 5/5] lint fix --- cmds/houndd/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmds/houndd/main.go b/cmds/houndd/main.go index b4673b05..d9e9a733 100644 --- a/cmds/houndd/main.go +++ b/cmds/houndd/main.go @@ -81,7 +81,6 @@ func unregisterShutdownSignal(shutdownCh chan os.Signal) { return } signal.Stop(shutdownCh) - shutdownCh = nil } // TODO: Automatically increment this when building a release @@ -129,6 +128,7 @@ func main() { } // unregister shutdown signal to create a new one unregisterShutdownSignal(shutdownCh) + shutdownCh = nil // It's not safe to be killed during makeSearchers, so register the // shutdown signal here and defer processing it until we are ready.