modify: cache search index for performance

internal/domain/content/entity/search.go

@@ -11,6 +11,8 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
+	"time"
 )
 
 type TypeService interface {
@@ -20,140 +22,88 @@ type TypeService interface {
 	IsAdminType(name string) bool
 }
 
-type Search struct {
-	TypeService TypeService
+var cleanupWaitDuration = 5 * time.Minute
 
-	Repo repository.Repository
-	Log  loggers.Logger
+type CacheIndex struct {
+	bleve.Index
 
-	IndicesMap map[string]map[string]bleve.Index
+	timer *time.Timer
 }
 
-// TypeQuery conducts a search and returns a set of Ponzu "targets", Type:ID pairs,
-// and an error. If there is no search index for the typeName (Type) provided,
-// db.ErrNoIndex will be returned as the error
-func (s *Search) TypeQuery(typeName, query string, count, offset int) ([]content.Identifier, error) {
-	s.setup()
-
-	idx, ok := s.IndicesMap[s.getSearchDir(typeName)][typeName]
-	if !ok {
-		s.Log.Debugln("Index for type ", typeName, " not found")
-		return nil, content.ErrNoIndex
-	}
-
-	s.Log.Debugln("TypeQuery: ", query)
-	q := bleve.NewQueryStringQuery(query)
-	req := bleve.NewSearchRequestOptions(q, count, offset, false)
-	res, err := idx.Search(req)
-	if err != nil {
-		return nil, err
-	}
-
-	var results []content.Identifier
-	for _, hit := range res.Hits {
-		results = append(results, valueobject.CreateIndex(hit.ID))
-	}
-
-	return results, nil
+func (ci *CacheIndex) close() error {
+	ci.timer.Stop()
+	return ci.Index.Close()
 }
 
-// UpdateIndex sets data into a content type's search index at the given
-// identifier
-func (s *Search) UpdateIndex(ns, id string, data []byte) error {
-	s.setup()
-
-	idx, ok := s.IndicesMap[s.getSearchDir(ns)][ns]
-	if ok {
-		// unmarshal json to struct, error if not registered
-		it, ok := s.TypeService.GetContentCreator(ns)
-		if !ok {
-			return fmt.Errorf("[search] UpdateIndex Error: type '%s' doesn't exist", ns)
-		}
-
-		p := it()
-		err := json.Unmarshal(data, &p)
-		if err != nil {
-			return err
-		}
-
-		// add data to search index
-		i := valueobject.NewIndex(ns, id)
-		err = idx.Index(i.String(), p)
+type Search struct {
+	TypeService TypeService
 
-		return err
-	}
+	Repo repository.Repository
+	Log  loggers.Logger
 
-	return nil
+	mu         sync.Mutex
+	IndicesMap map[string]*CacheIndex
 }
 
-// DeleteIndex removes data from a content type's search index at the
-// given identifier
-func (s *Search) DeleteIndex(id string) error {
-	s.setup()
-
-	// check if there is a search index to work with
-	target := strings.Split(id, ":")
-	ns := target[0]
-
-	idx, ok := s.IndicesMap[s.getSearchDir(ns)][ns]
-	if ok {
-		// add data to search index
-		err := idx.Delete(id)
-
-		return err
-	}
-
-	return nil
+func (s *Search) getSearchPath(ns string) string {
+	return fmt.Sprintf("%s-%s", s.getSearchDir(ns), ns)
 }
 
-func (s *Search) getSearchDir(ns string) string {
-	if s.TypeService.IsAdminType(ns) {
-		return s.adminSearchDir()
+func (s *Search) resetDBTimer(index *CacheIndex) {
+	if index.timer != nil {
+		index.timer.Stop()
 	}
-	return s.userSearchDir()
-}
 
-func (s *Search) userSearchDir() string {
-	return filepath.Join(s.Repo.UserDataDir(), "Search")
+	index.timer = time.NewTimer(cleanupWaitDuration)
+	go func() {
+		<-index.timer.C
+		s.cleanupIdleDB(index) // 触发统一的清理函数
+	}()
 }
 
-func (s *Search) adminSearchDir() string {
-	return filepath.Join(s.Repo.AdminDataDir(), "Search")
-}
+func (s *Search) cleanupIdleDB(idleIndex *CacheIndex) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
 
-func (s *Search) setup() {
-	s.setupAdminIndices()
-	s.setupUserIndices()
-}
+	for searchPath, idx := range s.IndicesMap {
+		if idx == idleIndex {
+			err := idx.close()
+			if err != nil {
+				s.Log.Errorln("Couldn't close search index.", err)
+				return
+			}
 
-func (s *Search) setupUserIndices() {
-	s.setupIndices(s.userSearchDir(), s.TypeService.AllContentTypeNames())
+			delete(s.IndicesMap, searchPath)
+			s.Log.Printf("Clean search index %s, %d", searchPath, len(s.IndicesMap))
+			return
+		}
+	}
 }
 
-func (s *Search) setupAdminIndices() {
-	s.setupIndices(s.adminSearchDir(), s.TypeService.AllAdminTypeNames())
-}
+func (s *Search) getSearchIndex(ns string) (bleve.Index, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
 
-func (s *Search) setupIndices(dir string, typeNames []string) {
-	_, ok := s.IndicesMap[dir]
-	if ok {
-		s.Log.Debugln("[search] Setup found: Index already exists for ", dir)
-		return
-	}
+	searchPath := s.getSearchPath(ns)
 
-	searchIndices := make(map[string]bleve.Index)
+	if idx, ok := s.IndicesMap[searchPath]; ok {
+		s.resetDBTimer(idx)
+		return idx.Index, nil
+	}
 
-	for _, t := range typeNames {
-		idx, err := s.mapIndex(t)
-		if err != nil {
-			s.Log.Errorln("[search] Setup Error", err)
-			return
-		}
+	index, err := s.mapIndex(ns)
+	if err != nil {
+		s.Log.Errorln("[search] Setup Error", searchPath, err)
+		return nil, err
+	}
 
-		searchIndices[t] = idx
+	ci := &CacheIndex{
+		Index: index,
 	}
+	s.resetDBTimer(ci)
 
-	s.IndicesMap[dir] = searchIndices
+	s.IndicesMap[searchPath] = ci
+	return ci.Index, nil
 }
 
 // MapIndex creates the mapping for a type and tracks the index to be used within
@@ -195,6 +145,7 @@ func (s *Search) mapIndex(typeName string) (bleve.Index, error) {
 		if err != nil {
 			return nil, err
 		}
+		idx.SetName(idxName)
 		s.Log.Debugf("[search] Index new created for %s\n", typeName)
 	} else {
 		idx, err = bleve.Open(idxPath)
@@ -204,7 +155,87 @@ func (s *Search) mapIndex(typeName string) (bleve.Index, error) {
 		s.Log.Debugf("[search] Index open created for %s\n", typeName)
 	}
 
-	idx.SetName(idxName)
-
 	return idx, nil
 }
+
+// TypeQuery conducts a search and returns a set of Ponzu "targets", Type:ID pairs,
+// and an error. If there is no search index for the typeName (Type) provided,
+// db.ErrNoIndex will be returned as the error
+func (s *Search) TypeQuery(typeName, query string, count, offset int) ([]content.Identifier, error) {
+	idx, err := s.getSearchIndex(typeName)
+	if err != nil {
+		s.Log.Debugln("Index for type ", typeName, " not found", err.Error())
+		return nil, content.ErrNoIndex
+	}
+
+	s.Log.Debugln("TypeQuery: ", query)
+	q := bleve.NewQueryStringQuery(query)
+	req := bleve.NewSearchRequestOptions(q, count, offset, false)
+	res, err := idx.Search(req)
+	if err != nil {
+		return nil, err
+	}
+
+	var results []content.Identifier
+	for _, hit := range res.Hits {
+		results = append(results, valueobject.CreateIndex(hit.ID))
+	}
+
+	return results, nil
+}
+
+// UpdateIndex sets data into a content type's search index at the given
+// identifier
+func (s *Search) UpdateIndex(ns, id string, data []byte) error {
+	idx, err := s.getSearchIndex(ns)
+	if err != nil {
+		return err
+	}
+
+	// unmarshal json to struct, error if not registered
+	it, ok := s.TypeService.GetContentCreator(ns)
+	if !ok {
+		return fmt.Errorf("[search] UpdateIndex Error: type '%s' doesn't exist", ns)
+	}
+
+	p := it()
+	err = json.Unmarshal(data, &p)
+	if err != nil {
+		return err
+	}
+
+	// add data to search index
+	i := valueobject.NewIndex(ns, id)
+	err = idx.Index(i.String(), p)
+
+	return err
+}
+
+// DeleteIndex removes data from a content type's search index at the
+// given identifier
+func (s *Search) DeleteIndex(id string) error {
+	target := strings.Split(id, ":")
+	ns := target[0]
+
+	idx, err := s.getSearchIndex(ns)
+	if err != nil {
+		return err
+	}
+
+	return idx.Delete(id)
+}
+
+func (s *Search) getSearchDir(ns string) string {
+	if s.TypeService.IsAdminType(ns) {
+		return s.adminSearchDir()
+	}
+	return s.userSearchDir()
+}
+
+func (s *Search) userSearchDir() string {
+	return filepath.Join(s.Repo.UserDataDir(), "Search")
+}
+
+func (s *Search) adminSearchDir() string {
+	return filepath.Join(s.Repo.AdminDataDir(), "Search")
+}

internal/domain/content/factory/content.go

@@ -1,7 +1,6 @@
 package factory
 
 import (
-	"github.com/blevesearch/bleve"
 	"github.com/gohugonet/hugoverse/internal/domain/content"
 	"github.com/gohugonet/hugoverse/internal/domain/content/entity"
 	"github.com/gohugonet/hugoverse/internal/domain/content/repository"
@@ -37,7 +36,7 @@ func NewContent(repo repository.Repository, dir content.DirService) *entity.Cont
 		Repo:        repo,
 		Log:         log,
 
-		IndicesMap: make(map[string]map[string]bleve.Index),
+		IndicesMap: make(map[string]*entity.CacheIndex),
 	}
 
 	return c

internal/interfaces/cli/vercurr.go

@@ -3,6 +3,6 @@ package cli
 var CurrentVersion = Version{
     Major:      0,
     Minor:      0,
-    PatchLevel: 11,
+    PatchLevel: 12,
     Suffix:     "",
 }

manifest.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.0.11",
+  "version": "0.0.12",
   "name": "Hugoverse",
   "description": "Headless CMS for Hugo",
   "author": "sunwei",