abc
-{{.Content}} --- layouts/_default/single.html -- -hello single page
-{{.Content}}` - data := txtar.Parse([]byte(files)) - for _, f := range data.Files { - filename := filepath.Join(workingDir, f.Name) - data := bytes.TrimSuffix(f.Data, []byte("\n")) - - err := afs.MkdirAll(filepath.Dir(filename), 0777) - if err != nil { - fmt.Println(err) - } - err = afero.WriteFile(afs, filename, data, 0666) - if err != nil { - fmt.Println(err) - } - } + return tmpDir, nil } diff --git a/internal/application/dir.go b/internal/application/dir.go new file mode 100644 index 0000000..8760c06 --- /dev/null +++ b/internal/application/dir.go @@ -0,0 +1,73 @@ +package application + +import ( + "fmt" + "log" + "os" + "path/filepath" +) + +var cachedHugoverseDir string + +func init() { + cachedHugoverseDir = hugoverseDir() + + err := ensureDirExists(cachedHugoverseDir) + if err != nil { + log.Fatalln(err) + } +} + +func TLSDir() string { + return filepath.Join(DataDir(), "tls") +} + +func UploadDir() string { + return filepath.Join(DataDir(), "uploads") +} + +func SearchDir() string { + return filepath.Join(DataDir(), "search") +} + +func DataDir() string { + return cachedHugoverseDir +} + +func hugoverseDir() string { + homeDir, err := os.UserHomeDir() + if err != nil { + fmt.Println("Error getting home directory:", err, "using current directory as working directory") + + return getWd() + } + + // 构建目录路径 ~/.local/share/hugoverse + hugoverseDir := filepath.Join(homeDir, ".local", "share", "hugoverse") + + return hugoverseDir +} + +func getWd() string { + wd, err := os.Getwd() + if err != nil { + log.Fatalln("Couldn't find working directory", err) + } + return wd +} + +func ensureDirExists(dir string) error { + // 使用 os.Stat 检查目录是否存在 + if _, err := os.Stat(dir); os.IsNotExist(err) { + // 目录不存在，使用 os.MkdirAll 创建目录 + err := os.MkdirAll(dir, 0755) + if err != nil { + return fmt.Errorf("failed to create directory: %w", err) + } + fmt.Println("Directory created:", dir) + } else if err != nil { + // 其他错误 + return fmt.Errorf("failed to check directory: %w", err) + } + return nil +} diff --git a/internal/domain/config/entity/language.go b/internal/domain/config/entity/language.go index 2673f92..89c15c3 100644 --- a/internal/domain/config/entity/language.go +++ b/internal/domain/config/entity/language.go @@ -103,3 +103,13 @@ func (l *Language) GetLanguageName(lang string) string { return "" } + +func (l *Language) GetLanguageFolder(lang string) string { + for c, v := range l.RootConfigs { + if c == lang { + return v.ContentDir + } + } + + return "" +} diff --git a/internal/domain/content/entity/content.go b/internal/domain/content/entity/content.go index 75bf562..3ee0e64 100644 --- a/internal/domain/content/entity/content.go +++ b/internal/domain/content/entity/content.go @@ -4,21 +4,40 @@ import ( "encoding/json" "errors" "fmt" - "github.com/gofrs/uuid" "github.com/gohugonet/hugoverse/internal/domain/content" "github.com/gohugonet/hugoverse/internal/domain/content/repository" "github.com/gohugonet/hugoverse/internal/domain/content/valueobject" "github.com/gohugonet/hugoverse/pkg/form" + "github.com/gohugonet/hugoverse/pkg/loggers" "github.com/gorilla/schema" "log" "net/url" "sort" - "strconv" ) +type contentSvc interface { + newContent(contentType string, ci any) (string, error) + search(contentType string, query string) ([][]byte, error) +} + type Content struct { Types map[string]content.Creator Repo repository.Repository + + *Search + *Hugo + + Log loggers.Logger +} + +func (c *Content) count(contentType string) { + all := c.Repo.AllContent(contentType) + + fmt.Println("all: ", contentType, len(all)) +} + +func (c *Content) LoadHugoProject() error { + return c.Hugo.LoadProject(c) } func (c *Content) AllContentTypeNames() []string { @@ -64,13 +83,24 @@ func (c *Content) DeleteContent(contentType, id, status string) error { return err } + ns := GetNamespace(contentType, status) if err := c.Repo.DeleteContent( - GetNamespace(contentType, status), + ns, id, cti.(content.Sluggable).ItemSlug()); err != nil { return err } + go func() { + // delete indexed data from search index + if isPublicNamespace(ns) { + err = c.Search.DeleteIndex(ns) + if err != nil { + log.Println("[search] DeleteIndex Error:", err) + } + } + }() + if err := c.SortContent(contentType); err != nil { return err } @@ -112,6 +142,19 @@ func (c *Content) UpdateContent(contentType string, data url.Values) error { return errors.New("invalid content type") } status := cis.ItemStatus() + + if cii, ok := ci.(content.Identifiable); ok { + go func() { + // update data in search index + if err := c.Search.UpdateIndex( + GetNamespace(contentType, string(cis.ItemStatus())), + fmt.Sprintf("%d", cii.ItemID()), b); err != nil { + + log.Println("[search] UpdateIndex Error:", err) + } + }() + } + if status == content.Public { go func() { err := c.SortContent(contentType) @@ -141,6 +184,10 @@ func (c *Content) SortContent(contentType string) error { // decode each (json) into type to then sort for i := range all { j := all[i] + if j == nil { + log.Println("Error decoding json while sorting", contentType, ": nil") + continue + } post := t() err := json.Unmarshal(j, &post) @@ -183,87 +230,6 @@ func (c *Content) SortContent(contentType string) error { return nil } -func (c *Content) NewContent(contentType string, data url.Values) (string, error) { - t, ok := c.GetContentCreator(contentType) - if !ok { - return "", errors.New("invalid content type") - } - ci := t() - - d, err := form.Convert(data) - if err != nil { - return "", err - } - // Decode Content - dec := schema.NewDecoder() - dec.SetAliasTag("json") // allows simpler struct tagging when creating a content type - dec.IgnoreUnknownKeys(true) // will skip over form values submitted, but not in struct - err = dec.Decode(ci, d) - if err != nil { - return "", err - } - - cii, ok := ci.(content.Identifiable) - if ok { - uid, err := uuid.NewV4() - if err != nil { - return "", err - } - cii.SetUniqueID(uid) - - id, err := c.Repo.NextContentId(contentType) - if err != nil { - return "", err - } - cii.SetItemID(int(id)) - } else { - return "", errors.New("content type does not implement Identifiable") - } - - slug, err := Slug(cii) - if err != nil { - return "", err - } - - slug, err = c.Repo.CheckSlugForDuplicate(slug) - if err != nil { - return "", err - } - - ciSlug, ok := ci.(content.Sluggable) - if ok { - ciSlug.SetSlug(slug) - } else { - return "", errors.New("content type does not implement Sluggable") - } - - cis, ok := ci.(content.Statusable) - if ok { - if cis.ItemStatus() == "" { - cis.SetItemStatus(content.Public) - } - } else { - return "", errors.New("content type does not implement Statusable") - } - - b, err := c.Marshal(ci) - if err != nil { - return "", err - } - - if err := c.Repo.NewContent(ci, b); err != nil { - return "", err - } - - if cis.ItemStatus() == content.Public { - if err := c.SortContent(contentType); err != nil { - return "", err - } - } - - return strconv.FormatInt(int64(cii.ItemID()), 10), nil -} - func (c *Content) Marshal(content any) ([]byte, error) { j, err := json.Marshal(content) if err != nil { diff --git a/internal/domain/content/entity/creator.go b/internal/domain/content/entity/creator.go new file mode 100644 index 0000000..7328f69 --- /dev/null +++ b/internal/domain/content/entity/creator.go @@ -0,0 +1,115 @@ +package entity + +import ( + "errors" + "fmt" + "github.com/gofrs/uuid" + "github.com/gohugonet/hugoverse/internal/domain/content" + "github.com/gohugonet/hugoverse/pkg/form" + "github.com/gorilla/schema" + "log" + "net/url" + "strconv" +) + +func (c *Content) NewContent(contentType string, data url.Values) (string, error) { + t, ok := c.GetContentCreator(contentType) + if !ok { + return "", errors.New("invalid content type") + } + ci := t() + + d, err := form.Convert(data) + if err != nil { + return "", err + } + // Decode Content + dec := schema.NewDecoder() + dec.SetAliasTag("json") // allows simpler struct tagging when creating a content type + dec.IgnoreUnknownKeys(true) // will skip over form values submitted, but not in struct + err = dec.Decode(ci, d) + if err != nil { + return "", err + } + + // TODO， need to sync content to file system + // in hugo project way + // check changes: file system existing check? hash? + return c.newContent(contentType, ci) +} + +func (c *Content) newContent(contentType string, ci any) (string, error) { + cii, ok := ci.(content.Identifiable) + if ok { + uid, err := uuid.NewV4() + if err != nil { + return "", err + } + cii.SetUniqueID(uid) + + id, err := c.Repo.NextContentId(contentType) + if err != nil { + return "", err + } + cii.SetItemID(int(id)) + } else { + return "", errors.New("content type does not implement Identifiable") + } + + slug, err := Slug(cii) + if err != nil { + return "", err + } + + slug, err = c.Repo.CheckSlugForDuplicate(slug) + if err != nil { + return "", err + } + + ciSlug, ok := ci.(content.Sluggable) + if ok { + ciSlug.SetSlug(slug) + } else { + return "", errors.New("content type does not implement Sluggable") + } + + cis, ok := ci.(content.Statusable) + if ok { + if cis.ItemStatus() == "" { + cis.SetItemStatus(content.Public) + } + } else { + return "", errors.New("content type does not implement Statusable") + } + + b, err := c.Marshal(ci) + if err != nil { + return "", err + } + + if err := c.Repo.NewContent(ci, b); err != nil { + return "", err + } + + if cis.ItemStatus() == content.Public { + go func() { + if err := c.SortContent(contentType); err != nil { + log.Println("sort content err: ", err) + } + }() + } + + id := int64(cii.ItemID()) + + go func() { + // update data in search index + if err := c.Search.UpdateIndex( + GetNamespace(contentType, string(cis.ItemStatus())), + fmt.Sprintf("%d", id), b); err != nil { + + log.Println("[search] UpdateIndex Error:", err) + } + }() + + return strconv.FormatInt(id, 10), nil +} diff --git a/internal/domain/content/entity/hugo.go b/internal/domain/content/entity/hugo.go new file mode 100644 index 0000000..f9b57a0 --- /dev/null +++ b/internal/domain/content/entity/hugo.go @@ -0,0 +1,254 @@ +package entity + +import ( + "encoding/json" + "fmt" + "github.com/gohugonet/hugoverse/internal/domain/content" + "github.com/gohugonet/hugoverse/internal/domain/content/valueobject" + "github.com/gohugonet/hugoverse/internal/domain/contenthub" + "github.com/gohugonet/hugoverse/pkg/loggers" + "gopkg.in/yaml.v3" + "path" + "strconv" +) + +type Hugo struct { + Services content.Services + + contentSvc contentSvc + + site *valueobject.Site + + Log loggers.Logger +} + +func (h *Hugo) LoadProject(c contentSvc) error { + h.contentSvc = c + + if err := h.loadSite(); err != nil { + return err + } + + if err := h.loadSiteLanguages(); err != nil { + return err + } + + if err := h.loadPosts(); err != nil { + return err + } + + return nil +} + +func (h *Hugo) loadPosts() error { + authorQueryStr, err := h.getAuthor() + if err != nil { + return err + } + + codes := h.Services.LanguageKeys() + for _, code := range codes { + langIndex, err := h.Services.GetLanguageIndex(code) + if err != nil { + return err + } + + if err := h.Services.WalkPages(langIndex, func(p contenthub.Page) error { + if p.PageIdentity().PageLanguage() != code { + return nil + } + h.Log.Printf("Loading post: %s, %s-%s\n", + p.PageFile().FileInfo().RelativeFilename(), code, p.PageIdentity().PageLanguage()) + + i, err := valueobject.NewItemWithNamespace("Post") + if err != nil { + return err + } + + post := &valueobject.Post{ + Item: *i, + Title: p.Title(), + Author: authorQueryStr, + Content: p.RawContent(), + } // TODO, page assets + id, err := h.contentSvc.newContent("Post", post) + if err != nil { + return err + } + + h.Log.Printf("Loaded post: %+v", post) + + num, err := strconv.Atoi(id) + if err != nil { + return err + } + post.ID = num + + spi, err := valueobject.NewItemWithNamespace("SitePost") + if err != nil { + return err + } + + sitePost := &valueobject.SitePost{ + Item: *spi, + Post: post.QueryString(), + Site: h.site.QueryString(), + Path: path.Join(p.PageFile().FileInfo().Root(), p.PageFile().FileInfo().RelativeFilename()), + } + _, err = h.contentSvc.newContent("SitePost", sitePost) + if err != nil { + return err + } + + h.Log.Printf("Loaded SitePost: %+v", sitePost) + + return nil + }); err != nil { + return err + } + } + + return nil +} + +func (h *Hugo) loadSiteLanguages() error { + codes := h.Services.LanguageKeys() + for _, code := range codes { + i, err := valueobject.NewItemWithNamespace("SiteLanguage") + if err != nil { + return err + } + + langQueryStr, err := h.getLanguage(code) + if err != nil { + return err + } + h.Log.Println("Get language", code, langQueryStr) + + siteLang := &valueobject.SiteLanguage{ + Item: *i, + Site: h.site.QueryString(), + Language: langQueryStr, + Default: code == h.Services.DefaultLanguage(), + Folder: h.Services.GetLanguageFolder(code), + } + h.Log.Printf("Loadeding SiteLanguage: %+v", *siteLang) + _, err = h.contentSvc.newContent("SiteLanguage", siteLang) + if err != nil { + return err + } + + h.Log.Printf("Loaded SiteLanguage: %+v", *siteLang) + } + return nil +} + +func (h *Hugo) loadSite() error { + i, err := valueobject.NewItemWithNamespace("Site") + if err != nil { + return err + } + + themeQueryStr, err := h.getTheme(h.Services.DefaultTheme()) + if err != nil { + return err + } + + site := &valueobject.Site{ + Item: *i, + Title: h.Services.SiteTitle(), + BaseURL: h.Services.BaseUrl(), + WorkingDir: h.Services.WorkingDir(), + Theme: themeQueryStr, + Owner: 1, + } + if h.Services.ConfigParams() != nil { + site.Params, err = mapToYAML(h.Services.ConfigParams()) + if err != nil { + return err + } + } + + id, err := h.contentSvc.newContent("Site", site) + if err != nil { + return err + } + + h.Log.Printf("Loaded Site: %+v", *site) + + num, err := strconv.Atoi(id) + if err != nil { + return err + } + site.ID = num + h.site = site + + return nil +} + +func (h *Hugo) getTheme(moduleUrl string) (string, error) { + data, err := h.contentSvc.search("Theme", fmt.Sprintf("module_url:%s", moduleUrl)) + if err != nil { + return "", err + } + + if len(data) > 0 { + firstData := data[0] + var result valueobject.Theme + if err := json.Unmarshal(firstData, &result); err != nil { + return "", err + } + + return result.QueryString(), nil + } + + return "", fmt.Errorf("no themes found") +} + +func (h *Hugo) getLanguage(code string) (string, error) { + data, err := h.contentSvc.search("Language", fmt.Sprintf("code:%s", code)) + if err != nil { + return "", err + } + + if len(data) > 0 { + firstData := data[0] + var result valueobject.Language + if err := json.Unmarshal(firstData, &result); err != nil { + return "", err + } + + return result.QueryString(), nil + } + + return "", fmt.Errorf("no languages found") +} + +func (h *Hugo) getAuthor() (string, error) { + //TODO get user email from token + + data, err := h.contentSvc.search("Author", fmt.Sprintf("email:%s", "me@sunwei.xyz")) + if err != nil { + return "", err + } + + if len(data) > 0 { + firstData := data[0] + var result valueobject.Author + if err := json.Unmarshal(firstData, &result); err != nil { + return "", err + } + + return result.QueryString(), nil + } + + return "", fmt.Errorf("no authors found") +} + +func mapToYAML(data map[string]any) (string, error) { + yamlData, err := yaml.Marshal(data) + if err != nil { + return "", err + } + return string(yamlData), nil +} diff --git a/internal/domain/content/entity/namespace.go b/internal/domain/content/entity/namespace.go index fb1f0c1..706b7ec 100644 --- a/internal/domain/content/entity/namespace.go +++ b/internal/domain/content/entity/namespace.go @@ -3,6 +3,7 @@ package entity import ( "fmt" "github.com/gohugonet/hugoverse/internal/domain/content" + "strings" ) func GetNamespace(contentType, status string) string { @@ -12,3 +13,7 @@ func GetNamespace(contentType, status string) string { } return ns } + +func isPublicNamespace(ns string) bool { + return !strings.Contains(ns, "__") +} diff --git a/internal/domain/content/entity/query.go b/internal/domain/content/entity/query.go new file mode 100644 index 0000000..33f9ee9 --- /dev/null +++ b/internal/domain/content/entity/query.go @@ -0,0 +1,29 @@ +package entity + +import ( + "errors" + "github.com/gohugonet/hugoverse/internal/interfaces/api/search" +) + +func (c *Content) search(contentType string, query string) ([][]byte, error) { + // execute search for query provided, if no index for type send 404 + indices, err := c.Search.TypeQuery(contentType, query, 10, 0) + if errors.Is(err, search.ErrNoIndex) { + c.Log.Errorf("Index for type %s not found", contentType) + + return nil, err + } + if err != nil { + c.Log.Errorf("Error searching for type %s: %v", contentType, err) + return nil, err + } + + // respond with json formatted results + bb, err := c.GetContents(indices) + if err != nil { + c.Log.Errorf("Error getting content: %v", err) + return nil, err + } + + return bb, nil +} diff --git a/internal/domain/content/entity/search.go b/internal/domain/content/entity/search.go new file mode 100644 index 0000000..f45b385 --- /dev/null +++ b/internal/domain/content/entity/search.go @@ -0,0 +1,167 @@ +package entity + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/blevesearch/bleve" + "github.com/gohugonet/hugoverse/internal/domain/content" + "github.com/gohugonet/hugoverse/internal/domain/content/valueobject" + "log" + "os" + "path/filepath" + "strings" +) + +var ( + // SearchIndices tracks all search indices to use throughout system + SearchIndices map[string]bleve.Index + + // ErrNoIndex is for failed checks for an index in Search map + ErrNoIndex = errors.New("no search index found for type provided") + + ContentTypes map[string]content.Creator +) + +type Search struct{} + +// TypeQuery conducts a search and returns a set of Ponzu "targets", Type:ID pairs, +// and an error. If there is no search index for the typeName (Type) provided, +// db.ErrNoIndex will be returned as the error +func (s *Search) TypeQuery(typeName, query string, count, offset int) ([]content.Identifier, error) { + idx, ok := SearchIndices[typeName] + if !ok { + fmt.Println("Index for type ", typeName, " not found") + return nil, ErrNoIndex + } + + q := bleve.NewQueryStringQuery(query) + req := bleve.NewSearchRequestOptions(q, count, offset, false) + res, err := idx.Search(req) + if err != nil { + return nil, err + } + + var results []content.Identifier + for _, hit := range res.Hits { + results = append(results, valueobject.CreateIndex(hit.ID)) + } + + return results, nil +} + +// UpdateIndex sets data into a content type's search index at the given +// identifier +func (s *Search) UpdateIndex(ns, id string, data []byte) error { + idx, ok := SearchIndices[ns] + if ok { + // unmarshal json to struct, error if not registered + it, ok := ContentTypes[ns] + if !ok { + return fmt.Errorf("[search] UpdateIndex Error: type '%s' doesn't exist", ns) + } + + p := it() + err := json.Unmarshal(data, &p) + if err != nil { + return err + } + + // add data to search index + i := valueobject.NewIndex(ns, id) + return idx.Index(i.String(), p) + } + + return nil +} + +// DeleteIndex removes data from a content type's search index at the +// given identifier +func (s *Search) DeleteIndex(id string) error { + // check if there is a search index to work with + target := strings.Split(id, ":") + ns := target[0] + + idx, ok := SearchIndices[ns] + if ok { + // add data to search index + return idx.Delete(id) + } + + return nil +} + +// Setup initializes Search Index for search to be functional +// This was moved out of db.Init and put to main(), because addon checker was initializing db together with +// search indexing initialisation in time when there were no item.Types defined so search index was always +// empty when using addons. We still have no guarentee whatsoever that item.Types is defined +// Should be called from a goroutine after SetContent is successful (SortContent requirement) +func (s *Search) Setup(cts map[string]content.Creator, searchDir string) { + SearchIndices = make(map[string]bleve.Index) + ContentTypes = cts + + for t := range ContentTypes { + err := MapIndex(t, searchDir) + if err != nil { + log.Fatalln(err) + return + } + } +} + +// MapIndex creates the mapping for a type and tracks the index to be used within +// the system for adding/deleting/checking data +func MapIndex(typeName string, searchDir string) error { + // type assert for Searchable, get configuration (which can be overridden) + // by Ponzu user if defines own SearchMapping() + it, ok := ContentTypes[typeName] + if !ok { + return fmt.Errorf("[search] MapIndex Error: Failed to MapIndex for %s, type doesn't exist", typeName) + } + s, ok := it().(content.Searchable) + if !ok { + return fmt.Errorf("[search] MapIndex Error: Item type %s doesn't implement search.Searchable", typeName) + } + + // skip setting or using index for types that shouldn't be indexed + if !s.IndexContent() { + fmt.Printf("[search] Index not created for %s\n", typeName) + return nil + } + + mapping, err := s.SearchMapping() + if err != nil { + fmt.Println(err) + return err + } + + idxName := typeName + ".index" + var idx bleve.Index + + searchPath := searchDir + + err = os.MkdirAll(searchPath, os.ModeDir|os.ModePerm) + if err != nil { + return err + } + + idxPath := filepath.Join(searchPath, idxName) + if _, err = os.Stat(idxPath); os.IsNotExist(err) { + idx, err = bleve.New(idxPath, mapping) + if err != nil { + return err + } + idx.SetName(idxName) + } else { + idx, err = bleve.Open(idxPath) + if err != nil { + return err + } + } + + // add the type name to the index and track the index + SearchIndices[typeName] = idx + fmt.Printf("[search] Index created for %s\n", typeName) + + return nil +} diff --git a/internal/domain/content/factory/content.go b/internal/domain/content/factory/content.go index 2f8c09b..b83ed0f 100644 --- a/internal/domain/content/factory/content.go +++ b/internal/domain/content/factory/content.go @@ -5,12 +5,15 @@ import ( "github.com/gohugonet/hugoverse/internal/domain/content/entity" "github.com/gohugonet/hugoverse/internal/domain/content/repository" "github.com/gohugonet/hugoverse/internal/domain/content/valueobject" + "github.com/gohugonet/hugoverse/pkg/loggers" ) -func NewContent(repo repository.Repository) content.Content { +func NewContent(repo repository.Repository, searchDir string) *entity.Content { c := &entity.Content{ Types: make(map[string]content.Creator), Repo: repo, + + Log: loggers.NewDefault(), } c.Types["Author"] = func() interface{} { return new(valueobject.Author) } @@ -21,5 +24,27 @@ func NewContent(repo repository.Repository) content.Content { c.Types["SiteLanguage"] = func() interface{} { return new(valueobject.SiteLanguage) } c.Types["SitePost"] = func() interface{} { return new(valueobject.SitePost) } + c.Search = newSearch(c, searchDir) + return c } + +func newSearch(c *entity.Content, searchDir string) *entity.Search { + s := &entity.Search{} + s.Setup(c.AllContentTypes(), searchDir) + return s +} + +func NewContentWithServices(repo repository.Repository, searchDir string, services content.Services) *entity.Content { + c := NewContent(repo, searchDir) + c.Hugo = &entity.Hugo{ + Services: services, + Log: c.Log, + } + + return c +} + +func NewItem() (*valueobject.Item, error) { + return valueobject.NewItem() +} diff --git a/internal/domain/content/factory/item.go b/internal/domain/content/factory/item.go deleted file mode 100644 index 75d64ac..0000000 --- a/internal/domain/content/factory/item.go +++ /dev/null @@ -1,24 +0,0 @@ -package factory - -import ( - "github.com/gofrs/uuid" - "github.com/gohugonet/hugoverse/internal/domain/content/valueobject" - "github.com/gohugonet/hugoverse/pkg/timestamp" -) - -func NewItem() (*valueobject.Item, error) { - uid, err := uuid.NewV4() - if err != nil { - return nil, err - } - - nowMillis := timestamp.CurrentTimeMillis() - - return &valueobject.Item{ - UUID: uid, - ID: -1, - Slug: "", - Timestamp: nowMillis, - Updated: nowMillis, - }, nil -} diff --git a/internal/domain/content/type.go b/internal/domain/content/type.go index b5f0fef..eebbcd2 100644 --- a/internal/domain/content/type.go +++ b/internal/domain/content/type.go @@ -4,8 +4,8 @@ import ( "errors" "github.com/blevesearch/bleve/mapping" "github.com/gofrs/uuid" + "github.com/gohugonet/hugoverse/internal/domain/contenthub" "net/http" - "net/url" ) type Creator func() interface{} @@ -15,20 +15,28 @@ type Identifier interface { ContentType() string } -type Content interface { - AllContentTypeNames() []string - AllContentTypes() map[string]Creator - NormalizeString(s string) (string, error) - GetContentCreator(string) (Creator, bool) - - GetContents([]Identifier) ([][]byte, error) +type Services interface { + SiteService + CHService +} - //GetContent Todo, convert to identifier +type SiteService interface { + SiteTitle() string + BaseUrl() string + ConfigParams() map[string]any + DefaultTheme() string + WorkingDir() string + + LanguageKeys() []string + DefaultLanguage() string + GetLanguageName(lang string) string + GetLanguageIndex(lang string) (int, error) + GetLanguageFolder(lang string) string +} - GetContent(contentType, id, status string) ([]byte, error) - DeleteContent(contentType, id, status string) error - NewContent(contentType string, data url.Values) (string, error) - UpdateContent(contentType string, data url.Values) error +type CHService interface { + WalkPages(langIndex int, walker contenthub.WalkFunc) error + GetPageSources(page contenthub.Page) ([]contenthub.PageSource, error) } type Status string diff --git a/internal/domain/content/valueobject/http.go b/internal/domain/content/valueobject/http.go new file mode 100644 index 0000000..880eae2 --- /dev/null +++ b/internal/domain/content/valueobject/http.go @@ -0,0 +1,97 @@ +package valueobject + +import ( + "bytes" + "encoding/json" + "fmt" + "gopkg.in/yaml.v3" + "io" + "net/http" + "net/url" + "reflect" +) + +func FetchAPIContent(contentType string, query string) ([]byte, error) { + // Create the URL with the parameters + fullURL := fmt.Sprintf("http://127.0.0.1:1314/api/search?type=%s&q=%s", url.QueryEscape(contentType), url.QueryEscape(query)) + + // Send the GET request + resp, err := http.Get(fullURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // Check for non-OK status codes + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("error: received status code %d", resp.StatusCode) + } + + // Read the response body + return io.ReadAll(resp.Body) +} + +func postAPIContent(data map[string]interface{}) (string, error) { + // Convert the map to JSON + jsonData, err := json.Marshal(data) + if err != nil { + return "", fmt.Errorf("error marshalling data: %v", err) + } + + // Send the POST request + resp, err := http.Post("http://127.0.0.1:1314/admin/edit", "application/json", bytes.NewBuffer(jsonData)) + if err != nil { + return "", fmt.Errorf("error making POST request: %v", err) + } + defer resp.Body.Close() + + // Check for non-OK status codes + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("error: received status code %d", resp.StatusCode) + } + + // Get the "Location" header from the response + location := resp.Header.Get("Location") + if location == "" { + return "", fmt.Errorf("no Location header found in response") + } + + // Return the modified URL + return location, nil +} + +func structToMap(obj interface{}) (map[string]interface{}, error) { + // Ensure that obj is a struct + v := reflect.ValueOf(obj) + if v.Kind() != reflect.Struct { + return nil, fmt.Errorf("expected a struct, got %T", obj) + } + + // Create a map to hold field names and values + result := make(map[string]interface{}) + + // Iterate over the struct fields + for i := 0; i < v.NumField(); i++ { + field := v.Type().Field(i) // Get the field metadata + fieldName := field.Name // Get the field name + + // Use the JSON tag if available, otherwise use the struct field name + jsonTag := field.Tag.Get("json") + if jsonTag == "" { + jsonTag = fieldName + } + + // Store the field value in the map + result[jsonTag] = v.Field(i).Interface() + } + + return result, nil +} + +func mapToYAML(data map[string]any) (string, error) { + yamlData, err := yaml.Marshal(data) + if err != nil { + return "", err + } + return string(yamlData), nil +} diff --git a/internal/domain/content/valueobject/item.go b/internal/domain/content/valueobject/item.go index 3b96b29..2880e12 100644 --- a/internal/domain/content/valueobject/item.go +++ b/internal/domain/content/valueobject/item.go @@ -6,6 +6,9 @@ import ( "github.com/blevesearch/bleve/mapping" "github.com/gofrs/uuid" "github.com/gohugonet/hugoverse/internal/domain/content" + "github.com/gohugonet/hugoverse/pkg/timestamp" + "golang.org/x/text/cases" + "golang.org/x/text/language" "net/http" ) @@ -20,6 +23,32 @@ type Item struct { Updated int64 `json:"updated"` } +func NewItem() (*Item, error) { + uid, err := uuid.NewV4() + if err != nil { + return nil, err + } + + nowMillis := timestamp.CurrentTimeMillis() + + return &Item{ + UUID: uid, + ID: -1, + Slug: "", + Timestamp: nowMillis, + Updated: nowMillis, + }, nil +} + +func NewItemWithNamespace(namespace string) (*Item, error) { + i, err := NewItem() + if err != nil { + return nil, err + } + i.Namespace = namespace + return i, nil +} + // Time partially implements the Sortable interface func (i *Item) Time() int64 { return i.Timestamp @@ -230,3 +259,16 @@ func (i *Item) SearchMapping() (*mapping.IndexMappingImpl, error) { func (i *Item) IndexContent() bool { return false } + +func (i *Item) QueryString() string { + return fmt.Sprintf("/api/content?type=%s&id=%d", capitalizeFirstLetter(i.Namespace), i.ID) +} + +func capitalizeFirstLetter(s string) string { + if len(s) == 0 { + return s + } + + caser := cases.Title(language.English) + return caser.String(s[:1]) + s[1:] +} diff --git a/internal/domain/content/valueobject/searchindex.go b/internal/domain/content/valueobject/searchindex.go new file mode 100644 index 0000000..eae88b4 --- /dev/null +++ b/internal/domain/content/valueobject/searchindex.go @@ -0,0 +1,42 @@ +package valueobject + +import ( + "fmt" + "strings" +) + +type Index struct { + ns string + id string +} + +func (i *Index) String() string { + return fmt.Sprintf("%s:%s", i.ns, i.id) +} + +func (i *Index) Namespace() string { + return i.ns +} + +func (i *Index) ContentType() string { + return i.ns +} + +func (i *Index) ID() string { + return i.id +} + +func CreateIndex(target string) *Index { + t := strings.Split(target, ":") + return &Index{ + ns: t[0], + id: t[1], + } +} + +func NewIndex(ns, id string) *Index { + return &Index{ + ns: ns, + id: id, + } +} diff --git a/internal/domain/content/valueobject/site.go b/internal/domain/content/valueobject/site.go index ec11811..674d7f6 100644 --- a/internal/domain/content/valueobject/site.go +++ b/internal/domain/content/valueobject/site.go @@ -16,6 +16,7 @@ type Site struct { Theme string `json:"theme"` Params string `json:"params"` Owner int `json:"owner"` + WorkingDir string `json:"working_dir"` } // MarshalEditor writes a buffer of html to edit a Song within the CMS @@ -65,10 +66,17 @@ func (s *Site) MarshalEditor() ([]byte, error) { "placeholder": "Enter the owner user id here", }), }, + editor.Field{ + View: editor.Input("WorkingDir", s, map[string]string{ + "label": "WorkingDir", + "type": "text", + "placeholder": "Enter the project dir here", + }), + }, ) if err != nil { - return nil, fmt.Errorf("failed to render Author editor view: %s", err.Error()) + return nil, fmt.Errorf("failed to render Site editor view: %s", err.Error()) } return view, nil diff --git a/internal/domain/content/valueobject/timer.go b/internal/domain/content/valueobject/timer.go index 33d241a..09e7c12 100644 --- a/internal/domain/content/valueobject/timer.go +++ b/internal/domain/content/valueobject/timer.go @@ -45,6 +45,7 @@ func EnoughTime(key string, cb func(key string) error) bool { <-enoughTimer.C lastInvocationAfterTimer, _ := lastInvocation(key) if !lastInvocationAfterTimer.After(lastInvocationBeforeTimer) { + log.Println("Time to trigger sort", key) if err := cb(key); err != nil { log.Println("Error while updating db with sorted", key, err) return diff --git a/internal/domain/contenthub/entity/contenthub.go b/internal/domain/contenthub/entity/contenthub.go index e4d4bb9..3b18a36 100644 --- a/internal/domain/contenthub/entity/contenthub.go +++ b/internal/domain/contenthub/entity/contenthub.go @@ -34,6 +34,20 @@ func (ch *ContentHub) RenderString(ctx context.Context, args ...any) (goTmpl.HTM return "", nil } +func (ch *ContentHub) ProcessPages(exec contenthub.Template) error { + ch.pagesLog = ch.Log.InfoCommand("ContentHub.ProcessPages") + defer loggers.TimeTrackf(ch.pagesLog, time.Now(), nil, "") + + ch.TemplateExecutor = exec + ch.PageMap.PageBuilder.TemplateSvc = exec + + if err := ch.process(); err != nil { + return fmt.Errorf("process: %w", err) + } + + return nil +} + func (ch *ContentHub) CollectPages(exec contenthub.Template) error { ch.pagesLog = ch.Log.InfoCommand("ContentHub.CollectPages") defer loggers.TimeTrackf(ch.pagesLog, time.Now(), nil, "") diff --git a/internal/interfaces/api/database/db.go b/internal/interfaces/api/database/db.go index 401de94..25fbfa1 100644 --- a/internal/interfaces/api/database/db.go +++ b/internal/interfaces/api/database/db.go @@ -5,9 +5,7 @@ import ( "errors" "fmt" "github.com/gohugonet/hugoverse/internal/domain/content" - "github.com/gohugonet/hugoverse/internal/interfaces/api/search" "github.com/gohugonet/hugoverse/pkg/db" - "log" "strconv" ) @@ -95,13 +93,6 @@ func (d *Database) PutContent(ci any, data []byte) error { return err } - go func() { - // update data in search index - if err := search.UpdateIndex(ns, fmt.Sprintf("%d", id), data); err != nil { - log.Println("[search] UpdateIndex Error:", err) - } - }() - return nil } diff --git a/internal/interfaces/api/handler/contentid.go b/internal/interfaces/api/handler/contentid.go deleted file mode 100644 index f64eb0c..0000000 --- a/internal/interfaces/api/handler/contentid.go +++ /dev/null @@ -1,34 +0,0 @@ -package handler - -import ( - "github.com/gohugonet/hugoverse/internal/domain/content" - "github.com/gohugonet/hugoverse/internal/interfaces/api/search" -) - -type contentIdentifier struct { - contentType string - id string -} - -func (c *contentIdentifier) ContentType() string { - return c.contentType -} - -func (c *contentIdentifier) ID() string { - return c.id -} - -func IndexToIdentifier(index search.Index) content.Identifier { - return &contentIdentifier{ - contentType: index.Namespace(), - id: index.ID(), - } -} - -func ConvertToIdentifiers(indices []search.Index) []content.Identifier { - var ids []content.Identifier - for _, i := range indices { - ids = append(ids, IndexToIdentifier(i)) - } - return ids -} diff --git a/internal/interfaces/api/handler/handler.go b/internal/interfaces/api/handler/handler.go index ad9bfb4..415095a 100644 --- a/internal/interfaces/api/handler/handler.go +++ b/internal/interfaces/api/handler/handler.go @@ -2,6 +2,7 @@ package handler import ( "github.com/gohugonet/hugoverse/internal/application" + "github.com/gohugonet/hugoverse/internal/domain/content/entity" "github.com/gohugonet/hugoverse/internal/interfaces/api/admin" "github.com/gohugonet/hugoverse/internal/interfaces/api/auth" "github.com/gohugonet/hugoverse/internal/interfaces/api/database" @@ -15,7 +16,7 @@ type Handler struct { uploadDir string db *database.Database - contentApp *application.ContentServer + contentApp *entity.Content adminApp *application.AdminServer adminView *admin.View @@ -23,7 +24,7 @@ type Handler struct { } func New(log log.Logger, uploadDir string, db *database.Database, - contentApp *application.ContentServer, adminApp *application.AdminServer) *Handler { + contentApp *entity.Content, adminApp *application.AdminServer) *Handler { adminView := admin.NewView(adminApp.Name(), contentApp.AllContentTypes()) diff --git a/internal/interfaces/api/handler/handlesearch.go b/internal/interfaces/api/handler/handlesearch.go index b9bee08..27c9577 100644 --- a/internal/interfaces/api/handler/handlesearch.go +++ b/internal/interfaces/api/handler/handlesearch.go @@ -3,6 +3,7 @@ package handler import ( "bytes" "encoding/json" + "errors" "github.com/gohugonet/hugoverse/internal/interfaces/api/search" "github.com/gohugonet/hugoverse/pkg/db" "github.com/gohugonet/hugoverse/pkg/editor" @@ -68,8 +69,8 @@ func (s *Handler) SearchContentHandler(res http.ResponseWriter, req *http.Reques } // execute search for query provided, if no index for type send 404 - indices, err := search.TypeQuery(t, q, count, offset) - if err == search.ErrNoIndex { + indices, err := s.contentApp.Search.TypeQuery(t, q, count, offset) + if errors.Is(err, search.ErrNoIndex) { s.log.Errorf("Index for type %s not found", t) res.WriteHeader(http.StatusNotFound) return @@ -81,7 +82,7 @@ func (s *Handler) SearchContentHandler(res http.ResponseWriter, req *http.Reques } // respond with json formatted results - bb, err := s.contentApp.GetContents(ConvertToIdentifiers(indices)) + bb, err := s.contentApp.GetContents(indices) if err != nil { s.log.Errorf("Error getting content: %v", err) res.WriteHeader(http.StatusInternalServerError) diff --git a/internal/interfaces/api/handlers.go b/internal/interfaces/api/handlers.go index d7e1670..c0da330 100644 --- a/internal/interfaces/api/handlers.go +++ b/internal/interfaces/api/handlers.go @@ -1,6 +1,7 @@ package api import ( + "github.com/gohugonet/hugoverse/internal/application" "net/http" ) @@ -43,7 +44,7 @@ func (s *Server) registerAdminHandler() { http.StripPrefix("/admin/static", http.FileServer(restrict(http.Dir(staticDir)))))) - uploadsDir := uploadDir() + uploadsDir := application.UploadDir() s.mux.Handle("/api/uploads/", s.record.Collect(s.cors.Handle(s.cache.Control( http.StripPrefix("/api/uploads/", http.FileServer(restrict(http.Dir(uploadsDir)))))))) diff --git a/internal/interfaces/api/path.go b/internal/interfaces/api/path.go index 4bd4132..8c78724 100644 --- a/internal/interfaces/api/path.go +++ b/internal/interfaces/api/path.go @@ -6,22 +6,6 @@ import ( "path/filepath" ) -func tlsDir() string { - tlsDir := os.Getenv("HUGOVERSE_TLS_DIR") - if tlsDir == "" { - tlsDir = filepath.Join(dataDir(), "tls") - } - return tlsDir -} - -func dataDir() string { - dataDir := os.Getenv("HUGOVERSE_DATA_DIR") - if dataDir == "" { - return getWd() - } - return dataDir -} - func adminStaticDir() string { staticDir := os.Getenv("HUGOVERSE_ADMIN_STATIC_DIR") if staticDir == "" { @@ -30,14 +14,6 @@ func adminStaticDir() string { return staticDir } -func uploadDir() string { - uploadDir := os.Getenv("HUGOVERSE_UPLOAD_DIR") - if uploadDir == "" { - uploadDir = filepath.Join(dataDir(), "uploads") - } - return uploadDir -} - func getWd() string { wd, err := os.Getwd() if err != nil { @@ -45,11 +21,3 @@ func getWd() string { } return wd } - -func searchDir() string { - searchDir := os.Getenv("HUGOVERSE_SEARCH_DIR") - if searchDir == "" { - searchDir = filepath.Join(dataDir(), "search") - } - return searchDir -} diff --git a/internal/interfaces/api/server.go b/internal/interfaces/api/server.go index 548c18e..ef335ca 100644 --- a/internal/interfaces/api/server.go +++ b/internal/interfaces/api/server.go @@ -10,7 +10,6 @@ import ( "github.com/gohugonet/hugoverse/internal/interfaces/api/database" "github.com/gohugonet/hugoverse/internal/interfaces/api/handler" "github.com/gohugonet/hugoverse/internal/interfaces/api/record" - "github.com/gohugonet/hugoverse/internal/interfaces/api/search" "github.com/gohugonet/hugoverse/internal/interfaces/api/tls" "github.com/gohugonet/hugoverse/pkg/log" "net/http" @@ -64,8 +63,8 @@ func NewServer(options ...func(s *Server) error) (*Server, error) { HttpsPort: 443, DevHttpsPort: 10443, - db: database.New(dataDir()), - record: record.New(dataDir()), + db: database.New(application.DataDir()), + record: record.New(application.DataDir()), auth: &auth.Auth{}, } for _, o := range options { @@ -92,11 +91,10 @@ func NewServer(options ...func(s *Server) error) (*Server, error) { s.cors = cors.New(s.Log, s.adminApp, s.cache) s.record.Start() - search.Setup(contentApp.AllContentTypes(), searchDir()) - s.tls = tls.NewTls(s, s.adminApp, tlsDir()) + s.tls = tls.NewTls(s, s.adminApp, application.TLSDir()) - s.handler = handler.New(s.Log, uploadDir(), s.db, contentApp, s.adminApp) + s.handler = handler.New(s.Log, application.UploadDir(), s.db, contentApp, s.adminApp) s.registerHandler() diff --git a/internal/interfaces/cli/load.go b/internal/interfaces/cli/load.go new file mode 100644 index 0000000..442cb77 --- /dev/null +++ b/internal/interfaces/cli/load.go @@ -0,0 +1,41 @@ +package cli + +import ( + "flag" + "github.com/gohugonet/hugoverse/internal/application" + "github.com/gohugonet/hugoverse/pkg/log" +) + +type loadCmd struct { + parent *flag.FlagSet + cmd *flag.FlagSet +} + +func NewLoadCmd(parent *flag.FlagSet) (*loadCmd, error) { + nCmd := &loadCmd{ + parent: parent, + } + + nCmd.cmd = flag.NewFlagSet("build", flag.ExitOnError) + err := nCmd.cmd.Parse(parent.Args()[1:]) + if err != nil { + return nil, err + } + + return nCmd, nil +} + +func (oc *loadCmd) Usage() { + oc.cmd.Usage() +} + +func (oc *loadCmd) Run() error { + l := log.NewStdLogger() + + if err := application.LoadHugoProject(); err != nil { + l.Fatalf("failed to generate static sites: %v", err) + return err + } + + return nil +} diff --git a/pkg/db/content.go b/pkg/db/content.go index b326350..655eb15 100644 --- a/pkg/db/content.go +++ b/pkg/db/content.go @@ -1,29 +1,38 @@ package db import ( + "fmt" bolt "go.etcd.io/bbolt" + "log" ) // ContentAll retrives all items from the database within the provided namespace func ContentAll(namespace string) [][]byte { var posts [][]byte - store.View(func(tx *bolt.Tx) error { + + if err := store.View(func(tx *bolt.Tx) error { b := tx.Bucket([]byte(namespace)) if b == nil { + fmt.Println("Bucket not found", namespace) return bolt.ErrBucketNotFound } numKeys := b.Stats().KeyN posts = make([][]byte, 0, numKeys) - b.ForEach(func(k, v []byte) error { + if err := b.ForEach(func(k, v []byte) error { posts = append(posts, v) return nil - }) + }); err != nil { + log.Println("Error reading from db", namespace, err) + return err + } return nil - }) + }); err != nil { + log.Println("Error reading from db", namespace, err) + } return posts } diff --git a/pkg/db/start.go b/pkg/db/start.go index 89e8143..e2d115f 100644 --- a/pkg/db/start.go +++ b/pkg/db/start.go @@ -29,7 +29,7 @@ func Start(dataDir string, contentTypes []string) { systemDb := filepath.Join(dataDir, "system.db") store, err = bolt.Open(systemDb, 0666, nil) if err != nil { - log.Fatalln(err) + log.Fatalln("Couldn't open db.", err) } err = store.Update(func(tx *bolt.Tx) error { diff --git a/pkg/testkit/post.go b/pkg/testkit/post.go index 162adff..a0d12dd 100644 --- a/pkg/testkit/post.go +++ b/pkg/testkit/post.go @@ -1,5 +1,93 @@ package testkit +const homeContent = ` +--- +title: Home Page Chinese +--- + +## Hugo源码阅读 + +你好，欢迎来到 Hugo 的源码阅读 + +` + +const homeContentEn = ` +--- +title: Home Page English +--- + +## Hugo Source Code Reading + +Welcome to the Hugo source code reading series. + +` + +const docsContent = ` +--- +title: Docs Section _index Chinese +--- + +## Hugo docs 源码阅读 + +你好，欢迎来到 Hugo 的源码阅读 + +` + +const docsContentEn = ` +--- +title: Docs Section _index English +--- + +## Hugo docs Source Code Reading + +Welcome to the Hugo source code reading series. + +` + +const docsApiContent = ` +--- +title: Docs Section api Chinese +--- + +## Hugo docs api 源码阅读 + +你好，欢迎来到 Hugo 的源码阅读 + +` + +const docsApiContentEn = ` +--- +title: Docs Section api English +--- + +## Hugo docs api Source Code Reading + +Welcome to the Hugo source code reading series. + +` + +const docsApiBookContent = ` +--- +title: Docs Section api book Chinese +--- + +## Hugo docs api book 源码阅读 + +你好，欢迎来到 Hugo 的源码阅读 + +` + +const docsApiBookContentEn = ` +--- +title: Docs Section api book English +--- + +## Hugo api book Source Code Reading + +Welcome to the Hugo source code reading series. + +` + const post1Content = ` --- title: Home Introduction diff --git a/pkg/testkit/site.go b/pkg/testkit/site.go index 9a512dc..5a634e5 100644 --- a/pkg/testkit/site.go +++ b/pkg/testkit/site.go @@ -48,3 +48,39 @@ func MkTestSite() (string, func(), error) { prepareFS(tempDir, files) return tempDir, clean, nil } + +func MkBookSite() (string, func(), error) { + tempDir, clean, err := MkTestTempDir(testOs, "go-hugoverse-temp-dir") + if err != nil { + return "", clean, err + } + + files := fmt.Sprintf(` +-- config.toml -- +%s +-- go.mod -- +%s +-- content/_index.md -- +%s +-- content/docs/_index.md -- +%s +-- content/docs/api/_index.md -- +%s +-- content/docs/api/book/index.md -- +%s +-- content.en/_index.md -- +%s +-- content.en/docs/_index.md -- +%s +-- content.en/docs/api/_index.md -- +%s +-- content.en/docs/api/book/index.md -- +%s +`, configContent, goModContent, + homeContent, docsContent, docsApiContent, docsApiBookContent, + homeContentEn, docsContentEn, docsApiContentEn, docsApiBookContentEn, + ) + + prepareFS(tempDir, files) + return tempDir, clean, nil +}