add cache for webdav redirect url

README.MD

@@ -152,6 +152,7 @@ dashboard:
     pwa-description: Go-Openbmclapi Internal Dashboard
 
 # 子存储节点列表
+# 注意: measure 测量请求总是以第一个存储为准
 storages:
     # local 为本地存储
     - type: local

cluster.go

@@ -160,7 +160,7 @@ func NewCluster(
 func (cr *Cluster) Init(ctx context.Context) error {
 	// Init storaged
 	for _, s := range cr.storages {
-		s.Init(ctx)
+		s.Init(ctx, cr)
 	}
 	// read old stats
 	if err := cr.stats.Load(cr.dataDir); err != nil {

cmd_webdav.go

@@ -61,7 +61,7 @@ func cmdUploadWebdav(args []string) {
 
 	var local LocalStorage
 	local.SetOptions(localOpt)
-	if err := local.Init(ctx); err != nil {
+	if err := local.Init(ctx, nil); err != nil {
 		logErrorf("Cannot initialize %s: %v", local.String(), err)
 		os.Exit(1)
 	}
@@ -71,7 +71,7 @@ func cmdUploadWebdav(args []string) {
 	for i, opt := range webdavOpts {
 		s := new(WebDavStorage)
 		s.SetOptions(opt)
-		if err := s.Init(ctx); err != nil {
+		if err := s.Init(ctx, nil); err != nil {
 			logErrorf("Cannot initialize %s: %v", s.String(), err)
 			os.Exit(1)
 		}

handler.go

@@ -312,7 +312,11 @@ func (cr *Cluster) handleDownload(rw http.ResponseWriter, req *http.Request, has
 			http.Error(rw, "404 Status Not Found", http.StatusNotFound)
 			return
 		}
-		http.Error(rw, err.Error(), http.StatusInternalServerError)
+		if _, ok := err.(*HTTPStatusError); ok {
+			http.Error(rw, err.Error(), http.StatusBadGateway)
+		} else {
+			http.Error(rw, err.Error(), http.StatusInternalServerError)
+		}
 		return
 	}
 	logDebug("[handler]: download served successed")

storage.go

@@ -39,7 +39,7 @@ type Storage interface {
 	// SetOptions will be called with the same type of the Options() result
 	SetOptions(any)
 	// Init will be called before start to use a storage
-	Init(context.Context) error
+	Init(context.Context, *Cluster) error
 
 	Size(hash string) (int64, error)
 	Open(hash string) (io.ReadCloser, error)

storage_local.go

@@ -67,7 +67,7 @@ func (s *LocalStorage) SetOptions(newOpts any) {
 	s.opt = *(newOpts.(*LocalStorageOption))
 }
 
-func (s *LocalStorage) Init(context.Context) (err error) {
+func (s *LocalStorage) Init(context.Context, *Cluster) (err error) {
 	tmpDir := s.opt.TmpPath()
 	os.RemoveAll(tmpDir)
 	// should be 0755 here because Windows permission issue

storage_mount.go

@@ -83,7 +83,7 @@ var checkerClient = &http.Client{
 	Timeout: time.Minute,
 }
 
-func (s *MountStorage) Init(ctx context.Context) (err error) {
+func (s *MountStorage) Init(ctx context.Context, _ *Cluster) (err error) {
 	logInfof("Initalizing mounted folder %s", s.opt.Path)
 	if err = initCache(s.opt.CachePath()); err != nil {
 		return

storage_webdav.go

@@ -38,12 +38,14 @@ import (
 )
 
 type WebDavStorageOption struct {
-	PreGenMeasures bool `yaml:"pre-gen-measures"`
+	PreGenMeasures    bool         `yaml:"pre-gen-measures"`
+	FollowRedirect    bool         `yaml:"follow-redirect"`
+	RedirectLinkCache YAMLDuration `yaml:"redirect-link-cache"`
 
-	Alias     string `yaml:"alias,omitempty"`
-	aliasUser *WebDavUser
+	Alias      string `yaml:"alias,omitempty"`
+	WebDavUser `yaml:",inline,omitempty"`
 
-	WebDavUser   `yaml:",inline,omitempty"`
+	aliasUser    *WebDavUser
 	fullEndPoint string
 }
 
@@ -95,7 +97,8 @@ func (o *WebDavStorageOption) GetPassword() string {
 type WebDavStorage struct {
 	opt WebDavStorageOption
 
-	cli *gowebdav.Client
+	cache Cache
+	cli   *gowebdav.Client
 }
 
 var _ Storage = (*WebDavStorage)(nil)
@@ -124,7 +127,7 @@ func webdavIsHTTPError(err error, code int) bool {
 	return strings.Contains(err.Error(), expect)
 }
 
-func (s *WebDavStorage) Init(ctx context.Context) (err error) {
+func (s *WebDavStorage) Init(ctx context.Context, cluster *Cluster) (err error) {
 	if alias := s.opt.Alias; alias != "" {
 		user, ok := config.WebdavUsers[alias]
 		if !ok {
@@ -149,6 +152,12 @@ func (s *WebDavStorage) Init(ctx context.Context) (err error) {
 		s.opt.fullEndPoint = s.opt.EndPoint
 	}
 
+	if cluster == nil {
+		s.cache = NoCache
+	} else {
+		s.cache = NewCacheWithNamespace(cluster.cache, fmt.Sprintf("redirect-cache@%s;%s@", s.opt.GetUsername(), s.opt.GetEndPoint()))
+	}
+
 	s.cli = gowebdav.NewClient(s.opt.GetEndPoint(), s.opt.GetUsername(), s.opt.GetPassword())
 	s.cli.SetHeader("User-Agent", ClusterUserAgentFull)
 
@@ -258,6 +267,26 @@ func copyHeader(key string, dst, src http.Header) {
 }
 
 func (s *WebDavStorage) ServeDownload(rw http.ResponseWriter, req *http.Request, hash string, size int64) (int64, error) {
+	if s.opt.RedirectLinkCache > 0 {
+		if location, ok := s.cache.Get(hash); ok {
+			// fix the size for Ranged request
+			rgs, err := gosrc.ParseRange(req.Header.Get("Range"), size)
+			if err == nil && len(rgs) > 0 {
+				var newSize int64 = 0
+				for _, r := range rgs {
+					newSize += r.Length
+				}
+				if newSize < size {
+					size = newSize
+				}
+			}
+			rw.Header().Set("Location", location)
+			rw.Header().Set("Cache-Control", fmt.Sprintf("public,max-age=%d", (int64)(s.opt.RedirectLinkCache.Dur().Seconds())))
+			rw.WriteHeader(http.StatusFound)
+			return size, nil
+		}
+	}
+
 	target, err := url.JoinPath(s.opt.GetEndPoint(), s.hashToPath(hash))
 	if err != nil {
 		return 0, err
@@ -296,9 +325,14 @@ func (s *WebDavStorage) ServeDownload(rw http.ResponseWriter, req *http.Request,
 				size = newSize
 			}
 		}
-		copyHeader("Location", rwh, resp.Header)
+		location := resp.Header.Get("Location")
+		rwh.Set("Location", location)
 		copyHeader("ETag", rwh, resp.Header)
 		copyHeader("Last-Modified", rwh, resp.Header)
+		if s.opt.RedirectLinkCache > 0 {
+			rwh.Set("Cache-Control", fmt.Sprintf("public,max-age=%d", (int64)(s.opt.RedirectLinkCache.Dur().Seconds())))
+			s.cache.Set(hash, location, CacheOpt{Expiration: s.opt.RedirectLinkCache.Dur()})
+		}
 		rw.WriteHeader(resp.StatusCode)
 		return size, nil
 	case 2:

util.go

@@ -409,7 +409,7 @@ var (
 	_ yaml.Unmarshaler = (*RawYAML)(nil)
 )
 
-func (r RawYAML) MarshalYAML() (interface{}, error) {
+func (r RawYAML) MarshalYAML() (any, error) {
 	return r.Node, nil
 }
 
@@ -418,6 +418,29 @@ func (r *RawYAML) UnmarshalYAML(n *yaml.Node) (err error) {
 	return nil
 }
 
+type YAMLDuration time.Duration
+
+func (d YAMLDuration) Dur() time.Duration {
+	return (time.Duration)(d)
+}
+
+func (d YAMLDuration) MarshalYAML() (any, error) {
+	return (time.Duration)(d).String(), nil
+}
+
+func (d *YAMLDuration) UnmarshalYAML(n *yaml.Node) (err error) {
+	var v string
+	if err = n.Decode(v); err != nil {
+		return
+	}
+	var td time.Duration
+	if td, err = time.ParseDuration(v); err != nil {
+		return
+	}
+	*d = (YAMLDuration)(td)
+	return nil
+}
+
 type slotInfo struct {
 	id  int
 	buf []byte