diff --git a/README.MD b/README.MD index 7a9ea4e6..d7e80bf0 100644 --- a/README.MD +++ b/README.MD @@ -152,6 +152,7 @@ dashboard: pwa-description: Go-Openbmclapi Internal Dashboard # 子存储节点列表 +# 注意: measure 测量请求总是以第一个存储为准 storages: # local 为本地存储 - type: local diff --git a/cluster.go b/cluster.go index 66cf12cc..e8430d33 100644 --- a/cluster.go +++ b/cluster.go @@ -160,7 +160,7 @@ func NewCluster( func (cr *Cluster) Init(ctx context.Context) error { // Init storaged for _, s := range cr.storages { - s.Init(ctx) + s.Init(ctx, cr) } // read old stats if err := cr.stats.Load(cr.dataDir); err != nil { diff --git a/cmd_webdav.go b/cmd_webdav.go index aa1e4107..12b6a686 100644 --- a/cmd_webdav.go +++ b/cmd_webdav.go @@ -61,7 +61,7 @@ func cmdUploadWebdav(args []string) { var local LocalStorage local.SetOptions(localOpt) - if err := local.Init(ctx); err != nil { + if err := local.Init(ctx, nil); err != nil { logErrorf("Cannot initialize %s: %v", local.String(), err) os.Exit(1) } @@ -71,7 +71,7 @@ func cmdUploadWebdav(args []string) { for i, opt := range webdavOpts { s := new(WebDavStorage) s.SetOptions(opt) - if err := s.Init(ctx); err != nil { + if err := s.Init(ctx, nil); err != nil { logErrorf("Cannot initialize %s: %v", s.String(), err) os.Exit(1) } diff --git a/handler.go b/handler.go index c8dc8689..3b6e9611 100644 --- a/handler.go +++ b/handler.go @@ -312,7 +312,11 @@ func (cr *Cluster) handleDownload(rw http.ResponseWriter, req *http.Request, has http.Error(rw, "404 Status Not Found", http.StatusNotFound) return } - http.Error(rw, err.Error(), http.StatusInternalServerError) + if _, ok := err.(*HTTPStatusError); ok { + http.Error(rw, err.Error(), http.StatusBadGateway) + } else { + http.Error(rw, err.Error(), http.StatusInternalServerError) + } return } logDebug("[handler]: download served successed") diff --git a/storage.go b/storage.go index c786a592..89e4811c 100644 --- a/storage.go +++ b/storage.go @@ -39,7 +39,7 @@ type Storage interface { // SetOptions will be called with the same type of the Options() result SetOptions(any) // Init will be called before start to use a storage - Init(context.Context) error + Init(context.Context, *Cluster) error Size(hash string) (int64, error) Open(hash string) (io.ReadCloser, error) diff --git a/storage_local.go b/storage_local.go index 34d1d3de..1fbc4dba 100644 --- a/storage_local.go +++ b/storage_local.go @@ -67,7 +67,7 @@ func (s *LocalStorage) SetOptions(newOpts any) { s.opt = *(newOpts.(*LocalStorageOption)) } -func (s *LocalStorage) Init(context.Context) (err error) { +func (s *LocalStorage) Init(context.Context, *Cluster) (err error) { tmpDir := s.opt.TmpPath() os.RemoveAll(tmpDir) // should be 0755 here because Windows permission issue diff --git a/storage_mount.go b/storage_mount.go index ba0416eb..ffcc1a50 100644 --- a/storage_mount.go +++ b/storage_mount.go @@ -83,7 +83,7 @@ var checkerClient = &http.Client{ Timeout: time.Minute, } -func (s *MountStorage) Init(ctx context.Context) (err error) { +func (s *MountStorage) Init(ctx context.Context, _ *Cluster) (err error) { logInfof("Initalizing mounted folder %s", s.opt.Path) if err = initCache(s.opt.CachePath()); err != nil { return diff --git a/storage_webdav.go b/storage_webdav.go index ed8336cb..bc1cde46 100644 --- a/storage_webdav.go +++ b/storage_webdav.go @@ -38,12 +38,14 @@ import ( ) type WebDavStorageOption struct { - PreGenMeasures bool `yaml:"pre-gen-measures"` + PreGenMeasures bool `yaml:"pre-gen-measures"` + FollowRedirect bool `yaml:"follow-redirect"` + RedirectLinkCache YAMLDuration `yaml:"redirect-link-cache"` - Alias string `yaml:"alias,omitempty"` - aliasUser *WebDavUser + Alias string `yaml:"alias,omitempty"` + WebDavUser `yaml:",inline,omitempty"` - WebDavUser `yaml:",inline,omitempty"` + aliasUser *WebDavUser fullEndPoint string } @@ -95,7 +97,8 @@ func (o *WebDavStorageOption) GetPassword() string { type WebDavStorage struct { opt WebDavStorageOption - cli *gowebdav.Client + cache Cache + cli *gowebdav.Client } var _ Storage = (*WebDavStorage)(nil) @@ -124,7 +127,7 @@ func webdavIsHTTPError(err error, code int) bool { return strings.Contains(err.Error(), expect) } -func (s *WebDavStorage) Init(ctx context.Context) (err error) { +func (s *WebDavStorage) Init(ctx context.Context, cluster *Cluster) (err error) { if alias := s.opt.Alias; alias != "" { user, ok := config.WebdavUsers[alias] if !ok { @@ -149,6 +152,12 @@ func (s *WebDavStorage) Init(ctx context.Context) (err error) { s.opt.fullEndPoint = s.opt.EndPoint } + if cluster == nil { + s.cache = NoCache + } else { + s.cache = NewCacheWithNamespace(cluster.cache, fmt.Sprintf("redirect-cache@%s;%s@", s.opt.GetUsername(), s.opt.GetEndPoint())) + } + s.cli = gowebdav.NewClient(s.opt.GetEndPoint(), s.opt.GetUsername(), s.opt.GetPassword()) s.cli.SetHeader("User-Agent", ClusterUserAgentFull) @@ -258,6 +267,26 @@ func copyHeader(key string, dst, src http.Header) { } func (s *WebDavStorage) ServeDownload(rw http.ResponseWriter, req *http.Request, hash string, size int64) (int64, error) { + if s.opt.RedirectLinkCache > 0 { + if location, ok := s.cache.Get(hash); ok { + // fix the size for Ranged request + rgs, err := gosrc.ParseRange(req.Header.Get("Range"), size) + if err == nil && len(rgs) > 0 { + var newSize int64 = 0 + for _, r := range rgs { + newSize += r.Length + } + if newSize < size { + size = newSize + } + } + rw.Header().Set("Location", location) + rw.Header().Set("Cache-Control", fmt.Sprintf("public,max-age=%d", (int64)(s.opt.RedirectLinkCache.Dur().Seconds()))) + rw.WriteHeader(http.StatusFound) + return size, nil + } + } + target, err := url.JoinPath(s.opt.GetEndPoint(), s.hashToPath(hash)) if err != nil { return 0, err @@ -296,9 +325,14 @@ func (s *WebDavStorage) ServeDownload(rw http.ResponseWriter, req *http.Request, size = newSize } } - copyHeader("Location", rwh, resp.Header) + location := resp.Header.Get("Location") + rwh.Set("Location", location) copyHeader("ETag", rwh, resp.Header) copyHeader("Last-Modified", rwh, resp.Header) + if s.opt.RedirectLinkCache > 0 { + rwh.Set("Cache-Control", fmt.Sprintf("public,max-age=%d", (int64)(s.opt.RedirectLinkCache.Dur().Seconds()))) + s.cache.Set(hash, location, CacheOpt{Expiration: s.opt.RedirectLinkCache.Dur()}) + } rw.WriteHeader(resp.StatusCode) return size, nil case 2: diff --git a/util.go b/util.go index e4eda8db..90528002 100644 --- a/util.go +++ b/util.go @@ -409,7 +409,7 @@ var ( _ yaml.Unmarshaler = (*RawYAML)(nil) ) -func (r RawYAML) MarshalYAML() (interface{}, error) { +func (r RawYAML) MarshalYAML() (any, error) { return r.Node, nil } @@ -418,6 +418,29 @@ func (r *RawYAML) UnmarshalYAML(n *yaml.Node) (err error) { return nil } +type YAMLDuration time.Duration + +func (d YAMLDuration) Dur() time.Duration { + return (time.Duration)(d) +} + +func (d YAMLDuration) MarshalYAML() (any, error) { + return (time.Duration)(d).String(), nil +} + +func (d *YAMLDuration) UnmarshalYAML(n *yaml.Node) (err error) { + var v string + if err = n.Decode(v); err != nil { + return + } + var td time.Duration + if td, err = time.ParseDuration(v); err != nil { + return + } + *d = (YAMLDuration)(td) + return nil +} + type slotInfo struct { id int buf []byte