diff --git a/cluster.go b/cluster.go index 2a1e3827..0f00885b 100644 --- a/cluster.go +++ b/cluster.go @@ -708,7 +708,7 @@ func (cr *Cluster) checkFileFor( slots = NewBufSlots(runtime.GOMAXPROCS(0) * 2) } - bar := pg.AddBar((int64)(len(files)), + bar := pg.AddBar((int64)(len(files))+0x100, mpb.BarRemoveOnComplete(), mpb.PrependDecorators( decor.Name("> Checking "+storage.String()), @@ -740,10 +740,20 @@ func (cr *Cluster) checkFileFor( defer bar.Abort(true) sizeMap := make(map[string]int64, len(files)) - storage.WalkDir(func(hash string, size int64) error { - sizeMap[hash] = size - return nil - }) + { + start := time.Now() + var checkedMp [256]bool + storage.WalkDir(func(hash string, size int64) error { + if n := HexTo256(hash); !checkedMp[n] { + checkedMp[n] = true + now := time.Now() + bar.EwmaIncrement(now.Sub(start)) + start = now + } + sizeMap[hash] = size + return nil + }) + } for _, f := range files { if ctx.Err() != nil { diff --git a/handler.go b/handler.go index 10dcba8f..2083cef9 100644 --- a/handler.go +++ b/handler.go @@ -204,7 +204,7 @@ func (cr *Cluster) ServeHTTP(rw http.ResponseWriter, req *http.Request) { } hash := rawpath[len("/download/"):] - if len(hash) < 4 { + if !IsHex(hash) { http.Error(rw, "404 Not Found", http.StatusNotFound) return } diff --git a/ishex_test.go b/ishex_test.go new file mode 100644 index 00000000..dec94d1d --- /dev/null +++ b/ishex_test.go @@ -0,0 +1,53 @@ + +/** + * OpenBmclAPI (Golang Edition) + * Copyright (C) 2023 Kevin Z + * All rights reserved + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package main + +import ( + "testing" +) + +func TestIsHex(t *testing.T){ + var data = []struct{ + S string + B bool + }{ + {"", false}, + {"0", false}, + {"00", true}, + {"000", false}, + {"0f", true}, + {"f0", true}, + {"af", true}, + {"fa", true}, + {"ff", true}, + {"aa", true}, + {"11", true}, + {"fg", false}, + {"58fe4669c65dbde8e0d58afb83e21620", true}, + {"1cad5d7f9ed285a04784429ab2a4615d5c59ea88", true}, + } + for _, d := range data { + ok := IsHex(d.S) + if ok != d.B { + t.Errorf("IsHex(%q) returned %v, but expected %v", d.S, ok, d.B) + } + } +} diff --git a/storage_local.go b/storage_local.go index 34d1d3de..f9895fac 100644 --- a/storage_local.go +++ b/storage_local.go @@ -80,6 +80,20 @@ func (s *LocalStorage) Init(context.Context) (err error) { return } +func initCache(base string) (err error) { + if err = os.MkdirAll(base, 0755); err != nil && !errors.Is(err, os.ErrExist) { + return + } + var b [1]byte + for i := 0; i < 0x100; i++ { + b[0] = (byte)(i) + if err = os.Mkdir(filepath.Join(base, hex.EncodeToString(b[:])), 0755); err != nil && !errors.Is(err, os.ErrExist) { + return + } + } + return nil +} + func (s *LocalStorage) hashToPath(hash string) string { return filepath.Join(s.opt.CachePath, hash[0:2], hash) } @@ -222,16 +236,6 @@ func (s *LocalStorage) ServeMeasure(rw http.ResponseWriter, req *http.Request, s return nil } -var hex256 = func() (hex256 []string) { - hex256 = make([]string, 0x100) - var b [1]byte - for i := 0; i < 0x100; i++ { - b[0] = (byte)(i) - hex256[i] = hex.EncodeToString(b[:]) - } - return -}() - func walkCacheDir(cacheDir string, walker func(hash string, size int64) (err error)) (err error) { for _, dir := range hex256 { files, err := os.ReadDir(filepath.Join(cacheDir, dir)) diff --git a/util.go b/util.go index e55c3a7c..610824bf 100644 --- a/util.go +++ b/util.go @@ -24,7 +24,6 @@ import ( "crypto" "crypto/x509" "encoding/base64" - "encoding/hex" "encoding/pem" "errors" "fmt" @@ -33,7 +32,6 @@ import ( "net/http" "net/url" "os" - "path/filepath" "strconv" "strings" "sync" @@ -326,20 +324,6 @@ func checkQuerySign(hash string, secret string, query url.Values) bool { return time.Now().UnixMilli() < before } -func initCache(base string) (err error) { - if err = os.MkdirAll(base, 0755); err != nil && !errors.Is(err, os.ErrExist) { - return - } - var b [1]byte - for i := 0; i < 0x100; i++ { - b[0] = (byte)(i) - if err = os.Mkdir(filepath.Join(base, hex.EncodeToString(b[:])), 0755); err != nil && !errors.Is(err, os.ErrExist) { - return - } - } - return nil -} - type SyncMap[K comparable, V any] struct { l sync.RWMutex m map[K]V @@ -589,3 +573,35 @@ func (s *Semaphore) ProxyReader(r io.Reader) io.ReadCloser { s: s, } } + +const numToHexMap = "0123456789abcdef" + +var hex256 = func() (hex256 []string) { + hex256 = make([]string, 0x100) + for i := 0; i < 0x100; i++ { + a, b := i>>4, i&0xf + hex256[i] = numToHexMap[a:a+1] + numToHexMap[b:b+1] + } + return +}() + +var hexToNumMap = [256]int{ + '0': 0x0, '1': 0x1, '2': 0x2, '3': 0x3, '4': 0x4, '5': 0x5, '6': 0x6, '7': 0x7, '8': 0x8, '9': 0x9, + 'a': 0xa, 'b': 0xb, 'c': 0xc, 'd': 0xd, 'e': 0xe, 'f': 0xf, +} + +func IsHex(s string) bool { + if len(s) < 2 || len(s)%2 != 0 { + return false + } + for i := 0; i < len(s); i++ { + if s[i] != '0' && hexToNumMap[s[i]] == 0 { + return false + } + } + return true +} + +func HexTo256(s string) (n int) { + return hexToNumMap[s[0]]*0x10 + hexToNumMap[s[1]] +}