add ishex

cluster.go

@@ -708,7 +708,7 @@ func (cr *Cluster) checkFileFor(
 		slots = NewBufSlots(runtime.GOMAXPROCS(0) * 2)
 	}
 
-	bar := pg.AddBar((int64)(len(files)),
+	bar := pg.AddBar((int64)(len(files))+0x100,
 		mpb.BarRemoveOnComplete(),
 		mpb.PrependDecorators(
 			decor.Name("> Checking "+storage.String()),
@@ -740,10 +740,20 @@ func (cr *Cluster) checkFileFor(
 	defer bar.Abort(true)
 
 	sizeMap := make(map[string]int64, len(files))
-	storage.WalkDir(func(hash string, size int64) error {
-		sizeMap[hash] = size
-		return nil
-	})
+	{
+		start := time.Now()
+		var checkedMp [256]bool
+		storage.WalkDir(func(hash string, size int64) error {
+			if n := HexTo256(hash); !checkedMp[n] {
+				checkedMp[n] = true
+				now := time.Now()
+				bar.EwmaIncrement(now.Sub(start))
+				start = now
+			}
+			sizeMap[hash] = size
+			return nil
+		})
+	}
 
 	for _, f := range files {
 		if ctx.Err() != nil {

handler.go

@@ -204,7 +204,7 @@ func (cr *Cluster) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 		}
 
 		hash := rawpath[len("/download/"):]
-		if len(hash) < 4 {
+		if !IsHex(hash) {
 			http.Error(rw, "404 Not Found", http.StatusNotFound)
 			return
 		}

ishex_test.go

@@ -0,0 +1,53 @@
+
+/**
+ * OpenBmclAPI (Golang Edition)
+ * Copyright (C) 2023 Kevin Z <[email protected]>
+ * All rights reserved
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as published
+ *  by the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package main
+
+import (
+	"testing"
+)
+
+func TestIsHex(t *testing.T){
+	var data = []struct{
+		S string
+		B bool
+	}{
+		{"", false},
+		{"0", false},
+		{"00", true},
+		{"000", false},
+		{"0f", true},
+		{"f0", true},
+		{"af", true},
+		{"fa", true},
+		{"ff", true},
+		{"aa", true},
+		{"11", true},
+		{"fg", false},
+		{"58fe4669c65dbde8e0d58afb83e21620", true},
+		{"1cad5d7f9ed285a04784429ab2a4615d5c59ea88", true},
+	}
+	for _, d := range data {
+		ok := IsHex(d.S)
+		if ok != d.B {
+			t.Errorf("IsHex(%q) returned %v, but expected %v", d.S, ok, d.B)
+		}
+	}
+}

storage_local.go

@@ -80,6 +80,20 @@ func (s *LocalStorage) Init(context.Context) (err error) {
 	return
 }
 
+func initCache(base string) (err error) {
+	if err = os.MkdirAll(base, 0755); err != nil && !errors.Is(err, os.ErrExist) {
+		return
+	}
+	var b [1]byte
+	for i := 0; i < 0x100; i++ {
+		b[0] = (byte)(i)
+		if err = os.Mkdir(filepath.Join(base, hex.EncodeToString(b[:])), 0755); err != nil && !errors.Is(err, os.ErrExist) {
+			return
+		}
+	}
+	return nil
+}
+
 func (s *LocalStorage) hashToPath(hash string) string {
 	return filepath.Join(s.opt.CachePath, hash[0:2], hash)
 }
@@ -222,16 +236,6 @@ func (s *LocalStorage) ServeMeasure(rw http.ResponseWriter, req *http.Request, s
 	return nil
 }
 
-var hex256 = func() (hex256 []string) {
-	hex256 = make([]string, 0x100)
-	var b [1]byte
-	for i := 0; i < 0x100; i++ {
-		b[0] = (byte)(i)
-		hex256[i] = hex.EncodeToString(b[:])
-	}
-	return
-}()
-
 func walkCacheDir(cacheDir string, walker func(hash string, size int64) (err error)) (err error) {
 	for _, dir := range hex256 {
 		files, err := os.ReadDir(filepath.Join(cacheDir, dir))

util.go

@@ -24,7 +24,6 @@ import (
 	"crypto"
 	"crypto/x509"
 	"encoding/base64"
-	"encoding/hex"
 	"encoding/pem"
 	"errors"
 	"fmt"
@@ -33,7 +32,6 @@ import (
 	"net/http"
 	"net/url"
 	"os"
-	"path/filepath"
 	"strconv"
 	"strings"
 	"sync"
@@ -326,20 +324,6 @@ func checkQuerySign(hash string, secret string, query url.Values) bool {
 	return time.Now().UnixMilli() < before
 }
 
-func initCache(base string) (err error) {
-	if err = os.MkdirAll(base, 0755); err != nil && !errors.Is(err, os.ErrExist) {
-		return
-	}
-	var b [1]byte
-	for i := 0; i < 0x100; i++ {
-		b[0] = (byte)(i)
-		if err = os.Mkdir(filepath.Join(base, hex.EncodeToString(b[:])), 0755); err != nil && !errors.Is(err, os.ErrExist) {
-			return
-		}
-	}
-	return nil
-}
-
 type SyncMap[K comparable, V any] struct {
 	l sync.RWMutex
 	m map[K]V
@@ -589,3 +573,35 @@ func (s *Semaphore) ProxyReader(r io.Reader) io.ReadCloser {
 		s:      s,
 	}
 }
+
+const numToHexMap = "0123456789abcdef"
+
+var hex256 = func() (hex256 []string) {
+	hex256 = make([]string, 0x100)
+	for i := 0; i < 0x100; i++ {
+		a, b := i>>4, i&0xf
+		hex256[i] = numToHexMap[a:a+1] + numToHexMap[b:b+1]
+	}
+	return
+}()
+
+var hexToNumMap = [256]int{
+	'0': 0x0, '1': 0x1, '2': 0x2, '3': 0x3, '4': 0x4, '5': 0x5, '6': 0x6, '7': 0x7, '8': 0x8, '9': 0x9,
+	'a': 0xa, 'b': 0xb, 'c': 0xc, 'd': 0xd, 'e': 0xe, 'f': 0xf,
+}
+
+func IsHex(s string) bool {
+	if len(s) < 2 || len(s)%2 != 0 {
+		return false
+	}
+	for i := 0; i < len(s); i++ {
+		if s[i] != '0' && hexToNumMap[s[i]] == 0 {
+			return false
+		}
+	}
+	return true
+}
+
+func HexTo256(s string) (n int) {
+	return hexToNumMap[s[0]]*0x10 + hexToNumMap[s[1]]
+}