From 9bca1f7a0669aee35c7b0a89016602970ca87fb9 Mon Sep 17 00:00:00 2001 From: Valentin Kuznetsov Date: Mon, 9 May 2022 13:28:49 -0400 Subject: [PATCH] convert utils functions to generics; re-factor usage of sets --- dbs/bulkblocks.go | 2 +- dbs/bulkblocks2.go | 2 +- dbs/fileparents.go | 2 +- dbs/migrate.go | 4 +-- go.mod | 1 + go.sum | 2 ++ test/bench_test.go | 39 +++++++++++++++++++++ test/main.go | 2 +- test/migrate_test.go | 2 +- test/utils_test.go | 15 +++++++- utils/utils.go | 82 +++++++++++++++----------------------------- 11 files changed, 90 insertions(+), 63 deletions(-) diff --git a/dbs/bulkblocks.go b/dbs/bulkblocks.go index bc6ec628..448e4330 100644 --- a/dbs/bulkblocks.go +++ b/dbs/bulkblocks.go @@ -673,7 +673,7 @@ func (a *API) InsertBulkBlocks() error { for _, d := range rec.DsParentList { datasetParentList = append(datasetParentList, d.ParentDataset) } - datasetParentList = utils.List2Set(datasetParentList) + datasetParentList = utils.Set(datasetParentList) for _, ds := range datasetParentList { // get file id for parent dataset pid, err := GetID(tx, "DATASETS", "dataset_id", "dataset", ds) diff --git a/dbs/bulkblocks2.go b/dbs/bulkblocks2.go index f778e341..91a62c44 100644 --- a/dbs/bulkblocks2.go +++ b/dbs/bulkblocks2.go @@ -834,7 +834,7 @@ func (a *API) InsertBulkBlocksConcurrently() error { for _, d := range rec.DsParentList { datasetParentList = append(datasetParentList, d.ParentDataset) } - datasetParentList = utils.List2Set(datasetParentList) + datasetParentList = utils.Set(datasetParentList) for _, ds := range datasetParentList { // get file id for parent dataset pid, err := GetID(tx, "DATASETS", "dataset_id", "dataset", ds) diff --git a/dbs/fileparents.go b/dbs/fileparents.go index b0949f38..2637bde6 100644 --- a/dbs/fileparents.go +++ b/dbs/fileparents.go @@ -342,7 +342,7 @@ func (a *API) InsertFileParentsBlockTxt(tx *sql.Tx) error { if utils.VERBOSE > 1 { log.Println("InsertFileParentsBlock fids", fids, "bfids", bfids) } - if !utils.Equal(utils.Set(fids), utils.Set(bfids)) { + if !utils.Equal(utils.OrderedSet(fids), utils.OrderedSet(bfids)) { log.Println("block fids != file ids") log.Println("block ids", bfids) log.Println("file ids", fids) diff --git a/dbs/migrate.go b/dbs/migrate.go index 59ad5272..2fb8f018 100644 --- a/dbs/migrate.go +++ b/dbs/migrate.go @@ -504,8 +504,8 @@ func startMigrationRequest(rec MigrationRequest) ([]MigrationReport, error) { // get parent blocks at source DBS instance for given input // srcParentBlocks = prepareMigrationList(localhost, input) srcParentBlocks = prepareMigrationListAtSource(localhost, dstParentBlocks) - dstParentBlocks = utils.List2Set(dstParentBlocks) - srcParentBlocks = utils.List2Set(srcParentBlocks) + dstParentBlocks = utils.Set(dstParentBlocks) + srcParentBlocks = utils.Set(srcParentBlocks) if utils.VERBOSE > 0 { log.Printf("Migration blocks from destination %s %+v", rurl, dstParentBlocks) log.Printf("Migration blocks from source %s %+v", localhost, srcParentBlocks) diff --git a/go.mod b/go.mod index a9db8229..68ec8dc1 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 // indirect + golang.org/x/exp v0.0.0-20220428152302-39d4317da171 // indirect golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12 // indirect diff --git a/go.sum b/go.sum index 4ba2bd0c..60d758b5 100644 --- a/go.sum +++ b/go.sum @@ -106,6 +106,8 @@ golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSO golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 h1:tkVvjkPTB7pnW3jnid7kNyAMPVWllTNOf/qKDze4p9o= golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20220428152302-39d4317da171 h1:TfdoLivD44QwvssI9Sv1xwa5DcL5XQr4au4sZ2F2NV4= +golang.org/x/exp v0.0.0-20220428152302-39d4317da171/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= diff --git a/test/bench_test.go b/test/bench_test.go index a3627a32..3c74ff44 100644 --- a/test/bench_test.go +++ b/test/bench_test.go @@ -1,7 +1,9 @@ package main import ( + "math/rand" "testing" + "time" "github.com/dmwm/dbs2go/dbs" "github.com/dmwm/dbs2go/utils" @@ -49,3 +51,40 @@ func BenchmarkUpdateOrderedDict(b *testing.B) { utils.UpdateOrderedDict(omap, nmap) } } + +// BenchmarkInList +func BenchmarkInList(b *testing.B) { + utils.VERBOSE = 0 + N := 1000 + list := make([]int, N) + for i := 0; i < N; i++ { + list[i] = i + } + rand.Seed(time.Now().UnixNano()) + for i := 0; i < b.N; i++ { + elem := rand.Intn(N) + res := utils.InList(elem, list) + if res != true { + b.Fatal("unable to find random element in a list") + } + } +} + +// BenchmarkEqual +func BenchmarkEqual(b *testing.B) { + utils.VERBOSE = 0 + N := 1000 + list := make([]int, N) + for i := 0; i < N; i++ { + list[i] = i + } + rand.Seed(time.Now().UnixNano()) + for i := 0; i < b.N; i++ { + set1 := utils.Set(list) + set2 := utils.Set(list) + res := utils.Equal(set1, set2) + if res != true { + b.Fatal("unable to compare sets") + } + } +} diff --git a/test/main.go b/test/main.go index 6aa8009a..27d44f9a 100644 --- a/test/main.go +++ b/test/main.go @@ -44,7 +44,7 @@ func initDB(dryRun bool) *sql.DB { log.Fatal("unable to get current working dir") } utils.STATICDIR = fmt.Sprintf("%s/../static", dir) - utils.VERBOSE = 2 + utils.VERBOSE = 1 dbtype := "sqlite3" dburi := "/tmp/dbs-test.db" dbowner := "sqlite" diff --git a/test/migrate_test.go b/test/migrate_test.go index e2556c0f..33049374 100644 --- a/test/migrate_test.go +++ b/test/migrate_test.go @@ -186,7 +186,7 @@ func TestMigrate(t *testing.T) { var sids []int64 for _, rrr := range statusRecords { sids = append(sids, rrr.MIGRATION_REQUEST_ID) - if !utils.InInt64List(rrr.MIGRATION_REQUEST_ID, rids) { + if !utils.InList(rrr.MIGRATION_REQUEST_ID, rids) { t.Errorf("unvalid status request id %d, expect %+v", rrr.MIGRATION_REQUEST_ID, rids) } } diff --git a/test/utils_test.go b/test/utils_test.go index 4ea4316f..c3792703 100644 --- a/test/utils_test.go +++ b/test/utils_test.go @@ -25,7 +25,7 @@ func TestUtilsInList(t *testing.T) { func TestUtilsSet(t *testing.T) { vals := []int64{1, 2, 3, 1} res := utils.Set(vals) - if !utils.Equal(res, utils.Set([]int64{3, 2, 1})) { + if !utils.Equal(res, utils.Set([]int64{1, 2, 3})) { t.Error("Fail TestUtilsSet") } arr := []int64{4, 5, 6} @@ -34,6 +34,19 @@ func TestUtilsSet(t *testing.T) { } } +// TestUtilsOrderedSet +func TestUtilsOrderedSet(t *testing.T) { + vals := []int64{1, 2, 3, 1} + res := utils.Set(vals) + if !utils.Equal(res, utils.OrderedSet([]int64{3, 2, 1})) { + t.Error("Fail TestUtilsSet") + } + arr := []int64{4, 5, 6} + if utils.Equal(res, arr) { + t.Error("Fail of Equal in TestUtilsOrderedSet") + } +} + // TestUtilsRecordSize func TestUtilsRecordSize(t *testing.T) { rec := make(map[string]int) diff --git a/utils/utils.go b/utils/utils.go index 4a3ecf06..f5ba6695 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -17,6 +17,8 @@ import ( "sort" "strconv" "strings" + + "golang.org/x/exp/constraints" ) // VERBOSE controls verbosity level of the package @@ -96,22 +98,13 @@ func GoDeferFunc(api string, f func()) { } } -// InInt64List checks item in a list -func InInt64List(a int64, list []int64) bool { - check := 0 - for _, b := range list { - if b == a { - check += 1 - } - } - if check != 0 { - return true - } - return false +// ListEntry identifies types used by list's generics function +type ListEntry interface { + int | int64 | float64 | string } // InList checks item in a list -func InList(a string, list []string) bool { +func InList[T ListEntry](a T, list []T) bool { check := 0 for _, b := range list { if b == a { @@ -124,33 +117,35 @@ func InList(a string, list []string) bool { return false } -// Int64List implement sort for []int type -type Int64List []int64 - -// Len provides length of the []int64 type -func (s Int64List) Len() int { return len(s) } - -// Swap implements swap function for []int64 type -func (s Int64List) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -// Less implements less function for []int64 type -func (s Int64List) Less(i, j int) bool { return s[i] < s[j] } - -// Set implementa basic set -func Set(list []int64) []int64 { - var out []int64 - for _, v := range list { - if !InInt64List(v, out) { +// Set converts input list into set +func Set[T ListEntry](arr []T) []T { + var out []T + for _, v := range arr { + if !InList(v, out) { out = append(out, v) } } - sort.Sort(Int64List(out)) + return out +} + +// sortSlice helper function on any ordered generic list +// https://gosamples.dev/generics-sort-slice/ +func sortSlice[T constraints.Ordered](s []T) { + sort.Slice(s, func(i, j int) bool { + return s[i] < s[j] + }) +} + +// OrderedSet implementa ordered set +func OrderedSet[T ListEntry](list []T) []T { + out := Set(list) + sortSlice(out) return out } // Equal tells whether a and b contain the same elements. // A nil argument is equivalent to an empty slice. -func Equal(a, b []int64) bool { +func Equal[T ListEntry](a, b []T) bool { if len(a) != len(b) { return false } @@ -180,29 +175,6 @@ func MapIntKeys(rec map[int]interface{}) []int { return keys } -// List2Set converts input list into set -func List2Set(arr []string) []string { - var out []string - for _, key := range arr { - if !InList(key, out) { - out = append(out, key) - } - } - return out -} - -// StringList implements sort for []string type -type StringList []string - -// Len provide length method of StringList -func (s StringList) Len() int { return len(s) } - -// Swap provide swap method of StringList -func (s StringList) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -// Less provide less method of StringList -func (s StringList) Less(i, j int) bool { return s[i] < s[j] } - // ListFiles lists files in a given directory func ListFiles(dir string) []string { var out []string