convert utils functions to generics; re-factor usage of sets

dbs/bulkblocks.go

@@ -673,7 +673,7 @@ func (a *API) InsertBulkBlocks() error {
 	for _, d := range rec.DsParentList {
 		datasetParentList = append(datasetParentList, d.ParentDataset)
 	}
-	datasetParentList = utils.List2Set(datasetParentList)
+	datasetParentList = utils.Set(datasetParentList)
 	for _, ds := range datasetParentList {
 		// get file id for parent dataset
 		pid, err := GetID(tx, "DATASETS", "dataset_id", "dataset", ds)

dbs/bulkblocks2.go

@@ -834,7 +834,7 @@ func (a *API) InsertBulkBlocksConcurrently() error {
 	for _, d := range rec.DsParentList {
 		datasetParentList = append(datasetParentList, d.ParentDataset)
 	}
-	datasetParentList = utils.List2Set(datasetParentList)
+	datasetParentList = utils.Set(datasetParentList)
 	for _, ds := range datasetParentList {
 		// get file id for parent dataset
 		pid, err := GetID(tx, "DATASETS", "dataset_id", "dataset", ds)

dbs/fileparents.go

@@ -342,7 +342,7 @@ func (a *API) InsertFileParentsBlockTxt(tx *sql.Tx) error {
 	if utils.VERBOSE > 1 {
 		log.Println("InsertFileParentsBlock fids", fids, "bfids", bfids)
 	}
-	if !utils.Equal(utils.Set(fids), utils.Set(bfids)) {
+	if !utils.Equal(utils.OrderedSet(fids), utils.OrderedSet(bfids)) {
 		log.Println("block fids != file ids")
 		log.Println("block ids", bfids)
 		log.Println("file  ids", fids)

dbs/migrate.go

@@ -504,8 +504,8 @@ func startMigrationRequest(rec MigrationRequest) ([]MigrationReport, error) {
 	// get parent blocks at source DBS instance for given input
 	//     srcParentBlocks = prepareMigrationList(localhost, input)
 	srcParentBlocks = prepareMigrationListAtSource(localhost, dstParentBlocks)
-	dstParentBlocks = utils.List2Set(dstParentBlocks)
-	srcParentBlocks = utils.List2Set(srcParentBlocks)
+	dstParentBlocks = utils.Set(dstParentBlocks)
+	srcParentBlocks = utils.Set(srcParentBlocks)
 	if utils.VERBOSE > 0 {
 		log.Printf("Migration blocks from destination %s %+v", rurl, dstParentBlocks)
 		log.Printf("Migration blocks from source %s %+v", localhost, srcParentBlocks)

go.mod

@@ -38,6 +38,7 @@ require (
 	github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 // indirect
+	golang.org/x/exp v0.0.0-20220428152302-39d4317da171 // indirect
 	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12 // indirect

go.sum

@@ -106,6 +106,8 @@ golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSO
 golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 h1:tkVvjkPTB7pnW3jnid7kNyAMPVWllTNOf/qKDze4p9o=
 golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20220428152302-39d4317da171 h1:TfdoLivD44QwvssI9Sv1xwa5DcL5XQr4au4sZ2F2NV4=
+golang.org/x/exp v0.0.0-20220428152302-39d4317da171/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=

test/bench_test.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"math/rand"
 	"testing"
+	"time"
 
 	"github.com/dmwm/dbs2go/dbs"
 	"github.com/dmwm/dbs2go/utils"
@@ -49,3 +51,40 @@ func BenchmarkUpdateOrderedDict(b *testing.B) {
 		utils.UpdateOrderedDict(omap, nmap)
 	}
 }
+
+// BenchmarkInList
+func BenchmarkInList(b *testing.B) {
+	utils.VERBOSE = 0
+	N := 1000
+	list := make([]int, N)
+	for i := 0; i < N; i++ {
+		list[i] = i
+	}
+	rand.Seed(time.Now().UnixNano())
+	for i := 0; i < b.N; i++ {
+		elem := rand.Intn(N)
+		res := utils.InList(elem, list)
+		if res != true {
+			b.Fatal("unable to find random element in a list")
+		}
+	}
+}
+
+// BenchmarkEqual
+func BenchmarkEqual(b *testing.B) {
+	utils.VERBOSE = 0
+	N := 1000
+	list := make([]int, N)
+	for i := 0; i < N; i++ {
+		list[i] = i
+	}
+	rand.Seed(time.Now().UnixNano())
+	for i := 0; i < b.N; i++ {
+		set1 := utils.Set(list)
+		set2 := utils.Set(list)
+		res := utils.Equal(set1, set2)
+		if res != true {
+			b.Fatal("unable to compare sets")
+		}
+	}
+}

test/main.go

@@ -44,7 +44,7 @@ func initDB(dryRun bool) *sql.DB {
 		log.Fatal("unable to get current working dir")
 	}
 	utils.STATICDIR = fmt.Sprintf("%s/../static", dir)
-	utils.VERBOSE = 2
+	utils.VERBOSE = 1
 	dbtype := "sqlite3"
 	dburi := "/tmp/dbs-test.db"
 	dbowner := "sqlite"

test/migrate_test.go

@@ -186,7 +186,7 @@ func TestMigrate(t *testing.T) {
 	var sids []int64
 	for _, rrr := range statusRecords {
 		sids = append(sids, rrr.MIGRATION_REQUEST_ID)
-		if !utils.InInt64List(rrr.MIGRATION_REQUEST_ID, rids) {
+		if !utils.InList(rrr.MIGRATION_REQUEST_ID, rids) {
 			t.Errorf("unvalid status request id %d, expect %+v", rrr.MIGRATION_REQUEST_ID, rids)
 		}
 	}

test/utils_test.go

@@ -25,7 +25,7 @@ func TestUtilsInList(t *testing.T) {
 func TestUtilsSet(t *testing.T) {
 	vals := []int64{1, 2, 3, 1}
 	res := utils.Set(vals)
-	if !utils.Equal(res, utils.Set([]int64{3, 2, 1})) {
+	if !utils.Equal(res, utils.Set([]int64{1, 2, 3})) {
 		t.Error("Fail TestUtilsSet")
 	}
 	arr := []int64{4, 5, 6}
@@ -34,6 +34,19 @@ func TestUtilsSet(t *testing.T) {
 	}
 }
 
+// TestUtilsOrderedSet
+func TestUtilsOrderedSet(t *testing.T) {
+	vals := []int64{1, 2, 3, 1}
+	res := utils.Set(vals)
+	if !utils.Equal(res, utils.OrderedSet([]int64{3, 2, 1})) {
+		t.Error("Fail TestUtilsSet")
+	}
+	arr := []int64{4, 5, 6}
+	if utils.Equal(res, arr) {
+		t.Error("Fail of Equal in TestUtilsOrderedSet")
+	}
+}
+
 // TestUtilsRecordSize
 func TestUtilsRecordSize(t *testing.T) {
 	rec := make(map[string]int)

utils/utils.go

@@ -17,6 +17,8 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+
+	"golang.org/x/exp/constraints"
 )
 
 // VERBOSE controls verbosity level of the package
@@ -96,22 +98,13 @@ func GoDeferFunc(api string, f func()) {
 	}
 }
 
-// InInt64List checks item in a list
-func InInt64List(a int64, list []int64) bool {
-	check := 0
-	for _, b := range list {
-		if b == a {
-			check += 1
-		}
-	}
-	if check != 0 {
-		return true
-	}
-	return false
+// ListEntry identifies types used by list's generics function
+type ListEntry interface {
+	int | int64 | float64 | string
 }
 
 // InList checks item in a list
-func InList(a string, list []string) bool {
+func InList[T ListEntry](a T, list []T) bool {
 	check := 0
 	for _, b := range list {
 		if b == a {
@@ -124,33 +117,35 @@ func InList(a string, list []string) bool {
 	return false
 }
 
-// Int64List implement sort for []int type
-type Int64List []int64
-
-// Len provides length of the []int64 type
-func (s Int64List) Len() int { return len(s) }
-
-// Swap implements swap function for []int64 type
-func (s Int64List) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
-
-// Less implements less function for []int64 type
-func (s Int64List) Less(i, j int) bool { return s[i] < s[j] }
-
-// Set implementa basic set
-func Set(list []int64) []int64 {
-	var out []int64
-	for _, v := range list {
-		if !InInt64List(v, out) {
+// Set converts input list into set
+func Set[T ListEntry](arr []T) []T {
+	var out []T
+	for _, v := range arr {
+		if !InList(v, out) {
 			out = append(out, v)
 		}
 	}
-	sort.Sort(Int64List(out))
+	return out
+}
+
+// sortSlice helper function on any ordered generic list
+// https://gosamples.dev/generics-sort-slice/
+func sortSlice[T constraints.Ordered](s []T) {
+    sort.Slice(s, func(i, j int) bool {
+        return s[i] < s[j]
+    })
+}
+
+// OrderedSet implementa ordered set
+func OrderedSet[T ListEntry](list []T) []T {
+	out := Set(list)
+	sortSlice(out)
 	return out
 }
 
 // Equal tells whether a and b contain the same elements.
 // A nil argument is equivalent to an empty slice.
-func Equal(a, b []int64) bool {
+func Equal[T ListEntry](a, b []T) bool {
 	if len(a) != len(b) {
 		return false
 	}
@@ -180,29 +175,6 @@ func MapIntKeys(rec map[int]interface{}) []int {
 	return keys
 }
 
-// List2Set converts input list into set
-func List2Set(arr []string) []string {
-	var out []string
-	for _, key := range arr {
-		if !InList(key, out) {
-			out = append(out, key)
-		}
-	}
-	return out
-}
-
-// StringList implements sort for []string type
-type StringList []string
-
-// Len provide length method of StringList
-func (s StringList) Len() int { return len(s) }
-
-// Swap provide swap method of StringList
-func (s StringList) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
-
-// Less provide less method of StringList
-func (s StringList) Less(i, j int) bool { return s[i] < s[j] }
-
 // ListFiles lists files in a given directory
 func ListFiles(dir string) []string {
 	var out []string