diff --git a/backend/backend.go b/backend/backend.go index 633152ba0..bbba7204c 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -20,11 +20,13 @@ import ( var processedLeaderboard structs.LeaderboardData -var lifterData = lifter.Build() +// var lifterData = lifter.Build() + +var QueryCache dbtools.QueryCache func getTest(c *gin.Context) { - hour, min, sec := time.Now().Clock() - retStruct := structs.TestPayload{Hour: hour, Min: min, Sec: sec} + hour, mins, sec := time.Now().Clock() + retStruct := structs.TestPayload{Hour: hour, Min: mins, Sec: sec} c.JSON(http.StatusOK, retStruct) } @@ -108,7 +110,7 @@ func postLeaderboard(c *gin.Context) { } leaderboardData := processedLeaderboard.Select(body.SortBy) // Selects either total or sinclair sorted leaderboard - fedData := dbtools.Filter(*leaderboardData, body, dbtools.WeightClassList[body.WeightClass], *lifterData) + fedData := dbtools.FilterLifts(*leaderboardData, body, dbtools.WeightClassList[body.WeightClass], &QueryCache) c.JSON(http.StatusOK, fedData) } @@ -137,7 +139,7 @@ func setupCORS(r *gin.Engine) { func buildServer() *gin.Engine { log.Println("Starting server...") - go dbtools.BuildDatabase(&processedLeaderboard) + dbtools.BuildDatabase(&processedLeaderboard) r := gin.Default() setupCORS(r) r.GET("test", getTest) @@ -149,8 +151,21 @@ func buildServer() *gin.Engine { return r } +// CacheMeOutsideHowBoutDat - Precaches data on startup on a separate thread due to container timeout constraints. +func CacheMeOutsideHowBoutDat() { + log.Println("Precaching data...") + for n, query := range dbtools.PreCacheQuery { + log.Println("Caching query: ", n) + _, _ = QueryCache.CheckQuery(query) + liftdata := processedLeaderboard.Select(query.SortBy) + dbtools.PreCacheFilter(*liftdata, query, dbtools.WeightClassList[query.WeightClass], &QueryCache) + } + log.Println("Caching complete") +} + func main() { apiServer := buildServer() + go CacheMeOutsideHowBoutDat() err := apiServer.Run() if err != nil { log.Fatal("Failed to run server") diff --git a/backend/dbtools/cache_handler.go b/backend/dbtools/cache_handler.go new file mode 100644 index 000000000..288e7bbe1 --- /dev/null +++ b/backend/dbtools/cache_handler.go @@ -0,0 +1,29 @@ +package dbtools + +import ( + "backend/structs" +) + +type QueryCache struct { + Store []Query +} + +type Query struct { + Filter structs.LeaderboardPayload + DataPositions []int +} + +// AddQuery - Adds a query to the cache. +func (q *QueryCache) AddQuery(query structs.LeaderboardPayload, dataPositions []int) { + q.Store = append(q.Store, Query{Filter: query, DataPositions: dataPositions}) +} + +// CheckQuery - Checks if the query has been run before, if so, return the data positions. +func (q *QueryCache) CheckQuery(query structs.LeaderboardPayload) (bool, []int) { + for _, cacheQuery := range q.Store { + if cacheQuery.Filter.SortBy == query.SortBy && cacheQuery.Filter.Federation == query.Federation && cacheQuery.Filter.WeightClass == query.WeightClass && cacheQuery.Filter.StartDate == query.StartDate && cacheQuery.Filter.EndDate == query.EndDate { + return true, cacheQuery.DataPositions + } + } + return false, nil +} diff --git a/backend/dbtools/cache_handler_test.go b/backend/dbtools/cache_handler_test.go new file mode 100644 index 000000000..27fb62ea3 --- /dev/null +++ b/backend/dbtools/cache_handler_test.go @@ -0,0 +1,63 @@ +package dbtools + +import ( + "backend/structs" + "testing" +) + +func TestQueryCache_AddQuery(t *testing.T) { + type fields struct { + Store []Query + } + type args struct { + query structs.LeaderboardPayload + dataPositions []int + } + tests := []struct { + name string + fields fields + args args + }{ + {"AddQuery", fields{Store: []Query{}}, args{query: structs.LeaderboardPayload{}, dataPositions: []int{1}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + q := &QueryCache{ + Store: tt.fields.Store, + } + q.AddQuery(tt.args.query, tt.args.dataPositions) + }) + } +} + +func TestQueryCache_CheckQuery(t *testing.T) { + type fields struct { + Store []Query + } + type args struct { + query structs.LeaderboardPayload + } + tests := []struct { + name string + fields fields + args args + want bool + want1 []int + }{ + {"CheckQuery", fields{Store: []Query{{Filter: structs.LeaderboardPayload{SortBy: "total", Federation: "IPF", WeightClass: "93", StartDate: "2018-01-01", EndDate: "2018-01-01"}, DataPositions: []int{1}}}}, args{query: structs.LeaderboardPayload{SortBy: "total", Federation: "IPF", WeightClass: "93", StartDate: "2018-01-01", EndDate: "2018-01-01"}}, true, []int{1}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + q := &QueryCache{ + Store: tt.fields.Store, + } + got, got1 := q.CheckQuery(tt.args.query) + if got != tt.want { + t.Errorf("QueryCache.CheckQuery() got = %v, want %v", got, tt.want) + } + if len(got1) != len(tt.want1) { + t.Errorf("QueryCache.CheckQuery() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} diff --git a/backend/dbtools/dbtools_test.go b/backend/dbtools/dbtools_test.go index 50589ef0f..ed633e311 100644 --- a/backend/dbtools/dbtools_test.go +++ b/backend/dbtools/dbtools_test.go @@ -42,23 +42,42 @@ func TestCollateAll(t *testing.T) { func TestFilter(t *testing.T) { type args struct { - bigData []structs.Entry - filterQuery structs.LeaderboardPayload - weightCat structs.WeightClass - lifterProfiles map[string]string + bigData []structs.Entry + filterQuery structs.LeaderboardPayload + weightCat string } tests := []struct { name string args args - wantFilteredData []structs.Entry + wantFilteredData structs.LeaderboardResponse }{ - // todo: Add test cases. - {}, + { + name: "FilterByFederation", + args: args{ + bigData: []structs.Entry{{Date: "2023-06-01", Name: "John Smith", Total: 100, Federation: "BWL", Gender: enum.Male, Bodyweight: 109.00}, {Date: "2023-06-01", Name: "Dave Smith", Total: 200, Federation: "BWL", Gender: enum.Male, Bodyweight: 109.00}, {Date: "2023-06-01", Name: "Ethan Smith", Total: 300, Federation: "BWL", Gender: enum.Male, Bodyweight: 109.00}}, + filterQuery: structs.LeaderboardPayload{ + Start: 0, + Stop: 10, + SortBy: enum.Total, + Federation: enum.ALLFEDS, + WeightClass: "MALL", + Year: 69, + StartDate: "2023-01-01", + EndDate: "2024-01-01", + }, + weightCat: "MALL", + }, + wantFilteredData: structs.LeaderboardResponse{ + Size: 3, + Data: []structs.Entry{{Date: "2023-06-01", Name: "John Smith", Total: 100, Federation: "BWL", Gender: enum.Male, Bodyweight: 109.00}, {Date: "2023-06-01", Name: "Dave Smith", Total: 200, Federation: "BWL", Gender: enum.Male, Bodyweight: 109.00}, {Date: "2023-06-01", Name: "Ethan Smith", Total: 300, Federation: "BWL", Gender: enum.Male, Bodyweight: 109.00}}, + }, + }, } + var cache QueryCache for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if gotFilteredData := Filter(tt.args.bigData, tt.args.filterQuery, tt.args.weightCat, tt.args.lifterProfiles); !reflect.DeepEqual(gotFilteredData, tt.wantFilteredData) { - t.Errorf("Filter() = %v, want %v", gotFilteredData, tt.wantFilteredData) + if gotFilteredData := FilterLifts(tt.args.bigData, tt.args.filterQuery, WeightClassList[tt.args.weightCat], &cache); !reflect.DeepEqual(gotFilteredData, tt.wantFilteredData) { + t.Errorf("FilterLifts() = %v, want %v", gotFilteredData, tt.wantFilteredData) } }) } @@ -273,26 +292,6 @@ func Test_loadAllFedEvents(t *testing.T) { } } -func Test_removeFollowingLifts(t *testing.T) { - type args struct { - bigData []structs.Entry - } - tests := []struct { - name string - args args - wantFilteredData []structs.Entry - }{ - {name: "RemoveFollowingLifts", args: args{bigData: []structs.Entry{{Name: "John Smith", Total: 100}, {Name: "John Smith", Total: 200}, {Name: "John Smith", Total: 300}}}, wantFilteredData: []structs.Entry{{Name: "John Smith", Total: 100}}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if gotFilteredData := removeFollowingLifts(tt.args.bigData); !reflect.DeepEqual(gotFilteredData, tt.wantFilteredData) { - t.Errorf("removeFollowingLifts() = %v, want %v", gotFilteredData, tt.wantFilteredData) - } - }) - } -} - func Test_setGender(t *testing.T) { type args struct { entry *structs.Entry diff --git a/backend/dbtools/precache.go b/backend/dbtools/precache.go new file mode 100644 index 000000000..93aeeba89 --- /dev/null +++ b/backend/dbtools/precache.go @@ -0,0 +1,38 @@ +package dbtools + +import ( + "backend/enum" + "backend/structs" +) + +// PreCacheQuery - Queries to pre-cache on startup. +var PreCacheQuery = []structs.LeaderboardPayload{ + { + SortBy: "total", + Federation: "allfeds", + WeightClass: "MALL", + StartDate: enum.ZeroDate, + EndDate: enum.MaxDate, + }, + { + SortBy: "total", + Federation: "allfeds", + WeightClass: "FALL", + StartDate: enum.ZeroDate, + EndDate: enum.MaxDate, + }, + { + SortBy: "sinclair", + Federation: "allfeds", + WeightClass: "MALL", + StartDate: enum.ZeroDate, + EndDate: enum.MaxDate, + }, + { + SortBy: "sinclair", + Federation: "allfeds", + WeightClass: "FALL", + StartDate: enum.ZeroDate, + EndDate: enum.MaxDate, + }, +} diff --git a/backend/dbtools/sortby.go b/backend/dbtools/sortby.go index 3ac89e594..8da3282c1 100644 --- a/backend/dbtools/sortby.go +++ b/backend/dbtools/sortby.go @@ -2,49 +2,81 @@ package dbtools import ( "backend/enum" - "backend/lifter" "backend/structs" "backend/utilities" "sort" "time" ) -func removeFollowingLifts(bigData []structs.Entry) (filteredData []structs.Entry) { +// FilterLifts - Returns a slice of structs relating to the selected filter selection +func FilterLifts(bigData []structs.Entry, filterQuery structs.LeaderboardPayload, weightCat structs.WeightClass, cache *QueryCache) (filteredData structs.LeaderboardResponse) { + exists, positions := cache.CheckQuery(filterQuery) + + if exists { + filteredData.Data, filteredData.Size = fetchLifts(&bigData, positions, &filterQuery) + return + } + var names []string - var position []int - for i, d := range bigData { - if !utilities.Contains(names, d.Name) { - position = append(position, i) - names = append(names, d.Name) + var liftPtr *structs.Entry + var liftPositions []int + for idx, lift := range bigData { + liftPtr = &bigData[idx] + if getGender(liftPtr) == weightCat.Gender && !utilities.Contains(names, lift.Name) { + if lift.SelectedFederation(filterQuery.Federation) && lift.WithinWeightClass(WeightClassList[filterQuery.WeightClass].Gender, weightCat) && lift.WithinDates(filterQuery.StartDate, filterQuery.EndDate) { + liftPositions = append(liftPositions, idx) + names = append(names, lift.Name) + filteredData.Data = append(filteredData.Data, lift) + } } } - for _, posInt := range position { - filteredData = append(filteredData, bigData[posInt]) + cache.AddQuery(filterQuery, liftPositions) + + if filterQuery.Stop > len(liftPositions) { + filterQuery.Stop = len(liftPositions) } + + if filterQuery.Start > len(liftPositions) { + filterQuery.Start = len(liftPositions) + } + + filteredData.Size = len(liftPositions) + filteredData.Data = filteredData.Data[filterQuery.Start:filterQuery.Stop] return } -// Filter - Returns a slice of structs relating to the selected filter selection -func Filter(bigData []structs.Entry, filterQuery structs.LeaderboardPayload, weightCat structs.WeightClass, lifterProfiles map[string]string) (filteredData []structs.Entry) { +func PreCacheFilter(bigData []structs.Entry, filterQuery structs.LeaderboardPayload, weightCat structs.WeightClass, cache *QueryCache) { + var names []string + var liftPtr *structs.Entry + var liftPositions []int for idx, lift := range bigData { - liftptr := &bigData[idx] - if getGender(liftptr) == weightCat.Gender { + liftPtr = &bigData[idx] + if getGender(liftPtr) == weightCat.Gender && !utilities.Contains(names, lift.Name) { if lift.SelectedFederation(filterQuery.Federation) && lift.WithinWeightClass(WeightClassList[filterQuery.WeightClass].Gender, weightCat) && lift.WithinDates(filterQuery.StartDate, filterQuery.EndDate) { - linkedIG, igHandle := lifter.CheckUserList(lift.Name, lifterProfiles) - if linkedIG { - lift.Instagram = igHandle - } - filteredData = append(filteredData, lift) - } - if len(filteredData) >= filterQuery.Stop { - filteredData = removeFollowingLifts(filteredData) - if len(filteredData) >= filterQuery.Stop { - return - } + liftPositions = append(liftPositions, idx) + names = append(names, lift.Name) } } } - filteredData = removeFollowingLifts(filteredData) + cache.AddQuery(filterQuery, liftPositions) +} + +// fetchLifts - Returns a slice of structs relating to the selected filter selection, it will also remove any duplicate entries. +func fetchLifts(bigData *[]structs.Entry, pos []int, query *structs.LeaderboardPayload) (lifts []structs.Entry, size int) { + for _, p := range pos { + lifts = append(lifts, (*bigData)[p]) + } + + if query.Stop > len(lifts) { + query.Stop = len(lifts) + } + + if query.Start > len(lifts) { + query.Start = len(lifts) + } + + size = len(lifts) + lifts = lifts[query.Start:query.Stop] return } diff --git a/backend/go.mod b/backend/go.mod index 4d04d4c73..b533b2c1c 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -1,6 +1,6 @@ module backend -go 1.20 +go 1.21 require ( github.com/gin-contrib/cors v1.4.0 diff --git a/backend/structs/structs.go b/backend/structs/structs.go index 5f2fe9552..7c9714220 100644 --- a/backend/structs/structs.go +++ b/backend/structs/structs.go @@ -77,3 +77,8 @@ type Entry struct { Federation string `json:"country"` Instagram string `json:"instagram"` } + +type LeaderboardResponse struct { + Size int `json:"size"` + Data []Entry `json:"data"` +}