From a3c35d213775c84eeee94db425c6452869139b9b Mon Sep 17 00:00:00 2001 From: Kristian Tsivkov <109758711+ktsivkov@users.noreply.github.com> Date: Tue, 18 Apr 2023 11:36:35 +0200 Subject: [PATCH] Release version 1.0.0 of the plugin (#1) * [prepare-release] Implemented easer & cacher functionality, and covered with tests * [prepare-release] Implemented and set in use the SetPointedValue instead of full reflection --- .gitignore | 1 + README.md | 233 +++++++++++++++++++++- cacher.go | 6 + cacher_test.go | 48 +++++ caches.go | 119 ++++++++++++ caches_test.go | 470 +++++++++++++++++++++++++++++++++++++++++++++ easer.go | 30 +++ easer_test.go | 102 ++++++++++ go.mod | 5 + go.sum | 6 + identifier.go | 26 +++ identifier_test.go | 20 ++ query_task.go | 17 ++ query_task_test.go | 38 ++++ reflection.go | 79 ++++++++ reflection_test.go | 215 +++++++++++++++++++++ task.go | 6 + task_test.go | 21 ++ 18 files changed, 1440 insertions(+), 2 deletions(-) create mode 100644 .gitignore create mode 100644 cacher.go create mode 100644 cacher_test.go create mode 100644 caches.go create mode 100644 caches_test.go create mode 100644 easer.go create mode 100644 easer_test.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 identifier.go create mode 100644 identifier_test.go create mode 100644 query_task.go create mode 100644 query_task_test.go create mode 100644 reflection.go create mode 100644 reflection_test.go create mode 100644 task.go create mode 100644 task_test.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..485dee6 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea diff --git a/README.md b/README.md index 6267674..3bfd6a7 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,231 @@ -# caches -Caches Plugin +# Gorm Caches + +Gorm Caches plugin using database request reductions (easer), and response caching mechanism provide you an easy way to optimize database performance. + +## Features + +- Database request reduction. If three identical requests are running at the same time, only the first one is going to be executed, and its response will be returned for all. +- Database response caching. By implementing the Cacher interface, you can easily setup a caching mechanism for your database queries. +- Supports all databases that are supported by gorm itself. + +## Install + +```bash +go get -u github.com/go-gorm/caches +``` + +## Usage + +Configure the `easer`, and the `cacher`, and then load the plugin to gorm. + +```go +package main + +import ( + "fmt" + "sync" + + "github.com/go-gorm/caches" + "gorm.io/driver/mysql" + "gorm.io/gorm" +) + +func main() { + db, _ := gorm.Open( + mysql.Open("DATABASE_DSN"), + &gorm.Config{}, + ) + cachesPlugin := &caches.Caches{Conf: &caches.Config{ + Easer: true, + Cacher: &yourCacherImplementation{}, + }} + _ = db.Use(cachesPlugin) +} +``` + +## Easer Example + +```go +package main + +import ( + "fmt" + "sync" + "time" + + "github.com/go-gorm/caches" + "gorm.io/driver/mysql" + "gorm.io/gorm" +) + +type UserRoleModel struct { + gorm.Model + Name string `gorm:"unique"` +} + +type UserModel struct { + gorm.Model + Name string + RoleId uint + Role *UserRoleModel `gorm:"foreignKey:role_id;references:id"` +} + +func main() { + db, _ := gorm.Open( + mysql.Open("DATABASE_DSN"), + &gorm.Config{}, + ) + + cachesPlugin := &caches.Caches{Conf: &caches.Config{ + Easer: true, + }} + + _ = db.Use(cachesPlugin) + + _ = db.AutoMigrate(&UserRoleModel{}) + + _ = db.AutoMigrate(&UserModel{}) + + adminRole := &UserRoleModel{ + Name: "Admin", + } + db.FirstOrCreate(adminRole, "Name = ?", "Admin") + + guestRole := &UserRoleModel{ + Name: "Guest", + } + db.FirstOrCreate(guestRole, "Name = ?", "Guest") + + db.Save(&UserModel{ + Name: "ktsivkov", + Role: adminRole, + }) + db.Save(&UserModel{ + Name: "anonymous", + Role: guestRole, + }) + + var ( + q1Users []UserModel + q2Users []UserModel + ) + wg := &sync.WaitGroup{} + wg.Add(2) + go func() { + db.Model(&UserModel{}).Joins("Role").Find(&q1Users, "Role.Name = ? AND Sleep(1) = false", "Admin") + wg.Done() + }() + go func() { + time.Sleep(500 * time.Millisecond) + db.Model(&UserModel{}).Joins("Role").Find(&q2Users, "Role.Name = ? AND Sleep(1) = false", "Admin") + wg.Done() + }() + wg.Wait() + + fmt.Println(fmt.Sprintf("%+v", q1Users)) + fmt.Println(fmt.Sprintf("%+v", q2Users)) +} +``` + +## Cacher Example + +```go +package main + +import ( + "fmt" + "sync" + + "github.com/go-gorm/caches" + "gorm.io/driver/mysql" + "gorm.io/gorm" +) + +type UserRoleModel struct { + gorm.Model + Name string `gorm:"unique"` +} + +type UserModel struct { + gorm.Model + Name string + RoleId uint + Role *UserRoleModel `gorm:"foreignKey:role_id;references:id"` +} + +type dummyCacher struct { + store *sync.Map +} + +func (c *dummyCacher) init() { + if c.store == nil { + c.store = &sync.Map{} + } +} + +func (c *dummyCacher) Get(key string) interface{} { + c.init() + val, _ := c.store.Load(key) + return val +} + +func (c *dummyCacher) Store(key string, val interface{}) error { + c.init() + c.store.Store(key, val) + return nil +} + +func main() { + db, _ := gorm.Open( + mysql.Open("DATABASE_DSN"), + &gorm.Config{}, + ) + + cachesPlugin := &caches.Caches{Conf: &caches.Config{ + Cacher: &dummyCacher{}, + }} + + _ = db.Use(cachesPlugin) + + _ = db.AutoMigrate(&UserRoleModel{}) + + _ = db.AutoMigrate(&UserModel{}) + + adminRole := &UserRoleModel{ + Name: "Admin", + } + db.FirstOrCreate(adminRole, "Name = ?", "Admin") + + guestRole := &UserRoleModel{ + Name: "Guest", + } + db.FirstOrCreate(guestRole, "Name = ?", "Guest") + + db.Save(&UserModel{ + Name: "ktsivkov", + Role: adminRole, + }) + db.Save(&UserModel{ + Name: "anonymous", + Role: guestRole, + }) + + var ( + q1Users []UserModel + q2Users []UserModel + ) + + db.Model(&UserModel{}).Joins("Role").Find(&q1Users, "Role.Name = ? AND Sleep(1) = false", "Admin") + fmt.Println(fmt.Sprintf("%+v", q1Users)) + + db.Model(&UserModel{}).Joins("Role").Find(&q2Users, "Role.Name = ? AND Sleep(1) = false", "Admin") + fmt.Println(fmt.Sprintf("%+v", q2Users)) +} +``` + +## License + +MIT license. + +## Easer +The easer is an adjusted version of the [ServantGo](https://github.com/ktsivkov/servantgo) library to fit the needs of this plugin. diff --git a/cacher.go b/cacher.go new file mode 100644 index 0000000..096877d --- /dev/null +++ b/cacher.go @@ -0,0 +1,6 @@ +package caches + +type Cacher interface { + Get(key string) interface{} + Store(key string, val interface{}) error +} diff --git a/cacher_test.go b/cacher_test.go new file mode 100644 index 0000000..d2b135b --- /dev/null +++ b/cacher_test.go @@ -0,0 +1,48 @@ +package caches + +import ( + "errors" + "sync" +) + +type cacherMock struct { + store *sync.Map +} + +func (c *cacherMock) init() { + if c.store == nil { + c.store = &sync.Map{} + } +} + +func (c *cacherMock) Get(key string) interface{} { + c.init() + val, _ := c.store.Load(key) + return val +} + +func (c *cacherMock) Store(key string, val interface{}) error { + c.init() + c.store.Store(key, val) + return nil +} + +type cacherStoreErrorMock struct { + store *sync.Map +} + +func (c *cacherStoreErrorMock) init() { + if c.store == nil { + c.store = &sync.Map{} + } +} + +func (c *cacherStoreErrorMock) Get(key string) interface{} { + c.init() + val, _ := c.store.Load(key) + return val +} + +func (c *cacherStoreErrorMock) Store(key string, val interface{}) error { + return errors.New("store-error") +} diff --git a/caches.go b/caches.go new file mode 100644 index 0000000..72a4165 --- /dev/null +++ b/caches.go @@ -0,0 +1,119 @@ +package caches + +import ( + "sync" + + "gorm.io/gorm" +) + +type Caches struct { + Conf *Config + + queue *sync.Map + queryCb func(*gorm.DB) +} + +type Config struct { + Easer bool + Cacher Cacher +} + +func (c *Caches) Name() string { + return "gorm:caches" +} + +func (c *Caches) Initialize(db *gorm.DB) error { + if c.Conf == nil { + c.Conf = &Config{ + Easer: false, + Cacher: nil, + } + } + + if c.Conf.Easer { + c.queue = &sync.Map{} + } + + c.queryCb = db.Callback().Query().Get("gorm:query") + + err := db.Callback().Query().Replace("gorm:query", c.Query) + if err != nil { + return err + } + + return nil +} + +func (c *Caches) Query(db *gorm.DB) { + if c.Conf.Easer == false && c.Conf.Cacher == nil { + c.queryCb(db) + return + } + + identifier := buildIdentifier(db) + + if c.checkCache(db, identifier) { + return + } + + c.ease(db, identifier) + if db.Error != nil { + return + } + + c.storeInCache(db, identifier) + if db.Error != nil { + return + } +} + +func (c *Caches) ease(db *gorm.DB, identifier string) { + if c.Conf.Easer == false { + c.queryCb(db) + return + } + + res := ease(&queryTask{ + id: identifier, + db: db, + queryCb: c.queryCb, + }, c.queue).(*queryTask) + + if db.Error != nil { + return + } + + if res.db.Statement.Dest == db.Statement.Dest { + return + } + + SetPointedValue(db.Statement.Dest, res.db.Statement.Dest) + //TODO: when dealing with timestamps the reflection fails, investigate further and find a durable solution for it + //if err := deepCopy(res.db.Statement.Dest, db.Statement.Dest); err != nil { + // _ = db.AddError(err) + // return + //} +} + +func (c *Caches) checkCache(db *gorm.DB, identifier string) bool { + if c.Conf.Cacher != nil { + if res := c.Conf.Cacher.Get(identifier); res != nil { + SetPointedValue(db.Statement.Dest, res) + //TODO: when dealing with timestamps the reflection fails, investigate further and find a durable solution for it + //if err := deepCopy(res, db.Statement.Dest); err != nil { + // _ = db.AddError(err) + //} + return true + } + } + return false +} + +func (c *Caches) storeInCache(db *gorm.DB, identifier string) { + if c.Conf.Cacher != nil { + err := c.Conf.Cacher.Store(identifier, db.Statement.Dest) + if err != nil { + _ = db.AddError(err) + } + } +} diff --git a/caches_test.go b/caches_test.go new file mode 100644 index 0000000..42e033c --- /dev/null +++ b/caches_test.go @@ -0,0 +1,470 @@ +package caches + +import ( + "fmt" + "reflect" + "sync" + "sync/atomic" + "testing" + "time" + + "gorm.io/gorm" + "gorm.io/gorm/utils/tests" +) + +type mockDest struct { + Result string +} + +type mockDestWithUnexportedField struct { + result string +} + +func TestCaches_Name(t *testing.T) { + caches := &Caches{ + Conf: &Config{ + Easer: true, + Cacher: nil, + }, + } + expectedName := "gorm:caches" + if act := caches.Name(); act != expectedName { + t.Errorf("Name on caches did not return the expected value, expected: %s, actual: %s", + expectedName, act) + } +} + +func TestCaches_Initialize(t *testing.T) { + t.Run("empty config", func(t *testing.T) { + caches := &Caches{} + db, err := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + if err != nil { + t.Fatalf("gorm initialization resulted into an unexpected error, %s", err.Error()) + } + + originalQueryCb := db.Callback().Query().Get("gorm:query") + + err = db.Use(caches) + if err != nil { + t.Fatalf("gorm:caches loading resulted into an unexpected error, %s", err.Error()) + } + + newQueryCallback := db.Callback().Query().Get("gorm:query") + + if reflect.ValueOf(originalQueryCb).Pointer() == reflect.ValueOf(newQueryCallback).Pointer() { + t.Errorf("loading of gorm:caches, expected to replace the `gorm:query` callback") + } + + if reflect.ValueOf(newQueryCallback).Pointer() != reflect.ValueOf(caches.Query).Pointer() { + t.Errorf("loading of gorm:caches, expected to replace the `gorm:query` callback, with caches.Query") + } + + if reflect.ValueOf(originalQueryCb).Pointer() != reflect.ValueOf(caches.queryCb).Pointer() { + t.Errorf("loading of gorm:caches, expected to load original `gorm:query` callback, to caches.queryCb") + } + }) + t.Run("config - easer", func(t *testing.T) { + caches := &Caches{ + Conf: &Config{ + Easer: true, + Cacher: nil, + }, + } + db, err := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + if err != nil { + t.Fatalf("gorm initialization resulted into an unexpected error, %s", err.Error()) + } + + originalQueryCb := db.Callback().Query().Get("gorm:query") + + err = db.Use(caches) + if err != nil { + t.Fatalf("gorm:caches loading resulted into an unexpected error, %s", err.Error()) + } + + newQueryCallback := db.Callback().Query().Get("gorm:query") + + if reflect.ValueOf(originalQueryCb).Pointer() == reflect.ValueOf(newQueryCallback).Pointer() { + t.Errorf("loading of gorm:caches, expected to replace the `gorm:query` callback") + } + + if reflect.ValueOf(newQueryCallback).Pointer() != reflect.ValueOf(caches.Query).Pointer() { + t.Errorf("loading of gorm:caches, expected to replace the `gorm:query` callback, with caches.Query") + } + + if reflect.ValueOf(originalQueryCb).Pointer() != reflect.ValueOf(caches.queryCb).Pointer() { + t.Errorf("loading of gorm:caches, expected to load original `gorm:query` callback, to caches.queryCb") + } + }) +} + +func TestCaches_Query(t *testing.T) { + t.Run("nothing enabled", func(t *testing.T) { + conf := &Config{ + Easer: false, + Cacher: nil, + } + db, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db.Statement.Dest = &mockDest{} + caches := &Caches{ + Conf: conf, + queryCb: func(db *gorm.DB) { + db.Statement.Dest.(*mockDest).Result = db.Statement.SQL.String() + }, + } + + // Set the query SQL into something specific + exampleQuery := "demo-query" + db.Statement.SQL.WriteString(exampleQuery) + + caches.Query(db) // Execute the query + + if db.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db.Error) + } + + if db.Statement.Dest == nil { + t.Fatal("no query result was set after caches Query was executed") + } + + if res := db.Statement.Dest.(*mockDest); res.Result != exampleQuery { + t.Errorf("the execution of the Query expected a result of `%s`, got `%s`", exampleQuery, res) + } + }) + + t.Run("easer only", func(t *testing.T) { + conf := &Config{ + Easer: true, + Cacher: nil, + } + + t.Run("one query", func(t *testing.T) { + db, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db.Statement.Dest = &mockDest{} + caches := &Caches{ + Conf: conf, + + queue: &sync.Map{}, + queryCb: func(db *gorm.DB) { + db.Statement.Dest.(*mockDest).Result = db.Statement.SQL.String() + }, + } + + // Set the query SQL into something specific + exampleQuery := "demo-query" + db.Statement.SQL.WriteString(exampleQuery) + + caches.Query(db) // Execute the query + + if db.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db.Error) + } + + if db.Statement.Dest == nil { + t.Fatal("no query result was set after caches Query was executed") + } + + if res := db.Statement.Dest.(*mockDest); res.Result != exampleQuery { + t.Errorf("the execution of the Query expected a result of `%s`, got `%s`", exampleQuery, res) + } + }) + + t.Run("two identical queries", func(t *testing.T) { + //t.Run("with error", func(t *testing.T) { + // var incr int32 + // db1, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + // db1.Statement.Dest = &mockDestWithUnexportedField{} + // db2, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + // db2.Statement.Dest = &mockDestWithUnexportedField{} + // + // caches := &Caches{ + // Conf: conf, + // + // queue: &sync.Map{}, + // queryCb: func(db *gorm.DB) { + // time.Sleep(1 * time.Second) + // atomic.AddInt32(&incr, 1) + // + // db.Statement.Dest.(*mockDestWithUnexportedField).result = fmt.Sprintf("%d", atomic.LoadInt32(&incr)) + // }, + // } + // + // // Set the queries' SQL into something specific + // exampleQuery := "demo-query" + // db1.Statement.SQL.WriteString(exampleQuery) + // db2.Statement.SQL.WriteString(exampleQuery) + // + // wg := &sync.WaitGroup{} + // wg.Add(2) + // go func() { + // caches.Query(db1) // Execute the query + // wg.Done() + // }() + // go func() { + // time.Sleep(500 * time.Millisecond) // Execute the second query half a second later + // caches.Query(db2) // Execute the query + // wg.Done() + // }() + // wg.Wait() + // + // if db2.Error == nil { + // t.Error("an error was expected, got none") + // } + //}) + + t.Run("without error", func(t *testing.T) { + var incr int32 + db1, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db1.Statement.Dest = &mockDest{} + db2, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db2.Statement.Dest = &mockDest{} + + caches := &Caches{ + Conf: conf, + + queue: &sync.Map{}, + queryCb: func(db *gorm.DB) { + time.Sleep(1 * time.Second) + atomic.AddInt32(&incr, 1) + + db.Statement.Dest.(*mockDest).Result = fmt.Sprintf("%d", atomic.LoadInt32(&incr)) + }, + } + + // Set the queries' SQL into something specific + exampleQuery := "demo-query" + db1.Statement.SQL.WriteString(exampleQuery) + db2.Statement.SQL.WriteString(exampleQuery) + + wg := &sync.WaitGroup{} + wg.Add(2) + go func() { + caches.Query(db1) // Execute the query + wg.Done() + }() + go func() { + time.Sleep(500 * time.Millisecond) // Execute the second query half a second later + caches.Query(db2) // Execute the query + wg.Done() + }() + wg.Wait() + + if db1.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db1.Error) + } + + if db2.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db2.Error) + } + + if act := atomic.LoadInt32(&incr); act != 1 { + t.Errorf("when executing two identical queries, expected to run %d time, but %d", 1, act) + } + }) + }) + + t.Run("two different queries", func(t *testing.T) { + var incr int32 + db1, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db1.Statement.Dest = &mockDest{} + db2, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db2.Statement.Dest = &mockDest{} + + caches := &Caches{ + Conf: conf, + + queue: &sync.Map{}, + queryCb: func(db *gorm.DB) { + time.Sleep(1 * time.Second) + atomic.AddInt32(&incr, 1) + + db.Statement.Dest.(*mockDest).Result = fmt.Sprintf("%d", atomic.LoadInt32(&incr)) + }, + } + + // Set the queries' SQL into something specific + exampleQuery1 := "demo-query-1" + db1.Statement.SQL.WriteString(exampleQuery1) + exampleQuery2 := "demo-query-2" + db2.Statement.SQL.WriteString(exampleQuery2) + + wg := &sync.WaitGroup{} + wg.Add(2) + go func() { + caches.Query(db1) // Execute the query + wg.Done() + }() + go func() { + time.Sleep(500 * time.Millisecond) // Execute the second query half a second later + caches.Query(db2) // Execute the query + wg.Done() + }() + wg.Wait() + + if db1.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db1.Error) + } + + if db2.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db2.Error) + } + + if act := atomic.LoadInt32(&incr); act != 2 { + t.Errorf("when executing two identical queries, expected to run %d times, but %d", 2, act) + } + }) + }) + + t.Run("cacher only", func(t *testing.T) { + t.Run("one query", func(t *testing.T) { + t.Run("with error", func(t *testing.T) { + db, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db.Statement.Dest = &mockDest{} + + caches := &Caches{ + Conf: &Config{ + Easer: false, + Cacher: &cacherStoreErrorMock{}, + }, + + queue: &sync.Map{}, + queryCb: func(db *gorm.DB) { + db.Statement.Dest.(*mockDest).Result = db.Statement.SQL.String() + }, + } + + // Set the query SQL into something specific + exampleQuery := "demo-query" + db.Statement.SQL.WriteString(exampleQuery) + + caches.Query(db) // Execute the query + + if db.Error == nil { + t.Error("an error was expected, got none") + } + }) + + t.Run("without error", func(t *testing.T) { + db, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db.Statement.Dest = &mockDest{} + + caches := &Caches{ + Conf: &Config{ + Easer: false, + Cacher: &cacherMock{}, + }, + + queue: &sync.Map{}, + queryCb: func(db *gorm.DB) { + db.Statement.Dest.(*mockDest).Result = db.Statement.SQL.String() + }, + } + + // Set the query SQL into something specific + exampleQuery := "demo-query" + db.Statement.SQL.WriteString(exampleQuery) + + caches.Query(db) // Execute the query + + if db.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db.Error) + } + + if db.Statement.Dest == nil { + t.Fatal("no query result was set after caches Query was executed") + } + + if res := db.Statement.Dest.(*mockDest); res.Result != exampleQuery { + t.Errorf("the execution of the Query expected a result of `%s`, got `%s`", exampleQuery, res) + } + }) + }) + + t.Run("two identical queries", func(t *testing.T) { + var incr int32 + db1, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db1.Statement.Dest = &mockDest{} + db2, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db2.Statement.Dest = &mockDest{} + + caches := &Caches{ + Conf: &Config{ + Easer: false, + Cacher: &cacherMock{}, + }, + + queue: &sync.Map{}, + queryCb: func(db *gorm.DB) { + time.Sleep(1 * time.Second) + atomic.AddInt32(&incr, 1) + + db.Statement.Dest.(*mockDest).Result = fmt.Sprintf("%d", atomic.LoadInt32(&incr)) + }, + } + + // Set the queries' SQL into something specific + exampleQuery := "demo-query" + db1.Statement.SQL.WriteString(exampleQuery) + db2.Statement.SQL.WriteString(exampleQuery) + + caches.Query(db1) + caches.Query(db2) + + if db1.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db1.Error) + } + + if db2.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db2.Error) + } + + if act := atomic.LoadInt32(&incr); act != 1 { + t.Errorf("when executing two identical queries, expected to run %d time, but %d", 1, act) + } + }) + + t.Run("two different queries", func(t *testing.T) { + + var incr int32 + db1, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db1.Statement.Dest = &mockDest{} + db2, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db2.Statement.Dest = &mockDest{} + + caches := &Caches{ + Conf: &Config{ + Easer: false, + Cacher: &cacherMock{}, + }, + + queue: &sync.Map{}, + queryCb: func(db *gorm.DB) { + time.Sleep(1 * time.Second) + atomic.AddInt32(&incr, 1) + + db.Statement.Dest.(*mockDest).Result = fmt.Sprintf("%d", atomic.LoadInt32(&incr)) + }, + } + + // Set the queries' SQL into something specific + exampleQuery1 := "demo-query-1" + db1.Statement.SQL.WriteString(exampleQuery1) + exampleQuery2 := "demo-query-2" + db2.Statement.SQL.WriteString(exampleQuery2) + + caches.Query(db1) + if db1.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db1.Error) + } + + caches.Query(db2) + if db2.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db2.Error) + } + + if act := atomic.LoadInt32(&incr); act != 2 { + t.Errorf("when executing two identical queries, expected to run %d times, but %d", 2, act) + } + }) + }) +} diff --git a/easer.go b/easer.go new file mode 100644 index 0000000..b313544 --- /dev/null +++ b/easer.go @@ -0,0 +1,30 @@ +package caches + +import "sync" + +func ease(t task, queue *sync.Map) task { + eq := &eased{ + task: t, + wg: &sync.WaitGroup{}, + } + eq.wg.Add(1) + + runner, ok := queue.LoadOrStore(t.GetId(), eq) + et := runner.(*eased) + + // If this request is the first of its kind, we execute the Run + if !ok { + et.task.Run() + + queue.Delete(et.task.GetId()) + et.wg.Done() + } + + et.wg.Wait() + return et.task +} + +type eased struct { + task task + wg *sync.WaitGroup +} diff --git a/easer_test.go b/easer_test.go new file mode 100644 index 0000000..c2b49e5 --- /dev/null +++ b/easer_test.go @@ -0,0 +1,102 @@ +package caches + +import ( + "sync" + "testing" + "time" +) + +func TestEase(t *testing.T) { + t.Run("same queries", func(t *testing.T) { + queue := &sync.Map{} + + myTask := &mockTask{ + delay: 1 * time.Second, + expRes: "expect-this", + id: "unique-id", + } + myDupTask := &mockTask{ + delay: 1 * time.Second, + expRes: "not-this", + id: "unique-id", + } + + wg := &sync.WaitGroup{} + wg.Add(2) + + var ( + myTaskRes *mockTask + myDupTaskRes *mockTask + ) + + // Both queries will run at the same time, the second one will run half a second later + go func() { + myTaskRes = ease(myTask, queue).(*mockTask) + wg.Done() + }() + go func() { + time.Sleep(500 * time.Millisecond) + myDupTaskRes = ease(myDupTask, queue).(*mockTask) + wg.Done() + }() + wg.Wait() + + if myTaskRes.actRes != myTaskRes.expRes { + t.Error("expected first query to be executed") + } + + if myTaskRes.actRes != myDupTaskRes.actRes { + t.Errorf("expected same result from both tasks, expected: %s, actual: %s", + myTaskRes.actRes, myDupTaskRes.actRes) + } + }) + + t.Run("different queries", func(t *testing.T) { + queue := &sync.Map{} + + myTask := &mockTask{ + delay: 1 * time.Second, + expRes: "expect-this", + id: "unique-id", + } + myDupTask := &mockTask{ + delay: 1 * time.Second, + expRes: "not-this", + id: "other-unique-id", + } + + wg := &sync.WaitGroup{} + wg.Add(2) + + var ( + myTaskRes *mockTask + myDupTaskRes *mockTask + ) + + // Both queries will run at the same time, the second one will run half a second later + go func() { + myTaskRes = ease(myTask, queue).(*mockTask) + wg.Done() + }() + go func() { + time.Sleep(500 * time.Millisecond) + myDupTaskRes = ease(myDupTask, queue).(*mockTask) + wg.Done() + }() + wg.Wait() + + if myTaskRes.actRes != myTaskRes.expRes { + t.Errorf("expected first query to be executed, expected: %s, actual: %s", + myTaskRes.actRes, myTaskRes.expRes) + } + + if myTaskRes.actRes == myDupTaskRes.actRes { + t.Errorf("expected different result from both tasks, expected: %s, actual: %s", + myTaskRes.actRes, myDupTaskRes.actRes) + } + + if myDupTaskRes.actRes != myDupTaskRes.expRes { + t.Error("expected second query to be executed") + } + }) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..a617ee9 --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/go-gorm/caches + +go 1.16 + +require gorm.io/gorm v1.25.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..f272274 --- /dev/null +++ b/go.sum @@ -0,0 +1,6 @@ +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +gorm.io/gorm v1.25.0 h1:+KtYtb2roDz14EQe4bla8CbQlmb9dN3VejSai3lprfU= +gorm.io/gorm v1.25.0/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= diff --git a/identifier.go b/identifier.go new file mode 100644 index 0000000..e99e0c3 --- /dev/null +++ b/identifier.go @@ -0,0 +1,26 @@ +package caches + +import ( + "fmt" + "gorm.io/gorm/callbacks" + + "gorm.io/gorm" +) + +func buildIdentifier(db *gorm.DB) string { + // Build query identifier, + // for that reason we need to compile all arguments into a string + // and concat them with the SQL query itself + + callbacks.BuildQuerySQL(db) + var ( + identifier string + query string + queryArgs string + ) + query = db.Statement.SQL.String() + queryArgs = fmt.Sprintf("%v", db.Statement.Vars) + identifier = fmt.Sprintf("%s-%s", query, queryArgs) + + return identifier +} diff --git a/identifier_test.go b/identifier_test.go new file mode 100644 index 0000000..68f9c94 --- /dev/null +++ b/identifier_test.go @@ -0,0 +1,20 @@ +package caches + +import ( + "testing" + + "gorm.io/gorm" +) + +func Test_buildIdentifier(t *testing.T) { + db := &gorm.DB{} + db.Statement = &gorm.Statement{} + db.Statement.SQL.WriteString("TEST-SQL") + db.Statement.Vars = append(db.Statement.Vars, "test", 123, 12.3, true, false, []string{"test", "me"}) + + actual := buildIdentifier(db) + expected := "TEST-SQL-[test 123 12.3 true false [test me]]" + if actual != expected { + t.Errorf("buildIdentifier expected to return `%s` but got `%s`", expected, actual) + } +} diff --git a/query_task.go b/query_task.go new file mode 100644 index 0000000..799ef77 --- /dev/null +++ b/query_task.go @@ -0,0 +1,17 @@ +package caches + +import "gorm.io/gorm" + +type queryTask struct { + id string + db *gorm.DB + queryCb func(db *gorm.DB) +} + +func (q *queryTask) GetId() string { + return q.id +} + +func (q *queryTask) Run() { + q.queryCb(q.db) +} diff --git a/query_task_test.go b/query_task_test.go new file mode 100644 index 0000000..e721d97 --- /dev/null +++ b/query_task_test.go @@ -0,0 +1,38 @@ +package caches + +import ( + "sync/atomic" + "testing" + + "gorm.io/gorm" +) + +func TestQueryTask_GetId(t *testing.T) { + task := &queryTask{ + id: "myId", + db: nil, + queryCb: func(db *gorm.DB) { + }, + } + + if task.GetId() != "myId" { + t.Error("GetId on queryTask returned an unexpected value") + } +} + +func TestQueryTask_Run(t *testing.T) { + var inc int32 + task := &queryTask{ + id: "myId", + db: nil, + queryCb: func(db *gorm.DB) { + atomic.AddInt32(&inc, 1) + }, + } + + task.Run() + + if atomic.LoadInt32(&inc) != 1 { + t.Error("Run on queryTask was expected to execute the callback specified once") + } +} diff --git a/reflection.go b/reflection.go new file mode 100644 index 0000000..c28a318 --- /dev/null +++ b/reflection.go @@ -0,0 +1,79 @@ +package caches + +import ( + "errors" + "fmt" + "reflect" + + "gorm.io/gorm/schema" +) + +func SetPointedValue(dest interface{}, src interface{}) { + reflect.ValueOf(dest).Elem().Set(reflect.ValueOf(src).Elem()) +} + +func deepCopy(src, dst interface{}) error { + srcVal := reflect.ValueOf(src) + dstVal := reflect.ValueOf(dst) + + if srcVal.Kind() == reflect.Ptr { + srcVal = srcVal.Elem() + } + + if srcVal.Type() != dstVal.Elem().Type() { + return errors.New("src and dst must be of the same type") + } + + return copyValue(srcVal, dstVal.Elem()) +} + +func copyValue(src, dst reflect.Value) error { + switch src.Kind() { + case reflect.Ptr: + src = src.Elem() + dst.Set(reflect.New(src.Type())) + err := copyValue(src, dst.Elem()) + if err != nil { + return err + } + + case reflect.Struct: + for i := 0; i < src.NumField(); i++ { + if src.Type().Field(i).PkgPath != "" { + return fmt.Errorf("%w: %+v", schema.ErrUnsupportedDataType, src.Type().Field(i).Name) + } + err := copyValue(src.Field(i), dst.Field(i)) + if err != nil { + return err + } + } + + case reflect.Slice: + newSlice := reflect.MakeSlice(src.Type(), src.Len(), src.Cap()) + for i := 0; i < src.Len(); i++ { + err := copyValue(src.Index(i), newSlice.Index(i)) + if err != nil { + return err + } + } + dst.Set(newSlice) + + case reflect.Map: + newMap := reflect.MakeMapWithSize(src.Type(), src.Len()) + for _, key := range src.MapKeys() { + value := src.MapIndex(key) + newValue := reflect.New(value.Type()).Elem() + err := copyValue(value, newValue) + if err != nil { + return err + } + newMap.SetMapIndex(key, newValue) + } + dst.Set(newMap) + + default: + dst.Set(src) + } + + return nil +} diff --git a/reflection_test.go b/reflection_test.go new file mode 100644 index 0000000..7d8683e --- /dev/null +++ b/reflection_test.go @@ -0,0 +1,215 @@ +package caches + +import ( + "reflect" + "testing" +) + +type unsupportedMockStruct struct { + ExportedField string + unexportedField string + ExportedSliceField []string + unexportedSliceField []string + ExportedMapField map[string]string + unexportedMapField map[string]string +} + +type supportedMockStruct struct { + ExportedField string + ExportedSliceField []string + ExportedMapField map[string]string +} + +func Test_SetPointedValue(t *testing.T) { + src := &struct { + Name string + }{ + Name: "Test", + } + + dest := &struct { + Name string + }{} + + SetPointedValue(dest, src) + + if !reflect.DeepEqual(src, dest) { + t.Error("SetPointedValue was expected to point the dest to the source") + } + + if dest.Name != src.Name { + t.Errorf("src and dest were expected to have the same name, src.Name `%s`, dest.Name `%s`", src.Name, dest.Name) + } +} + +func Test_deepCopy(t *testing.T) { + t.Run("struct", func(t *testing.T) { + t.Run("supported", func(t *testing.T) { + srcStruct := supportedMockStruct{ + ExportedField: "exported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + } + dstStruct := supportedMockStruct{} + + if err := deepCopy(srcStruct, &dstStruct); err != nil { + t.Errorf("deepCopy returned an unexpected error %+v", err) + } + + if !reflect.DeepEqual(srcStruct, dstStruct) { + t.Errorf("deepCopy failed to copy structure: got %+v, want %+v", dstStruct, srcStruct) + } + }) + t.Run("unsupported", func(t *testing.T) { + srcStruct := unsupportedMockStruct{ + ExportedField: "exported field", + unexportedField: "unexported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + unexportedSliceField: []string{"1st elem of an unexported slice field", "2nd elem of an unexported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + unexportedMapField: map[string]string{ + "key1": "unexported map elem", + "key2": "unexported map elem", + }, + } + dstStruct := unsupportedMockStruct{} + + if err := deepCopy(srcStruct, &dstStruct); err == nil { + t.Error("deepCopy was expected to fail copying an structure with unexported fields") + } + }) + }) + + t.Run("map", func(t *testing.T) { + t.Run("map[string]string", func(t *testing.T) { + srcMap := map[string]string{ + "key1": "value1", + "key2": "value2", + } + dstMap := make(map[string]string) + + if err := deepCopy(srcMap, &dstMap); err != nil { + t.Errorf("deepCopy returned an unexpected error %+v", err) + } + + if !reflect.DeepEqual(srcMap, dstMap) { + t.Errorf("deepCopy failed to copy map: got %+v, want %+v", dstMap, srcMap) + } + }) + + t.Run("map[string]struct", func(t *testing.T) { + srcMap := map[string]supportedMockStruct{ + "key1": { + ExportedField: "exported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + }, + "key2": { + ExportedField: "exported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + }, + } + dstMap := make(map[string]supportedMockStruct) + + if err := deepCopy(srcMap, &dstMap); err != nil { + t.Errorf("deepCopy returned an unexpected error %+v", err) + } + + if !reflect.DeepEqual(srcMap, dstMap) { + t.Errorf("deepCopy failed to copy map: got %+v, want %+v", dstMap, srcMap) + } + }) + }) + + t.Run("slice", func(t *testing.T) { + t.Run("[]string", func(t *testing.T) { + srcSlice := []string{"A", "B", "C"} + dstSlice := make([]string, len(srcSlice)) + + if err := deepCopy(srcSlice, &dstSlice); err != nil { + t.Errorf("deepCopy returned an unexpected error %+v", err) + } + + if !reflect.DeepEqual(srcSlice, dstSlice) { + t.Errorf("deepCopy failed to copy slice: got %+v, want %+v", dstSlice, srcSlice) + } + }) + t.Run("[]struct", func(t *testing.T) { + srcSlice := []supportedMockStruct{ + { + ExportedField: "exported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + }, { + ExportedField: "exported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + }, { + ExportedField: "exported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + }, + } + dstSlice := make([]supportedMockStruct, len(srcSlice)) + + if err := deepCopy(srcSlice, &dstSlice); err != nil { + t.Errorf("deepCopy returned an unexpected error %+v", err) + } + + if !reflect.DeepEqual(srcSlice, dstSlice) { + t.Errorf("deepCopy failed to copy slice: got %+v, want %+v", dstSlice, srcSlice) + } + }) + }) + + t.Run("pointer", func(t *testing.T) { + srcStruct := &supportedMockStruct{ + ExportedField: "exported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + } + dstStruct := &supportedMockStruct{} + + if err := deepCopy(srcStruct, dstStruct); err != nil { + t.Errorf("deepCopy returned an unexpected error %+v", err) + } + + if !reflect.DeepEqual(srcStruct, dstStruct) { + t.Errorf("deepCopy failed to copy structure: got %+v, want %+v", dstStruct, srcStruct) + } + }) + + t.Run("mismatched", func(t *testing.T) { + src := "a string" + dst := 123 + + if err := deepCopy(src, &dst); err == nil { + t.Error("deepCopy did not return an error when provided mismatched types") + } + }) +} diff --git a/task.go b/task.go new file mode 100644 index 0000000..2b470ff --- /dev/null +++ b/task.go @@ -0,0 +1,6 @@ +package caches + +type task interface { + GetId() string + Run() +} diff --git a/task_test.go b/task_test.go new file mode 100644 index 0000000..431a196 --- /dev/null +++ b/task_test.go @@ -0,0 +1,21 @@ +package caches + +import ( + "time" +) + +type mockTask struct { + delay time.Duration + actRes string + expRes string + id string +} + +func (q *mockTask) GetId() string { + return q.id +} + +func (q *mockTask) Run() { + time.Sleep(q.delay) + q.actRes = q.expRes +}