From d627811ea9a3824e7173be1e072b61e3ff0f1156 Mon Sep 17 00:00:00 2001 From: Kristina Date: Thu, 17 Jan 2019 09:53:27 -0800 Subject: [PATCH] Db Package Updates - Added Comments - Did some linting fixes - Split up insert to call helper functions - Added some unit tests; some difficult due to no interfaces in gocb itself --- Gopkg.lock | 56 +++++++++-- db/db.go | 137 +++++++++++++++----------- db/db_test.go | 246 +++++++++++++++++++++++++++++++++++++++++++++++ db/mocks_test.go | 59 ++++++++++++ 4 files changed, 434 insertions(+), 64 deletions(-) create mode 100644 db/db_test.go create mode 100644 db/mocks_test.go diff --git a/Gopkg.lock b/Gopkg.lock index 28c376f..667d4b5 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1,6 +1,18 @@ # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. +[[projects]] + digest = "1:142cf0c63cd0fc3cc0fdd26854cb028bb33d569f1b80de05f353cd45850e688b" + name = "github.com/DATA-DOG/godog" + packages = [ + ".", + "colors", + "gherkin", + ] + pruneopts = "UT" + revision = "75562b12d67aa758aba65b0072c763bccf7e9b00" + version = "v0.7.9" + [[projects]] digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" name = "github.com/davecgh/go-spew" @@ -41,6 +53,22 @@ revision = "4cdd86c173cfed1f47be88bd88327140f81bcede" version = "v0.16.0" +[[projects]] + digest = "1:c79fb010be38a59d657c48c6ba1d003a8aa651fa56b579d959d74573b7dff8e1" + name = "github.com/gorilla/context" + packages = ["."] + pruneopts = "UT" + revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42" + version = "v1.1.1" + +[[projects]] + digest = "1:e73f5b0152105f18bc131fba127d9949305c8693f8a762588a82a48f61756f5f" + name = "github.com/gorilla/mux" + packages = ["."] + pruneopts = "UT" + revision = "e3702bed27f0d39777b0b37b664b6280e8ef8fbf" + version = "v1.6.2" + [[projects]] digest = "1:0ade334594e69404d80d9d323445d2297ff8161637f9b2d347cc6973d2d6f05b" name = "github.com/hashicorp/errwrap" @@ -246,9 +274,20 @@ version = "v1.3.1" [[projects]] - digest = "1:972c2427413d41a1e06ca4897e8528e5a1622894050e2f527b38ddf0f343f759" + digest = "1:ac83cf90d08b63ad5f7e020ef480d319ae890c208f8524622a2f3136e2686b02" + name = "github.com/stretchr/objx" + packages = ["."] + pruneopts = "UT" + revision = "477a77ecc69700c7cdeb1fa9e129548e1c1c393c" + version = "v0.1.1" + +[[projects]] + digest = "1:0bcc464dabcfad5393daf87c3f8142911d0f6c52569b837e91a1c15e890265f3" name = "github.com/stretchr/testify" - packages = ["assert"] + packages = [ + "assert", + "mock", + ] pruneopts = "UT" revision = "ffdc059bfe9ce6a4e144ba849dbedead332c6053" version = "v1.3.0" @@ -269,11 +308,11 @@ [[projects]] branch = "master" - digest = "1:fe2af5c0e6b4188bb1907e051cd086dae4f7ab3a2f4c1b62c03fefca848ab900" + digest = "1:f6f6d9d6f80c9e32acc0f4f3b904e0b9f5b18a7757c2ef941121bd79a10e7bdf" name = "golang.org/x/sys" packages = ["unix"] pruneopts = "UT" - revision = "a457fd036447854c0c02e89ea439481bdcf941a2" + revision = "11f53e03133963fb11ae0588e08b5e0b85be8be5" [[projects]] digest = "1:a2ab62866c75542dd18d2b069fec854577a20211d7c0ea6ae746072a1dccdd18" @@ -315,12 +354,12 @@ version = "v1.5.2" [[projects]] - digest = "1:bce1f41989cec84a73eb4eb37cce730ccada525ed3df565f4edf9a916be3d65b" + digest = "1:95fc600062cee3516d382b9c4d6ef4b039b4a796110c9adaafc2566815e10ed6" name = "gopkg.in/couchbase/gocbcore.v7" packages = ["."] pruneopts = "UT" - revision = "168eae0b31c401fbefc79e9a9d4354e19ec50ad7" - version = "v7.1.10" + revision = "d6f6cc0c51cba2983dbff50e106394b755f847c4" + version = "v7.1.11" [[projects]] digest = "1:002e24bae6c5c2d00148cf04983fc5add942f5a929d38721d2f377dd7b5b6da9" @@ -350,10 +389,13 @@ analyzer-name = "dep" analyzer-version = 1 input-imports = [ + "github.com/DATA-DOG/godog", "github.com/goph/emperror", + "github.com/gorilla/mux", "github.com/hashicorp/vault/api", "github.com/spf13/viper", "github.com/stretchr/testify/assert", + "github.com/stretchr/testify/mock", "gopkg.in/couchbase/gocb.v1", ] solver-name = "gps-cdcl" diff --git a/db/db.go b/db/db.go index a0d0fd7..632e77f 100644 --- a/db/db.go +++ b/db/db.go @@ -26,15 +26,25 @@ import ( "time" ) +// Interface describes the main functionality needed to connect to a db type Interface interface { Initialize() error - GetHistory(deviceId string) (History, error) - GetTombstone(deviceId string) (map[string]Event, error) - UpdateHistory(deviceId string, events []Event) error - InsertEvent(deviceId string, event Event, tombstoneKey string) error + GetHistory(deviceI string) (History, error) + GetTombstone(deviceID string) (map[string]Event, error) + UpdateHistory(deviceID string, events []Event) error + InsertEvent(deviceID string, event Event, tombstoneKey string) error RemoveAll() error } +type bucketWrapper interface { + Manager(username, password string) *gocb.BucketManager + Get(key string, valuePtr interface{}) (gocb.Cas, error) + MutateIn(key string, cas gocb.Cas, expiry uint32) *gocb.MutateInBuilder + Counter(key string, delta, initial int64, expiry uint32) (uint64, gocb.Cas, error) + Insert(key string, value interface{}, expiry uint32) (gocb.Cas, error) + ExecuteN1qlQuery(q *gocb.N1qlQuery, params interface{}) (gocb.QueryResults, error) +} + // the prefixes for the different documents being stored in couchbase const ( historyDoc = "history" @@ -42,10 +52,15 @@ const ( tombstoneDoc = "tombstone" ) +var ( + errInvaliddeviceID = errors.New("Invalid device ID") + errInvalidEvent = errors.New("Invalid event") +) + // TODO: Add a way to try to reconnect to the database after a command fails because the connection broke -// DbConnection contains the bucket connection and configuration values -type DbConnection struct { +// Connection contains the bucket connection and configuration values +type Connection struct { Server string Username string Password string @@ -54,7 +69,7 @@ type DbConnection struct { NumRetries int // the time duration to add when creating TTLs for history documents Timeout time.Duration - bucketConn *gocb.Bucket + bucketConn bucketWrapper } // History is a list of events related to a device id, @@ -73,7 +88,7 @@ type Event struct { // the id for the event // // required: true - Id string `json:"id"` + ID string `json:"id"` // the time this event was found // @@ -112,7 +127,7 @@ type Event struct { } // Initialize creates the connection with couchbase and opens the specified bucket -func (db *DbConnection) Initialize() error { +func (db *Connection) Initialize() error { var err error cluster, err := gocb.Connect("couchbase://" + db.Server) @@ -152,122 +167,130 @@ func (db *DbConnection) Initialize() error { } // GetHistory returns the history (list of events) for a given device -func (db *DbConnection) GetHistory(deviceId string) (History, error) { +func (db *Connection) GetHistory(deviceID string) (History, error) { var ( deviceInfo History ) - if deviceId == "" { + if deviceID == "" { return History{}, emperror.WrapWith(errors.New("Invalid device id"), "Get history not attempted", - "device id", deviceId) + "device id", deviceID) } - key := strings.Join([]string{historyDoc, deviceId}, ":") + key := strings.Join([]string{historyDoc, deviceID}, ":") _, err := db.bucketConn.Get(key, &deviceInfo) if err != nil { - return History{}, emperror.WrapWith(err, "Getting history from database failed", "device id", deviceId) + return History{}, emperror.WrapWith(err, "Getting history from database failed", "device id", deviceID) } return deviceInfo, nil } // GetTombstone returns the tombstone (map of events) for a given device -func (db *DbConnection) GetTombstone(deviceId string) (map[string]Event, error) { +func (db *Connection) GetTombstone(deviceID string) (map[string]Event, error) { var ( deviceInfo map[string]Event ) - if deviceId == "" { + if deviceID == "" { return map[string]Event{}, emperror.WrapWith(errors.New("Invalid device id"), "Get tombstone not attempted", - "device id", deviceId) + "device id", deviceID) } - key := strings.Join([]string{tombstoneDoc, deviceId}, ":") + key := strings.Join([]string{tombstoneDoc, deviceID}, ":") _, err := db.bucketConn.Get(key, &deviceInfo) if err != nil { - return map[string]Event{}, emperror.WrapWith(err, "Getting tombstone from database failed", "device id", deviceId) + return map[string]Event{}, emperror.WrapWith(err, "Getting tombstone from database failed", "device id", deviceID) } return deviceInfo, nil } // UpdateHistory updates the history to the list of events given for a given device -func (db *DbConnection) UpdateHistory(deviceId string, events []Event) error { - key := strings.Join([]string{historyDoc, deviceId}, ":") +func (db *Connection) UpdateHistory(deviceID string, events []Event) error { + key := strings.Join([]string{historyDoc, deviceID}, ":") newTimeout := uint32(time.Now().Add(db.Timeout).Unix()) _, err := db.bucketConn.MutateIn(key, 0, newTimeout).Upsert("events", &events, false).Execute() if err != nil { - return emperror.WrapWith(err, "Update history failed", "device id", deviceId, + return emperror.WrapWith(err, "Update history failed", "device id", deviceID, "events", events) } return nil } // InsertEvent adds an event to the history of the given device id and adds it to the tombstone if a key is given -func (db *DbConnection) InsertEvent(deviceId string, event Event, tombstoneMapKey string) error { - if valid, err := isEventValid(deviceId, event); !valid { - return emperror.WrapWith(err, "Insert event not attempted", "device id", deviceId, +func (db *Connection) InsertEvent(deviceID string, event Event, tombstoneMapKey string) error { + if valid, err := isEventValid(deviceID, event); !valid { + return emperror.WrapWith(err, "Insert event not attempted", "device id", deviceID, "event", event) } - // get event id using the device id - counterKey := strings.Join([]string{counterDoc, deviceId}, ":") + counterKey := strings.Join([]string{counterDoc, deviceID}, ":") eventID, _, err := db.bucketConn.Counter(counterKey, 1, 0, 0) if err != nil { - return emperror.WrapWith(err, "Failed to get event id", "device id", deviceId) + return emperror.WrapWith(err, "Failed to get event id", "device id", deviceID) } - - event.Id = strconv.FormatUint(eventID, 10) + event.ID = strconv.FormatUint(eventID, 10) //if tombstonekey isn't empty string, then set the tombstone map at that key if tombstoneMapKey != "" { - tombstoneKey := strings.Join([]string{tombstoneDoc, deviceId}, ":") - events := make(map[string]Event) - events[tombstoneMapKey] = event - _, err = db.bucketConn.Insert(tombstoneKey, &events, 0) - if err != nil && err != gocb.ErrKeyExists { - return emperror.WrapWith(err, "Failed to create tombstone", "device id", deviceId, - "event id", eventID, "event", event) + err = db.upsertToTombstone(deviceID, tombstoneMapKey, event) + if err != nil { + return err } + } + // append to the history, create if it doesn't exist + err = db.upsertToHistory(deviceID, event) + return err +} + +func (db *Connection) upsertToTombstone(deviceID string, tombstoneMapKey string, event Event) error { + tombstoneKey := strings.Join([]string{tombstoneDoc, deviceID}, ":") + events := map[string]Event{tombstoneMapKey: event} + _, err := db.bucketConn.Insert(tombstoneKey, &events, 0) + if err != nil && err != gocb.ErrKeyExists { + return emperror.WrapWith(err, "Failed to create tombstone", "device id", deviceID, + "event id", event.ID, "event", event) + } + if err != nil { + _, err = db.bucketConn.MutateIn(tombstoneKey, 0, 0). + Upsert(tombstoneMapKey, &event, false). + Execute() if err != nil { - _, err = db.bucketConn.MutateIn(tombstoneKey, 0, 0). - Upsert(tombstoneMapKey, &event, false). - Execute() - if err != nil { - return emperror.WrapWith(err, "Failed to add event to tombstone", "device id", deviceId, - "event id", eventID, "event", event) - } + return emperror.WrapWith(err, "Failed to add event to tombstone", "device id", deviceID, + "event id", event.ID, "event", event) } } + return nil +} - // append to the history, create if it doesn't exist +func (db *Connection) upsertToHistory(deviceID string, event Event) error { newTimeout := uint32(time.Now().Add(db.Timeout).Unix()) - historyKey := strings.Join([]string{historyDoc, deviceId}, ":") + historyKey := strings.Join([]string{historyDoc, deviceID}, ":") eventDoc := History{ Events: []Event{event}, } - _, err = db.bucketConn.Insert(historyKey, &eventDoc, newTimeout) + _, err := db.bucketConn.Insert(historyKey, &eventDoc, newTimeout) if err != nil && err != gocb.ErrKeyExists { - return emperror.WrapWith(err, "Failed to create history document", "device id", deviceId, - "event id", eventID, "event", event) + return emperror.WrapWith(err, "Failed to create history document", "device id", deviceID, + "event id", event.ID, "event", event) } if err != nil { _, err = db.bucketConn.MutateIn(historyKey, 0, newTimeout).ArrayPrepend("events", &event, false).Execute() if err != nil { - return emperror.WrapWith(err, "Failed to add event to history", "device id", deviceId, - "event id", eventID, "event", event) + return emperror.WrapWith(err, "Failed to add event to history", "device id", deviceID, + "event id", event.ID, "event", event) } } - return nil } -func isEventValid(deviceId string, event Event) (bool, error) { - if deviceId == "" { - return false, errors.New("Invalid device id") +func isEventValid(deviceID string, event Event) (bool, error) { + if deviceID == "" { + return false, errInvaliddeviceID } if event.Source == "" || event.Destination == "" || len(event.Details) == 0 { - return false, errors.New("Invalid event") + return false, errInvalidEvent } return true, nil } // RemoveAll removes everything in the database. Used for testing -func (db *DbConnection) RemoveAll() error { +func (db *Connection) RemoveAll() error { _, err := db.bucketConn.ExecuteN1qlQuery(gocb.NewN1qlQuery("DELETE FROM devices;"), nil) if err != nil { return emperror.Wrap(err, "Removing all devices from database failed") diff --git a/db/db_test.go b/db/db_test.go new file mode 100644 index 0000000..dfda0bb --- /dev/null +++ b/db/db_test.go @@ -0,0 +1,246 @@ +/** + * Copyright 2019 Comcast Cable Communications Management, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package db + +import ( + "encoding/json" + "errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "gopkg.in/couchbase/gocb.v1" + "testing" +) + +var ( + goodEvent = Event{ + Source: "test source", + Destination: "test destination", + Details: map[string]interface{}{"test key": "test value"}, + } +) + +func TestIsEventValid(t *testing.T) { + tests := []struct { + description string + deviceID string + event Event + expectedValidity bool + expectedErr error + }{ + { + description: "Success", + deviceID: "1234", + event: Event{ + Source: "test source", + Destination: "test destination", + Details: map[string]interface{}{"test": "test value"}, + }, + expectedValidity: true, + expectedErr: nil, + }, + { + description: "Empty device id error", + deviceID: "", + event: Event{}, + expectedValidity: false, + expectedErr: errInvaliddeviceID, + }, + { + description: "Invalid event error", + deviceID: "1234", + event: Event{}, + expectedValidity: false, + expectedErr: errInvalidEvent, + }, + } + + for _, tc := range tests { + t.Run(tc.description, func(t *testing.T) { + assert := assert.New(t) + valid, err := isEventValid(tc.deviceID, tc.event) + assert.Equal(tc.expectedValidity, valid) + assert.Equal(tc.expectedErr, err) + }) + } + +} + +func TestGetHistory(t *testing.T) { + var cas gocb.Cas + tests := []struct { + description string + deviceID string + expectedHistory History + expectedErr error + expectedCalls int + }{ + { + description: "Success", + deviceID: "1234", + expectedHistory: History{ + Events: []Event{ + { + ID: "1234", + }, + }, + }, + expectedErr: nil, + expectedCalls: 1, + }, + { + description: "Invalid Device Error", + deviceID: "", + expectedHistory: History{}, + expectedErr: errors.New("Invalid device id"), + expectedCalls: 0, + }, + { + description: "Get Error", + deviceID: "1234", + expectedHistory: History{}, + expectedErr: errors.New("test Get error"), + expectedCalls: 1, + }, + } + + for _, tc := range tests { + t.Run(tc.description, func(t *testing.T) { + assert := assert.New(t) + mockObj := new(mockBucket) + dbConnection := Connection{ + bucketConn: mockObj, + } + if tc.expectedCalls > 0 { + marshalledHistory, err := json.Marshal(tc.expectedHistory) + assert.Nil(err) + mockObj.On("Get", mock.Anything, mock.Anything).Return(cas, tc.expectedErr, marshalledHistory).Times(tc.expectedCalls) + } + history, err := dbConnection.GetHistory(tc.deviceID) + mockObj.AssertExpectations(t) + if tc.expectedErr == nil || err == nil { + assert.Equal(tc.expectedErr, err) + } else { + assert.Contains(err.Error(), tc.expectedErr.Error()) + } + assert.Equal(tc.expectedHistory, history) + }) + } +} + +func TestGetTombstone(t *testing.T) { + var cas gocb.Cas + tests := []struct { + description string + deviceID string + expectedTombstone map[string]Event + expectedErr error + expectedCalls int + }{ + { + description: "Success", + deviceID: "1234", + expectedTombstone: map[string]Event{ + "test": { + ID: "1234", + }, + }, + expectedErr: nil, + expectedCalls: 1, + }, + { + description: "Invalid Device Error", + deviceID: "", + expectedTombstone: map[string]Event{}, + expectedErr: errors.New("Invalid device id"), + expectedCalls: 0, + }, + { + description: "Get Error", + deviceID: "1234", + expectedTombstone: map[string]Event{}, + expectedErr: errors.New("test Get error"), + expectedCalls: 1, + }, + } + + for _, tc := range tests { + t.Run(tc.description, func(t *testing.T) { + assert := assert.New(t) + mockObj := new(mockBucket) + dbConnection := Connection{ + bucketConn: mockObj, + } + if tc.expectedCalls > 0 { + marshalledTombstone, err := json.Marshal(tc.expectedTombstone) + assert.Nil(err) + mockObj.On("Get", mock.Anything, mock.Anything).Return(cas, tc.expectedErr, marshalledTombstone).Times(tc.expectedCalls) + } + tombstone, err := dbConnection.GetTombstone(tc.deviceID) + mockObj.AssertExpectations(t) + if tc.expectedErr == nil || err == nil { + assert.Equal(tc.expectedErr, err) + } else { + assert.Contains(err.Error(), tc.expectedErr.Error()) + } + assert.Equal(tc.expectedTombstone, tombstone) + }) + } +} + +func TestUpdateHistory(t *testing.T) { + +} + +func TestInsertEventSuccess(t *testing.T) { + var cas gocb.Cas + assert := assert.New(t) + mockObj := new(mockBucket) + dbConnection := Connection{ + bucketConn: mockObj, + } + mockObj.On("Counter", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(uint64(1), cas, nil).Once() + mockObj.On("Insert", mock.Anything, mock.Anything, mock.Anything).Return(cas, nil).Twice() + err := dbConnection.InsertEvent("1234", goodEvent, "test") + mockObj.AssertExpectations(t) + assert.Nil(err) +} + +func TestInsertEventCounterFail(t *testing.T) { + var cas gocb.Cas + assert := assert.New(t) + mockObj := new(mockBucket) + dbConnection := Connection{ + bucketConn: mockObj, + } + expectedErr := errors.New("test counter fail") + mockObj.On("Counter", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(uint64(1), cas, expectedErr).Once() + err := dbConnection.InsertEvent("1234", goodEvent, "test") + mockObj.AssertExpectations(t) + assert.NotNil(err) + if err != nil { + assert.Contains(err.Error(), expectedErr.Error()) + } +} + +func TestUpsertToTombstone(t *testing.T) { + +} + +func TestUpsertToHistory(t *testing.T) { + +} diff --git a/db/mocks_test.go b/db/mocks_test.go new file mode 100644 index 0000000..e05f4db --- /dev/null +++ b/db/mocks_test.go @@ -0,0 +1,59 @@ +/** + * Copyright 2019 Comcast Cable Communications Management, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package db + +import ( + "encoding/json" + "github.com/stretchr/testify/mock" + "gopkg.in/couchbase/gocb.v1" +) + +type mockBucket struct { + mock.Mock +} + +func (m *mockBucket) Manager(username, password string) *gocb.BucketManager { + args := m.Called(username, password) + return args.Get(0).(*gocb.BucketManager) +} + +func (m *mockBucket) Get(key string, valuePtr interface{}) (gocb.Cas, error) { + args := m.Called(key, valuePtr) + json.Unmarshal(args.Get(2).([]byte), valuePtr) + return args.Get(0).(gocb.Cas), args.Error(1) +} + +func (m *mockBucket) MutateIn(key string, cas gocb.Cas, expiry uint32) *gocb.MutateInBuilder { + args := m.Called(key, cas, expiry) + return args.Get(0).(*gocb.MutateInBuilder) +} + +func (m *mockBucket) Counter(key string, delta, initial int64, expiry uint32) (uint64, gocb.Cas, error) { + args := m.Called(key, delta, initial, expiry) + return args.Get(0).(uint64), args.Get(1).(gocb.Cas), args.Error(2) +} + +func (m *mockBucket) Insert(key string, value interface{}, expiry uint32) (gocb.Cas, error) { + args := m.Called(key, value, expiry) + return args.Get(0).(gocb.Cas), args.Error(1) +} + +func (m *mockBucket) ExecuteN1qlQuery(q *gocb.N1qlQuery, params interface{}) (gocb.QueryResults, error) { + args := m.Called(q, params) + return args.Get(0).(gocb.QueryResults), args.Error(1) +}