diff --git a/Makefile b/Makefile index 484f2d24..0c955a5f 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,10 @@ container: gofmt: find . -path ./vendor -prune -o -name '*.go' -print | xargs -L 1 -I % gofmt -s -w % +# Same as gofmt, but also orders imports +goimports: + find . -path ./vendor -prune -o -name '*.go' -print | xargs -L 1 -I % goimports -w % + clean: rm -f kube-monkey diff --git a/chaos/chaos.go b/chaos/chaos.go index de0094d1..500bcfc7 100644 --- a/chaos/chaos.go +++ b/chaos/chaos.go @@ -4,7 +4,7 @@ import ( "fmt" "time" - "github.com/golang/glog" + "github.com/pkg/errors" "github.com/asobti/kube-monkey/config" "github.com/asobti/kube-monkey/kubernetes" @@ -102,14 +102,14 @@ func (c *Chaos) verifyExecution(clientset kube.Interface) error { func (c *Chaos) terminate(clientset kube.Interface) error { killType, err := c.Victim().KillType(clientset) if err != nil { - glog.Errorf("Failed to check KillType label for %s %s. Proceeding with termination of a single pod. Error: %v", c.Victim().Kind(), c.Victim().Name(), err.Error()) - return c.terminatePod(clientset) + return errors.Wrapf(err, "Failed to check KillType label for %s %s", c.Victim().Kind(), c.Victim().Name()) } - killValue, err := c.Victim().KillValue(clientset) - if err != nil { - glog.Errorf("Failed to check KillValue label for %s %s. Proceeding with termination of a single pod. Error: %v", c.Victim().Kind(), c.Victim().Name(), err.Error()) - return c.terminatePod(clientset) + killValue, err := c.getKillValue(clientset) + + // KillAll is the only kill type that does not require a kill-value + if killType != config.KillAllLabelValue && err != nil { + return err } // Validate killtype @@ -117,17 +117,35 @@ func (c *Chaos) terminate(clientset kube.Interface) error { case config.KillFixedLabelValue: return c.Victim().DeleteRandomPods(clientset, killValue) case config.KillAllLabelValue: - killNum := c.Victim().KillNumberForKillingAll(clientset, killValue) + killNum, err := c.Victim().KillNumberForKillingAll(clientset) + if err != nil { + return err + } return c.Victim().DeleteRandomPods(clientset, killNum) case config.KillRandomMaxLabelValue: - killNum := c.Victim().KillNumberForMaxPercentage(clientset, killValue) + killNum, err := c.Victim().KillNumberForMaxPercentage(clientset, killValue) + if err != nil { + return err + } return c.Victim().DeleteRandomPods(clientset, killNum) case config.KillFixedPercentageLabelValue: - killNum := c.Victim().KillNumberForFixedPercentage(clientset, killValue) + killNum, err := c.Victim().KillNumberForFixedPercentage(clientset, killValue) + if err != nil { + return err + } return c.Victim().DeleteRandomPods(clientset, killNum) default: - return fmt.Errorf("Failed to recognize KillType label for %s %s", c.Victim().Kind(), c.Victim().Name()) + return fmt.Errorf("failed to recognize KillType label for %s %s", c.Victim().Kind(), c.Victim().Name()) + } +} + +func (c *Chaos) getKillValue(clientset kube.Interface) (int, error) { + killValue, err := c.Victim().KillValue(clientset) + if err != nil { + return 0, errors.Wrapf(err, "Failed to check KillValue label for %s %s", c.Victim().Kind(), c.Victim().Name()) } + + return killValue, nil } // Redundant for DeleteRandomPods(clientset,1) but DeleteRandomPod is faster diff --git a/chaos/chaos_test.go b/chaos/chaos_test.go index 6d76008c..9c87e3f0 100644 --- a/chaos/chaos_test.go +++ b/chaos/chaos_test.go @@ -63,20 +63,19 @@ func (s *ChaosTestSuite) TestVerifyExecutionWhitelisted() { func (s *ChaosTestSuite) TestTerminateKillTypeError() { v := s.chaos.victim.(*victimMock) - errMsg := "KillType Error" - v.On("KillType", s.client).Return("", errors.New(errMsg)) - v.On("DeleteRandomPod", s.client).Return(nil) - _ = s.chaos.terminate(s.client) + err := errors.New("KillType Error") + v.On("KillType", s.client).Return("", err) + + s.NotNil(s.chaos.terminate(s.client)) v.AssertExpectations(s.T()) } func (s *ChaosTestSuite) TestTerminateKillValueError() { v := s.chaos.victim.(*victimMock) errMsg := "KillValue Error" - v.On("KillType", s.client).Return("", nil) + v.On("KillType", s.client).Return(config.KillFixedLabelValue, nil) v.On("KillValue", s.client).Return(0, errors.New(errMsg)) - v.On("DeleteRandomPod", s.client).Return(nil) - _ = s.chaos.terminate(s.client) + s.NotNil(s.chaos.terminate(s.client)) v.AssertExpectations(s.T()) } @@ -92,10 +91,9 @@ func (s *ChaosTestSuite) TestTerminateKillFixed() { func (s *ChaosTestSuite) TestTerminateAllPods() { v := s.chaos.victim.(*victimMock) - killValue := 1 v.On("KillType", s.client).Return(config.KillAllLabelValue, nil) - v.On("KillValue", s.client).Return(killValue, nil) - v.On("KillNumberForKillingAll", s.client, killValue).Return(0) + v.On("KillValue", s.client).Return(0, nil) + v.On("KillNumberForKillingAll", s.client).Return(0, nil) v.On("DeleteRandomPods", s.client, 0).Return(nil) _ = s.chaos.terminate(s.client) v.AssertExpectations(s.T()) @@ -106,7 +104,7 @@ func (s *ChaosTestSuite) TestTerminateKillRandomMaxPercentage() { killValue := 1 v.On("KillType", s.client).Return(config.KillRandomMaxLabelValue, nil) v.On("KillValue", s.client).Return(killValue, nil) - v.On("KillNumberForMaxPercentage", s.client, mock.AnythingOfType("int")).Return(0) + v.On("KillNumberForMaxPercentage", s.client, mock.AnythingOfType("int")).Return(0, nil) v.On("DeleteRandomPods", s.client, 0).Return(nil) _ = s.chaos.terminate(s.client) v.AssertExpectations(s.T()) @@ -117,7 +115,7 @@ func (s *ChaosTestSuite) TestTerminateKillFixedPercentage() { killValue := 1 v.On("KillType", s.client).Return(config.KillFixedPercentageLabelValue, nil) v.On("KillValue", s.client).Return(killValue, nil) - v.On("KillNumberForFixedPercentage", s.client, mock.AnythingOfType("int")).Return(0) + v.On("KillNumberForFixedPercentage", s.client, mock.AnythingOfType("int")).Return(0, nil) v.On("DeleteRandomPods", s.client, 0).Return(nil) _ = s.chaos.terminate(s.client) v.AssertExpectations(s.T()) @@ -125,12 +123,27 @@ func (s *ChaosTestSuite) TestTerminateKillFixedPercentage() { func (s *ChaosTestSuite) TestInvalidKillType() { v := s.chaos.victim.(*victimMock) - killValue := 1 v.On("KillType", s.client).Return("InvalidKillTypeHere", nil) - v.On("KillValue", s.client).Return(killValue, nil) + v.On("KillValue", s.client).Return(0, nil) err := s.chaos.terminate(s.client) v.AssertExpectations(s.T()) - s.EqualError(err, "Failed to recognize KillType label for Pod "+v.Name()+"") + s.NotNil(err) +} + +func (s *ChaosTestSuite) TestGetKillValue() { + v := s.chaos.victim.(*victimMock) + killValue := 5 + v.On("KillValue", s.client).Return(killValue, nil) + result, err := s.chaos.getKillValue(s.client) + s.Nil(err) + s.Equal(killValue, result) +} + +func (s *ChaosTestSuite) TestGetKillValueReturnsError() { + v := s.chaos.victim.(*victimMock) + v.On("KillValue", s.client).Return(0, errors.New("InvalidKillValue")) + _, err := s.chaos.getKillValue(s.client) + s.NotNil(err) } func (s *ChaosTestSuite) TestDurationToKillTime() { diff --git a/chaos/chaosmock.go b/chaos/chaosmock.go index 59ddeb80..461f1336 100644 --- a/chaos/chaosmock.go +++ b/chaos/chaosmock.go @@ -46,19 +46,19 @@ func (vm *victimMock) DeleteRandomPods(clientset kube.Interface, killValue int) return args.Error(0) } -func (vm *victimMock) KillNumberForKillingAll(clientset kube.Interface, killValue int) int { - args := vm.Called(clientset, killValue) - return args.Int(0) +func (vm *victimMock) KillNumberForKillingAll(clientset kube.Interface) (int, error) { + args := vm.Called(clientset) + return args.Int(0), args.Error(1) } -func (vm *victimMock) KillNumberForMaxPercentage(clientset kube.Interface, killValue int) int { +func (vm *victimMock) KillNumberForMaxPercentage(clientset kube.Interface, killValue int) (int, error) { args := vm.Called(clientset, killValue) - return args.Int(0) + return args.Int(0), args.Error(1) } -func (vm *victimMock) KillNumberForFixedPercentage(clientset kube.Interface, killValue int) int { +func (vm *victimMock) KillNumberForFixedPercentage(clientset kube.Interface, killValue int) (int, error) { args := vm.Called(clientset, killValue) - return args.Int(0) + return args.Int(0), args.Error(1) } func (vm *victimMock) IsBlacklisted() bool { diff --git a/victims/victims.go b/victims/victims.go index e2142bb5..65b06db0 100644 --- a/victims/victims.go +++ b/victims/victims.go @@ -55,9 +55,9 @@ type VictimAPICalls interface { } type VictimKillNumberGenerator interface { - KillNumberForMaxPercentage(kube.Interface, int) int - KillNumberForKillingAll(kube.Interface, int) int - KillNumberForFixedPercentage(kube.Interface, int) int + KillNumberForMaxPercentage(kube.Interface, int) (int, error) + KillNumberForKillingAll(kube.Interface) (int, error) + KillNumberForFixedPercentage(kube.Interface, int) (int, error) } type VictimBase struct { @@ -166,35 +166,33 @@ func (v *VictimBase) DeleteRandomPods(clientset kube.Interface, killNum int) err switch { case numPods == 0: return fmt.Errorf("%s %s has no running pods at the moment", v.kind, v.name) + case killNum == 0: + return fmt.Errorf("no terminations requested for %s %s", v.kind, v.name) case numPods < killNum: glog.Warningf("%s %s has only %d currently running pods, but %d terminations requested", v.kind, v.name, numPods, killNum) fallthrough case numPods == killNum: glog.V(6).Infof("Killing ALL %d running pods for %s %s", numPods, v.kind, v.name) - case killNum == 0: - return fmt.Errorf("No terminations requested for %s %s", v.kind, v.name) case killNum < 0: - return fmt.Errorf("Cannot request negative terminations %d for %s %s", numPods, v.kind, v.name) + return fmt.Errorf("cannot request negative terminations %d for %s %s", killNum, v.kind, v.name) case numPods > killNum: - glog.V(6).Infof("Killing %d running pods for %s %s", numPods, v.kind, v.name) + glog.V(6).Infof("Killing %d running pods for %s %s", killNum, v.kind, v.name) default: return fmt.Errorf("unexpected behavior for terminating %s %s", v.kind, v.name) } r := rand.New(rand.NewSource(time.Now().UnixNano())) - killCount := 0 - for _, i := range r.Perm(numPods) { - if killCount == killNum { - // Report success - return nil - } - targetPod := pods[i].Name - glog.V(6).Infof("Terminating pod %s for %s %s\n", targetPod, v.kind, v.name) + + for i := 0; i < killNum; i++ { + victimIndex := r.Intn(numPods) + targetPod := pods[victimIndex].Name + + glog.V(6).Infof("Terminating pod %s for %s %s/%s\n", targetPod, v.kind, v.namespace, v.name) + err = v.DeletePod(clientset, targetPod) if err != nil { return err } - killCount++ } // Successful termination @@ -264,69 +262,67 @@ func RandomPodName(pods []v1.Pod) string { } // Returns the number of pods to kill based on the number of all running pods -func (v *VictimBase) KillNumberForKillingAll(clientset kube.Interface, killPercentage int) int { - killNum := v.numberOfRunningPods(clientset) +func (v *VictimBase) KillNumberForKillingAll(clientset kube.Interface) (int, error) { + killNum, err := v.numberOfRunningPods(clientset) + if err != nil { + return 0, err + } - return killNum + return killNum, nil } // Returns the number of pods to kill based on a kill percentage and the number of running pods -func (v *VictimBase) KillNumberForFixedPercentage(clientset kube.Interface, killPercentage int) int { +func (v *VictimBase) KillNumberForFixedPercentage(clientset kube.Interface, killPercentage int) (int, error) { if killPercentage == 0 { glog.V(6).Infof("Not terminating any pods for %s %s as kill percentage is 0\n", v.kind, v.name) // Report success - return 0 + return 0, nil } - if killPercentage < 0 { - glog.V(6).Infof("Expected kill percentage config %d to be between 0 and 100 for %s %s. Defaulting to 0", killPercentage, v.kind, v.name) - killPercentage = 0 - } - if killPercentage > 100 { - glog.V(6).Infof("Expected kill percentage config %d to be between 0 and 100 for %s %s. Defaulting to 100", killPercentage, v.kind, v.name) - killPercentage = 100 + if killPercentage < 0 || killPercentage > 100 { + return 0, fmt.Errorf("percentage value of %d is invalid. Must be [0-100]", killPercentage) } - numRunningPods := v.numberOfRunningPods(clientset) + numRunningPods, err := v.numberOfRunningPods(clientset) + if err != nil { + return 0, err + } numberOfPodsToKill := float64(numRunningPods) * float64(killPercentage) / 100 killNum := int(math.Floor(numberOfPodsToKill)) - return killNum + return killNum, nil } // Returns a number of pods to kill based on a a random kill percentage (between 0 and maxPercentage) and the number of running pods -func (v *VictimBase) KillNumberForMaxPercentage(clientset kube.Interface, maxPercentage int) int { +func (v *VictimBase) KillNumberForMaxPercentage(clientset kube.Interface, maxPercentage int) (int, error) { if maxPercentage == 0 { - glog.V(6).Infof("Not terminating any pods for %s %s as kill percentage is 0\n", v.kind, v.name) + glog.V(6).Infof("Not terminating any pods for %s %s as kill percentage is 0", v.kind, v.name) // Report success - return 0 + return 0, nil } - if maxPercentage < 0 { - glog.V(6).Infof("Expected kill percentage config %d to be between 0 and 100 for %s %s. Defaulting to 0%%", maxPercentage, v.kind, v.name) - maxPercentage = 0 - } - if maxPercentage > 100 { - glog.V(6).Infof("Expected kill percentage config %d to be between 0 and 100 for %s %s. Defaulting to 100%%", maxPercentage, v.kind, v.name) - maxPercentage = 100 + if maxPercentage < 0 || maxPercentage > 100 { + return 0, fmt.Errorf("percentage value of %d is invalid. Must be [0-100]", maxPercentage) } - numRunningPods := v.numberOfRunningPods(clientset) + numRunningPods, err := v.numberOfRunningPods(clientset) + if err != nil { + return 0, err + } r := rand.New(rand.NewSource(time.Now().UnixNano())) killPercentage := r.Intn(maxPercentage + 1) // + 1 because Intn works with half open interval [0,n) and we want [0,n] numberOfPodsToKill := float64(numRunningPods) * float64(killPercentage) / 100 killNum := int(math.Floor(numberOfPodsToKill)) - return killNum + return killNum, nil } // Returns the number of running pods or 0 if the operation fails -func (v *VictimBase) numberOfRunningPods(clientset kube.Interface) int { +func (v *VictimBase) numberOfRunningPods(clientset kube.Interface) (int, error) { pods, err := v.RunningPods(clientset) if err != nil { - glog.V(6).Infof("Failed to get list of running pods %s %s", v.kind, v.name) - return 0 + return 0, errors.Wrapf(err, "Failed to get running pods for victim %s %s", v.kind, v.name) } - return len(pods) + return len(pods), nil } diff --git a/victims/victims_test.go b/victims/victims_test.go index b2c7c325..e874dff8 100644 --- a/victims/victims_test.go +++ b/victims/victims_test.go @@ -41,7 +41,20 @@ func newPod(name string, status v1.PodPhase) v1.Pod { Phase: status, }, } +} + +func generateNPods(namePrefix string, n int, status v1.PodPhase) []runtime.Object { + var pods []runtime.Object + for i := 0; i < n; i++ { + pod := newPod(fmt.Sprintf("%s%d", namePrefix, i), status) + pods = append(pods, &pod) + } + + return pods +} +func generateNRunningPods(namePrefix string, n int) []runtime.Object { + return generateNPods(namePrefix, n, v1.PodRunning) } func newVictimBase() *VictimBase { @@ -137,10 +150,10 @@ func TestDeleteRandomPods(t *testing.T) { assert.Lenf(t, podList, 3, "Expected 3 items in podList, got %d", len(podList)) err := v.DeleteRandomPods(client, 0) - assert.EqualError(t, err, "No terminations requested for Pod name") + assert.NotNil(t, err, "expected err for killNum=0 but got nil") err = v.DeleteRandomPods(client, -1) - assert.EqualError(t, err, "Cannot request negative terminations 2 for Pod name") + assert.NotNil(t, err, "expected err for negative terminations but got nil") _ = v.DeleteRandomPods(client, 1) podList = getPodList(client).Items @@ -156,83 +169,130 @@ func TestDeleteRandomPods(t *testing.T) { assert.EqualError(t, err, KIND+" "+NAME+" has no running pods at the moment") } -func TestInvalidInputsForDeletePodsRandomMaxPercentage(t *testing.T) { - - v := newVictimBase() - - var pods []runtime.Object - for i := 0; i < 100; i++ { - pod := newPod(fmt.Sprintf("app%d", i), v1.PodRunning) - pods = append(pods, &pod) - } - - client := fake.NewSimpleClientset(pods...) - - killNum := v.KillNumberForMaxPercentage(client, -1) - assert.Equalf(t, 0, killNum, "Should default to 0 percent when percentage has a negative value, got %d", killNum) - - killNum = v.KillNumberForMaxPercentage(client, 101) - assert.Truef(t, killNum > 0 && killNum <= 100, "Should default to 100 percent pods when percentage is greater than 100, got %d", killNum) -} - func TestKillNumberForMaxPercentage(t *testing.T) { v := newVictimBase() - var pods []runtime.Object - for i := 0; i < 100; i++ { - pod := newPod(fmt.Sprintf("app%d", i), v1.PodRunning) - pods = append(pods, &pod) - } + pods := generateNRunningPods("app", 100) client := fake.NewSimpleClientset(pods...) - killNum := v.KillNumberForMaxPercentage(client, 0) // 0% means we don't kill any pods - assert.Equal(t, killNum, 0, "Expected 0 pods to be killed, got %d", killNum) - - killNum = v.KillNumberForMaxPercentage(client, 50) // 50% means we kill between at most 50 pods of the 100 that are running + killNum, err := v.KillNumberForMaxPercentage(client, 50) // 50% means we kill between at most 50 pods of the 100 that are running + assert.Nil(t, err, "Expected err to be nil but got %v", err) assert.Truef(t, killNum >= 0 && killNum <= 50, "Expected kill number between 0 and 50 pods, got %d", killNum) } -func TestInvalidInputsForDeletePodsFixedMaxPercentage(t *testing.T) { - - v := newVictimBase() +func TestKillNumberForMaxPercentageInvalidValues(t *testing.T) { + type TestCase struct { + name string + maxPercentage int + expectedNum int + expectedErr bool + } - var pods []runtime.Object - for i := 0; i < 100; i++ { - pod := newPod(fmt.Sprintf("app%d", i), v1.PodRunning) - pods = append(pods, &pod) + tcs := []TestCase{ + { + name: "Negative value for maxPercentage", + maxPercentage: -1, + expectedNum: 0, + expectedErr: true, + }, + { + name: "0 value for maxPercentage", + maxPercentage: 0, + expectedNum: 0, + expectedErr: false, + }, + { + name: "maxPercentage greater than 100", + maxPercentage: 110, + expectedNum: 0, + expectedErr: true, + }, } - client := fake.NewSimpleClientset(pods...) + for _, tc := range tcs { + v := newVictimBase() + client := fake.NewSimpleClientset() - killNum := v.KillNumberForFixedPercentage(client, -1) - assert.Equalf(t, 0, killNum, "Should default to 0 percent when percentage has a negative value, got %d", killNum) + result, err := v.KillNumberForMaxPercentage(client, tc.maxPercentage) - killNum = v.KillNumberForFixedPercentage(client, 101) - assert.Equalf(t, 100, killNum, "Should default to 100 percent when percentage is greater than 100, got %d", killNum) + if tc.expectedErr { + assert.NotNil(t, err, tc.name) + } else { + assert.Nil(t, err, tc.name) + assert.Equal(t, result, tc.expectedNum, tc.name) + } + } } func TestDeletePodsFixedPercentage(t *testing.T) { + type TestCase struct { + name string + killPercentage int + pods []runtime.Object + expectedNum int + expectedErr bool + } - v := newVictimBase() - pod1 := newPod("app1", v1.PodRunning) - pod2 := newPod("app2", v1.PodPending) // not running - pod3 := newPod("app3", v1.PodRunning) - pod4 := newPod("app4", v1.PodRunning) - pod5 := newPod("app5", v1.PodRunning) - pod6 := newPod("app6", v1.PodRunning) + tcs := []TestCase{ + { + name: "negative value for killPercentage", + killPercentage: -1, + expectedNum: 0, + expectedErr: true, + }, + { + name: "0 value for killPercentage", + killPercentage: 0, + expectedNum: 0, + expectedErr: false, + }, + { + name: "killPercentage greater than 100", + killPercentage: 110, + expectedNum: 0, + expectedErr: true, + }, + { + name: "correctly calculates pods to kill based on killPercentage", + killPercentage: 50, + pods: generateNRunningPods("app", 10), + expectedNum: 5, + expectedErr: false, + }, + { + name: "correctly floors fractional values for the number of pods to kill", + killPercentage: 33, + pods: generateNRunningPods("app", 10), + expectedNum: 3, + expectedErr: false, + }, + { + name: "does not count pending pods when calculating num of pods to kill", + killPercentage: 80, + pods: append( + generateNPods("running", 1, v1.PodRunning), + generateNPods("pending", 1, v1.PodPending)...), + expectedNum: 0, + expectedErr: false, + }, + } - client := fake.NewSimpleClientset(&pod1, &pod2, &pod3, &pod4, &pod5, &pod6) + for _, tc := range tcs { + client := fake.NewSimpleClientset(tc.pods...) + v := newVictimBase() - killNum := v.KillNumberForFixedPercentage(client, 0) // 0% means we don't kill any pods - assert.Equalf(t, killNum, 0, "Expected 0 pods to be killed, got %d", killNum) + result, err := v.KillNumberForFixedPercentage(client, tc.killPercentage) - killNum = v.KillNumberForFixedPercentage(client, 50) // 50% means we kill 2 (rounded down from 2.5) out of 5 running pods - assert.Equalf(t, killNum, 2, "Expected 2 pods to be killed, got %d", killNum) + if tc.expectedErr { + assert.NotNil(t, err, tc.name) + } else { + assert.Nil(t, err, tc.name) + assert.Equal(t, tc.expectedNum, result, tc.name) + } + } - killNum = v.KillNumberForFixedPercentage(client, 100) // 100% means we kill all 6 running pods - assert.Equalf(t, killNum, 5, "Expected 5 pods to be killed, got %d", killNum) } func TestDeleteRandomPod(t *testing.T) {