Skip to content

Commit

Permalink
Run karpenter by default in the kube-system namespace
Browse files Browse the repository at this point in the history
  • Loading branch information
jonathan-innis committed Nov 30, 2023
1 parent 504c71e commit 75c31f4
Show file tree
Hide file tree
Showing 8 changed files with 21 additions and 32 deletions.
2 changes: 1 addition & 1 deletion .github/actions/e2e/create-cluster/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ runs:
serviceAccounts:
- metadata:
name: karpenter
namespace: karpenter
namespace: kube-system
attachPolicyARNs:
- "arn:aws:iam::${{ inputs.account_id }}:policy/KarpenterControllerPolicy-${{ inputs.cluster_name }}"
permissionsBoundary: "arn:aws:iam::${{ inputs.account_id }}:policy/GithubActionsPermissionsBoundary"
Expand Down
4 changes: 2 additions & 2 deletions .github/actions/e2e/install-karpenter/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ runs:
fi
helm upgrade --install karpenter oci://${{ inputs.ecr_account_id }}.dkr.ecr.${{ inputs.ecr_region }}.amazonaws.com/karpenter/snapshot/karpenter \
-n karpenter \
-n kube-system \
--version "v0-$(git rev-parse HEAD)" \
--set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:aws:iam::${{ inputs.account_id }}:role/karpenter-irsa-${{ inputs.cluster_name }}" \
--set webhook.enabled=${WEBHOOK_ENABLED} \
Expand All @@ -82,7 +82,7 @@ runs:
- name: diff-karpenter
shell: bash
run: |
helm diff upgrade --namespace karpenter \
helm diff upgrade --namespace kube-system \
karpenter oci://${{ inputs.ecr_account_id }}.dkr.ecr.${{ inputs.ecr_region }}.amazonaws.com/karpenter/snapshot/karpenter \
--version v0-$(git rev-parse HEAD) \
--reuse-values --three-way-merge --detailed-exitcode
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ HELM_OPTS ?= --set serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn=${K
--create-namespace

# CR for local builds of Karpenter
KARPENTER_NAMESPACE ?= karpenter
KARPENTER_NAMESPACE ?= kube-system
KARPENTER_VERSION ?= $(shell git tag --sort=committerdate | tail -1)
KO_DOCKER_REPO ?= ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/dev
GETTING_STARTED_SCRIPT_DIR = website/content/en/preview/getting-started/getting-started-with-karpenter/scripts
Expand Down
12 changes: 0 additions & 12 deletions test/pkg/debug/events.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,24 +42,12 @@ func NewEventClient(kubeClient client.Client) *EventClient {

func (c *EventClient) DumpEvents(ctx context.Context) error {
return multierr.Combine(
c.dumpKarpenterEvents(ctx),
c.dumpPodEvents(ctx),
c.dumpNodeEvents(ctx),
)

}

func (c *EventClient) dumpKarpenterEvents(ctx context.Context) error {
el := &v1.EventList{}
if err := c.kubeClient.List(ctx, el, client.InNamespace("karpenter")); err != nil {
return err
}
for k, v := range coallateEvents(filterTestEvents(el.Items, c.start)) {
fmt.Print(getEventInformation(k, v))
}
return nil
}

func (c *EventClient) dumpPodEvents(ctx context.Context) error {
el := &v1.EventList{}
if err := c.kubeClient.List(ctx, el, &client.ListOptions{
Expand Down
2 changes: 1 addition & 1 deletion test/pkg/debug/pod.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ func (c *PodController) Builder(_ context.Context, m manager.Manager) corecontro
},
},
predicate.NewPredicateFuncs(func(o client.Object) bool {
return o.GetNamespace() != "kube-system" && o.GetNamespace() != "karpenter"
return o.GetNamespace() != "kube-system"
}),
)).
WithOptions(controller.Options{MaxConcurrentReconciles: 10}))
Expand Down
2 changes: 1 addition & 1 deletion test/pkg/environment/common/environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ func NewEnvironment(t *testing.T) *Environment {
config := NewConfig()
client := NewClient(ctx, config)

lo.Must0(os.Setenv(system.NamespaceEnvKey, "karpenter"))
lo.Must0(os.Setenv(system.NamespaceEnvKey, "kube-system"))
if val, ok := os.LookupEnv("GIT_REF"); ok {
ctx = context.WithValue(ctx, GitRefContextKey, val)
}
Expand Down
25 changes: 13 additions & 12 deletions test/pkg/environment/common/expectations.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ func (env *Environment) ExpectSettings() (res []v1.EnvVar) {
GinkgoHelper()

d := &appsv1.Deployment{}
Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "karpenter", Name: "karpenter"}, d)).To(Succeed())
Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "kube-system", Name: "karpenter"}, d)).To(Succeed())
Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1))
return lo.Map(d.Spec.Template.Spec.Containers[0].Env, func(v v1.EnvVar, _ int) v1.EnvVar {
return *v.DeepCopy()
Expand All @@ -110,7 +110,7 @@ func (env *Environment) ExpectSettingsReplaced(vars ...v1.EnvVar) {
GinkgoHelper()

d := &appsv1.Deployment{}
Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "karpenter", Name: "karpenter"}, d)).To(Succeed())
Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "kube-system", Name: "karpenter"}, d)).To(Succeed())
Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1))

stored := d.DeepCopy()
Expand All @@ -127,7 +127,7 @@ func (env *Environment) ExpectSettingsOverridden(vars ...v1.EnvVar) {
GinkgoHelper()

d := &appsv1.Deployment{}
Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "karpenter", Name: "karpenter"}, d)).To(Succeed())
Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "kube-system", Name: "karpenter"}, d)).To(Succeed())
Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1))

stored := d.DeepCopy()
Expand All @@ -153,7 +153,7 @@ func (env *Environment) ExpectSettingsRemoved(vars ...v1.EnvVar) {
varNames := sets.New[string](lo.Map(vars, func(v v1.EnvVar, _ int) string { return v.Name })...)

d := &appsv1.Deployment{}
Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "karpenter", Name: "karpenter"}, d)).To(Succeed())
Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "kube-system", Name: "karpenter"}, d)).To(Succeed())
Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1))

stored := d.DeepCopy()
Expand Down Expand Up @@ -272,7 +272,7 @@ func (env *Environment) EventuallyExpectHealthyWithTimeout(timeout time.Duration
func (env *Environment) EventuallyExpectKarpenterRestarted() {
GinkgoHelper()
By("rolling out the new karpenter deployment")
env.EventuallyExpectRollout("karpenter", "karpenter")
env.EventuallyExpectRollout("karpenter", "kube-system")
env.ExpectKarpenterLeaseOwnerChanged()
}

Expand Down Expand Up @@ -332,7 +332,7 @@ func (env *Environment) ExpectKarpenterPods() []*v1.Pod {
func (env *Environment) ExpectActiveKarpenterPodName() string {
GinkgoHelper()
lease := &coordinationv1.Lease{}
Expect(env.Client.Get(env.Context, types.NamespacedName{Name: "karpenter-leader-election", Namespace: "karpenter"}, lease)).To(Succeed())
Expect(env.Client.Get(env.Context, types.NamespacedName{Name: "karpenter-leader-election", Namespace: "kube-system"}, lease)).To(Succeed())

// Holder identity for lease is always in the format "<pod-name>_<pseudo-random-value>
holderArr := strings.Split(lo.FromPtr(lease.Spec.HolderIdentity), "_")
Expand All @@ -346,7 +346,7 @@ func (env *Environment) ExpectActiveKarpenterPod() *v1.Pod {
podName := env.ExpectActiveKarpenterPodName()

pod := &v1.Pod{}
Expect(env.Client.Get(env.Context, types.NamespacedName{Name: podName, Namespace: "karpenter"}, pod)).To(Succeed())
Expect(env.Client.Get(env.Context, types.NamespacedName{Name: podName, Namespace: "kube-system"}, pod)).To(Succeed())
return pod
}

Expand Down Expand Up @@ -574,10 +574,11 @@ func (env *Environment) GetNode(nodeName string) v1.Node {

func (env *Environment) ExpectNoCrashes() {
GinkgoHelper()
_, crashed := lo.Find(lo.Values(env.Monitor.RestartCount()), func(restartCount int) bool {
return restartCount > 0
})
Expect(crashed).To(BeFalse(), "expected karpenter containers to not crash")
for k, v := range env.Monitor.RestartCount("kube-system") {
if strings.Contains(k, "karpenter") && v > 0 {
Fail("expected karpenter containers to not crash")
}
}
}

var (
Expand All @@ -601,7 +602,7 @@ func (env *Environment) printControllerLogs(options *v1.PodLogOptions) {
fmt.Printf("[PREVIOUS CONTAINER LOGS]\n")
temp.Previous = true
}
stream, err := env.KubeClient.CoreV1().Pods("karpenter").GetLogs(pod.Name, temp).Stream(env.Context)
stream, err := env.KubeClient.CoreV1().Pods("kube-system").GetLogs(pod.Name, temp).Stream(env.Context)
if err != nil {
logging.FromContext(env.Context).Errorf("fetching controller logs: %s", err)
return
Expand Down
4 changes: 2 additions & 2 deletions test/pkg/environment/common/monitor.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,14 +70,14 @@ func (m *Monitor) Reset() {

// RestartCount returns the containers and number of restarts for that container for all containers in the pods in the
// given namespace
func (m *Monitor) RestartCount() map[string]int {
func (m *Monitor) RestartCount(namespace string) map[string]int {
st := m.poll()

m.mu.RLock()
defer m.mu.RUnlock()
restarts := map[string]int{}
for _, pod := range st.pods.Items {
if pod.Namespace != "karpenter" {
if pod.Namespace != namespace {
continue
}
for _, cs := range pod.Status.ContainerStatuses {
Expand Down

0 comments on commit 75c31f4

Please sign in to comment.