diff --git a/go.sum b/go.sum index 48a5712b65b1..6f838d2fec3c 100644 --- a/go.sum +++ b/go.sum @@ -57,8 +57,6 @@ github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHS github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.45.25 h1:c4fLlh5sLdK2DCRTY1z0hyuJZU4ygxX8m1FswL6/nF4= github.com/aws/aws-sdk-go v1.45.25/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/karpenter-core v0.31.1-0.20231013203304-4239902b18b9 h1:j0iZuhoAKHrt0oqfSiKDqvHMnV/t45wi0loG1lEqdUw= -github.com/aws/karpenter-core v0.31.1-0.20231013203304-4239902b18b9/go.mod h1:rb3kp/3cj38tACF6udfpmIvKoQMwirSVoHNlrd66LyE= github.com/aws/karpenter-core v0.31.1-0.20231016061404-e22c2eb518be h1:EOrv6wPMVSck/XyHEBtfG2gaH5ovDHTRw1mcswB/7xs= github.com/aws/karpenter-core v0.31.1-0.20231016061404-e22c2eb518be/go.mod h1:rb3kp/3cj38tACF6udfpmIvKoQMwirSVoHNlrd66LyE= github.com/aws/karpenter/tools/kompat v0.0.0-20231010173459-62c25a3ea85c h1:oXWwIttmjYLbBKhLazG21aQvpJ3NOOr8IXhCJ/p6e/M= diff --git a/hack/code/prices_gen.go b/hack/code/prices_gen.go index e6dfe86cb0bb..73bd67f0fb07 100644 --- a/hack/code/prices_gen.go +++ b/hack/code/prices_gen.go @@ -34,7 +34,7 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/aws/karpenter/pkg/apis/settings" + awsoptions "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/test" ) @@ -93,7 +93,7 @@ func main() { os.Setenv("AWS_SDK_LOAD_CONFIG", "true") os.Setenv("AWS_REGION", region) ctx := context.Background() - ctx = settings.ToContext(ctx, test.Settings()) + ctx = awsoptions.ToContext(ctx, test.Options()) sess := session.Must(session.NewSession()) ec2 := ec22.New(sess) src := &bytes.Buffer{} diff --git a/hack/docs/configuration_gen_docs.go b/hack/docs/configuration_gen_docs.go index 844ce0bd87c7..4cfa57b17fee 100644 --- a/hack/docs/configuration_gen_docs.go +++ b/hack/docs/configuration_gen_docs.go @@ -21,7 +21,8 @@ import ( "os" "strings" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" + "github.com/aws/karpenter/pkg/operator/options" ) func main() { @@ -48,11 +49,13 @@ func main() { topDoc := fmt.Sprintf("%s%s\n\n", startDocSections[0], genStart) bottomDoc := fmt.Sprintf("\n%s%s", genEnd, endDocSections[1]) - opts := options.New() + fs := flag.NewFlagSet("karpenter", flag.ContinueOnError) + (&coreoptions.Options{}).AddFlags(fs) + (&options.Options{}).AddFlags(fs) envVarsBlock := "| Environment Variable | CLI Flag | Description |\n" envVarsBlock += "|--|--|--|\n" - opts.FlagSet.VisitAll(func(f *flag.Flag) { + fs.VisitAll(func(f *flag.Flag) { if f.DefValue == "" { envVarsBlock += fmt.Sprintf("| %s | %s | %s|\n", strings.ReplaceAll(strings.ToUpper(f.Name), "-", "_"), "\\-\\-"+f.Name, f.Usage) } else { diff --git a/hack/docs/instancetypes_gen_docs.go b/hack/docs/instancetypes_gen_docs.go index dcbbc9f1eb83..3c924b328682 100644 --- a/hack/docs/instancetypes_gen_docs.go +++ b/hack/docs/instancetypes_gen_docs.go @@ -35,14 +35,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/manager" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" coreoperator "github.com/aws/karpenter-core/pkg/operator" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" coretest "github.com/aws/karpenter-core/pkg/test" nodepoolutil "github.com/aws/karpenter-core/pkg/utils/nodepool" - "github.com/aws/karpenter/pkg/apis/settings" awscloudprovider "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/operator" + awsoptions "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter-core/pkg/cloudprovider" @@ -77,8 +77,8 @@ func main() { lo.Must0(os.Setenv("AWS_SDK_LOAD_CONFIG", "true")) lo.Must0(os.Setenv("AWS_REGION", "us-east-1")) - ctx := coresettings.ToContext(context.Background(), coretest.Settings()) - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx := coreoptions.ToContext(context.Background(), coretest.Options()) + ctx = awsoptions.ToContext(ctx, test.Options(test.OptionsFields{ ClusterName: lo.ToPtr("docs-gen"), ClusterEndpoint: lo.ToPtr("https://docs-gen.aws"), IsolatedVPC: lo.ToPtr(true), // disable pricing lookup diff --git a/pkg/apis/settings/settings.go b/pkg/apis/settings/settings.go index 5a3d6467b225..184e4770f8ce 100644 --- a/pkg/apis/settings/settings.go +++ b/pkg/apis/settings/settings.go @@ -22,6 +22,8 @@ import ( v1 "k8s.io/api/core/v1" "knative.dev/pkg/configmap" + + coresettings "github.com/aws/karpenter-core/pkg/apis/settings" ) type settingsKeyType struct{} @@ -92,6 +94,10 @@ func (*Settings) Inject(ctx context.Context, cm *v1.ConfigMap) (context.Context, return ToContext(ctx, s), nil } +func (*Settings) FromContext(ctx context.Context) coresettings.Injectable { + return FromContext(ctx) +} + func ToContext(ctx context.Context, s *Settings) context.Context { return context.WithValue(ctx, ContextKey, s) } @@ -99,8 +105,7 @@ func ToContext(ctx context.Context, s *Settings) context.Context { func FromContext(ctx context.Context) *Settings { data := ctx.Value(ContextKey) if data == nil { - // This is developer error if this happens, so we should panic - panic("settings doesn't exist in context") + return nil } return data.(*Settings) } diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index 14a5ce4deb98..28c9121dfb09 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -30,15 +30,15 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/pkg/cloudprovider" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/controllers/provisioning" "github.com/aws/karpenter-core/pkg/controllers/state" "github.com/aws/karpenter-core/pkg/events" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -46,7 +46,6 @@ import ( var ctx context.Context var stop context.CancelFunc -var opts *options.Options var env *coretest.Environment var awsEnv *test.Environment var prov *provisioning.Provisioner @@ -63,7 +62,8 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) awsEnv = test.NewEnvironment(ctx, env) @@ -81,8 +81,8 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = options.ToContext(ctx, opts) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) cluster.Reset() diff --git a/pkg/controllers/controllers.go b/pkg/controllers/controllers.go index e91c0c4710f8..816f7413e15e 100644 --- a/pkg/controllers/controllers.go +++ b/pkg/controllers/controllers.go @@ -24,13 +24,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/aws/karpenter-core/pkg/events" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/controllers/interruption" nodeclaimgarbagecollection "github.com/aws/karpenter/pkg/controllers/nodeclaim/garbagecollection" nodeclaimlink "github.com/aws/karpenter/pkg/controllers/nodeclaim/link" "github.com/aws/karpenter/pkg/controllers/nodeclass" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter/pkg/providers/instanceprofile" "github.com/aws/karpenter/pkg/providers/pricing" @@ -54,10 +54,10 @@ func NewControllers(ctx context.Context, sess *session.Session, clk clock.Clock, linkController, nodeclaimgarbagecollection.NewController(kubeClient, cloudProvider, linkController), } - if settings.FromContext(ctx).InterruptionQueueName != "" { + if options.FromContext(ctx).InterruptionQueueName != "" { controllers = append(controllers, interruption.NewController(kubeClient, clk, recorder, interruption.NewSQSProvider(sqs.New(sess)), unavailableOfferings)) } - if settings.FromContext(ctx).IsolatedVPC { + if options.FromContext(ctx).IsolatedVPC { logging.FromContext(ctx).Infof("assuming isolated VPC, pricing information will not be updated") } else { controllers = append(controllers, pricing.NewController(pricingProvider)) diff --git a/pkg/controllers/interruption/controller.go b/pkg/controllers/interruption/controller.go index 305c6b918599..e5d4651e610b 100644 --- a/pkg/controllers/interruption/controller.go +++ b/pkg/controllers/interruption/controller.go @@ -32,12 +32,12 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1beta1" "github.com/aws/karpenter-core/pkg/utils/pretty" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/cache" interruptionevents "github.com/aws/karpenter/pkg/controllers/interruption/events" "github.com/aws/karpenter/pkg/controllers/interruption/messages" "github.com/aws/karpenter/pkg/controllers/interruption/messages/statechange" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/utils" "github.com/aws/karpenter-core/pkg/events" @@ -80,8 +80,8 @@ func NewController(kubeClient client.Client, clk clock.Clock, recorder events.Re } func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { - ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("queue", settings.FromContext(ctx).InterruptionQueueName)) - if c.cm.HasChanged(settings.FromContext(ctx).InterruptionQueueName, nil) { + ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("queue", options.FromContext(ctx).InterruptionQueueName)) + if c.cm.HasChanged(options.FromContext(ctx).InterruptionQueueName, nil) { logging.FromContext(ctx).Debugf("watching interruption queue") } sqsMessages, err := c.sqsProvider.GetSQSMessages(ctx) diff --git a/pkg/controllers/interruption/interruption_benchmark_test.go b/pkg/controllers/interruption/interruption_benchmark_test.go index 6149bdddd35c..b0459b542a46 100644 --- a/pkg/controllers/interruption/interruption_benchmark_test.go +++ b/pkg/controllers/interruption/interruption_benchmark_test.go @@ -43,6 +43,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/aws/karpenter-core/pkg/operator/scheme" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/apis/settings" awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/controllers/interruption" @@ -50,7 +51,7 @@ import ( "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/test" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" coretest "github.com/aws/karpenter-core/pkg/test" ) @@ -77,12 +78,13 @@ func BenchmarkNotification100(b *testing.B) { func benchmarkNotificationController(b *testing.B, messageCount int) { ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("message-count", messageCount)) fakeClock = &clock.FakeClock{} - ctx = coresettings.ToContext(ctx, coretest.Settings()) - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ClusterName: lo.ToPtr("karpenter-notification-benchmarking"), IsolatedVPC: lo.ToPtr(true), InterruptionQueueName: lo.ToPtr("test-cluster"), })) + ctx = settings.ToContext(ctx, test.Settings()) env = coretest.NewEnvironment(scheme.Scheme) // Stop the coretest environment after the coretest completes defer func() { @@ -181,7 +183,7 @@ func newProviders(kubeClient client.Client) providerSet { func (p *providerSet) makeInfrastructure(ctx context.Context) error { if _, err := p.sqsAPI.CreateQueueWithContext(ctx, &sqs.CreateQueueInput{ - QueueName: lo.ToPtr(settings.FromContext(ctx).InterruptionQueueName), + QueueName: lo.ToPtr(options.FromContext(ctx).InterruptionQueueName), Attributes: map[string]*string{ sqs.QueueAttributeNameMessageRetentionPeriod: aws.String("1200"), // 20 minutes for this test }, diff --git a/pkg/controllers/interruption/sqs.go b/pkg/controllers/interruption/sqs.go index 873688801d3d..6cf518955820 100644 --- a/pkg/controllers/interruption/sqs.go +++ b/pkg/controllers/interruption/sqs.go @@ -26,8 +26,8 @@ import ( "github.com/samber/lo" "github.com/aws/karpenter-core/pkg/utils/atomic" - "github.com/aws/karpenter/pkg/apis/settings" awserrors "github.com/aws/karpenter/pkg/errors" + "github.com/aws/karpenter/pkg/operator/options" ) type SQSProvider struct { @@ -43,7 +43,7 @@ func NewSQSProvider(client sqsiface.SQSAPI) *SQSProvider { } provider.queueURL.Resolve = func(ctx context.Context) (string, error) { input := &sqs.GetQueueUrlInput{ - QueueName: aws.String(settings.FromContext(ctx).InterruptionQueueName), + QueueName: aws.String(options.FromContext(ctx).InterruptionQueueName), } ret, err := provider.client.GetQueueUrlWithContext(ctx, input) if err != nil { @@ -66,12 +66,12 @@ func (s *SQSProvider) QueueExists(ctx context.Context) (bool, error) { } func (s *SQSProvider) DiscoverQueueURL(ctx context.Context) (string, error) { - if settings.FromContext(ctx).InterruptionQueueName != lo.FromPtr(s.queueName.Load()) { + if options.FromContext(ctx).InterruptionQueueName != lo.FromPtr(s.queueName.Load()) { res, err := s.queueURL.TryGet(ctx, atomic.IgnoreCacheOption) if err != nil { return res, err } - s.queueName.Store(lo.ToPtr(settings.FromContext(ctx).InterruptionQueueName)) + s.queueName.Store(lo.ToPtr(options.FromContext(ctx).InterruptionQueueName)) return res, nil } return s.queueURL.TryGet(ctx) diff --git a/pkg/controllers/interruption/suite_test.go b/pkg/controllers/interruption/suite_test.go index 23d3ef8ab57e..8cb557e1efbf 100644 --- a/pkg/controllers/interruption/suite_test.go +++ b/pkg/controllers/interruption/suite_test.go @@ -37,10 +37,10 @@ import ( _ "knative.dev/pkg/system/testing" "sigs.k8s.io/controller-runtime/pkg/client" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" "github.com/aws/karpenter-core/pkg/events" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -53,6 +53,7 @@ import ( "github.com/aws/karpenter/pkg/controllers/interruption/messages/spotinterruption" "github.com/aws/karpenter/pkg/controllers/interruption/messages/statechange" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/pkg/utils" ) @@ -91,10 +92,11 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = coresettings.ToContext(ctx, coretest.Settings()) - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ InterruptionQueueName: lo.ToPtr("test-cluster"), })) + ctx = settings.ToContext(ctx, test.Settings()) unavailableOfferingsCache.Flush() sqsapi.Reset() sqsProvider.Reset() diff --git a/pkg/controllers/nodeclaim/garbagecollection/machine_test.go b/pkg/controllers/nodeclaim/garbagecollection/machine_test.go index bab441c35ab2..b9dcca9dd97e 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/machine_test.go +++ b/pkg/controllers/nodeclaim/garbagecollection/machine_test.go @@ -33,9 +33,9 @@ import ( coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" ) @@ -60,7 +60,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -69,7 +69,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -125,7 +125,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -134,7 +134,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -179,7 +179,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -188,7 +188,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -286,7 +286,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -295,7 +295,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), diff --git a/pkg/controllers/nodeclaim/garbagecollection/nodeclaim_test.go b/pkg/controllers/nodeclaim/garbagecollection/nodeclaim_test.go index fd098c57e989..9574bb4ca248 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/nodeclaim_test.go +++ b/pkg/controllers/nodeclaim/garbagecollection/nodeclaim_test.go @@ -35,8 +35,8 @@ import ( nodeclaimutil "github.com/aws/karpenter-core/pkg/utils/nodeclaim" nodepoolutil "github.com/aws/karpenter-core/pkg/utils/nodepool" "github.com/aws/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter/pkg/operator/options" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/test" ) @@ -67,7 +67,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -76,7 +76,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -132,7 +132,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -141,7 +141,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -186,7 +186,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -195,7 +195,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -303,7 +303,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -312,7 +312,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), diff --git a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go index 40dd367df7ea..888f5935bdb7 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go +++ b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go @@ -32,7 +32,6 @@ import ( "k8s.io/client-go/tools/record" . "knative.dev/pkg/logging/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" @@ -50,6 +49,7 @@ import ( "github.com/aws/karpenter/pkg/controllers/nodeclaim/garbagecollection" "github.com/aws/karpenter/pkg/controllers/nodeclaim/link" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" ) @@ -67,7 +67,7 @@ func TestAPIs(t *testing.T) { } var _ = BeforeSuite(func() { - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) awsEnv = test.NewEnvironment(ctx, env) @@ -108,7 +108,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -117,7 +117,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -143,7 +143,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -152,7 +152,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -197,7 +197,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -206,7 +206,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -247,7 +247,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -256,7 +256,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), diff --git a/pkg/controllers/nodeclaim/link/suite_test.go b/pkg/controllers/nodeclaim/link/suite_test.go index 15e059bf15ec..7fe54382750d 100644 --- a/pkg/controllers/nodeclaim/link/suite_test.go +++ b/pkg/controllers/nodeclaim/link/suite_test.go @@ -33,10 +33,10 @@ import ( . "knative.dev/pkg/logging/testing" "sigs.k8s.io/controller-runtime/pkg/client" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/events" "github.com/aws/karpenter-core/pkg/operator/controller" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -47,6 +47,7 @@ import ( "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/controllers/nodeclaim/link" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/pkg/utils" ) @@ -64,7 +65,8 @@ func TestAPIs(t *testing.T) { } var _ = BeforeSuite(func() { - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) awsEnv = test.NewEnvironment(ctx, env) @@ -107,7 +109,7 @@ var _ = Describe("MachineLink", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -254,7 +256,7 @@ var _ = Describe("MachineLink", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -344,7 +346,7 @@ var _ = Describe("MachineLink", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, }, diff --git a/pkg/controllers/nodeclaim/tagging/suite_test.go b/pkg/controllers/nodeclaim/tagging/suite_test.go index 69324b86a03d..fefab7c37ee5 100644 --- a/pkg/controllers/nodeclaim/tagging/suite_test.go +++ b/pkg/controllers/nodeclaim/tagging/suite_test.go @@ -31,7 +31,6 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" . "knative.dev/pkg/logging/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" coretest "github.com/aws/karpenter-core/pkg/test" "github.com/aws/karpenter/pkg/apis" @@ -39,10 +38,12 @@ import ( "github.com/aws/karpenter/pkg/apis/v1beta1" "github.com/aws/karpenter/pkg/controllers/nodeclaim/tagging" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter-core/pkg/operator/controller" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" ) @@ -59,7 +60,8 @@ func TestAPIs(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) awsEnv = test.NewEnvironment(ctx, env) taggingController = tagging.NewController(env.Client, awsEnv.InstanceProvider) @@ -86,7 +88,7 @@ var _ = Describe("TaggingController", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -95,7 +97,7 @@ var _ = Describe("TaggingController", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), diff --git a/pkg/controllers/nodeclass/suite_test.go b/pkg/controllers/nodeclass/suite_test.go index 830c155dd389..862a13320af7 100644 --- a/pkg/controllers/nodeclass/suite_test.go +++ b/pkg/controllers/nodeclass/suite_test.go @@ -24,23 +24,22 @@ import ( . "knative.dev/pkg/logging/testing" _ "knative.dev/pkg/system/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/events" corecontroller "github.com/aws/karpenter-core/pkg/operator/controller" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/controllers/nodeclass" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" ) var ctx context.Context var env *coretest.Environment var awsEnv *test.Environment -var opts *options.Options var nodeTemplateController corecontroller.Controller var nodeClassController corecontroller.Controller @@ -52,7 +51,8 @@ func TestAPIs(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...), coretest.WithFieldIndexers(test.EC2NodeClassFieldIndexer(ctx))) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) awsEnv = test.NewEnvironment(ctx, env) @@ -65,7 +65,7 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = options.ToContext(ctx, opts) + ctx = coreoptions.ToContext(ctx, coretest.Options()) awsEnv.Reset() }) diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index d07f0dd98f7a..d683c8f5362d 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -47,8 +47,8 @@ import ( "knative.dev/pkg/ptr" "github.com/aws/karpenter-core/pkg/operator" - "github.com/aws/karpenter/pkg/apis/settings" awscache "github.com/aws/karpenter/pkg/cache" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/providers/instanceprofile" @@ -85,7 +85,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont STSRegionalEndpoint: endpoints.RegionalSTSEndpoint, } - if assumeRoleARN := settings.FromContext(ctx).AssumeRoleARN; assumeRoleARN != "" { + if assumeRoleARN := options.FromContext(ctx).AssumeRoleARN; assumeRoleARN != "" { config.Credentials = stscreds.NewCredentials(session.Must(session.NewSession()), assumeRoleARN, func(provider *stscreds.AssumeRoleProvider) { setDurationAndExpiry(ctx, provider) }) } @@ -204,12 +204,12 @@ func checkEC2Connectivity(ctx context.Context, api *ec2.EC2) error { } func ResolveClusterEndpoint(ctx context.Context, eksAPI eksiface.EKSAPI) (string, error) { - clusterEndpointFromSettings := settings.FromContext(ctx).ClusterEndpoint - if clusterEndpointFromSettings != "" { - return clusterEndpointFromSettings, nil // cluster endpoint is explicitly set + clusterEndpointFromOptions := options.FromContext(ctx).ClusterEndpoint + if clusterEndpointFromOptions != "" { + return clusterEndpointFromOptions, nil // cluster endpoint is explicitly set } out, err := eksAPI.DescribeClusterWithContext(ctx, &eks.DescribeClusterInput{ - Name: aws.String(settings.FromContext(ctx).ClusterName), + Name: aws.String(options.FromContext(ctx).ClusterName), }) if err != nil { return "", fmt.Errorf("failed to resolve cluster endpoint, %w", err) @@ -222,7 +222,7 @@ func getCABundle(ctx context.Context, restConfig *rest.Config) (*string, error) // have used the simpler client-go InClusterConfig() method. // However, that only works when Karpenter is running as a Pod // within the same cluster it's managing. - if caBundle := settings.FromContext(ctx).ClusterCABundle; caBundle != "" { + if caBundle := options.FromContext(ctx).ClusterCABundle; caBundle != "" { return lo.ToPtr(caBundle), nil } transportConfig, err := restConfig.TransportConfig() @@ -252,6 +252,6 @@ func kubeDNSIP(ctx context.Context, kubernetesInterface kubernetes.Interface) (n } func setDurationAndExpiry(ctx context.Context, provider *stscreds.AssumeRoleProvider) { - provider.Duration = settings.FromContext(ctx).AssumeRoleDuration + provider.Duration = options.FromContext(ctx).AssumeRoleDuration provider.ExpiryWindow = time.Duration(10) * time.Second } diff --git a/pkg/operator/options/options.go b/pkg/operator/options/options.go new file mode 100644 index 000000000000..86caa969fa5f --- /dev/null +++ b/pkg/operator/options/options.go @@ -0,0 +1,141 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "strings" + "time" + + "k8s.io/apimachinery/pkg/util/sets" + + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" + "github.com/aws/karpenter-core/pkg/utils/env" + "github.com/aws/karpenter/pkg/apis/settings" +) + +func init() { + coreoptions.Injectables = append(coreoptions.Injectables, &Options{}) +} + +type optionsKey struct{} + +type Options struct { + AssumeRoleARN string + AssumeRoleDuration time.Duration + ClusterCABundle string + ClusterName string + ClusterEndpoint string + IsolatedVPC bool + VMMemoryOverheadPercent float64 + InterruptionQueueName string + ReservedENIs int + + setFlags map[string]bool +} + +func (o *Options) AddFlags(fs *flag.FlagSet) { + fs.StringVar(&o.AssumeRoleARN, "assume-role-arn", env.WithDefaultString("ASSUME_ROLE_ARN", ""), "Role to assume for calling AWS services.") + fs.DurationVar(&o.AssumeRoleDuration, "assume-role-duration", env.WithDefaultDuration("ASSUME_ROLE_DURATION", 15*time.Minute), "Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless aws.assumeRole set.") + fs.StringVar(&o.ClusterCABundle, "cluster-ca-bundle", env.WithDefaultString("CLUSTER_CA_BUNDLE", ""), "Cluster CA bundle for nodes to use for TLS connections with the API server. If not set, this is taken from the controller's TLS configuration.") + fs.StringVar(&o.ClusterName, "cluster-name", env.WithDefaultString("CLUSTER_NAME", ""), "[REQUIRED] The kubernetes cluster name for resource discovery.") + fs.StringVar(&o.ClusterEndpoint, "cluster-endpoint", env.WithDefaultString("CLUSTER_ENDPOINT", ""), "The external kubernetes cluster endpoint for new nodes to connect with. If not specified, will discover the cluster endpoint using DescribeCluster API.") + fs.BoolVar(&o.IsolatedVPC, "isolated-vpc", env.WithDefaultBool("ISOLATED_VPC", false), "If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS pricing endpoint.") + fs.Float64Var(&o.VMMemoryOverheadPercent, "vm-memory-overhead-percent", env.WithDefaultFloat64("VM_MEMORY_OVERHEAD_PERCENT", 0.075), "The VM memory overhead as a percent that will be subtracted from the total memory for all instance types.") + fs.StringVar(&o.InterruptionQueueName, "interruption-queue-name", env.WithDefaultString("INTERRUPTION_QUEUE_NAME", ""), "Interruption queue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.") + fs.IntVar(&o.ReservedENIs, "reserved-enis", env.WithDefaultInt("RESERVED_ENIS", 0), "Reserved ENIs are not included in the calculations for max-pods or kube-reserved. This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html.") +} + +func (o *Options) Parse(fs *flag.FlagSet, args ...string) error { + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + os.Exit(0) + } + return fmt.Errorf("parsing flags, %w", err) + } + + if err := o.Validate(); err != nil { + return fmt.Errorf("validating options, %w", err) + } + + // Check if each option has been set. This is a little brute force and better options might exist, + // but this only needs to be here for one version + o.setFlags = map[string]bool{} + cliFlags := sets.New[string]() + fs.Visit(func(f *flag.Flag) { + cliFlags.Insert(f.Name) + }) + fs.VisitAll(func(f *flag.Flag) { + envName := strings.ReplaceAll(strings.ToUpper(f.Name), "-", "_") + _, ok := os.LookupEnv(envName) + o.setFlags[f.Name] = ok || cliFlags.Has(f.Name) + }) + + return nil +} + +func (o *Options) ToContext(ctx context.Context) context.Context { + return ToContext(ctx, o) +} + +func (o *Options) MergeSettings(ctx context.Context) { + s := settings.FromContext(ctx) + if s == nil { + return + } + if !o.setFlags["assume-role-arn"] { + o.AssumeRoleARN = s.AssumeRoleARN + } + if !o.setFlags["assume-role-duration"] { + o.AssumeRoleDuration = s.AssumeRoleDuration + } + if !o.setFlags["cluster-ca-bundle"] { + o.ClusterCABundle = s.ClusterCABundle + } + if !o.setFlags["cluster-name"] { + o.ClusterName = s.ClusterName + } + if !o.setFlags["cluster-endpoint"] { + o.ClusterEndpoint = s.ClusterEndpoint + } + if !o.setFlags["isolated-vpc"] { + o.IsolatedVPC = s.IsolatedVPC + } + if !o.setFlags["vm-memory-overhead-percent"] { + o.VMMemoryOverheadPercent = s.VMMemoryOverheadPercent + } + if !o.setFlags["interruption-queue-name"] { + o.InterruptionQueueName = s.InterruptionQueueName + } + if !o.setFlags["reserved-enis"] { + o.ReservedENIs = s.ReservedENIs + } +} + +func ToContext(ctx context.Context, opts *Options) context.Context { + return context.WithValue(ctx, optionsKey{}, opts) +} + +func FromContext(ctx context.Context) *Options { + retval := ctx.Value(optionsKey{}) + if retval == nil { + return nil + } + return retval.(*Options) +} diff --git a/pkg/operator/options/options_validation.go b/pkg/operator/options/options_validation.go new file mode 100644 index 000000000000..480a1fb0bea8 --- /dev/null +++ b/pkg/operator/options/options_validation.go @@ -0,0 +1,78 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "fmt" + "net/url" + "time" + + "knative.dev/pkg/apis" +) + +func (o Options) Validate() (errs *apis.FieldError) { + return errs.Also( + o.validateEndpoint(), + o.validateClusterName(), + o.validateVMMemoryOverheadPercent(), + o.validateAssumeRoleDuration(), + ).ViaField("aws") +} + +func (o Options) validateAssumeRoleDuration() (errs *apis.FieldError) { + // TODO: Remove with karpenter-global-settings + if !o.setFlags["aws-assume-role-arn"] { + return nil + } + if o.AssumeRoleDuration < time.Minute*15 { + return errs.Also(apis.ErrInvalidValue("assumeRoleDuration cannot be less than 15 Minutes", "assumeRoleDuration")) + } + return nil +} + +func (o Options) validateClusterName() (errs *apis.FieldError) { + // TODO: Remove with karpenter-global-settings + if !o.setFlags["aws-cluster-name"] { + return nil + } + if o.ClusterName == "" { + return errs.Also(apis.ErrMissingField("clusterName is required", "clusterName")) + } + return nil +} + +func (o Options) validateEndpoint() (errs *apis.FieldError) { + if o.ClusterEndpoint == "" { + return nil + } + endpoint, err := url.Parse(o.ClusterEndpoint) + // url.Parse() will accept a lot of input without error; make + // sure it's a real URL + if err != nil || !endpoint.IsAbs() || endpoint.Hostname() == "" { + return errs.Also(apis.ErrInvalidValue(fmt.Sprintf("%q not a valid clusterEndpoint URL", o.ClusterEndpoint), "clusterEndpoint")) + } + return nil +} + +func (o Options) validateVMMemoryOverheadPercent() (errs *apis.FieldError) { + // TODO: Remove with karpenter-global-settings + if !o.setFlags["aws-vm-memory-overhead-percent"] { + return nil + } + if o.VMMemoryOverheadPercent < 0 { + return errs.Also(apis.ErrInvalidValue("cannot be negative", "vmMemoryOverheadPercent")) + } + return nil +} diff --git a/pkg/operator/suite_test.go b/pkg/operator/suite_test.go index dea32a9fedc9..5146ed6d9e9b 100644 --- a/pkg/operator/suite_test.go +++ b/pkg/operator/suite_test.go @@ -26,12 +26,11 @@ import ( . "knative.dev/pkg/logging/testing" "github.com/aws/karpenter/pkg/apis" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/fake" awscontext "github.com/aws/karpenter/pkg/operator" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -50,8 +49,6 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) - ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) fakeEKSAPI = &fake.EKSAPI{} @@ -72,7 +69,7 @@ var _ = AfterEach(func() { var _ = Describe("Operator", func() { It("should resolve endpoint if set via configuration", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ClusterEndpoint: lo.ToPtr("https://api.test-cluster.k8s.local"), })) endpoint, err := awscontext.ResolveClusterEndpoint(ctx, fakeEKSAPI) @@ -80,7 +77,7 @@ var _ = Describe("Operator", func() { Expect(endpoint).To(Equal("https://api.test-cluster.k8s.local")) }) It("should resolve endpoint if not set, via call to API", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ClusterEndpoint: lo.ToPtr(""), })) fakeEKSAPI.DescribeClusterBehavior.Output.Set( @@ -96,7 +93,7 @@ var _ = Describe("Operator", func() { Expect(endpoint).To(Equal("https://cluster-endpoint.test-cluster.k8s.local")) }) It("should propagate error if API fails", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ClusterEndpoint: lo.ToPtr(""), })) fakeEKSAPI.DescribeClusterBehavior.Error.Set(errors.New("test error")) diff --git a/pkg/providers/amifamily/ami_test.go b/pkg/providers/amifamily/ami_test.go index 9da5c2058dff..279ce74dd88b 100644 --- a/pkg/providers/amifamily/ami_test.go +++ b/pkg/providers/amifamily/ami_test.go @@ -29,14 +29,15 @@ import ( v1 "k8s.io/api/core/v1" . "knative.dev/pkg/logging/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" "github.com/aws/karpenter-core/pkg/scheduling" coretest "github.com/aws/karpenter-core/pkg/test" "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter/pkg/test" ) @@ -61,7 +62,8 @@ const ( var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) awsEnv = test.NewEnvironment(ctx, env) }) diff --git a/pkg/providers/instance/instance.go b/pkg/providers/instance/instance.go index c8b555f20e29..46776de51bf5 100644 --- a/pkg/providers/instance/instance.go +++ b/pkg/providers/instance/instance.go @@ -40,6 +40,7 @@ import ( "github.com/aws/karpenter/pkg/batcher" "github.com/aws/karpenter/pkg/cache" awserrors "github.com/aws/karpenter/pkg/errors" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/instancetype" "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/subnet" @@ -105,7 +106,7 @@ func (p *Provider) Create(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, func (p *Provider) Link(ctx context.Context, id, provisionerName string) error { if err := p.CreateTags(ctx, id, map[string]string{ - v1alpha5.MachineManagedByAnnotationKey: settings.FromContext(ctx).ClusterName, + v1alpha5.MachineManagedByAnnotationKey: options.FromContext(ctx).ClusterName, v1alpha5.ProvisionerNameLabelKey: provisionerName, }); err != nil { return fmt.Errorf("linking tags, %w", err) @@ -148,7 +149,7 @@ func (p *Provider) List(ctx context.Context) ([]*Instance, error) { }, { Name: aws.String("tag-key"), - Values: aws.StringSlice([]string{fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)}), + Values: aws.StringSlice([]string{fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)}), }, instanceStateFilter, }, @@ -262,15 +263,15 @@ func getTags(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *co "Name": fmt.Sprintf("%s/%s", v1alpha5.ProvisionerNameLabelKey, nodeClaim.Labels[v1alpha5.ProvisionerNameLabelKey]), } staticTags = map[string]string{ - fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName): "owned", - v1alpha5.ProvisionerNameLabelKey: nodeClaim.Labels[v1alpha5.ProvisionerNameLabelKey], - v1alpha5.MachineManagedByAnnotationKey: settings.FromContext(ctx).ClusterName, + fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName): "owned", + v1alpha5.ProvisionerNameLabelKey: nodeClaim.Labels[v1alpha5.ProvisionerNameLabelKey], + v1alpha5.MachineManagedByAnnotationKey: options.FromContext(ctx).ClusterName, } } else { staticTags = map[string]string{ - fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName): "owned", + fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName): "owned", corev1beta1.NodePoolLabelKey: nodeClaim.Labels[corev1beta1.NodePoolLabelKey], - corev1beta1.ManagedByAnnotationKey: settings.FromContext(ctx).ClusterName, + corev1beta1.ManagedByAnnotationKey: options.FromContext(ctx).ClusterName, } } return lo.Assign(overridableTags, settings.FromContext(ctx).Tags, nodeClass.Spec.Tags, staticTags) diff --git a/pkg/providers/instance/nodeclass_test.go b/pkg/providers/instance/nodeclass_test.go index 4f711b38da72..a2fcaa913142 100644 --- a/pkg/providers/instance/nodeclass_test.go +++ b/pkg/providers/instance/nodeclass_test.go @@ -31,9 +31,9 @@ import ( corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/test" ) @@ -100,7 +100,7 @@ var _ = Describe("NodeClass/InstanceProvider", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -109,7 +109,7 @@ var _ = Describe("NodeClass/InstanceProvider", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), diff --git a/pkg/providers/instance/nodetemplate_test.go b/pkg/providers/instance/nodetemplate_test.go index aec5c71ea45c..4894eb9bcddc 100644 --- a/pkg/providers/instance/nodetemplate_test.go +++ b/pkg/providers/instance/nodetemplate_test.go @@ -33,9 +33,9 @@ import ( . "github.com/aws/karpenter-core/pkg/test/expectations" nodeclaimutil "github.com/aws/karpenter-core/pkg/utils/nodeclaim" nodepoolutil "github.com/aws/karpenter-core/pkg/utils/nodepool" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/test" nodeclassutil "github.com/aws/karpenter/pkg/utils/nodeclass" @@ -114,7 +114,7 @@ var _ = Describe("NodeTemplate/InstanceProvider", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -123,7 +123,7 @@ var _ = Describe("NodeTemplate/InstanceProvider", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), diff --git a/pkg/providers/instance/suite_test.go b/pkg/providers/instance/suite_test.go index e7239953bf06..6714d7b2929b 100644 --- a/pkg/providers/instance/suite_test.go +++ b/pkg/providers/instance/suite_test.go @@ -29,11 +29,10 @@ import ( "k8s.io/client-go/tools/record" . "knative.dev/pkg/logging/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" "github.com/aws/karpenter-core/pkg/events" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" nodeclaimutil "github.com/aws/karpenter-core/pkg/utils/nodeclaim" @@ -42,12 +41,12 @@ import ( "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/test" ) var ctx context.Context -var opts *options.Options var env *coretest.Environment var awsEnv *test.Environment var cloudProvider *cloudprovider.CloudProvider @@ -60,7 +59,8 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) awsEnv = test.NewEnvironment(ctx, env) cloudProvider = cloudprovider.New(awsEnv.InstanceTypesProvider, awsEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), @@ -74,8 +74,6 @@ var _ = AfterSuite(func() { var _ = BeforeEach(func() { nodepoolutil.EnableNodePools = true nodeclaimutil.EnableNodeClaims = true - ctx = options.ToContext(ctx, opts) - ctx = coresettings.ToContext(ctx, coretest.Settings()) ctx = settings.ToContext(ctx, test.Settings()) awsEnv.Reset() }) @@ -94,7 +92,7 @@ var _ = Describe("Combined/InstanceProvider", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -103,7 +101,7 @@ var _ = Describe("Combined/InstanceProvider", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -129,7 +127,7 @@ var _ = Describe("Combined/InstanceProvider", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -138,7 +136,7 @@ var _ = Describe("Combined/InstanceProvider", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -196,7 +194,7 @@ var _ = Describe("Combined/InstanceProvider", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -205,7 +203,7 @@ var _ = Describe("Combined/InstanceProvider", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -231,7 +229,7 @@ var _ = Describe("Combined/InstanceProvider", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -240,7 +238,7 @@ var _ = Describe("Combined/InstanceProvider", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), diff --git a/pkg/providers/instanceprofile/instanceprofile.go b/pkg/providers/instanceprofile/instanceprofile.go index e5b6f9437399..f9bb39d9cf8a 100644 --- a/pkg/providers/instanceprofile/instanceprofile.go +++ b/pkg/providers/instanceprofile/instanceprofile.go @@ -27,9 +27,9 @@ import ( v1 "k8s.io/api/core/v1" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" awserrors "github.com/aws/karpenter/pkg/errors" + "github.com/aws/karpenter/pkg/operator/options" ) type Provider struct { @@ -48,10 +48,10 @@ func NewProvider(region string, iamapi iamiface.IAMAPI, cache *cache.Cache) *Pro func (p *Provider) Create(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) (string, error) { tags := lo.Assign(nodeClass.Spec.Tags, map[string]string{ - fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName): "owned", - corev1beta1.ManagedByAnnotationKey: settings.FromContext(ctx).ClusterName, - v1beta1.LabelNodeClass: nodeClass.Name, - v1.LabelTopologyRegion: p.region, + fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName): "owned", + corev1beta1.ManagedByAnnotationKey: options.FromContext(ctx).ClusterName, + v1beta1.LabelNodeClass: nodeClass.Name, + v1.LabelTopologyRegion: p.region, }) profileName := GetProfileName(ctx, p.region, nodeClass) @@ -129,5 +129,5 @@ func (p *Provider) Delete(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) // GetProfileName gets the string for the profile name based on the cluster name and the NodeClass UUID. // The length of this string can never exceed the maximum instance profile name limit of 128 characters. func GetProfileName(ctx context.Context, region string, nodeClass *v1beta1.EC2NodeClass) string { - return fmt.Sprintf("%s_%d", settings.FromContext(ctx).ClusterName, lo.Must(hashstructure.Hash(fmt.Sprintf("%s%s", region, nodeClass.Name), hashstructure.FormatV2, nil))) + return fmt.Sprintf("%s_%d", options.FromContext(ctx).ClusterName, lo.Must(hashstructure.Hash(fmt.Sprintf("%s%s", region, nodeClass.Name), hashstructure.FormatV2, nil))) } diff --git a/pkg/providers/instancetype/nodeclass_test.go b/pkg/providers/instancetype/nodeclass_test.go index d2b4cdae9a00..b58bd3212ec6 100644 --- a/pkg/providers/instancetype/nodeclass_test.go +++ b/pkg/providers/instancetype/nodeclass_test.go @@ -40,6 +40,7 @@ import ( "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/providers/instancetype" "github.com/aws/karpenter/pkg/test" @@ -427,20 +428,6 @@ var _ = Describe("NodeClass/InstanceTypes", func() { ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) }) - It("should fail to launch AWS Pod ENI if the setting enabling it isn't set", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ - EnablePodENI: lo.ToPtr(false), - })) - ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pod := coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1beta1.ResourceAWSPodENI: resource.MustParse("1")}, - Limits: v1.ResourceList{v1beta1.ResourceAWSPodENI: resource.MustParse("1")}, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) It("should launch AWS Pod ENI on a compatible instance type", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod(coretest.PodOptions{ @@ -645,16 +632,8 @@ var _ = Describe("NodeClass/InstanceTypes", func() { Context("Overhead", func() { var info *ec2.InstanceTypeInfo BeforeEach(func() { - ctx, err := (&settings.Settings{}).Inject(ctx, &v1.ConfigMap{ - Data: map[string]string{ - "aws.clusterName": "karpenter-cluster", - }, - }) - Expect(err).To(BeNil()) - - s := settings.FromContext(ctx) - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ - VMMemoryOverheadPercent: &s.VMMemoryOverheadPercent, + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + ClusterName: lo.ToPtr("CLUSTER_NAME"), })) var ok bool @@ -713,7 +692,7 @@ var _ = Describe("NodeClass/InstanceTypes", func() { }) Context("Eviction Thresholds", func() { BeforeEach(func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) }) @@ -950,7 +929,7 @@ var _ = Describe("NodeClass/InstanceTypes", func() { } }) It("should reserve ENIs when aws.reservedENIs is set and is used in max-pods calculation", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ReservedENIs: lo.ToPtr(1), })) @@ -970,7 +949,7 @@ var _ = Describe("NodeClass/InstanceTypes", func() { Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", maxPods)) }) It("should reserve ENIs when aws.reservedENIs is set and not go below 0 ENIs in max-pods calculation", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ReservedENIs: lo.ToPtr(1_000_000), })) diff --git a/pkg/providers/instancetype/nodetemplate_test.go b/pkg/providers/instancetype/nodetemplate_test.go index ba650c3b4ca7..874829062b94 100644 --- a/pkg/providers/instancetype/nodetemplate_test.go +++ b/pkg/providers/instancetype/nodetemplate_test.go @@ -40,6 +40,7 @@ import ( . "github.com/aws/karpenter-core/pkg/test/expectations" nodepoolutil "github.com/aws/karpenter-core/pkg/utils/nodepool" "github.com/aws/karpenter-core/pkg/utils/resources" + "github.com/aws/karpenter/pkg/operator/options" nodeclassutil "github.com/aws/karpenter/pkg/utils/nodeclass" "github.com/aws/karpenter/pkg/apis/settings" @@ -434,20 +435,6 @@ var _ = Describe("NodeTemplate/InstanceTypes", func() { ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) }) - It("should fail to launch AWS Pod ENI if the setting enabling it isn't set", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ - EnablePodENI: lo.ToPtr(false), - })) - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) - pod := coretest.UnschedulablePod(coretest.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1alpha1.ResourceAWSPodENI: resource.MustParse("1")}, - Limits: v1.ResourceList{v1alpha1.ResourceAWSPodENI: resource.MustParse("1")}, - }, - }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) - }) It("should launch AWS Pod ENI on a compatible instance type", func() { ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod(coretest.PodOptions{ @@ -664,16 +651,8 @@ var _ = Describe("NodeTemplate/InstanceTypes", func() { Context("Overhead", func() { var info *ec2.InstanceTypeInfo BeforeEach(func() { - ctx, err := (&settings.Settings{}).Inject(ctx, &v1.ConfigMap{ - Data: map[string]string{ - "aws.clusterName": "karpenter-cluster", - }, - }) - Expect(err).To(BeNil()) - - s := settings.FromContext(ctx) - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ - VMMemoryOverheadPercent: &s.VMMemoryOverheadPercent, + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + ClusterName: lo.ToPtr("karpenter-cluster"), })) var ok bool @@ -734,7 +713,7 @@ var _ = Describe("NodeTemplate/InstanceTypes", func() { }) Context("Eviction Thresholds", func() { BeforeEach(func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) }) @@ -990,7 +969,7 @@ var _ = Describe("NodeTemplate/InstanceTypes", func() { } }) It("should reserve ENIs when aws.reservedENIs is set and is used in max-pods calculation", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ReservedENIs: lo.ToPtr(1), })) @@ -1010,7 +989,7 @@ var _ = Describe("NodeTemplate/InstanceTypes", func() { Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", maxPods)) }) It("should reserve ENIs when aws.reservedENIs is set and not go below 0 ENIs in max-pods calculation", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ReservedENIs: lo.ToPtr(1_000_000), })) diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index c237cbcd3da6..4a56a7bcbc62 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -29,13 +29,12 @@ import ( clock "k8s.io/utils/clock/testing" . "knative.dev/pkg/logging/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" "github.com/aws/karpenter-core/pkg/controllers/provisioning" "github.com/aws/karpenter-core/pkg/controllers/state" "github.com/aws/karpenter-core/pkg/events" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -43,12 +42,12 @@ import ( "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/test" ) var ctx context.Context -var opts *options.Options var env *coretest.Environment var awsEnv *test.Environment var fakeClock *clock.FakeClock @@ -64,7 +63,8 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) awsEnv = test.NewEnvironment(ctx, env) fakeClock = &clock.FakeClock{} @@ -79,8 +79,8 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = options.ToContext(ctx, opts) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) cluster.Reset() awsEnv.Reset() @@ -128,7 +128,7 @@ func generateSpotPricing(cp *cloudprovider.CloudProvider, nodePool *corev1beta1. func makeFakeInstances() []*ec2.InstanceTypeInfo { var instanceTypes []*ec2.InstanceTypeInfo - ctx := settings.ToContext(context.Background(), &settings.Settings{IsolatedVPC: true}) + ctx := options.ToContext(context.Background(), &options.Options{IsolatedVPC: true}) // Use keys from the static pricing data so that we guarantee pricing for the data // Create uniform instance data so all of them schedule for a given pod for _, it := range pricing.NewProvider(ctx, nil, nil, "us-east-1").InstanceTypes() { diff --git a/pkg/providers/instancetype/types.go b/pkg/providers/instancetype/types.go index a126ac84615d..25878e994973 100644 --- a/pkg/providers/instancetype/types.go +++ b/pkg/providers/instancetype/types.go @@ -30,9 +30,10 @@ import ( "knative.dev/pkg/ptr" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" - awssettings "github.com/aws/karpenter/pkg/apis/settings" + "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter-core/pkg/cloudprovider" @@ -176,7 +177,7 @@ func computeCapacity(ctx context.Context, info *ec2.InstanceTypeInfo, amiFamily v1.ResourceMemory: *memory(ctx, info), v1.ResourceEphemeralStorage: *ephemeralStorage(amiFamily, blockDeviceMappings), v1.ResourcePods: *pods(ctx, info, amiFamily, kc), - v1beta1.ResourceAWSPodENI: *awsPodENI(ctx, aws.StringValue(info.InstanceType)), + v1beta1.ResourceAWSPodENI: *awsPodENI(aws.StringValue(info.InstanceType)), v1beta1.ResourceNVIDIAGPU: *nvidiaGPUs(info), v1beta1.ResourceAMDGPU: *amdGPUs(info), v1beta1.ResourceAWSNeuron: *awsNeurons(info), @@ -201,7 +202,7 @@ func memory(ctx context.Context, info *ec2.InstanceTypeInfo) *resource.Quantity } mem := resources.Quantity(fmt.Sprintf("%dMi", sizeInMib)) // Account for VM overhead in calculation - mem.Sub(resource.MustParse(fmt.Sprintf("%dMi", int64(math.Ceil(float64(mem.Value())*awssettings.FromContext(ctx).VMMemoryOverheadPercent/1024/1024))))) + mem.Sub(resource.MustParse(fmt.Sprintf("%dMi", int64(math.Ceil(float64(mem.Value())*options.FromContext(ctx).VMMemoryOverheadPercent/1024/1024))))) return mem } @@ -237,10 +238,10 @@ func ephemeralStorage(amiFamily amifamily.AMIFamily, blockDeviceMappings []*v1be return amifamily.DefaultEBS.VolumeSize } -func awsPodENI(ctx context.Context, name string) *resource.Quantity { +func awsPodENI(name string) *resource.Quantity { // https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html#supported-instance-types limits, ok := Limits[name] - if awssettings.FromContext(ctx).EnablePodENI && ok && limits.IsTrunkingCompatible { + if ok && limits.IsTrunkingCompatible { return resources.Quantity(fmt.Sprint(limits.BranchInterface)) } return resources.Quantity("0") @@ -308,7 +309,7 @@ func ENILimitedPods(ctx context.Context, info *ec2.InstanceTypeInfo) *resource.Q // VPC CNI only uses the default network interface // https://github.com/aws/amazon-vpc-cni-k8s/blob/3294231c0dce52cfe473bf6c62f47956a3b333b6/scripts/gen_vpc_ip_limits.go#L162 networkInterfaces := *info.NetworkInfo.NetworkCards[*info.NetworkInfo.DefaultNetworkCardIndex].MaximumNetworkInterfaces - usableNetworkInterfaces := lo.Max([]int64{(networkInterfaces - int64(awssettings.FromContext(ctx).ReservedENIs)), 0}) + usableNetworkInterfaces := lo.Max([]int64{(networkInterfaces - int64(options.FromContext(ctx).ReservedENIs)), 0}) if usableNetworkInterfaces == 0 { return resource.NewQuantity(0, resource.DecimalSI) } @@ -403,7 +404,7 @@ func pods(ctx context.Context, info *ec2.InstanceTypeInfo, amiFamily amifamily.A switch { case kc != nil && kc.MaxPods != nil: count = int64(ptr.Int32Value(kc.MaxPods)) - case awssettings.FromContext(ctx).EnableENILimitedPodDensity && amiFamily.FeatureFlags().SupportsENILimitedPodDensity: + case settings.FromContext(ctx).EnableENILimitedPodDensity && amiFamily.FeatureFlags().SupportsENILimitedPodDensity: count = ENILimitedPods(ctx, info).Value() default: count = 110 diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index 015f344e07a3..40a84ee734ce 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -39,6 +39,7 @@ import ( "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" awserrors "github.com/aws/karpenter/pkg/errors" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter/pkg/providers/instanceprofile" "github.com/aws/karpenter/pkg/providers/securitygroup" @@ -172,7 +173,7 @@ func (p *Provider) createAMIOptions(ctx context.Context, nodeClass *v1beta1.EC2N return nil, fmt.Errorf("no security groups exist given constraints") } options := &amifamily.Options{ - ClusterName: settings.FromContext(ctx).ClusterName, + ClusterName: options.FromContext(ctx).ClusterName, ClusterEndpoint: p.ClusterEndpoint, AWSENILimitedPodDensity: settings.FromContext(ctx).EnableENILimitedPodDensity, InstanceProfile: instanceProfile, @@ -328,7 +329,7 @@ func (p *Provider) volumeSize(quantity *resource.Quantity) *int64 { // hydrateCache queries for existing Launch Templates created by Karpenter for the current cluster and adds to the LT cache. // Any error during hydration will result in a panic func (p *Provider) hydrateCache(ctx context.Context) { - clusterName := settings.FromContext(ctx).ClusterName + clusterName := options.FromContext(ctx).ClusterName ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("tag-key", karpenterManagedTagKey, "tag-value", clusterName)) if err := p.ec2api.DescribeLaunchTemplatesPagesWithContext(ctx, &ec2.DescribeLaunchTemplatesInput{ Filters: []*ec2.Filter{{Name: aws.String(fmt.Sprintf("tag:%s", karpenterManagedTagKey)), Values: []*string{aws.String(clusterName)}}}, diff --git a/pkg/providers/launchtemplate/nodeclass_test.go b/pkg/providers/launchtemplate/nodeclass_test.go index 0d1f2281aada..471849d15aeb 100644 --- a/pkg/providers/launchtemplate/nodeclass_test.go +++ b/pkg/providers/launchtemplate/nodeclass_test.go @@ -42,6 +42,7 @@ import ( "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/amifamily/bootstrap" "github.com/aws/karpenter/pkg/providers/instancetype" "github.com/aws/karpenter/pkg/test" @@ -692,7 +693,10 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() { It("should calculate memory overhead based on eni limited pods when ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 @@ -703,7 +707,10 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() { It("should calculate memory overhead based on eni limited pods when not ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 @@ -742,7 +749,10 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() { It("should calculate memory overhead based on eni limited pods when ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(true), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket @@ -753,7 +763,10 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() { It("should calculate memory overhead based on max pods when not ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket diff --git a/pkg/providers/launchtemplate/nodetemplate_test.go b/pkg/providers/launchtemplate/nodetemplate_test.go index 84107056148a..0be09622f2ab 100644 --- a/pkg/providers/launchtemplate/nodetemplate_test.go +++ b/pkg/providers/launchtemplate/nodetemplate_test.go @@ -39,6 +39,7 @@ import ( "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/amifamily/bootstrap" "github.com/aws/karpenter/pkg/providers/instancetype" "github.com/aws/karpenter/pkg/test" @@ -707,7 +708,10 @@ var _ = Describe("NodeTemplate/LaunchTemplates", func() { It("should calculate memory overhead based on eni limited pods when ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 @@ -718,7 +722,10 @@ var _ = Describe("NodeTemplate/LaunchTemplates", func() { It("should calculate memory overhead based on eni limited pods when not ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 @@ -757,7 +764,10 @@ var _ = Describe("NodeTemplate/LaunchTemplates", func() { It("should calculate memory overhead based on eni limited pods when ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(true), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket @@ -768,7 +778,10 @@ var _ = Describe("NodeTemplate/LaunchTemplates", func() { It("should calculate memory overhead based on max pods when not ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index 27c5fc85204a..046648db9f71 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -30,23 +30,22 @@ import ( clock "k8s.io/utils/clock/testing" . "knative.dev/pkg/logging/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/controllers/provisioning" "github.com/aws/karpenter-core/pkg/controllers/state" "github.com/aws/karpenter-core/pkg/events" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/cloudprovider" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" ) var ctx context.Context var stop context.CancelFunc -var opts *options.Options var env *coretest.Environment var awsEnv *test.Environment var fakeClock *clock.FakeClock @@ -62,7 +61,8 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) awsEnv = test.NewEnvironment(ctx, env) @@ -80,8 +80,8 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = options.ToContext(ctx, opts) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) cluster.Reset() awsEnv.Reset() diff --git a/pkg/providers/pricing/suite_test.go b/pkg/providers/pricing/suite_test.go index 4f7e5416b017..029ed70322a5 100644 --- a/pkg/providers/pricing/suite_test.go +++ b/pkg/providers/pricing/suite_test.go @@ -29,8 +29,7 @@ import ( "k8s.io/apimachinery/pkg/types" . "knative.dev/pkg/logging/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -38,13 +37,13 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/test" ) var ctx context.Context var stop context.CancelFunc -var opts *options.Options var env *coretest.Environment var awsEnv *test.Environment var controller *pricing.Controller @@ -57,7 +56,8 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) awsEnv = test.NewEnvironment(ctx, env) @@ -70,8 +70,8 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = options.ToContext(ctx, opts) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) awsEnv.Reset() diff --git a/pkg/providers/securitygroup/suite_test.go b/pkg/providers/securitygroup/suite_test.go index f3de49b1a452..6cd9bd8cf88e 100644 --- a/pkg/providers/securitygroup/suite_test.go +++ b/pkg/providers/securitygroup/suite_test.go @@ -28,10 +28,10 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -39,7 +39,6 @@ import ( var ctx context.Context var stop context.CancelFunc -var opts *options.Options var env *coretest.Environment var awsEnv *test.Environment var nodeClass *v1beta1.EC2NodeClass @@ -52,7 +51,8 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) awsEnv = test.NewEnvironment(ctx, env) @@ -64,8 +64,8 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = options.ToContext(ctx, opts) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ Spec: v1beta1.EC2NodeClassSpec{ diff --git a/pkg/providers/subnet/suite_test.go b/pkg/providers/subnet/suite_test.go index 8e644cbbf7cd..1c8b8f240e6d 100644 --- a/pkg/providers/subnet/suite_test.go +++ b/pkg/providers/subnet/suite_test.go @@ -28,10 +28,10 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -39,7 +39,6 @@ import ( var ctx context.Context var stop context.CancelFunc -var opts *options.Options var env *coretest.Environment var awsEnv *test.Environment var nodeClass *v1beta1.EC2NodeClass @@ -52,7 +51,8 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) awsEnv = test.NewEnvironment(ctx, env) @@ -64,8 +64,8 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = options.ToContext(ctx, opts) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ Spec: v1beta1.EC2NodeClassSpec{ diff --git a/pkg/test/options.go b/pkg/test/options.go new file mode 100644 index 000000000000..378b8e2dbc5a --- /dev/null +++ b/pkg/test/options.go @@ -0,0 +1,57 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "fmt" + "time" + + "github.com/imdario/mergo" + "github.com/samber/lo" + + "github.com/aws/karpenter/pkg/operator/options" +) + +type OptionsFields struct { + AssumeRoleARN *string + AssumeRoleDuration *time.Duration + ClusterCABundle *string + ClusterName *string + ClusterEndpoint *string + IsolatedVPC *bool + VMMemoryOverheadPercent *float64 + InterruptionQueueName *string + ReservedENIs *int +} + +func Options(overrides ...OptionsFields) *options.Options { + opts := OptionsFields{} + for _, override := range overrides { + if err := mergo.Merge(&opts, override, mergo.WithOverride); err != nil { + panic(fmt.Sprintf("Failed to merge settings: %s", err)) + } + } + return &options.Options{ + AssumeRoleARN: lo.FromPtrOr(opts.AssumeRoleARN, ""), + AssumeRoleDuration: lo.FromPtrOr(opts.AssumeRoleDuration, 15*time.Minute), + ClusterCABundle: lo.FromPtrOr(opts.ClusterCABundle, ""), + ClusterName: lo.FromPtrOr(opts.ClusterName, "test-cluster"), + ClusterEndpoint: lo.FromPtrOr(opts.ClusterEndpoint, "https://test-cluster"), + IsolatedVPC: lo.FromPtrOr(opts.IsolatedVPC, false), + VMMemoryOverheadPercent: lo.FromPtrOr(opts.VMMemoryOverheadPercent, 0.075), + InterruptionQueueName: lo.FromPtrOr(opts.InterruptionQueueName, ""), + ReservedENIs: lo.FromPtrOr(opts.ReservedENIs, 0), + } +} diff --git a/test/suites/chaos/suite_test.go b/test/suites/chaos/suite_test.go index 1f12d51e6afd..471151070fbd 100644 --- a/test/suites/chaos/suite_test.go +++ b/test/suites/chaos/suite_test.go @@ -17,6 +17,7 @@ package chaos_test import ( "context" "fmt" + "os" "sync/atomic" "testing" "time" @@ -38,7 +39,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" nodeutils "github.com/aws/karpenter-core/pkg/utils/node" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/test/pkg/debug" @@ -69,8 +69,8 @@ var _ = Describe("Chaos", func() { defer cancel() provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ @@ -114,8 +114,8 @@ var _ = Describe("Chaos", func() { defer cancel() provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ diff --git a/test/suites/consolidation/suite_test.go b/test/suites/consolidation/suite_test.go index 1aed325442f8..cfcec68c2f96 100644 --- a/test/suites/consolidation/suite_test.go +++ b/test/suites/consolidation/suite_test.go @@ -16,11 +16,13 @@ package consolidation_test import ( "fmt" + "os" "strings" "testing" "time" "github.com/aws/aws-sdk-go/aws" + "github.com/samber/lo" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -28,7 +30,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/test/pkg/debug" @@ -60,8 +61,8 @@ var _ = AfterEach(func() { env.AfterEach() }) var _ = Describe("Consolidation", func() { It("should consolidate nodes (delete)", Label(debug.NoWatch), Label(debug.NoEvents), func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ @@ -126,8 +127,8 @@ var _ = Describe("Consolidation", func() { }) It("should consolidate on-demand nodes (replace)", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ @@ -242,8 +243,8 @@ var _ = Describe("Consolidation", func() { }) It("should consolidate on-demand nodes to spot (replace)", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ diff --git a/test/suites/drift/suite_test.go b/test/suites/drift/suite_test.go index 0aae96f77991..4605710f4762 100644 --- a/test/suites/drift/suite_test.go +++ b/test/suites/drift/suite_test.go @@ -16,6 +16,7 @@ package drift_test import ( "fmt" + "os" "sort" "strings" "testing" @@ -39,7 +40,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -73,8 +73,8 @@ var _ = Describe("Drift", Label("AWS"), func() { BeforeEach(func() { customAMI = env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", 1) nodeTemplate = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner = test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{{Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeOnDemand}}}, @@ -101,7 +101,7 @@ var _ = Describe("Drift", Label("AWS"), func() { oldCustomAMI := *parameter.Parameter.Value nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyCustom nodeTemplate.Spec.AMISelector = map[string]string{"aws-ids": oldCustomAMI} - nodeTemplate.Spec.UserData = awssdk.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", settings.FromContext(env.Context).ClusterName)) + nodeTemplate.Spec.UserData = awssdk.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", lo.Must(os.LookupEnv("CLUSTER_NAME")))) env.ExpectCreated(pod, nodeTemplate, provisioner) env.EventuallyExpectHealthy(pod) @@ -134,7 +134,7 @@ var _ = Describe("Drift", Label("AWS"), func() { oldCustomAMI := *parameter.Parameter.Value nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyCustom nodeTemplate.Spec.AMISelector = map[string]string{"aws-ids": oldCustomAMI} - nodeTemplate.Spec.UserData = awssdk.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", settings.FromContext(env.Context).ClusterName)) + nodeTemplate.Spec.UserData = awssdk.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", lo.Must(os.LookupEnv("CLUSTER_NAME")))) env.ExpectCreated(pod, nodeTemplate, provisioner) env.EventuallyExpectHealthy(pod) @@ -151,7 +151,7 @@ var _ = Describe("Drift", Label("AWS"), func() { }) It("should deprovision nodes that have drifted due to securitygroup", func() { By("getting the cluster vpc id") - output, err := env.EKSAPI.DescribeCluster(&eks.DescribeClusterInput{Name: awssdk.String(settings.FromContext(env.Context).ClusterName)}) + output, err := env.EKSAPI.DescribeCluster(&eks.DescribeClusterInput{Name: awssdk.String(lo.Must(os.LookupEnv("CLUSTER_NAME")))}) Expect(err).To(BeNil()) By("creating new security group") @@ -165,11 +165,11 @@ var _ = Describe("Drift", Label("AWS"), func() { Tags: []*ec2.Tag{ { Key: awssdk.String("karpenter.sh/discovery"), - Value: awssdk.String(settings.FromContext(env.Context).ClusterName), + Value: awssdk.String(lo.Must(os.LookupEnv("CLUSTER_NAME"))), }, { Key: awssdk.String(test.DiscoveryLabel), - Value: awssdk.String(settings.FromContext(env.Context).ClusterName), + Value: awssdk.String(lo.Must(os.LookupEnv("CLUSTER_NAME"))), }, { Key: awssdk.String("creation-date"), @@ -185,7 +185,7 @@ var _ = Describe("Drift", Label("AWS"), func() { var securitygroups []aws.SecurityGroup var testSecurityGroup aws.SecurityGroup Eventually(func(g Gomega) { - securitygroups = env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + securitygroups = env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}) testSecurityGroup, _ = lo.Find(securitygroups, func(sg aws.SecurityGroup) bool { return awssdk.StringValue(sg.GroupName) == "security-group-drift" }) @@ -222,7 +222,7 @@ var _ = Describe("Drift", Label("AWS"), func() { env.EventuallyExpectNotFound(pod, machine, node) }) It("should deprovision nodes that have drifted due to subnets", func() { - subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}) Expect(len(subnets)).To(BeNumerically(">", 1)) nodeTemplate.Spec.SubnetSelector = map[string]string{"aws-ids": subnets[0].ID} @@ -300,7 +300,7 @@ var _ = Describe("Drift", Label("AWS"), func() { ) DescribeTable("AWSNodeTemplate Drift", func(fieldName string, nodeTemplateSpec v1alpha1.AWSNodeTemplateSpec) { if fieldName == "InstanceProfile" { - nodeTemplateSpec.AWS.InstanceProfile = awssdk.String(fmt.Sprintf("KarpenterNodeInstanceProfile-Drift-%s", settings.FromContext(env.Context).ClusterName)) + nodeTemplateSpec.AWS.InstanceProfile = awssdk.String(fmt.Sprintf("KarpenterNodeInstanceProfile-Drift-%s", lo.Must(os.LookupEnv("CLUSTER_NAME")))) ExpectInstanceProfileCreated(nodeTemplateSpec.AWS.InstanceProfile) } @@ -476,7 +476,7 @@ func ExpectInstanceProfileCreated(instanceProfileName *string) { Tags: []*iam.Tag{ { Key: awssdk.String(test.DiscoveryLabel), - Value: awssdk.String(settings.FromContext(env.Context).ClusterName), + Value: awssdk.String(lo.Must(os.LookupEnv("CLUSTER_NAME"))), }, }, } @@ -485,7 +485,7 @@ func ExpectInstanceProfileCreated(instanceProfileName *string) { Expect(ignoreAlreadyExists(err)).ToNot(HaveOccurred()) addInstanceProfile := &iam.AddRoleToInstanceProfileInput{ InstanceProfileName: instanceProfileName, - RoleName: awssdk.String(fmt.Sprintf("KarpenterNodeRole-%s", settings.FromContext(env.Context).ClusterName)), + RoleName: awssdk.String(fmt.Sprintf("KarpenterNodeRole-%s", lo.Must(os.LookupEnv("CLUSTER_NAME")))), } _, err = env.IAMAPI.AddRoleToInstanceProfile(addInstanceProfile) Expect(ignoreAlreadyContainsRole(err)).ToNot(HaveOccurred()) diff --git a/test/suites/expiration/expiration_test.go b/test/suites/expiration/expiration_test.go index 7a3fbc83abbd..e1a38c21cf5d 100644 --- a/test/suites/expiration/expiration_test.go +++ b/test/suites/expiration/expiration_test.go @@ -15,6 +15,7 @@ limitations under the License. package expiration_test import ( + "os" "testing" "time" @@ -34,7 +35,6 @@ import ( "github.com/aws/aws-sdk-go/service/ssm" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" - "github.com/aws/karpenter/pkg/apis/settings" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -67,8 +67,8 @@ var _ = Describe("Expiration", func() { var provisioner *v1alpha5.Provisioner BeforeEach(func() { nodeTemplate = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner = test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: nodeTemplate.Name}, diff --git a/test/suites/integration/ami_test.go b/test/suites/integration/ami_test.go index 358c5f5dd526..85150fb167ab 100644 --- a/test/suites/integration/ami_test.go +++ b/test/suites/integration/ami_test.go @@ -26,13 +26,13 @@ import ( "github.com/aws/aws-sdk-go/service/ssm" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -48,8 +48,8 @@ var _ = Describe("AMI", func() { It("should use the AMI defined by the AMI Selector", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, AMIFamily: &v1alpha1.AMIFamilyAL2, }, AMISelector: map[string]string{"aws-ids": customAMI}, @@ -71,12 +71,12 @@ var _ = Describe("AMI", func() { Expect(err).To(BeNil()) oldCustomAMI := *parameter.Parameter.Value provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, AMIFamily: &v1alpha1.AMIFamilyCustom, }, AMISelector: map[string]string{"aws-ids": fmt.Sprintf("%s,%s", customAMI, oldCustomAMI)}, - UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", settings.FromContext(env.Context).ClusterName)), + UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", lo.Must(os.LookupEnv("CLUSTER_NAME")))), }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) pod := test.Pod() @@ -95,12 +95,12 @@ var _ = Describe("AMI", func() { Expect(output.Images).To(HaveLen(1)) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, AMIFamily: &v1alpha1.AMIFamilyCustom, }, AMISelector: map[string]string{"aws::name": *output.Images[0].Name, "aws::owners": "fakeOwnerValue"}, - UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", settings.FromContext(env.Context).ClusterName)), + UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", lo.Must(os.LookupEnv("CLUSTER_NAME")))), }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) @@ -118,12 +118,12 @@ var _ = Describe("AMI", func() { Expect(output.Images).To(HaveLen(1)) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, AMIFamily: &v1alpha1.AMIFamilyCustom, }, AMISelector: map[string]string{"aws::name": *output.Images[0].Name}, - UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", settings.FromContext(env.Context).ClusterName)), + UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", lo.Must(os.LookupEnv("CLUSTER_NAME")))), }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) @@ -137,12 +137,12 @@ var _ = Describe("AMI", func() { }) It("should support ami selector aws::ids", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, AMIFamily: &v1alpha1.AMIFamilyCustom, }, AMISelector: map[string]string{"aws::ids": customAMI}, - UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", settings.FromContext(env.Context).ClusterName)), + UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", lo.Must(os.LookupEnv("CLUSTER_NAME")))), }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) pod := test.Pod() @@ -157,8 +157,8 @@ var _ = Describe("AMI", func() { Context("AMIFamily", func() { It("should provision a node using the AL2 family", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, @@ -170,8 +170,8 @@ var _ = Describe("AMI", func() { }) It("should provision a node using the Bottlerocket family", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, AMIFamily: &v1alpha1.AMIFamilyBottlerocket, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ @@ -184,8 +184,8 @@ var _ = Describe("AMI", func() { }) It("should provision a node using the Ubuntu family", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, AMIFamily: &v1alpha1.AMIFamilyUbuntu, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ @@ -212,12 +212,12 @@ var _ = Describe("AMI", func() { }) It("should support Custom AMIFamily with AMI Selectors", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, AMIFamily: &v1alpha1.AMIFamilyCustom, }, AMISelector: map[string]string{"aws-ids": customAMI}, - UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", settings.FromContext(env.Context).ClusterName)), + UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", lo.Must(os.LookupEnv("CLUSTER_NAME")))), }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) pod := test.Pod() @@ -231,8 +231,8 @@ var _ = Describe("AMI", func() { It("should have the AWSNodeTemplateStatus for AMIs using wildcard", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }, AMISelector: map[string]string{"aws::name": "*"}, }) @@ -244,8 +244,8 @@ var _ = Describe("AMI", func() { It("should have the AWSNodeTemplateStatus for AMIs using tags", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }, AMISelector: map[string]string{"aws-ids": customAMI}, }) @@ -263,8 +263,8 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/al2_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, AMIFamily: &v1alpha1.AMIFamilyAL2, }, UserData: aws.String(string(content)), @@ -292,8 +292,8 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/al2_no_mime_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, AMIFamily: &v1alpha1.AMIFamilyAL2, }, UserData: aws.String(string(content)), @@ -321,8 +321,8 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/br_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, AMIFamily: &v1alpha1.AMIFamilyBottlerocket, }, UserData: aws.String(string(content)), @@ -353,8 +353,8 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/windows_userdata_input.ps1") Expect(err).ToNot(HaveOccurred()) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, AMIFamily: &v1alpha1.AMIFamilyWindows2022, }, UserData: aws.String(string(content)), diff --git a/test/suites/integration/aws_metadata_test.go b/test/suites/integration/aws_metadata_test.go index 018d29010c9a..25b9689f966f 100644 --- a/test/suites/integration/aws_metadata_test.go +++ b/test/suites/integration/aws_metadata_test.go @@ -15,14 +15,16 @@ limitations under the License. package integration_test import ( + "os" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -32,8 +34,8 @@ var _ = Describe("MetadataOptions", func() { It("should use specified metadata options", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, LaunchTemplate: v1alpha1.LaunchTemplate{ MetadataOptions: &v1alpha1.MetadataOptions{ HTTPEndpoint: aws.String("enabled"), diff --git a/test/suites/integration/backwards_compatability_test.go b/test/suites/integration/backwards_compatability_test.go index 87c609a201a5..02bbd512def7 100644 --- a/test/suites/integration/backwards_compatability_test.go +++ b/test/suites/integration/backwards_compatability_test.go @@ -15,13 +15,14 @@ limitations under the License. package integration_test import ( + "os" + "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/samber/lo" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" ) @@ -30,8 +31,8 @@ var _ = Describe("BackwardsCompatability", func() { provisioner := test.Provisioner( test.ProvisionerOptions{ Provider: &v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, Tags: map[string]string{ "custom-tag": "custom-value", "custom-tag2": "custom-value2", diff --git a/test/suites/integration/block_device_mappings_test.go b/test/suites/integration/block_device_mappings_test.go index 3e11d74e5154..8d1596ec5b79 100644 --- a/test/suites/integration/block_device_mappings_test.go +++ b/test/suites/integration/block_device_mappings_test.go @@ -15,14 +15,16 @@ limitations under the License. package integration_test import ( + "os" + "github.com/aws/aws-sdk-go/aws" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" "github.com/aws/karpenter-core/pkg/utils/resources" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -32,8 +34,8 @@ var _ = Describe("BlockDeviceMappings", func() { It("should use specified block device mappings", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, LaunchTemplate: v1alpha1.LaunchTemplate{ BlockDeviceMappings: []*v1alpha1.BlockDeviceMapping{ { diff --git a/test/suites/integration/cni_test.go b/test/suites/integration/cni_test.go index d686de9b2707..12b4c24afd2c 100644 --- a/test/suites/integration/cni_test.go +++ b/test/suites/integration/cni_test.go @@ -15,18 +15,19 @@ limitations under the License. package integration_test import ( + "os" "strconv" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -39,8 +40,8 @@ var _ = Describe("CNITests", func() { }) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, AMIFamily: &v1alpha1.AMIFamilyAL2, }, }) @@ -58,8 +59,8 @@ var _ = Describe("CNITests", func() { It("should set eni-limited maxPods when AWSENILimited when AWS_ENI_LIMITED_POD_DENSITY is true", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, AMIFamily: &v1alpha1.AMIFamilyAL2, }, }) @@ -79,8 +80,8 @@ var _ = Describe("CNITests", func() { }) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, AMIFamily: &v1alpha1.AMIFamilyAL2, }, }) diff --git a/test/suites/integration/daemonset_test.go b/test/suites/integration/daemonset_test.go index 3498429f1e0c..0be728094234 100644 --- a/test/suites/integration/daemonset_test.go +++ b/test/suites/integration/daemonset_test.go @@ -15,6 +15,8 @@ limitations under the License. package integration_test import ( + "os" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" schedulingv1 "k8s.io/api/scheduling/v1" @@ -26,7 +28,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -45,8 +46,8 @@ var _ = Describe("DaemonSet", func() { BeforeEach(func() { provider = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner = test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, diff --git a/test/suites/integration/emptiness_test.go b/test/suites/integration/emptiness_test.go index 3bfa2eaa2bdb..e5dde1537546 100644 --- a/test/suites/integration/emptiness_test.go +++ b/test/suites/integration/emptiness_test.go @@ -15,6 +15,8 @@ limitations under the License. package integration_test import ( + "os" + "k8s.io/apimachinery/pkg/labels" "knative.dev/pkg/ptr" @@ -22,10 +24,10 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" ) @@ -33,8 +35,8 @@ import ( var _ = Describe("Emptiness", func() { It("should terminate an empty node", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, diff --git a/test/suites/integration/extended_resources_test.go b/test/suites/integration/extended_resources_test.go index 93850bdc87f8..1299f218c6f1 100644 --- a/test/suites/integration/extended_resources_test.go +++ b/test/suites/integration/extended_resources_test.go @@ -30,7 +30,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -41,8 +40,8 @@ var _ = Describe("Extended Resources", func() { ExpectNvidiaDevicePluginCreated() provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, @@ -80,8 +79,8 @@ var _ = Describe("Extended Resources", func() { // For Bottlerocket, we are testing that resources are initialized without needing a device plugin provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ AMIFamily: &v1alpha1.AMIFamilyBottlerocket, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, @@ -124,8 +123,8 @@ var _ = Describe("Extended Resources", func() { "aws.enablePodENI": "true", }) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, @@ -178,8 +177,8 @@ var _ = Describe("Extended Resources", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ AMIFamily: &v1alpha1.AMIFamilyCustom, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }, AMISelector: map[string]string{ "aws-ids": customAMI, @@ -195,8 +194,8 @@ var _ = Describe("Extended Resources", func() { }, }, }) - provider.Spec.UserData = lo.ToPtr(fmt.Sprintf(string(rawContent), settings.FromContext(env.Context).ClusterName, - settings.FromContext(env.Context).ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)) + provider.Spec.UserData = lo.ToPtr(fmt.Sprintf(string(rawContent), lo.Must(os.LookupEnv("CLUSTER_NAME")), + lo.Must(os.LookupEnv("CLUSTER_ENDPOINT")), env.ExpectCABundle(), provisioner.Name)) numPods := 1 dep := test.Deployment(test.DeploymentOptions{ @@ -231,8 +230,8 @@ var _ = Describe("Extended Resources", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }, AMISelector: map[string]string{"aws-ids": "ami-0fae925f94979981f"}, }) diff --git a/test/suites/integration/hash_test.go b/test/suites/integration/hash_test.go index 84fa5302c209..d6a786c70b5a 100644 --- a/test/suites/integration/hash_test.go +++ b/test/suites/integration/hash_test.go @@ -15,13 +15,15 @@ limitations under the License. package integration_test import ( + "os" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -31,8 +33,8 @@ var _ = Describe("CRD Hash", func() { It("should have Provisioner hash", func() { nodeTemplate := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }, }) provisioner := test.Provisioner(test.ProvisionerOptions{ @@ -54,8 +56,8 @@ var _ = Describe("CRD Hash", func() { It("should have AWSNodeTemplate hash", func() { nodeTemplate := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }, }) env.ExpectCreated(nodeTemplate) diff --git a/test/suites/integration/instance_profile_test.go b/test/suites/integration/instance_profile_test.go index bdb19893d9f6..6d020dc3a818 100644 --- a/test/suites/integration/instance_profile_test.go +++ b/test/suites/integration/instance_profile_test.go @@ -16,6 +16,7 @@ package integration_test import ( "fmt" + "os" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/iam" @@ -25,7 +26,6 @@ import ( corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" coretest "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" awserrors "github.com/aws/karpenter/pkg/errors" "github.com/aws/karpenter/pkg/providers/instanceprofile" @@ -49,7 +49,7 @@ var _ = Describe("InstanceProfile Generation", func() { Tags: map[string]string{"*": "*"}, }, }, - Role: fmt.Sprintf("KarpenterNodeRole-%s", settings.FromContext(env.Context).ClusterName), + Role: fmt.Sprintf("KarpenterNodeRole-%s", lo.Must(os.LookupEnv("CLUSTER_NAME"))), }, }) }) diff --git a/test/suites/integration/kubelet_config_test.go b/test/suites/integration/kubelet_config_test.go index 02eba693982f..cec679d420e2 100644 --- a/test/suites/integration/kubelet_config_test.go +++ b/test/suites/integration/kubelet_config_test.go @@ -16,9 +16,11 @@ package integration_test import ( "math" + "os" "time" . "github.com/onsi/ginkgo/v2" + "github.com/samber/lo" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -26,7 +28,6 @@ import ( "knative.dev/pkg/ptr" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" - "github.com/aws/karpenter/pkg/apis/settings" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -41,8 +42,8 @@ var _ = Describe("KubeletConfiguration Overrides", func() { BeforeEach(func() { nodeTemplate = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) // MaxPods needs to account for the daemonsets that will run on the nodes @@ -176,8 +177,8 @@ var _ = Describe("KubeletConfiguration Overrides", func() { }) It("should schedule pods onto separate nodes when maxPods is set", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) // MaxPods needs to account for the daemonsets that will run on the nodes @@ -219,8 +220,8 @@ var _ = Describe("KubeletConfiguration Overrides", func() { }) It("should schedule pods onto separate nodes when podsPerCore is set", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) // PodsPerCore needs to account for the daemonsets that will run on the nodes // This will have 4 pods available on each node (2 taken by daemonset pods) @@ -273,8 +274,8 @@ var _ = Describe("KubeletConfiguration Overrides", func() { }) It("should ignore podsPerCore value when Bottlerocket is used", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, AMIFamily: &v1alpha1.AMIFamilyBottlerocket, }}) // All pods should schedule to a single node since we are ignoring podsPerCore value diff --git a/test/suites/integration/scheduling_test.go b/test/suites/integration/scheduling_test.go index 72dd94c75ff4..d35064a77bd6 100644 --- a/test/suites/integration/scheduling_test.go +++ b/test/suites/integration/scheduling_test.go @@ -16,6 +16,7 @@ package integration_test import ( "fmt" + "os" "time" . "github.com/onsi/ginkgo/v2" @@ -29,7 +30,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/test/pkg/debug" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -44,8 +44,8 @@ var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { BeforeEach(func() { provider = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner = test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, diff --git a/test/suites/integration/security_group_test.go b/test/suites/integration/security_group_test.go index 17775726209f..36489e49880b 100644 --- a/test/suites/integration/security_group_test.go +++ b/test/suites/integration/security_group_test.go @@ -16,6 +16,7 @@ package integration_test import ( "fmt" + "os" "strings" "time" @@ -28,7 +29,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -37,14 +37,14 @@ import ( var _ = Describe("SecurityGroups", func() { It("should use the security-group-id selector", func() { - securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}) Expect(len(securityGroups)).To(BeNumerically(">", 1)) ids := strings.Join([]string{*securityGroups[0].GroupId, *securityGroups[1].GroupId}, ",") provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{"aws-ids": ids}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }, }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) @@ -58,7 +58,7 @@ var _ = Describe("SecurityGroups", func() { }) It("should use the security group selector with multiple tag values", func() { - securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}) Expect(len(securityGroups)).To(BeNumerically(">", 1)) first := securityGroups[0] last := securityGroups[len(securityGroups)-1] @@ -69,7 +69,7 @@ var _ = Describe("SecurityGroups", func() { lo.FromPtr(lo.FindOrElse(first.Tags, &ec2.Tag{}, func(tag *ec2.Tag) bool { return lo.FromPtr(tag.Key) == "Name" }).Value), lo.FromPtr(lo.FindOrElse(last.Tags, &ec2.Tag{}, func(tag *ec2.Tag) bool { return lo.FromPtr(tag.Key) == "Name" }).Value), )}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }, }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) @@ -85,8 +85,8 @@ var _ = Describe("SecurityGroups", func() { It("should update the AWSNodeTemplateStatus for security groups", func() { nodeTemplate := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }, }) @@ -96,7 +96,7 @@ var _ = Describe("SecurityGroups", func() { }) func EventuallyExpectSecurityGroups(env *aws.Environment, nodeTemplate *v1alpha1.AWSNodeTemplate) { - securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}) Expect(securityGroups).ToNot(HaveLen(0)) ids := sets.New(lo.Map(securityGroups, func(s aws.SecurityGroup, _ int) string { diff --git a/test/suites/integration/storage_test.go b/test/suites/integration/storage_test.go index 5dd8acf231ae..5947e3f4b0f0 100644 --- a/test/suites/integration/storage_test.go +++ b/test/suites/integration/storage_test.go @@ -16,8 +16,10 @@ package integration_test import ( "fmt" + "os" "github.com/aws/aws-sdk-go/aws" + "github.com/samber/lo" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" @@ -28,7 +30,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -53,8 +54,8 @@ var _ = Describe("Dynamic PVC", func() { } provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) @@ -91,8 +92,8 @@ var _ = Describe("Dynamic PVC", func() { var _ = Describe("Static PVC", func() { It("should run a pod with a static persistent volume", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) diff --git a/test/suites/integration/subnet_test.go b/test/suites/integration/subnet_test.go index 2158c483ce6e..0b88c73caa93 100644 --- a/test/suites/integration/subnet_test.go +++ b/test/suites/integration/subnet_test.go @@ -16,6 +16,7 @@ package integration_test import ( "fmt" + "os" "time" "github.com/aws/aws-sdk-go/service/ec2" @@ -29,7 +30,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -37,14 +37,14 @@ import ( var _ = Describe("Subnets", func() { It("should use the subnet-id selector", func() { - subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}) Expect(len(subnets)).ToNot(Equal(0)) shuffledAZs := lo.Shuffle(lo.Keys(subnets)) firstSubnet := subnets[shuffledAZs[0]][0] provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, SubnetSelector: map[string]string{"aws-ids": firstSubnet}, }, }) @@ -58,7 +58,7 @@ var _ = Describe("Subnets", func() { env.ExpectInstance(pod.Spec.NodeName).To(HaveField("SubnetId", HaveValue(Equal(firstSubnet)))) }) It("should use resource based naming as node names", func() { - subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}) Expect(len(subnets)).ToNot(Equal(0)) allSubnets := lo.Flatten(lo.Values(subnets)) @@ -70,8 +70,8 @@ var _ = Describe("Subnets", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }, }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) @@ -85,14 +85,14 @@ var _ = Describe("Subnets", func() { }) It("should use the subnet tag selector with multiple tag values", func() { // Get all the subnets for the cluster - subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}) Expect(len(subnets)).To(BeNumerically(">", 1)) firstSubnet := subnets[0] lastSubnet := subnets[len(subnets)-1] provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, SubnetSelector: map[string]string{"Name": fmt.Sprintf("%s,%s", firstSubnet.Name, lastSubnet.Name)}, }, }) @@ -107,14 +107,14 @@ var _ = Describe("Subnets", func() { }) It("should use a subnet within the AZ requested", func() { - subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}) Expect(len(subnets)).ToNot(Equal(0)) shuffledAZs := lo.Shuffle(lo.Keys(subnets)) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }, }) provisioner := test.Provisioner(test.ProvisionerOptions{ @@ -141,8 +141,8 @@ var _ = Describe("Subnets", func() { It("should have the AWSNodeTemplateStatus for subnets", func() { nodeTemplate := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }, }) @@ -198,7 +198,7 @@ type SubnetInfo struct { } func EventuallyExpectSubnets(env *aws.Environment, nodeTemplate *v1alpha1.AWSNodeTemplate) { - subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}) Expect(subnets).ToNot(HaveLen(0)) ids := sets.New(lo.Flatten(lo.Values(subnets))...) diff --git a/test/suites/integration/tags_test.go b/test/suites/integration/tags_test.go index fb958b140b51..bd8402fbd9de 100644 --- a/test/suites/integration/tags_test.go +++ b/test/suites/integration/tags_test.go @@ -15,6 +15,8 @@ limitations under the License. package integration_test import ( + "os" + "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -29,7 +31,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" coretest "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/apis/v1beta1" "github.com/aws/karpenter/pkg/providers/instance" @@ -41,8 +42,8 @@ var _ = Describe("Tags", func() { It("should tag all associated resources", func() { provider := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, Tags: map[string]string{"TestTag": "TestVal"}, }, }) @@ -62,8 +63,8 @@ var _ = Describe("Tags", func() { It("should tag all associated resources with global tags", func() { provider := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }, }) @@ -94,10 +95,10 @@ var _ = Describe("Tags", func() { BeforeEach(func() { nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{ SecurityGroupSelectorTerms: []v1beta1.SecurityGroupSelectorTerm{{ - Tags: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + Tags: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}, SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{{ - Tags: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + Tags: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}, }}) @@ -169,8 +170,8 @@ var _ = Describe("Tags", func() { It("shouldn't tag nodes provisioned by v1alpha5 provisioner", func() { nodeTemplate := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := coretest.Provisioner(coretest.ProvisionerOptions{ diff --git a/test/suites/integration/termination_test.go b/test/suites/integration/termination_test.go index 9eb99453016f..3a5ead7a0584 100644 --- a/test/suites/integration/termination_test.go +++ b/test/suites/integration/termination_test.go @@ -15,6 +15,7 @@ limitations under the License. package integration_test import ( + "os" "time" . "github.com/onsi/ginkgo/v2" @@ -23,7 +24,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" ) @@ -31,8 +31,8 @@ import ( var _ = Describe("Termination", func() { It("should terminate the node and the instance on deletion", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, diff --git a/test/suites/integration/webhook_test.go b/test/suites/integration/webhook_test.go index a946f0be33bb..9112e8606263 100644 --- a/test/suites/integration/webhook_test.go +++ b/test/suites/integration/webhook_test.go @@ -16,16 +16,17 @@ package integration_test import ( "fmt" + "os" v1 "k8s.io/api/core/v1" "knative.dev/pkg/ptr" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" ) @@ -133,8 +134,8 @@ var _ = Describe("Webhooks", func() { It("should error when provider and providerRef are combined", func() { Expect(env.Client.Create(env, test.Provisioner(test.ProvisionerOptions{ Provider: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }, ProviderRef: &v1alpha5.MachineTemplateRef{Name: "test"}, }))).ToNot(Succeed()) @@ -266,15 +267,15 @@ var _ = Describe("Webhooks", func() { It("should error when amiSelector is not defined for amiFamily Custom", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ AMIFamily: &v1alpha1.AMIFamilyCustom, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}))).ToNot(Succeed()) }) It("should fail if both userdata and launchTemplate are set", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ LaunchTemplate: v1alpha1.LaunchTemplate{LaunchTemplateName: ptr.String("lt")}, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }, UserData: ptr.String("data"), }))).ToNot(Succeed()) @@ -282,47 +283,47 @@ var _ = Describe("Webhooks", func() { It("should fail if both amiSelector and launchTemplate are set", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ LaunchTemplate: v1alpha1.LaunchTemplate{LaunchTemplateName: ptr.String("lt")}, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }, AMISelector: map[string]string{"foo": "bar"}, }))).ToNot(Succeed()) }) It("should fail for poorly formatted aws-ids", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }, AMISelector: map[string]string{"aws-ids": "must-start-with-ami"}, }))).ToNot(Succeed()) }) It("should succeed when tags don't contain restricted keys", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, Tags: map[string]string{"karpenter.sh/custom-key": "custom-value", "kubernetes.io/role/key": "custom-value"}, }, }))).To(Succeed()) }) It("should error when tags contains a restricted key", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, Tags: map[string]string{"karpenter.sh/provisioner-name": "custom-value"}, }, }))).ToNot(Succeed()) Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - Tags: map[string]string{"karpenter.sh/managed-by": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + Tags: map[string]string{"karpenter.sh/managed-by": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }, }))).ToNot(Succeed()) Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - Tags: map[string]string{fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(env.Context).ClusterName): "owned"}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + Tags: map[string]string{fmt.Sprintf("kubernetes.io/cluster/%s", lo.Must(os.LookupEnv("CLUSTER_NAME"))): "owned"}, }, }))).ToNot(Succeed()) }) @@ -330,22 +331,22 @@ var _ = Describe("Webhooks", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME")), "aws-ids": "sg-12345", }, SubnetSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME")), }, }, }))).ToNot(Succeed()) Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME")), "aws-ids": "sg-12345", }, SubnetSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME")), }, }, }))).ToNot(Succeed()) @@ -354,10 +355,10 @@ var _ = Describe("Webhooks", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME")), }, SubnetSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME")), "aws-ids": "subnet-12345", }, }, @@ -365,10 +366,10 @@ var _ = Describe("Webhooks", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME")), }, SubnetSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME")), "aws-ids": "subnet-12345", }, }, @@ -378,10 +379,10 @@ var _ = Describe("Webhooks", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME")), }, SubnetSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME")), }, }, AMISelector: map[string]string{ @@ -392,10 +393,10 @@ var _ = Describe("Webhooks", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME")), }, SubnetSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME")), }, }, AMISelector: map[string]string{ diff --git a/test/suites/interruption/suite_test.go b/test/suites/interruption/suite_test.go index a7b4f6128a3b..db852f467cd0 100644 --- a/test/suites/interruption/suite_test.go +++ b/test/suites/interruption/suite_test.go @@ -16,11 +16,13 @@ package interruption_test import ( "fmt" + "os" "testing" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -29,10 +31,10 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/controllers/interruption/messages" "github.com/aws/karpenter/pkg/controllers/interruption/messages/scheduledchange" + "github.com/aws/karpenter/pkg/operator/options" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/pkg/utils" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -53,6 +55,9 @@ func TestInterruption(t *testing.T) { } var _ = BeforeEach(func() { + env.Context = options.ToContext(env.Context, awstest.Options(awstest.OptionsFields{ + InterruptionQueueName: lo.ToPtr(lo.Must(os.LookupEnv("INTERRUPTION_QUEUE_NAME"))), + })) env.BeforeEach() env.ExpectQueueExists() }) @@ -63,8 +68,8 @@ var _ = Describe("Interruption", Label("AWS"), func() { It("should terminate the spot instance and spin-up a new node on spot interruption warning", func() { By("Creating a single healthy node with a healthy deployment") provider = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ @@ -111,8 +116,8 @@ var _ = Describe("Interruption", Label("AWS"), func() { It("should terminate the node at the API server when the EC2 instance is stopped", func() { By("Creating a single healthy node with a healthy deployment") provider = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ @@ -151,8 +156,8 @@ var _ = Describe("Interruption", Label("AWS"), func() { It("should terminate the node at the API server when the EC2 instance is terminated", func() { By("Creating a single healthy node with a healthy deployment") provider = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ @@ -191,8 +196,8 @@ var _ = Describe("Interruption", Label("AWS"), func() { It("should terminate the node when receiving a scheduled change health event", func() { By("Creating a single healthy node with a healthy deployment") provider = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ diff --git a/test/suites/ipv6/suite_test.go b/test/suites/ipv6/suite_test.go index 2813c51c171c..6758485e65a7 100644 --- a/test/suites/ipv6/suite_test.go +++ b/test/suites/ipv6/suite_test.go @@ -16,6 +16,7 @@ package ipv6_test import ( "net" + "os" "testing" . "github.com/onsi/ginkgo/v2" @@ -25,7 +26,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -52,8 +52,8 @@ var _ = Describe("IPv6", func() { It("should provision an IPv6 node by discovering kube-dns IPv6", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, Requirements: []v1.NodeSelectorRequirement{ { @@ -82,8 +82,8 @@ var _ = Describe("IPv6", func() { clusterDNSAddr := env.ExpectIPv6ClusterDNS() provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, Requirements: []v1.NodeSelectorRequirement{ { diff --git a/test/suites/machine/garbage_collection_test.go b/test/suites/machine/garbage_collection_test.go index 4062a12f7f9e..fd808cb70713 100644 --- a/test/suites/machine/garbage_collection_test.go +++ b/test/suites/machine/garbage_collection_test.go @@ -43,8 +43,8 @@ var _ = Describe("NodeClaimGarbageCollection", func() { BeforeEach(func() { provisioner = test.Provisioner() - securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) - subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}) + subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}) Expect(securityGroups).ToNot(HaveLen(0)) Expect(subnets).ToNot(HaveLen(0)) @@ -75,7 +75,7 @@ var _ = Describe("NodeClaimGarbageCollection", func() { ResourceType: aws.String(ec2.ResourceTypeInstance), Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(env.Context).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", lo.Must(os.LookupEnv("CLUSTER_NAME")))), Value: aws.String("owned"), }, { @@ -93,8 +93,8 @@ var _ = Describe("NodeClaimGarbageCollection", func() { // Update the userData for the instance input with the correct provisionerName rawContent, err := os.ReadFile("testdata/al2_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) - instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), settings.FromContext(env.Context).ClusterName, - settings.FromContext(env.Context).ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)))) + instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), lo.Must(os.LookupEnv("CLUSTER_NAME")), + lo.Must(os.LookupEnv("CLUSTER_ENDPOINT")), env.ExpectCABundle(), provisioner.Name)))) // Create an instance manually to mock Karpenter launching an instance out := env.ExpectRunInstances(instanceInput) @@ -120,7 +120,7 @@ var _ = Describe("NodeClaimGarbageCollection", func() { Tags: []*ec2.Tag{ { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(env.Context).ClusterName), + Value: aws.String(lo.Must(os.LookupEnv("CLUSTER_NAME"))), }, }, }) @@ -139,8 +139,8 @@ var _ = Describe("NodeClaimGarbageCollection", func() { }) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, diff --git a/test/suites/machine/link_test.go b/test/suites/machine/link_test.go index 87b1c0739192..5fbf13a9d56f 100644 --- a/test/suites/machine/link_test.go +++ b/test/suites/machine/link_test.go @@ -42,8 +42,8 @@ var _ = Describe("MachineLink", func() { var instanceInput *ec2.RunInstancesInput BeforeEach(func() { - securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) - subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}) + subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}) Expect(securityGroups).ToNot(HaveLen(0)) Expect(subnets).ToNot(HaveLen(0)) @@ -74,7 +74,7 @@ var _ = Describe("MachineLink", func() { ResourceType: aws.String(ec2.ResourceTypeInstance), Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(env.Context).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", lo.Must(os.LookupEnv("CLUSTER_NAME")))), Value: aws.String("owned"), }, }, @@ -87,8 +87,8 @@ var _ = Describe("MachineLink", func() { It("should succeed to link a Machine for an existing instance launched by Karpenter", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ AMIFamily: &v1alpha1.AMIFamilyAL2, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, @@ -102,8 +102,8 @@ var _ = Describe("MachineLink", func() { Key: aws.String(v1alpha5.ProvisionerNameLabelKey), Value: aws.String(provisioner.Name), }) - instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), settings.FromContext(env.Context).ClusterName, - settings.FromContext(env.Context).ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)))) + instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), lo.Must(os.LookupEnv("CLUSTER_NAME")), + lo.Must(os.LookupEnv("CLUSTER_ENDPOINT")), env.ExpectCABundle(), provisioner.Name)))) // Create an instance manually to mock Karpenter launching an instance out := env.ExpectRunInstances(instanceInput) @@ -141,15 +141,15 @@ var _ = Describe("MachineLink", func() { return aws.StringValue(t.Key) == v1alpha5.MachineManagedByAnnotationKey }) g.Expect(ok).To(BeTrue()) - g.Expect(aws.StringValue(tag.Value)).To(Equal(settings.FromContext(env.Context).ClusterName)) + g.Expect(aws.StringValue(tag.Value)).To(Equal(lo.Must(os.LookupEnv("CLUSTER_NAME")))) }, time.Minute, time.Second).Should(Succeed()) }) It("should succeed to link a Machine for an existing instance launched by Karpenter with provider", func() { provisioner := test.Provisioner(test.ProvisionerOptions{ Provider: &v1alpha1.AWS{ AMIFamily: &v1alpha1.AMIFamilyAL2, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }, }) env.ExpectCreated(provisioner) @@ -161,8 +161,8 @@ var _ = Describe("MachineLink", func() { Key: aws.String(v1alpha5.ProvisionerNameLabelKey), Value: aws.String(provisioner.Name), }) - instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), settings.FromContext(env.Context).ClusterName, - settings.FromContext(env.Context).ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)))) + instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), lo.Must(os.LookupEnv("CLUSTER_NAME")), + lo.Must(os.LookupEnv("CLUSTER_ENDPOINT")), env.ExpectCABundle(), provisioner.Name)))) // Create an instance manually to mock Karpenter launching an instance out := env.ExpectRunInstances(instanceInput) @@ -200,14 +200,14 @@ var _ = Describe("MachineLink", func() { return aws.StringValue(t.Key) == v1alpha5.MachineManagedByAnnotationKey }) g.Expect(ok).To(BeTrue()) - g.Expect(aws.StringValue(tag.Value)).To(Equal(settings.FromContext(env.Context).ClusterName)) + g.Expect(aws.StringValue(tag.Value)).To(Equal(lo.Must(os.LookupEnv("CLUSTER_NAME")))) }, time.Minute, time.Second).Should(Succeed()) }) It("should succeed to link a Machine for an existing instance re-owned by Karpenter", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ AMIFamily: &v1alpha1.AMIFamilyAL2, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, @@ -219,8 +219,8 @@ var _ = Describe("MachineLink", func() { Expect(err).ToNot(HaveOccurred()) // No tag specifications since we're mocking an instance not launched by Karpenter - instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), settings.FromContext(env.Context).ClusterName, - settings.FromContext(env.Context).ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)))) + instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), lo.Must(os.LookupEnv("CLUSTER_NAME")), + lo.Must(os.LookupEnv("CLUSTER_ENDPOINT")), env.ExpectCABundle(), provisioner.Name)))) // Create an instance manually to mock Karpenter launching an instance out := env.ExpectRunInstances(instanceInput) @@ -263,7 +263,7 @@ var _ = Describe("MachineLink", func() { return aws.StringValue(t.Key) == v1alpha5.MachineManagedByAnnotationKey }) g.Expect(ok).To(BeTrue()) - g.Expect(aws.StringValue(tag.Value)).To(Equal(settings.FromContext(env.Context).ClusterName)) + g.Expect(aws.StringValue(tag.Value)).To(Equal(lo.Must(os.LookupEnv("CLUSTER_NAME")))) tag, ok = lo.Find(instance.Tags, func(t *ec2.Tag) bool { return aws.StringValue(t.Key) == v1alpha5.ProvisionerNameLabelKey }) diff --git a/test/suites/machine/machine_test.go b/test/suites/machine/machine_test.go index f92d4ec81734..c570e4bf11b9 100644 --- a/test/suites/machine/machine_test.go +++ b/test/suites/machine/machine_test.go @@ -31,7 +31,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" "github.com/aws/karpenter-core/pkg/utils/resources" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" ) @@ -40,8 +39,8 @@ var _ = Describe("StandaloneMachine", func() { var nodeTemplate *v1alpha1.AWSNodeTemplate BeforeEach(func() { nodeTemplate = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) }) It("should create a standard machine within the 'c' instance family", func() { @@ -273,8 +272,8 @@ var _ = Describe("StandaloneMachine", func() { // Create userData that adds custom labels through the --kubelet-extra-args nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyCustom nodeTemplate.Spec.AMISelector = map[string]string{"aws-ids": customAMI} - nodeTemplate.Spec.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), settings.FromContext(env.Context).ClusterName, - settings.FromContext(env.Context).ClusterEndpoint, env.ExpectCABundle())))) + nodeTemplate.Spec.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), lo.Must(os.LookupEnv("CLUSTER_NAME")), + lo.Must(os.LookupEnv("CLUSTER_ENDPOINT")), env.ExpectCABundle())))) machine := test.Machine(v1alpha5.Machine{ Spec: v1alpha5.MachineSpec{ diff --git a/test/suites/scale/deprovisioning_test.go b/test/suites/scale/deprovisioning_test.go index e4f818297530..f007f099ee22 100644 --- a/test/suites/scale/deprovisioning_test.go +++ b/test/suites/scale/deprovisioning_test.go @@ -17,6 +17,7 @@ package scale_test import ( "context" "fmt" + "os" "strconv" "sync" "time" @@ -32,7 +33,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/controllers/interruption/messages" "github.com/aws/karpenter/pkg/controllers/interruption/messages/scheduledchange" @@ -86,8 +86,8 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), "featureGates.driftEnabled": "true", }) nodeTemplate = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisionerOptions = test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{ diff --git a/test/suites/scale/provisioning_test.go b/test/suites/scale/provisioning_test.go index b59667ce00fd..dba5acefe91e 100644 --- a/test/suites/scale/provisioning_test.go +++ b/test/suites/scale/provisioning_test.go @@ -16,6 +16,7 @@ package scale_test import ( "context" + "os" "strconv" "time" @@ -28,7 +29,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/test/pkg/debug" @@ -46,8 +46,8 @@ var _ = Describe("Provisioning", Label(debug.NoWatch), Label(debug.NoEvents), fu BeforeEach(func() { nodeTemplate = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner = test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{ diff --git a/test/suites/utilization/suite_test.go b/test/suites/utilization/suite_test.go index 9504da9fa598..b0084f6d181c 100644 --- a/test/suites/utilization/suite_test.go +++ b/test/suites/utilization/suite_test.go @@ -15,17 +15,18 @@ limitations under the License. package utilization_test import ( + "os" "testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/labels" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/test/pkg/debug" @@ -53,8 +54,8 @@ var _ = AfterEach(func() { env.AfterEach() }) var _ = Describe("Utilization", Label(debug.NoWatch), Label(debug.NoEvents), func() { It("should provision one pod per node", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": lo.Must(os.LookupEnv("CLUSTER_NAME"))}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, Requirements: []v1.NodeSelectorRequirement{{ Key: v1.LabelInstanceTypeStable,