From da2f03c61fe1066aef096651f6f399958f2bc102 Mon Sep 17 00:00:00 2001 From: Jason Deal Date: Wed, 4 Oct 2023 16:58:23 -0700 Subject: [PATCH 1/4] add options --- .../main.go} | 0 .../main.go} | 0 .../{prices_gen.go => prices_gen/main.go} | 4 +- .../main.go} | 4 +- hack/codegen.sh | 8 +- hack/docs/configuration_gen_docs.go | 4 +- hack/docs/instancetypes_gen_docs.go | 8 +- pkg/apis/settings/settings.go | 9 ++ pkg/apis/settings/settings_validation.go | 8 -- pkg/apis/settings/suite_test.go | 9 -- pkg/cloudprovider/suite_test.go | 12 +- pkg/controllers/controllers.go | 6 +- pkg/controllers/interruption/controller.go | 6 +- .../interruption_benchmark_test.go | 10 +- pkg/controllers/interruption/sqs.go | 8 +- pkg/controllers/interruption/suite_test.go | 8 +- .../garbagecollection/machine_test.go | 18 +-- .../garbagecollection/nodeclaim_test.go | 18 +-- .../nodeclaim/garbagecollection/suite_test.go | 20 +-- pkg/controllers/nodeclaim/link/suite_test.go | 12 +- .../nodeclaim/tagging/suite_test.go | 10 +- pkg/controllers/nodeclass/suite_test.go | 10 +- pkg/operator/operator.go | 16 +-- pkg/operator/options/options.go | 134 ++++++++++++++++++ pkg/operator/options/options_validation.go | 82 +++++++++++ pkg/operator/options/suite_test.go | 118 +++++++++++++++ pkg/operator/suite_test.go | 11 +- pkg/providers/amifamily/ami_test.go | 6 +- pkg/providers/instance/instance.go | 15 +- pkg/providers/instance/nodeclass_test.go | 6 +- pkg/providers/instance/nodetemplate_test.go | 6 +- pkg/providers/instance/suite_test.go | 16 +-- .../instanceprofile/instanceprofile.go | 12 +- pkg/providers/instancetype/nodeclass_test.go | 19 +-- .../instancetype/nodetemplate_test.go | 19 +-- pkg/providers/instancetype/suite_test.go | 14 +- pkg/providers/instancetype/types.go | 11 +- .../launchtemplate/launchtemplate.go | 5 +- .../launchtemplate/nodeclass_test.go | 21 ++- .../launchtemplate/nodetemplate_test.go | 21 ++- pkg/providers/launchtemplate/suite_test.go | 12 +- pkg/providers/pricing/suite_test.go | 12 +- pkg/providers/securitygroup/suite_test.go | 12 +- pkg/providers/subnet/suite_test.go | 12 +- pkg/test/options.go | 57 ++++++++ test/pkg/environment/aws/environment.go | 9 ++ test/suites/chaos/suite_test.go | 15 +- test/suites/consolidation/suite_test.go | 13 +- test/suites/drift/suite_test.go | 25 ++-- test/suites/expiration/expiration_test.go | 5 +- test/suites/integration/ami_test.go | 71 +++++----- test/suites/integration/aws_metadata_test.go | 5 +- .../backwards_compatability_test.go | 5 +- .../integration/block_device_mappings_test.go | 5 +- test/suites/integration/cni_test.go | 13 +- test/suites/integration/daemonset_test.go | 5 +- test/suites/integration/emptiness_test.go | 5 +- .../integration/extended_resources_test.go | 25 ++-- test/suites/integration/hash_test.go | 9 +- .../integration/instance_profile_test.go | 3 +- .../suites/integration/kubelet_config_test.go | 17 ++- test/suites/integration/scheduling_test.go | 5 +- .../suites/integration/security_group_test.go | 15 +- test/suites/integration/storage_test.go | 9 +- test/suites/integration/subnet_test.go | 27 ++-- test/suites/integration/tags_test.go | 17 ++- test/suites/integration/termination_test.go | 5 +- test/suites/integration/webhook_test.go | 65 +++++---- test/suites/interruption/suite_test.go | 22 +-- test/suites/ipv6/suite_test.go | 9 +- .../suites/machine/garbage_collection_test.go | 16 +-- test/suites/machine/link_test.go | 36 ++--- test/suites/machine/machine_test.go | 9 +- test/suites/scale/deprovisioning_test.go | 5 +- test/suites/scale/provisioning_test.go | 5 +- test/suites/utilization/suite_test.go | 5 +- 76 files changed, 851 insertions(+), 458 deletions(-) rename hack/code/{bandwidth_gen.go => bandwidth_gen/main.go} (100%) rename hack/code/{instancetype_testdata_gen.go => instancetype_testdata_gen/main.go} (100%) rename hack/code/{prices_gen.go => prices_gen/main.go} (98%) rename hack/code/{vpc_limits_gen.go => vpc_limits_gen/main.go} (97%) create mode 100644 pkg/operator/options/options.go create mode 100644 pkg/operator/options/options_validation.go create mode 100644 pkg/operator/options/suite_test.go create mode 100644 pkg/test/options.go diff --git a/hack/code/bandwidth_gen.go b/hack/code/bandwidth_gen/main.go similarity index 100% rename from hack/code/bandwidth_gen.go rename to hack/code/bandwidth_gen/main.go diff --git a/hack/code/instancetype_testdata_gen.go b/hack/code/instancetype_testdata_gen/main.go similarity index 100% rename from hack/code/instancetype_testdata_gen.go rename to hack/code/instancetype_testdata_gen/main.go diff --git a/hack/code/prices_gen.go b/hack/code/prices_gen/main.go similarity index 98% rename from hack/code/prices_gen.go rename to hack/code/prices_gen/main.go index e6dfe86cb0bb..c2bac3781adc 100644 --- a/hack/code/prices_gen.go +++ b/hack/code/prices_gen/main.go @@ -34,7 +34,7 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/aws/karpenter/pkg/apis/settings" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/test" ) @@ -93,7 +93,7 @@ func main() { os.Setenv("AWS_SDK_LOAD_CONFIG", "true") os.Setenv("AWS_REGION", region) ctx := context.Background() - ctx = settings.ToContext(ctx, test.Settings()) + ctx = options.ToContext(ctx, test.Options()) sess := session.Must(session.NewSession()) ec2 := ec22.New(sess) src := &bytes.Buffer{} diff --git a/hack/code/vpc_limits_gen.go b/hack/code/vpc_limits_gen/main.go similarity index 97% rename from hack/code/vpc_limits_gen.go rename to hack/code/vpc_limits_gen/main.go index 773777fc567c..5baaa0460050 100644 --- a/hack/code/vpc_limits_gen.go +++ b/hack/code/vpc_limits_gen/main.go @@ -26,13 +26,13 @@ import ( "time" ) -type options struct { +type Options struct { sourceOutput string urlInput string } func main() { - opts := options{} + opts := Options{} flag.StringVar(&opts.urlInput, "url", "https://raw.githubusercontent.com/aws/amazon-vpc-resource-controller-k8s/master/pkg/aws/vpc/limits.go", "url of the raw vpc/limits.go file in the github.com/aws/amazon-vpc-resource-controller-k8s repo") flag.StringVar(&opts.sourceOutput, "output", "pkg/providers/instancetype/zz_generated.vpclimits.go", "output location for the generated go source file") diff --git a/hack/codegen.sh b/hack/codegen.sh index 966c70c7cf5f..2781722d3bf2 100755 --- a/hack/codegen.sh +++ b/hack/codegen.sh @@ -12,7 +12,7 @@ bandwidth() { NO_UPDATE='' SUBJECT="Bandwidth" - go run hack/code/bandwidth_gen.go -- "${GENERATED_FILE}" + go run hack/code/bandwidth_gen/main.go -- "${GENERATED_FILE}" GIT_DIFF=$(git diff --stat "${GENERATED_FILE}") checkForUpdates "${GIT_DIFF}" "${NO_UPDATE}" "${SUBJECT}" "${GENERATED_FILE}" @@ -30,7 +30,7 @@ pricing() { NO_UPDATE=" ${GENERATED_FILE} "$'| 4 ++--\n 1 file changed, 2 insertions(+), 2 deletions(-)' SUBJECT="Pricing" - go run hack/code/prices_gen.go --partition "$partition" --output "$GENERATED_FILE" + go run hack/code/prices_gen/main.go --partition "$partition" --output "$GENERATED_FILE" GIT_DIFF=$(git diff --stat "${GENERATED_FILE}") checkForUpdates "${GIT_DIFF}" "${NO_UPDATE}" "${SUBJECT} beside timestamps since last update" "${GENERATED_FILE}" @@ -42,7 +42,7 @@ vpcLimits() { NO_UPDATE='' SUBJECT="VPC Limits" - go run hack/code/vpc_limits_gen.go -- \ + go run hack/code/vpc_limits_gen/main.go -- \ --url=https://raw.githubusercontent.com/aws/amazon-vpc-resource-controller-k8s/master/pkg/aws/vpc/limits.go \ --output="${GENERATED_FILE}" @@ -55,7 +55,7 @@ instanceTypeTestData() { NO_UPDATE='' SUBJECT="Instance Type Test Data" - go run hack/code/instancetype_testdata_gen.go --out-file ${GENERATED_FILE} \ + go run hack/code/instancetype_testdata_gen/main.go --out-file ${GENERATED_FILE} \ --instance-types t3.large,m5.large,m5.xlarge,p3.8xlarge,g4dn.8xlarge,c6g.large,inf1.2xlarge,inf1.6xlarge,trn1.2xlarge,m5.metal,dl1.24xlarge,m6idn.32xlarge,t4g.small,t4g.xlarge,t4g.medium GIT_DIFF=$(git diff --stat "${GENERATED_FILE}") diff --git a/hack/docs/configuration_gen_docs.go b/hack/docs/configuration_gen_docs.go index 7ae092d7efd4..2624a41f7e20 100644 --- a/hack/docs/configuration_gen_docs.go +++ b/hack/docs/configuration_gen_docs.go @@ -21,7 +21,8 @@ import ( "os" "strings" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" + "github.com/aws/karpenter/pkg/operator/options" ) func main() { @@ -49,6 +50,7 @@ func main() { bottomDoc := fmt.Sprintf("\n%s%s", genEnd, endDocSections[1]) fs := &options.FlagSet{FlagSet: flag.NewFlagSet("karpenter", flag.ContinueOnError)} + (&coreoptions.Options{}).AddFlags(fs) (&options.Options{}).AddFlags(fs) envVarsBlock := "| Environment Variable | CLI Flag | Description |\n" diff --git a/hack/docs/instancetypes_gen_docs.go b/hack/docs/instancetypes_gen_docs.go index 64bc679fc405..b87f3cb435ce 100644 --- a/hack/docs/instancetypes_gen_docs.go +++ b/hack/docs/instancetypes_gen_docs.go @@ -35,14 +35,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/manager" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" coreoperator "github.com/aws/karpenter-core/pkg/operator" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" coretest "github.com/aws/karpenter-core/pkg/test" nodepoolutil "github.com/aws/karpenter-core/pkg/utils/nodepool" - "github.com/aws/karpenter/pkg/apis/settings" awscloudprovider "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/operator" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter-core/pkg/cloudprovider" @@ -87,8 +87,8 @@ func main() { lo.Must0(os.Setenv("AWS_SDK_LOAD_CONFIG", "true")) lo.Must0(os.Setenv("AWS_REGION", "us-east-1")) - ctx := coresettings.ToContext(context.Background(), coretest.Settings()) - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx := coreoptions.ToContext(context.Background(), coretest.Options()) + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ClusterName: lo.ToPtr("docs-gen"), ClusterEndpoint: lo.ToPtr("https://docs-gen.aws"), IsolatedVPC: lo.ToPtr(true), // disable pricing lookup diff --git a/pkg/apis/settings/settings.go b/pkg/apis/settings/settings.go index 5a3d6467b225..052d6eb0341b 100644 --- a/pkg/apis/settings/settings.go +++ b/pkg/apis/settings/settings.go @@ -22,6 +22,8 @@ import ( v1 "k8s.io/api/core/v1" "knative.dev/pkg/configmap" + + coresettings "github.com/aws/karpenter-core/pkg/apis/settings" ) type settingsKeyType struct{} @@ -68,6 +70,9 @@ func (*Settings) ConfigMap() string { // Inject creates a Settings from the supplied ConfigMap func (*Settings) Inject(ctx context.Context, cm *v1.ConfigMap) (context.Context, error) { s := defaultSettings.DeepCopy() + if cm == nil { + return ToContext(ctx, s), nil + } if err := configmap.Parse(cm.Data, configmap.AsString("aws.assumeRoleARN", &s.AssumeRoleARN), @@ -92,6 +97,10 @@ func (*Settings) Inject(ctx context.Context, cm *v1.ConfigMap) (context.Context, return ToContext(ctx, s), nil } +func (*Settings) FromContext(ctx context.Context) coresettings.Injectable { + return FromContext(ctx) +} + func ToContext(ctx context.Context, s *Settings) context.Context { return context.WithValue(ctx, ContextKey, s) } diff --git a/pkg/apis/settings/settings_validation.go b/pkg/apis/settings/settings_validation.go index e8c3e6f9dbe3..467040bb84ff 100644 --- a/pkg/apis/settings/settings_validation.go +++ b/pkg/apis/settings/settings_validation.go @@ -28,7 +28,6 @@ func (s Settings) Validate() (errs *apis.FieldError) { return errs.Also( s.validateEndpoint(), s.validateTags(), - s.validateClusterName(), s.validateVMMemoryOverheadPercent(), s.validateReservedENIs(), s.validateAssumeRoleDuration(), @@ -42,13 +41,6 @@ func (s Settings) validateAssumeRoleDuration() (errs *apis.FieldError) { return nil } -func (s Settings) validateClusterName() (errs *apis.FieldError) { - if s.ClusterName == "" { - return errs.Also(apis.ErrMissingField("clusterName is required", "clusterName")) - } - return nil -} - func (s Settings) validateEndpoint() (errs *apis.FieldError) { if s.ClusterEndpoint == "" { return nil diff --git a/pkg/apis/settings/suite_test.go b/pkg/apis/settings/suite_test.go index 6bf00189afdb..af34c73a55c3 100644 --- a/pkg/apis/settings/suite_test.go +++ b/pkg/apis/settings/suite_test.go @@ -136,15 +136,6 @@ var _ = Describe("Validation", func() { Expect(s.Tags).To(HaveKeyWithValue("kubernetes.io/role/key", "value2")) Expect(s.Tags).To(HaveKeyWithValue("kubernetes.io/cluster/other-tag/hello", "value3")) }) - It("should fail validation with panic when clusterName not included", func() { - cm := &v1.ConfigMap{ - Data: map[string]string{ - "aws.clusterEndpoint": "https://00000000000000000000000.gr7.us-west-2.eks.amazonaws.com", - }, - } - _, err := (&settings.Settings{}).Inject(ctx, cm) - Expect(err).To(HaveOccurred()) - }) It("should fail validation when clusterEndpoint is invalid (not absolute)", func() { cm := &v1.ConfigMap{ Data: map[string]string{ diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index 14a5ce4deb98..28c9121dfb09 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -30,15 +30,15 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/pkg/cloudprovider" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/controllers/provisioning" "github.com/aws/karpenter-core/pkg/controllers/state" "github.com/aws/karpenter-core/pkg/events" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -46,7 +46,6 @@ import ( var ctx context.Context var stop context.CancelFunc -var opts *options.Options var env *coretest.Environment var awsEnv *test.Environment var prov *provisioning.Provisioner @@ -63,7 +62,8 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) awsEnv = test.NewEnvironment(ctx, env) @@ -81,8 +81,8 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = options.ToContext(ctx, opts) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) cluster.Reset() diff --git a/pkg/controllers/controllers.go b/pkg/controllers/controllers.go index 8cc8b7bea492..6ac84393cf09 100644 --- a/pkg/controllers/controllers.go +++ b/pkg/controllers/controllers.go @@ -25,7 +25,6 @@ import ( "github.com/aws/karpenter-core/pkg/events" "github.com/aws/karpenter-core/pkg/operator/controller" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/controllers/interruption" @@ -33,6 +32,7 @@ import ( nodeclaimlink "github.com/aws/karpenter/pkg/controllers/nodeclaim/link" nodeclaimtagging "github.com/aws/karpenter/pkg/controllers/nodeclaim/tagging" "github.com/aws/karpenter/pkg/controllers/nodeclass" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/providers/instanceprofile" @@ -54,10 +54,10 @@ func NewControllers(ctx context.Context, sess *session.Session, clk clock.Clock, nodeclaimgarbagecollection.NewController(kubeClient, cloudProvider, linkController), nodeclaimtagging.NewController(kubeClient, instanceProvider), } - if settings.FromContext(ctx).InterruptionQueueName != "" { + if options.FromContext(ctx).InterruptionQueueName != "" { controllers = append(controllers, interruption.NewController(kubeClient, clk, recorder, interruption.NewSQSProvider(sqs.New(sess)), unavailableOfferings)) } - if settings.FromContext(ctx).IsolatedVPC { + if options.FromContext(ctx).IsolatedVPC { logging.FromContext(ctx).Infof("assuming isolated VPC, pricing information will not be updated") } else { controllers = append(controllers, pricing.NewController(pricingProvider)) diff --git a/pkg/controllers/interruption/controller.go b/pkg/controllers/interruption/controller.go index 305c6b918599..e5d4651e610b 100644 --- a/pkg/controllers/interruption/controller.go +++ b/pkg/controllers/interruption/controller.go @@ -32,12 +32,12 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1beta1" "github.com/aws/karpenter-core/pkg/utils/pretty" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/cache" interruptionevents "github.com/aws/karpenter/pkg/controllers/interruption/events" "github.com/aws/karpenter/pkg/controllers/interruption/messages" "github.com/aws/karpenter/pkg/controllers/interruption/messages/statechange" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/utils" "github.com/aws/karpenter-core/pkg/events" @@ -80,8 +80,8 @@ func NewController(kubeClient client.Client, clk clock.Clock, recorder events.Re } func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { - ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("queue", settings.FromContext(ctx).InterruptionQueueName)) - if c.cm.HasChanged(settings.FromContext(ctx).InterruptionQueueName, nil) { + ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("queue", options.FromContext(ctx).InterruptionQueueName)) + if c.cm.HasChanged(options.FromContext(ctx).InterruptionQueueName, nil) { logging.FromContext(ctx).Debugf("watching interruption queue") } sqsMessages, err := c.sqsProvider.GetSQSMessages(ctx) diff --git a/pkg/controllers/interruption/interruption_benchmark_test.go b/pkg/controllers/interruption/interruption_benchmark_test.go index 6149bdddd35c..b0459b542a46 100644 --- a/pkg/controllers/interruption/interruption_benchmark_test.go +++ b/pkg/controllers/interruption/interruption_benchmark_test.go @@ -43,6 +43,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/aws/karpenter-core/pkg/operator/scheme" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/apis/settings" awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/controllers/interruption" @@ -50,7 +51,7 @@ import ( "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/test" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" coretest "github.com/aws/karpenter-core/pkg/test" ) @@ -77,12 +78,13 @@ func BenchmarkNotification100(b *testing.B) { func benchmarkNotificationController(b *testing.B, messageCount int) { ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("message-count", messageCount)) fakeClock = &clock.FakeClock{} - ctx = coresettings.ToContext(ctx, coretest.Settings()) - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ClusterName: lo.ToPtr("karpenter-notification-benchmarking"), IsolatedVPC: lo.ToPtr(true), InterruptionQueueName: lo.ToPtr("test-cluster"), })) + ctx = settings.ToContext(ctx, test.Settings()) env = coretest.NewEnvironment(scheme.Scheme) // Stop the coretest environment after the coretest completes defer func() { @@ -181,7 +183,7 @@ func newProviders(kubeClient client.Client) providerSet { func (p *providerSet) makeInfrastructure(ctx context.Context) error { if _, err := p.sqsAPI.CreateQueueWithContext(ctx, &sqs.CreateQueueInput{ - QueueName: lo.ToPtr(settings.FromContext(ctx).InterruptionQueueName), + QueueName: lo.ToPtr(options.FromContext(ctx).InterruptionQueueName), Attributes: map[string]*string{ sqs.QueueAttributeNameMessageRetentionPeriod: aws.String("1200"), // 20 minutes for this test }, diff --git a/pkg/controllers/interruption/sqs.go b/pkg/controllers/interruption/sqs.go index 873688801d3d..6cf518955820 100644 --- a/pkg/controllers/interruption/sqs.go +++ b/pkg/controllers/interruption/sqs.go @@ -26,8 +26,8 @@ import ( "github.com/samber/lo" "github.com/aws/karpenter-core/pkg/utils/atomic" - "github.com/aws/karpenter/pkg/apis/settings" awserrors "github.com/aws/karpenter/pkg/errors" + "github.com/aws/karpenter/pkg/operator/options" ) type SQSProvider struct { @@ -43,7 +43,7 @@ func NewSQSProvider(client sqsiface.SQSAPI) *SQSProvider { } provider.queueURL.Resolve = func(ctx context.Context) (string, error) { input := &sqs.GetQueueUrlInput{ - QueueName: aws.String(settings.FromContext(ctx).InterruptionQueueName), + QueueName: aws.String(options.FromContext(ctx).InterruptionQueueName), } ret, err := provider.client.GetQueueUrlWithContext(ctx, input) if err != nil { @@ -66,12 +66,12 @@ func (s *SQSProvider) QueueExists(ctx context.Context) (bool, error) { } func (s *SQSProvider) DiscoverQueueURL(ctx context.Context) (string, error) { - if settings.FromContext(ctx).InterruptionQueueName != lo.FromPtr(s.queueName.Load()) { + if options.FromContext(ctx).InterruptionQueueName != lo.FromPtr(s.queueName.Load()) { res, err := s.queueURL.TryGet(ctx, atomic.IgnoreCacheOption) if err != nil { return res, err } - s.queueName.Store(lo.ToPtr(settings.FromContext(ctx).InterruptionQueueName)) + s.queueName.Store(lo.ToPtr(options.FromContext(ctx).InterruptionQueueName)) return res, nil } return s.queueURL.TryGet(ctx) diff --git a/pkg/controllers/interruption/suite_test.go b/pkg/controllers/interruption/suite_test.go index 23d3ef8ab57e..8cb557e1efbf 100644 --- a/pkg/controllers/interruption/suite_test.go +++ b/pkg/controllers/interruption/suite_test.go @@ -37,10 +37,10 @@ import ( _ "knative.dev/pkg/system/testing" "sigs.k8s.io/controller-runtime/pkg/client" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" "github.com/aws/karpenter-core/pkg/events" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -53,6 +53,7 @@ import ( "github.com/aws/karpenter/pkg/controllers/interruption/messages/spotinterruption" "github.com/aws/karpenter/pkg/controllers/interruption/messages/statechange" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/pkg/utils" ) @@ -91,10 +92,11 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = coresettings.ToContext(ctx, coretest.Settings()) - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ InterruptionQueueName: lo.ToPtr("test-cluster"), })) + ctx = settings.ToContext(ctx, test.Settings()) unavailableOfferingsCache.Flush() sqsapi.Reset() sqsProvider.Reset() diff --git a/pkg/controllers/nodeclaim/garbagecollection/machine_test.go b/pkg/controllers/nodeclaim/garbagecollection/machine_test.go index bab441c35ab2..b9dcca9dd97e 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/machine_test.go +++ b/pkg/controllers/nodeclaim/garbagecollection/machine_test.go @@ -33,9 +33,9 @@ import ( coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" ) @@ -60,7 +60,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -69,7 +69,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -125,7 +125,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -134,7 +134,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -179,7 +179,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -188,7 +188,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -286,7 +286,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -295,7 +295,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), diff --git a/pkg/controllers/nodeclaim/garbagecollection/nodeclaim_test.go b/pkg/controllers/nodeclaim/garbagecollection/nodeclaim_test.go index ef1241a7278d..897c83ec4b60 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/nodeclaim_test.go +++ b/pkg/controllers/nodeclaim/garbagecollection/nodeclaim_test.go @@ -33,8 +33,8 @@ import ( coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" "github.com/aws/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter/pkg/operator/options" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/test" ) @@ -65,7 +65,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -74,7 +74,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -130,7 +130,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -139,7 +139,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -184,7 +184,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -193,7 +193,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -301,7 +301,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -310,7 +310,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), diff --git a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go index 5a9384a3f1fb..03cf6b5fee4f 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go +++ b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go @@ -32,7 +32,6 @@ import ( "k8s.io/client-go/tools/record" . "knative.dev/pkg/logging/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" @@ -48,6 +47,7 @@ import ( "github.com/aws/karpenter/pkg/controllers/nodeclaim/garbagecollection" "github.com/aws/karpenter/pkg/controllers/nodeclaim/link" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" ) @@ -65,7 +65,7 @@ func TestAPIs(t *testing.T) { } var _ = BeforeSuite(func() { - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) awsEnv = test.NewEnvironment(ctx, env) @@ -104,7 +104,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -113,7 +113,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -139,7 +139,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -148,7 +148,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -193,7 +193,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -202,7 +202,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -243,7 +243,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -252,7 +252,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), diff --git a/pkg/controllers/nodeclaim/link/suite_test.go b/pkg/controllers/nodeclaim/link/suite_test.go index 15e059bf15ec..7fe54382750d 100644 --- a/pkg/controllers/nodeclaim/link/suite_test.go +++ b/pkg/controllers/nodeclaim/link/suite_test.go @@ -33,10 +33,10 @@ import ( . "knative.dev/pkg/logging/testing" "sigs.k8s.io/controller-runtime/pkg/client" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/events" "github.com/aws/karpenter-core/pkg/operator/controller" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -47,6 +47,7 @@ import ( "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/controllers/nodeclaim/link" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/pkg/utils" ) @@ -64,7 +65,8 @@ func TestAPIs(t *testing.T) { } var _ = BeforeSuite(func() { - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) awsEnv = test.NewEnvironment(ctx, env) @@ -107,7 +109,7 @@ var _ = Describe("MachineLink", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -254,7 +256,7 @@ var _ = Describe("MachineLink", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -344,7 +346,7 @@ var _ = Describe("MachineLink", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, }, diff --git a/pkg/controllers/nodeclaim/tagging/suite_test.go b/pkg/controllers/nodeclaim/tagging/suite_test.go index 69324b86a03d..fefab7c37ee5 100644 --- a/pkg/controllers/nodeclaim/tagging/suite_test.go +++ b/pkg/controllers/nodeclaim/tagging/suite_test.go @@ -31,7 +31,6 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" . "knative.dev/pkg/logging/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" coretest "github.com/aws/karpenter-core/pkg/test" "github.com/aws/karpenter/pkg/apis" @@ -39,10 +38,12 @@ import ( "github.com/aws/karpenter/pkg/apis/v1beta1" "github.com/aws/karpenter/pkg/controllers/nodeclaim/tagging" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter-core/pkg/operator/controller" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" ) @@ -59,7 +60,8 @@ func TestAPIs(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) awsEnv = test.NewEnvironment(ctx, env) taggingController = tagging.NewController(env.Client, awsEnv.InstanceProvider) @@ -86,7 +88,7 @@ var _ = Describe("TaggingController", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -95,7 +97,7 @@ var _ = Describe("TaggingController", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), diff --git a/pkg/controllers/nodeclass/suite_test.go b/pkg/controllers/nodeclass/suite_test.go index 830c155dd389..862a13320af7 100644 --- a/pkg/controllers/nodeclass/suite_test.go +++ b/pkg/controllers/nodeclass/suite_test.go @@ -24,23 +24,22 @@ import ( . "knative.dev/pkg/logging/testing" _ "knative.dev/pkg/system/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/events" corecontroller "github.com/aws/karpenter-core/pkg/operator/controller" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/controllers/nodeclass" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" ) var ctx context.Context var env *coretest.Environment var awsEnv *test.Environment -var opts *options.Options var nodeTemplateController corecontroller.Controller var nodeClassController corecontroller.Controller @@ -52,7 +51,8 @@ func TestAPIs(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...), coretest.WithFieldIndexers(test.EC2NodeClassFieldIndexer(ctx))) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) awsEnv = test.NewEnvironment(ctx, env) @@ -65,7 +65,7 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = options.ToContext(ctx, opts) + ctx = coreoptions.ToContext(ctx, coretest.Options()) awsEnv.Reset() }) diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index b40f33c6e9ff..1a3be6d4a687 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -49,8 +49,8 @@ import ( corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" "github.com/aws/karpenter-core/pkg/operator" - "github.com/aws/karpenter/pkg/apis/settings" awscache "github.com/aws/karpenter/pkg/cache" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/providers/instanceprofile" @@ -86,7 +86,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont STSRegionalEndpoint: endpoints.RegionalSTSEndpoint, } - if assumeRoleARN := settings.FromContext(ctx).AssumeRoleARN; assumeRoleARN != "" { + if assumeRoleARN := options.FromContext(ctx).AssumeRoleARN; assumeRoleARN != "" { config.Credentials = stscreds.NewCredentials(session.Must(session.NewSession()), assumeRoleARN, func(provider *stscreds.AssumeRoleProvider) { setDurationAndExpiry(ctx, provider) }) } @@ -213,12 +213,12 @@ func checkEC2Connectivity(ctx context.Context, api *ec2.EC2) error { } func ResolveClusterEndpoint(ctx context.Context, eksAPI eksiface.EKSAPI) (string, error) { - clusterEndpointFromSettings := settings.FromContext(ctx).ClusterEndpoint - if clusterEndpointFromSettings != "" { - return clusterEndpointFromSettings, nil // cluster endpoint is explicitly set + clusterEndpointFromOptions := options.FromContext(ctx).ClusterEndpoint + if clusterEndpointFromOptions != "" { + return clusterEndpointFromOptions, nil // cluster endpoint is explicitly set } out, err := eksAPI.DescribeClusterWithContext(ctx, &eks.DescribeClusterInput{ - Name: aws.String(settings.FromContext(ctx).ClusterName), + Name: aws.String(options.FromContext(ctx).ClusterName), }) if err != nil { return "", fmt.Errorf("failed to resolve cluster endpoint, %w", err) @@ -231,7 +231,7 @@ func getCABundle(ctx context.Context, restConfig *rest.Config) (*string, error) // have used the simpler client-go InClusterConfig() method. // However, that only works when Karpenter is running as a Pod // within the same cluster it's managing. - if caBundle := settings.FromContext(ctx).ClusterCABundle; caBundle != "" { + if caBundle := options.FromContext(ctx).ClusterCABundle; caBundle != "" { return lo.ToPtr(caBundle), nil } transportConfig, err := restConfig.TransportConfig() @@ -261,6 +261,6 @@ func kubeDNSIP(ctx context.Context, kubernetesInterface kubernetes.Interface) (n } func setDurationAndExpiry(ctx context.Context, provider *stscreds.AssumeRoleProvider) { - provider.Duration = settings.FromContext(ctx).AssumeRoleDuration + provider.Duration = options.FromContext(ctx).AssumeRoleDuration provider.ExpiryWindow = time.Duration(10) * time.Second } diff --git a/pkg/operator/options/options.go b/pkg/operator/options/options.go new file mode 100644 index 000000000000..960fa080e4d6 --- /dev/null +++ b/pkg/operator/options/options.go @@ -0,0 +1,134 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "strings" + "time" + + "k8s.io/apimachinery/pkg/util/sets" + + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" + "github.com/aws/karpenter-core/pkg/utils/env" + "github.com/aws/karpenter/pkg/apis/settings" +) + +func init() { + coreoptions.Injectables = append(coreoptions.Injectables, &Options{}) +} + +type optionsKey struct{} + +type Options struct { + AssumeRoleARN string + AssumeRoleDuration time.Duration + ClusterCABundle string + ClusterName string + ClusterEndpoint string + IsolatedVPC bool + VMMemoryOverheadPercent float64 + InterruptionQueueName string + ReservedENIs int + + isolatedVPC string + setFlags map[string]bool +} + +func (o *Options) AddFlags(fs *flag.FlagSet) { + fs.StringVar(&o.AssumeRoleARN, "assume-role-arn", env.WithDefaultString("ASSUME_ROLE_ARN", ""), "Role to assume for calling AWS services.") + fs.DurationVar(&o.AssumeRoleDuration, "assume-role-duration", env.WithDefaultDuration("ASSUME_ROLE_DURATION", 15*time.Minute), "Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless aws.assumeRole set.") + fs.StringVar(&o.ClusterCABundle, "cluster-ca-bundle", env.WithDefaultString("CLUSTER_CA_BUNDLE", ""), "Cluster CA bundle for nodes to use for TLS connections with the API server. If not set, this is taken from the controller's TLS configuration.") + fs.StringVar(&o.ClusterName, "cluster-name", env.WithDefaultString("CLUSTER_NAME", ""), "[REQUIRED] The kubernetes cluster name for resource discovery.") + fs.StringVar(&o.ClusterEndpoint, "cluster-endpoint", env.WithDefaultString("CLUSTER_ENDPOINT", ""), "The external kubernetes cluster endpoint for new nodes to connect with. If not specified, will discover the cluster endpoint using DescribeCluster API.") + fs.StringVar(&o.isolatedVPC, "isolated-vpc", env.WithDefaultString("ISOLATED_VPC", "false"), "If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS pricing endpoint.") + fs.Float64Var(&o.VMMemoryOverheadPercent, "vm-memory-overhead-percent", env.WithDefaultFloat64("VM_MEMORY_OVERHEAD_PERCENT", 0.075), "The VM memory overhead as a percent that will be subtracted from the total memory for all instance types.") + fs.StringVar(&o.InterruptionQueueName, "interruption-queue-name", env.WithDefaultString("INTERRUPTION_QUEUE_NAME", ""), "Interruption queue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.") + fs.IntVar(&o.ReservedENIs, "reserved-enis", env.WithDefaultInt("RESERVED_ENIS", 0), "Reserved ENIs are not included in the calculations for max-pods or kube-reserved. This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html.") +} + +func (o *Options) Parse(fs *flag.FlagSet, args ...string) error { + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + os.Exit(0) + } + return fmt.Errorf("parsing flags, %w", err) + } + + // Check if each option has been set. This is a little brute force and better options might exist, + // but this only needs to be here for one version + o.setFlags = map[string]bool{} + cliFlags := sets.New[string]() + fs.Visit(func(f *flag.Flag) { + cliFlags.Insert(f.Name) + }) + fs.VisitAll(func(f *flag.Flag) { + envName := strings.ReplaceAll(strings.ToUpper(f.Name), "-", "_") + _, ok := os.LookupEnv(envName) + o.setFlags[f.Name] = ok || cliFlags.Has(f.Name) + }) + + if err := o.Validate(); err != nil { + return fmt.Errorf("validating options, %w", err) + } + + // Parse options which can't be directly parsed by flag + o.IsolatedVPC = (o.isolatedVPC == "true") + + return nil +} + +func (o *Options) ToContext(ctx context.Context) context.Context { + return ToContext(ctx, o) +} + +func (o *Options) MergeSettings(ctx context.Context) { + s := settings.FromContext(ctx) + mergeField(&o.AssumeRoleARN, s.AssumeRoleARN, o.setFlags["assume-role-arn"]) + mergeField(&o.AssumeRoleDuration, s.AssumeRoleDuration, o.setFlags["assume-role-duration"]) + mergeField(&o.ClusterCABundle, s.ClusterCABundle, o.setFlags["cluster-ca-bundle"]) + mergeField(&o.ClusterName, s.ClusterName, o.setFlags["cluster-name"]) + mergeField(&o.ClusterEndpoint, s.ClusterEndpoint, o.setFlags["cluster-endpoint"]) + mergeField(&o.IsolatedVPC, s.IsolatedVPC, o.setFlags["isolated-vpc"]) + mergeField(&o.VMMemoryOverheadPercent, s.VMMemoryOverheadPercent, o.setFlags["vm-memory-overhead-percent"]) + mergeField(&o.InterruptionQueueName, s.InterruptionQueueName, o.setFlags["interruption-queue-name"]) + mergeField(&o.ReservedENIs, s.ReservedENIs, o.setFlags["reserved-enis"]) + if err := o.validateRequiredFields(); err != nil { + panic(fmt.Errorf("checking required fields, %w", err)) + } +} + +func ToContext(ctx context.Context, opts *Options) context.Context { + return context.WithValue(ctx, optionsKey{}, opts) +} + +func FromContext(ctx context.Context) *Options { + retval := ctx.Value(optionsKey{}) + if retval == nil { + return nil + } + return retval.(*Options) +} + +// Note: Separated out to help with cyclomatic complexity check +func mergeField[T any](dest *T, src T, isDestSet bool) { + if !isDestSet { + *dest = src + } +} diff --git a/pkg/operator/options/options_validation.go b/pkg/operator/options/options_validation.go new file mode 100644 index 000000000000..976543d12d75 --- /dev/null +++ b/pkg/operator/options/options_validation.go @@ -0,0 +1,82 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "fmt" + "net/url" + "time" + + "go.uber.org/multierr" +) + +func (o Options) Validate() error { + return multierr.Combine( + o.validateEndpoint(), + o.validateVMMemoryOverheadPercent(), + o.validateAssumeRoleDuration(), + o.validateReservedENIs(), + o.validateIsolateVPC(), + ) +} + +func (o Options) validateAssumeRoleDuration() error { + if o.AssumeRoleDuration < time.Minute*15 { + return fmt.Errorf("assume-role-duration cannot be less than 15 minutes") + } + return nil +} + +func (o Options) validateIsolateVPC() error { + if o.isolatedVPC != "true" && o.isolatedVPC != "false" { + return fmt.Errorf("%q is not a valid value for isolated-vpc, options are true or false", o.isolatedVPC) + } + return nil +} + +func (o Options) validateEndpoint() error { + if o.ClusterEndpoint == "" { + return nil + } + endpoint, err := url.Parse(o.ClusterEndpoint) + // url.Parse() will accept a lot of input without error; make + // sure it's a real URL + if err != nil || !endpoint.IsAbs() || endpoint.Hostname() == "" { + return fmt.Errorf("%q is not a valid cluster-endpoint URL", o.ClusterEndpoint) + } + return nil +} + +func (o Options) validateVMMemoryOverheadPercent() error { + if o.VMMemoryOverheadPercent < 0 { + return fmt.Errorf("vm-memory-overhead-percent cannot be negative") + } + return nil +} + +func (o Options) validateReservedENIs() error { + if o.ReservedENIs < 0 { + return fmt.Errorf("reserved-enis cannot be negative") + } + return nil +} + +// Note: add back to Validate when karpenter-global-settings (and merge logic) are completely removed +func (o Options) validateRequiredFields() error { + if o.ClusterName == "" { + return fmt.Errorf("missing field, cluster-name") + } + return nil +} diff --git a/pkg/operator/options/suite_test.go b/pkg/operator/options/suite_test.go new file mode 100644 index 000000000000..b69131348cc0 --- /dev/null +++ b/pkg/operator/options/suite_test.go @@ -0,0 +1,118 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options_test + +import ( + "context" + "flag" + "fmt" + "testing" + "time" + + "github.com/aws/karpenter/pkg/apis/settings" + "github.com/aws/karpenter/pkg/operator/options" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "knative.dev/pkg/logging/testing" +) + +var ctx context.Context + +func TestAPIs(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "Options") +} + +var _ = Describe("Options", func() { + var fs *flag.FlagSet + var opts *options.Options + BeforeEach(func() { + fs = flag.NewFlagSet("karpenter", flag.ContinueOnError) + opts = &options.Options{} + opts.AddFlags(fs) + + // Inject default settings + var err error + ctx, err = (&settings.Settings{}).Inject(ctx, nil) + Expect(err).To(BeNil()) + }) + + Context("Merging", func() { + It("shouldn't overwrite cli flags / env vars", func() { + err := opts.Parse( + fs, + "--assume-role-arn", "options-cluster-role", + "--cluster-ca-bundle", "options-bundle", + "--cluster-name", "options-cluster", + "--cluster-endpoint", "https://options-cluster", + "--isolated-vpc", "false", + "--interruption-queue-name", "options-cluster", + ) + Expect(err).ToNot(HaveOccurred()) + ctx = settings.ToContext(ctx, &settings.Settings{ + AssumeRoleARN: "settings-cluster-role", + AssumeRoleDuration: time.Minute * 20, + ClusterCABundle: "settings-bundle", + ClusterName: "settings-cluster", + ClusterEndpoint: "https://settings-cluster", + IsolatedVPC: true, + VMMemoryOverheadPercent: 0.1, + InterruptionQueueName: "settings-cluster", + ReservedENIs: 10, + }) + opts.MergeSettings(ctx) + Expect(opts.AssumeRoleARN).To(Equal("options-cluster-role")) + Expect(opts.AssumeRoleDuration).To(Equal(time.Minute * 20)) + Expect(opts.ClusterCABundle).To(Equal("options-bundle")) + Expect(opts.ClusterName).To(Equal("options-cluster")) + Expect(opts.ClusterEndpoint).To(Equal("https://options-cluster")) + Expect(opts.IsolatedVPC).To(BeFalse()) + Expect(opts.VMMemoryOverheadPercent).To(Equal(0.1)) + Expect(opts.InterruptionQueueName).To(Equal("options-cluster")) + Expect(opts.ReservedENIs).To(Equal(10)) + }) + + }) + + Context("Validation", func() { + It("should fail when cluster name is not set", func() { + err := opts.Parse(fs) + // Overwrite ClusterName since it is commonly set by environment variables in dev environments + opts.ClusterName = "" + Expect(err).ToNot(HaveOccurred()) + Expect(func() { + opts.MergeSettings(ctx) + fmt.Printf("%#v", opts) + }).To(Panic()) + }) + It("should fail when assume role duration is less than 15 minutes", func() { + err := opts.Parse(fs, "--assume-role-duration", "1s") + Expect(err).To(HaveOccurred()) + }) + It("should fail when clusterEndpoint is invalid (not absolute)", func() { + err := opts.Parse(fs, "--cluster-endpoint", "00000000000000000000000.gr7.us-west-2.eks.amazonaws.com") + Expect(err).To(HaveOccurred()) + }) + It("should fail when vmMemoryOverheadPercent is negative", func() { + err := opts.Parse(fs, "--vm-memory-overhead-percent", "-0.01") + Expect(err).To(HaveOccurred()) + }) + It("should fail when reservedENIs is negative", func() { + err := opts.Parse(fs, "--reserved-enis", "-1") + Expect(err).To(HaveOccurred()) + }) + }) +}) diff --git a/pkg/operator/suite_test.go b/pkg/operator/suite_test.go index dea32a9fedc9..5146ed6d9e9b 100644 --- a/pkg/operator/suite_test.go +++ b/pkg/operator/suite_test.go @@ -26,12 +26,11 @@ import ( . "knative.dev/pkg/logging/testing" "github.com/aws/karpenter/pkg/apis" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/fake" awscontext "github.com/aws/karpenter/pkg/operator" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -50,8 +49,6 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) - ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) fakeEKSAPI = &fake.EKSAPI{} @@ -72,7 +69,7 @@ var _ = AfterEach(func() { var _ = Describe("Operator", func() { It("should resolve endpoint if set via configuration", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ClusterEndpoint: lo.ToPtr("https://api.test-cluster.k8s.local"), })) endpoint, err := awscontext.ResolveClusterEndpoint(ctx, fakeEKSAPI) @@ -80,7 +77,7 @@ var _ = Describe("Operator", func() { Expect(endpoint).To(Equal("https://api.test-cluster.k8s.local")) }) It("should resolve endpoint if not set, via call to API", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ClusterEndpoint: lo.ToPtr(""), })) fakeEKSAPI.DescribeClusterBehavior.Output.Set( @@ -96,7 +93,7 @@ var _ = Describe("Operator", func() { Expect(endpoint).To(Equal("https://cluster-endpoint.test-cluster.k8s.local")) }) It("should propagate error if API fails", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ClusterEndpoint: lo.ToPtr(""), })) fakeEKSAPI.DescribeClusterBehavior.Error.Set(errors.New("test error")) diff --git a/pkg/providers/amifamily/ami_test.go b/pkg/providers/amifamily/ami_test.go index 9da5c2058dff..279ce74dd88b 100644 --- a/pkg/providers/amifamily/ami_test.go +++ b/pkg/providers/amifamily/ami_test.go @@ -29,14 +29,15 @@ import ( v1 "k8s.io/api/core/v1" . "knative.dev/pkg/logging/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" "github.com/aws/karpenter-core/pkg/scheduling" coretest "github.com/aws/karpenter-core/pkg/test" "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter/pkg/test" ) @@ -61,7 +62,8 @@ const ( var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) awsEnv = test.NewEnvironment(ctx, env) }) diff --git a/pkg/providers/instance/instance.go b/pkg/providers/instance/instance.go index d44f33ac659e..35690da3c5ea 100644 --- a/pkg/providers/instance/instance.go +++ b/pkg/providers/instance/instance.go @@ -39,6 +39,7 @@ import ( "github.com/aws/karpenter/pkg/batcher" "github.com/aws/karpenter/pkg/cache" awserrors "github.com/aws/karpenter/pkg/errors" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/instancetype" "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/subnet" @@ -104,7 +105,7 @@ func (p *Provider) Create(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, func (p *Provider) Link(ctx context.Context, id, provisionerName string) error { if err := p.CreateTags(ctx, id, map[string]string{ - v1alpha5.MachineManagedByAnnotationKey: settings.FromContext(ctx).ClusterName, + v1alpha5.MachineManagedByAnnotationKey: options.FromContext(ctx).ClusterName, v1alpha5.ProvisionerNameLabelKey: provisionerName, }); err != nil { return fmt.Errorf("linking tags, %w", err) @@ -143,7 +144,7 @@ func (p *Provider) List(ctx context.Context) ([]*Instance, error) { }, { Name: aws.String("tag-key"), - Values: aws.StringSlice([]string{fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)}), + Values: aws.StringSlice([]string{fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)}), }, instanceStateFilter, }, @@ -257,15 +258,15 @@ func getTags(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *co "Name": fmt.Sprintf("%s/%s", v1alpha5.ProvisionerNameLabelKey, nodeClaim.Labels[v1alpha5.ProvisionerNameLabelKey]), } staticTags = map[string]string{ - fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName): "owned", - v1alpha5.ProvisionerNameLabelKey: nodeClaim.Labels[v1alpha5.ProvisionerNameLabelKey], - v1alpha5.MachineManagedByAnnotationKey: settings.FromContext(ctx).ClusterName, + fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName): "owned", + v1alpha5.ProvisionerNameLabelKey: nodeClaim.Labels[v1alpha5.ProvisionerNameLabelKey], + v1alpha5.MachineManagedByAnnotationKey: options.FromContext(ctx).ClusterName, } } else { staticTags = map[string]string{ - fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName): "owned", + fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName): "owned", corev1beta1.NodePoolLabelKey: nodeClaim.Labels[corev1beta1.NodePoolLabelKey], - corev1beta1.ManagedByAnnotationKey: settings.FromContext(ctx).ClusterName, + corev1beta1.ManagedByAnnotationKey: options.FromContext(ctx).ClusterName, } } return lo.Assign(overridableTags, settings.FromContext(ctx).Tags, nodeClass.Spec.Tags, staticTags) diff --git a/pkg/providers/instance/nodeclass_test.go b/pkg/providers/instance/nodeclass_test.go index 4f711b38da72..a2fcaa913142 100644 --- a/pkg/providers/instance/nodeclass_test.go +++ b/pkg/providers/instance/nodeclass_test.go @@ -31,9 +31,9 @@ import ( corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/test" ) @@ -100,7 +100,7 @@ var _ = Describe("NodeClass/InstanceProvider", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -109,7 +109,7 @@ var _ = Describe("NodeClass/InstanceProvider", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), diff --git a/pkg/providers/instance/nodetemplate_test.go b/pkg/providers/instance/nodetemplate_test.go index aec5c71ea45c..4894eb9bcddc 100644 --- a/pkg/providers/instance/nodetemplate_test.go +++ b/pkg/providers/instance/nodetemplate_test.go @@ -33,9 +33,9 @@ import ( . "github.com/aws/karpenter-core/pkg/test/expectations" nodeclaimutil "github.com/aws/karpenter-core/pkg/utils/nodeclaim" nodepoolutil "github.com/aws/karpenter-core/pkg/utils/nodepool" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/test" nodeclassutil "github.com/aws/karpenter/pkg/utils/nodeclass" @@ -114,7 +114,7 @@ var _ = Describe("NodeTemplate/InstanceProvider", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -123,7 +123,7 @@ var _ = Describe("NodeTemplate/InstanceProvider", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), diff --git a/pkg/providers/instance/suite_test.go b/pkg/providers/instance/suite_test.go index 9a46f6393b26..45e163c0a7c2 100644 --- a/pkg/providers/instance/suite_test.go +++ b/pkg/providers/instance/suite_test.go @@ -29,23 +29,22 @@ import ( "k8s.io/client-go/tools/record" . "knative.dev/pkg/logging/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" "github.com/aws/karpenter-core/pkg/events" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/test" ) var ctx context.Context -var opts *options.Options var env *coretest.Environment var awsEnv *test.Environment var cloudProvider *cloudprovider.CloudProvider @@ -58,7 +57,8 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) awsEnv = test.NewEnvironment(ctx, env) cloudProvider = cloudprovider.New(awsEnv.InstanceTypesProvider, awsEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), @@ -90,7 +90,7 @@ var _ = Describe("Combined/InstanceProvider", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -99,7 +99,7 @@ var _ = Describe("Combined/InstanceProvider", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -125,7 +125,7 @@ var _ = Describe("Combined/InstanceProvider", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -134,7 +134,7 @@ var _ = Describe("Combined/InstanceProvider", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), diff --git a/pkg/providers/instanceprofile/instanceprofile.go b/pkg/providers/instanceprofile/instanceprofile.go index e5b6f9437399..f9bb39d9cf8a 100644 --- a/pkg/providers/instanceprofile/instanceprofile.go +++ b/pkg/providers/instanceprofile/instanceprofile.go @@ -27,9 +27,9 @@ import ( v1 "k8s.io/api/core/v1" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" awserrors "github.com/aws/karpenter/pkg/errors" + "github.com/aws/karpenter/pkg/operator/options" ) type Provider struct { @@ -48,10 +48,10 @@ func NewProvider(region string, iamapi iamiface.IAMAPI, cache *cache.Cache) *Pro func (p *Provider) Create(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) (string, error) { tags := lo.Assign(nodeClass.Spec.Tags, map[string]string{ - fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName): "owned", - corev1beta1.ManagedByAnnotationKey: settings.FromContext(ctx).ClusterName, - v1beta1.LabelNodeClass: nodeClass.Name, - v1.LabelTopologyRegion: p.region, + fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName): "owned", + corev1beta1.ManagedByAnnotationKey: options.FromContext(ctx).ClusterName, + v1beta1.LabelNodeClass: nodeClass.Name, + v1.LabelTopologyRegion: p.region, }) profileName := GetProfileName(ctx, p.region, nodeClass) @@ -129,5 +129,5 @@ func (p *Provider) Delete(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) // GetProfileName gets the string for the profile name based on the cluster name and the NodeClass UUID. // The length of this string can never exceed the maximum instance profile name limit of 128 characters. func GetProfileName(ctx context.Context, region string, nodeClass *v1beta1.EC2NodeClass) string { - return fmt.Sprintf("%s_%d", settings.FromContext(ctx).ClusterName, lo.Must(hashstructure.Hash(fmt.Sprintf("%s%s", region, nodeClass.Name), hashstructure.FormatV2, nil))) + return fmt.Sprintf("%s_%d", options.FromContext(ctx).ClusterName, lo.Must(hashstructure.Hash(fmt.Sprintf("%s%s", region, nodeClass.Name), hashstructure.FormatV2, nil))) } diff --git a/pkg/providers/instancetype/nodeclass_test.go b/pkg/providers/instancetype/nodeclass_test.go index d2b4cdae9a00..16cb06e0f502 100644 --- a/pkg/providers/instancetype/nodeclass_test.go +++ b/pkg/providers/instancetype/nodeclass_test.go @@ -40,6 +40,7 @@ import ( "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/providers/instancetype" "github.com/aws/karpenter/pkg/test" @@ -645,16 +646,8 @@ var _ = Describe("NodeClass/InstanceTypes", func() { Context("Overhead", func() { var info *ec2.InstanceTypeInfo BeforeEach(func() { - ctx, err := (&settings.Settings{}).Inject(ctx, &v1.ConfigMap{ - Data: map[string]string{ - "aws.clusterName": "karpenter-cluster", - }, - }) - Expect(err).To(BeNil()) - - s := settings.FromContext(ctx) - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ - VMMemoryOverheadPercent: &s.VMMemoryOverheadPercent, + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + ClusterName: lo.ToPtr("karpenter-cluster"), })) var ok bool @@ -713,7 +706,7 @@ var _ = Describe("NodeClass/InstanceTypes", func() { }) Context("Eviction Thresholds", func() { BeforeEach(func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) }) @@ -950,7 +943,7 @@ var _ = Describe("NodeClass/InstanceTypes", func() { } }) It("should reserve ENIs when aws.reservedENIs is set and is used in max-pods calculation", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ReservedENIs: lo.ToPtr(1), })) @@ -970,7 +963,7 @@ var _ = Describe("NodeClass/InstanceTypes", func() { Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", maxPods)) }) It("should reserve ENIs when aws.reservedENIs is set and not go below 0 ENIs in max-pods calculation", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ReservedENIs: lo.ToPtr(1_000_000), })) diff --git a/pkg/providers/instancetype/nodetemplate_test.go b/pkg/providers/instancetype/nodetemplate_test.go index ba650c3b4ca7..30e03bdfa179 100644 --- a/pkg/providers/instancetype/nodetemplate_test.go +++ b/pkg/providers/instancetype/nodetemplate_test.go @@ -40,6 +40,7 @@ import ( . "github.com/aws/karpenter-core/pkg/test/expectations" nodepoolutil "github.com/aws/karpenter-core/pkg/utils/nodepool" "github.com/aws/karpenter-core/pkg/utils/resources" + "github.com/aws/karpenter/pkg/operator/options" nodeclassutil "github.com/aws/karpenter/pkg/utils/nodeclass" "github.com/aws/karpenter/pkg/apis/settings" @@ -664,16 +665,8 @@ var _ = Describe("NodeTemplate/InstanceTypes", func() { Context("Overhead", func() { var info *ec2.InstanceTypeInfo BeforeEach(func() { - ctx, err := (&settings.Settings{}).Inject(ctx, &v1.ConfigMap{ - Data: map[string]string{ - "aws.clusterName": "karpenter-cluster", - }, - }) - Expect(err).To(BeNil()) - - s := settings.FromContext(ctx) - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ - VMMemoryOverheadPercent: &s.VMMemoryOverheadPercent, + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + ClusterName: lo.ToPtr("karpenter-cluster"), })) var ok bool @@ -734,7 +727,7 @@ var _ = Describe("NodeTemplate/InstanceTypes", func() { }) Context("Eviction Thresholds", func() { BeforeEach(func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) }) @@ -990,7 +983,7 @@ var _ = Describe("NodeTemplate/InstanceTypes", func() { } }) It("should reserve ENIs when aws.reservedENIs is set and is used in max-pods calculation", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ReservedENIs: lo.ToPtr(1), })) @@ -1010,7 +1003,7 @@ var _ = Describe("NodeTemplate/InstanceTypes", func() { Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", maxPods)) }) It("should reserve ENIs when aws.reservedENIs is set and not go below 0 ENIs in max-pods calculation", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ReservedENIs: lo.ToPtr(1_000_000), })) diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index c237cbcd3da6..4a56a7bcbc62 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -29,13 +29,12 @@ import ( clock "k8s.io/utils/clock/testing" . "knative.dev/pkg/logging/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" "github.com/aws/karpenter-core/pkg/controllers/provisioning" "github.com/aws/karpenter-core/pkg/controllers/state" "github.com/aws/karpenter-core/pkg/events" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -43,12 +42,12 @@ import ( "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/test" ) var ctx context.Context -var opts *options.Options var env *coretest.Environment var awsEnv *test.Environment var fakeClock *clock.FakeClock @@ -64,7 +63,8 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) awsEnv = test.NewEnvironment(ctx, env) fakeClock = &clock.FakeClock{} @@ -79,8 +79,8 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = options.ToContext(ctx, opts) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) cluster.Reset() awsEnv.Reset() @@ -128,7 +128,7 @@ func generateSpotPricing(cp *cloudprovider.CloudProvider, nodePool *corev1beta1. func makeFakeInstances() []*ec2.InstanceTypeInfo { var instanceTypes []*ec2.InstanceTypeInfo - ctx := settings.ToContext(context.Background(), &settings.Settings{IsolatedVPC: true}) + ctx := options.ToContext(context.Background(), &options.Options{IsolatedVPC: true}) // Use keys from the static pricing data so that we guarantee pricing for the data // Create uniform instance data so all of them schedule for a given pod for _, it := range pricing.NewProvider(ctx, nil, nil, "us-east-1").InstanceTypes() { diff --git a/pkg/providers/instancetype/types.go b/pkg/providers/instancetype/types.go index a126ac84615d..5f132f1c6cb3 100644 --- a/pkg/providers/instancetype/types.go +++ b/pkg/providers/instancetype/types.go @@ -30,9 +30,10 @@ import ( "knative.dev/pkg/ptr" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" - awssettings "github.com/aws/karpenter/pkg/apis/settings" + "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter-core/pkg/cloudprovider" @@ -201,7 +202,7 @@ func memory(ctx context.Context, info *ec2.InstanceTypeInfo) *resource.Quantity } mem := resources.Quantity(fmt.Sprintf("%dMi", sizeInMib)) // Account for VM overhead in calculation - mem.Sub(resource.MustParse(fmt.Sprintf("%dMi", int64(math.Ceil(float64(mem.Value())*awssettings.FromContext(ctx).VMMemoryOverheadPercent/1024/1024))))) + mem.Sub(resource.MustParse(fmt.Sprintf("%dMi", int64(math.Ceil(float64(mem.Value())*options.FromContext(ctx).VMMemoryOverheadPercent/1024/1024))))) return mem } @@ -240,7 +241,7 @@ func ephemeralStorage(amiFamily amifamily.AMIFamily, blockDeviceMappings []*v1be func awsPodENI(ctx context.Context, name string) *resource.Quantity { // https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html#supported-instance-types limits, ok := Limits[name] - if awssettings.FromContext(ctx).EnablePodENI && ok && limits.IsTrunkingCompatible { + if settings.FromContext(ctx).EnablePodENI && ok && limits.IsTrunkingCompatible { return resources.Quantity(fmt.Sprint(limits.BranchInterface)) } return resources.Quantity("0") @@ -308,7 +309,7 @@ func ENILimitedPods(ctx context.Context, info *ec2.InstanceTypeInfo) *resource.Q // VPC CNI only uses the default network interface // https://github.com/aws/amazon-vpc-cni-k8s/blob/3294231c0dce52cfe473bf6c62f47956a3b333b6/scripts/gen_vpc_ip_limits.go#L162 networkInterfaces := *info.NetworkInfo.NetworkCards[*info.NetworkInfo.DefaultNetworkCardIndex].MaximumNetworkInterfaces - usableNetworkInterfaces := lo.Max([]int64{(networkInterfaces - int64(awssettings.FromContext(ctx).ReservedENIs)), 0}) + usableNetworkInterfaces := lo.Max([]int64{(networkInterfaces - int64(options.FromContext(ctx).ReservedENIs)), 0}) if usableNetworkInterfaces == 0 { return resource.NewQuantity(0, resource.DecimalSI) } @@ -403,7 +404,7 @@ func pods(ctx context.Context, info *ec2.InstanceTypeInfo, amiFamily amifamily.A switch { case kc != nil && kc.MaxPods != nil: count = int64(ptr.Int32Value(kc.MaxPods)) - case awssettings.FromContext(ctx).EnableENILimitedPodDensity && amiFamily.FeatureFlags().SupportsENILimitedPodDensity: + case settings.FromContext(ctx).EnableENILimitedPodDensity && amiFamily.FeatureFlags().SupportsENILimitedPodDensity: count = ENILimitedPods(ctx, info).Value() default: count = 110 diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index 015f344e07a3..40a84ee734ce 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -39,6 +39,7 @@ import ( "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" awserrors "github.com/aws/karpenter/pkg/errors" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter/pkg/providers/instanceprofile" "github.com/aws/karpenter/pkg/providers/securitygroup" @@ -172,7 +173,7 @@ func (p *Provider) createAMIOptions(ctx context.Context, nodeClass *v1beta1.EC2N return nil, fmt.Errorf("no security groups exist given constraints") } options := &amifamily.Options{ - ClusterName: settings.FromContext(ctx).ClusterName, + ClusterName: options.FromContext(ctx).ClusterName, ClusterEndpoint: p.ClusterEndpoint, AWSENILimitedPodDensity: settings.FromContext(ctx).EnableENILimitedPodDensity, InstanceProfile: instanceProfile, @@ -328,7 +329,7 @@ func (p *Provider) volumeSize(quantity *resource.Quantity) *int64 { // hydrateCache queries for existing Launch Templates created by Karpenter for the current cluster and adds to the LT cache. // Any error during hydration will result in a panic func (p *Provider) hydrateCache(ctx context.Context) { - clusterName := settings.FromContext(ctx).ClusterName + clusterName := options.FromContext(ctx).ClusterName ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("tag-key", karpenterManagedTagKey, "tag-value", clusterName)) if err := p.ec2api.DescribeLaunchTemplatesPagesWithContext(ctx, &ec2.DescribeLaunchTemplatesInput{ Filters: []*ec2.Filter{{Name: aws.String(fmt.Sprintf("tag:%s", karpenterManagedTagKey)), Values: []*string{aws.String(clusterName)}}}, diff --git a/pkg/providers/launchtemplate/nodeclass_test.go b/pkg/providers/launchtemplate/nodeclass_test.go index 7392e6af47a2..09f0f2f04bb6 100644 --- a/pkg/providers/launchtemplate/nodeclass_test.go +++ b/pkg/providers/launchtemplate/nodeclass_test.go @@ -42,6 +42,7 @@ import ( "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/amifamily/bootstrap" "github.com/aws/karpenter/pkg/providers/instancetype" "github.com/aws/karpenter/pkg/test" @@ -692,7 +693,10 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() { It("should calculate memory overhead based on eni limited pods when ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 @@ -703,7 +707,10 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() { It("should calculate memory overhead based on eni limited pods when not ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 @@ -742,7 +749,10 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() { It("should calculate memory overhead based on eni limited pods when ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(true), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket @@ -753,7 +763,10 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() { It("should calculate memory overhead based on max pods when not ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket diff --git a/pkg/providers/launchtemplate/nodetemplate_test.go b/pkg/providers/launchtemplate/nodetemplate_test.go index 84107056148a..0be09622f2ab 100644 --- a/pkg/providers/launchtemplate/nodetemplate_test.go +++ b/pkg/providers/launchtemplate/nodetemplate_test.go @@ -39,6 +39,7 @@ import ( "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/amifamily/bootstrap" "github.com/aws/karpenter/pkg/providers/instancetype" "github.com/aws/karpenter/pkg/test" @@ -707,7 +708,10 @@ var _ = Describe("NodeTemplate/LaunchTemplates", func() { It("should calculate memory overhead based on eni limited pods when ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 @@ -718,7 +722,10 @@ var _ = Describe("NodeTemplate/LaunchTemplates", func() { It("should calculate memory overhead based on eni limited pods when not ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 @@ -757,7 +764,10 @@ var _ = Describe("NodeTemplate/LaunchTemplates", func() { It("should calculate memory overhead based on eni limited pods when ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(true), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket @@ -768,7 +778,10 @@ var _ = Describe("NodeTemplate/LaunchTemplates", func() { It("should calculate memory overhead based on max pods when not ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index 27c5fc85204a..046648db9f71 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -30,23 +30,22 @@ import ( clock "k8s.io/utils/clock/testing" . "knative.dev/pkg/logging/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/controllers/provisioning" "github.com/aws/karpenter-core/pkg/controllers/state" "github.com/aws/karpenter-core/pkg/events" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/cloudprovider" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" ) var ctx context.Context var stop context.CancelFunc -var opts *options.Options var env *coretest.Environment var awsEnv *test.Environment var fakeClock *clock.FakeClock @@ -62,7 +61,8 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) awsEnv = test.NewEnvironment(ctx, env) @@ -80,8 +80,8 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = options.ToContext(ctx, opts) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) cluster.Reset() awsEnv.Reset() diff --git a/pkg/providers/pricing/suite_test.go b/pkg/providers/pricing/suite_test.go index 4f7e5416b017..029ed70322a5 100644 --- a/pkg/providers/pricing/suite_test.go +++ b/pkg/providers/pricing/suite_test.go @@ -29,8 +29,7 @@ import ( "k8s.io/apimachinery/pkg/types" . "knative.dev/pkg/logging/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -38,13 +37,13 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/test" ) var ctx context.Context var stop context.CancelFunc -var opts *options.Options var env *coretest.Environment var awsEnv *test.Environment var controller *pricing.Controller @@ -57,7 +56,8 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) awsEnv = test.NewEnvironment(ctx, env) @@ -70,8 +70,8 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = options.ToContext(ctx, opts) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) awsEnv.Reset() diff --git a/pkg/providers/securitygroup/suite_test.go b/pkg/providers/securitygroup/suite_test.go index f3de49b1a452..6cd9bd8cf88e 100644 --- a/pkg/providers/securitygroup/suite_test.go +++ b/pkg/providers/securitygroup/suite_test.go @@ -28,10 +28,10 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -39,7 +39,6 @@ import ( var ctx context.Context var stop context.CancelFunc -var opts *options.Options var env *coretest.Environment var awsEnv *test.Environment var nodeClass *v1beta1.EC2NodeClass @@ -52,7 +51,8 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) awsEnv = test.NewEnvironment(ctx, env) @@ -64,8 +64,8 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = options.ToContext(ctx, opts) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ Spec: v1beta1.EC2NodeClassSpec{ diff --git a/pkg/providers/subnet/suite_test.go b/pkg/providers/subnet/suite_test.go index 8e644cbbf7cd..1c8b8f240e6d 100644 --- a/pkg/providers/subnet/suite_test.go +++ b/pkg/providers/subnet/suite_test.go @@ -28,10 +28,10 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -39,7 +39,6 @@ import ( var ctx context.Context var stop context.CancelFunc -var opts *options.Options var env *coretest.Environment var awsEnv *test.Environment var nodeClass *v1beta1.EC2NodeClass @@ -52,7 +51,8 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) awsEnv = test.NewEnvironment(ctx, env) @@ -64,8 +64,8 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = options.ToContext(ctx, opts) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ Spec: v1beta1.EC2NodeClassSpec{ diff --git a/pkg/test/options.go b/pkg/test/options.go new file mode 100644 index 000000000000..378b8e2dbc5a --- /dev/null +++ b/pkg/test/options.go @@ -0,0 +1,57 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "fmt" + "time" + + "github.com/imdario/mergo" + "github.com/samber/lo" + + "github.com/aws/karpenter/pkg/operator/options" +) + +type OptionsFields struct { + AssumeRoleARN *string + AssumeRoleDuration *time.Duration + ClusterCABundle *string + ClusterName *string + ClusterEndpoint *string + IsolatedVPC *bool + VMMemoryOverheadPercent *float64 + InterruptionQueueName *string + ReservedENIs *int +} + +func Options(overrides ...OptionsFields) *options.Options { + opts := OptionsFields{} + for _, override := range overrides { + if err := mergo.Merge(&opts, override, mergo.WithOverride); err != nil { + panic(fmt.Sprintf("Failed to merge settings: %s", err)) + } + } + return &options.Options{ + AssumeRoleARN: lo.FromPtrOr(opts.AssumeRoleARN, ""), + AssumeRoleDuration: lo.FromPtrOr(opts.AssumeRoleDuration, 15*time.Minute), + ClusterCABundle: lo.FromPtrOr(opts.ClusterCABundle, ""), + ClusterName: lo.FromPtrOr(opts.ClusterName, "test-cluster"), + ClusterEndpoint: lo.FromPtrOr(opts.ClusterEndpoint, "https://test-cluster"), + IsolatedVPC: lo.FromPtrOr(opts.IsolatedVPC, false), + VMMemoryOverheadPercent: lo.FromPtrOr(opts.VMMemoryOverheadPercent, 0.075), + InterruptionQueueName: lo.FromPtrOr(opts.InterruptionQueueName, ""), + ReservedENIs: lo.FromPtrOr(opts.ReservedENIs, 0), + } +} diff --git a/test/pkg/environment/aws/environment.go b/test/pkg/environment/aws/environment.go index 7961472461b2..7d4ab57064ce 100644 --- a/test/pkg/environment/aws/environment.go +++ b/test/pkg/environment/aws/environment.go @@ -15,6 +15,7 @@ limitations under the License. package aws import ( + "os" "testing" "github.com/aws/aws-sdk-go/aws" @@ -54,6 +55,10 @@ type Environment struct { TimeStreamAPI timestreamwriteiface.TimestreamWriteAPI SQSProvider *interruption.SQSProvider + + ClusterName string + ClusterEndpoint string + InterruptionQueueName string } func NewEnvironment(t *testing.T) *Environment { @@ -80,6 +85,10 @@ func NewEnvironment(t *testing.T) *Environment { EKSAPI: eks.New(session), SQSProvider: interruption.NewSQSProvider(sqs.New(session)), TimeStreamAPI: GetTimeStreamAPI(session), + + ClusterName: lo.Must(os.LookupEnv("CLUSTER_NAME")), + ClusterEndpoint: lo.Must(os.LookupEnv("CLUSTER_ENDPOINT")), + InterruptionQueueName: lo.Must(os.LookupEnv("INTERRUPTION_QUEUE_NAME")), } } diff --git a/test/suites/chaos/suite_test.go b/test/suites/chaos/suite_test.go index 1f12d51e6afd..b2d2dedb01b5 100644 --- a/test/suites/chaos/suite_test.go +++ b/test/suites/chaos/suite_test.go @@ -38,19 +38,18 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" nodeutils "github.com/aws/karpenter-core/pkg/utils/node" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/test/pkg/debug" - "github.com/aws/karpenter/test/pkg/environment/common" + "github.com/aws/karpenter/test/pkg/environment/aws" ) -var env *common.Environment +var env *aws.Environment func TestChaos(t *testing.T) { RegisterFailHandler(Fail) BeforeSuite(func() { - env = common.NewEnvironment(t) + env = aws.NewEnvironment(t) }) AfterSuite(func() { env.Stop() @@ -69,8 +68,8 @@ var _ = Describe("Chaos", func() { defer cancel() provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ @@ -114,8 +113,8 @@ var _ = Describe("Chaos", func() { defer cancel() provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ diff --git a/test/suites/consolidation/suite_test.go b/test/suites/consolidation/suite_test.go index 1aed325442f8..6be4afe97e3f 100644 --- a/test/suites/consolidation/suite_test.go +++ b/test/suites/consolidation/suite_test.go @@ -28,7 +28,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/test/pkg/debug" @@ -60,8 +59,8 @@ var _ = AfterEach(func() { env.AfterEach() }) var _ = Describe("Consolidation", func() { It("should consolidate nodes (delete)", Label(debug.NoWatch), Label(debug.NoEvents), func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ @@ -126,8 +125,8 @@ var _ = Describe("Consolidation", func() { }) It("should consolidate on-demand nodes (replace)", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ @@ -242,8 +241,8 @@ var _ = Describe("Consolidation", func() { }) It("should consolidate on-demand nodes to spot (replace)", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ diff --git a/test/suites/drift/suite_test.go b/test/suites/drift/suite_test.go index 0aae96f77991..baf91ee74faa 100644 --- a/test/suites/drift/suite_test.go +++ b/test/suites/drift/suite_test.go @@ -39,7 +39,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -73,8 +72,8 @@ var _ = Describe("Drift", Label("AWS"), func() { BeforeEach(func() { customAMI = env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", 1) nodeTemplate = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner = test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{{Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeOnDemand}}}, @@ -101,7 +100,7 @@ var _ = Describe("Drift", Label("AWS"), func() { oldCustomAMI := *parameter.Parameter.Value nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyCustom nodeTemplate.Spec.AMISelector = map[string]string{"aws-ids": oldCustomAMI} - nodeTemplate.Spec.UserData = awssdk.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", settings.FromContext(env.Context).ClusterName)) + nodeTemplate.Spec.UserData = awssdk.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)) env.ExpectCreated(pod, nodeTemplate, provisioner) env.EventuallyExpectHealthy(pod) @@ -134,7 +133,7 @@ var _ = Describe("Drift", Label("AWS"), func() { oldCustomAMI := *parameter.Parameter.Value nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyCustom nodeTemplate.Spec.AMISelector = map[string]string{"aws-ids": oldCustomAMI} - nodeTemplate.Spec.UserData = awssdk.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", settings.FromContext(env.Context).ClusterName)) + nodeTemplate.Spec.UserData = awssdk.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)) env.ExpectCreated(pod, nodeTemplate, provisioner) env.EventuallyExpectHealthy(pod) @@ -151,7 +150,7 @@ var _ = Describe("Drift", Label("AWS"), func() { }) It("should deprovision nodes that have drifted due to securitygroup", func() { By("getting the cluster vpc id") - output, err := env.EKSAPI.DescribeCluster(&eks.DescribeClusterInput{Name: awssdk.String(settings.FromContext(env.Context).ClusterName)}) + output, err := env.EKSAPI.DescribeCluster(&eks.DescribeClusterInput{Name: awssdk.String(env.ClusterName)}) Expect(err).To(BeNil()) By("creating new security group") @@ -165,11 +164,11 @@ var _ = Describe("Drift", Label("AWS"), func() { Tags: []*ec2.Tag{ { Key: awssdk.String("karpenter.sh/discovery"), - Value: awssdk.String(settings.FromContext(env.Context).ClusterName), + Value: awssdk.String(env.ClusterName), }, { Key: awssdk.String(test.DiscoveryLabel), - Value: awssdk.String(settings.FromContext(env.Context).ClusterName), + Value: awssdk.String(env.ClusterName), }, { Key: awssdk.String("creation-date"), @@ -185,7 +184,7 @@ var _ = Describe("Drift", Label("AWS"), func() { var securitygroups []aws.SecurityGroup var testSecurityGroup aws.SecurityGroup Eventually(func(g Gomega) { - securitygroups = env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + securitygroups = env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": env.ClusterName}) testSecurityGroup, _ = lo.Find(securitygroups, func(sg aws.SecurityGroup) bool { return awssdk.StringValue(sg.GroupName) == "security-group-drift" }) @@ -222,7 +221,7 @@ var _ = Describe("Drift", Label("AWS"), func() { env.EventuallyExpectNotFound(pod, machine, node) }) It("should deprovision nodes that have drifted due to subnets", func() { - subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(len(subnets)).To(BeNumerically(">", 1)) nodeTemplate.Spec.SubnetSelector = map[string]string{"aws-ids": subnets[0].ID} @@ -300,7 +299,7 @@ var _ = Describe("Drift", Label("AWS"), func() { ) DescribeTable("AWSNodeTemplate Drift", func(fieldName string, nodeTemplateSpec v1alpha1.AWSNodeTemplateSpec) { if fieldName == "InstanceProfile" { - nodeTemplateSpec.AWS.InstanceProfile = awssdk.String(fmt.Sprintf("KarpenterNodeInstanceProfile-Drift-%s", settings.FromContext(env.Context).ClusterName)) + nodeTemplateSpec.AWS.InstanceProfile = awssdk.String(fmt.Sprintf("KarpenterNodeInstanceProfile-Drift-%s", env.ClusterName)) ExpectInstanceProfileCreated(nodeTemplateSpec.AWS.InstanceProfile) } @@ -476,7 +475,7 @@ func ExpectInstanceProfileCreated(instanceProfileName *string) { Tags: []*iam.Tag{ { Key: awssdk.String(test.DiscoveryLabel), - Value: awssdk.String(settings.FromContext(env.Context).ClusterName), + Value: awssdk.String(env.ClusterName), }, }, } @@ -485,7 +484,7 @@ func ExpectInstanceProfileCreated(instanceProfileName *string) { Expect(ignoreAlreadyExists(err)).ToNot(HaveOccurred()) addInstanceProfile := &iam.AddRoleToInstanceProfileInput{ InstanceProfileName: instanceProfileName, - RoleName: awssdk.String(fmt.Sprintf("KarpenterNodeRole-%s", settings.FromContext(env.Context).ClusterName)), + RoleName: awssdk.String(fmt.Sprintf("KarpenterNodeRole-%s", env.ClusterName)), } _, err = env.IAMAPI.AddRoleToInstanceProfile(addInstanceProfile) Expect(ignoreAlreadyContainsRole(err)).ToNot(HaveOccurred()) diff --git a/test/suites/expiration/expiration_test.go b/test/suites/expiration/expiration_test.go index 7a3fbc83abbd..9c3c7ff9ab96 100644 --- a/test/suites/expiration/expiration_test.go +++ b/test/suites/expiration/expiration_test.go @@ -34,7 +34,6 @@ import ( "github.com/aws/aws-sdk-go/service/ssm" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" - "github.com/aws/karpenter/pkg/apis/settings" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -67,8 +66,8 @@ var _ = Describe("Expiration", func() { var provisioner *v1alpha5.Provisioner BeforeEach(func() { nodeTemplate = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner = test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: nodeTemplate.Name}, diff --git a/test/suites/integration/ami_test.go b/test/suites/integration/ami_test.go index 358c5f5dd526..44625235af31 100644 --- a/test/suites/integration/ami_test.go +++ b/test/suites/integration/ami_test.go @@ -32,7 +32,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -48,8 +47,8 @@ var _ = Describe("AMI", func() { It("should use the AMI defined by the AMI Selector", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyAL2, }, AMISelector: map[string]string{"aws-ids": customAMI}, @@ -71,12 +70,12 @@ var _ = Describe("AMI", func() { Expect(err).To(BeNil()) oldCustomAMI := *parameter.Parameter.Value provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyCustom, }, AMISelector: map[string]string{"aws-ids": fmt.Sprintf("%s,%s", customAMI, oldCustomAMI)}, - UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", settings.FromContext(env.Context).ClusterName)), + UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)), }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) pod := test.Pod() @@ -95,12 +94,12 @@ var _ = Describe("AMI", func() { Expect(output.Images).To(HaveLen(1)) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyCustom, }, AMISelector: map[string]string{"aws::name": *output.Images[0].Name, "aws::owners": "fakeOwnerValue"}, - UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", settings.FromContext(env.Context).ClusterName)), + UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)), }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) @@ -118,12 +117,12 @@ var _ = Describe("AMI", func() { Expect(output.Images).To(HaveLen(1)) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyCustom, }, AMISelector: map[string]string{"aws::name": *output.Images[0].Name}, - UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", settings.FromContext(env.Context).ClusterName)), + UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)), }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) @@ -137,12 +136,12 @@ var _ = Describe("AMI", func() { }) It("should support ami selector aws::ids", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyCustom, }, AMISelector: map[string]string{"aws::ids": customAMI}, - UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", settings.FromContext(env.Context).ClusterName)), + UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)), }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) pod := test.Pod() @@ -157,8 +156,8 @@ var _ = Describe("AMI", func() { Context("AMIFamily", func() { It("should provision a node using the AL2 family", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, @@ -170,8 +169,8 @@ var _ = Describe("AMI", func() { }) It("should provision a node using the Bottlerocket family", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyBottlerocket, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ @@ -184,8 +183,8 @@ var _ = Describe("AMI", func() { }) It("should provision a node using the Ubuntu family", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyUbuntu, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ @@ -212,12 +211,12 @@ var _ = Describe("AMI", func() { }) It("should support Custom AMIFamily with AMI Selectors", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyCustom, }, AMISelector: map[string]string{"aws-ids": customAMI}, - UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", settings.FromContext(env.Context).ClusterName)), + UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)), }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) pod := test.Pod() @@ -231,8 +230,8 @@ var _ = Describe("AMI", func() { It("should have the AWSNodeTemplateStatus for AMIs using wildcard", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, AMISelector: map[string]string{"aws::name": "*"}, }) @@ -244,8 +243,8 @@ var _ = Describe("AMI", func() { It("should have the AWSNodeTemplateStatus for AMIs using tags", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, AMISelector: map[string]string{"aws-ids": customAMI}, }) @@ -263,8 +262,8 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/al2_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyAL2, }, UserData: aws.String(string(content)), @@ -292,8 +291,8 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/al2_no_mime_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyAL2, }, UserData: aws.String(string(content)), @@ -321,8 +320,8 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/br_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyBottlerocket, }, UserData: aws.String(string(content)), @@ -353,8 +352,8 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/windows_userdata_input.ps1") Expect(err).ToNot(HaveOccurred()) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyWindows2022, }, UserData: aws.String(string(content)), diff --git a/test/suites/integration/aws_metadata_test.go b/test/suites/integration/aws_metadata_test.go index 018d29010c9a..157fb3dc1348 100644 --- a/test/suites/integration/aws_metadata_test.go +++ b/test/suites/integration/aws_metadata_test.go @@ -22,7 +22,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -32,8 +31,8 @@ var _ = Describe("MetadataOptions", func() { It("should use specified metadata options", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, LaunchTemplate: v1alpha1.LaunchTemplate{ MetadataOptions: &v1alpha1.MetadataOptions{ HTTPEndpoint: aws.String("enabled"), diff --git a/test/suites/integration/backwards_compatability_test.go b/test/suites/integration/backwards_compatability_test.go index 87c609a201a5..17d707277f56 100644 --- a/test/suites/integration/backwards_compatability_test.go +++ b/test/suites/integration/backwards_compatability_test.go @@ -21,7 +21,6 @@ import ( "github.com/samber/lo" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" ) @@ -30,8 +29,8 @@ var _ = Describe("BackwardsCompatability", func() { provisioner := test.Provisioner( test.ProvisionerOptions{ Provider: &v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, Tags: map[string]string{ "custom-tag": "custom-value", "custom-tag2": "custom-value2", diff --git a/test/suites/integration/block_device_mappings_test.go b/test/suites/integration/block_device_mappings_test.go index 3e11d74e5154..7d8f4ffb6a64 100644 --- a/test/suites/integration/block_device_mappings_test.go +++ b/test/suites/integration/block_device_mappings_test.go @@ -22,7 +22,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" "github.com/aws/karpenter-core/pkg/utils/resources" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -32,8 +31,8 @@ var _ = Describe("BlockDeviceMappings", func() { It("should use specified block device mappings", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, LaunchTemplate: v1alpha1.LaunchTemplate{ BlockDeviceMappings: []*v1alpha1.BlockDeviceMapping{ { diff --git a/test/suites/integration/cni_test.go b/test/suites/integration/cni_test.go index d686de9b2707..85c78ff9b64e 100644 --- a/test/suites/integration/cni_test.go +++ b/test/suites/integration/cni_test.go @@ -26,7 +26,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -39,8 +38,8 @@ var _ = Describe("CNITests", func() { }) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyAL2, }, }) @@ -58,8 +57,8 @@ var _ = Describe("CNITests", func() { It("should set eni-limited maxPods when AWSENILimited when AWS_ENI_LIMITED_POD_DENSITY is true", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyAL2, }, }) @@ -79,8 +78,8 @@ var _ = Describe("CNITests", func() { }) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyAL2, }, }) diff --git a/test/suites/integration/daemonset_test.go b/test/suites/integration/daemonset_test.go index 3498429f1e0c..ba89b43d28c5 100644 --- a/test/suites/integration/daemonset_test.go +++ b/test/suites/integration/daemonset_test.go @@ -26,7 +26,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -45,8 +44,8 @@ var _ = Describe("DaemonSet", func() { BeforeEach(func() { provider = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner = test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, diff --git a/test/suites/integration/emptiness_test.go b/test/suites/integration/emptiness_test.go index 3bfa2eaa2bdb..837d0b8bdba3 100644 --- a/test/suites/integration/emptiness_test.go +++ b/test/suites/integration/emptiness_test.go @@ -25,7 +25,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" ) @@ -33,8 +32,8 @@ import ( var _ = Describe("Emptiness", func() { It("should terminate an empty node", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, diff --git a/test/suites/integration/extended_resources_test.go b/test/suites/integration/extended_resources_test.go index 93850bdc87f8..7e222743d734 100644 --- a/test/suites/integration/extended_resources_test.go +++ b/test/suites/integration/extended_resources_test.go @@ -30,7 +30,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -41,8 +40,8 @@ var _ = Describe("Extended Resources", func() { ExpectNvidiaDevicePluginCreated() provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, @@ -80,8 +79,8 @@ var _ = Describe("Extended Resources", func() { // For Bottlerocket, we are testing that resources are initialized without needing a device plugin provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ AMIFamily: &v1alpha1.AMIFamilyBottlerocket, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, @@ -124,8 +123,8 @@ var _ = Describe("Extended Resources", func() { "aws.enablePodENI": "true", }) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, @@ -178,8 +177,8 @@ var _ = Describe("Extended Resources", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ AMIFamily: &v1alpha1.AMIFamilyCustom, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, AMISelector: map[string]string{ "aws-ids": customAMI, @@ -195,8 +194,8 @@ var _ = Describe("Extended Resources", func() { }, }, }) - provider.Spec.UserData = lo.ToPtr(fmt.Sprintf(string(rawContent), settings.FromContext(env.Context).ClusterName, - settings.FromContext(env.Context).ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)) + provider.Spec.UserData = lo.ToPtr(fmt.Sprintf(string(rawContent), env.ClusterName, + env.ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)) numPods := 1 dep := test.Deployment(test.DeploymentOptions{ @@ -231,8 +230,8 @@ var _ = Describe("Extended Resources", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, AMISelector: map[string]string{"aws-ids": "ami-0fae925f94979981f"}, }) diff --git a/test/suites/integration/hash_test.go b/test/suites/integration/hash_test.go index 84fa5302c209..08e6f6dbc365 100644 --- a/test/suites/integration/hash_test.go +++ b/test/suites/integration/hash_test.go @@ -21,7 +21,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -31,8 +30,8 @@ var _ = Describe("CRD Hash", func() { It("should have Provisioner hash", func() { nodeTemplate := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, }) provisioner := test.Provisioner(test.ProvisionerOptions{ @@ -54,8 +53,8 @@ var _ = Describe("CRD Hash", func() { It("should have AWSNodeTemplate hash", func() { nodeTemplate := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, }) env.ExpectCreated(nodeTemplate) diff --git a/test/suites/integration/instance_profile_test.go b/test/suites/integration/instance_profile_test.go index bdb19893d9f6..5e8e29292d5d 100644 --- a/test/suites/integration/instance_profile_test.go +++ b/test/suites/integration/instance_profile_test.go @@ -25,7 +25,6 @@ import ( corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" coretest "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" awserrors "github.com/aws/karpenter/pkg/errors" "github.com/aws/karpenter/pkg/providers/instanceprofile" @@ -49,7 +48,7 @@ var _ = Describe("InstanceProfile Generation", func() { Tags: map[string]string{"*": "*"}, }, }, - Role: fmt.Sprintf("KarpenterNodeRole-%s", settings.FromContext(env.Context).ClusterName), + Role: fmt.Sprintf("KarpenterNodeRole-%s", env.ClusterName), }, }) }) diff --git a/test/suites/integration/kubelet_config_test.go b/test/suites/integration/kubelet_config_test.go index 02eba693982f..aec27cb931c9 100644 --- a/test/suites/integration/kubelet_config_test.go +++ b/test/suites/integration/kubelet_config_test.go @@ -26,7 +26,6 @@ import ( "knative.dev/pkg/ptr" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" - "github.com/aws/karpenter/pkg/apis/settings" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -41,8 +40,8 @@ var _ = Describe("KubeletConfiguration Overrides", func() { BeforeEach(func() { nodeTemplate = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) // MaxPods needs to account for the daemonsets that will run on the nodes @@ -176,8 +175,8 @@ var _ = Describe("KubeletConfiguration Overrides", func() { }) It("should schedule pods onto separate nodes when maxPods is set", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) // MaxPods needs to account for the daemonsets that will run on the nodes @@ -219,8 +218,8 @@ var _ = Describe("KubeletConfiguration Overrides", func() { }) It("should schedule pods onto separate nodes when podsPerCore is set", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) // PodsPerCore needs to account for the daemonsets that will run on the nodes // This will have 4 pods available on each node (2 taken by daemonset pods) @@ -273,8 +272,8 @@ var _ = Describe("KubeletConfiguration Overrides", func() { }) It("should ignore podsPerCore value when Bottlerocket is used", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyBottlerocket, }}) // All pods should schedule to a single node since we are ignoring podsPerCore value diff --git a/test/suites/integration/scheduling_test.go b/test/suites/integration/scheduling_test.go index 72dd94c75ff4..352b0a526f85 100644 --- a/test/suites/integration/scheduling_test.go +++ b/test/suites/integration/scheduling_test.go @@ -29,7 +29,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/test/pkg/debug" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -44,8 +43,8 @@ var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { BeforeEach(func() { provider = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner = test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, diff --git a/test/suites/integration/security_group_test.go b/test/suites/integration/security_group_test.go index 17775726209f..e13e994919ea 100644 --- a/test/suites/integration/security_group_test.go +++ b/test/suites/integration/security_group_test.go @@ -28,7 +28,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -37,14 +36,14 @@ import ( var _ = Describe("SecurityGroups", func() { It("should use the security-group-id selector", func() { - securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(len(securityGroups)).To(BeNumerically(">", 1)) ids := strings.Join([]string{*securityGroups[0].GroupId, *securityGroups[1].GroupId}, ",") provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{"aws-ids": ids}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) @@ -58,7 +57,7 @@ var _ = Describe("SecurityGroups", func() { }) It("should use the security group selector with multiple tag values", func() { - securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(len(securityGroups)).To(BeNumerically(">", 1)) first := securityGroups[0] last := securityGroups[len(securityGroups)-1] @@ -69,7 +68,7 @@ var _ = Describe("SecurityGroups", func() { lo.FromPtr(lo.FindOrElse(first.Tags, &ec2.Tag{}, func(tag *ec2.Tag) bool { return lo.FromPtr(tag.Key) == "Name" }).Value), lo.FromPtr(lo.FindOrElse(last.Tags, &ec2.Tag{}, func(tag *ec2.Tag) bool { return lo.FromPtr(tag.Key) == "Name" }).Value), )}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) @@ -85,8 +84,8 @@ var _ = Describe("SecurityGroups", func() { It("should update the AWSNodeTemplateStatus for security groups", func() { nodeTemplate := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, }) @@ -96,7 +95,7 @@ var _ = Describe("SecurityGroups", func() { }) func EventuallyExpectSecurityGroups(env *aws.Environment, nodeTemplate *v1alpha1.AWSNodeTemplate) { - securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(securityGroups).ToNot(HaveLen(0)) ids := sets.New(lo.Map(securityGroups, func(s aws.SecurityGroup, _ int) string { diff --git a/test/suites/integration/storage_test.go b/test/suites/integration/storage_test.go index 5dd8acf231ae..f0fc0322883e 100644 --- a/test/suites/integration/storage_test.go +++ b/test/suites/integration/storage_test.go @@ -28,7 +28,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -53,8 +52,8 @@ var _ = Describe("Dynamic PVC", func() { } provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) @@ -91,8 +90,8 @@ var _ = Describe("Dynamic PVC", func() { var _ = Describe("Static PVC", func() { It("should run a pod with a static persistent volume", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) diff --git a/test/suites/integration/subnet_test.go b/test/suites/integration/subnet_test.go index 2158c483ce6e..45901a433c64 100644 --- a/test/suites/integration/subnet_test.go +++ b/test/suites/integration/subnet_test.go @@ -29,7 +29,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -37,14 +36,14 @@ import ( var _ = Describe("Subnets", func() { It("should use the subnet-id selector", func() { - subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(len(subnets)).ToNot(Equal(0)) shuffledAZs := lo.Shuffle(lo.Keys(subnets)) firstSubnet := subnets[shuffledAZs[0]][0] provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, SubnetSelector: map[string]string{"aws-ids": firstSubnet}, }, }) @@ -58,7 +57,7 @@ var _ = Describe("Subnets", func() { env.ExpectInstance(pod.Spec.NodeName).To(HaveField("SubnetId", HaveValue(Equal(firstSubnet)))) }) It("should use resource based naming as node names", func() { - subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(len(subnets)).ToNot(Equal(0)) allSubnets := lo.Flatten(lo.Values(subnets)) @@ -70,8 +69,8 @@ var _ = Describe("Subnets", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) @@ -85,14 +84,14 @@ var _ = Describe("Subnets", func() { }) It("should use the subnet tag selector with multiple tag values", func() { // Get all the subnets for the cluster - subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(len(subnets)).To(BeNumerically(">", 1)) firstSubnet := subnets[0] lastSubnet := subnets[len(subnets)-1] provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, SubnetSelector: map[string]string{"Name": fmt.Sprintf("%s,%s", firstSubnet.Name, lastSubnet.Name)}, }, }) @@ -107,14 +106,14 @@ var _ = Describe("Subnets", func() { }) It("should use a subnet within the AZ requested", func() { - subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(len(subnets)).ToNot(Equal(0)) shuffledAZs := lo.Shuffle(lo.Keys(subnets)) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, }) provisioner := test.Provisioner(test.ProvisionerOptions{ @@ -141,8 +140,8 @@ var _ = Describe("Subnets", func() { It("should have the AWSNodeTemplateStatus for subnets", func() { nodeTemplate := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, }) @@ -198,7 +197,7 @@ type SubnetInfo struct { } func EventuallyExpectSubnets(env *aws.Environment, nodeTemplate *v1alpha1.AWSNodeTemplate) { - subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(subnets).ToNot(HaveLen(0)) ids := sets.New(lo.Flatten(lo.Values(subnets))...) diff --git a/test/suites/integration/tags_test.go b/test/suites/integration/tags_test.go index fb958b140b51..cd97aea53bf5 100644 --- a/test/suites/integration/tags_test.go +++ b/test/suites/integration/tags_test.go @@ -29,7 +29,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" coretest "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/apis/v1beta1" "github.com/aws/karpenter/pkg/providers/instance" @@ -41,8 +40,8 @@ var _ = Describe("Tags", func() { It("should tag all associated resources", func() { provider := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, Tags: map[string]string{"TestTag": "TestVal"}, }, }) @@ -62,8 +61,8 @@ var _ = Describe("Tags", func() { It("should tag all associated resources with global tags", func() { provider := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, }) @@ -94,10 +93,10 @@ var _ = Describe("Tags", func() { BeforeEach(func() { nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{ SecurityGroupSelectorTerms: []v1beta1.SecurityGroupSelectorTerm{{ - Tags: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + Tags: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}, SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{{ - Tags: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + Tags: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}, }}) @@ -169,8 +168,8 @@ var _ = Describe("Tags", func() { It("shouldn't tag nodes provisioned by v1alpha5 provisioner", func() { nodeTemplate := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := coretest.Provisioner(coretest.ProvisionerOptions{ diff --git a/test/suites/integration/termination_test.go b/test/suites/integration/termination_test.go index 9eb99453016f..3fac773e06eb 100644 --- a/test/suites/integration/termination_test.go +++ b/test/suites/integration/termination_test.go @@ -23,7 +23,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" ) @@ -31,8 +30,8 @@ import ( var _ = Describe("Termination", func() { It("should terminate the node and the instance on deletion", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, diff --git a/test/suites/integration/webhook_test.go b/test/suites/integration/webhook_test.go index a946f0be33bb..996666d7633a 100644 --- a/test/suites/integration/webhook_test.go +++ b/test/suites/integration/webhook_test.go @@ -25,7 +25,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" ) @@ -133,8 +132,8 @@ var _ = Describe("Webhooks", func() { It("should error when provider and providerRef are combined", func() { Expect(env.Client.Create(env, test.Provisioner(test.ProvisionerOptions{ Provider: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, ProviderRef: &v1alpha5.MachineTemplateRef{Name: "test"}, }))).ToNot(Succeed()) @@ -266,15 +265,15 @@ var _ = Describe("Webhooks", func() { It("should error when amiSelector is not defined for amiFamily Custom", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ AMIFamily: &v1alpha1.AMIFamilyCustom, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}))).ToNot(Succeed()) }) It("should fail if both userdata and launchTemplate are set", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ LaunchTemplate: v1alpha1.LaunchTemplate{LaunchTemplateName: ptr.String("lt")}, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, UserData: ptr.String("data"), }))).ToNot(Succeed()) @@ -282,47 +281,47 @@ var _ = Describe("Webhooks", func() { It("should fail if both amiSelector and launchTemplate are set", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ LaunchTemplate: v1alpha1.LaunchTemplate{LaunchTemplateName: ptr.String("lt")}, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, AMISelector: map[string]string{"foo": "bar"}, }))).ToNot(Succeed()) }) It("should fail for poorly formatted aws-ids", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, AMISelector: map[string]string{"aws-ids": "must-start-with-ami"}, }))).ToNot(Succeed()) }) It("should succeed when tags don't contain restricted keys", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, Tags: map[string]string{"karpenter.sh/custom-key": "custom-value", "kubernetes.io/role/key": "custom-value"}, }, }))).To(Succeed()) }) It("should error when tags contains a restricted key", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, Tags: map[string]string{"karpenter.sh/provisioner-name": "custom-value"}, }, }))).ToNot(Succeed()) Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - Tags: map[string]string{"karpenter.sh/managed-by": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + Tags: map[string]string{"karpenter.sh/managed-by": env.ClusterName}, }, }))).ToNot(Succeed()) Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - Tags: map[string]string{fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(env.Context).ClusterName): "owned"}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + Tags: map[string]string{fmt.Sprintf("kubernetes.io/cluster/%s", env.ClusterName): "owned"}, }, }))).ToNot(Succeed()) }) @@ -330,22 +329,22 @@ var _ = Describe("Webhooks", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, "aws-ids": "sg-12345", }, SubnetSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, }, }, }))).ToNot(Succeed()) Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, "aws-ids": "sg-12345", }, SubnetSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, }, }, }))).ToNot(Succeed()) @@ -354,10 +353,10 @@ var _ = Describe("Webhooks", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, }, SubnetSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, "aws-ids": "subnet-12345", }, }, @@ -365,10 +364,10 @@ var _ = Describe("Webhooks", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, }, SubnetSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, "aws-ids": "subnet-12345", }, }, @@ -378,10 +377,10 @@ var _ = Describe("Webhooks", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, }, SubnetSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, }, }, AMISelector: map[string]string{ @@ -392,10 +391,10 @@ var _ = Describe("Webhooks", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, }, SubnetSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, }, }, AMISelector: map[string]string{ diff --git a/test/suites/interruption/suite_test.go b/test/suites/interruption/suite_test.go index a7b4f6128a3b..f370ba99d16d 100644 --- a/test/suites/interruption/suite_test.go +++ b/test/suites/interruption/suite_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -29,10 +30,10 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/controllers/interruption/messages" "github.com/aws/karpenter/pkg/controllers/interruption/messages/scheduledchange" + "github.com/aws/karpenter/pkg/operator/options" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/pkg/utils" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -53,6 +54,9 @@ func TestInterruption(t *testing.T) { } var _ = BeforeEach(func() { + env.Context = options.ToContext(env.Context, awstest.Options(awstest.OptionsFields{ + InterruptionQueueName: lo.ToPtr(env.InterruptionQueueName), + })) env.BeforeEach() env.ExpectQueueExists() }) @@ -63,8 +67,8 @@ var _ = Describe("Interruption", Label("AWS"), func() { It("should terminate the spot instance and spin-up a new node on spot interruption warning", func() { By("Creating a single healthy node with a healthy deployment") provider = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ @@ -111,8 +115,8 @@ var _ = Describe("Interruption", Label("AWS"), func() { It("should terminate the node at the API server when the EC2 instance is stopped", func() { By("Creating a single healthy node with a healthy deployment") provider = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ @@ -151,8 +155,8 @@ var _ = Describe("Interruption", Label("AWS"), func() { It("should terminate the node at the API server when the EC2 instance is terminated", func() { By("Creating a single healthy node with a healthy deployment") provider = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ @@ -191,8 +195,8 @@ var _ = Describe("Interruption", Label("AWS"), func() { It("should terminate the node when receiving a scheduled change health event", func() { By("Creating a single healthy node with a healthy deployment") provider = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ diff --git a/test/suites/ipv6/suite_test.go b/test/suites/ipv6/suite_test.go index 2813c51c171c..7e6bad6f0e95 100644 --- a/test/suites/ipv6/suite_test.go +++ b/test/suites/ipv6/suite_test.go @@ -25,7 +25,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -52,8 +51,8 @@ var _ = Describe("IPv6", func() { It("should provision an IPv6 node by discovering kube-dns IPv6", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, Requirements: []v1.NodeSelectorRequirement{ { @@ -82,8 +81,8 @@ var _ = Describe("IPv6", func() { clusterDNSAddr := env.ExpectIPv6ClusterDNS() provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, Requirements: []v1.NodeSelectorRequirement{ { diff --git a/test/suites/machine/garbage_collection_test.go b/test/suites/machine/garbage_collection_test.go index 4062a12f7f9e..74247796d631 100644 --- a/test/suites/machine/garbage_collection_test.go +++ b/test/suites/machine/garbage_collection_test.go @@ -43,8 +43,8 @@ var _ = Describe("NodeClaimGarbageCollection", func() { BeforeEach(func() { provisioner = test.Provisioner() - securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) - subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": env.ClusterName}) + subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(securityGroups).ToNot(HaveLen(0)) Expect(subnets).ToNot(HaveLen(0)) @@ -75,7 +75,7 @@ var _ = Describe("NodeClaimGarbageCollection", func() { ResourceType: aws.String(ec2.ResourceTypeInstance), Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(env.Context).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", env.ClusterName)), Value: aws.String("owned"), }, { @@ -93,8 +93,8 @@ var _ = Describe("NodeClaimGarbageCollection", func() { // Update the userData for the instance input with the correct provisionerName rawContent, err := os.ReadFile("testdata/al2_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) - instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), settings.FromContext(env.Context).ClusterName, - settings.FromContext(env.Context).ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)))) + instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), env.ClusterName, + env.ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)))) // Create an instance manually to mock Karpenter launching an instance out := env.ExpectRunInstances(instanceInput) @@ -120,7 +120,7 @@ var _ = Describe("NodeClaimGarbageCollection", func() { Tags: []*ec2.Tag{ { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(env.Context).ClusterName), + Value: aws.String(env.ClusterName), }, }, }) @@ -139,8 +139,8 @@ var _ = Describe("NodeClaimGarbageCollection", func() { }) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, diff --git a/test/suites/machine/link_test.go b/test/suites/machine/link_test.go index 87b1c0739192..18958b98daf5 100644 --- a/test/suites/machine/link_test.go +++ b/test/suites/machine/link_test.go @@ -42,8 +42,8 @@ var _ = Describe("MachineLink", func() { var instanceInput *ec2.RunInstancesInput BeforeEach(func() { - securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) - subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": env.ClusterName}) + subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(securityGroups).ToNot(HaveLen(0)) Expect(subnets).ToNot(HaveLen(0)) @@ -74,7 +74,7 @@ var _ = Describe("MachineLink", func() { ResourceType: aws.String(ec2.ResourceTypeInstance), Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(env.Context).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", env.ClusterName)), Value: aws.String("owned"), }, }, @@ -87,8 +87,8 @@ var _ = Describe("MachineLink", func() { It("should succeed to link a Machine for an existing instance launched by Karpenter", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ AMIFamily: &v1alpha1.AMIFamilyAL2, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, @@ -102,8 +102,8 @@ var _ = Describe("MachineLink", func() { Key: aws.String(v1alpha5.ProvisionerNameLabelKey), Value: aws.String(provisioner.Name), }) - instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), settings.FromContext(env.Context).ClusterName, - settings.FromContext(env.Context).ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)))) + instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), env.ClusterName, + env.ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)))) // Create an instance manually to mock Karpenter launching an instance out := env.ExpectRunInstances(instanceInput) @@ -141,15 +141,15 @@ var _ = Describe("MachineLink", func() { return aws.StringValue(t.Key) == v1alpha5.MachineManagedByAnnotationKey }) g.Expect(ok).To(BeTrue()) - g.Expect(aws.StringValue(tag.Value)).To(Equal(settings.FromContext(env.Context).ClusterName)) + g.Expect(aws.StringValue(tag.Value)).To(Equal(env.ClusterName)) }, time.Minute, time.Second).Should(Succeed()) }) It("should succeed to link a Machine for an existing instance launched by Karpenter with provider", func() { provisioner := test.Provisioner(test.ProvisionerOptions{ Provider: &v1alpha1.AWS{ AMIFamily: &v1alpha1.AMIFamilyAL2, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, }) env.ExpectCreated(provisioner) @@ -161,8 +161,8 @@ var _ = Describe("MachineLink", func() { Key: aws.String(v1alpha5.ProvisionerNameLabelKey), Value: aws.String(provisioner.Name), }) - instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), settings.FromContext(env.Context).ClusterName, - settings.FromContext(env.Context).ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)))) + instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), env.ClusterName, + env.ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)))) // Create an instance manually to mock Karpenter launching an instance out := env.ExpectRunInstances(instanceInput) @@ -200,14 +200,14 @@ var _ = Describe("MachineLink", func() { return aws.StringValue(t.Key) == v1alpha5.MachineManagedByAnnotationKey }) g.Expect(ok).To(BeTrue()) - g.Expect(aws.StringValue(tag.Value)).To(Equal(settings.FromContext(env.Context).ClusterName)) + g.Expect(aws.StringValue(tag.Value)).To(Equal(env.ClusterName)) }, time.Minute, time.Second).Should(Succeed()) }) It("should succeed to link a Machine for an existing instance re-owned by Karpenter", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ AMIFamily: &v1alpha1.AMIFamilyAL2, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, @@ -219,8 +219,8 @@ var _ = Describe("MachineLink", func() { Expect(err).ToNot(HaveOccurred()) // No tag specifications since we're mocking an instance not launched by Karpenter - instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), settings.FromContext(env.Context).ClusterName, - settings.FromContext(env.Context).ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)))) + instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), env.ClusterName, + env.ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)))) // Create an instance manually to mock Karpenter launching an instance out := env.ExpectRunInstances(instanceInput) @@ -263,7 +263,7 @@ var _ = Describe("MachineLink", func() { return aws.StringValue(t.Key) == v1alpha5.MachineManagedByAnnotationKey }) g.Expect(ok).To(BeTrue()) - g.Expect(aws.StringValue(tag.Value)).To(Equal(settings.FromContext(env.Context).ClusterName)) + g.Expect(aws.StringValue(tag.Value)).To(Equal(env.ClusterName)) tag, ok = lo.Find(instance.Tags, func(t *ec2.Tag) bool { return aws.StringValue(t.Key) == v1alpha5.ProvisionerNameLabelKey }) diff --git a/test/suites/machine/machine_test.go b/test/suites/machine/machine_test.go index f92d4ec81734..b86c5d51917f 100644 --- a/test/suites/machine/machine_test.go +++ b/test/suites/machine/machine_test.go @@ -31,7 +31,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" "github.com/aws/karpenter-core/pkg/utils/resources" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" ) @@ -40,8 +39,8 @@ var _ = Describe("StandaloneMachine", func() { var nodeTemplate *v1alpha1.AWSNodeTemplate BeforeEach(func() { nodeTemplate = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) }) It("should create a standard machine within the 'c' instance family", func() { @@ -273,8 +272,8 @@ var _ = Describe("StandaloneMachine", func() { // Create userData that adds custom labels through the --kubelet-extra-args nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyCustom nodeTemplate.Spec.AMISelector = map[string]string{"aws-ids": customAMI} - nodeTemplate.Spec.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), settings.FromContext(env.Context).ClusterName, - settings.FromContext(env.Context).ClusterEndpoint, env.ExpectCABundle())))) + nodeTemplate.Spec.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), env.ClusterName, + env.ClusterEndpoint, env.ExpectCABundle())))) machine := test.Machine(v1alpha5.Machine{ Spec: v1alpha5.MachineSpec{ diff --git a/test/suites/scale/deprovisioning_test.go b/test/suites/scale/deprovisioning_test.go index e4f818297530..1ae51e001155 100644 --- a/test/suites/scale/deprovisioning_test.go +++ b/test/suites/scale/deprovisioning_test.go @@ -32,7 +32,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/controllers/interruption/messages" "github.com/aws/karpenter/pkg/controllers/interruption/messages/scheduledchange" @@ -86,8 +85,8 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), "featureGates.driftEnabled": "true", }) nodeTemplate = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisionerOptions = test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{ diff --git a/test/suites/scale/provisioning_test.go b/test/suites/scale/provisioning_test.go index b59667ce00fd..0ebf0268815b 100644 --- a/test/suites/scale/provisioning_test.go +++ b/test/suites/scale/provisioning_test.go @@ -28,7 +28,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/test/pkg/debug" @@ -46,8 +45,8 @@ var _ = Describe("Provisioning", Label(debug.NoWatch), Label(debug.NoEvents), fu BeforeEach(func() { nodeTemplate = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner = test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{ diff --git a/test/suites/utilization/suite_test.go b/test/suites/utilization/suite_test.go index 9504da9fa598..9c09f49b66b4 100644 --- a/test/suites/utilization/suite_test.go +++ b/test/suites/utilization/suite_test.go @@ -25,7 +25,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/test/pkg/debug" @@ -53,8 +52,8 @@ var _ = AfterEach(func() { env.AfterEach() }) var _ = Describe("Utilization", Label(debug.NoWatch), Label(debug.NoEvents), func() { It("should provision one pod per node", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, Requirements: []v1.NodeSelectorRequirement{{ Key: v1.LabelInstanceTypeStable, From 450622bc9ddf9b75d090e14103c5872af9441d83 Mon Sep 17 00:00:00 2001 From: jmdeal Date: Wed, 18 Oct 2023 10:03:23 -0700 Subject: [PATCH 2/4] tests --- pkg/operator/options/options.go | 10 +- pkg/operator/options/options_validation.go | 8 - pkg/operator/options/suite_test.go | 178 +++++++++++++++++++-- 3 files changed, 165 insertions(+), 31 deletions(-) diff --git a/pkg/operator/options/options.go b/pkg/operator/options/options.go index 960fa080e4d6..2d836d798aa9 100644 --- a/pkg/operator/options/options.go +++ b/pkg/operator/options/options.go @@ -47,23 +47,22 @@ type Options struct { InterruptionQueueName string ReservedENIs int - isolatedVPC string setFlags map[string]bool } -func (o *Options) AddFlags(fs *flag.FlagSet) { +func (o *Options) AddFlags(fs *coreoptions.FlagSet) { fs.StringVar(&o.AssumeRoleARN, "assume-role-arn", env.WithDefaultString("ASSUME_ROLE_ARN", ""), "Role to assume for calling AWS services.") fs.DurationVar(&o.AssumeRoleDuration, "assume-role-duration", env.WithDefaultDuration("ASSUME_ROLE_DURATION", 15*time.Minute), "Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless aws.assumeRole set.") fs.StringVar(&o.ClusterCABundle, "cluster-ca-bundle", env.WithDefaultString("CLUSTER_CA_BUNDLE", ""), "Cluster CA bundle for nodes to use for TLS connections with the API server. If not set, this is taken from the controller's TLS configuration.") fs.StringVar(&o.ClusterName, "cluster-name", env.WithDefaultString("CLUSTER_NAME", ""), "[REQUIRED] The kubernetes cluster name for resource discovery.") fs.StringVar(&o.ClusterEndpoint, "cluster-endpoint", env.WithDefaultString("CLUSTER_ENDPOINT", ""), "The external kubernetes cluster endpoint for new nodes to connect with. If not specified, will discover the cluster endpoint using DescribeCluster API.") - fs.StringVar(&o.isolatedVPC, "isolated-vpc", env.WithDefaultString("ISOLATED_VPC", "false"), "If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS pricing endpoint.") + fs.BoolVarWithEnv(&o.IsolatedVPC, "isolated-vpc", "ISOLATED_VPC", false, "If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS pricing endpoint.") fs.Float64Var(&o.VMMemoryOverheadPercent, "vm-memory-overhead-percent", env.WithDefaultFloat64("VM_MEMORY_OVERHEAD_PERCENT", 0.075), "The VM memory overhead as a percent that will be subtracted from the total memory for all instance types.") fs.StringVar(&o.InterruptionQueueName, "interruption-queue-name", env.WithDefaultString("INTERRUPTION_QUEUE_NAME", ""), "Interruption queue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.") fs.IntVar(&o.ReservedENIs, "reserved-enis", env.WithDefaultInt("RESERVED_ENIS", 0), "Reserved ENIs are not included in the calculations for max-pods or kube-reserved. This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html.") } -func (o *Options) Parse(fs *flag.FlagSet, args ...string) error { +func (o *Options) Parse(fs *coreoptions.FlagSet, args ...string) error { if err := fs.Parse(args); err != nil { if errors.Is(err, flag.ErrHelp) { os.Exit(0) @@ -88,9 +87,6 @@ func (o *Options) Parse(fs *flag.FlagSet, args ...string) error { return fmt.Errorf("validating options, %w", err) } - // Parse options which can't be directly parsed by flag - o.IsolatedVPC = (o.isolatedVPC == "true") - return nil } diff --git a/pkg/operator/options/options_validation.go b/pkg/operator/options/options_validation.go index 976543d12d75..984058d8b8e2 100644 --- a/pkg/operator/options/options_validation.go +++ b/pkg/operator/options/options_validation.go @@ -28,7 +28,6 @@ func (o Options) Validate() error { o.validateVMMemoryOverheadPercent(), o.validateAssumeRoleDuration(), o.validateReservedENIs(), - o.validateIsolateVPC(), ) } @@ -39,13 +38,6 @@ func (o Options) validateAssumeRoleDuration() error { return nil } -func (o Options) validateIsolateVPC() error { - if o.isolatedVPC != "true" && o.isolatedVPC != "false" { - return fmt.Errorf("%q is not a valid value for isolated-vpc, options are true or false", o.isolatedVPC) - } - return nil -} - func (o Options) validateEndpoint() error { if o.ClusterEndpoint == "" { return nil diff --git a/pkg/operator/options/suite_test.go b/pkg/operator/options/suite_test.go index b69131348cc0..267f77f0cbd2 100644 --- a/pkg/operator/options/suite_test.go +++ b/pkg/operator/options/suite_test.go @@ -18,14 +18,19 @@ import ( "context" "flag" "fmt" + "os" "testing" "time" - "github.com/aws/karpenter/pkg/apis/settings" - "github.com/aws/karpenter/pkg/operator/options" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" . "knative.dev/pkg/logging/testing" + + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" + "github.com/aws/karpenter/pkg/apis/settings" + "github.com/aws/karpenter/pkg/operator/options" + "github.com/aws/karpenter/pkg/test" ) var ctx context.Context @@ -37,10 +42,34 @@ func TestAPIs(t *testing.T) { } var _ = Describe("Options", func() { - var fs *flag.FlagSet + var envState map[string]string + var environmentVariables = []string{ + "ASSUME_ROLE_ARN", + "ASSUME_ROLE_DURATION", + "CLUSTER_CA_BUNDLE", + "CLUSTER_NAME", + "CLUSTER_ENDPOINT", + "ISOLATED_VPC", + "VM_MOMORY_OVERHEAD_PERCENT", + "INTERRUPTION_QUEUE_NAME", + "RESERVED_ENIS", + } + + var fs *coreoptions.FlagSet var opts *options.Options + BeforeEach(func() { - fs = flag.NewFlagSet("karpenter", flag.ContinueOnError) + envState = map[string]string{} + for _, ev := range environmentVariables { + val, ok := os.LookupEnv(ev) + if ok { + envState[ev] = val + } + os.Unsetenv(ev) + } + fs = &coreoptions.FlagSet{ + FlagSet: flag.NewFlagSet("karpenter", flag.ContinueOnError), + } opts = &options.Options{} opts.AddFlags(fs) @@ -50,15 +79,90 @@ var _ = Describe("Options", func() { Expect(err).To(BeNil()) }) + AfterEach(func() { + for _, ev := range environmentVariables { + os.Unsetenv(ev) + } + for ev, val := range envState { + os.Setenv(ev, val) + } + }) + Context("Merging", func() { - It("shouldn't overwrite cli flags / env vars", func() { + It("shouldn't overwrite options when all are set", func() { + err := opts.Parse( + fs, + "--assume-role-arn", "options-cluster-role", + "--assume-role-duration", "20m", + "--cluster-ca-bundle", "options-bundle", + "--cluster-name", "options-cluster", + "--cluster-endpoint", "https://options-cluster", + "--isolated-vpc", + "--vm-memory-overhead-percent", "0.1", + "--interruption-queue-name", "options-cluster", + "--reserved-enis", "10", + ) + Expect(err).ToNot(HaveOccurred()) + ctx = settings.ToContext(ctx, &settings.Settings{ + AssumeRoleARN: "settings-cluster-role", + AssumeRoleDuration: time.Minute * 22, + ClusterCABundle: "settings-bundle", + ClusterName: "settings-cluster", + ClusterEndpoint: "https://settings-cluster", + IsolatedVPC: true, + VMMemoryOverheadPercent: 0.05, + InterruptionQueueName: "settings-cluster", + ReservedENIs: 8, + }) + opts.MergeSettings(ctx) + expectOptionsEqual(opts, test.Options(test.OptionsFields{ + AssumeRoleARN: lo.ToPtr("options-cluster-role"), + AssumeRoleDuration: lo.ToPtr(20 * time.Minute), + ClusterCABundle: lo.ToPtr("options-bundle"), + ClusterName: lo.ToPtr("options-cluster"), + ClusterEndpoint: lo.ToPtr("https://options-cluster"), + IsolatedVPC: lo.ToPtr(true), + VMMemoryOverheadPercent: lo.ToPtr[float64](0.1), + InterruptionQueueName: lo.ToPtr("options-cluster"), + ReservedENIs: lo.ToPtr(10), + })) + + }) + It("should overwrite options when none are set", func() { + err := opts.Parse(fs) + Expect(err).ToNot(HaveOccurred()) + ctx = settings.ToContext(ctx, &settings.Settings{ + AssumeRoleARN: "settings-cluster-role", + AssumeRoleDuration: time.Minute * 22, + ClusterCABundle: "settings-bundle", + ClusterName: "settings-cluster", + ClusterEndpoint: "https://settings-cluster", + IsolatedVPC: true, + VMMemoryOverheadPercent: 0.05, + InterruptionQueueName: "settings-cluster", + ReservedENIs: 8, + }) + opts.MergeSettings(ctx) + expectOptionsEqual(opts, test.Options(test.OptionsFields{ + AssumeRoleARN: lo.ToPtr("settings-cluster-role"), + AssumeRoleDuration: lo.ToPtr(22 * time.Minute), + ClusterCABundle: lo.ToPtr("settings-bundle"), + ClusterName: lo.ToPtr("settings-cluster"), + ClusterEndpoint: lo.ToPtr("https://settings-cluster"), + IsolatedVPC: lo.ToPtr(true), + VMMemoryOverheadPercent: lo.ToPtr[float64](0.05), + InterruptionQueueName: lo.ToPtr("settings-cluster"), + ReservedENIs: lo.ToPtr(8), + })) + + }) + It("should correctly merge options and settings when mixed", func() { err := opts.Parse( fs, "--assume-role-arn", "options-cluster-role", "--cluster-ca-bundle", "options-bundle", "--cluster-name", "options-cluster", "--cluster-endpoint", "https://options-cluster", - "--isolated-vpc", "false", "--interruption-queue-name", "options-cluster", ) Expect(err).ToNot(HaveOccurred()) @@ -74,17 +178,46 @@ var _ = Describe("Options", func() { ReservedENIs: 10, }) opts.MergeSettings(ctx) - Expect(opts.AssumeRoleARN).To(Equal("options-cluster-role")) - Expect(opts.AssumeRoleDuration).To(Equal(time.Minute * 20)) - Expect(opts.ClusterCABundle).To(Equal("options-bundle")) - Expect(opts.ClusterName).To(Equal("options-cluster")) - Expect(opts.ClusterEndpoint).To(Equal("https://options-cluster")) - Expect(opts.IsolatedVPC).To(BeFalse()) - Expect(opts.VMMemoryOverheadPercent).To(Equal(0.1)) - Expect(opts.InterruptionQueueName).To(Equal("options-cluster")) - Expect(opts.ReservedENIs).To(Equal(10)) + expectOptionsEqual(opts, test.Options(test.OptionsFields{ + AssumeRoleARN: lo.ToPtr("options-cluster-role"), + AssumeRoleDuration: lo.ToPtr(20 * time.Minute), + ClusterCABundle: lo.ToPtr("options-bundle"), + ClusterName: lo.ToPtr("options-cluster"), + ClusterEndpoint: lo.ToPtr("https://options-cluster"), + IsolatedVPC: lo.ToPtr(true), + VMMemoryOverheadPercent: lo.ToPtr[float64](0.1), + InterruptionQueueName: lo.ToPtr("options-cluster"), + ReservedENIs: lo.ToPtr(10), + })) }) + It("should correctly fallback to env vars when CLI flags aren't set", func() { + os.Setenv("ASSUME_ROLE_ARN", "env-role") + os.Setenv("ASSUME_ROLE_DURATION", "20m") + os.Setenv("CLUSTER_CA_BUNDLE", "env-bundle") + os.Setenv("CLUSTER_NAME", "env-cluster") + os.Setenv("CLUSTER_ENDPOINT", "https://env-cluster") + os.Setenv("ISOLATED_VPC", "true") + os.Setenv("VM_MEMORY_OVERHEAD_PERCENT", "0.1") + os.Setenv("INTERRUPTION_QUEUE_NAME", "env-cluster") + os.Setenv("RESERVED_ENIS", "10") + fs = &coreoptions.FlagSet{ + FlagSet: flag.NewFlagSet("karpenter", flag.ContinueOnError), + } + opts.AddFlags(fs) + opts.Parse(fs) + expectOptionsEqual(opts, test.Options(test.OptionsFields{ + AssumeRoleARN: lo.ToPtr("env-role"), + AssumeRoleDuration: lo.ToPtr(20 * time.Minute), + ClusterCABundle: lo.ToPtr("env-bundle"), + ClusterName: lo.ToPtr("env-cluster"), + ClusterEndpoint: lo.ToPtr("https://env-cluster"), + IsolatedVPC: lo.ToPtr(true), + VMMemoryOverheadPercent: lo.ToPtr[float64](0.1), + InterruptionQueueName: lo.ToPtr("env-cluster"), + ReservedENIs: lo.ToPtr(10), + })) + }) }) Context("Validation", func() { @@ -96,7 +229,7 @@ var _ = Describe("Options", func() { Expect(func() { opts.MergeSettings(ctx) fmt.Printf("%#v", opts) - }).To(Panic()) + }).To(Panic()) }) It("should fail when assume role duration is less than 15 minutes", func() { err := opts.Parse(fs, "--assume-role-duration", "1s") @@ -116,3 +249,16 @@ var _ = Describe("Options", func() { }) }) }) + +func expectOptionsEqual(optsA *options.Options, optsB *options.Options) { + GinkgoHelper() + Expect(optsA.AssumeRoleARN).To(Equal(optsB.AssumeRoleARN)) + Expect(optsA.AssumeRoleDuration).To(Equal(optsB.AssumeRoleDuration)) + Expect(optsA.ClusterCABundle).To(Equal(optsB.ClusterCABundle)) + Expect(optsA.ClusterName).To(Equal(optsB.ClusterName)) + Expect(optsA.ClusterEndpoint).To(Equal(optsB.ClusterEndpoint)) + Expect(optsA.IsolatedVPC).To(Equal(optsB.IsolatedVPC)) + Expect(optsA.VMMemoryOverheadPercent).To(Equal(optsB.VMMemoryOverheadPercent)) + Expect(optsA.InterruptionQueueName).To(Equal(optsB.InterruptionQueueName)) + Expect(optsA.ReservedENIs).To(Equal(optsB.ReservedENIs)) +} From 189b369be9515f76a5ae11c2704ba441648652fa Mon Sep 17 00:00:00 2001 From: Jason Deal Date: Thu, 19 Oct 2023 10:04:48 -0700 Subject: [PATCH 3/4] rebase fixes --- hack/docs/configuration_gen_docs.go | 4 +++- pkg/controllers/controllers.go | 2 +- pkg/controllers/interruption/controller.go | 4 ++-- pkg/controllers/interruption/sqs.go | 6 +++--- pkg/controllers/interruption/suite_test.go | 2 +- pkg/operator/options/options.go | 8 ++++---- pkg/operator/options/suite_test.go | 23 +++++++++++----------- pkg/providers/instance/suite_test.go | 4 ++-- pkg/test/options.go | 4 ++-- test/pkg/environment/aws/environment.go | 12 +++++------ test/suites/interruption/suite_test.go | 2 +- 11 files changed, 37 insertions(+), 34 deletions(-) diff --git a/hack/docs/configuration_gen_docs.go b/hack/docs/configuration_gen_docs.go index 2624a41f7e20..828369d0e405 100644 --- a/hack/docs/configuration_gen_docs.go +++ b/hack/docs/configuration_gen_docs.go @@ -49,7 +49,9 @@ func main() { topDoc := fmt.Sprintf("%s%s\n\n", startDocSections[0], genStart) bottomDoc := fmt.Sprintf("\n%s%s", genEnd, endDocSections[1]) - fs := &options.FlagSet{FlagSet: flag.NewFlagSet("karpenter", flag.ContinueOnError)} + fs := &coreoptions.FlagSet { + FlagSet: flag.NewFlagSet("karpenter", flag.ContinueOnError), + } (&coreoptions.Options{}).AddFlags(fs) (&options.Options{}).AddFlags(fs) diff --git a/pkg/controllers/controllers.go b/pkg/controllers/controllers.go index 6ac84393cf09..33088cf74079 100644 --- a/pkg/controllers/controllers.go +++ b/pkg/controllers/controllers.go @@ -54,7 +54,7 @@ func NewControllers(ctx context.Context, sess *session.Session, clk clock.Clock, nodeclaimgarbagecollection.NewController(kubeClient, cloudProvider, linkController), nodeclaimtagging.NewController(kubeClient, instanceProvider), } - if options.FromContext(ctx).InterruptionQueueName != "" { + if options.FromContext(ctx).InterruptionQueue != "" { controllers = append(controllers, interruption.NewController(kubeClient, clk, recorder, interruption.NewSQSProvider(sqs.New(sess)), unavailableOfferings)) } if options.FromContext(ctx).IsolatedVPC { diff --git a/pkg/controllers/interruption/controller.go b/pkg/controllers/interruption/controller.go index e5d4651e610b..f6d594764755 100644 --- a/pkg/controllers/interruption/controller.go +++ b/pkg/controllers/interruption/controller.go @@ -80,8 +80,8 @@ func NewController(kubeClient client.Client, clk clock.Clock, recorder events.Re } func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { - ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("queue", options.FromContext(ctx).InterruptionQueueName)) - if c.cm.HasChanged(options.FromContext(ctx).InterruptionQueueName, nil) { + ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("queue", options.FromContext(ctx).InterruptionQueue)) + if c.cm.HasChanged(options.FromContext(ctx).InterruptionQueue, nil) { logging.FromContext(ctx).Debugf("watching interruption queue") } sqsMessages, err := c.sqsProvider.GetSQSMessages(ctx) diff --git a/pkg/controllers/interruption/sqs.go b/pkg/controllers/interruption/sqs.go index 6cf518955820..7090da489a13 100644 --- a/pkg/controllers/interruption/sqs.go +++ b/pkg/controllers/interruption/sqs.go @@ -43,7 +43,7 @@ func NewSQSProvider(client sqsiface.SQSAPI) *SQSProvider { } provider.queueURL.Resolve = func(ctx context.Context) (string, error) { input := &sqs.GetQueueUrlInput{ - QueueName: aws.String(options.FromContext(ctx).InterruptionQueueName), + QueueName: aws.String(options.FromContext(ctx).InterruptionQueue), } ret, err := provider.client.GetQueueUrlWithContext(ctx, input) if err != nil { @@ -66,12 +66,12 @@ func (s *SQSProvider) QueueExists(ctx context.Context) (bool, error) { } func (s *SQSProvider) DiscoverQueueURL(ctx context.Context) (string, error) { - if options.FromContext(ctx).InterruptionQueueName != lo.FromPtr(s.queueName.Load()) { + if options.FromContext(ctx).InterruptionQueue != lo.FromPtr(s.queueName.Load()) { res, err := s.queueURL.TryGet(ctx, atomic.IgnoreCacheOption) if err != nil { return res, err } - s.queueName.Store(lo.ToPtr(options.FromContext(ctx).InterruptionQueueName)) + s.queueName.Store(lo.ToPtr(options.FromContext(ctx).InterruptionQueue)) return res, nil } return s.queueURL.TryGet(ctx) diff --git a/pkg/controllers/interruption/suite_test.go b/pkg/controllers/interruption/suite_test.go index 8cb557e1efbf..5b2bd879190a 100644 --- a/pkg/controllers/interruption/suite_test.go +++ b/pkg/controllers/interruption/suite_test.go @@ -94,7 +94,7 @@ var _ = AfterSuite(func() { var _ = BeforeEach(func() { ctx = coreoptions.ToContext(ctx, coretest.Options()) ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ - InterruptionQueueName: lo.ToPtr("test-cluster"), + InterruptionQueue: lo.ToPtr("test-cluster"), })) ctx = settings.ToContext(ctx, test.Settings()) unavailableOfferingsCache.Flush() diff --git a/pkg/operator/options/options.go b/pkg/operator/options/options.go index 2d836d798aa9..9134b944662d 100644 --- a/pkg/operator/options/options.go +++ b/pkg/operator/options/options.go @@ -44,10 +44,10 @@ type Options struct { ClusterEndpoint string IsolatedVPC bool VMMemoryOverheadPercent float64 - InterruptionQueueName string + InterruptionQueue string ReservedENIs int - setFlags map[string]bool + setFlags map[string]bool } func (o *Options) AddFlags(fs *coreoptions.FlagSet) { @@ -58,7 +58,7 @@ func (o *Options) AddFlags(fs *coreoptions.FlagSet) { fs.StringVar(&o.ClusterEndpoint, "cluster-endpoint", env.WithDefaultString("CLUSTER_ENDPOINT", ""), "The external kubernetes cluster endpoint for new nodes to connect with. If not specified, will discover the cluster endpoint using DescribeCluster API.") fs.BoolVarWithEnv(&o.IsolatedVPC, "isolated-vpc", "ISOLATED_VPC", false, "If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS pricing endpoint.") fs.Float64Var(&o.VMMemoryOverheadPercent, "vm-memory-overhead-percent", env.WithDefaultFloat64("VM_MEMORY_OVERHEAD_PERCENT", 0.075), "The VM memory overhead as a percent that will be subtracted from the total memory for all instance types.") - fs.StringVar(&o.InterruptionQueueName, "interruption-queue-name", env.WithDefaultString("INTERRUPTION_QUEUE_NAME", ""), "Interruption queue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.") + fs.StringVar(&o.InterruptionQueue, "interruption-queue", env.WithDefaultString("INTERRUPTION_QUEUE", ""), "Interruption queue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.") fs.IntVar(&o.ReservedENIs, "reserved-enis", env.WithDefaultInt("RESERVED_ENIS", 0), "Reserved ENIs are not included in the calculations for max-pods or kube-reserved. This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html.") } @@ -103,7 +103,7 @@ func (o *Options) MergeSettings(ctx context.Context) { mergeField(&o.ClusterEndpoint, s.ClusterEndpoint, o.setFlags["cluster-endpoint"]) mergeField(&o.IsolatedVPC, s.IsolatedVPC, o.setFlags["isolated-vpc"]) mergeField(&o.VMMemoryOverheadPercent, s.VMMemoryOverheadPercent, o.setFlags["vm-memory-overhead-percent"]) - mergeField(&o.InterruptionQueueName, s.InterruptionQueueName, o.setFlags["interruption-queue-name"]) + mergeField(&o.InterruptionQueue, s.InterruptionQueueName, o.setFlags["interruption-queue"]) mergeField(&o.ReservedENIs, s.ReservedENIs, o.setFlags["reserved-enis"]) if err := o.validateRequiredFields(); err != nil { panic(fmt.Errorf("checking required fields, %w", err)) diff --git a/pkg/operator/options/suite_test.go b/pkg/operator/options/suite_test.go index 267f77f0cbd2..6703ee13598c 100644 --- a/pkg/operator/options/suite_test.go +++ b/pkg/operator/options/suite_test.go @@ -50,8 +50,8 @@ var _ = Describe("Options", func() { "CLUSTER_NAME", "CLUSTER_ENDPOINT", "ISOLATED_VPC", - "VM_MOMORY_OVERHEAD_PERCENT", - "INTERRUPTION_QUEUE_NAME", + "VM_MEMORY_OVERHEAD_PERCENT", + "INTERRUPTION_QUEUE", "RESERVED_ENIS", } @@ -99,7 +99,7 @@ var _ = Describe("Options", func() { "--cluster-endpoint", "https://options-cluster", "--isolated-vpc", "--vm-memory-overhead-percent", "0.1", - "--interruption-queue-name", "options-cluster", + "--interruption-queue", "options-cluster", "--reserved-enis", "10", ) Expect(err).ToNot(HaveOccurred()) @@ -123,7 +123,7 @@ var _ = Describe("Options", func() { ClusterEndpoint: lo.ToPtr("https://options-cluster"), IsolatedVPC: lo.ToPtr(true), VMMemoryOverheadPercent: lo.ToPtr[float64](0.1), - InterruptionQueueName: lo.ToPtr("options-cluster"), + InterruptionQueue: lo.ToPtr("options-cluster"), ReservedENIs: lo.ToPtr(10), })) @@ -151,7 +151,7 @@ var _ = Describe("Options", func() { ClusterEndpoint: lo.ToPtr("https://settings-cluster"), IsolatedVPC: lo.ToPtr(true), VMMemoryOverheadPercent: lo.ToPtr[float64](0.05), - InterruptionQueueName: lo.ToPtr("settings-cluster"), + InterruptionQueue: lo.ToPtr("settings-cluster"), ReservedENIs: lo.ToPtr(8), })) @@ -163,7 +163,7 @@ var _ = Describe("Options", func() { "--cluster-ca-bundle", "options-bundle", "--cluster-name", "options-cluster", "--cluster-endpoint", "https://options-cluster", - "--interruption-queue-name", "options-cluster", + "--interruption-queue", "options-cluster", ) Expect(err).ToNot(HaveOccurred()) ctx = settings.ToContext(ctx, &settings.Settings{ @@ -186,7 +186,7 @@ var _ = Describe("Options", func() { ClusterEndpoint: lo.ToPtr("https://options-cluster"), IsolatedVPC: lo.ToPtr(true), VMMemoryOverheadPercent: lo.ToPtr[float64](0.1), - InterruptionQueueName: lo.ToPtr("options-cluster"), + InterruptionQueue: lo.ToPtr("options-cluster"), ReservedENIs: lo.ToPtr(10), })) }) @@ -199,13 +199,14 @@ var _ = Describe("Options", func() { os.Setenv("CLUSTER_ENDPOINT", "https://env-cluster") os.Setenv("ISOLATED_VPC", "true") os.Setenv("VM_MEMORY_OVERHEAD_PERCENT", "0.1") - os.Setenv("INTERRUPTION_QUEUE_NAME", "env-cluster") + os.Setenv("INTERRUPTION_QUEUE", "env-cluster") os.Setenv("RESERVED_ENIS", "10") fs = &coreoptions.FlagSet{ FlagSet: flag.NewFlagSet("karpenter", flag.ContinueOnError), } opts.AddFlags(fs) - opts.Parse(fs) + err := opts.Parse(fs) + Expect(err).ToNot(HaveOccurred()) expectOptionsEqual(opts, test.Options(test.OptionsFields{ AssumeRoleARN: lo.ToPtr("env-role"), AssumeRoleDuration: lo.ToPtr(20 * time.Minute), @@ -214,7 +215,7 @@ var _ = Describe("Options", func() { ClusterEndpoint: lo.ToPtr("https://env-cluster"), IsolatedVPC: lo.ToPtr(true), VMMemoryOverheadPercent: lo.ToPtr[float64](0.1), - InterruptionQueueName: lo.ToPtr("env-cluster"), + InterruptionQueue: lo.ToPtr("env-cluster"), ReservedENIs: lo.ToPtr(10), })) }) @@ -259,6 +260,6 @@ func expectOptionsEqual(optsA *options.Options, optsB *options.Options) { Expect(optsA.ClusterEndpoint).To(Equal(optsB.ClusterEndpoint)) Expect(optsA.IsolatedVPC).To(Equal(optsB.IsolatedVPC)) Expect(optsA.VMMemoryOverheadPercent).To(Equal(optsB.VMMemoryOverheadPercent)) - Expect(optsA.InterruptionQueueName).To(Equal(optsB.InterruptionQueueName)) + Expect(optsA.InterruptionQueue).To(Equal(optsB.InterruptionQueue)) Expect(optsA.ReservedENIs).To(Equal(optsB.ReservedENIs)) } diff --git a/pkg/providers/instance/suite_test.go b/pkg/providers/instance/suite_test.go index 45e163c0a7c2..44d155b95004 100644 --- a/pkg/providers/instance/suite_test.go +++ b/pkg/providers/instance/suite_test.go @@ -70,8 +70,8 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = options.ToContext(ctx, opts) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) awsEnv.Reset() }) diff --git a/pkg/test/options.go b/pkg/test/options.go index 378b8e2dbc5a..609c118bcb2a 100644 --- a/pkg/test/options.go +++ b/pkg/test/options.go @@ -32,7 +32,7 @@ type OptionsFields struct { ClusterEndpoint *string IsolatedVPC *bool VMMemoryOverheadPercent *float64 - InterruptionQueueName *string + InterruptionQueue *string ReservedENIs *int } @@ -51,7 +51,7 @@ func Options(overrides ...OptionsFields) *options.Options { ClusterEndpoint: lo.FromPtrOr(opts.ClusterEndpoint, "https://test-cluster"), IsolatedVPC: lo.FromPtrOr(opts.IsolatedVPC, false), VMMemoryOverheadPercent: lo.FromPtrOr(opts.VMMemoryOverheadPercent, 0.075), - InterruptionQueueName: lo.FromPtrOr(opts.InterruptionQueueName, ""), + InterruptionQueue: lo.FromPtrOr(opts.InterruptionQueue, ""), ReservedENIs: lo.FromPtrOr(opts.ReservedENIs, 0), } } diff --git a/test/pkg/environment/aws/environment.go b/test/pkg/environment/aws/environment.go index 7d4ab57064ce..9fc111b9759c 100644 --- a/test/pkg/environment/aws/environment.go +++ b/test/pkg/environment/aws/environment.go @@ -56,9 +56,9 @@ type Environment struct { SQSProvider *interruption.SQSProvider - ClusterName string - ClusterEndpoint string - InterruptionQueueName string + ClusterName string + ClusterEndpoint string + InterruptionQueue string } func NewEnvironment(t *testing.T) *Environment { @@ -86,9 +86,9 @@ func NewEnvironment(t *testing.T) *Environment { SQSProvider: interruption.NewSQSProvider(sqs.New(session)), TimeStreamAPI: GetTimeStreamAPI(session), - ClusterName: lo.Must(os.LookupEnv("CLUSTER_NAME")), - ClusterEndpoint: lo.Must(os.LookupEnv("CLUSTER_ENDPOINT")), - InterruptionQueueName: lo.Must(os.LookupEnv("INTERRUPTION_QUEUE_NAME")), + ClusterName: lo.Must(os.LookupEnv("CLUSTER_NAME")), + ClusterEndpoint: lo.Must(os.LookupEnv("CLUSTER_ENDPOINT")), + InterruptionQueue: lo.Must(os.LookupEnv("INTERRUPTION_QUEUE")), } } diff --git a/test/suites/interruption/suite_test.go b/test/suites/interruption/suite_test.go index f370ba99d16d..0e1fc563efd5 100644 --- a/test/suites/interruption/suite_test.go +++ b/test/suites/interruption/suite_test.go @@ -55,7 +55,7 @@ func TestInterruption(t *testing.T) { var _ = BeforeEach(func() { env.Context = options.ToContext(env.Context, awstest.Options(awstest.OptionsFields{ - InterruptionQueueName: lo.ToPtr(env.InterruptionQueueName), + InterruptionQueue: lo.ToPtr(env.InterruptionQueue), })) env.BeforeEach() env.ExpectQueueExists() From 006f7f5c6ae32ed8eab46c5abfd48e3885a6c7f1 Mon Sep 17 00:00:00 2001 From: Jason Deal Date: Thu, 19 Oct 2023 12:10:38 -0700 Subject: [PATCH 4/4] gh comment --- pkg/providers/launchtemplate/nodeclass_test.go | 5 +---- pkg/providers/launchtemplate/nodetemplate_test.go | 4 +--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/pkg/providers/launchtemplate/nodeclass_test.go b/pkg/providers/launchtemplate/nodeclass_test.go index 09f0f2f04bb6..86a5d9941caf 100644 --- a/pkg/providers/launchtemplate/nodeclass_test.go +++ b/pkg/providers/launchtemplate/nodeclass_test.go @@ -747,10 +747,7 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() { }) It("should calculate memory overhead based on eni limited pods when ENI limited", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ - EnableENILimitedPodDensity: lo.ToPtr(true), - })) - + ctx = settings.ToContext(ctx, test.Settings()) ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) diff --git a/pkg/providers/launchtemplate/nodetemplate_test.go b/pkg/providers/launchtemplate/nodetemplate_test.go index 0be09622f2ab..f5ba265aaee1 100644 --- a/pkg/providers/launchtemplate/nodetemplate_test.go +++ b/pkg/providers/launchtemplate/nodetemplate_test.go @@ -762,9 +762,7 @@ var _ = Describe("NodeTemplate/LaunchTemplates", func() { }) It("should calculate memory overhead based on eni limited pods when ENI limited", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ - EnableENILimitedPodDensity: lo.ToPtr(true), - })) + ctx = settings.ToContext(ctx, test.Settings()) ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ VMMemoryOverheadPercent: lo.ToPtr[float64](0),