From 98effff473d935e403b750639ac820b9e26e7719 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Mon, 8 Apr 2024 23:04:40 -0700 Subject: [PATCH] Add global config parsing instead of context injection --- cmd/controller/main.go | 2 + go.mod | 2 + go.sum | 4 +- hack/code/prices_gen/main.go | 5 +- hack/docs/configuration_gen_docs.go | 10 +- hack/docs/instancetypes_gen_docs.go | 21 +-- pkg/cloudprovider/suite_test.go | 12 +- pkg/controllers/controllers.go | 6 +- .../interruption_benchmark_test.go | 18 +-- pkg/controllers/interruption/suite_test.go | 2 - .../nodeclaim/garbagecollection/suite_test.go | 21 ++- .../nodeclaim/tagging/suite_test.go | 14 +- pkg/controllers/nodeclass/suite_test.go | 14 +- pkg/controllers/pricing/suite_test.go | 21 +-- pkg/fake/utils.go | 5 +- pkg/global/config.go | 75 +++++++++ .../config_validation.go} | 38 ++--- pkg/global/suite_test.go | 128 +++++++++++++++ pkg/operator/operator.go | 27 ++-- pkg/operator/options/options.go | 86 ---------- pkg/operator/options/suite_test.go | 149 ------------------ pkg/operator/suite_test.go | 29 ++-- pkg/providers/amifamily/suite_test.go | 4 - pkg/providers/instance/instance.go | 16 +- pkg/providers/instance/suite_test.go | 11 +- .../instanceprofile/instanceprofile.go | 18 +-- pkg/providers/instancetype/suite_test.go | 27 +--- pkg/providers/instancetype/types.go | 12 +- .../launchtemplate/launchtemplate.go | 10 +- pkg/providers/launchtemplate/suite_test.go | 24 +-- pkg/providers/pricing/pricing.go | 8 +- pkg/providers/securitygroup/suite_test.go | 8 +- pkg/providers/subnet/suite_test.go | 8 +- pkg/test/environment.go | 8 +- pkg/test/options.go | 57 ------- test/suites/interruption/suite_test.go | 6 - tools/allocatable-diff/main.go | 3 +- 37 files changed, 358 insertions(+), 551 deletions(-) create mode 100644 pkg/global/config.go rename pkg/{operator/options/options_validation.go => global/config_validation.go} (61%) create mode 100644 pkg/global/suite_test.go delete mode 100644 pkg/operator/options/options.go delete mode 100644 pkg/operator/options/suite_test.go delete mode 100644 pkg/test/options.go diff --git a/cmd/controller/main.go b/cmd/controller/main.go index b1871bd91f1e..fb15306e92c0 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -19,6 +19,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" "github.com/aws/karpenter-provider-aws/pkg/controllers" + "github.com/aws/karpenter-provider-aws/pkg/global" "github.com/aws/karpenter-provider-aws/pkg/operator" "github.com/aws/karpenter-provider-aws/pkg/webhooks" @@ -30,6 +31,7 @@ import ( ) func main() { + lo.Must0(global.Initialize()) ctx, op := operator.NewOperator(coreoperator.NewOperator()) awsCloudProvider := cloudprovider.New( op.InstanceTypesProvider, diff --git a/go.mod b/go.mod index 8d4d7b805121..88e30b3335da 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,8 @@ require ( sigs.k8s.io/yaml v1.4.0 ) +replace sigs.k8s.io/karpenter => github.com/jonathan-innis/karpenter v0.0.4-0.20240409062722-d59d53db306d + require ( contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d // indirect contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect diff --git a/go.sum b/go.sum index 7d13c780cc6d..ff89af277ab9 100644 --- a/go.sum +++ b/go.sum @@ -220,6 +220,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jonathan-innis/karpenter v0.0.4-0.20240409062722-d59d53db306d h1:iHQZtrsr3PxEdOHReLaMmH4rFyjzNweuHx+3ksCAdTc= +github.com/jonathan-innis/karpenter v0.0.4-0.20240409062722-d59d53db306d/go.mod h1:fieFojxOec/l0tDmFT7R+g/Y+SGQbL9VlcYO8xb3sLo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -759,8 +761,6 @@ sigs.k8s.io/controller-runtime v0.17.2 h1:FwHwD1CTUemg0pW2otk7/U5/i5m2ymzvOXdbeG sigs.k8s.io/controller-runtime v0.17.2/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/karpenter v0.35.1-0.20240404200702-545d88a836d4 h1:eb3ubw9tFiV62Gq1Ci5r5BCJ8OlKu2Mqf7h3tJIRe0U= -sigs.k8s.io/karpenter v0.35.1-0.20240404200702-545d88a836d4/go.mod h1:fieFojxOec/l0tDmFT7R+g/Y+SGQbL9VlcYO8xb3sLo= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= diff --git a/hack/code/prices_gen/main.go b/hack/code/prices_gen/main.go index 7b73fdfc8564..cd466eda20d9 100644 --- a/hack/code/prices_gen/main.go +++ b/hack/code/prices_gen/main.go @@ -35,9 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" controllerspricing "github.com/aws/karpenter-provider-aws/pkg/controllers/pricing" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/pricing" - "github.com/aws/karpenter-provider-aws/pkg/test" ) func getAWSRegions(partition string) []string { @@ -94,7 +92,6 @@ func main() { os.Setenv("AWS_SDK_LOAD_CONFIG", "true") os.Setenv("AWS_REGION", region) ctx := context.Background() - ctx = options.ToContext(ctx, test.Options()) sess := session.Must(session.NewSession()) ec2 := ec22.New(sess) src := &bytes.Buffer{} @@ -108,7 +105,7 @@ func main() { // record prices for each region we are interested in for _, region := range getAWSRegions(opts.partition) { log.Println("fetching for", region) - pricingProvider := pricing.NewDefaultProvider(ctx, pricing.NewAPI(sess, region), ec2, region) + pricingProvider := pricing.NewDefaultProvider(pricing.NewAPI(sess, region), ec2, region) controller := controllerspricing.NewController(pricingProvider) _, err := controller.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{}}) if err != nil { diff --git a/hack/docs/configuration_gen_docs.go b/hack/docs/configuration_gen_docs.go index 7b1616c1a123..bc9ce8ef0962 100644 --- a/hack/docs/configuration_gen_docs.go +++ b/hack/docs/configuration_gen_docs.go @@ -21,9 +21,9 @@ import ( "os" "strings" - coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" + coreglobal "sigs.k8s.io/karpenter/pkg/global" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" + "github.com/aws/karpenter-provider-aws/pkg/global" ) func main() { @@ -50,11 +50,11 @@ func main() { topDoc := fmt.Sprintf("%s%s\n\n", startDocSections[0], genStart) bottomDoc := fmt.Sprintf("\n%s%s", genEnd, endDocSections[1]) - fs := &coreoptions.FlagSet{ + fs := &coreglobal.FlagSet{ FlagSet: flag.NewFlagSet("karpenter", flag.ContinueOnError), } - (&coreoptions.Options{}).AddFlags(fs) - (&options.Options{}).AddFlags(fs) + coreglobal.AddFlags(fs) + global.AddFlags(fs) envVarsBlock := "| Environment Variable | CLI Flag | Description |\n" envVarsBlock += "|--|--|--|\n" diff --git a/hack/docs/instancetypes_gen_docs.go b/hack/docs/instancetypes_gen_docs.go index 68736fb5e823..1a9f57842637 100644 --- a/hack/docs/instancetypes_gen_docs.go +++ b/hack/docs/instancetypes_gen_docs.go @@ -34,17 +34,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "sigs.k8s.io/karpenter/pkg/cloudprovider" coreoperator "sigs.k8s.io/karpenter/pkg/operator" - coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" - coretest "sigs.k8s.io/karpenter/pkg/test" + "sigs.k8s.io/karpenter/pkg/utils/resources" awscloudprovider "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" + "github.com/aws/karpenter-provider-aws/pkg/global" "github.com/aws/karpenter-provider-aws/pkg/operator" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" - "github.com/aws/karpenter-provider-aws/pkg/test" - - "sigs.k8s.io/karpenter/pkg/cloudprovider" - "sigs.k8s.io/karpenter/pkg/utils/resources" ) // FakeManager is a manager that takes all the utilized calls from the operator setup @@ -84,14 +80,11 @@ func main() { lo.Must0(os.Setenv("AWS_SDK_LOAD_CONFIG", "true")) lo.Must0(os.Setenv("AWS_REGION", "us-east-1")) - ctx := coreoptions.ToContext(context.Background(), coretest.Options()) - ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ - ClusterName: lo.ToPtr("docs-gen"), - ClusterEndpoint: lo.ToPtr("https://docs-gen.aws"), - IsolatedVPC: lo.ToPtr(true), // disable pricing lookup - })) + global.Initialize("--cluster-name", "docs-gen", + "--cluster-endpoint", "https://docs-gen.aws", + "--isolated-vpc") - ctx, op := operator.NewOperator(ctx, &coreoperator.Operator{ + ctx, op := operator.NewOperator(context.Background(), &coreoperator.Operator{ Manager: &FakeManager{}, KubernetesInterface: kubernetes.NewForConfigOrDie(&rest.Config{}), }) diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index 4e980174a4ba..ddbcd2fc8379 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -40,7 +40,6 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" "github.com/aws/karpenter-provider-aws/pkg/fake" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/test" corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" @@ -48,7 +47,6 @@ import ( "sigs.k8s.io/karpenter/pkg/controllers/provisioning" "sigs.k8s.io/karpenter/pkg/controllers/state" "sigs.k8s.io/karpenter/pkg/events" - coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" @@ -76,8 +74,6 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coreoptions.ToContext(ctx, coretest.Options()) - ctx = options.ToContext(ctx, test.Options()) ctx, stop = context.WithCancel(ctx) awsEnv = test.NewEnvironment(ctx, env) fakeClock = clock.NewFakeClock(time.Now()) @@ -94,18 +90,14 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = coreoptions.ToContext(ctx, coretest.Options()) - ctx = options.ToContext(ctx, test.Options()) - - cluster.Reset() - awsEnv.Reset() - awsEnv.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") awsEnv.LaunchTemplateProvider.ClusterEndpoint = "https://test-cluster" }) var _ = AfterEach(func() { ExpectCleanedUp(ctx, env.Client) + cluster.Reset() + awsEnv.Reset() }) var _ = Describe("CloudProvider", func() { diff --git a/pkg/controllers/controllers.go b/pkg/controllers/controllers.go index e428418123f1..8c360fa4f936 100644 --- a/pkg/controllers/controllers.go +++ b/pkg/controllers/controllers.go @@ -20,6 +20,7 @@ import ( "sigs.k8s.io/karpenter/pkg/cloudprovider" controllerspricing "github.com/aws/karpenter-provider-aws/pkg/controllers/pricing" + "github.com/aws/karpenter-provider-aws/pkg/global" "github.com/aws/karpenter-provider-aws/pkg/providers/launchtemplate" "github.com/aws/aws-sdk-go/aws/session" @@ -36,7 +37,6 @@ import ( nodeclaimgarbagecollection "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclaim/garbagecollection" nodeclaimtagging "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclaim/tagging" "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" "github.com/aws/karpenter-provider-aws/pkg/providers/instance" "github.com/aws/karpenter-provider-aws/pkg/providers/instanceprofile" @@ -57,8 +57,8 @@ func NewControllers(ctx context.Context, sess *session.Session, clk clock.Clock, nodeclaimtagging.NewController(kubeClient, instanceProvider), controllerspricing.NewController(pricingProvider), } - if options.FromContext(ctx).InterruptionQueue != "" { - controllers = append(controllers, interruption.NewController(kubeClient, clk, recorder, lo.Must(sqs.NewProvider(ctx, servicesqs.New(sess), options.FromContext(ctx).InterruptionQueue)), unavailableOfferings)) + if global.Config.InterruptionQueue != "" { + controllers = append(controllers, interruption.NewController(kubeClient, clk, recorder, lo.Must(sqs.NewProvider(ctx, servicesqs.New(sess), global.Config.InterruptionQueue)), unavailableOfferings)) } return controllers } diff --git a/pkg/controllers/interruption/interruption_benchmark_test.go b/pkg/controllers/interruption/interruption_benchmark_test.go index 585fd22596c7..225f44a0de85 100644 --- a/pkg/controllers/interruption/interruption_benchmark_test.go +++ b/pkg/controllers/interruption/interruption_benchmark_test.go @@ -45,16 +45,13 @@ import ( "sigs.k8s.io/karpenter/pkg/operator/scheme" + coretest "sigs.k8s.io/karpenter/pkg/test" + awscache "github.com/aws/karpenter-provider-aws/pkg/cache" "github.com/aws/karpenter-provider-aws/pkg/controllers/interruption" "github.com/aws/karpenter-provider-aws/pkg/controllers/interruption/events" "github.com/aws/karpenter-provider-aws/pkg/fake" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/sqs" - "github.com/aws/karpenter-provider-aws/pkg/test" - - coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" - coretest "sigs.k8s.io/karpenter/pkg/test" ) var r = rand.New(rand.NewSource(time.Now().Unix())) @@ -79,12 +76,9 @@ func BenchmarkNotification100(b *testing.B) { func benchmarkNotificationController(b *testing.B, messageCount int) { ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("message-count", messageCount)) fakeClock = &clock.FakeClock{} - ctx = coreoptions.ToContext(ctx, coretest.Options()) - ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ - ClusterName: lo.ToPtr("karpenter-notification-benchmarking"), - IsolatedVPC: lo.ToPtr(true), - InterruptionQueue: lo.ToPtr("test-cluster"), - })) + global.Config.ClusterName = "karpenter-notification-benchmarking" + global.Config.IsolatedVPC = true + global.Config.InterruptionQueue = "test-cluster" env = coretest.NewEnvironment(scheme.Scheme) // Stop the coretest environment after the coretest completes defer func() { @@ -183,7 +177,7 @@ func newProviders(ctx context.Context, kubeClient client.Client) providerSet { func (p *providerSet) makeInfrastructure(ctx context.Context) error { if _, err := p.sqsAPI.CreateQueueWithContext(ctx, &servicesqs.CreateQueueInput{ - QueueName: lo.ToPtr(options.FromContext(ctx).InterruptionQueueName), + QueueName: lo.ToPtr(global.Config.InterruptionQueueName), Attributes: map[string]*string{ servicesqs.QueueAttributeNameMessageRetentionPeriod: aws.String("1200"), // 20 minutes for this test }, diff --git a/pkg/controllers/interruption/suite_test.go b/pkg/controllers/interruption/suite_test.go index d0248494f0d5..f4b0caf4b723 100644 --- a/pkg/controllers/interruption/suite_test.go +++ b/pkg/controllers/interruption/suite_test.go @@ -36,7 +36,6 @@ import ( corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/events" - coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" @@ -91,7 +90,6 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = coreoptions.ToContext(ctx, coretest.Options()) unavailableOfferingsCache.Flush() sqsapi.Reset() }) diff --git a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go index 6527c20a2a4a..df987ede54f2 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go +++ b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go @@ -40,7 +40,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclaim/garbagecollection" "github.com/aws/karpenter-provider-aws/pkg/fake" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" + "github.com/aws/karpenter-provider-aws/pkg/global" "github.com/aws/karpenter-provider-aws/pkg/test" . "github.com/onsi/ginkgo/v2" @@ -62,7 +62,6 @@ func TestAPIs(t *testing.T) { } var _ = BeforeSuite(func() { - ctx = options.ToContext(ctx, test.Options()) env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) awsEnv = test.NewEnvironment(ctx, env) cloudProvider = cloudprovider.New(awsEnv.InstanceTypesProvider, awsEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), @@ -74,7 +73,7 @@ var _ = AfterSuite(func() { Expect(env.Stop()).To(Succeed(), "Failed to stop environment") }) -var _ = BeforeEach(func() { +var _ = AfterEach(func() { awsEnv.Reset() }) @@ -104,7 +103,7 @@ var _ = Describe("GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", global.Config.ClusterName)), Value: aws.String("owned"), }, { @@ -117,7 +116,7 @@ var _ = Describe("GarbageCollection", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(options.FromContext(ctx).ClusterName), + Value: aws.String(global.Config.ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -130,8 +129,8 @@ var _ = Describe("GarbageCollection", func() { }) AfterEach(func() { ExpectCleanedUp(ctx, env.Client) + awsEnv.Reset() }) - It("should delete an instance if there is no NodeClaim owner", func() { // Launch time was 1m ago instance.LaunchTime = aws.Time(time.Now().Add(-time.Minute)) @@ -172,7 +171,7 @@ var _ = Describe("GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", global.Config.ClusterName)), Value: aws.String("owned"), }, { @@ -185,7 +184,7 @@ var _ = Describe("GarbageCollection", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(options.FromContext(ctx).ClusterName), + Value: aws.String(global.Config.ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -230,7 +229,7 @@ var _ = Describe("GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", global.Config.ClusterName)), Value: aws.String("owned"), }, }, @@ -339,7 +338,7 @@ var _ = Describe("GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", global.Config.ClusterName)), Value: aws.String("owned"), }, { @@ -348,7 +347,7 @@ var _ = Describe("GarbageCollection", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(options.FromContext(ctx).ClusterName), + Value: aws.String(global.Config.ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), diff --git a/pkg/controllers/nodeclaim/tagging/suite_test.go b/pkg/controllers/nodeclaim/tagging/suite_test.go index 6addad95b22b..8504bfe56d94 100644 --- a/pkg/controllers/nodeclaim/tagging/suite_test.go +++ b/pkg/controllers/nodeclaim/tagging/suite_test.go @@ -32,12 +32,11 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclaim/tagging" "github.com/aws/karpenter-provider-aws/pkg/fake" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" + "github.com/aws/karpenter-provider-aws/pkg/global" "github.com/aws/karpenter-provider-aws/pkg/providers/instance" "github.com/aws/karpenter-provider-aws/pkg/test" "sigs.k8s.io/karpenter/pkg/operator/controller" - coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" "sigs.k8s.io/karpenter/pkg/operator/scheme" . "github.com/onsi/ginkgo/v2" @@ -59,8 +58,6 @@ func TestAPIs(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coreoptions.ToContext(ctx, coretest.Options()) - ctx = options.ToContext(ctx, test.Options()) awsEnv = test.NewEnvironment(ctx, env) taggingController = tagging.NewController(env.Client, awsEnv.InstanceProvider) }) @@ -68,12 +65,9 @@ var _ = AfterSuite(func() { Expect(env.Stop()).To(Succeed(), "Failed to stop environment") }) -var _ = BeforeEach(func() { - awsEnv.Reset() -}) - var _ = AfterEach(func() { ExpectCleanedUp(ctx, env.Client) + awsEnv.Reset() }) var _ = Describe("TaggingController", func() { @@ -86,7 +80,7 @@ var _ = Describe("TaggingController", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", global.Config.ClusterName)), Value: aws.String("owned"), }, { @@ -95,7 +89,7 @@ var _ = Describe("TaggingController", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(options.FromContext(ctx).ClusterName), + Value: aws.String(global.Config.ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), diff --git a/pkg/controllers/nodeclass/suite_test.go b/pkg/controllers/nodeclass/suite_test.go index 0c0a436697c3..a7b93a3e6bf1 100644 --- a/pkg/controllers/nodeclass/suite_test.go +++ b/pkg/controllers/nodeclass/suite_test.go @@ -35,7 +35,6 @@ import ( corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/events" corecontroller "sigs.k8s.io/karpenter/pkg/operator/controller" - coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" @@ -43,7 +42,6 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass" "github.com/aws/karpenter-provider-aws/pkg/fake" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/instanceprofile" "github.com/aws/karpenter-provider-aws/pkg/test" @@ -66,8 +64,6 @@ func TestAPIs(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...), coretest.WithFieldIndexers(test.EC2NodeClassFieldIndexer(ctx))) - ctx = coreoptions.ToContext(ctx, coretest.Options()) - ctx = options.ToContext(ctx, test.Options()) awsEnv = test.NewEnvironment(ctx, env) nodeClassController = nodeclass.NewController(env.Client, events.NewRecorder(&record.FakeRecorder{}), awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider) @@ -77,13 +73,9 @@ var _ = AfterSuite(func() { Expect(env.Stop()).To(Succeed(), "Failed to stop environment") }) -var _ = BeforeEach(func() { - ctx = coreoptions.ToContext(ctx, coretest.Options()) - awsEnv.Reset() -}) - var _ = AfterEach(func() { ExpectCleanedUp(ctx, env.Client) + awsEnv.Reset() }) var _ = Describe("NodeClassController", func() { @@ -1016,7 +1008,7 @@ var _ = Describe("NodeClassController", func() { Context("NodeClass Termination", func() { var profileName string BeforeEach(func() { - profileName = instanceprofile.GetProfileName(ctx, fake.DefaultRegion, nodeClass) + profileName = instanceprofile.GetProfileName(fake.DefaultRegion, nodeClass) }) It("should not delete the NodeClass if launch template deletion fails", func() { launchTemplateName := aws.String(fake.LaunchTemplateName()) @@ -1189,7 +1181,7 @@ var _ = Describe("NodeClassController", func() { Context("Instance Profile Status", func() { var profileName string BeforeEach(func() { - profileName = instanceprofile.GetProfileName(ctx, fake.DefaultRegion, nodeClass) + profileName = instanceprofile.GetProfileName(fake.DefaultRegion, nodeClass) }) It("should create the instance profile when it doesn't exist", func() { nodeClass.Spec.Role = "test-role" diff --git a/pkg/controllers/pricing/suite_test.go b/pkg/controllers/pricing/suite_test.go index 382a398a2088..e3b35d32f68e 100644 --- a/pkg/controllers/pricing/suite_test.go +++ b/pkg/controllers/pricing/suite_test.go @@ -25,14 +25,13 @@ import ( awspricing "github.com/aws/aws-sdk-go/service/pricing" "github.com/samber/lo" "k8s.io/apimachinery/pkg/types" - coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" "github.com/aws/karpenter-provider-aws/pkg/apis" controllerspricing "github.com/aws/karpenter-provider-aws/pkg/controllers/pricing" "github.com/aws/karpenter-provider-aws/pkg/fake" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" + "github.com/aws/karpenter-provider-aws/pkg/global" "github.com/aws/karpenter-provider-aws/pkg/providers/pricing" "github.com/aws/karpenter-provider-aws/pkg/test" @@ -56,8 +55,6 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coreoptions.ToContext(ctx, coretest.Options()) - ctx = options.ToContext(ctx, test.Options()) ctx, stop = context.WithCancel(ctx) awsEnv = test.NewEnvironment(ctx, env) controller = controllerspricing.NewController(awsEnv.PricingProvider) @@ -68,15 +65,9 @@ var _ = AfterSuite(func() { Expect(env.Stop()).To(Succeed(), "Failed to stop environment") }) -var _ = BeforeEach(func() { - ctx = coreoptions.ToContext(ctx, coretest.Options()) - ctx = options.ToContext(ctx, test.Options()) - - awsEnv.Reset() -}) - var _ = AfterEach(func() { ExpectCleanedUp(ctx, env.Client) + awsEnv.Reset() }) var _ = Describe("Pricing", func() { @@ -84,7 +75,7 @@ var _ = Describe("Pricing", func() { "should return correct static data for all partitions", func(staticPricing map[string]map[string]float64) { for region, prices := range staticPricing { - provider := pricing.NewDefaultProvider(ctx, awsEnv.PricingAPI, awsEnv.EC2API, region) + provider := pricing.NewDefaultProvider(awsEnv.PricingAPI, awsEnv.EC2API, region) for instance, price := range prices { val, ok := provider.OnDemandPrice(instance) Expect(ok).To(BeTrue()) @@ -259,9 +250,7 @@ var _ = Describe("Pricing", func() { To(ContainElements("Linux/UNIX", "Linux/UNIX (Amazon VPC)")) }) It("should return static on-demand data when in isolated-vpc", func() { - ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ - IsolatedVPC: lo.ToPtr(true), - })) + global.Config.IsolatedVPC = true now := time.Now() awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ SpotPriceHistory: []*ec2.SpotPrice{ @@ -298,7 +287,7 @@ var _ = Describe("Pricing", func() { Expect(price).To(BeNumerically("==", 1.10)) }) It("should update on-demand pricing with response from the pricing API when in the CN partition", func() { - tmpPricingProvider := pricing.NewDefaultProvider(ctx, awsEnv.PricingAPI, awsEnv.EC2API, "cn-anywhere-1") + tmpPricingProvider := pricing.NewDefaultProvider(awsEnv.PricingAPI, awsEnv.EC2API, "cn-anywhere-1") tmpController := controllerspricing.NewController(tmpPricingProvider) now := time.Now() diff --git a/pkg/fake/utils.go b/pkg/fake/utils.go index 6e3067c41174..60890af63a34 100644 --- a/pkg/fake/utils.go +++ b/pkg/fake/utils.go @@ -15,7 +15,6 @@ limitations under the License. package fake import ( - "context" "fmt" "strings" @@ -25,7 +24,6 @@ import ( "github.com/samber/lo" "k8s.io/apimachinery/pkg/util/sets" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/pricing" ) @@ -163,10 +161,9 @@ func matchTags(tags []*ec2.Tag, filter *ec2.Filter) bool { func MakeInstances() []*ec2.InstanceTypeInfo { var instanceTypes []*ec2.InstanceTypeInfo - ctx := options.ToContext(context.Background(), &options.Options{IsolatedVPC: true}) // Use keys from the static pricing data so that we guarantee pricing for the data // Create uniform instance data so all of them schedule for a given pod - for _, it := range pricing.NewDefaultProvider(ctx, nil, nil, "us-east-1").InstanceTypes() { + for _, it := range pricing.NewDefaultProvider(nil, nil, "us-east-1").InstanceTypes() { instanceTypes = append(instanceTypes, &ec2.InstanceTypeInfo{ InstanceType: aws.String(it), ProcessorInfo: &ec2.ProcessorInfo{ diff --git a/pkg/global/config.go b/pkg/global/config.go new file mode 100644 index 000000000000..5d4e7708b56e --- /dev/null +++ b/pkg/global/config.go @@ -0,0 +1,75 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package global + +import ( + "errors" + "flag" + "fmt" + "os" + "time" + + coreglobal "sigs.k8s.io/karpenter/pkg/global" + "sigs.k8s.io/karpenter/pkg/utils/env" +) + +var Config config + +type config struct { + AssumeRoleARN string + AssumeRoleDuration time.Duration + ClusterCABundle string + ClusterName string + ClusterEndpoint string + IsolatedVPC bool + VMMemoryOverheadPercent float64 + InterruptionQueue string + ReservedENIs int +} + +func AddFlags(fs *coreglobal.FlagSet) { + fs.StringVar(&Config.AssumeRoleARN, "assume-role-arn", env.WithDefaultString("ASSUME_ROLE_ARN", ""), "Role to assume for calling AWS services.") + fs.DurationVar(&Config.AssumeRoleDuration, "assume-role-duration", env.WithDefaultDuration("ASSUME_ROLE_DURATION", 15*time.Minute), "Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless aws.assumeRole set.") + fs.StringVar(&Config.ClusterCABundle, "cluster-ca-bundle", env.WithDefaultString("CLUSTER_CA_BUNDLE", ""), "Cluster CA bundle for nodes to use for TLS connections with the API server. If not set, this is taken from the controller's TLS configuration.") + fs.StringVar(&Config.ClusterName, "cluster-name", env.WithDefaultString("CLUSTER_NAME", ""), "[REQUIRED] The kubernetes cluster name for resource discovery.") + fs.StringVar(&Config.ClusterEndpoint, "cluster-endpoint", env.WithDefaultString("CLUSTER_ENDPOINT", ""), "The external kubernetes cluster endpoint for new nodes to connect with. If not specified, will discover the cluster endpoint using DescribeCluster API.") + fs.BoolVarWithEnv(&Config.IsolatedVPC, "isolated-vpc", "ISOLATED_VPC", false, "If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS on-demand pricing endpoint.") + fs.Float64Var(&Config.VMMemoryOverheadPercent, "vm-memory-overhead-percent", env.WithDefaultFloat64("VM_MEMORY_OVERHEAD_PERCENT", 0.075), "The VM memory overhead as a percent that will be subtracted from the total memory for all instance types.") + fs.StringVar(&Config.InterruptionQueue, "interruption-queue", env.WithDefaultString("INTERRUPTION_QUEUE", ""), "Interruption queue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.") + fs.IntVar(&Config.ReservedENIs, "reserved-enis", env.WithDefaultInt("RESERVED_ENIS", 0), "Reserved ENIs are not included in the calculations for max-pods or kube-reserved. This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html.") +} + +func Initialize(args ...string) error { + fs := coreglobal.NewFlagSet() + coreglobal.AddFlags(fs) + AddFlags(fs) + + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + os.Exit(0) + } + return fmt.Errorf("parsing flags, %w", err) + } + if err := coreglobal.ParseFeatureGates(); err != nil { + return fmt.Errorf("parsing feature gates, %w", err) + } + if err := coreglobal.Config.Validate(); err != nil { + return fmt.Errorf("validating config, %w", err) + } + if err := Config.Validate(); err != nil { + return fmt.Errorf("validating config, %w", err) + } + return nil +} diff --git a/pkg/operator/options/options_validation.go b/pkg/global/config_validation.go similarity index 61% rename from pkg/operator/options/options_validation.go rename to pkg/global/config_validation.go index 1c4a733700ec..4410bfe5e7c2 100644 --- a/pkg/operator/options/options_validation.go +++ b/pkg/global/config_validation.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package options +package global import ( "fmt" @@ -22,52 +22,52 @@ import ( "go.uber.org/multierr" ) -func (o Options) Validate() error { +func (c config) Validate() error { return multierr.Combine( - o.validateEndpoint(), - o.validateVMMemoryOverheadPercent(), - o.validateAssumeRoleDuration(), - o.validateReservedENIs(), - o.validateRequiredFields(), + c.validateEndpoint(), + c.validateVMMemoryOverheadPercent(), + c.validateAssumeRoleDuration(), + c.validateReservedENIs(), + c.validateRequiredFields(), ) } -func (o Options) validateAssumeRoleDuration() error { - if o.AssumeRoleDuration < time.Minute*15 { +func (c config) validateAssumeRoleDuration() error { + if c.AssumeRoleDuration < time.Minute*15 { return fmt.Errorf("assume-role-duration cannot be less than 15 minutes") } return nil } -func (o Options) validateEndpoint() error { - if o.ClusterEndpoint == "" { +func (c config) validateEndpoint() error { + if c.ClusterEndpoint == "" { return nil } - endpoint, err := url.Parse(o.ClusterEndpoint) + endpoint, err := url.Parse(c.ClusterEndpoint) // url.Parse() will accept a lot of input without error; make // sure it's a real URL if err != nil || !endpoint.IsAbs() || endpoint.Hostname() == "" { - return fmt.Errorf("%q is not a valid cluster-endpoint URL", o.ClusterEndpoint) + return fmt.Errorf("%q is not a valid cluster-endpoint URL", c.ClusterEndpoint) } return nil } -func (o Options) validateVMMemoryOverheadPercent() error { - if o.VMMemoryOverheadPercent < 0 { +func (c config) validateVMMemoryOverheadPercent() error { + if c.VMMemoryOverheadPercent < 0 { return fmt.Errorf("vm-memory-overhead-percent cannot be negative") } return nil } -func (o Options) validateReservedENIs() error { - if o.ReservedENIs < 0 { +func (c config) validateReservedENIs() error { + if c.ReservedENIs < 0 { return fmt.Errorf("reserved-enis cannot be negative") } return nil } -func (o Options) validateRequiredFields() error { - if o.ClusterName == "" { +func (c config) validateRequiredFields() error { + if c.ClusterName == "" { return fmt.Errorf("missing field, cluster-name") } return nil diff --git a/pkg/global/suite_test.go b/pkg/global/suite_test.go new file mode 100644 index 000000000000..3199c69410cb --- /dev/null +++ b/pkg/global/suite_test.go @@ -0,0 +1,128 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package global + +import ( + "context" + "os" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "knative.dev/pkg/logging/testing" +) + +var ctx context.Context + +func TestAPIs(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "Options") +} + +var _ = Describe("Config", func() { + AfterEach(func() { + Config = config{} + os.Clearenv() + }) + + It("should correctly override default vars when CLI flags are set", func() { + err := Initialize( + "--assume-role-arn", "env-role", + "--assume-role-duration", "20m", + "--cluster-ca-bundle", "env-bundle", + "--cluster-name", "env-cluster", + "--cluster-endpoint", "https://env-cluster", + "--isolated-vpc", + "--vm-memory-overhead-percent", "0.1", + "--interruption-queue", "env-cluster", + "--reserved-enis", "10") + Expect(err).ToNot(HaveOccurred()) + expectConfigEqual(Config, config{ + AssumeRoleARN: "env-role", + AssumeRoleDuration: 20 * time.Minute, + ClusterCABundle: "env-bundle", + ClusterName: "env-cluster", + ClusterEndpoint: "https://env-cluster", + IsolatedVPC: true, + VMMemoryOverheadPercent: 0.1, + InterruptionQueue: "env-cluster", + ReservedENIs: 10, + }) + }) + It("should correctly fallback to env vars when CLI flags aren't set", func() { + os.Setenv("ASSUME_ROLE_ARN", "env-role") + os.Setenv("ASSUME_ROLE_DURATION", "20m") + os.Setenv("CLUSTER_CA_BUNDLE", "env-bundle") + os.Setenv("CLUSTER_NAME", "env-cluster") + os.Setenv("CLUSTER_ENDPOINT", "https://env-cluster") + os.Setenv("ISOLATED_VPC", "true") + os.Setenv("VM_MEMORY_OVERHEAD_PERCENT", "0.1") + os.Setenv("INTERRUPTION_QUEUE", "env-cluster") + os.Setenv("RESERVED_ENIS", "10") + + err := Initialize() + Expect(err).ToNot(HaveOccurred()) + + expectConfigEqual(Config, config{ + AssumeRoleARN: "env-role", + AssumeRoleDuration: 20 * time.Minute, + ClusterCABundle: "env-bundle", + ClusterName: "env-cluster", + ClusterEndpoint: "https://env-cluster", + IsolatedVPC: true, + VMMemoryOverheadPercent: 0.1, + InterruptionQueue: "env-cluster", + ReservedENIs: 10, + }) + }) + + Context("Validation", func() { + It("should fail when cluster name is not set", func() { + err := Initialize() + Expect(err).To(HaveOccurred()) + }) + It("should fail when assume role duration is less than 15 minutes", func() { + err := Initialize("--cluster-name", "test-cluster", "--assume-role-duration", "1s") + Expect(err).To(HaveOccurred()) + }) + It("should fail when clusterEndpoint is invalid (not absolute)", func() { + err := Initialize("--cluster-name", "test-cluster", "--cluster-endpoint", "00000000000000000000000.gr7.us-west-2.eks.amazonaws.com") + Expect(err).To(HaveOccurred()) + }) + It("should fail when vmMemoryOverheadPercent is negative", func() { + err := Initialize("--cluster-name", "test-cluster", "--vm-memory-overhead-percent", "-0.01") + Expect(err).To(HaveOccurred()) + }) + It("should fail when reservedENIs is negative", func() { + err := Initialize("--cluster-name", "test-cluster", "--reserved-enis", "-1") + Expect(err).To(HaveOccurred()) + }) + }) +}) + +func expectConfigEqual(configA, configB config) { + GinkgoHelper() + Expect(configA.AssumeRoleARN).To(Equal(configB.AssumeRoleARN)) + Expect(configA.AssumeRoleDuration).To(Equal(configB.AssumeRoleDuration)) + Expect(configA.ClusterCABundle).To(Equal(configB.ClusterCABundle)) + Expect(configA.ClusterName).To(Equal(configB.ClusterName)) + Expect(configA.ClusterEndpoint).To(Equal(configB.ClusterEndpoint)) + Expect(configA.IsolatedVPC).To(Equal(configB.IsolatedVPC)) + Expect(configA.VMMemoryOverheadPercent).To(Equal(configB.VMMemoryOverheadPercent)) + Expect(configA.InterruptionQueue).To(Equal(configB.InterruptionQueue)) + Expect(configA.ReservedENIs).To(Equal(configB.ReservedENIs)) +} diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 52de88629ce8..3ce71bf3f06a 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -52,7 +52,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis" awscache "github.com/aws/karpenter-provider-aws/pkg/cache" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" + "github.com/aws/karpenter-provider-aws/pkg/global" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" "github.com/aws/karpenter-provider-aws/pkg/providers/instance" "github.com/aws/karpenter-provider-aws/pkg/providers/instanceprofile" @@ -89,13 +89,18 @@ type Operator struct { } func NewOperator(ctx context.Context, operator *operator.Operator) (context.Context, *Operator) { + lo.Must0(global.Initialize()) config := &aws.Config{ STSRegionalEndpoint: endpoints.RegionalSTSEndpoint, } - if assumeRoleARN := options.FromContext(ctx).AssumeRoleARN; assumeRoleARN != "" { + if assumeRoleARN := global.Config.AssumeRoleARN; assumeRoleARN != "" { config.Credentials = stscreds.NewCredentials(session.Must(session.NewSession()), assumeRoleARN, - func(provider *stscreds.AssumeRoleProvider) { setDurationAndExpiry(ctx, provider) }) + func(provider *stscreds.AssumeRoleProvider) { + provider.Duration = global.Config.AssumeRoleDuration + provider.ExpiryWindow = time.Duration(10) * time.Second + }, + ) } sess := withUserAgent(session.Must(session.NewSession( @@ -136,7 +141,6 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont securityGroupProvider := securitygroup.NewDefaultProvider(ec2api, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) instanceProfileProvider := instanceprofile.NewProvider(*sess.Config.Region, iam.New(sess), cache.New(awscache.InstanceProfileTTL, awscache.DefaultCleanupInterval)) pricingProvider := pricing.NewDefaultProvider( - ctx, pricing.NewAPI(sess, *sess.Config.Region), ec2api, *sess.Config.Region, @@ -153,7 +157,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont securityGroupProvider, subnetProvider, instanceProfileProvider, - lo.Must(getCABundle(ctx, operator.GetConfig())), + lo.Must(getCABundle(operator.GetConfig())), operator.Elected(), kubeDNSIP, clusterEndpoint, @@ -213,12 +217,12 @@ func checkEC2Connectivity(ctx context.Context, api *ec2.EC2) error { } func ResolveClusterEndpoint(ctx context.Context, eksAPI eksiface.EKSAPI) (string, error) { - clusterEndpointFromOptions := options.FromContext(ctx).ClusterEndpoint + clusterEndpointFromOptions := global.Config.ClusterEndpoint if clusterEndpointFromOptions != "" { return clusterEndpointFromOptions, nil // cluster endpoint is explicitly set } out, err := eksAPI.DescribeClusterWithContext(ctx, &eks.DescribeClusterInput{ - Name: aws.String(options.FromContext(ctx).ClusterName), + Name: aws.String(global.Config.ClusterName), }) if err != nil { return "", fmt.Errorf("failed to resolve cluster endpoint, %w", err) @@ -226,12 +230,12 @@ func ResolveClusterEndpoint(ctx context.Context, eksAPI eksiface.EKSAPI) (string return *out.Cluster.Endpoint, nil } -func getCABundle(ctx context.Context, restConfig *rest.Config) (*string, error) { +func getCABundle(restConfig *rest.Config) (*string, error) { // Discover CA Bundle from the REST client. We could alternatively // have used the simpler client-go InClusterConfig() method. // However, that only works when Karpenter is running as a Pod // within the same cluster it's managing. - if caBundle := options.FromContext(ctx).ClusterCABundle; caBundle != "" { + if caBundle := global.Config.ClusterCABundle; caBundle != "" { return lo.ToPtr(caBundle), nil } transportConfig, err := restConfig.TransportConfig() @@ -259,8 +263,3 @@ func kubeDNSIP(ctx context.Context, kubernetesInterface kubernetes.Interface) (n } return kubeDNSIP, nil } - -func setDurationAndExpiry(ctx context.Context, provider *stscreds.AssumeRoleProvider) { - provider.Duration = options.FromContext(ctx).AssumeRoleDuration - provider.ExpiryWindow = time.Duration(10) * time.Second -} diff --git a/pkg/operator/options/options.go b/pkg/operator/options/options.go deleted file mode 100644 index 84d9c38c5268..000000000000 --- a/pkg/operator/options/options.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "context" - "errors" - "flag" - "fmt" - "os" - "time" - - coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" - "sigs.k8s.io/karpenter/pkg/utils/env" -) - -func init() { - coreoptions.Injectables = append(coreoptions.Injectables, &Options{}) -} - -type optionsKey struct{} - -type Options struct { - AssumeRoleARN string - AssumeRoleDuration time.Duration - ClusterCABundle string - ClusterName string - ClusterEndpoint string - IsolatedVPC bool - VMMemoryOverheadPercent float64 - InterruptionQueue string - ReservedENIs int -} - -func (o *Options) AddFlags(fs *coreoptions.FlagSet) { - fs.StringVar(&o.AssumeRoleARN, "assume-role-arn", env.WithDefaultString("ASSUME_ROLE_ARN", ""), "Role to assume for calling AWS services.") - fs.DurationVar(&o.AssumeRoleDuration, "assume-role-duration", env.WithDefaultDuration("ASSUME_ROLE_DURATION", 15*time.Minute), "Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless aws.assumeRole set.") - fs.StringVar(&o.ClusterCABundle, "cluster-ca-bundle", env.WithDefaultString("CLUSTER_CA_BUNDLE", ""), "Cluster CA bundle for nodes to use for TLS connections with the API server. If not set, this is taken from the controller's TLS configuration.") - fs.StringVar(&o.ClusterName, "cluster-name", env.WithDefaultString("CLUSTER_NAME", ""), "[REQUIRED] The kubernetes cluster name for resource discovery.") - fs.StringVar(&o.ClusterEndpoint, "cluster-endpoint", env.WithDefaultString("CLUSTER_ENDPOINT", ""), "The external kubernetes cluster endpoint for new nodes to connect with. If not specified, will discover the cluster endpoint using DescribeCluster API.") - fs.BoolVarWithEnv(&o.IsolatedVPC, "isolated-vpc", "ISOLATED_VPC", false, "If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS on-demand pricing endpoint.") - fs.Float64Var(&o.VMMemoryOverheadPercent, "vm-memory-overhead-percent", env.WithDefaultFloat64("VM_MEMORY_OVERHEAD_PERCENT", 0.075), "The VM memory overhead as a percent that will be subtracted from the total memory for all instance types.") - fs.StringVar(&o.InterruptionQueue, "interruption-queue", env.WithDefaultString("INTERRUPTION_QUEUE", ""), "Interruption queue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.") - fs.IntVar(&o.ReservedENIs, "reserved-enis", env.WithDefaultInt("RESERVED_ENIS", 0), "Reserved ENIs are not included in the calculations for max-pods or kube-reserved. This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html.") -} - -func (o *Options) Parse(fs *coreoptions.FlagSet, args ...string) error { - if err := fs.Parse(args); err != nil { - if errors.Is(err, flag.ErrHelp) { - os.Exit(0) - } - return fmt.Errorf("parsing flags, %w", err) - } - if err := o.Validate(); err != nil { - return fmt.Errorf("validating options, %w", err) - } - return nil -} - -func (o *Options) ToContext(ctx context.Context) context.Context { - return ToContext(ctx, o) -} - -func ToContext(ctx context.Context, opts *Options) context.Context { - return context.WithValue(ctx, optionsKey{}, opts) -} - -func FromContext(ctx context.Context) *Options { - retval := ctx.Value(optionsKey{}) - if retval == nil { - return nil - } - return retval.(*Options) -} diff --git a/pkg/operator/options/suite_test.go b/pkg/operator/options/suite_test.go deleted file mode 100644 index 6b7e39715149..000000000000 --- a/pkg/operator/options/suite_test.go +++ /dev/null @@ -1,149 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options_test - -import ( - "context" - "flag" - "os" - "testing" - "time" - - "github.com/samber/lo" - coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" - - "github.com/aws/karpenter-provider-aws/pkg/operator/options" - "github.com/aws/karpenter-provider-aws/pkg/test" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - . "knative.dev/pkg/logging/testing" -) - -var ctx context.Context - -func TestAPIs(t *testing.T) { - ctx = TestContextWithLogger(t) - RegisterFailHandler(Fail) - RunSpecs(t, "Options") -} - -var _ = Describe("Options", func() { - var fs *coreoptions.FlagSet - var opts *options.Options - - BeforeEach(func() { - fs = &coreoptions.FlagSet{ - FlagSet: flag.NewFlagSet("karpenter", flag.ContinueOnError), - } - opts = &options.Options{} - }) - AfterEach(func() { - os.Clearenv() - }) - - It("should correctly override default vars when CLI flags are set", func() { - opts.AddFlags(fs) - err := opts.Parse(fs, - "--assume-role-arn", "env-role", - "--assume-role-duration", "20m", - "--cluster-ca-bundle", "env-bundle", - "--cluster-name", "env-cluster", - "--cluster-endpoint", "https://env-cluster", - "--isolated-vpc", - "--vm-memory-overhead-percent", "0.1", - "--interruption-queue", "env-cluster", - "--reserved-enis", "10") - Expect(err).ToNot(HaveOccurred()) - expectOptionsEqual(opts, test.Options(test.OptionsFields{ - AssumeRoleARN: lo.ToPtr("env-role"), - AssumeRoleDuration: lo.ToPtr(20 * time.Minute), - ClusterCABundle: lo.ToPtr("env-bundle"), - ClusterName: lo.ToPtr("env-cluster"), - ClusterEndpoint: lo.ToPtr("https://env-cluster"), - IsolatedVPC: lo.ToPtr(true), - VMMemoryOverheadPercent: lo.ToPtr[float64](0.1), - InterruptionQueue: lo.ToPtr("env-cluster"), - ReservedENIs: lo.ToPtr(10), - })) - }) - It("should correctly fallback to env vars when CLI flags aren't set", func() { - os.Setenv("ASSUME_ROLE_ARN", "env-role") - os.Setenv("ASSUME_ROLE_DURATION", "20m") - os.Setenv("CLUSTER_CA_BUNDLE", "env-bundle") - os.Setenv("CLUSTER_NAME", "env-cluster") - os.Setenv("CLUSTER_ENDPOINT", "https://env-cluster") - os.Setenv("ISOLATED_VPC", "true") - os.Setenv("VM_MEMORY_OVERHEAD_PERCENT", "0.1") - os.Setenv("INTERRUPTION_QUEUE", "env-cluster") - os.Setenv("RESERVED_ENIS", "10") - - // Add flags after we set the environment variables so that the parsing logic correctly refers - // to the new environment variable values - opts.AddFlags(fs) - err := opts.Parse(fs) - Expect(err).ToNot(HaveOccurred()) - expectOptionsEqual(opts, test.Options(test.OptionsFields{ - AssumeRoleARN: lo.ToPtr("env-role"), - AssumeRoleDuration: lo.ToPtr(20 * time.Minute), - ClusterCABundle: lo.ToPtr("env-bundle"), - ClusterName: lo.ToPtr("env-cluster"), - ClusterEndpoint: lo.ToPtr("https://env-cluster"), - IsolatedVPC: lo.ToPtr(true), - VMMemoryOverheadPercent: lo.ToPtr[float64](0.1), - InterruptionQueue: lo.ToPtr("env-cluster"), - ReservedENIs: lo.ToPtr(10), - })) - }) - - Context("Validation", func() { - BeforeEach(func() { - opts.AddFlags(fs) - }) - It("should fail when cluster name is not set", func() { - err := opts.Parse(fs) - Expect(err).To(HaveOccurred()) - }) - It("should fail when assume role duration is less than 15 minutes", func() { - err := opts.Parse(fs, "--cluster-name", "test-cluster", "--assume-role-duration", "1s") - Expect(err).To(HaveOccurred()) - }) - It("should fail when clusterEndpoint is invalid (not absolute)", func() { - err := opts.Parse(fs, "--cluster-name", "test-cluster", "--cluster-endpoint", "00000000000000000000000.gr7.us-west-2.eks.amazonaws.com") - Expect(err).To(HaveOccurred()) - }) - It("should fail when vmMemoryOverheadPercent is negative", func() { - err := opts.Parse(fs, "--cluster-name", "test-cluster", "--vm-memory-overhead-percent", "-0.01") - Expect(err).To(HaveOccurred()) - }) - It("should fail when reservedENIs is negative", func() { - err := opts.Parse(fs, "--cluster-name", "test-cluster", "--reserved-enis", "-1") - Expect(err).To(HaveOccurred()) - }) - }) -}) - -func expectOptionsEqual(optsA *options.Options, optsB *options.Options) { - GinkgoHelper() - Expect(optsA.AssumeRoleARN).To(Equal(optsB.AssumeRoleARN)) - Expect(optsA.AssumeRoleDuration).To(Equal(optsB.AssumeRoleDuration)) - Expect(optsA.ClusterCABundle).To(Equal(optsB.ClusterCABundle)) - Expect(optsA.ClusterName).To(Equal(optsB.ClusterName)) - Expect(optsA.ClusterEndpoint).To(Equal(optsB.ClusterEndpoint)) - Expect(optsA.IsolatedVPC).To(Equal(optsB.IsolatedVPC)) - Expect(optsA.VMMemoryOverheadPercent).To(Equal(optsB.VMMemoryOverheadPercent)) - Expect(optsA.InterruptionQueue).To(Equal(optsB.InterruptionQueue)) - Expect(optsA.ReservedENIs).To(Equal(optsB.ReservedENIs)) -} diff --git a/pkg/operator/suite_test.go b/pkg/operator/suite_test.go index f725b2f23652..93a2730da15e 100644 --- a/pkg/operator/suite_test.go +++ b/pkg/operator/suite_test.go @@ -25,16 +25,15 @@ import ( "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" - "github.com/aws/karpenter-provider-aws/pkg/apis" - "github.com/aws/karpenter-provider-aws/pkg/fake" - awscontext "github.com/aws/karpenter-provider-aws/pkg/operator" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" - "github.com/aws/karpenter-provider-aws/pkg/test" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "knative.dev/pkg/logging/testing" . "sigs.k8s.io/karpenter/pkg/test/expectations" + + "github.com/aws/karpenter-provider-aws/pkg/apis" + "github.com/aws/karpenter-provider-aws/pkg/fake" + "github.com/aws/karpenter-provider-aws/pkg/global" + "github.com/aws/karpenter-provider-aws/pkg/operator" ) var ctx context.Context @@ -70,17 +69,13 @@ var _ = AfterEach(func() { var _ = Describe("Operator", func() { It("should resolve endpoint if set via configuration", func() { - ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ - ClusterEndpoint: lo.ToPtr("https://api.test-cluster.k8s.local"), - })) - endpoint, err := awscontext.ResolveClusterEndpoint(ctx, fakeEKSAPI) + global.Config.ClusterEndpoint = "https://api.test-cluster.k8s.local" + endpoint, err := operator.ResolveClusterEndpoint(ctx, fakeEKSAPI) Expect(err).ToNot(HaveOccurred()) Expect(endpoint).To(Equal("https://api.test-cluster.k8s.local")) }) It("should resolve endpoint if not set, via call to API", func() { - ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ - ClusterEndpoint: lo.ToPtr(""), - })) + global.Config.ClusterEndpoint = "" fakeEKSAPI.DescribeClusterBehavior.Output.Set( &eks.DescribeClusterOutput{ Cluster: &eks.Cluster{ @@ -89,17 +84,15 @@ var _ = Describe("Operator", func() { }, ) - endpoint, err := awscontext.ResolveClusterEndpoint(ctx, fakeEKSAPI) + endpoint, err := operator.ResolveClusterEndpoint(ctx, fakeEKSAPI) Expect(err).ToNot(HaveOccurred()) Expect(endpoint).To(Equal("https://cluster-endpoint.test-cluster.k8s.local")) }) It("should propagate error if API fails", func() { - ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ - ClusterEndpoint: lo.ToPtr(""), - })) + global.Config.ClusterEndpoint = "" fakeEKSAPI.DescribeClusterBehavior.Error.Set(errors.New("test error")) - _, err := awscontext.ResolveClusterEndpoint(ctx, fakeEKSAPI) + _, err := operator.ResolveClusterEndpoint(ctx, fakeEKSAPI) Expect(err).To(HaveOccurred()) }) }) diff --git a/pkg/providers/amifamily/suite_test.go b/pkg/providers/amifamily/suite_test.go index 7dfc5ebe0d0b..d5c65b46d6c1 100644 --- a/pkg/providers/amifamily/suite_test.go +++ b/pkg/providers/amifamily/suite_test.go @@ -30,14 +30,12 @@ import ( . "knative.dev/pkg/logging/testing" corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" "sigs.k8s.io/karpenter/pkg/operator/scheme" "sigs.k8s.io/karpenter/pkg/scheduling" coretest "sigs.k8s.io/karpenter/pkg/test" "github.com/aws/karpenter-provider-aws/pkg/apis" "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" "github.com/aws/karpenter-provider-aws/pkg/test" ) @@ -62,8 +60,6 @@ const ( var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coreoptions.ToContext(ctx, coretest.Options()) - ctx = options.ToContext(ctx, test.Options()) awsEnv = test.NewEnvironment(ctx, env) }) diff --git a/pkg/providers/instance/instance.go b/pkg/providers/instance/instance.go index 50abbaf45826..efd5a5e85c92 100644 --- a/pkg/providers/instance/instance.go +++ b/pkg/providers/instance/instance.go @@ -39,7 +39,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/batcher" "github.com/aws/karpenter-provider-aws/pkg/cache" awserrors "github.com/aws/karpenter-provider-aws/pkg/errors" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" + "github.com/aws/karpenter-provider-aws/pkg/global" "github.com/aws/karpenter-provider-aws/pkg/providers/instancetype" "github.com/aws/karpenter-provider-aws/pkg/providers/launchtemplate" "github.com/aws/karpenter-provider-aws/pkg/providers/subnet" @@ -95,7 +95,7 @@ func (p *DefaultProvider) Create(ctx context.Context, nodeClass *v1beta1.EC2Node if !schedulingRequirements.HasMinValues() { instanceTypes = p.filterInstanceTypes(nodeClaim, instanceTypes) } - tags := getTags(ctx, nodeClass, nodeClaim) + tags := getTags(nodeClass, nodeClaim) fleetInstance, err := p.launchInstance(ctx, nodeClass, nodeClaim, instanceTypes, tags) if awserrors.IsLaunchTemplateNotFound(err) { // retry once if launch template is not found. This allows karpenter to generate a new LT if the @@ -144,7 +144,7 @@ func (p *DefaultProvider) List(ctx context.Context) ([]*Instance, error) { }, { Name: aws.String("tag-key"), - Values: aws.StringSlice([]string{fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)}), + Values: aws.StringSlice([]string{fmt.Sprintf("kubernetes.io/cluster/%s", global.Config.ClusterName)}), }, instanceStateFilter, }, @@ -251,12 +251,12 @@ func (p *DefaultProvider) launchInstance(ctx context.Context, nodeClass *v1beta1 return createFleetOutput.Instances[0], nil } -func getTags(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim) map[string]string { +func getTags(nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim) map[string]string { staticTags := map[string]string{ - fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName): "owned", - corev1beta1.NodePoolLabelKey: nodeClaim.Labels[corev1beta1.NodePoolLabelKey], - corev1beta1.ManagedByAnnotationKey: options.FromContext(ctx).ClusterName, - v1beta1.LabelNodeClass: nodeClass.Name, + fmt.Sprintf("kubernetes.io/cluster/%s", global.Config.ClusterName): "owned", + corev1beta1.NodePoolLabelKey: nodeClaim.Labels[corev1beta1.NodePoolLabelKey], + corev1beta1.ManagedByAnnotationKey: global.Config.ClusterName, + v1beta1.LabelNodeClass: nodeClass.Name, } return lo.Assign(nodeClass.Spec.Tags, staticTags) } diff --git a/pkg/providers/instance/suite_test.go b/pkg/providers/instance/suite_test.go index 45d3504bafb6..540ee280d616 100644 --- a/pkg/providers/instance/suite_test.go +++ b/pkg/providers/instance/suite_test.go @@ -29,7 +29,6 @@ import ( corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" corecloudprovider "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/events" - coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" @@ -37,7 +36,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" "github.com/aws/karpenter-provider-aws/pkg/fake" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" + "github.com/aws/karpenter-provider-aws/pkg/global" "github.com/aws/karpenter-provider-aws/pkg/providers/instance" "github.com/aws/karpenter-provider-aws/pkg/test" @@ -60,8 +59,6 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coreoptions.ToContext(ctx, coretest.Options()) - ctx = options.ToContext(ctx, test.Options()) awsEnv = test.NewEnvironment(ctx, env) cloudProvider = cloudprovider.New(awsEnv.InstanceTypesProvider, awsEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), env.Client, awsEnv.AMIProvider, awsEnv.SecurityGroupProvider, awsEnv.SubnetProvider) @@ -72,8 +69,6 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = coreoptions.ToContext(ctx, coretest.Options()) - ctx = options.ToContext(ctx, test.Options()) awsEnv.Reset() }) @@ -139,7 +134,7 @@ var _ = Describe("InstanceProvider", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", global.Config.ClusterName)), Value: aws.String("owned"), }, { @@ -152,7 +147,7 @@ var _ = Describe("InstanceProvider", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(options.FromContext(ctx).ClusterName), + Value: aws.String(global.Config.ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), diff --git a/pkg/providers/instanceprofile/instanceprofile.go b/pkg/providers/instanceprofile/instanceprofile.go index 31296181b127..d7962b43d63c 100644 --- a/pkg/providers/instanceprofile/instanceprofile.go +++ b/pkg/providers/instanceprofile/instanceprofile.go @@ -30,7 +30,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" awserrors "github.com/aws/karpenter-provider-aws/pkg/errors" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" + "github.com/aws/karpenter-provider-aws/pkg/global" ) type Provider interface { @@ -54,12 +54,12 @@ func NewProvider(region string, iamapi iamiface.IAMAPI, cache *cache.Cache) *Def func (p *DefaultProvider) Create(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) (string, error) { tags := lo.Assign(nodeClass.Spec.Tags, map[string]string{ - fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName): "owned", - corev1beta1.ManagedByAnnotationKey: options.FromContext(ctx).ClusterName, - v1beta1.LabelNodeClass: nodeClass.Name, - v1.LabelTopologyRegion: p.region, + fmt.Sprintf("kubernetes.io/cluster/%s", global.Config.ClusterName): "owned", + corev1beta1.ManagedByAnnotationKey: global.Config.ClusterName, + v1beta1.LabelNodeClass: nodeClass.Name, + v1.LabelTopologyRegion: p.region, }) - profileName := GetProfileName(ctx, p.region, nodeClass) + profileName := GetProfileName(p.region, nodeClass) // An instance profile exists for this NodeClass if _, ok := p.cache.Get(string(nodeClass.UID)); ok { @@ -107,7 +107,7 @@ func (p *DefaultProvider) Create(ctx context.Context, nodeClass *v1beta1.EC2Node } func (p *DefaultProvider) Delete(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) error { - profileName := GetProfileName(ctx, p.region, nodeClass) + profileName := GetProfileName(p.region, nodeClass) out, err := p.iamapi.GetInstanceProfileWithContext(ctx, &iam.GetInstanceProfileInput{ InstanceProfileName: aws.String(profileName), }) @@ -134,6 +134,6 @@ func (p *DefaultProvider) Delete(ctx context.Context, nodeClass *v1beta1.EC2Node // GetProfileName gets the string for the profile name based on the cluster name and the NodeClass UUID. // The length of this string can never exceed the maximum instance profile name limit of 128 characters. -func GetProfileName(ctx context.Context, region string, nodeClass *v1beta1.EC2NodeClass) string { - return fmt.Sprintf("%s_%d", options.FromContext(ctx).ClusterName, lo.Must(hashstructure.Hash(fmt.Sprintf("%s%s", region, nodeClass.Name), hashstructure.FormatV2, nil))) +func GetProfileName(region string, nodeClass *v1beta1.EC2NodeClass) string { + return fmt.Sprintf("%s_%d", global.Config.ClusterName, lo.Must(hashstructure.Hash(fmt.Sprintf("%s%s", region, nodeClass.Name), hashstructure.FormatV2, nil))) } diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index a597f987701c..15595612bfa2 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -42,7 +42,6 @@ import ( "sigs.k8s.io/karpenter/pkg/controllers/provisioning" "sigs.k8s.io/karpenter/pkg/controllers/state" "sigs.k8s.io/karpenter/pkg/events" - coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" "sigs.k8s.io/karpenter/pkg/operator/scheme" "sigs.k8s.io/karpenter/pkg/scheduling" coretest "sigs.k8s.io/karpenter/pkg/test" @@ -53,7 +52,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" "github.com/aws/karpenter-provider-aws/pkg/fake" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" + "github.com/aws/karpenter-provider-aws/pkg/global" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" "github.com/aws/karpenter-provider-aws/pkg/providers/instancetype" "github.com/aws/karpenter-provider-aws/pkg/test" @@ -75,8 +74,6 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coreoptions.ToContext(ctx, coretest.Options()) - ctx = options.ToContext(ctx, test.Options()) awsEnv = test.NewEnvironment(ctx, env) fakeClock = &clock.FakeClock{} cloudProvider = cloudprovider.New(awsEnv.InstanceTypesProvider, awsEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), @@ -90,16 +87,14 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = coreoptions.ToContext(ctx, coretest.Options()) - ctx = options.ToContext(ctx, test.Options()) - cluster.Reset() - awsEnv.Reset() awsEnv.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") awsEnv.LaunchTemplateProvider.ClusterEndpoint = "https://test-cluster" }) var _ = AfterEach(func() { ExpectCleanedUp(ctx, env.Client) + cluster.Reset() + awsEnv.Reset() }) var _ = Describe("InstanceTypeProvider", func() { @@ -857,9 +852,7 @@ var _ = Describe("InstanceTypeProvider", func() { Context("Overhead", func() { var info *ec2.InstanceTypeInfo BeforeEach(func() { - ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ - ClusterName: lo.ToPtr("karpenter-cluster"), - })) + global.Config.ClusterName = "karpenter-cluster" var ok bool instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) @@ -975,9 +968,7 @@ var _ = Describe("InstanceTypeProvider", func() { }) Context("Eviction Thresholds", func() { BeforeEach(func() { - ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ - VMMemoryOverheadPercent: lo.ToPtr[float64](0), - })) + global.Config.VMMemoryOverheadPercent = 0 }) Context("Eviction Hard", func() { It("should override eviction threshold when specified as a quantity", func() { @@ -1433,9 +1424,7 @@ var _ = Describe("InstanceTypeProvider", func() { } }) It("should reserve ENIs when aws.reservedENIs is set and is used in max-pods calculation", func() { - ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ - ReservedENIs: lo.ToPtr(1), - })) + global.Config.ReservedENIs = 1 instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) @@ -1467,9 +1456,7 @@ var _ = Describe("InstanceTypeProvider", func() { Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", maxPods)) }) It("should reserve ENIs when aws.reservedENIs is set and not go below 0 ENIs in max-pods calculation", func() { - ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ - ReservedENIs: lo.ToPtr(1_000_000), - })) + global.Config.ReservedENIs = 1_000_000 instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) diff --git a/pkg/providers/instancetype/types.go b/pkg/providers/instancetype/types.go index 6cebec2f523b..46cf07e0bbc3 100644 --- a/pkg/providers/instancetype/types.go +++ b/pkg/providers/instancetype/types.go @@ -32,7 +32,7 @@ import ( corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" + "github.com/aws/karpenter-provider-aws/pkg/global" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" "sigs.k8s.io/karpenter/pkg/cloudprovider" @@ -62,7 +62,7 @@ func NewInstanceType(ctx context.Context, info *ec2.InstanceTypeInfo, region str Overhead: &cloudprovider.InstanceTypeOverhead{ KubeReserved: kubeReservedResources(cpu(info), pods(ctx, info, amiFamily, maxPods, podsPerCore), ENILimitedPods(ctx, info), amiFamily, kubeReserved), SystemReserved: systemReservedResources(systemReserved), - EvictionThreshold: evictionThreshold(memory(ctx, info), ephemeralStorage(info, amiFamily, blockDeviceMappings, instanceStorePolicy), amiFamily, evictionHard, evictionSoft), + EvictionThreshold: evictionThreshold(memory(info), ephemeralStorage(info, amiFamily, blockDeviceMappings, instanceStorePolicy), amiFamily, evictionHard, evictionSoft), }, } if it.Requirements.Compatible(scheduling.NewRequirements(scheduling.NewRequirement(v1.LabelOSStable, v1.NodeSelectorOpIn, string(v1.Windows)))) == nil { @@ -180,7 +180,7 @@ func computeCapacity(ctx context.Context, info *ec2.InstanceTypeInfo, amiFamily resourceList := v1.ResourceList{ v1.ResourceCPU: *cpu(info), - v1.ResourceMemory: *memory(ctx, info), + v1.ResourceMemory: *memory(info), v1.ResourceEphemeralStorage: *ephemeralStorage(info, amiFamily, blockDeviceMapping, instanceStorePolicy), v1.ResourcePods: *pods(ctx, info, amiFamily, maxPods, podsPerCore), v1beta1.ResourceAWSPodENI: *awsPodENI(aws.StringValue(info.InstanceType)), @@ -197,7 +197,7 @@ func cpu(info *ec2.InstanceTypeInfo) *resource.Quantity { return resources.Quantity(fmt.Sprint(*info.VCpuInfo.DefaultVCpus)) } -func memory(ctx context.Context, info *ec2.InstanceTypeInfo) *resource.Quantity { +func memory(info *ec2.InstanceTypeInfo) *resource.Quantity { sizeInMib := *info.MemoryInfo.SizeInMiB // Gravitons have an extra 64 MiB of cma reserved memory that we can't use if len(info.ProcessorInfo.SupportedArchitectures) > 0 && *info.ProcessorInfo.SupportedArchitectures[0] == "arm64" { @@ -205,7 +205,7 @@ func memory(ctx context.Context, info *ec2.InstanceTypeInfo) *resource.Quantity } mem := resources.Quantity(fmt.Sprintf("%dMi", sizeInMib)) // Account for VM overhead in calculation - mem.Sub(resource.MustParse(fmt.Sprintf("%dMi", int64(math.Ceil(float64(mem.Value())*options.FromContext(ctx).VMMemoryOverheadPercent/1024/1024))))) + mem.Sub(resource.MustParse(fmt.Sprintf("%dMi", int64(math.Ceil(float64(mem.Value())*global.Config.VMMemoryOverheadPercent/1024/1024))))) return mem } @@ -326,7 +326,7 @@ func ENILimitedPods(ctx context.Context, info *ec2.InstanceTypeInfo) *resource.Q // VPC CNI only uses the default network interface // https://github.com/aws/amazon-vpc-cni-k8s/blob/3294231c0dce52cfe473bf6c62f47956a3b333b6/scripts/gen_vpc_ip_limits.go#L162 networkInterfaces := *info.NetworkInfo.NetworkCards[*info.NetworkInfo.DefaultNetworkCardIndex].MaximumNetworkInterfaces - usableNetworkInterfaces := lo.Max([]int64{(networkInterfaces - int64(options.FromContext(ctx).ReservedENIs)), 0}) + usableNetworkInterfaces := lo.Max([]int64{(networkInterfaces - int64(global.Config.ReservedENIs)), 0}) if usableNetworkInterfaces == 0 { return resource.NewQuantity(0, resource.DecimalSI) } diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index 9589950f8fee..d8ef9fb1d286 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -42,7 +42,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" awserrors "github.com/aws/karpenter-provider-aws/pkg/errors" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" + "github.com/aws/karpenter-provider-aws/pkg/global" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" "github.com/aws/karpenter-provider-aws/pkg/providers/instanceprofile" "github.com/aws/karpenter-provider-aws/pkg/providers/securitygroup" @@ -184,7 +184,7 @@ func (p *DefaultProvider) createAMIOptions(ctx context.Context, nodeClass *v1bet return nil, fmt.Errorf("no security groups exist given constraints") } options := &amifamily.Options{ - ClusterName: options.FromContext(ctx).ClusterName, + ClusterName: global.Config.ClusterName, ClusterEndpoint: p.ClusterEndpoint, ClusterCIDR: p.ClusterCIDR.Load(), InstanceProfile: instanceProfile, @@ -361,7 +361,7 @@ func (p *DefaultProvider) volumeSize(quantity *resource.Quantity) *int64 { // hydrateCache queries for existing Launch Templates created by Karpenter for the current cluster and adds to the LT cache. // Any error during hydration will result in a panic func (p *DefaultProvider) hydrateCache(ctx context.Context) { - clusterName := options.FromContext(ctx).ClusterName + clusterName := global.Config.ClusterName ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("tag-key", karpenterManagedTagKey, "tag-value", clusterName)) if err := p.ec2api.DescribeLaunchTemplatesPagesWithContext(ctx, &ec2.DescribeLaunchTemplatesInput{ Filters: []*ec2.Filter{{Name: aws.String(fmt.Sprintf("tag:%s", karpenterManagedTagKey)), Values: []*string{aws.String(clusterName)}}}, @@ -410,7 +410,7 @@ func (p *DefaultProvider) getInstanceProfile(nodeClass *v1beta1.EC2NodeClass) (s } func (p *DefaultProvider) DeleteAll(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) error { - clusterName := options.FromContext(ctx).ClusterName + clusterName := global.Config.ClusterName var ltNames []*string if err := p.ec2api.DescribeLaunchTemplatesPagesWithContext(ctx, &ec2.DescribeLaunchTemplatesInput{ Filters: []*ec2.Filter{ @@ -445,7 +445,7 @@ func (p *DefaultProvider) ResolveClusterCIDR(ctx context.Context) error { return nil } out, err := p.eksapi.DescribeClusterWithContext(ctx, &eks.DescribeClusterInput{ - Name: aws.String(options.FromContext(ctx).ClusterName), + Name: aws.String(global.Config.ClusterName), }) if err != nil { return err diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index 03b36bdcd2a5..2b2422594d9f 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -48,7 +48,6 @@ import ( "sigs.k8s.io/karpenter/pkg/controllers/provisioning" "sigs.k8s.io/karpenter/pkg/controllers/state" "sigs.k8s.io/karpenter/pkg/events" - coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" . "sigs.k8s.io/karpenter/pkg/test/expectations" @@ -57,7 +56,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" "github.com/aws/karpenter-provider-aws/pkg/fake" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" + "github.com/aws/karpenter-provider-aws/pkg/global" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap/mime" @@ -82,8 +81,6 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coreoptions.ToContext(ctx, coretest.Options()) - ctx = options.ToContext(ctx, test.Options()) ctx, stop = context.WithCancel(ctx) awsEnv = test.NewEnvironment(ctx, env) @@ -100,11 +97,6 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = coreoptions.ToContext(ctx, coretest.Options()) - ctx = options.ToContext(ctx, test.Options()) - cluster.Reset() - awsEnv.Reset() - awsEnv.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") awsEnv.LaunchTemplateProvider.ClusterEndpoint = "https://test-cluster" awsEnv.LaunchTemplateProvider.CABundle = lo.ToPtr("ca-bundle") @@ -112,6 +104,8 @@ var _ = BeforeEach(func() { var _ = AfterEach(func() { ExpectCleanedUp(ctx, env.Client) + cluster.Reset() + awsEnv.Reset() }) var _ = Describe("LaunchTemplate Provider", func() { @@ -819,9 +813,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) It("should calculate memory overhead based on eni limited pods", func() { - ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ - VMMemoryOverheadPercent: lo.ToPtr[float64](0), - })) + global.Config.VMMemoryOverheadPercent = 0 nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) @@ -872,9 +864,7 @@ var _ = Describe("LaunchTemplate Provider", func() { }) It("should calculate memory overhead based on eni limited pods", func() { - ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ - VMMemoryOverheadPercent: lo.ToPtr[float64](0), - })) + global.Config.VMMemoryOverheadPercent = 0 nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) @@ -897,9 +887,7 @@ var _ = Describe("LaunchTemplate Provider", func() { Expect(overhead.Memory().String()).To(Equal("993Mi")) }) It("should calculate memory overhead based on max pods", func() { - ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ - VMMemoryOverheadPercent: lo.ToPtr[float64](0), - })) + global.Config.VMMemoryOverheadPercent = 0 nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](110)} diff --git a/pkg/providers/pricing/pricing.go b/pkg/providers/pricing/pricing.go index 242416c4a25a..0192b38f822e 100644 --- a/pkg/providers/pricing/pricing.go +++ b/pkg/providers/pricing/pricing.go @@ -25,8 +25,6 @@ import ( "sync" "time" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" @@ -38,6 +36,8 @@ import ( "knative.dev/pkg/logging" "sigs.k8s.io/karpenter/pkg/utils/pretty" + + "github.com/aws/karpenter-provider-aws/pkg/global" ) var initialOnDemandPrices = lo.Assign(InitialOnDemandPricesAWS, InitialOnDemandPricesUSGov, InitialOnDemandPricesCN) @@ -105,7 +105,7 @@ func NewAPI(sess *session.Session, region string) pricingiface.PricingAPI { return pricing.New(sess, &aws.Config{Region: aws.String(pricingAPIRegion)}) } -func NewDefaultProvider(_ context.Context, pricing pricingiface.PricingAPI, ec2Api ec2iface.EC2API, region string) *DefaultProvider { +func NewDefaultProvider(pricing pricingiface.PricingAPI, ec2Api ec2iface.EC2API, region string) *DefaultProvider { p := &DefaultProvider{ region: region, ec2: ec2Api, @@ -164,7 +164,7 @@ func (p *DefaultProvider) UpdateOnDemandPricing(ctx context.Context) error { // if we are in isolated vpc, skip updating on demand pricing // as pricing api may not be available - if options.FromContext(ctx).IsolatedVPC { + if global.Config.IsolatedVPC { if p.cm.HasChanged("on-demand-prices", nil) { logging.FromContext(ctx).Debug("running in an isolated VPC, on-demand pricing information will not be updated") } diff --git a/pkg/providers/securitygroup/suite_test.go b/pkg/providers/securitygroup/suite_test.go index 901dbcdd5d35..8bef03187b9c 100644 --- a/pkg/providers/securitygroup/suite_test.go +++ b/pkg/providers/securitygroup/suite_test.go @@ -24,10 +24,8 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis" "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/test" - coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" @@ -51,8 +49,6 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coreoptions.ToContext(ctx, coretest.Options()) - ctx = options.ToContext(ctx, test.Options()) ctx, stop = context.WithCancel(ctx) awsEnv = test.NewEnvironment(ctx, env) }) @@ -63,8 +59,6 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = coreoptions.ToContext(ctx, coretest.Options()) - ctx = options.ToContext(ctx, test.Options()) nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ Spec: v1beta1.EC2NodeClassSpec{ AMIFamily: aws.String(v1beta1.AMIFamilyAL2), @@ -84,11 +78,11 @@ var _ = BeforeEach(func() { }, }, }) - awsEnv.Reset() }) var _ = AfterEach(func() { ExpectCleanedUp(ctx, env.Client) + awsEnv.Reset() }) var _ = Describe("SecurityGroupProvider", func() { diff --git a/pkg/providers/subnet/suite_test.go b/pkg/providers/subnet/suite_test.go index 08b7354f7924..276394f1eeef 100644 --- a/pkg/providers/subnet/suite_test.go +++ b/pkg/providers/subnet/suite_test.go @@ -24,10 +24,8 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis" "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/test" - coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" @@ -51,8 +49,6 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coreoptions.ToContext(ctx, coretest.Options()) - ctx = options.ToContext(ctx, test.Options()) ctx, stop = context.WithCancel(ctx) awsEnv = test.NewEnvironment(ctx, env) }) @@ -63,8 +59,6 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = coreoptions.ToContext(ctx, coretest.Options()) - ctx = options.ToContext(ctx, test.Options()) nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ Spec: v1beta1.EC2NodeClassSpec{ AMIFamily: aws.String(v1beta1.AMIFamilyAL2), @@ -84,11 +78,11 @@ var _ = BeforeEach(func() { }, }, }) - awsEnv.Reset() }) var _ = AfterEach(func() { ExpectCleanedUp(ctx, env.Client) + awsEnv.Reset() }) var _ = Describe("SubnetProvider", func() { diff --git a/pkg/test/environment.go b/pkg/test/environment.go index 7f5c64db3674..76b7682fe979 100644 --- a/pkg/test/environment.go +++ b/pkg/test/environment.go @@ -29,6 +29,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis" awscache "github.com/aws/karpenter-provider-aws/pkg/cache" "github.com/aws/karpenter-provider-aws/pkg/fake" + "github.com/aws/karpenter-provider-aws/pkg/global" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" "github.com/aws/karpenter-provider-aws/pkg/providers/instance" "github.com/aws/karpenter-provider-aws/pkg/providers/instanceprofile" @@ -99,7 +100,7 @@ func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment fakePricingAPI := &fake.PricingAPI{} // Providers - pricingProvider := pricing.NewDefaultProvider(ctx, fakePricingAPI, ec2api, fake.DefaultRegion) + pricingProvider := pricing.NewDefaultProvider(fakePricingAPI, ec2api, fake.DefaultRegion) subnetProvider := subnet.NewDefaultProvider(ec2api, subnetCache) securityGroupProvider := securitygroup.NewDefaultProvider(ec2api, securityGroupCache) versionProvider := version.NewDefaultProvider(env.KubernetesInterface, kubernetesVersionCache) @@ -188,4 +189,9 @@ func (env *Environment) Reset() { } } } + _ = global.Initialize( + "--cluster-name", "test-cluster", + "--cluster-endpoint", "https://test-cluster", + "--vm-memory-overhead-percent", "0.075", + ) } diff --git a/pkg/test/options.go b/pkg/test/options.go deleted file mode 100644 index 4657f28e9c53..000000000000 --- a/pkg/test/options.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package test - -import ( - "fmt" - "time" - - "github.com/imdario/mergo" - "github.com/samber/lo" - - "github.com/aws/karpenter-provider-aws/pkg/operator/options" -) - -type OptionsFields struct { - AssumeRoleARN *string - AssumeRoleDuration *time.Duration - ClusterCABundle *string - ClusterName *string - ClusterEndpoint *string - IsolatedVPC *bool - VMMemoryOverheadPercent *float64 - InterruptionQueue *string - ReservedENIs *int -} - -func Options(overrides ...OptionsFields) *options.Options { - opts := OptionsFields{} - for _, override := range overrides { - if err := mergo.Merge(&opts, override, mergo.WithOverride); err != nil { - panic(fmt.Sprintf("Failed to merge settings: %s", err)) - } - } - return &options.Options{ - AssumeRoleARN: lo.FromPtrOr(opts.AssumeRoleARN, ""), - AssumeRoleDuration: lo.FromPtrOr(opts.AssumeRoleDuration, 15*time.Minute), - ClusterCABundle: lo.FromPtrOr(opts.ClusterCABundle, ""), - ClusterName: lo.FromPtrOr(opts.ClusterName, "test-cluster"), - ClusterEndpoint: lo.FromPtrOr(opts.ClusterEndpoint, "https://test-cluster"), - IsolatedVPC: lo.FromPtrOr(opts.IsolatedVPC, false), - VMMemoryOverheadPercent: lo.FromPtrOr(opts.VMMemoryOverheadPercent, 0.075), - InterruptionQueue: lo.FromPtrOr(opts.InterruptionQueue, ""), - ReservedENIs: lo.FromPtrOr(opts.ReservedENIs, 0), - } -} diff --git a/test/suites/interruption/suite_test.go b/test/suites/interruption/suite_test.go index f2adf66c1437..b7a199ba3d4f 100644 --- a/test/suites/interruption/suite_test.go +++ b/test/suites/interruption/suite_test.go @@ -19,7 +19,6 @@ import ( "testing" "time" - "github.com/samber/lo" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -32,8 +31,6 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/controllers/interruption/messages" "github.com/aws/karpenter-provider-aws/pkg/controllers/interruption/messages/scheduledchange" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" - "github.com/aws/karpenter-provider-aws/pkg/test" "github.com/aws/karpenter-provider-aws/pkg/utils" "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" @@ -57,9 +54,6 @@ func TestInterruption(t *testing.T) { } var _ = BeforeEach(func() { - env.Context = options.ToContext(env.Context, test.Options(test.OptionsFields{ - InterruptionQueue: lo.ToPtr(env.InterruptionQueue), - })) env.BeforeEach() nodeClass = env.DefaultEC2NodeClass() nodePool = env.DefaultNodePool(nodeClass) diff --git a/tools/allocatable-diff/main.go b/tools/allocatable-diff/main.go index 0a675056025a..0f20b1f7e22c 100644 --- a/tools/allocatable-diff/main.go +++ b/tools/allocatable-diff/main.go @@ -30,12 +30,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/controller-runtime/pkg/manager" + "github.com/aws/karpenter-provider-aws/pkg/operator/options" + corecloudprovider "sigs.k8s.io/karpenter/pkg/cloudprovider" coreoperator "sigs.k8s.io/karpenter/pkg/operator" "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" "github.com/aws/karpenter-provider-aws/pkg/operator" - "github.com/aws/karpenter-provider-aws/pkg/operator/options" ) var clusterName string