add options

hack/code/prices_gen.go

@@ -34,7 +34,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
-	"github.com/aws/karpenter/pkg/apis/settings"
+	awsoptions "github.com/aws/karpenter/pkg/operator/options"
 	"github.com/aws/karpenter/pkg/providers/pricing"
 	"github.com/aws/karpenter/pkg/test"
 )
@@ -93,7 +93,7 @@ func main() {
 	os.Setenv("AWS_SDK_LOAD_CONFIG", "true")
 	os.Setenv("AWS_REGION", region)
 	ctx := context.Background()
-	ctx = settings.ToContext(ctx, test.Settings())
+	ctx = awsoptions.ToContext(ctx, test.Options())
 	sess := session.Must(session.NewSession())
 	ec2 := ec22.New(sess)
 	src := &bytes.Buffer{}

hack/docs/configuration_gen_docs.go

@@ -21,7 +21,8 @@ import (
 	"os"
 	"strings"
 
-	"github.com/aws/karpenter-core/pkg/operator/options"
+	coreoptions "github.com/aws/karpenter-core/pkg/operator/options"
+	"github.com/aws/karpenter/pkg/operator/options"
 )
 
 func main() {
@@ -48,17 +49,20 @@ func main() {
 	topDoc := fmt.Sprintf("%s%s\n\n", startDocSections[0], genStart)
 	bottomDoc := fmt.Sprintf("\n%s%s", genEnd, endDocSections[1])
 
+	coreopts := coreoptions.New()
 	opts := options.New()
 
 	envVarsBlock := "| Environment Variable | CLI Flag | Description |\n"
 	envVarsBlock += "|--|--|--|\n"
-	opts.FlagSet.VisitAll(func(f *flag.Flag) {
+	addFlagEntry := func(f *flag.Flag) {
 		if f.DefValue == "" {
 			envVarsBlock += fmt.Sprintf("| %s | %s | %s|\n", strings.ReplaceAll(strings.ToUpper(f.Name), "-", "_"), "\\-\\-"+f.Name, f.Usage)
 		} else {
 			envVarsBlock += fmt.Sprintf("| %s | %s | %s (default = %s)|\n", strings.ReplaceAll(strings.ToUpper(f.Name), "-", "_"), "\\-\\-"+f.Name, f.Usage, f.DefValue)
 		}
-	})
+	}
+	coreopts.FlagSet.VisitAll(addFlagEntry)
+	opts.VisitAll(addFlagEntry)
 
 	log.Println("writing output to", outputFileName)
 	f, err := os.Create(outputFileName)

hack/docs/instancetypes_gen_docs.go

@@ -35,14 +35,14 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 
-	coresettings "github.com/aws/karpenter-core/pkg/apis/settings"
 	"github.com/aws/karpenter-core/pkg/apis/v1alpha5"
 	coreoperator "github.com/aws/karpenter-core/pkg/operator"
+	coreoptions "github.com/aws/karpenter-core/pkg/operator/options"
 	coretest "github.com/aws/karpenter-core/pkg/test"
 	nodepoolutil "github.com/aws/karpenter-core/pkg/utils/nodepool"
-	"github.com/aws/karpenter/pkg/apis/settings"
 	awscloudprovider "github.com/aws/karpenter/pkg/cloudprovider"
 	"github.com/aws/karpenter/pkg/operator"
+	awsoptions "github.com/aws/karpenter/pkg/operator/options"
 	"github.com/aws/karpenter/pkg/test"
 
 	"github.com/aws/karpenter-core/pkg/cloudprovider"
@@ -77,8 +77,8 @@ func main() {
 	lo.Must0(os.Setenv("AWS_SDK_LOAD_CONFIG", "true"))
 	lo.Must0(os.Setenv("AWS_REGION", "us-east-1"))
 
-	ctx := coresettings.ToContext(context.Background(), coretest.Settings())
-	ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{
+	ctx := coreoptions.ToContext(context.Background(), coretest.Options())
+	ctx = awsoptions.ToContext(ctx, test.Options(test.OptionsFields{
 		ClusterName:     lo.ToPtr("docs-gen"),
 		ClusterEndpoint: lo.ToPtr("https://docs-gen.aws"),
 		IsolatedVPC:     lo.ToPtr(true), // disable pricing lookup

pkg/apis/apis.go

@@ -24,11 +24,13 @@ import (
 	"github.com/aws/karpenter-core/pkg/operator/scheme"
 	"github.com/aws/karpenter/pkg/apis/settings"
 	"github.com/aws/karpenter/pkg/apis/v1beta1"
+	"github.com/aws/karpenter/pkg/operator/options"
 
 	"github.com/samber/lo"
 
 	"github.com/aws/karpenter-core/pkg/apis"
 	coresettings "github.com/aws/karpenter-core/pkg/apis/settings"
+	coreoptions "github.com/aws/karpenter-core/pkg/operator/options"
 	"github.com/aws/karpenter-core/pkg/utils/functional"
 	"github.com/aws/karpenter/pkg/apis/v1alpha1"
 )
@@ -42,6 +44,7 @@ var (
 	// AddToScheme may be used to add all resources defined in the project to a Scheme
 	AddToScheme = Builder.AddToScheme
 	Settings    = []coresettings.Injectable{&settings.Settings{}}
+	Options     = []coreoptions.Injectable{&options.Options{}}
 )
 
 //go:generate controller-gen crd object:headerFile="../../hack/boilerplate.go.txt" paths="./..." output:crd:artifacts:config=crds

pkg/apis/settings/settings.go

@@ -22,6 +22,8 @@ import (
 
 	v1 "k8s.io/api/core/v1"
 	"knative.dev/pkg/configmap"
+
+	coresettings "github.com/aws/karpenter-core/pkg/apis/settings"
 )
 
 type settingsKeyType struct{}
@@ -92,15 +94,18 @@ func (*Settings) Inject(ctx context.Context, cm *v1.ConfigMap) (context.Context,
 	return ToContext(ctx, s), nil
 }
 
+func (*Settings) FromContext(ctx context.Context) coresettings.Injectable {
+	return FromContext(ctx)
+}
+
 func ToContext(ctx context.Context, s *Settings) context.Context {
 	return context.WithValue(ctx, ContextKey, s)
 }
 
 func FromContext(ctx context.Context) *Settings {
 	data := ctx.Value(ContextKey)
 	if data == nil {
-		// This is developer error if this happens, so we should panic
-		panic("settings doesn't exist in context")
+		return nil
 	}
 	return data.(*Settings)
 }

pkg/cloudprovider/cloudprovider.go

@@ -63,6 +63,7 @@ func init() {
 	v1alpha5.NormalizedLabels = lo.Assign(v1alpha5.NormalizedLabels, map[string]string{"topology.ebs.csi.aws.com/zone": v1.LabelTopologyZone})
 	corev1beta1.NormalizedLabels = lo.Assign(corev1beta1.NormalizedLabels, map[string]string{"topology.ebs.csi.aws.com/zone": v1.LabelTopologyZone})
 	coreapis.Settings = append(coreapis.Settings, apis.Settings...)
+	coreapis.Options = append(coreapis.Options, apis.Options...)
 }
 
 var _ cloudprovider.CloudProvider = (*CloudProvider)(nil)

pkg/cloudprovider/suite_test.go

@@ -30,23 +30,22 @@ import (
 
 	"github.com/aws/karpenter/pkg/apis"
 	"github.com/aws/karpenter/pkg/apis/settings"
+	"github.com/aws/karpenter/pkg/operator/options"
 	"github.com/aws/karpenter/pkg/test"
 
 	"github.com/aws/karpenter/pkg/cloudprovider"
 
-	coresettings "github.com/aws/karpenter-core/pkg/apis/settings"
 	"github.com/aws/karpenter-core/pkg/controllers/provisioning"
 	"github.com/aws/karpenter-core/pkg/controllers/state"
 	"github.com/aws/karpenter-core/pkg/events"
-	"github.com/aws/karpenter-core/pkg/operator/options"
+	coreoptions "github.com/aws/karpenter-core/pkg/operator/options"
 	"github.com/aws/karpenter-core/pkg/operator/scheme"
 	coretest "github.com/aws/karpenter-core/pkg/test"
 	. "github.com/aws/karpenter-core/pkg/test/expectations"
 )
 
 var ctx context.Context
 var stop context.CancelFunc
-var opts *options.Options
 var env *coretest.Environment
 var awsEnv *test.Environment
 var prov *provisioning.Provisioner
@@ -63,7 +62,8 @@ func TestAWS(t *testing.T) {
 
 var _ = BeforeSuite(func() {
 	env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...))
-	ctx = coresettings.ToContext(ctx, coretest.Settings())
+	ctx = coreoptions.ToContext(ctx, coretest.Options())
+	ctx = options.ToContext(ctx, test.Options())
 	ctx = settings.ToContext(ctx, test.Settings())
 	ctx, stop = context.WithCancel(ctx)
 	awsEnv = test.NewEnvironment(ctx, env)
@@ -81,8 +81,8 @@ var _ = AfterSuite(func() {
 })
 
 var _ = BeforeEach(func() {
-	ctx = options.ToContext(ctx, opts)
-	ctx = coresettings.ToContext(ctx, coretest.Settings())
+	ctx = coreoptions.ToContext(ctx, coretest.Options())
+	ctx = options.ToContext(ctx, test.Options())
 	ctx = settings.ToContext(ctx, test.Settings())
 
 	cluster.Reset()

pkg/controllers/controllers.go

@@ -24,13 +24,13 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/aws/karpenter-core/pkg/events"
-	"github.com/aws/karpenter/pkg/apis/settings"
 	"github.com/aws/karpenter/pkg/cache"
 	"github.com/aws/karpenter/pkg/cloudprovider"
 	"github.com/aws/karpenter/pkg/controllers/interruption"
 	nodeclaimgarbagecollection "github.com/aws/karpenter/pkg/controllers/nodeclaim/garbagecollection"
 	nodeclaimlink "github.com/aws/karpenter/pkg/controllers/nodeclaim/link"
 	"github.com/aws/karpenter/pkg/controllers/nodeclass"
+	"github.com/aws/karpenter/pkg/operator/options"
 	"github.com/aws/karpenter/pkg/providers/amifamily"
 	"github.com/aws/karpenter/pkg/providers/instanceprofile"
 	"github.com/aws/karpenter/pkg/providers/pricing"
@@ -54,10 +54,10 @@ func NewControllers(ctx context.Context, sess *session.Session, clk clock.Clock,
 		linkController,
 		nodeclaimgarbagecollection.NewController(kubeClient, cloudProvider, linkController),
 	}
-	if settings.FromContext(ctx).InterruptionQueueName != "" {
+	if options.FromContext(ctx).InterruptionQueueName != "" {
 		controllers = append(controllers, interruption.NewController(kubeClient, clk, recorder, interruption.NewSQSProvider(sqs.New(sess)), unavailableOfferings))
 	}
-	if settings.FromContext(ctx).IsolatedVPC {
+	if options.FromContext(ctx).IsolatedVPC {
 		logging.FromContext(ctx).Infof("assuming isolated VPC, pricing information will not be updated")
 	} else {
 		controllers = append(controllers, pricing.NewController(pricingProvider))

pkg/controllers/interruption/controller.go

@@ -32,12 +32,12 @@ import (
 
 	"github.com/aws/karpenter-core/pkg/apis/v1beta1"
 	"github.com/aws/karpenter-core/pkg/utils/pretty"
-	"github.com/aws/karpenter/pkg/apis/settings"
 	"github.com/aws/karpenter/pkg/apis/v1alpha1"
 	"github.com/aws/karpenter/pkg/cache"
 	interruptionevents "github.com/aws/karpenter/pkg/controllers/interruption/events"
 	"github.com/aws/karpenter/pkg/controllers/interruption/messages"
 	"github.com/aws/karpenter/pkg/controllers/interruption/messages/statechange"
+	"github.com/aws/karpenter/pkg/operator/options"
 	"github.com/aws/karpenter/pkg/utils"
 
 	"github.com/aws/karpenter-core/pkg/events"
@@ -80,8 +80,8 @@ func NewController(kubeClient client.Client, clk clock.Clock, recorder events.Re
 }
 
 func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
-	ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("queue", settings.FromContext(ctx).InterruptionQueueName))
-	if c.cm.HasChanged(settings.FromContext(ctx).InterruptionQueueName, nil) {
+	ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("queue", options.FromContext(ctx).InterruptionQueueName))
+	if c.cm.HasChanged(options.FromContext(ctx).InterruptionQueueName, nil) {
 		logging.FromContext(ctx).Debugf("watching interruption queue")
 	}
 	sqsMessages, err := c.sqsProvider.GetSQSMessages(ctx)

pkg/controllers/interruption/interruption_benchmark_test.go

@@ -43,14 +43,15 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/aws/karpenter-core/pkg/operator/scheme"
+	"github.com/aws/karpenter/pkg/operator/options"
 	"github.com/aws/karpenter/pkg/apis/settings"
 	awscache "github.com/aws/karpenter/pkg/cache"
 	"github.com/aws/karpenter/pkg/controllers/interruption"
 	"github.com/aws/karpenter/pkg/controllers/interruption/events"
 	"github.com/aws/karpenter/pkg/fake"
 	"github.com/aws/karpenter/pkg/test"
 
-	coresettings "github.com/aws/karpenter-core/pkg/apis/settings"
+	coreoptions "github.com/aws/karpenter-core/pkg/operator/options"
 	"github.com/aws/karpenter-core/pkg/apis/v1alpha5"
 	coretest "github.com/aws/karpenter-core/pkg/test"
 )
@@ -77,12 +78,13 @@ func BenchmarkNotification100(b *testing.B) {
 func benchmarkNotificationController(b *testing.B, messageCount int) {
 	ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("message-count", messageCount))
 	fakeClock = &clock.FakeClock{}
-	ctx = coresettings.ToContext(ctx, coretest.Settings())
-	ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{
+	ctx = coreoptions.ToContext(ctx, coretest.Options())
+	ctx = options.ToContext(ctx, test.Options(test.OptionsFields{
 		ClusterName:           lo.ToPtr("karpenter-notification-benchmarking"),
 		IsolatedVPC:           lo.ToPtr(true),
 		InterruptionQueueName: lo.ToPtr("test-cluster"),
 	}))
+	ctx = settings.ToContext(ctx, test.Settings())
 	env = coretest.NewEnvironment(scheme.Scheme)
 	// Stop the coretest environment after the coretest completes
 	defer func() {
@@ -181,7 +183,7 @@ func newProviders(kubeClient client.Client) providerSet {
 
 func (p *providerSet) makeInfrastructure(ctx context.Context) error {
 	if _, err := p.sqsAPI.CreateQueueWithContext(ctx, &sqs.CreateQueueInput{
-		QueueName: lo.ToPtr(settings.FromContext(ctx).InterruptionQueueName),
+		QueueName: lo.ToPtr(options.FromContext(ctx).InterruptionQueueName),
 		Attributes: map[string]*string{
 			sqs.QueueAttributeNameMessageRetentionPeriod: aws.String("1200"), // 20 minutes for this test
 		},

pkg/controllers/interruption/sqs.go

@@ -26,8 +26,8 @@ import (
 	"github.com/samber/lo"
 
 	"github.com/aws/karpenter-core/pkg/utils/atomic"
-	"github.com/aws/karpenter/pkg/apis/settings"
 	awserrors "github.com/aws/karpenter/pkg/errors"
+	"github.com/aws/karpenter/pkg/operator/options"
 )
 
 type SQSProvider struct {
@@ -43,7 +43,7 @@ func NewSQSProvider(client sqsiface.SQSAPI) *SQSProvider {
 	}
 	provider.queueURL.Resolve = func(ctx context.Context) (string, error) {
 		input := &sqs.GetQueueUrlInput{
-			QueueName: aws.String(settings.FromContext(ctx).InterruptionQueueName),
+			QueueName: aws.String(options.FromContext(ctx).InterruptionQueueName),
 		}
 		ret, err := provider.client.GetQueueUrlWithContext(ctx, input)
 		if err != nil {
@@ -66,12 +66,12 @@ func (s *SQSProvider) QueueExists(ctx context.Context) (bool, error) {
 }
 
 func (s *SQSProvider) DiscoverQueueURL(ctx context.Context) (string, error) {
-	if settings.FromContext(ctx).InterruptionQueueName != lo.FromPtr(s.queueName.Load()) {
+	if options.FromContext(ctx).InterruptionQueueName != lo.FromPtr(s.queueName.Load()) {
 		res, err := s.queueURL.TryGet(ctx, atomic.IgnoreCacheOption)
 		if err != nil {
 			return res, err
 		}
-		s.queueName.Store(lo.ToPtr(settings.FromContext(ctx).InterruptionQueueName))
+		s.queueName.Store(lo.ToPtr(options.FromContext(ctx).InterruptionQueueName))
 		return res, nil
 	}
 	return s.queueURL.TryGet(ctx)

pkg/controllers/interruption/suite_test.go

@@ -37,10 +37,10 @@ import (
 	_ "knative.dev/pkg/system/testing"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	coresettings "github.com/aws/karpenter-core/pkg/apis/settings"
 	"github.com/aws/karpenter-core/pkg/apis/v1alpha5"
 	corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1"
 	"github.com/aws/karpenter-core/pkg/events"
+	coreoptions "github.com/aws/karpenter-core/pkg/operator/options"
 	"github.com/aws/karpenter-core/pkg/operator/scheme"
 	coretest "github.com/aws/karpenter-core/pkg/test"
 	. "github.com/aws/karpenter-core/pkg/test/expectations"
@@ -53,6 +53,7 @@ import (
 	"github.com/aws/karpenter/pkg/controllers/interruption/messages/spotinterruption"
 	"github.com/aws/karpenter/pkg/controllers/interruption/messages/statechange"
 	"github.com/aws/karpenter/pkg/fake"
+	"github.com/aws/karpenter/pkg/operator/options"
 	"github.com/aws/karpenter/pkg/test"
 	"github.com/aws/karpenter/pkg/utils"
 )
@@ -91,10 +92,11 @@ var _ = AfterSuite(func() {
 })
 
 var _ = BeforeEach(func() {
-	ctx = coresettings.ToContext(ctx, coretest.Settings())
-	ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{
+	ctx = coreoptions.ToContext(ctx, coretest.Options())
+	ctx = options.ToContext(ctx, test.Options(test.OptionsFields{
 		InterruptionQueueName: lo.ToPtr("test-cluster"),
 	}))
+	ctx = settings.ToContext(ctx, test.Settings())
 	unavailableOfferingsCache.Flush()
 	sqsapi.Reset()
 	sqsProvider.Reset()