From be3754fd5044f7dc9a4b85b95d0bb85a271bbc65 Mon Sep 17 00:00:00 2001 From: Max Cao Date: Thu, 3 Oct 2024 10:47:41 -0700 Subject: [PATCH 01/12] Add BoundServiceAccountToken trigger authentication type Signed-off-by: Max Cao --- .../v1alpha1/triggerauthentication_types.go | 9 + apis/keda/v1alpha1/zz_generated.deepcopy.go | 20 ++ cmd/operator/main.go | 14 +- ...keda.sh_clustertriggerauthentications.yaml | 15 ++ .../bases/keda.sh_triggerauthentications.yaml | 15 ++ config/rbac/role.yaml | 7 + controllers/keda/scaledjob_controller.go | 7 +- controllers/keda/scaledobject_controller.go | 1 + go.mod | 2 +- pkg/eventemitter/eventemitter.go | 10 +- .../authentication/authentication_helpers.go | 9 + pkg/scaling/resolver/scale_resolvers.go | 226 +++++++++++++++++- pkg/scaling/resolver/scale_resolvers_test.go | 8 +- pkg/scaling/scale_handler.go | 8 +- pkg/scaling/scalers_builder.go | 4 +- 15 files changed, 325 insertions(+), 30 deletions(-) diff --git a/apis/keda/v1alpha1/triggerauthentication_types.go b/apis/keda/v1alpha1/triggerauthentication_types.go index 0b0d9ffa315..aa85def04b8 100644 --- a/apis/keda/v1alpha1/triggerauthentication_types.go +++ b/apis/keda/v1alpha1/triggerauthentication_types.go @@ -95,6 +95,9 @@ type TriggerAuthenticationSpec struct { // +optional AwsSecretManager *AwsSecretManager `json:"awsSecretManager,omitempty"` + + // +optional + BoundServiceAccountToken []BoundServiceAccountToken `json:"boundServiceAccountToken,omitempty"` } // TriggerAuthenticationStatus defines the observed state of TriggerAuthentication @@ -378,6 +381,12 @@ type AwsSecretManagerSecret struct { VersionStage string `json:"versionStage,omitempty"` } +type BoundServiceAccountToken struct { + Parameter string `json:"parameter"` + ServiceAccountName string `json:"serviceAccountName"` + Expiry string `json:"expiry"` +} + func init() { SchemeBuilder.Register(&ClusterTriggerAuthentication{}, &ClusterTriggerAuthenticationList{}) SchemeBuilder.Register(&TriggerAuthentication{}, &TriggerAuthenticationList{}) diff --git a/apis/keda/v1alpha1/zz_generated.deepcopy.go b/apis/keda/v1alpha1/zz_generated.deepcopy.go index 70b40bac583..5248166f09f 100755 --- a/apis/keda/v1alpha1/zz_generated.deepcopy.go +++ b/apis/keda/v1alpha1/zz_generated.deepcopy.go @@ -354,6 +354,21 @@ func (in *AzureKeyVaultSecret) DeepCopy() *AzureKeyVaultSecret { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BoundServiceAccountToken) DeepCopyInto(out *BoundServiceAccountToken) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BoundServiceAccountToken. +func (in *BoundServiceAccountToken) DeepCopy() *BoundServiceAccountToken { + if in == nil { + return nil + } + out := new(BoundServiceAccountToken) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterTriggerAuthentication) DeepCopyInto(out *ClusterTriggerAuthentication) { *out = *in @@ -1202,6 +1217,11 @@ func (in *TriggerAuthenticationSpec) DeepCopyInto(out *TriggerAuthenticationSpec *out = new(AwsSecretManager) (*in).DeepCopyInto(*out) } + if in.BoundServiceAccountToken != nil { + in, out := &in.BoundServiceAccountToken, &out.BoundServiceAccountToken + *out = make([]BoundServiceAccountToken, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TriggerAuthenticationSpec. diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 1bdaed01954..389ccca8317 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -46,6 +46,7 @@ import ( "github.com/kedacore/keda/v2/pkg/k8s" "github.com/kedacore/keda/v2/pkg/metricscollector" "github.com/kedacore/keda/v2/pkg/metricsservice" + "github.com/kedacore/keda/v2/pkg/scalers/authentication" "github.com/kedacore/keda/v2/pkg/scaling" kedautil "github.com/kedacore/keda/v2/pkg/util" //+kubebuilder:scaffold:imports @@ -225,8 +226,14 @@ func main() { os.Exit(1) } - scaledHandler := scaling.NewScaleHandler(mgr.GetClient(), scaleClient, mgr.GetScheme(), globalHTTPTimeout, eventRecorder, secretInformer.Lister()) - eventEmitter := eventemitter.NewEventEmitter(mgr.GetClient(), eventRecorder, k8sClusterName, secretInformer.Lister()) + authClientSet := &authentication.AuthClientSet{ + TokenReviewInterface: kubeClientset.AuthenticationV1().TokenReviews(), + CoreV1Interface: kubeClientset.CoreV1(), + SecretLister: secretInformer.Lister(), + } + + scaledHandler := scaling.NewScaleHandler(mgr.GetClient(), scaleClient, mgr.GetScheme(), globalHTTPTimeout, eventRecorder, authClientSet) + eventEmitter := eventemitter.NewEventEmitter(mgr.GetClient(), eventRecorder, k8sClusterName, authClientSet) if err = (&kedacontrollers.ScaledObjectReconciler{ Client: mgr.GetClient(), @@ -245,8 +252,7 @@ func main() { Scheme: mgr.GetScheme(), GlobalHTTPTimeout: globalHTTPTimeout, EventEmitter: eventEmitter, - SecretsLister: secretInformer.Lister(), - SecretsSynced: secretInformer.Informer().HasSynced, + AuthClientSet: authClientSet, }).SetupWithManager(mgr, controller.Options{ MaxConcurrentReconciles: scaledJobMaxReconciles, }); err != nil { diff --git a/config/crd/bases/keda.sh_clustertriggerauthentications.yaml b/config/crd/bases/keda.sh_clustertriggerauthentications.yaml index d8a74647ffc..fcc28494184 100644 --- a/config/crd/bases/keda.sh_clustertriggerauthentications.yaml +++ b/config/crd/bases/keda.sh_clustertriggerauthentications.yaml @@ -302,6 +302,21 @@ spec: - secrets - vaultUri type: object + boundServiceAccountToken: + items: + properties: + expiry: + type: string + parameter: + type: string + serviceAccountName: + type: string + required: + - expiry + - parameter + - serviceAccountName + type: object + type: array configMapTargetRef: items: description: AuthConfigMapTargetRef is used to authenticate using diff --git a/config/crd/bases/keda.sh_triggerauthentications.yaml b/config/crd/bases/keda.sh_triggerauthentications.yaml index 9c38fa2ada4..3a7eaea02c8 100644 --- a/config/crd/bases/keda.sh_triggerauthentications.yaml +++ b/config/crd/bases/keda.sh_triggerauthentications.yaml @@ -301,6 +301,21 @@ spec: - secrets - vaultUri type: object + boundServiceAccountToken: + items: + properties: + expiry: + type: string + parameter: + type: string + serviceAccountName: + type: string + required: + - expiry + - parameter + - serviceAccountName + type: object + type: array configMapTargetRef: items: description: AuthConfigMapTargetRef is used to authenticate using diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 87362c08e13..a056f3111c1 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -32,6 +32,13 @@ rules: verbs: - list - watch +- apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create + - get - apiGroups: - '*' resources: diff --git a/controllers/keda/scaledjob_controller.go b/controllers/keda/scaledjob_controller.go index 386ff71ab25..6cdc1f7fa90 100755 --- a/controllers/keda/scaledjob_controller.go +++ b/controllers/keda/scaledjob_controller.go @@ -29,7 +29,6 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - corev1listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -45,6 +44,7 @@ import ( "github.com/kedacore/keda/v2/pkg/eventemitter" "github.com/kedacore/keda/v2/pkg/eventreason" "github.com/kedacore/keda/v2/pkg/metricscollector" + "github.com/kedacore/keda/v2/pkg/scalers/authentication" "github.com/kedacore/keda/v2/pkg/scaling" kedastatus "github.com/kedacore/keda/v2/pkg/status" "github.com/kedacore/keda/v2/pkg/util" @@ -59,11 +59,10 @@ type ScaledJobReconciler struct { Scheme *runtime.Scheme GlobalHTTPTimeout time.Duration EventEmitter eventemitter.EventHandler + AuthClientSet *authentication.AuthClientSet scaledJobGenerations *sync.Map scaleHandler scaling.ScaleHandler - SecretsLister corev1listers.SecretLister - SecretsSynced cache.InformerSynced } type scaledJobMetricsData struct { @@ -83,7 +82,7 @@ func init() { // SetupWithManager initializes the ScaledJobReconciler instance and starts a new controller managed by the passed Manager instance. func (r *ScaledJobReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error { - r.scaleHandler = scaling.NewScaleHandler(mgr.GetClient(), nil, mgr.GetScheme(), r.GlobalHTTPTimeout, mgr.GetEventRecorderFor("scale-handler"), r.SecretsLister) + r.scaleHandler = scaling.NewScaleHandler(mgr.GetClient(), nil, mgr.GetScheme(), r.GlobalHTTPTimeout, mgr.GetEventRecorderFor("scale-handler"), r.AuthClientSet) r.scaledJobGenerations = &sync.Map{} return ctrl.NewControllerManagedBy(mgr). WithOptions(options). diff --git a/controllers/keda/scaledobject_controller.go b/controllers/keda/scaledobject_controller.go index c480dc380c4..21c88e6757c 100755 --- a/controllers/keda/scaledobject_controller.go +++ b/controllers/keda/scaledobject_controller.go @@ -58,6 +58,7 @@ import ( // +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;update;patch;create;delete // +kubebuilder:rbac:groups="",resources=configmaps;configmaps/status,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch +// +kubebuilder:rbac:groups="",resources=serviceaccounts/token,verbs=create;get // +kubebuilder:rbac:groups="",resources=pods;services;services;secrets;external,verbs=get;list;watch // +kubebuilder:rbac:groups="*",resources="*/scale",verbs=get;list;watch;update;patch // +kubebuilder:rbac:groups="",resources="serviceaccounts",verbs=list;watch diff --git a/go.mod b/go.mod index 5fc7977aee5..2e248c4eaaf 100644 --- a/go.mod +++ b/go.mod @@ -250,7 +250,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang-jwt/jwt/v4 v4.5.1 // indirect - github.com/golang-jwt/jwt/v5 v5.2.1 // indirect + github.com/golang-jwt/jwt/v5 v5.2.1 github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect diff --git a/pkg/eventemitter/eventemitter.go b/pkg/eventemitter/eventemitter.go index 91d57f3ca4d..4624018735f 100644 --- a/pkg/eventemitter/eventemitter.go +++ b/pkg/eventemitter/eventemitter.go @@ -36,7 +36,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - corev1listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -44,6 +43,7 @@ import ( eventingv1alpha1 "github.com/kedacore/keda/v2/apis/eventing/v1alpha1" "github.com/kedacore/keda/v2/pkg/eventemitter/eventdata" "github.com/kedacore/keda/v2/pkg/metricscollector" + "github.com/kedacore/keda/v2/pkg/scalers/authentication" "github.com/kedacore/keda/v2/pkg/scaling/resolver" kedastatus "github.com/kedacore/keda/v2/pkg/status" ) @@ -66,7 +66,7 @@ type EventEmitter struct { eventFilterCacheLock *sync.RWMutex eventLoopContexts *sync.Map cloudEventProcessingChan chan eventdata.EventData - secretsLister corev1listers.SecretLister + authClientSet *authentication.AuthClientSet } // EventHandler defines the behavior for EventEmitter clients @@ -96,7 +96,7 @@ const ( ) // NewEventEmitter creates a new EventEmitter -func NewEventEmitter(client client.Client, recorder record.EventRecorder, clusterName string, secretsLister corev1listers.SecretLister) EventHandler { +func NewEventEmitter(client client.Client, recorder record.EventRecorder, clusterName string, authClientSet *authentication.AuthClientSet) EventHandler { return &EventEmitter{ log: logf.Log.WithName("event_emitter"), client: client, @@ -108,7 +108,7 @@ func NewEventEmitter(client client.Client, recorder record.EventRecorder, cluste eventFilterCacheLock: &sync.RWMutex{}, eventLoopContexts: &sync.Map{}, cloudEventProcessingChan: make(chan eventdata.EventData, maxChannelBuffer), - secretsLister: secretsLister, + authClientSet: authClientSet, } } @@ -188,7 +188,7 @@ func (e *EventEmitter) createEventHandlers(ctx context.Context, cloudEventSource } // Resolve auth related - authParams, podIdentity, err := resolver.ResolveAuthRefAndPodIdentity(ctx, e.client, e.log, spec.AuthenticationRef, nil, cloudEventSourceI.GetNamespace(), e.secretsLister) + authParams, podIdentity, err := resolver.ResolveAuthRefAndPodIdentity(ctx, e.client, e.log, spec.AuthenticationRef, nil, cloudEventSourceI.GetNamespace(), e.authClientSet) if err != nil { e.log.Error(err, "error resolving auth params", "cloudEventSource", cloudEventSourceI) return diff --git a/pkg/scalers/authentication/authentication_helpers.go b/pkg/scalers/authentication/authentication_helpers.go index a8e8255122d..f8ac658e01c 100644 --- a/pkg/scalers/authentication/authentication_helpers.go +++ b/pkg/scalers/authentication/authentication_helpers.go @@ -13,10 +13,19 @@ import ( libs "github.com/dysnix/predictkube-libs/external/configs" "github.com/dysnix/predictkube-libs/external/http_transport" pConfig "github.com/prometheus/common/config" + authenticationv1client "k8s.io/client-go/kubernetes/typed/authentication/v1" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + corev1listers "k8s.io/client-go/listers/core/v1" kedautil "github.com/kedacore/keda/v2/pkg/util" ) +type AuthClientSet struct { + authenticationv1client.TokenReviewInterface + corev1client.CoreV1Interface + corev1listers.SecretLister +} + const ( AuthModesKey = "authModes" ) diff --git a/pkg/scaling/resolver/scale_resolvers.go b/pkg/scaling/resolver/scale_resolvers.go index 6ca40672fc6..08cd98603bc 100644 --- a/pkg/scaling/resolver/scale_resolvers.go +++ b/pkg/scaling/resolver/scale_resolvers.go @@ -19,22 +19,31 @@ package resolver import ( "bytes" "context" + "encoding/base64" + "errors" "fmt" "strconv" "strings" + "time" + "github.com/aws/smithy-go/ptr" "github.com/go-logr/logr" + "github.com/golang-jwt/jwt/v5" appsv1 "k8s.io/api/apps/v1" + authenticationv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" corev1listers "k8s.io/client-go/listers/core/v1" + "knative.dev/pkg/apis/duck" duckv1 "knative.dev/pkg/apis/duck/v1" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + "github.com/kedacore/keda/v2/pkg/scalers/authentication" "github.com/kedacore/keda/v2/pkg/util" ) @@ -47,6 +56,19 @@ const ( defaultServiceAccount = "default" ) +type triggerAuthType interface { + SetAnnotations(map[string]string) + GetAnnotations() map[string]string +} + +type tokenStatus int + +const ( + tokenStatusValid tokenStatus = iota + tokenStatusInvalid + tokenStatusUnknown +) + var ( kedaNamespace, _ = util.GetClusterObjectNamespace() restrictSecretAccess = util.GetRestrictSecretAccess() @@ -178,9 +200,9 @@ func ResolveContainerEnv(ctx context.Context, client client.Client, logger logr. // ResolveAuthRefAndPodIdentity provides authentication parameters and pod identity needed authenticate scaler with the environment. func ResolveAuthRefAndPodIdentity(ctx context.Context, client client.Client, logger logr.Logger, triggerAuthRef *kedav1alpha1.AuthenticationRef, podTemplateSpec *corev1.PodTemplateSpec, - namespace string, secretsLister corev1listers.SecretLister) (map[string]string, kedav1alpha1.AuthPodIdentity, error) { + namespace string, authClientSet *authentication.AuthClientSet) (map[string]string, kedav1alpha1.AuthPodIdentity, error) { if podTemplateSpec != nil { - authParams, podIdentity, err := resolveAuthRef(ctx, client, logger, triggerAuthRef, &podTemplateSpec.Spec, namespace, secretsLister) + authParams, podIdentity, err := resolveAuthRef(ctx, client, logger, triggerAuthRef, &podTemplateSpec.Spec, namespace, authClientSet) if err != nil { return authParams, podIdentity, err @@ -220,14 +242,14 @@ func ResolveAuthRefAndPodIdentity(ctx context.Context, client client.Client, log return authParams, podIdentity, nil } - return resolveAuthRef(ctx, client, logger, triggerAuthRef, nil, namespace, secretsLister) + return resolveAuthRef(ctx, client, logger, triggerAuthRef, nil, namespace, authClientSet) } // resolveAuthRef provides authentication parameters needed authenticate scaler with the environment. // based on authentication method defined in TriggerAuthentication, authParams and podIdentity is returned func resolveAuthRef(ctx context.Context, client client.Client, logger logr.Logger, triggerAuthRef *kedav1alpha1.AuthenticationRef, podSpec *corev1.PodSpec, - namespace string, secretsLister corev1listers.SecretLister) (map[string]string, kedav1alpha1.AuthPodIdentity, error) { + namespace string, authClientSet *authentication.AuthClientSet) (map[string]string, kedav1alpha1.AuthPodIdentity, error) { result := make(map[string]string) podIdentity := kedav1alpha1.AuthPodIdentity{Provider: kedav1alpha1.PodIdentityProviderNone} var err error @@ -246,7 +268,7 @@ func resolveAuthRef(ctx context.Context, client client.Client, logger logr.Logge result[e.Parameter] = "" continue } - env, err := ResolveContainerEnv(ctx, client, logger, podSpec, e.ContainerName, namespace, secretsLister) + env, err := ResolveContainerEnv(ctx, client, logger, podSpec, e.ContainerName, namespace, authClientSet.SecretLister) if err != nil { result[e.Parameter] = "" } else { @@ -261,7 +283,7 @@ func resolveAuthRef(ctx context.Context, client client.Client, logger logr.Logge } if triggerAuthSpec.SecretTargetRef != nil { for _, e := range triggerAuthSpec.SecretTargetRef { - result[e.Parameter] = resolveAuthSecret(ctx, client, logger, e.Name, triggerNamespace, e.Key, secretsLister) + result[e.Parameter] = resolveAuthSecret(ctx, client, logger, e.Name, triggerNamespace, e.Key, authClientSet.SecretLister) } } if triggerAuthSpec.HashiCorpVault != nil && len(triggerAuthSpec.HashiCorpVault.Secrets) > 0 { @@ -287,7 +309,7 @@ func resolveAuthRef(ctx context.Context, client client.Client, logger logr.Logge } if triggerAuthSpec.AzureKeyVault != nil && len(triggerAuthSpec.AzureKeyVault.Secrets) > 0 { vaultHandler := NewAzureKeyVaultHandler(triggerAuthSpec.AzureKeyVault) - err := vaultHandler.Initialize(ctx, client, logger, triggerNamespace, secretsLister) + err := vaultHandler.Initialize(ctx, client, logger, triggerNamespace, authClientSet.SecretLister) if err != nil { logger.Error(err, "error authenticating to Azure Key Vault", "triggerAuthRef.Name", triggerAuthRef.Name) return result, podIdentity, err @@ -306,7 +328,7 @@ func resolveAuthRef(ctx context.Context, client client.Client, logger logr.Logge } if triggerAuthSpec.GCPSecretManager != nil && len(triggerAuthSpec.GCPSecretManager.Secrets) > 0 { secretManagerHandler := NewGCPSecretManagerHandler(triggerAuthSpec.GCPSecretManager) - err := secretManagerHandler.Initialize(ctx, client, logger, triggerNamespace, secretsLister) + err := secretManagerHandler.Initialize(ctx, client, logger, triggerNamespace, authClientSet.SecretLister) if err != nil { logger.Error(err, "error authenticating to GCP Secret Manager", "triggerAuthRef.Name", triggerAuthRef.Name) } else { @@ -327,7 +349,7 @@ func resolveAuthRef(ctx context.Context, client client.Client, logger logr.Logge } if triggerAuthSpec.AwsSecretManager != nil && len(triggerAuthSpec.AwsSecretManager.Secrets) > 0 { awsSecretManagerHandler := NewAwsSecretManagerHandler(triggerAuthSpec.AwsSecretManager) - err := awsSecretManagerHandler.Initialize(ctx, client, logger, triggerNamespace, secretsLister, podSpec) + err := awsSecretManagerHandler.Initialize(ctx, client, logger, triggerNamespace, authClientSet.SecretLister, podSpec) defer awsSecretManagerHandler.Stop() if err != nil { logger.Error(err, "error authenticating to Aws Secret Manager", "triggerAuthRef.Name", triggerAuthRef.Name) @@ -343,12 +365,36 @@ func resolveAuthRef(ctx context.Context, client client.Client, logger logr.Logge } } } + if triggerAuthSpec.BoundServiceAccountToken != nil { + for _, e := range triggerAuthSpec.BoundServiceAccountToken { + result[e.Parameter] = resolveBoundServiceAccountToken(ctx, client, logger, triggerNamespace, &e, triggerAuthRef, authClientSet) + } + } } } return result, podIdentity, err } +func getTriggerAuth(ctx context.Context, client client.Client, triggerAuthRef *kedav1alpha1.AuthenticationRef, namespace string) (triggerAuthType, string, error) { + if triggerAuthRef.Kind == "" || triggerAuthRef.Kind == "TriggerAuthentication" { + triggerAuth := &kedav1alpha1.TriggerAuthentication{} + err := client.Get(ctx, types.NamespacedName{Name: triggerAuthRef.Name, Namespace: namespace}, triggerAuth) + if err != nil { + return nil, "", err + } + return triggerAuth, namespace, nil + } else if triggerAuthRef.Kind == "ClusterTriggerAuthentication" { + triggerAuth := &kedav1alpha1.ClusterTriggerAuthentication{} + err := client.Get(ctx, types.NamespacedName{Name: triggerAuthRef.Name}, triggerAuth) + if err != nil { + return nil, "", err + } + return triggerAuth, kedaNamespace, nil + } + return nil, "", fmt.Errorf("unknown trigger auth kind %s", triggerAuthRef.Kind) +} + func getTriggerAuthSpec(ctx context.Context, client client.Client, triggerAuthRef *kedav1alpha1.AuthenticationRef, namespace string) (*kedav1alpha1.TriggerAuthenticationSpec, string, error) { if triggerAuthRef.Kind == "" || triggerAuthRef.Kind == "TriggerAuthentication" { triggerAuth := &kedav1alpha1.TriggerAuthentication{} @@ -597,6 +643,168 @@ func resolveAuthSecret(ctx context.Context, client client.Client, logger logr.Lo return string(result) } +func resolveBoundServiceAccountToken(ctx context.Context, client client.Client, logger logr.Logger, namespace string, bsat *kedav1alpha1.BoundServiceAccountToken, triggerAuthRef *kedav1alpha1.AuthenticationRef, acs *authentication.AuthClientSet) string { + serviceAccountName, expiry := bsat.ServiceAccountName, bsat.Expiry + if serviceAccountName == "" { + logger.Error(fmt.Errorf("error trying to get token"), "serviceAccountName is required") + return "" + } + var err error + var expirySeconds *int64 = ptr.Int64(3600) + if expiry != "" { + duration, err := time.ParseDuration(expiry) + if err != nil { + logger.Error(err, "error trying to parse expiry duration", "expiry", expiry) + return "" + } + // convert duration to seconds + expirySeconds = ptr.Int64(int64(duration.Seconds())) + } + + triggerAuth, _, err := getTriggerAuth(ctx, client, triggerAuthRef, namespace) + if err != nil { + logger.Error(err, "error trying to get [cluster]triggerAuth", "TriggerAuth.Namespace", namespace, "TriggerAuth.Name", triggerAuthRef.Name) + return "" + } + currentAnnotations := triggerAuth.GetAnnotations() + encodedToken := currentAnnotations["keda-serviceAccountToken"] + + // check if service account exists in the namespace + serviceAccount := &corev1.ServiceAccount{} + err = client.Get(ctx, types.NamespacedName{Name: serviceAccountName, Namespace: namespace}, serviceAccount) + if err != nil { + logger.Error(err, "error trying to get service account from namespace", "ServiceAccount.Namespace", namespace, "ServiceAccount.Name", serviceAccountName) + return "" + } + + // check if token is already referenced in the TriggerAuthentication annotation + if encodedToken != "" { + tokenValid := checkTokenValidity(ctx, logger, encodedToken, expirySeconds, acs) + if tokenValid == tokenStatusInvalid { + // token is invalid, or if more than 50% of the token's expiry has passed, create new token + return generateAndAnnotateNewToken(ctx, client, logger, serviceAccountName, namespace, expirySeconds, triggerAuth, acs) + } else if tokenValid == tokenStatusValid { + return encodedToken + } else { + return "" + } + } else { + // token doesn't exist; create new token and embed it in the the TriggerAuth + logger.Info("Token doesn't exist; creating new token and embedding it in the triggerauth", "ServiceAccount.Namespace", namespace, "ServiceAccount.Name", serviceAccountName) + return generateAndAnnotateNewToken(ctx, client, logger, serviceAccountName, namespace, expirySeconds, triggerAuth, acs) + } +} + +func generateAndAnnotateNewToken(ctx context.Context, client client.Client, logger logr.Logger, serviceAccountName, namespace string, expirySeconds *int64, triggerAuth triggerAuthType, acs *authentication.AuthClientSet) string { + newToken := generateToken(ctx, serviceAccountName, namespace, expirySeconds, acs) + // encode and embed the new token in the TriggerAuth + encodedToken := base64.StdEncoding.EncodeToString([]byte(newToken)) + currentAnnotations := triggerAuth.GetAnnotations() + currentAnnotations["keda-serviceAccountToken"] = encodedToken + triggerAuth.SetAnnotations(currentAnnotations) + switch underlyingTriggerAuth := triggerAuth.(type) { + case *kedav1alpha1.TriggerAuthentication: + err := client.Update(ctx, underlyingTriggerAuth) + if err != nil { + logger.Error(err, "error trying to update TriggerAuth", "TriggerAuth.Namespace", namespace, "TriggerAuth.Name", underlyingTriggerAuth.Name) + return "" + } + return newToken + case *kedav1alpha1.ClusterTriggerAuthentication: + err := client.Update(ctx, underlyingTriggerAuth) + if err != nil { + logger.Error(err, "error trying to update ClusterTriggerAuth", "ClusterTriggerAuth.Name", underlyingTriggerAuth.Name) + return "" + } + return newToken + } + return "" +} + +func checkTokenValidity(ctx context.Context, logger logr.Logger, encodedToken string, expiry *int64, acs *authentication.AuthClientSet) tokenStatus { + byteToken, err := base64.StdEncoding.DecodeString(encodedToken) + if err != nil { + logger.Error(err, "error trying to base64 decode token", "Token", encodedToken) + return tokenStatusInvalid + } + token := string(byteToken) + + // verify the token + tr := &authenticationv1.TokenReview{ + Spec: authenticationv1.TokenReviewSpec{ + Token: token, + }, + } + result, err := acs.TokenReviewInterface.Create(ctx, tr, metav1.CreateOptions{}) + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + // try again + logger.Error(err, "retrying to verify token", "Token", token) + return tokenStatusUnknown + } + + if err != nil { + logger.Error(err, "error trying to verify token", "Token", token) + return tokenStatusInvalid + } + + if result.Status.User.Username == "" || (!result.Status.Authenticated) { + logger.Error(fmt.Errorf("error trying to verify token"), "token is invalid", "Token", token) + return tokenStatusInvalid + } + + // parse the token and check the expiry + jwtToken, _ := jwt.Parse(token, nil) + // err will always be non-nil since we can't verify the token without the public key, but we already verified using TokenReview API + // so only check if the jwt token is nil + if jwtToken == nil { + logger.Error(err, "jwt token parse resulted in an error", "token", token, "expiry", expiry) + return tokenStatusInvalid + } + + claims, ok := jwtToken.Claims.(jwt.MapClaims) + if !ok { + logger.Error(nil, "jwt token claims are invalid", "token", token, "expiry", expiry) + return tokenStatusInvalid + } + + exp, err := claims.GetExpirationTime() + if err != nil { + logger.Error(err, "error trying to parse expiry time from jwt token") + return tokenStatusInvalid + } + + // rotate if more than 50% of the token's expiry has passed + expSeconds := exp.Unix() + currentTime := time.Now().Unix() + timeLeft := expSeconds - currentTime + + if timeLeft < (*expiry / 2) { + logger.Info("Rotating token as more than 50% of the token's expiry has passed", "Expires at", exp, "Time left", timeLeft) + return tokenStatusInvalid + } + + return tokenStatusValid +} + +func generateToken(ctx context.Context, serviceAccountName, namespace string, expiry *int64, acs *authentication.AuthClientSet) string { + // create new token and embed it in the secret + token, err := acs.CoreV1Interface.ServiceAccounts(namespace).CreateToken( + ctx, + serviceAccountName, + &authenticationv1.TokenRequest{ + Spec: authenticationv1.TokenRequestSpec{ + ExpirationSeconds: expiry, // kubernetes prevents token expiry to be less than 10 minutes + }, + }, + metav1.CreateOptions{}, + ) + if err != nil { + log.Error(err, "error trying to create token for service account", "ServiceAccount.Name", serviceAccountName) + return "" + } + return token.Status.Token +} + // resolveServiceAccountAnnotation retrieves the value of a specific annotation // from the annotations of a given Kubernetes ServiceAccount. func resolveServiceAccountAnnotation(ctx context.Context, client client.Client, name, namespace, annotation string, required bool) (string, error) { diff --git a/pkg/scaling/resolver/scale_resolvers_test.go b/pkg/scaling/resolver/scale_resolvers_test.go index 13f3c92c07a..951b291d6fc 100644 --- a/pkg/scaling/resolver/scale_resolvers_test.go +++ b/pkg/scaling/resolver/scale_resolvers_test.go @@ -33,6 +33,7 @@ import ( kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" mock_v1 "github.com/kedacore/keda/v2/pkg/mock/mock_secretlister" + "github.com/kedacore/keda/v2/pkg/scalers/authentication" ) var ( @@ -622,7 +623,12 @@ func TestResolveAuthRef(t *testing.T) { test.soar, test.podSpec, namespace, - secretsLister) + &authentication.AuthClientSet{ + SecretLister: secretsLister, + CoreV1Interface: nil, + TokenReviewInterface: nil, + }, + ) if err != nil && !test.isError { t.Errorf("Expected success because %s got error, %s", test.comment, err) diff --git a/pkg/scaling/scale_handler.go b/pkg/scaling/scale_handler.go index a8c50a057d0..8edaf4c0aa6 100644 --- a/pkg/scaling/scale_handler.go +++ b/pkg/scaling/scale_handler.go @@ -29,7 +29,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - corev1listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/scale" "k8s.io/client-go/tools/record" "k8s.io/metrics/pkg/apis/external_metrics" @@ -42,6 +41,7 @@ import ( "github.com/kedacore/keda/v2/pkg/fallback" "github.com/kedacore/keda/v2/pkg/metricscollector" "github.com/kedacore/keda/v2/pkg/scalers" + "github.com/kedacore/keda/v2/pkg/scalers/authentication" "github.com/kedacore/keda/v2/pkg/scalers/scalersconfig" "github.com/kedacore/keda/v2/pkg/scaling/cache" "github.com/kedacore/keda/v2/pkg/scaling/cache/metricscache" @@ -73,11 +73,11 @@ type scaleHandler struct { scalerCaches map[string]*cache.ScalersCache scalerCachesLock *sync.RWMutex scaledObjectsMetricCache metricscache.MetricsCache - secretsLister corev1listers.SecretLister + authClientSet *authentication.AuthClientSet } // NewScaleHandler creates a ScaleHandler object -func NewScaleHandler(client client.Client, scaleClient scale.ScalesGetter, reconcilerScheme *runtime.Scheme, globalHTTPTimeout time.Duration, recorder record.EventRecorder, secretsLister corev1listers.SecretLister) ScaleHandler { +func NewScaleHandler(client client.Client, scaleClient scale.ScalesGetter, reconcilerScheme *runtime.Scheme, globalHTTPTimeout time.Duration, recorder record.EventRecorder, authClientSet *authentication.AuthClientSet) ScaleHandler { return &scaleHandler{ client: client, scaleLoopContexts: &sync.Map{}, @@ -87,7 +87,7 @@ func NewScaleHandler(client client.Client, scaleClient scale.ScalesGetter, recon scalerCaches: map[string]*cache.ScalersCache{}, scalerCachesLock: &sync.RWMutex{}, scaledObjectsMetricCache: metricscache.NewMetricsCache(), - secretsLister: secretsLister, + authClientSet: authClientSet, } } diff --git a/pkg/scaling/scalers_builder.go b/pkg/scaling/scalers_builder.go index 11d294bbec3..cad09e6c10e 100644 --- a/pkg/scaling/scalers_builder.go +++ b/pkg/scaling/scalers_builder.go @@ -48,7 +48,7 @@ func (h *scaleHandler) buildScalers(ctx context.Context, withTriggers *kedav1alp factory := func() (scalers.Scaler, *scalersconfig.ScalerConfig, error) { if podTemplateSpec != nil { - resolvedEnv, err = resolver.ResolveContainerEnv(ctx, h.client, logger, &podTemplateSpec.Spec, containerName, withTriggers.Namespace, h.secretsLister) + resolvedEnv, err = resolver.ResolveContainerEnv(ctx, h.client, logger, &podTemplateSpec.Spec, containerName, withTriggers.Namespace, h.authClientSet.SecretLister) if err != nil { return nil, nil, fmt.Errorf("error resolving secrets for ScaleTarget: %w", err) } @@ -72,7 +72,7 @@ func (h *scaleHandler) buildScalers(ctx context.Context, withTriggers *kedav1alp TriggerUniqueKey: fmt.Sprintf("%s-%s-%s-%d", withTriggers.Kind, withTriggers.Namespace, withTriggers.Name, triggerIndex), } - authParams, podIdentity, err := resolver.ResolveAuthRefAndPodIdentity(ctx, h.client, logger, trigger.AuthenticationRef, podTemplateSpec, withTriggers.Namespace, h.secretsLister) + authParams, podIdentity, err := resolver.ResolveAuthRefAndPodIdentity(ctx, h.client, logger, trigger.AuthenticationRef, podTemplateSpec, withTriggers.Namespace, h.authClientSet) switch podIdentity.Provider { case kedav1alpha1.PodIdentityProviderAwsEKS: // FIXME: Delete this for v3 From 1424706f06870bfed311b3f8e5b912a1774a0585 Mon Sep 17 00:00:00 2001 From: Max Cao Date: Thu, 24 Oct 2024 13:03:25 -0700 Subject: [PATCH 02/12] Fix broken tests Signed-off-by: Max Cao --- controllers/keda/suite_test.go | 12 ++++++++---- pkg/scaling/scale_handler_test.go | 4 ++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/controllers/keda/suite_test.go b/controllers/keda/suite_test.go index 482a45b135c..fec3acd241b 100644 --- a/controllers/keda/suite_test.go +++ b/controllers/keda/suite_test.go @@ -36,6 +36,7 @@ import ( kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" "github.com/kedacore/keda/v2/pkg/eventemitter" "github.com/kedacore/keda/v2/pkg/k8s" + "github.com/kedacore/keda/v2/pkg/scalers/authentication" "github.com/kedacore/keda/v2/pkg/scaling" //+kubebuilder:scaffold:imports ) @@ -91,19 +92,22 @@ var _ = BeforeSuite(func() { scaleClient, _, err := k8s.InitScaleClient(k8sManager) Expect(err).ToNot(HaveOccurred()) + authClientSet := &authentication.AuthClientSet{} + err = (&ScaledObjectReconciler{ Client: k8sManager.GetClient(), Scheme: k8sManager.GetScheme(), - ScaleHandler: scaling.NewScaleHandler(k8sManager.GetClient(), scaleClient, k8sManager.GetScheme(), time.Duration(10), k8sManager.GetEventRecorderFor("keda-operator"), nil), + ScaleHandler: scaling.NewScaleHandler(k8sManager.GetClient(), scaleClient, k8sManager.GetScheme(), time.Duration(10), k8sManager.GetEventRecorderFor("keda-operator"), authClientSet), ScaleClient: scaleClient, EventEmitter: eventemitter.NewEventEmitter(k8sManager.GetClient(), k8sManager.GetEventRecorderFor("keda-operator"), "kubernetes-default", nil), }).SetupWithManager(k8sManager, controller.Options{}) Expect(err).ToNot(HaveOccurred()) err = (&ScaledJobReconciler{ - Client: k8sManager.GetClient(), - Scheme: k8sManager.GetScheme(), - EventEmitter: eventemitter.NewEventEmitter(k8sManager.GetClient(), k8sManager.GetEventRecorderFor("keda-operator"), "kubernetes-default", nil), + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + EventEmitter: eventemitter.NewEventEmitter(k8sManager.GetClient(), k8sManager.GetEventRecorderFor("keda-operator"), "kubernetes-default", nil), + AuthClientSet: authClientSet, }).SetupWithManager(k8sManager, controller.Options{}) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/scaling/scale_handler_test.go b/pkg/scaling/scale_handler_test.go index 1dd00da0fcc..cca23893f2a 100644 --- a/pkg/scaling/scale_handler_test.go +++ b/pkg/scaling/scale_handler_test.go @@ -43,6 +43,7 @@ import ( mock_scalers "github.com/kedacore/keda/v2/pkg/mock/mock_scaler" "github.com/kedacore/keda/v2/pkg/mock/mock_scaling/mock_executor" "github.com/kedacore/keda/v2/pkg/scalers" + "github.com/kedacore/keda/v2/pkg/scalers/authentication" "github.com/kedacore/keda/v2/pkg/scalers/scalersconfig" "github.com/kedacore/keda/v2/pkg/scaling/cache" "github.com/kedacore/keda/v2/pkg/scaling/cache/metricscache" @@ -538,6 +539,9 @@ func TestCheckScaledObjectScalersWithTriggerAuthError(t *testing.T) { scalerCaches: map[string]*cache.ScalersCache{}, scalerCachesLock: &sync.RWMutex{}, scaledObjectsMetricCache: metricscache.NewMetricsCache(), + authClientSet: &authentication.AuthClientSet{ + SecretLister: nil, + }, } isActive, isError, _, activeTriggers, _ := sh.getScaledObjectState(context.TODO(), &scaledObject) From 161b6eb294ceedfc279454cbde4a542ef74d6762 Mon Sep 17 00:00:00 2001 From: Max Cao Date: Thu, 24 Oct 2024 14:30:41 -0700 Subject: [PATCH 03/12] Fix golangci-lint Signed-off-by: Max Cao --- pkg/scaling/resolver/scale_resolvers.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/scaling/resolver/scale_resolvers.go b/pkg/scaling/resolver/scale_resolvers.go index 08cd98603bc..a67ec84f1db 100644 --- a/pkg/scaling/resolver/scale_resolvers.go +++ b/pkg/scaling/resolver/scale_resolvers.go @@ -36,7 +36,6 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" corev1listers "k8s.io/client-go/listers/core/v1" - "knative.dev/pkg/apis/duck" duckv1 "knative.dev/pkg/apis/duck/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -650,7 +649,7 @@ func resolveBoundServiceAccountToken(ctx context.Context, client client.Client, return "" } var err error - var expirySeconds *int64 = ptr.Int64(3600) + expirySeconds := ptr.Int64(3600) if expiry != "" { duration, err := time.ParseDuration(expiry) if err != nil { @@ -680,12 +679,14 @@ func resolveBoundServiceAccountToken(ctx context.Context, client client.Client, // check if token is already referenced in the TriggerAuthentication annotation if encodedToken != "" { tokenValid := checkTokenValidity(ctx, logger, encodedToken, expirySeconds, acs) - if tokenValid == tokenStatusInvalid { + switch tokenValid { + case tokenStatusInvalid: // token is invalid, or if more than 50% of the token's expiry has passed, create new token return generateAndAnnotateNewToken(ctx, client, logger, serviceAccountName, namespace, expirySeconds, triggerAuth, acs) - } else if tokenValid == tokenStatusValid { + case tokenStatusValid: return encodedToken - } else { + default: + // tokenStatusUnknown return "" } } else { From 084a53d6e90ff9b4c21095c06959f058f28599a7 Mon Sep 17 00:00:00 2001 From: Max Cao Date: Tue, 5 Nov 2024 11:50:03 -0800 Subject: [PATCH 04/12] Remove verification code; add test case Signed-off-by: Max Cao --- go.mod | 2 +- .../mock_serviceaccounts/mock_interfaces.go | 94 +++++++++++ pkg/scaling/resolver/scale_resolvers.go | 159 +----------------- pkg/scaling/resolver/scale_resolvers_test.go | 53 +++++- 4 files changed, 150 insertions(+), 158 deletions(-) create mode 100644 pkg/mock/mock_serviceaccounts/mock_interfaces.go diff --git a/go.mod b/go.mod index 2e248c4eaaf..5fc7977aee5 100644 --- a/go.mod +++ b/go.mod @@ -250,7 +250,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang-jwt/jwt/v4 v4.5.1 // indirect - github.com/golang-jwt/jwt/v5 v5.2.1 + github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect diff --git a/pkg/mock/mock_serviceaccounts/mock_interfaces.go b/pkg/mock/mock_serviceaccounts/mock_interfaces.go new file mode 100644 index 00000000000..2f9767461b6 --- /dev/null +++ b/pkg/mock/mock_serviceaccounts/mock_interfaces.go @@ -0,0 +1,94 @@ +// Generated from these commands and then edited: +// +// mockgen -source=k8s.io/client-go/kubernetes/typed/core/v1/serviceaccount.go -imports=k8s.io/client-go/kubernetes/typed/core/v1/core_client.go +// mockgen k8s.io/client-go/kubernetes/typed/core/v1 CoreV1Interface +// +// Package mock_v1 is a generated GoMock package from various generated sources and edited to remove unnecessary code. +// + +package mock_v1 //nolint:revive,stylecheck + +import ( + context "context" + reflect "reflect" + + gomock "go.uber.org/mock/gomock" + v10 "k8s.io/api/authentication/v1" + v12 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/client-go/kubernetes/typed/core/v1" +) + +// MockCoreV1Interface is a mock of CoreV1Interface interface. +type MockCoreV1Interface struct { + v1.CoreV1Interface + mockServiceAccount *MockServiceAccountInterface + ctrl *gomock.Controller + recorder *MockCoreV1InterfaceMockRecorder +} + +// MockCoreV1InterfaceMockRecorder is the mock recorder for MockCoreV1Interface. +type MockCoreV1InterfaceMockRecorder struct { + mock *MockCoreV1Interface +} + +// NewMockCoreV1Interface creates a new mock instance. +func NewMockCoreV1Interface(ctrl *gomock.Controller) *MockCoreV1Interface { + mock := &MockCoreV1Interface{ctrl: ctrl} + mock.mockServiceAccount = NewMockServiceAccountInterface(ctrl) + mock.recorder = &MockCoreV1InterfaceMockRecorder{mock} + return mock +} + +// GetServiceAccountInterface returns the mock for ServiceAccountInterface. +func (m *MockCoreV1Interface) GetServiceAccountInterface() *MockServiceAccountInterface { + return m.mockServiceAccount +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCoreV1Interface) EXPECT() *MockCoreV1InterfaceMockRecorder { + return m.recorder +} + +// ServiceAccounts mocks base method. +func (m *MockCoreV1Interface) ServiceAccounts(_ string) v1.ServiceAccountInterface { + return m.mockServiceAccount +} + +// MockServiceAccountInterface is a mock of ServiceAccountInterface interface. +type MockServiceAccountInterface struct { + v1.ServiceAccountInterface + ctrl *gomock.Controller + recorder *MockServiceAccountInterfaceMockRecorder +} + +// MockServiceAccountInterfaceMockRecorder is the mock recorder for MockServiceAccountInterface. +type MockServiceAccountInterfaceMockRecorder struct { + mock *MockServiceAccountInterface +} + +// NewMockServiceAccountInterface creates a new mock instance. +func NewMockServiceAccountInterface(ctrl *gomock.Controller) *MockServiceAccountInterface { + mock := &MockServiceAccountInterface{ctrl: ctrl} + mock.recorder = &MockServiceAccountInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockServiceAccountInterface) EXPECT() *MockServiceAccountInterfaceMockRecorder { + return m.recorder +} + +// CreateToken mocks base method. +func (m *MockServiceAccountInterface) CreateToken(ctx context.Context, serviceAccountName string, tokenRequest *v10.TokenRequest, opts v12.CreateOptions) (*v10.TokenRequest, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateToken", ctx, serviceAccountName, tokenRequest, opts) + ret0, _ := ret[0].(*v10.TokenRequest) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateToken indicates an expected call of CreateToken. +func (mr *MockServiceAccountInterfaceMockRecorder) CreateToken(ctx, serviceAccountName, tokenRequest, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateToken", reflect.TypeOf((*MockServiceAccountInterface)(nil).CreateToken), ctx, serviceAccountName, tokenRequest, opts) +} diff --git a/pkg/scaling/resolver/scale_resolvers.go b/pkg/scaling/resolver/scale_resolvers.go index a67ec84f1db..fa5e6dee7b9 100644 --- a/pkg/scaling/resolver/scale_resolvers.go +++ b/pkg/scaling/resolver/scale_resolvers.go @@ -19,8 +19,6 @@ package resolver import ( "bytes" "context" - "encoding/base64" - "errors" "fmt" "strconv" "strings" @@ -28,7 +26,6 @@ import ( "github.com/aws/smithy-go/ptr" "github.com/go-logr/logr" - "github.com/golang-jwt/jwt/v5" appsv1 "k8s.io/api/apps/v1" authenticationv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" @@ -55,19 +52,6 @@ const ( defaultServiceAccount = "default" ) -type triggerAuthType interface { - SetAnnotations(map[string]string) - GetAnnotations() map[string]string -} - -type tokenStatus int - -const ( - tokenStatusValid tokenStatus = iota - tokenStatusInvalid - tokenStatusUnknown -) - var ( kedaNamespace, _ = util.GetClusterObjectNamespace() restrictSecretAccess = util.GetRestrictSecretAccess() @@ -366,7 +350,7 @@ func resolveAuthRef(ctx context.Context, client client.Client, logger logr.Logge } if triggerAuthSpec.BoundServiceAccountToken != nil { for _, e := range triggerAuthSpec.BoundServiceAccountToken { - result[e.Parameter] = resolveBoundServiceAccountToken(ctx, client, logger, triggerNamespace, &e, triggerAuthRef, authClientSet) + result[e.Parameter] = resolveBoundServiceAccountToken(ctx, client, logger, triggerNamespace, &e, authClientSet) } } } @@ -375,25 +359,6 @@ func resolveAuthRef(ctx context.Context, client client.Client, logger logr.Logge return result, podIdentity, err } -func getTriggerAuth(ctx context.Context, client client.Client, triggerAuthRef *kedav1alpha1.AuthenticationRef, namespace string) (triggerAuthType, string, error) { - if triggerAuthRef.Kind == "" || triggerAuthRef.Kind == "TriggerAuthentication" { - triggerAuth := &kedav1alpha1.TriggerAuthentication{} - err := client.Get(ctx, types.NamespacedName{Name: triggerAuthRef.Name, Namespace: namespace}, triggerAuth) - if err != nil { - return nil, "", err - } - return triggerAuth, namespace, nil - } else if triggerAuthRef.Kind == "ClusterTriggerAuthentication" { - triggerAuth := &kedav1alpha1.ClusterTriggerAuthentication{} - err := client.Get(ctx, types.NamespacedName{Name: triggerAuthRef.Name}, triggerAuth) - if err != nil { - return nil, "", err - } - return triggerAuth, kedaNamespace, nil - } - return nil, "", fmt.Errorf("unknown trigger auth kind %s", triggerAuthRef.Kind) -} - func getTriggerAuthSpec(ctx context.Context, client client.Client, triggerAuthRef *kedav1alpha1.AuthenticationRef, namespace string) (*kedav1alpha1.TriggerAuthenticationSpec, string, error) { if triggerAuthRef.Kind == "" || triggerAuthRef.Kind == "TriggerAuthentication" { triggerAuth := &kedav1alpha1.TriggerAuthentication{} @@ -642,7 +607,7 @@ func resolveAuthSecret(ctx context.Context, client client.Client, logger logr.Lo return string(result) } -func resolveBoundServiceAccountToken(ctx context.Context, client client.Client, logger logr.Logger, namespace string, bsat *kedav1alpha1.BoundServiceAccountToken, triggerAuthRef *kedav1alpha1.AuthenticationRef, acs *authentication.AuthClientSet) string { +func resolveBoundServiceAccountToken(ctx context.Context, client client.Client, logger logr.Logger, namespace string, bsat *kedav1alpha1.BoundServiceAccountToken, acs *authentication.AuthClientSet) string { serviceAccountName, expiry := bsat.ServiceAccountName, bsat.Expiry if serviceAccountName == "" { logger.Error(fmt.Errorf("error trying to get token"), "serviceAccountName is required") @@ -660,14 +625,6 @@ func resolveBoundServiceAccountToken(ctx context.Context, client client.Client, expirySeconds = ptr.Int64(int64(duration.Seconds())) } - triggerAuth, _, err := getTriggerAuth(ctx, client, triggerAuthRef, namespace) - if err != nil { - logger.Error(err, "error trying to get [cluster]triggerAuth", "TriggerAuth.Namespace", namespace, "TriggerAuth.Name", triggerAuthRef.Name) - return "" - } - currentAnnotations := triggerAuth.GetAnnotations() - encodedToken := currentAnnotations["keda-serviceAccountToken"] - // check if service account exists in the namespace serviceAccount := &corev1.ServiceAccount{} err = client.Get(ctx, types.NamespacedName{Name: serviceAccountName, Namespace: namespace}, serviceAccount) @@ -675,116 +632,7 @@ func resolveBoundServiceAccountToken(ctx context.Context, client client.Client, logger.Error(err, "error trying to get service account from namespace", "ServiceAccount.Namespace", namespace, "ServiceAccount.Name", serviceAccountName) return "" } - - // check if token is already referenced in the TriggerAuthentication annotation - if encodedToken != "" { - tokenValid := checkTokenValidity(ctx, logger, encodedToken, expirySeconds, acs) - switch tokenValid { - case tokenStatusInvalid: - // token is invalid, or if more than 50% of the token's expiry has passed, create new token - return generateAndAnnotateNewToken(ctx, client, logger, serviceAccountName, namespace, expirySeconds, triggerAuth, acs) - case tokenStatusValid: - return encodedToken - default: - // tokenStatusUnknown - return "" - } - } else { - // token doesn't exist; create new token and embed it in the the TriggerAuth - logger.Info("Token doesn't exist; creating new token and embedding it in the triggerauth", "ServiceAccount.Namespace", namespace, "ServiceAccount.Name", serviceAccountName) - return generateAndAnnotateNewToken(ctx, client, logger, serviceAccountName, namespace, expirySeconds, triggerAuth, acs) - } -} - -func generateAndAnnotateNewToken(ctx context.Context, client client.Client, logger logr.Logger, serviceAccountName, namespace string, expirySeconds *int64, triggerAuth triggerAuthType, acs *authentication.AuthClientSet) string { - newToken := generateToken(ctx, serviceAccountName, namespace, expirySeconds, acs) - // encode and embed the new token in the TriggerAuth - encodedToken := base64.StdEncoding.EncodeToString([]byte(newToken)) - currentAnnotations := triggerAuth.GetAnnotations() - currentAnnotations["keda-serviceAccountToken"] = encodedToken - triggerAuth.SetAnnotations(currentAnnotations) - switch underlyingTriggerAuth := triggerAuth.(type) { - case *kedav1alpha1.TriggerAuthentication: - err := client.Update(ctx, underlyingTriggerAuth) - if err != nil { - logger.Error(err, "error trying to update TriggerAuth", "TriggerAuth.Namespace", namespace, "TriggerAuth.Name", underlyingTriggerAuth.Name) - return "" - } - return newToken - case *kedav1alpha1.ClusterTriggerAuthentication: - err := client.Update(ctx, underlyingTriggerAuth) - if err != nil { - logger.Error(err, "error trying to update ClusterTriggerAuth", "ClusterTriggerAuth.Name", underlyingTriggerAuth.Name) - return "" - } - return newToken - } - return "" -} - -func checkTokenValidity(ctx context.Context, logger logr.Logger, encodedToken string, expiry *int64, acs *authentication.AuthClientSet) tokenStatus { - byteToken, err := base64.StdEncoding.DecodeString(encodedToken) - if err != nil { - logger.Error(err, "error trying to base64 decode token", "Token", encodedToken) - return tokenStatusInvalid - } - token := string(byteToken) - - // verify the token - tr := &authenticationv1.TokenReview{ - Spec: authenticationv1.TokenReviewSpec{ - Token: token, - }, - } - result, err := acs.TokenReviewInterface.Create(ctx, tr, metav1.CreateOptions{}) - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - // try again - logger.Error(err, "retrying to verify token", "Token", token) - return tokenStatusUnknown - } - - if err != nil { - logger.Error(err, "error trying to verify token", "Token", token) - return tokenStatusInvalid - } - - if result.Status.User.Username == "" || (!result.Status.Authenticated) { - logger.Error(fmt.Errorf("error trying to verify token"), "token is invalid", "Token", token) - return tokenStatusInvalid - } - - // parse the token and check the expiry - jwtToken, _ := jwt.Parse(token, nil) - // err will always be non-nil since we can't verify the token without the public key, but we already verified using TokenReview API - // so only check if the jwt token is nil - if jwtToken == nil { - logger.Error(err, "jwt token parse resulted in an error", "token", token, "expiry", expiry) - return tokenStatusInvalid - } - - claims, ok := jwtToken.Claims.(jwt.MapClaims) - if !ok { - logger.Error(nil, "jwt token claims are invalid", "token", token, "expiry", expiry) - return tokenStatusInvalid - } - - exp, err := claims.GetExpirationTime() - if err != nil { - logger.Error(err, "error trying to parse expiry time from jwt token") - return tokenStatusInvalid - } - - // rotate if more than 50% of the token's expiry has passed - expSeconds := exp.Unix() - currentTime := time.Now().Unix() - timeLeft := expSeconds - currentTime - - if timeLeft < (*expiry / 2) { - logger.Info("Rotating token as more than 50% of the token's expiry has passed", "Expires at", exp, "Time left", timeLeft) - return tokenStatusInvalid - } - - return tokenStatusValid + return generateToken(ctx, serviceAccountName, namespace, expirySeconds, acs) } func generateToken(ctx context.Context, serviceAccountName, namespace string, expiry *int64, acs *authentication.AuthClientSet) string { @@ -803,6 +651,7 @@ func generateToken(ctx context.Context, serviceAccountName, namespace string, ex log.Error(err, "error trying to create token for service account", "ServiceAccount.Name", serviceAccountName) return "" } + log.Info("Service account token created successfully", "ServiceAccount.Name", serviceAccountName, "Token", token.Status.Token) return token.Status.Token } diff --git a/pkg/scaling/resolver/scale_resolvers_test.go b/pkg/scaling/resolver/scale_resolvers_test.go index 951b291d6fc..e86af2593dc 100644 --- a/pkg/scaling/resolver/scale_resolvers_test.go +++ b/pkg/scaling/resolver/scale_resolvers_test.go @@ -23,16 +23,19 @@ import ( "github.com/google/go-cmp/cmp" "go.uber.org/mock/gomock" + authv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/scheme" + authenticationv1client "k8s.io/client-go/kubernetes/typed/authentication/v1" corev1listers "k8s.io/client-go/listers/core/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" logf "sigs.k8s.io/controller-runtime/pkg/log" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" mock_v1 "github.com/kedacore/keda/v2/pkg/mock/mock_secretlister" + mock_serviceaccounts "github.com/kedacore/keda/v2/pkg/mock/mock_serviceaccounts" "github.com/kedacore/keda/v2/pkg/scalers/authentication" ) @@ -46,6 +49,9 @@ var ( cmName = "supercm" cmKey = "mycmkey" cmData = "cmDataHere" + bsatSAName = "bsatServiceAccount" + bsatExpiry = "10m" + bsatData = "k8s-bsat-token" trueValue = true falseValue = false envKey = "test-env-key" @@ -449,6 +455,38 @@ func TestResolveAuthRef(t *testing.T) { expected: map[string]string{"host-secret": secretData, "host-configmap": cmData}, expectedPodIdentity: kedav1alpha1.AuthPodIdentity{Provider: kedav1alpha1.PodIdentityProviderNone}, }, + { + name: "triggerauth exists bound service account token", + existing: []runtime.Object{ + &kedav1alpha1.TriggerAuthentication{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: triggerAuthenticationName, + }, + Spec: kedav1alpha1.TriggerAuthenticationSpec{ + PodIdentity: &kedav1alpha1.AuthPodIdentity{ + Provider: kedav1alpha1.PodIdentityProviderNone, + }, + BoundServiceAccountToken: []kedav1alpha1.BoundServiceAccountToken{ + { + Parameter: "token", + ServiceAccountName: bsatSAName, + Expiry: bsatExpiry, + }, + }, + }, + }, + &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: bsatSAName, + }, + }, + }, + soar: &kedav1alpha1.AuthenticationRef{Name: triggerAuthenticationName}, + expected: map[string]string{"token": bsatData}, + expectedPodIdentity: kedav1alpha1.AuthPodIdentity{Provider: kedav1alpha1.PodIdentityProviderNone}, + }, { name: "clustertriggerauth exists, podidentity nil", existing: []runtime.Object{ @@ -610,7 +648,18 @@ func TestResolveAuthRef(t *testing.T) { expectedPodIdentity: kedav1alpha1.AuthPodIdentity{Provider: kedav1alpha1.PodIdentityProviderGCP}, }, } + ctrl := gomock.NewController(t) var secretsLister corev1listers.SecretLister + var tokenReviewInterface authenticationv1client.TokenReviewInterface + mockCoreV1Interface := mock_serviceaccounts.NewMockCoreV1Interface(ctrl) + mockServiceAccountInterface := mockCoreV1Interface.GetServiceAccountInterface() + tokenRequest := &authv1.TokenRequest{ + Status: authv1.TokenRequestStatus{ + Token: bsatData, + }, + } + mockServiceAccountInterface.EXPECT().CreateToken(gomock.Any(), gomock.Eq(bsatSAName), gomock.Any(), gomock.Any()).Return(tokenRequest, nil) + for _, test := range tests { test := test t.Run(test.name, func(t *testing.T) { @@ -625,8 +674,8 @@ func TestResolveAuthRef(t *testing.T) { namespace, &authentication.AuthClientSet{ SecretLister: secretsLister, - CoreV1Interface: nil, - TokenReviewInterface: nil, + CoreV1Interface: mockCoreV1Interface, + TokenReviewInterface: tokenReviewInterface, }, ) From 1b45dffa7ff0438f690112723183deef5a199b4d Mon Sep 17 00:00:00 2001 From: Max Cao Date: Tue, 5 Nov 2024 15:30:57 -0800 Subject: [PATCH 05/12] Rollback privilege escalation Signed-off-by: Max Cao --- config/rbac/role.yaml | 7 ------- controllers/keda/scaledobject_controller.go | 1 - pkg/scaling/resolver/scale_resolvers.go | 4 ++-- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index a056f3111c1..87362c08e13 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -32,13 +32,6 @@ rules: verbs: - list - watch -- apiGroups: - - "" - resources: - - serviceaccounts/token - verbs: - - create - - get - apiGroups: - '*' resources: diff --git a/controllers/keda/scaledobject_controller.go b/controllers/keda/scaledobject_controller.go index 21c88e6757c..c480dc380c4 100755 --- a/controllers/keda/scaledobject_controller.go +++ b/controllers/keda/scaledobject_controller.go @@ -58,7 +58,6 @@ import ( // +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;update;patch;create;delete // +kubebuilder:rbac:groups="",resources=configmaps;configmaps/status,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch -// +kubebuilder:rbac:groups="",resources=serviceaccounts/token,verbs=create;get // +kubebuilder:rbac:groups="",resources=pods;services;services;secrets;external,verbs=get;list;watch // +kubebuilder:rbac:groups="*",resources="*/scale",verbs=get;list;watch;update;patch // +kubebuilder:rbac:groups="",resources="serviceaccounts",verbs=list;watch diff --git a/pkg/scaling/resolver/scale_resolvers.go b/pkg/scaling/resolver/scale_resolvers.go index fa5e6dee7b9..d7fd558a1bb 100644 --- a/pkg/scaling/resolver/scale_resolvers.go +++ b/pkg/scaling/resolver/scale_resolvers.go @@ -614,7 +614,7 @@ func resolveBoundServiceAccountToken(ctx context.Context, client client.Client, return "" } var err error - expirySeconds := ptr.Int64(3600) + expirySeconds := ptr.Int64(3600) // default expiry is 1 hour if expiry != "" { duration, err := time.ParseDuration(expiry) if err != nil { @@ -651,7 +651,7 @@ func generateToken(ctx context.Context, serviceAccountName, namespace string, ex log.Error(err, "error trying to create token for service account", "ServiceAccount.Name", serviceAccountName) return "" } - log.Info("Service account token created successfully", "ServiceAccount.Name", serviceAccountName, "Token", token.Status.Token) + log.Info("Service account token created successfully", "ServiceAccount.Name", serviceAccountName) return token.Status.Token } From 6dffd6ec8ea346884a2699bb766e23f55c994f3e Mon Sep 17 00:00:00 2001 From: Max Cao Date: Tue, 5 Nov 2024 15:42:17 -0800 Subject: [PATCH 06/12] Additional test cases Signed-off-by: Max Cao --- pkg/scaling/resolver/scale_resolvers_test.go | 96 +++++++++++++++++++- 1 file changed, 95 insertions(+), 1 deletion(-) diff --git a/pkg/scaling/resolver/scale_resolvers_test.go b/pkg/scaling/resolver/scale_resolvers_test.go index e86af2593dc..9b50d595426 100644 --- a/pkg/scaling/resolver/scale_resolvers_test.go +++ b/pkg/scaling/resolver/scale_resolvers_test.go @@ -487,6 +487,38 @@ func TestResolveAuthRef(t *testing.T) { expected: map[string]string{"token": bsatData}, expectedPodIdentity: kedav1alpha1.AuthPodIdentity{Provider: kedav1alpha1.PodIdentityProviderNone}, }, + { + name: "triggerauth exists bound service account token, but expiry invalid", + existing: []runtime.Object{ + &kedav1alpha1.TriggerAuthentication{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: triggerAuthenticationName, + }, + Spec: kedav1alpha1.TriggerAuthenticationSpec{ + PodIdentity: &kedav1alpha1.AuthPodIdentity{ + Provider: kedav1alpha1.PodIdentityProviderNone, + }, + BoundServiceAccountToken: []kedav1alpha1.BoundServiceAccountToken{ + { + Parameter: "token", + ServiceAccountName: bsatSAName, + Expiry: "10g", + }, + }, + }, + }, + &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: bsatSAName, + }, + }, + }, + soar: &kedav1alpha1.AuthenticationRef{Name: triggerAuthenticationName}, + expected: map[string]string{"token": ""}, + expectedPodIdentity: kedav1alpha1.AuthPodIdentity{Provider: kedav1alpha1.PodIdentityProviderNone}, + }, { name: "clustertriggerauth exists, podidentity nil", existing: []runtime.Object{ @@ -647,6 +679,68 @@ func TestResolveAuthRef(t *testing.T) { expected: map[string]string{}, expectedPodIdentity: kedav1alpha1.AuthPodIdentity{Provider: kedav1alpha1.PodIdentityProviderGCP}, }, + { + name: "clustertriggerauth exists bound service account token", + existing: []runtime.Object{ + &kedav1alpha1.ClusterTriggerAuthentication{ + ObjectMeta: metav1.ObjectMeta{ + Name: triggerAuthenticationName, + }, + Spec: kedav1alpha1.TriggerAuthenticationSpec{ + PodIdentity: &kedav1alpha1.AuthPodIdentity{ + Provider: kedav1alpha1.PodIdentityProviderNone, + }, + BoundServiceAccountToken: []kedav1alpha1.BoundServiceAccountToken{ + { + Parameter: "token", + ServiceAccountName: bsatSAName, + Expiry: bsatExpiry, + }, + }, + }, + }, + &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: clusterNamespace, + Name: bsatSAName, + }, + }, + }, + soar: &kedav1alpha1.AuthenticationRef{Name: triggerAuthenticationName, Kind: "ClusterTriggerAuthentication"}, + expected: map[string]string{"token": bsatData}, + expectedPodIdentity: kedav1alpha1.AuthPodIdentity{Provider: kedav1alpha1.PodIdentityProviderNone}, + }, + { + name: "clustertriggerauth exists bound service account token but service account in the wrong namespace", + existing: []runtime.Object{ + &kedav1alpha1.ClusterTriggerAuthentication{ + ObjectMeta: metav1.ObjectMeta{ + Name: triggerAuthenticationName, + }, + Spec: kedav1alpha1.TriggerAuthenticationSpec{ + PodIdentity: &kedav1alpha1.AuthPodIdentity{ + Provider: kedav1alpha1.PodIdentityProviderNone, + }, + BoundServiceAccountToken: []kedav1alpha1.BoundServiceAccountToken{ + { + Parameter: "token", + ServiceAccountName: bsatSAName, + Expiry: bsatExpiry, + }, + }, + }, + }, + &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: bsatSAName, + }, + }, + }, + soar: &kedav1alpha1.AuthenticationRef{Name: triggerAuthenticationName, Kind: "ClusterTriggerAuthentication"}, + expected: map[string]string{"token": ""}, + expectedPodIdentity: kedav1alpha1.AuthPodIdentity{Provider: kedav1alpha1.PodIdentityProviderNone}, + }, } ctrl := gomock.NewController(t) var secretsLister corev1listers.SecretLister @@ -658,7 +752,7 @@ func TestResolveAuthRef(t *testing.T) { Token: bsatData, }, } - mockServiceAccountInterface.EXPECT().CreateToken(gomock.Any(), gomock.Eq(bsatSAName), gomock.Any(), gomock.Any()).Return(tokenRequest, nil) + mockServiceAccountInterface.EXPECT().CreateToken(gomock.Any(), gomock.Eq(bsatSAName), gomock.Any(), gomock.Any()).Return(tokenRequest, nil).AnyTimes() for _, test := range tests { test := test From 24064054f22cf663c48e8a5fea3ef598d5986ba2 Mon Sep 17 00:00:00 2001 From: Max Cao Date: Tue, 5 Nov 2024 16:26:53 -0800 Subject: [PATCH 07/12] Update CHANGELOG.md Signed-off-by: Max Cao --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 963665fd4b7..2f63ae1bf4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio ### New +- **General**: Add support for time-bound Kubernetes ServiceAccount tokens as a source for TriggerAuthentication ([#6136](https://github.com/kedacore/keda/issues/6136)) - **General**: Enable OpenSSF Scorecard to enhance security practices across the project ([#5913](https://github.com/kedacore/keda/issues/5913)) - **General**: Introduce new NSQ scaler ([#3281](https://github.com/kedacore/keda/issues/3281)) - **General**: Operator flag to control patching of webhook resources certificates ([#6184](https://github.com/kedacore/keda/issues/6184)) From 47e48b52aad9a4cad6b3b2a5961b249bbc322956 Mon Sep 17 00:00:00 2001 From: Max Cao Date: Fri, 29 Nov 2024 13:01:30 -0800 Subject: [PATCH 08/12] fixup: rebase changes Signed-off-by: Max Cao --- go.mod | 2 +- pkg/scaling/resolver/scale_resolvers.go | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5fc7977aee5..3d53b2099c6 100644 --- a/go.mod +++ b/go.mod @@ -211,7 +211,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.24.8 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7 // indirect - github.com/aws/smithy-go v1.22.1 // indirect + github.com/aws/smithy-go v1.22.1 github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect diff --git a/pkg/scaling/resolver/scale_resolvers.go b/pkg/scaling/resolver/scale_resolvers.go index d7fd558a1bb..92c768b5aaf 100644 --- a/pkg/scaling/resolver/scale_resolvers.go +++ b/pkg/scaling/resolver/scale_resolvers.go @@ -621,7 +621,6 @@ func resolveBoundServiceAccountToken(ctx context.Context, client client.Client, logger.Error(err, "error trying to parse expiry duration", "expiry", expiry) return "" } - // convert duration to seconds expirySeconds = ptr.Int64(int64(duration.Seconds())) } @@ -636,7 +635,6 @@ func resolveBoundServiceAccountToken(ctx context.Context, client client.Client, } func generateToken(ctx context.Context, serviceAccountName, namespace string, expiry *int64, acs *authentication.AuthClientSet) string { - // create new token and embed it in the secret token, err := acs.CoreV1Interface.ServiceAccounts(namespace).CreateToken( ctx, serviceAccountName, From da7b3211b7a836bd270a5a9bd9cb345e46d9dbf6 Mon Sep 17 00:00:00 2001 From: Max Cao Date: Thu, 5 Dec 2024 11:54:46 -0800 Subject: [PATCH 09/12] Remove expiry field and hardcode the token expiry to 1 hour Signed-off-by: Max Cao --- .../v1alpha1/triggerauthentication_types.go | 1 - ...keda.sh_clustertriggerauthentications.yaml | 3 -- .../bases/keda.sh_triggerauthentications.yaml | 3 -- pkg/scaling/resolver/scale_resolvers.go | 20 +++-------- pkg/scaling/resolver/scale_resolvers_test.go | 36 ------------------- 5 files changed, 5 insertions(+), 58 deletions(-) diff --git a/apis/keda/v1alpha1/triggerauthentication_types.go b/apis/keda/v1alpha1/triggerauthentication_types.go index aa85def04b8..826dd5249c0 100644 --- a/apis/keda/v1alpha1/triggerauthentication_types.go +++ b/apis/keda/v1alpha1/triggerauthentication_types.go @@ -384,7 +384,6 @@ type AwsSecretManagerSecret struct { type BoundServiceAccountToken struct { Parameter string `json:"parameter"` ServiceAccountName string `json:"serviceAccountName"` - Expiry string `json:"expiry"` } func init() { diff --git a/config/crd/bases/keda.sh_clustertriggerauthentications.yaml b/config/crd/bases/keda.sh_clustertriggerauthentications.yaml index fcc28494184..108ca117ef0 100644 --- a/config/crd/bases/keda.sh_clustertriggerauthentications.yaml +++ b/config/crd/bases/keda.sh_clustertriggerauthentications.yaml @@ -305,14 +305,11 @@ spec: boundServiceAccountToken: items: properties: - expiry: - type: string parameter: type: string serviceAccountName: type: string required: - - expiry - parameter - serviceAccountName type: object diff --git a/config/crd/bases/keda.sh_triggerauthentications.yaml b/config/crd/bases/keda.sh_triggerauthentications.yaml index 3a7eaea02c8..dbdeed4dd55 100644 --- a/config/crd/bases/keda.sh_triggerauthentications.yaml +++ b/config/crd/bases/keda.sh_triggerauthentications.yaml @@ -304,14 +304,11 @@ spec: boundServiceAccountToken: items: properties: - expiry: - type: string parameter: type: string serviceAccountName: type: string required: - - expiry - parameter - serviceAccountName type: object diff --git a/pkg/scaling/resolver/scale_resolvers.go b/pkg/scaling/resolver/scale_resolvers.go index 92c768b5aaf..2473f4701eb 100644 --- a/pkg/scaling/resolver/scale_resolvers.go +++ b/pkg/scaling/resolver/scale_resolvers.go @@ -22,7 +22,6 @@ import ( "fmt" "strconv" "strings" - "time" "github.com/aws/smithy-go/ptr" "github.com/go-logr/logr" @@ -608,39 +607,30 @@ func resolveAuthSecret(ctx context.Context, client client.Client, logger logr.Lo } func resolveBoundServiceAccountToken(ctx context.Context, client client.Client, logger logr.Logger, namespace string, bsat *kedav1alpha1.BoundServiceAccountToken, acs *authentication.AuthClientSet) string { - serviceAccountName, expiry := bsat.ServiceAccountName, bsat.Expiry + serviceAccountName := bsat.ServiceAccountName if serviceAccountName == "" { logger.Error(fmt.Errorf("error trying to get token"), "serviceAccountName is required") return "" } var err error - expirySeconds := ptr.Int64(3600) // default expiry is 1 hour - if expiry != "" { - duration, err := time.ParseDuration(expiry) - if err != nil { - logger.Error(err, "error trying to parse expiry duration", "expiry", expiry) - return "" - } - expirySeconds = ptr.Int64(int64(duration.Seconds())) - } - // check if service account exists in the namespace serviceAccount := &corev1.ServiceAccount{} err = client.Get(ctx, types.NamespacedName{Name: serviceAccountName, Namespace: namespace}, serviceAccount) if err != nil { logger.Error(err, "error trying to get service account from namespace", "ServiceAccount.Namespace", namespace, "ServiceAccount.Name", serviceAccountName) return "" } - return generateToken(ctx, serviceAccountName, namespace, expirySeconds, acs) + return generateToken(ctx, serviceAccountName, namespace, acs) } -func generateToken(ctx context.Context, serviceAccountName, namespace string, expiry *int64, acs *authentication.AuthClientSet) string { +func generateToken(ctx context.Context, serviceAccountName, namespace string, acs *authentication.AuthClientSet) string { + expirationSeconds := ptr.Int64(3600) // We default the token expiry to 1 hour token, err := acs.CoreV1Interface.ServiceAccounts(namespace).CreateToken( ctx, serviceAccountName, &authenticationv1.TokenRequest{ Spec: authenticationv1.TokenRequestSpec{ - ExpirationSeconds: expiry, // kubernetes prevents token expiry to be less than 10 minutes + ExpirationSeconds: expirationSeconds, }, }, metav1.CreateOptions{}, diff --git a/pkg/scaling/resolver/scale_resolvers_test.go b/pkg/scaling/resolver/scale_resolvers_test.go index 9b50d595426..6609de65013 100644 --- a/pkg/scaling/resolver/scale_resolvers_test.go +++ b/pkg/scaling/resolver/scale_resolvers_test.go @@ -50,7 +50,6 @@ var ( cmKey = "mycmkey" cmData = "cmDataHere" bsatSAName = "bsatServiceAccount" - bsatExpiry = "10m" bsatData = "k8s-bsat-token" trueValue = true falseValue = false @@ -471,7 +470,6 @@ func TestResolveAuthRef(t *testing.T) { { Parameter: "token", ServiceAccountName: bsatSAName, - Expiry: bsatExpiry, }, }, }, @@ -487,38 +485,6 @@ func TestResolveAuthRef(t *testing.T) { expected: map[string]string{"token": bsatData}, expectedPodIdentity: kedav1alpha1.AuthPodIdentity{Provider: kedav1alpha1.PodIdentityProviderNone}, }, - { - name: "triggerauth exists bound service account token, but expiry invalid", - existing: []runtime.Object{ - &kedav1alpha1.TriggerAuthentication{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: triggerAuthenticationName, - }, - Spec: kedav1alpha1.TriggerAuthenticationSpec{ - PodIdentity: &kedav1alpha1.AuthPodIdentity{ - Provider: kedav1alpha1.PodIdentityProviderNone, - }, - BoundServiceAccountToken: []kedav1alpha1.BoundServiceAccountToken{ - { - Parameter: "token", - ServiceAccountName: bsatSAName, - Expiry: "10g", - }, - }, - }, - }, - &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: bsatSAName, - }, - }, - }, - soar: &kedav1alpha1.AuthenticationRef{Name: triggerAuthenticationName}, - expected: map[string]string{"token": ""}, - expectedPodIdentity: kedav1alpha1.AuthPodIdentity{Provider: kedav1alpha1.PodIdentityProviderNone}, - }, { name: "clustertriggerauth exists, podidentity nil", existing: []runtime.Object{ @@ -694,7 +660,6 @@ func TestResolveAuthRef(t *testing.T) { { Parameter: "token", ServiceAccountName: bsatSAName, - Expiry: bsatExpiry, }, }, }, @@ -725,7 +690,6 @@ func TestResolveAuthRef(t *testing.T) { { Parameter: "token", ServiceAccountName: bsatSAName, - Expiry: bsatExpiry, }, }, }, From 91dc6886737a3b47f98b5d56f796d6c87a5686a3 Mon Sep 17 00:00:00 2001 From: Max Cao Date: Mon, 9 Dec 2024 11:58:25 -0800 Subject: [PATCH 10/12] Allow bound service account token expiry to be configurable through env vars Signed-off-by: Max Cao --- cmd/operator/main.go | 11 +++++++--- .../authentication/authentication_helpers.go | 2 -- pkg/scaling/resolver/scale_resolvers.go | 20 ++++++++++-------- pkg/scaling/resolver/scale_resolvers_test.go | 7 ++----- pkg/util/env_resolver.go | 21 +++++++++++++++++++ 5 files changed, 42 insertions(+), 19 deletions(-) diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 389ccca8317..55e9638b6f0 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -202,6 +202,12 @@ func main() { os.Exit(1) } + _, err = kedautil.GetBoundServiceAccountTokenExpiry() + if err != nil { + setupLog.Error(err, "invalid "+kedautil.BoundServiceAccountTokenExpiryEnvVar) + os.Exit(1) + } + globalHTTPTimeout := time.Duration(globalHTTPTimeoutMS) * time.Millisecond eventRecorder := mgr.GetEventRecorderFor("keda-operator") @@ -227,9 +233,8 @@ func main() { } authClientSet := &authentication.AuthClientSet{ - TokenReviewInterface: kubeClientset.AuthenticationV1().TokenReviews(), - CoreV1Interface: kubeClientset.CoreV1(), - SecretLister: secretInformer.Lister(), + CoreV1Interface: kubeClientset.CoreV1(), + SecretLister: secretInformer.Lister(), } scaledHandler := scaling.NewScaleHandler(mgr.GetClient(), scaleClient, mgr.GetScheme(), globalHTTPTimeout, eventRecorder, authClientSet) diff --git a/pkg/scalers/authentication/authentication_helpers.go b/pkg/scalers/authentication/authentication_helpers.go index f8ac658e01c..1b89c048590 100644 --- a/pkg/scalers/authentication/authentication_helpers.go +++ b/pkg/scalers/authentication/authentication_helpers.go @@ -13,7 +13,6 @@ import ( libs "github.com/dysnix/predictkube-libs/external/configs" "github.com/dysnix/predictkube-libs/external/http_transport" pConfig "github.com/prometheus/common/config" - authenticationv1client "k8s.io/client-go/kubernetes/typed/authentication/v1" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" corev1listers "k8s.io/client-go/listers/core/v1" @@ -21,7 +20,6 @@ import ( ) type AuthClientSet struct { - authenticationv1client.TokenReviewInterface corev1client.CoreV1Interface corev1listers.SecretLister } diff --git a/pkg/scaling/resolver/scale_resolvers.go b/pkg/scaling/resolver/scale_resolvers.go index 2473f4701eb..96133d5b1ad 100644 --- a/pkg/scaling/resolver/scale_resolvers.go +++ b/pkg/scaling/resolver/scale_resolvers.go @@ -23,7 +23,6 @@ import ( "strconv" "strings" - "github.com/aws/smithy-go/ptr" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" authenticationv1 "k8s.io/api/authentication/v1" @@ -32,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/utils/ptr" "knative.dev/pkg/apis/duck" duckv1 "knative.dev/pkg/apis/duck/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -52,9 +52,10 @@ const ( ) var ( - kedaNamespace, _ = util.GetClusterObjectNamespace() - restrictSecretAccess = util.GetRestrictSecretAccess() - log = logf.Log.WithName("scale_resolvers") + kedaNamespace, _ = util.GetClusterObjectNamespace() + restrictSecretAccess = util.GetRestrictSecretAccess() + boundServiceAccountTokenExpiry, _ = util.GetBoundServiceAccountTokenExpiry() + log = logf.Log.WithName("scale_resolvers") ) // isSecretAccessRestricted returns whether secret access need to be restricted in KEDA namespace @@ -620,11 +621,12 @@ func resolveBoundServiceAccountToken(ctx context.Context, client client.Client, logger.Error(err, "error trying to get service account from namespace", "ServiceAccount.Namespace", namespace, "ServiceAccount.Name", serviceAccountName) return "" } - return generateToken(ctx, serviceAccountName, namespace, acs) + return generateBoundServiceAccountToken(ctx, serviceAccountName, namespace, acs) } -func generateToken(ctx context.Context, serviceAccountName, namespace string, acs *authentication.AuthClientSet) string { - expirationSeconds := ptr.Int64(3600) // We default the token expiry to 1 hour +// generateBoundServiceAccountToken creates a Kubernetes token for a namespaced service account with a runtime-configurable expiration time and returns the token string. +func generateBoundServiceAccountToken(ctx context.Context, serviceAccountName, namespace string, acs *authentication.AuthClientSet) string { + expirationSeconds := ptr.To[int64](int64(boundServiceAccountTokenExpiry.Seconds())) token, err := acs.CoreV1Interface.ServiceAccounts(namespace).CreateToken( ctx, serviceAccountName, @@ -636,10 +638,10 @@ func generateToken(ctx context.Context, serviceAccountName, namespace string, ac metav1.CreateOptions{}, ) if err != nil { - log.Error(err, "error trying to create token for service account", "ServiceAccount.Name", serviceAccountName) + log.V(1).Error(err, "error trying to create bound service account token for service account", "ServiceAccount.Name", serviceAccountName) return "" } - log.Info("Service account token created successfully", "ServiceAccount.Name", serviceAccountName) + log.V(1).Info("Bound service account token created successfully", "ServiceAccount.Name", serviceAccountName) return token.Status.Token } diff --git a/pkg/scaling/resolver/scale_resolvers_test.go b/pkg/scaling/resolver/scale_resolvers_test.go index 6609de65013..4485d6554e0 100644 --- a/pkg/scaling/resolver/scale_resolvers_test.go +++ b/pkg/scaling/resolver/scale_resolvers_test.go @@ -28,7 +28,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/scheme" - authenticationv1client "k8s.io/client-go/kubernetes/typed/authentication/v1" corev1listers "k8s.io/client-go/listers/core/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -708,7 +707,6 @@ func TestResolveAuthRef(t *testing.T) { } ctrl := gomock.NewController(t) var secretsLister corev1listers.SecretLister - var tokenReviewInterface authenticationv1client.TokenReviewInterface mockCoreV1Interface := mock_serviceaccounts.NewMockCoreV1Interface(ctrl) mockServiceAccountInterface := mockCoreV1Interface.GetServiceAccountInterface() tokenRequest := &authv1.TokenRequest{ @@ -731,9 +729,8 @@ func TestResolveAuthRef(t *testing.T) { test.podSpec, namespace, &authentication.AuthClientSet{ - SecretLister: secretsLister, - CoreV1Interface: mockCoreV1Interface, - TokenReviewInterface: tokenReviewInterface, + SecretLister: secretsLister, + CoreV1Interface: mockCoreV1Interface, }, ) diff --git a/pkg/util/env_resolver.go b/pkg/util/env_resolver.go index b8e9cfa8763..1dcb3598c4c 100644 --- a/pkg/util/env_resolver.go +++ b/pkg/util/env_resolver.go @@ -17,12 +17,16 @@ limitations under the License. package util import ( + "fmt" "os" "strconv" "time" + + "k8s.io/utils/ptr" ) const RestrictSecretAccessEnvVar = "KEDA_RESTRICT_SECRET_ACCESS" +const BoundServiceAccountTokenExpiryEnvVar = "KEDA_BOUND_SERVICE_ACCOUNT_TOKEN_EXPIRY" var clusterObjectNamespaceCache *string @@ -90,3 +94,20 @@ func GetPodNamespace() string { func GetRestrictSecretAccess() string { return os.Getenv(RestrictSecretAccessEnvVar) } + +// GetBoundServiceAccountTokenExpiry retrieves the value of the environment variable of KEDA_BOUND_SERVICE_ACCOUNT_TOKEN_EXPIRY +func GetBoundServiceAccountTokenExpiry() (*time.Duration, error) { + expiry, err := ResolveOsEnvDuration(BoundServiceAccountTokenExpiryEnvVar) + if err != nil { + return nil, err + } + if expiry == nil { + return ptr.To[time.Duration](time.Hour), nil // if blank, default to 1 hour + + } + if *expiry < time.Hour || *expiry > 6*time.Hour { + return nil, fmt.Errorf("invalid value for %s: %s, must be between 1h and 6h", BoundServiceAccountTokenExpiryEnvVar, expiry.String()) // Must be between 1 hour and 6 hours + + } + return expiry, nil +} From 261a7fea4d88aa4ec14fb11186be56a6db3e90ca Mon Sep 17 00:00:00 2001 From: Max Cao Date: Tue, 10 Dec 2024 17:03:51 -0800 Subject: [PATCH 11/12] Add bound service account token e2e test Signed-off-by: Max Cao --- pkg/util/env_resolver.go | 2 - ...r_auth_bound_service_account_token_test.go | 358 ++++++++++++++++++ 2 files changed, 358 insertions(+), 2 deletions(-) create mode 100644 tests/secret-providers/trigger_auth_bound_service_account_token/trigger_auth_bound_service_account_token_test.go diff --git a/pkg/util/env_resolver.go b/pkg/util/env_resolver.go index 1dcb3598c4c..965c688a370 100644 --- a/pkg/util/env_resolver.go +++ b/pkg/util/env_resolver.go @@ -103,11 +103,9 @@ func GetBoundServiceAccountTokenExpiry() (*time.Duration, error) { } if expiry == nil { return ptr.To[time.Duration](time.Hour), nil // if blank, default to 1 hour - } if *expiry < time.Hour || *expiry > 6*time.Hour { return nil, fmt.Errorf("invalid value for %s: %s, must be between 1h and 6h", BoundServiceAccountTokenExpiryEnvVar, expiry.String()) // Must be between 1 hour and 6 hours - } return expiry, nil } diff --git a/tests/secret-providers/trigger_auth_bound_service_account_token/trigger_auth_bound_service_account_token_test.go b/tests/secret-providers/trigger_auth_bound_service_account_token/trigger_auth_bound_service_account_token_test.go new file mode 100644 index 00000000000..04bdd738f25 --- /dev/null +++ b/tests/secret-providers/trigger_auth_bound_service_account_token/trigger_auth_bound_service_account_token_test.go @@ -0,0 +1,358 @@ +//go:build e2e +// +build e2e + +package trigger_auth_bound_service_account_token_test + +import ( + "fmt" + "testing" + + "github.com/joho/godotenv" + "github.com/stretchr/testify/assert" + "k8s.io/client-go/kubernetes" + + . "github.com/kedacore/keda/v2/tests/helper" +) + +// Load environment variables from .env file +var _ = godotenv.Load("../../.env") + +const ( + testName = "trigger-auth-bound-service-account-token-test" +) + +var ( + testNamespace = fmt.Sprintf("%s-ns", testName) + deploymentName = fmt.Sprintf("%s-deployment", testName) + metricsServerDeploymentName = fmt.Sprintf("%s-metrics-server", testName) + triggerAuthName = fmt.Sprintf("%s-ta", testName) + scaledObjectName = fmt.Sprintf("%s-so", testName) + metricsServerServiceName = fmt.Sprintf("%s-service", testName) + metricsServerEndpoint = fmt.Sprintf("http://%s.%s.svc.cluster.local:8080/api/value", metricsServerServiceName, testNamespace) + serviceAccountName = fmt.Sprintf("%s-sa", testName) + serviceAccountTokenCreationRole = fmt.Sprintf("%s-sa-role", testName) + serviceAccountTokenCreationRoleBinding = fmt.Sprintf("%s-sa-role-binding", testName) + minReplicaCount = 0 + maxReplicaCount = 1 +) + +type templateData struct { + TestNamespace string + ServiceAccountName string + ServiceAccountTokenCreationRole string + ServiceAccountTokenCreationRoleBinding string + DeploymentName string + MetricsServerDeploymentName string + MetricsServerServiceName string + TriggerAuthName string + ScaledObjectName string + MetricsServerEndpoint string + MetricValue int + MinReplicaCount string + MaxReplicaCount string +} + +const ( + serviceAccountTemplate = ` +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{.ServiceAccountName}} + namespace: {{.TestNamespace}} +` + // arbitrary k8s rbac permissions that the test metrics-api container requires requesters to have + serviceAccountClusterRoleTemplate = ` +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{.ServiceAccountName}} +rules: +- nonResourceURLs: + - /api/value + verbs: + - get +` + serviceAccountClusterRoleBindingTemplate = ` +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{.ServiceAccountName}} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{.ServiceAccountName}} +subjects: +- kind: ServiceAccount + name: {{.ServiceAccountName}} + namespace: {{.TestNamespace}} +` + serviceAccountTokenCreationRoleTemplate = ` +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{.ServiceAccountTokenCreationRole}} + namespace: {{.TestNamespace}} +rules: +- apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create + - get +` + serviceAccountTokenCreationRoleBindingTemplate = ` +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{.ServiceAccountTokenCreationRoleBinding}} + namespace: {{.TestNamespace}} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{.ServiceAccountTokenCreationRole}} +subjects: +- kind: ServiceAccount + name: keda-operator + namespace: keda +` + tokenReviewAndSubjectAccessReviewClusterRoleTemplate = ` +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: token-review-and-subject-access-review-role +rules: +- apiGroups: + - "authentication.k8s.io" + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +` + tokenReviewAndSubjectAccessReviewClusterRoleBindingTemplate = ` +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: token-review-and-subject-access-review-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: token-review-and-subject-access-review-role +subjects: +- kind: ServiceAccount + name: default + namespace: {{.TestNamespace}} +` + // TODO(macao): replace the image when https://github.com/kedacore/test-tools/pull/186 merged + metricsServerDeploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: {{.MetricsServerDeploymentName}} + name: {{.MetricsServerDeploymentName}} + namespace: {{.TestNamespace}} +spec: + replicas: 1 + selector: + matchLabels: + app: {{.MetricsServerDeploymentName}} + template: + metadata: + labels: + app: {{.MetricsServerDeploymentName}} + type: keda-testing + spec: + containers: + - name: k8s-protected-metrics-api + image: quay.io/macao/metrics-api:latest + imagePullPolicy: Always + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault +` + metricsServerService = ` +apiVersion: v1 +kind: Service +metadata: + name: {{.MetricsServerServiceName}} + namespace: {{.TestNamespace}} +spec: + ports: + - name: http + port: 8080 + targetPort: 8080 + selector: + app: {{.MetricsServerDeploymentName}} +` + deploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: {{.DeploymentName}} + name: {{.DeploymentName}} + namespace: {{.TestNamespace}} +spec: + replicas: 0 + selector: + matchLabels: + app: {{.DeploymentName}} + template: + metadata: + labels: + app: {{.DeploymentName}} + type: keda-testing + spec: + containers: + - name: prom-test-app + image: ghcr.io/kedacore/tests-prometheus:latest + imagePullPolicy: Always + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault +` + triggerAuthTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: {{.TriggerAuthName}} + namespace: {{.TestNamespace}} +spec: + boundServiceAccountToken: + - parameter: token + serviceAccountName: {{.ServiceAccountName}} +` + scaledObjectTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{.ScaledObjectName}} + namespace: {{.TestNamespace}} +spec: + scaleTargetRef: + name: {{.DeploymentName}} + pollingInterval: 5 + minReplicaCount: 0 + maxReplicaCount: 1 + cooldownPeriod: 10 + triggers: + - type: metrics-api + metadata: + targetValue: "10" + url: "{{.MetricsServerEndpoint}}" + valueLocation: 'value' + authMode: "bearer" + authenticationRef: + name: {{.TriggerAuthName}} +` + updateMetricTemplate = `apiVersion: batch/v1 +kind: Job +metadata: + name: update-metric-value + namespace: {{.TestNamespace}} +spec: + ttlSecondsAfterFinished: 0 + template: + spec: + containers: + - name: curl-client + image: curlimages/curl + imagePullPolicy: Always + command: ["curl", "-X", "POST", "{{.MetricsServerEndpoint}}/{{.MetricValue}}"] + restartPolicy: Never` +) + +func TestScaler(t *testing.T) { + // setup + // ctx := context.Background() + t.Log("--- setting up ---") + + // Create kubernetes resources + kc := GetKubernetesClient(t) + data, templates := getTemplateData() + + CreateKubernetesResources(t, kc, testNamespace, data, templates) + + // wait for metrics server to be ready; scale target to start at 0 replicas + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, metricsServerDeploymentName, testNamespace, 1, 60, 1), + "replica count should be 1 after 1 minute") + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 0, 60, 1), + "replica count should be 0 after 1 minute") + + // test scaling + testScaleOut(t, kc, data) + testScaleIn(t, kc, data) + + // cleanup + DeleteKubernetesResources(t, testNamespace, data, templates) +} + +func getTemplateData() (templateData, []Template) { + return templateData{ + TestNamespace: testNamespace, + ServiceAccountName: serviceAccountName, + ServiceAccountTokenCreationRole: serviceAccountTokenCreationRole, + ServiceAccountTokenCreationRoleBinding: serviceAccountTokenCreationRoleBinding, + MetricsServerDeploymentName: metricsServerDeploymentName, + MetricsServerEndpoint: metricsServerEndpoint, + MetricsServerServiceName: metricsServerServiceName, + DeploymentName: deploymentName, + TriggerAuthName: triggerAuthName, + ScaledObjectName: scaledObjectName, + MinReplicaCount: fmt.Sprintf("%d", minReplicaCount), + MaxReplicaCount: fmt.Sprintf("%d", maxReplicaCount), + MetricValue: 1, + }, []Template{ + // required for the keda to act as the service account which has the necessary permissions + {Name: "serviceAccountTemplate", Config: serviceAccountTemplate}, + {Name: "serviceAccountClusterRoleTemplate", Config: serviceAccountClusterRoleTemplate}, + {Name: "serviceAccountClusterRoleBindingTemplate", Config: serviceAccountClusterRoleBindingTemplate}, + // required for the keda to request token creations for the service account + {Name: "serviceAccountTokenCreationRoleTemplate", Config: serviceAccountTokenCreationRoleTemplate}, + {Name: "serviceAccountTokenCreationRoleBindingTemplate", Config: serviceAccountTokenCreationRoleBindingTemplate}, + // required for the metrics-api container to delegate authenticate/authorize requests to k8s apiserver + {Name: "tokenReviewAndSubjectAccessReviewClusterRoleTemplate", Config: tokenReviewAndSubjectAccessReviewClusterRoleTemplate}, + {Name: "tokenReviewAndSubjectAccessReviewClusterRoleBindingTemplate", Config: tokenReviewAndSubjectAccessReviewClusterRoleBindingTemplate}, + {Name: "metricsServerDeploymentTemplate", Config: metricsServerDeploymentTemplate}, + {Name: "metricsServerService", Config: metricsServerService}, + // scale target and trigger auths + {Name: "deploymentTemplate", Config: deploymentTemplate}, + {Name: "triggerAuthTemplate", Config: triggerAuthTemplate}, + {Name: "scaledObjectTemplate", Config: scaledObjectTemplate}, + } +} + +func testScaleOut(t *testing.T, kc *kubernetes.Clientset, data templateData) { + t.Log("--- testing scale out ---") + data.MetricValue = 50 + KubectlReplaceWithTemplate(t, data, "updateMetricTemplate", updateMetricTemplate) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 60, maxReplicaCount), + "replica count should be %d after 3 minutes", maxReplicaCount) +} + +func testScaleIn(t *testing.T, kc *kubernetes.Clientset, data templateData) { + t.Log("--- testing scale in ---") + data.MetricValue = 0 + KubectlReplaceWithTemplate(t, data, "updateMetricTemplate", updateMetricTemplate) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 60, 3), + "replica count should be %d after 3 minutes", minReplicaCount) +} From 42ea3fb138e977cb67ecbf86d71583727a2cb042 Mon Sep 17 00:00:00 2001 From: Max Cao Date: Thu, 2 Jan 2025 10:00:22 -0800 Subject: [PATCH 12/12] go mod tidy; replace bound-service-account-token test image with correct merged ghcr test image Signed-off-by: Max Cao --- go.mod | 2 +- .../trigger_auth_bound_service_account_token_test.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3d53b2099c6..5fc7977aee5 100644 --- a/go.mod +++ b/go.mod @@ -211,7 +211,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.24.8 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7 // indirect - github.com/aws/smithy-go v1.22.1 + github.com/aws/smithy-go v1.22.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect diff --git a/tests/secret-providers/trigger_auth_bound_service_account_token/trigger_auth_bound_service_account_token_test.go b/tests/secret-providers/trigger_auth_bound_service_account_token/trigger_auth_bound_service_account_token_test.go index 04bdd738f25..94a4dba01bb 100644 --- a/tests/secret-providers/trigger_auth_bound_service_account_token/trigger_auth_bound_service_account_token_test.go +++ b/tests/secret-providers/trigger_auth_bound_service_account_token/trigger_auth_bound_service_account_token_test.go @@ -149,7 +149,6 @@ subjects: name: default namespace: {{.TestNamespace}} ` - // TODO(macao): replace the image when https://github.com/kedacore/test-tools/pull/186 merged metricsServerDeploymentTemplate = ` apiVersion: apps/v1 kind: Deployment @@ -171,7 +170,7 @@ spec: spec: containers: - name: k8s-protected-metrics-api - image: quay.io/macao/metrics-api:latest + image: ghcr.io/kedacore/tests-bound-service-account-token:latest imagePullPolicy: Always securityContext: allowPrivilegeEscalation: false