Add option to pass configmap with mapping between SA and IAM role (#142)

This adds an option for the webhook to watch a configmap for additional service account to Role mappings. This is useful where there are tooling that creates IAM Roles and already know what service account should use them. In particular kOps already has this mapping. Adding the annotation to the service accounts is then just additional manual work.

* Add option to use a configmap as source of mappings between service account and IAM role.
* Creates a separate informer and cache for the configmap.
* Enforces precedence between service account annotations and configmap where service accounts have the higher precedence.

README.md

@@ -143,11 +143,12 @@ When running a container with a non-root user, you need to give the container ac
 
 ```
 Usage of amazon-eks-pod-identity-webhook:
+      --add_dir_header                   If true, adds the file directory to the header
       --alsologtostderr                  log to standard error as well as files
       --annotation-prefix string         The Service Account annotation to look for (default "eks.amazonaws.com")
       --aws-default-region string        If set, AWS_DEFAULT_REGION and AWS_REGION will be set to this value in mutated containers
+      --enable-debugging-handlers        Enable debugging handlers. Currently /debug/alpha/cache is supported
       --in-cluster                       Use in-cluster authentication and certificate request API (default true)
-      --enable-debugging-handlers        Enable debugging handlers on the metrics port (http). Currently /debug/alpha/cache is supported (default false) [ALPHA]
       --kube-api string                  (out-of-cluster) The url to the API server
       --kubeconfig string                (out-of-cluster) Absolute path to the API server kubeconfig file
       --log_backtrace_at traceLocation   when logging hits line file:N, emit a stack trace (default :0)
@@ -156,11 +157,11 @@ Usage of amazon-eks-pod-identity-webhook:
       --log_file_max_size uint           Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800)
       --logtostderr                      log to standard error instead of files (default true)
       --metrics-port int                 Port to listen on for metrics and healthz (http) (default 9999)
-      --namespace string                 (in-cluster) The namespace name this webhook and the tls secret resides in (default "eks")
+      --namespace string                 (in-cluster) The namespace name this webhook, the TLS secret, and configmap resides in (default "eks")
       --port int                         Port to listen on (default 443)
       --service-name string              (in-cluster) The service name fronting this webhook (default "pod-identity-webhook")
       --skip_headers                     If true, avoid header prefixes in the log messages
-      --skip_log_headers                 If true, avoid headers when openning log files
+      --skip_log_headers                 If true, avoid headers when opening log files
       --stderrthreshold severity         logs at or above this threshold go to stderr (default 2)
       --sts-regional-endpoint false      Whether to inject the AWS_STS_REGIONAL_ENDPOINTS=regional env var in mutated pods. Defaults to false.
       --tls-cert string                  (out-of-cluster) TLS certificate file path (default "/etc/webhook/certs/tls.crt")
@@ -172,6 +173,7 @@ Usage of amazon-eks-pod-identity-webhook:
   -v, --v Level                          number for the log level verbosity
       --version                          Display the version and exit
       --vmodule moduleSpec               comma-separated list of pattern=N settings for file-filtered logging
+      --watch-config-map                 Enables watching serviceaccounts that are configured through the pod-identity-webhook configmap instead of using annotations
 ```
 
 ### AWS_DEFAULT_REGION Injection
@@ -191,6 +193,37 @@ account](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_en
 You can also enable this per-service account with the annotation
 `eks.amazonaws.com/sts-regional-endpoints` set to `"true"`.
 
+### pod-identity-webhook ConfigMap
+
+The purpose of the `pod-identity-webhook` ConfigMap is to simplify the mapping of IAM roles and ServiceAccount
+when using tools/installers like [kOps](https://kops.sigs.k8s.io/) that directly manage IAM roles and trust policies. When using these tools,
+users do not need to configure annotations on the ServiceAccounts as the tools already know the relationship can relay it to the webhook.
+
+When the `watch-config-map` flag is set to `true`, the webhook will watch the
+`pod-identity-webhook` ConfigMap in the namespace configured by the `--namespace` flag
+for additional ServiceAccounts. The webhook will mutate Pods configured to use these
+ServiceAccounts even if they have no annotations.
+
+Should the same ServiceAccount both be referenced both in the ConfigMap and have annotations, the annotations takes presedence. 
+
+Here is an example ConfigMap:
+
+```
+apiVersion: v1
+data:
+  config: '{"default/myserviceaccount":{"RoleARN":"arn:aws-test:iam::123456789012:role/myserviceaccount.default.sa.minimal.example.com","Audience":"amazonaws.com","UseRegionalSTS":true,"TokenExpiration":0},"myapp/myotherserviceaccount":{"RoleARN":"arn:aws-test:iam::123456789012:role/myotherserviceaccount.myapp.sa.minimal.example.com","Audience":"amazonaws.com","UseRegionalSTS":true,"TokenExpiration":0},"test-*/myserviceaccount":{"RoleARN":"arn:aws-test:iam::123456789012:role/myserviceaccount.test-wildcard.sa.minimal.example.com","Audience":"amazonaws.com","UseRegionalSTS":true,"TokenExpiration":0}}'
+kind: ConfigMap
+metadata:
+  annotations:
+    prometheus.io/port: "443"
+    prometheus.io/scheme: https
+    prometheus.io/scrape: "true"
+  creationTimestamp: null
+  name: pod-identity-webhook
+  namespace: kube-system
+```
+
+
 ## Container Images
 
 Container images for amazon-eks-pod-identity-webhook can be found on [Docker Hub](https://hub.docker.com/r/amazon/amazon-eks-pod-identity-webhook).

main.go

@@ -33,6 +33,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	flag "github.com/spf13/pflag"
 	"k8s.io/client-go/informers"
+	v1 "k8s.io/client-go/informers/core/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog"
@@ -58,7 +59,7 @@ func main() {
 	// in-cluster TLS options
 	inCluster := flag.Bool("in-cluster", true, "Use in-cluster authentication and certificate request API")
 	serviceName := flag.String("service-name", "pod-identity-webhook", "(in-cluster) The service name fronting this webhook")
-	namespaceName := flag.String("namespace", "eks", "(in-cluster) The namespace name this webhook and the tls secret resides in")
+	namespaceName := flag.String("namespace", "eks", "(in-cluster) The namespace name this webhook, the TLS secret, and configmap resides in")
 	tlsSecret := flag.String("tls-secret", "pod-identity-webhook", "(in-cluster) The secret name for storing the TLS serving cert")
 
 	// annotation/volume configurations
@@ -68,6 +69,7 @@ func main() {
 	tokenExpiration := flag.Int64("token-expiration", pkg.DefaultTokenExpiration, "The token expiration")
 	region := flag.String("aws-default-region", "", "If set, AWS_DEFAULT_REGION and AWS_REGION will be set to this value in mutated containers")
 	regionalSTS := flag.Bool("sts-regional-endpoint", false, "Whether to inject the AWS_STS_REGIONAL_ENDPOINTS=regional env var in mutated pods. Defaults to `false`.")
+	watchConfigMap := flag.Bool("watch-config-map", false, "Enables watching serviceaccounts that are configured through the pod-identity-webhook configmap instead of using annotations")
 
 	version := flag.Bool("version", false, "Display the version and exit")
 
@@ -101,18 +103,33 @@ func main() {
 		klog.Fatalf("Error creating clientset: %v", err.Error())
 	}
 	informerFactory := informers.NewSharedInformerFactory(clientset, 60*time.Second)
-	informer := informerFactory.Core().V1().ServiceAccounts()
+
+	var cmInformer v1.ConfigMapInformer
+	var nsInformerFactory informers.SharedInformerFactory
+	if *watchConfigMap {
+		klog.Infof("Watching ConfigMap pod-identity-webhook in %s namespace", *namespaceName)
+		nsInformerFactory = informers.NewSharedInformerFactoryWithOptions(clientset, 60*time.Second, informers.WithNamespace(*namespaceName))
+		cmInformer = nsInformerFactory.Core().V1().ConfigMaps()
+	}
+
+	saInformer := informerFactory.Core().V1().ServiceAccounts()
 
 	*tokenExpiration = pkg.ValidateMinTokenExpiration(*tokenExpiration)
 	saCache := cache.New(
 		*audience,
 		*annotationPrefix,
 		*regionalSTS,
 		*tokenExpiration,
-		informer,
+		saInformer,
+		cmInformer,
 	)
 	stop := make(chan struct{})
 	informerFactory.Start(stop)
+
+	if *watchConfigMap {
+		nsInformerFactory.Start(stop)
+	}
+
 	saCache.Start(stop)
 	defer close(stop)
 

pkg/cache/cache.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"strconv"
+	"strings"
 	"sync"
 
 	"github.com/aws/amazon-eks-pod-identity-webhook/pkg"
@@ -47,7 +48,8 @@ type ServiceAccountCache interface {
 
 type serviceAccountCache struct {
 	mu                     sync.RWMutex // guards cache
-	cache                  map[string]*CacheResponse
+	saCache                map[string]*CacheResponse
+	cmCache                map[string]*CacheResponse
 	hasSynced              cache.InformerSynced
 	clientset              kubernetes.Interface
 	annotationPrefix       string
@@ -71,38 +73,66 @@ func init() {
 	prometheus.MustRegister(webhookUsage)
 }
 
+// Get will return the cached configuration of the given ServiceAccount.
+// It will first look at the set of ServiceAccounts configured using annotations. If none are found, it will look for any
+// ServiceAccount configured through the pod-identity-webhook ConfigMap.
 func (c *serviceAccountCache) Get(name, namespace string) (role, aud string, useRegionalSTS bool, tokenExpiration int64) {
 	klog.V(5).Infof("Fetching sa %s/%s from cache", namespace, name)
-	resp := c.get(name, namespace)
-	if resp == nil {
-		klog.V(4).Infof("Service account %s/%s not found in cache", namespace, name)
-		return "", "", false, pkg.DefaultTokenExpiration
+	{
+		resp := c.getSA(name, namespace)
+		if resp != nil && resp.RoleARN != "" {
+			return resp.RoleARN, resp.Audience, resp.UseRegionalSTS, resp.TokenExpiration
+		}
+	}
+	{
+		resp := c.getCM(name, namespace)
+		if resp != nil {
+			return resp.RoleARN, resp.Audience, resp.UseRegionalSTS, resp.TokenExpiration
+		}
 	}
-	return resp.RoleARN, resp.Audience, resp.UseRegionalSTS, resp.TokenExpiration
+	klog.V(5).Infof("Service account %s/%s not found in cache", namespace, name)
+	return "", "", false, pkg.DefaultTokenExpiration
 }
 
-func (c *serviceAccountCache) get(name, namespace string) *CacheResponse {
+func (c *serviceAccountCache) getSA(name, namespace string) *CacheResponse {
 	c.mu.RLock()
 	defer c.mu.RUnlock()
-	resp, ok := c.cache[namespace+"/"+name]
+	resp, ok := c.saCache[namespace+"/"+name]
 	if !ok {
 		return nil
 	}
 	return resp
 }
 
-func (c *serviceAccountCache) pop(name, namespace string) {
-	klog.V(5).Infof("Removing sa %s/%s from cache", namespace, name)
+func (c *serviceAccountCache) getCM(name, namespace string) *CacheResponse {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	resp, ok := c.cmCache[namespace+"/"+name]
+	if !ok {
+		return nil
+	}
+	return resp
+}
+
+func (c *serviceAccountCache) popSA(name, namespace string) {
+	klog.V(5).Infof("Removing SA %s/%s from SA cache", namespace, name)
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	delete(c.saCache, namespace+"/"+name)
+}
+
+func (c *serviceAccountCache) popCM(name, namespace string) {
+	klog.V(5).Infof("Removing SA %s/%s from CM cache", namespace, name)
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	delete(c.cache, namespace+"/"+name)
+	delete(c.cmCache, namespace+"/"+name)
 }
 
 // Log cache contents for debugginqg
 func (c *serviceAccountCache) ToJSON() string {
 	c.mu.RLock()
 	defer c.mu.RUnlock()
-	contents, err := json.MarshalIndent(c.cache, "", " ")
+	contents, err := json.MarshalIndent(c.saCache, "", " ")
 	if err != nil {
 		klog.Errorf("Json marshal error: %v", err.Error())
 		return ""
@@ -140,28 +170,43 @@ func (c *serviceAccountCache) addSA(sa *v1.ServiceAccount) {
 		}
 		c.webhookUsage.Set(1)
 	}
-	klog.V(5).Infof("Adding sa %s/%s to cache: %+v", sa.Name, sa.Namespace, resp)
-	c.set(sa.Name, sa.Namespace, resp)
+	c.setSA(sa.Name, sa.Namespace, resp)
+}
+
+func (c *serviceAccountCache) setSA(name, namespace string, resp *CacheResponse) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	klog.V(5).Infof("Adding SA %s/%s to SA cache: %+v", namespace, name, resp)
+	c.saCache[namespace+"/"+name] = resp
 }
 
-func (c *serviceAccountCache) set(name, namespace string, resp *CacheResponse) {
+func (c *serviceAccountCache) setCM(name, namespace string, resp *CacheResponse) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	c.cache[namespace+"/"+name] = resp
+	klog.V(5).Infof("Adding SA %s/%s to CM cache: %+v", namespace, name, resp)
+	c.cmCache[namespace+"/"+name] = resp
 }
 
-func New(defaultAudience, prefix string, defaultRegionalSTS bool, defaultTokenExpiration int64, informer coreinformers.ServiceAccountInformer) ServiceAccountCache {
+func New(defaultAudience, prefix string, defaultRegionalSTS bool, defaultTokenExpiration int64, saInformer coreinformers.ServiceAccountInformer, cmInformer coreinformers.ConfigMapInformer) ServiceAccountCache {
+	hasSynced := func() bool {
+		if cmInformer != nil {
+			return saInformer.Informer().HasSynced() && cmInformer.Informer().HasSynced()
+		} else {
+			return saInformer.Informer().HasSynced()
+		}
+	}
 	c := &serviceAccountCache{
-		cache:                  map[string]*CacheResponse{},
+		saCache:                map[string]*CacheResponse{},
+		cmCache:                map[string]*CacheResponse{},
 		defaultAudience:        defaultAudience,
 		annotationPrefix:       prefix,
 		defaultRegionalSTS:     defaultRegionalSTS,
 		defaultTokenExpiration: defaultTokenExpiration,
-		hasSynced:              informer.Informer().HasSynced,
+		hasSynced:              hasSynced,
 		webhookUsage:           webhookUsage,
 	}
 
-	informer.Informer().AddEventHandler(
+	saInformer.Informer().AddEventHandler(
 		cache.ResourceEventHandlerFuncs{
 			AddFunc: func(obj interface{}) {
 				sa := obj.(*v1.ServiceAccount)
@@ -181,17 +226,70 @@ func New(defaultAudience, prefix string, defaultRegionalSTS bool, defaultTokenEx
 						return
 					}
 				}
-				c.pop(sa.Name, sa.Namespace)
+				c.popSA(sa.Name, sa.Namespace)
 			},
 			UpdateFunc: func(oldObj, newObj interface{}) {
 				sa := newObj.(*v1.ServiceAccount)
 				c.addSA(sa)
 			},
 		},
 	)
+	if cmInformer != nil {
+		cmInformer.Informer().AddEventHandler(
+			cache.ResourceEventHandlerFuncs{
+				AddFunc: func(obj interface{}) {
+					err := c.populateCacheFromCM(nil, obj.(*v1.ConfigMap))
+					if err != nil {
+						utilruntime.HandleError(err)
+					}
+				},
+				UpdateFunc: func(oldObj, newObj interface{}) {
+					err := c.populateCacheFromCM(oldObj.(*v1.ConfigMap), newObj.(*v1.ConfigMap))
+					if err != nil {
+						utilruntime.HandleError(err)
+					}
+				},
+			},
+		)
+	}
 	return c
 }
 
+func (c *serviceAccountCache) populateCacheFromCM(oldCM, newCM *v1.ConfigMap) error {
+	if newCM.Name != "pod-identity-webhook" {
+		return nil
+	}
+	newConfig := newCM.Data["config"]
+	sas := make(map[string]*CacheResponse)
+	err := json.Unmarshal([]byte(newConfig), &sas)
+	if err != nil {
+		return fmt.Errorf("failed to unmarshal new config %q: %v", newConfig, err)
+	}
+	for key, resp := range sas {
+		parts := strings.Split(key, "/")
+		if resp.TokenExpiration == 0 {
+			resp.TokenExpiration = c.defaultTokenExpiration
+		}
+		c.setCM(parts[1], parts[0], resp)
+	}
+
+	if oldCM != nil {
+		oldConfig := oldCM.Data["config"]
+		oldCache := make(map[string]*CacheResponse)
+		err := json.Unmarshal([]byte(oldConfig), &oldCache)
+		if err != nil {
+			return fmt.Errorf("failed to unmarshal old config %q: %v", oldConfig, err)
+		}
+		for key := range oldCache {
+			if _, found := sas[key]; !found {
+				parts := strings.Split(key, "/")
+				c.popCM(parts[1], parts[0])
+			}
+		}
+	}
+	return nil
+}
+
 func (c *serviceAccountCache) start(stop chan struct{}) {
 
 	if !cache.WaitForCacheSync(stop, c.hasSynced) {