From c44ee59c2cabb23d46a84096dad72ce93b712622 Mon Sep 17 00:00:00 2001 From: Antony Dovgal Date: Mon, 5 Feb 2024 21:52:34 +0200 Subject: [PATCH] Add Argo Rollouts support --- go.mod | 3 ++- go.sum | 12 +++++++----- main.go | 8 ++++++++ pkg/cli/cli.go | 2 +- pkg/cli/cli_test.go | 4 ++-- pkg/registry/checker.go | 30 ++++++++++++++++++++++++++++++ pkg/registry/indexers.go | 23 ++++++++++++++++++++++- 7 files changed, 72 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 9ea05bf..0bfbf27 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/flant/k8s-image-availability-exporter go 1.21 require ( + github.com/argoproj/argo-rollouts v1.6.5 github.com/gammazero/deque v0.2.1 github.com/google/go-containerregistry v0.18.0 github.com/google/go-containerregistry/pkg/authn/kubernetes v0.0.0-20231202142526-55ffb0092afd @@ -36,7 +37,7 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/imdario/mergo v0.3.12 // indirect + github.com/imdario/mergo v0.3.13 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.5 // indirect diff --git a/go.sum b/go.sum index 7baa946..e2d989c 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/argoproj/argo-rollouts v1.6.5 h1:VDAp9PGboRbzd9tQJ/8IkaI+KrvWIRrpfSV5aeX0GUQ= +github.com/argoproj/argo-rollouts v1.6.5/go.mod h1:X2kTiBaYCSounmw1kmONdIZTwJNzNQYC0SrXUgSw9UI= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= @@ -20,8 +22,8 @@ github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryef github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= -github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= +github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0= github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -60,8 +62,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJY github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -190,10 +192,10 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.1.0 h1:rVV8Tcg/8jHUkPUorwjaMTtemIMVXfIPKiOqnhEhakk= diff --git a/main.go b/main.go index fe58bca..c85458d 100644 --- a/main.go +++ b/main.go @@ -20,6 +20,8 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + argo "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" @@ -62,6 +64,11 @@ func main() { logrus.Fatalf("Error building kubernetes clientset: %s", err.Error()) } + argoKubeClient, err := argo.NewForConfig(cfg) + if err != nil { + logrus.Fatalf("Error building argo clientset: %s", err.Error()) + } + liveTicksCounter := prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "k8s_image_availability_exporter", @@ -82,6 +89,7 @@ func main() { registryChecker := registry.NewChecker( stopCh.Done(), kubeClient, + argoKubeClient, *insecureSkipVerify, *plainHTTP, *cp, diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 724ee46..af2bf37 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -40,6 +40,6 @@ flagLoop: func NewForceCheckDisabledControllerKindsParser() *ForceCheckDisabledControllerKindsParser { parser := &ForceCheckDisabledControllerKindsParser{} - parser.allowedControllerKinds = []string{"deployment", "statefulset", "daemonset", "cronjob"} + parser.allowedControllerKinds = []string{"deployment", "statefulset", "daemonset", "cronjob", "rollout"} return parser } diff --git a/pkg/cli/cli_test.go b/pkg/cli/cli_test.go index 18d0cde..5425b46 100644 --- a/pkg/cli/cli_test.go +++ b/pkg/cli/cli_test.go @@ -12,7 +12,7 @@ func Test_ForceCheckDisabledControllerKindsParser(t *testing.T) { const ( allKinds = "*" goodKinds = "deployment,statefulset" - goodKindsWithDuplicates = "deployment,deployment,statefulset,cronjob,cronjob" + goodKindsWithDuplicates = "deployment,deployment,statefulset,cronjob,cronjob,rollout" goodKindsWithWildcard = "deployment,statefulset,*" badKinds = "deployment,job" ) @@ -29,7 +29,7 @@ func Test_ForceCheckDisabledControllerKindsParser(t *testing.T) { err = parser.Parse(goodKindsWithDuplicates) require.NoError(t, err) - require.Equal(t, parser.ParsedKinds, []string{"deployment", "statefulset", "cronjob"}) + require.Equal(t, parser.ParsedKinds, []string{"deployment", "statefulset", "cronjob", "rollout"}) err = parser.Parse(goodKindsWithWildcard) require.Error(t, expectedErr, err) diff --git a/pkg/registry/checker.go b/pkg/registry/checker.go index 5a51b56..3c5bc0c 100644 --- a/pkg/registry/checker.go +++ b/pkg/registry/checker.go @@ -20,12 +20,15 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/sirupsen/logrus" + argov1informers "github.com/argoproj/argo-rollouts/pkg/client/informers/externalversions/rollouts/v1alpha1" appsv1informers "k8s.io/client-go/informers/apps/v1" batchv1informers "k8s.io/client-go/informers/batch/v1" corev1informers "k8s.io/client-go/informers/core/v1" + argoinformers "github.com/argoproj/argo-rollouts/pkg/client/informers/externalversions" "k8s.io/client-go/informers" + argo "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned" "k8s.io/client-go/kubernetes" "github.com/flant/k8s-image-availability-exporter/pkg/store" @@ -51,6 +54,7 @@ type Checker struct { daemonSetsInformer appsv1informers.DaemonSetInformer cronJobsInformer batchv1informers.CronJobInformer secretsInformer corev1informers.SecretInformer + rolloutsInformer argov1informers.RolloutInformer controllerIndexers ControllerIndexers @@ -66,6 +70,7 @@ type Checker struct { func NewChecker( stopCh <-chan struct{}, kubeClient *kubernetes.Clientset, + argoKubeClient *argo.Clientset, skipVerify bool, plainHTTP bool, caPths []string, @@ -75,6 +80,7 @@ func NewChecker( namespaceLabel string, ) *Checker { informerFactory := informers.NewSharedInformerFactory(kubeClient, time.Hour) + argoInformerFactory := argoinformers.NewSharedInformerFactory(argoKubeClient, time.Hour) customTransport := http.DefaultTransport.(*http.Transport).Clone() if skipVerify { @@ -104,6 +110,7 @@ func NewChecker( daemonSetsInformer: informerFactory.Apps().V1().DaemonSets(), cronJobsInformer: informerFactory.Batch().V1().CronJobs(), secretsInformer: informerFactory.Core().V1().Secrets(), + rolloutsInformer: argoInformerFactory.Argoproj().V1alpha1().Rollouts(), ignoredImagesRegex: ignoredImages, @@ -172,6 +179,27 @@ func NewChecker( } rc.controllerIndexers.statefulSetIndexer = rc.statefulSetsInformer.Informer().GetIndexer() + _, _ = rc.rolloutsInformer.Informer().AddEventHandlerWithResyncPeriod(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + rc.reconcile(obj) + }, + UpdateFunc: func(_, newObj interface{}) { + rc.reconcile(newObj) + }, + DeleteFunc: func(obj interface{}) { + rc.reconcile(obj) + }, + }, time.Minute) + err = rc.rolloutsInformer.Informer().AddIndexers(imageIndexers) + if err != nil { + panic(err) + } + err = rc.rolloutsInformer.Informer().SetTransform(getImagesFromRollout) + if err != nil { + panic(err) + } + rc.controllerIndexers.rolloutIndexer = rc.rolloutsInformer.Informer().GetIndexer() + _, _ = rc.daemonSetsInformer.Informer().AddEventHandlerWithResyncPeriod(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { rc.reconcile(obj) @@ -219,8 +247,10 @@ func NewChecker( rc.controllerIndexers.forceCheckDisabledControllerKinds = forceCheckDisabledControllerKinds go informerFactory.Start(stopCh) + go argoInformerFactory.Start(stopCh) logrus.Info("Waiting for cache sync") informerFactory.WaitForCacheSync(stopCh) + argoInformerFactory.WaitForCacheSync(stopCh) logrus.Info("Caches populated successfully") rc.imageStore.RunGC(rc.controllerIndexers.GetContainerInfosForImage) diff --git a/pkg/registry/indexers.go b/pkg/registry/indexers.go index 086e15e..733baab 100644 --- a/pkg/registry/indexers.go +++ b/pkg/registry/indexers.go @@ -6,6 +6,7 @@ import ( "slices" "strings" + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" "github.com/flant/k8s-image-availability-exporter/pkg/store" "github.com/google/go-containerregistry/pkg/authn" kubeauth "github.com/google/go-containerregistry/pkg/authn/kubernetes" @@ -30,6 +31,7 @@ type ControllerIndexers struct { daemonSetIndexer cache.Indexer cronJobIndexer cache.Indexer secretIndexer cache.Indexer + rolloutIndexer cache.Indexer forceCheckDisabledControllerKinds []string } @@ -103,6 +105,25 @@ func getImagesFromDeployment(obj interface{}) (interface{}, error) { }, nil } +func getImagesFromRollout(obj interface{}) (interface{}, error) { + if cis, ok := obj.(*controllerWithContainerInfos); ok { + return cis, nil + } + + rollout := obj.(*v1alpha1.Rollout) + + rolloutCopy := rollout.DeepCopy() + + return &controllerWithContainerInfos{ + ObjectMeta: rolloutCopy.ObjectMeta, + controllerKind: "Rollout", + containerToImages: extractImagesFromContainers(rolloutCopy.Spec.Template.Spec.Containers), + pullSecretReferences: rolloutCopy.Spec.Template.Spec.ImagePullSecrets, + serviceAccountName: rolloutCopy.Spec.Template.Spec.ServiceAccountName, + enabled: *rolloutCopy.Spec.Replicas > 0, + }, nil +} + func getImagesFromStatefulSet(obj interface{}) (interface{}, error) { if cis, ok := obj.(*controllerWithContainerInfos); ok { return cis, nil @@ -219,7 +240,7 @@ func (ci ControllerIndexers) ExtractPullSecretRefs(obj interface{}) (ret []strin } func (ci ControllerIndexers) GetObjectsByImageIndex(image string) (ret []interface{}) { - for _, indexer := range []cache.Indexer{ci.deploymentIndexer, ci.statefulSetIndexer, ci.daemonSetIndexer, ci.cronJobIndexer} { + for _, indexer := range []cache.Indexer{ci.deploymentIndexer, ci.statefulSetIndexer, ci.daemonSetIndexer, ci.cronJobIndexer, ci.rolloutIndexer} { objs, err := indexer.ByIndex(imageIndexName, image) if err != nil { panic(err)