From 58c329380020a6841ad8cdaae33b71c8c4414749 Mon Sep 17 00:00:00 2001 From: Paul Laffitte Date: Fri, 17 Nov 2023 12:34:29 +0100 Subject: [PATCH] feat: allow to add additional root CAs to trust for caching and proxying --- cmd/cache/main.go | 9 ++++++ cmd/proxy/main.go | 9 +++++- controllers/cachedimage_controller.go | 4 ++- .../templates/controller-deployment.yaml | 20 +++++++++++-- .../templates/proxy-daemonset.yaml | 18 +++++++++++ helm/kube-image-keeper/values.yaml | 4 +++ internal/proxy/server.go | 18 ++++++----- internal/proxy/server_test.go | 2 +- internal/registry/certificates.go | 30 +++++++++++++++++++ internal/registry/registry.go | 13 ++++---- internal/registry/registry_test.go | 2 +- 11 files changed, 109 insertions(+), 20 deletions(-) create mode 100644 internal/registry/certificates.go diff --git a/cmd/cache/main.go b/cmd/cache/main.go index 6dc25f08..2cc1d33d 100644 --- a/cmd/cache/main.go +++ b/cmd/cache/main.go @@ -38,6 +38,7 @@ func main() { var architectures internal.ArrayFlags var maxConcurrentCachedImageReconciles int var insecureRegistries internal.ArrayFlags + var rootCAPaths internal.ArrayFlags flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, @@ -50,6 +51,7 @@ func main() { flag.StringVar(®istry.Endpoint, "registry-endpoint", "kube-image-keeper-registry:5000", "The address of the registry where cached images are stored.") flag.IntVar(&maxConcurrentCachedImageReconciles, "max-concurrent-cached-image-reconciles", 3, "Maximum number of CachedImages that can be handled and reconciled at the same time (put or removed from cache).") flag.Var(&insecureRegistries, "insecure-registries", "Insecure registries to allow to cache and proxify images from (this flag can be used multiple times).") + flag.Var(&rootCAPaths, "root-certificate-authorities", "Root certificate authorities to trust.") opts := zap.Options{ Development: true, @@ -74,6 +76,12 @@ func main() { os.Exit(1) } + rootCAs, err := registry.LoadRootCAPoolFromFiles(rootCAPaths) + if err != nil { + setupLog.Error(err, "could not load root certificate authorities") + os.Exit(1) + } + if err = (&controllers.CachedImageReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), @@ -82,6 +90,7 @@ func main() { ExpiryDelay: time.Duration(expiryDelay*24) * time.Hour, Architectures: []string(architectures), InsecureRegistries: []string(insecureRegistries), + RootCAs: rootCAs, }).SetupWithManager(mgr, maxConcurrentCachedImageReconciles); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CachedImage") os.Exit(1) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 47d8a869..47b05097 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -25,6 +25,7 @@ var ( rateLimitQPS int rateLimitBurst int insecureRegistries internal.ArrayFlags + rootCAPaths internal.ArrayFlags ) func initFlags() { @@ -38,6 +39,7 @@ func initFlags() { flag.IntVar(&rateLimitQPS, "kube-api-rate-limit-qps", 0, "Kubernetes API request rate limit") flag.IntVar(&rateLimitBurst, "kube-api-rate-limit-burst", 0, "Kubernetes API request burst") flag.Var(&insecureRegistries, "insecure-registries", "Insecure registries to allow to cache and proxify images from (this flag can be used multiple times).") + flag.Var(&rootCAPaths, "root-certificate-authorities", "Root certificate authorities to trust.") flag.Parse() } @@ -81,5 +83,10 @@ func main() { panic(err) } - <-proxy.New(k8sClient, metricsAddr, []string(insecureRegistries)).Run() + rootCAs, err := registry.LoadRootCAPoolFromFiles(rootCAPaths) + if err != nil { + panic(fmt.Errorf("could not load root certificate authorities: %s", err)) + } + + <-proxy.New(k8sClient, metricsAddr, []string(insecureRegistries), rootCAs).Run() } diff --git a/controllers/cachedimage_controller.go b/controllers/cachedimage_controller.go index 29666806..675536a7 100644 --- a/controllers/cachedimage_controller.go +++ b/controllers/cachedimage_controller.go @@ -2,6 +2,7 @@ package controllers import ( "context" + "crypto/x509" "net/http" "time" @@ -42,6 +43,7 @@ type CachedImageReconciler struct { ExpiryDelay time.Duration Architectures []string InsecureRegistries []string + RootCAs *x509.CertPool } //+kubebuilder:rbac:groups=kuik.enix.io,resources=cachedimages,verbs=get;list;watch;create;update;patch;delete @@ -159,7 +161,7 @@ func (r *CachedImageReconciler) Reconcile(ctx context.Context, req ctrl.Request) if !isCached { r.Recorder.Eventf(&cachedImage, "Normal", "Caching", "Start caching image %s", cachedImage.Spec.SourceImage) keychain := registry.NewKubernetesKeychain(r.ApiReader, cachedImage.Spec.PullSecretsNamespace, cachedImage.Spec.PullSecretNames) - if err := registry.CacheImage(cachedImage.Spec.SourceImage, keychain, r.Architectures, r.InsecureRegistries); err != nil { + if err := registry.CacheImage(cachedImage.Spec.SourceImage, keychain, r.Architectures, r.InsecureRegistries, r.RootCAs); err != nil { log.Error(err, "failed to cache image") r.Recorder.Eventf(&cachedImage, "Warning", "CacheFailed", "Failed to cache image %s, reason: %s", cachedImage.Spec.SourceImage, err) return ctrl.Result{}, err diff --git a/helm/kube-image-keeper/templates/controller-deployment.yaml b/helm/kube-image-keeper/templates/controller-deployment.yaml index 7612ba62..7ab05dbc 100644 --- a/helm/kube-image-keeper/templates/controller-deployment.yaml +++ b/helm/kube-image-keeper/templates/controller-deployment.yaml @@ -53,6 +53,11 @@ spec: {{- range .Values.insecureRegistries }} - -insecure-registries={{- . }} {{- end }} + {{- with .Values.rootCertificateAuthorities }} + {{- range .keys }} + - -root-certificate-authorities=/etc/ssl/certs/registry-certificate-authorities/{{- . }} + {{- end }} + {{- end }} ports: - containerPort: 9443 name: webhook-server @@ -62,8 +67,13 @@ spec: protocol: TCP volumeMounts: - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert + name: webhook-cert readOnly: true + {{- if .Values.rootCertificateAuthorities }} + - mountPath: /etc/ssl/certs/registry-certificate-authorities + name: registry-certificate-authorities + readOnly: true + {{- end }} {{- with .Values.controllers.resources }} resources: {{- toYaml . | nindent 12 }} @@ -81,7 +91,13 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: - - name: cert + - name: webhook-cert secret: defaultMode: 420 secretName: {{ include "kube-image-keeper.fullname" . }}-webhook-server-cert + {{- with .Values.rootCertificateAuthorities }} + - name: registry-certificate-authorities + secret: + defaultMode: 420 + secretName: {{ .secretName }} + {{- end }} diff --git a/helm/kube-image-keeper/templates/proxy-daemonset.yaml b/helm/kube-image-keeper/templates/proxy-daemonset.yaml index 6ad09a7a..faf97f8c 100644 --- a/helm/kube-image-keeper/templates/proxy-daemonset.yaml +++ b/helm/kube-image-keeper/templates/proxy-daemonset.yaml @@ -52,6 +52,17 @@ spec: {{- range .Values.insecureRegistries }} - -insecure-registries={{- . }} {{- end }} + {{- with .Values.rootCertificateAuthorities }} + {{- range .keys }} + - -root-certificate-authorities=/etc/ssl/certs/registry-certificate-authorities/{{- . }} + {{- end }} + {{- end }} + {{- if .Values.rootCertificateAuthorities }} + volumeMounts: + - mountPath: /etc/ssl/certs/registry-certificate-authorities + name: registry-certificate-authorities + readOnly: true + {{- end }} {{- with .Values.proxy.resources }} resources: {{- toYaml . | nindent 12 }} @@ -68,3 +79,10 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.rootCertificateAuthorities }} + volumes: + - name: registry-certificate-authorities + secret: + defaultMode: 420 + secretName: {{ .secretName }} + {{- end }} diff --git a/helm/kube-image-keeper/values.yaml b/helm/kube-image-keeper/values.yaml index 6c2188f5..7490bc6f 100644 --- a/helm/kube-image-keeper/values.yaml +++ b/helm/kube-image-keeper/values.yaml @@ -10,6 +10,10 @@ installCRD: true architectures: [amd64] # -- Insecure registries to allow to cache and proxify images from insecureRegistries: [] +# -- Root certificate authorities to trust +rootCertificateAuthorities: {} + # secretName: some-secret + # keys: [] controllers: # Maximum number of CachedImages that can be handled and reconciled at the same time (put or remove from cache) diff --git a/internal/proxy/server.go b/internal/proxy/server.go index 6e04cd88..1c9a4b59 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -3,6 +3,7 @@ package proxy import ( "context" "crypto/tls" + "crypto/x509" "errors" "fmt" "net/http" @@ -29,9 +30,10 @@ type Proxy struct { collector *Collector exporter *metrics.Exporter insecureRegistries []string + rootCAs *x509.CertPool } -func New(k8sClient client.Client, metricsAddr string, insecureRegistries []string) *Proxy { +func New(k8sClient client.Client, metricsAddr string, insecureRegistries []string, rootCAs *x509.CertPool) *Proxy { collector := NewCollector() return &Proxy{ k8sClient: k8sClient, @@ -39,6 +41,7 @@ func New(k8sClient client.Client, metricsAddr string, insecureRegistries []strin collector: collector, exporter: metrics.New(collector, metricsAddr), insecureRegistries: insecureRegistries, + rootCAs: rootCAs, } } @@ -144,7 +147,7 @@ func (p *Proxy) routeProxy(c *gin.Context) { if err := p.proxyRegistry(c, registry.Protocol+registry.Endpoint, false, nil); err != nil { klog.InfoS("cached image is not available, proxying origin", "originRegistry", originRegistry, "error", err) - transport, err := p.getAuthentifiedTransport(originRegistry, repository, p.insecureRegistries) + transport, err := p.getAuthentifiedTransport(originRegistry, repository) if err != nil { _ = c.AbortWithError(http.StatusUnauthorized, err) return @@ -227,7 +230,7 @@ func (p *Proxy) proxyRegistry(c *gin.Context, endpoint string, endpointIsOrigin return proxyError } -func (p *Proxy) getAuthentifiedTransport(registryDomain string, repository string, insecureRegistries []string) (http.RoundTripper, error) { +func (p *Proxy) getAuthentifiedTransport(registryDomain string, repository string) (http.RoundTripper, error) { repositoryLabel := registry.RepositoryLabel(registryDomain + "/" + repository) cachedImages := &kuikenixiov1alpha1.CachedImageList{} @@ -259,11 +262,10 @@ func (p *Proxy) getAuthentifiedTransport(registryDomain string, repository strin return nil, err } - originalTransport := http.DefaultTransport - if slices.Contains(insecureRegistries, ref.Context().Registry.RegistryStr()) { - originalTransport = &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } + originalTransport := http.DefaultTransport.(*http.Transport).Clone() + originalTransport.TLSClientConfig = &tls.Config{RootCAs: p.rootCAs} + if slices.Contains(p.insecureRegistries, ref.Context().Registry.RegistryStr()) { + originalTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} } return transport.NewWithContext(context.Background(), ref.Context().Registry, auth, originalTransport, []string{ref.Scope(transport.PullScope)}) diff --git a/internal/proxy/server_test.go b/internal/proxy/server_test.go index f6b144a2..5f7405ba 100644 --- a/internal/proxy/server_test.go +++ b/internal/proxy/server_test.go @@ -28,7 +28,7 @@ func init() { func TestNew(t *testing.T) { g := NewWithT(t) - proxy := New(dummyK8sClient, ":8080", []string{}) + proxy := New(dummyK8sClient, ":8080", []string{}, nil) g.Expect(proxy).To(Not(BeNil())) g.Expect(proxy.engine).To(Not(BeNil())) } diff --git a/internal/registry/certificates.go b/internal/registry/certificates.go new file mode 100644 index 00000000..61cebb29 --- /dev/null +++ b/internal/registry/certificates.go @@ -0,0 +1,30 @@ +package registry + +import ( + "crypto/x509" + "fmt" + "os" +) + +func LoadRootCAPoolFromFiles(certificatePaths []string) (*x509.CertPool, error) { + rootCAs, err := x509.SystemCertPool() + if err != nil { + return nil, err + } + + if rootCAs == nil { + rootCAs = x509.NewCertPool() + } + + for _, path := range certificatePaths { + caCert, err := os.ReadFile(path) + if err != nil { + return nil, err + } + if !rootCAs.AppendCertsFromPEM(caCert) { + return nil, fmt.Errorf("failed to add certificate from %s", path) + } + } + + return rootCAs, nil +} diff --git a/internal/registry/registry.go b/internal/registry/registry.go index c2a410d3..5296a45a 100644 --- a/internal/registry/registry.go +++ b/internal/registry/registry.go @@ -4,6 +4,7 @@ import ( "crypto/sha1" "crypto/sha256" "crypto/tls" + "crypto/x509" "errors" "fmt" "net/http" @@ -99,7 +100,7 @@ func DeleteImage(imageName string) error { return remote.Delete(digest) } -func CacheImage(imageName string, keychain authn.Keychain, architectures []string, insecureRegistries []string) error { +func CacheImage(imageName string, keychain authn.Keychain, architectures []string, insecureRegistries []string, rootCAs *x509.CertPool) error { destRef, err := parseLocalReference(imageName) if err != nil { return err @@ -111,18 +112,18 @@ func CacheImage(imageName string, keychain authn.Keychain, architectures []strin auth := remote.WithAuthFromKeychain(keychain) opts := []remote.Option{auth} + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{RootCAs: rootCAs} if slices.Contains(insecureRegistries, sourceRef.Context().Registry.RegistryStr()) { - transportOption := remote.WithTransport(&http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }) - opts = append(opts, transportOption) + transport.TLSClientConfig.InsecureSkipVerify = true } + opts = append(opts, remote.WithTransport(transport)) + desc, err := remote.Get(sourceRef, opts...) if err != nil { if errIsImageNotFound(err) { - return errors.New("could not find source image") } return err diff --git a/internal/registry/registry_test.go b/internal/registry/registry_test.go index fea3e7ca..a7be96fe 100644 --- a/internal/registry/registry_test.go +++ b/internal/registry/registry_test.go @@ -325,7 +325,7 @@ func Test_CacheImage(t *testing.T) { Endpoint = cacheRegistry.Addr() keychain := NewKubernetesKeychain(nil, "default", []string{}) - err := CacheImage(originRegistry.Addr()+"/"+tt.image, keychain, []string{"amd64"}, []string{}) + err := CacheImage(originRegistry.Addr()+"/"+tt.image, keychain, []string{"amd64"}, []string{}, nil) if tt.wantErr != "" { g.Expect(err).To(BeAssignableToTypeOf(tt.errType)) g.Expect(err).To(MatchError(ContainSubstring(tt.wantErr)))