feat: allow to add additional root CAs to trust for caching and proxying

cmd/cache/main.go

@@ -38,6 +38,7 @@ func main() {
 	var architectures internal.ArrayFlags
 	var maxConcurrentCachedImageReconciles int
 	var insecureRegistries internal.ArrayFlags
+	var rootCAPaths internal.ArrayFlags
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
@@ -50,6 +51,7 @@ func main() {
 	flag.StringVar(&registry.Endpoint, "registry-endpoint", "kube-image-keeper-registry:5000", "The address of the registry where cached images are stored.")
 	flag.IntVar(&maxConcurrentCachedImageReconciles, "max-concurrent-cached-image-reconciles", 3, "Maximum number of CachedImages that can be handled and reconciled at the same time (put or removed from cache).")
 	flag.Var(&insecureRegistries, "insecure-registries", "Insecure registries to allow to cache and proxify images from (this flag can be used multiple times).")
+	flag.Var(&rootCAPaths, "root-certificate-authorities", "Root certificate authorities to trust.")
 
 	opts := zap.Options{
 		Development:     true,
@@ -74,6 +76,12 @@ func main() {
 		os.Exit(1)
 	}
 
+	rootCAs, err := registry.LoadRootCAPoolFromFiles(rootCAPaths)
+	if err != nil {
+		setupLog.Error(err, "could not load root certificate authorities")
+		os.Exit(1)
+	}
+
 	if err = (&controllers.CachedImageReconciler{
 		Client:             mgr.GetClient(),
 		Scheme:             mgr.GetScheme(),
@@ -82,6 +90,7 @@ func main() {
 		ExpiryDelay:        time.Duration(expiryDelay*24) * time.Hour,
 		Architectures:      []string(architectures),
 		InsecureRegistries: []string(insecureRegistries),
+		RootCAs:            rootCAs,
 	}).SetupWithManager(mgr, maxConcurrentCachedImageReconciles); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "CachedImage")
 		os.Exit(1)

cmd/proxy/main.go

@@ -25,6 +25,7 @@ var (
 	rateLimitQPS       int
 	rateLimitBurst     int
 	insecureRegistries internal.ArrayFlags
+	rootCAPaths        internal.ArrayFlags
 )
 
 func initFlags() {
@@ -38,6 +39,7 @@ func initFlags() {
 	flag.IntVar(&rateLimitQPS, "kube-api-rate-limit-qps", 0, "Kubernetes API request rate limit")
 	flag.IntVar(&rateLimitBurst, "kube-api-rate-limit-burst", 0, "Kubernetes API request burst")
 	flag.Var(&insecureRegistries, "insecure-registries", "Insecure registries to allow to cache and proxify images from (this flag can be used multiple times).")
+	flag.Var(&rootCAPaths, "root-certificate-authorities", "Root certificate authorities to trust.")
 
 	flag.Parse()
 }
@@ -81,5 +83,10 @@ func main() {
 		panic(err)
 	}
 
-	<-proxy.New(k8sClient, metricsAddr, []string(insecureRegistries)).Run()
+	rootCAs, err := registry.LoadRootCAPoolFromFiles(rootCAPaths)
+	if err != nil {
+		panic(fmt.Errorf("could not load root certificate authorities: %s", err))
+	}
+
+	<-proxy.New(k8sClient, metricsAddr, []string(insecureRegistries), rootCAs).Run()
 }

controllers/cachedimage_controller.go

@@ -2,6 +2,7 @@ package controllers
 
 import (
 	"context"
+	"crypto/x509"
 	"net/http"
 	"time"
 
@@ -42,6 +43,7 @@ type CachedImageReconciler struct {
 	ExpiryDelay        time.Duration
 	Architectures      []string
 	InsecureRegistries []string
+	RootCAs            *x509.CertPool
 }
 
 //+kubebuilder:rbac:groups=kuik.enix.io,resources=cachedimages,verbs=get;list;watch;create;update;patch;delete
@@ -159,7 +161,7 @@ func (r *CachedImageReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	if !isCached {
 		r.Recorder.Eventf(&cachedImage, "Normal", "Caching", "Start caching image %s", cachedImage.Spec.SourceImage)
 		keychain := registry.NewKubernetesKeychain(r.ApiReader, cachedImage.Spec.PullSecretsNamespace, cachedImage.Spec.PullSecretNames)
-		if err := registry.CacheImage(cachedImage.Spec.SourceImage, keychain, r.Architectures, r.InsecureRegistries); err != nil {
+		if err := registry.CacheImage(cachedImage.Spec.SourceImage, keychain, r.Architectures, r.InsecureRegistries, r.RootCAs); err != nil {
 			log.Error(err, "failed to cache image")
 			r.Recorder.Eventf(&cachedImage, "Warning", "CacheFailed", "Failed to cache image %s, reason: %s", cachedImage.Spec.SourceImage, err)
 			return ctrl.Result{}, err

helm/kube-image-keeper/templates/controller-deployment.yaml

@@ -53,6 +53,11 @@ spec:
             {{- range .Values.insecureRegistries }}
             - -insecure-registries={{- . }}
             {{- end }}
+            {{- with .Values.rootCertificateAuthorities }}
+            {{- range .keys }}
+            - -root-certificate-authorities=/etc/ssl/certs/registry-certificate-authorities/{{- . }}
+            {{- end }}
+            {{- end }}
           ports:
             - containerPort: 9443
               name: webhook-server
@@ -62,8 +67,13 @@ spec:
               protocol: TCP
           volumeMounts:
             - mountPath: /tmp/k8s-webhook-server/serving-certs
-              name: cert
+              name: webhook-cert
               readOnly: true
+            {{- if .Values.rootCertificateAuthorities }}
+            - mountPath: /etc/ssl/certs/registry-certificate-authorities
+              name: registry-certificate-authorities
+              readOnly: true
+            {{- end }}
           {{- with .Values.controllers.resources }}
           resources:
             {{- toYaml . | nindent 12 }}
@@ -81,7 +91,13 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
-      - name: cert
+      - name: webhook-cert
         secret:
           defaultMode: 420
           secretName: {{ include "kube-image-keeper.fullname" . }}-webhook-server-cert
+      {{- with .Values.rootCertificateAuthorities }}
+      - name: registry-certificate-authorities
+        secret:
+          defaultMode: 420
+          secretName: {{ .secretName }}
+      {{- end }}

helm/kube-image-keeper/templates/proxy-daemonset.yaml

@@ -52,6 +52,17 @@ spec:
             {{- range .Values.insecureRegistries }}
             - -insecure-registries={{- . }}
             {{- end }}
+            {{- with .Values.rootCertificateAuthorities }}
+            {{- range .keys }}
+            - -root-certificate-authorities=/etc/ssl/certs/registry-certificate-authorities/{{- . }}
+            {{- end }}
+            {{- end }}
+          {{- if .Values.rootCertificateAuthorities }}
+          volumeMounts:
+            - mountPath: /etc/ssl/certs/registry-certificate-authorities
+              name: registry-certificate-authorities
+              readOnly: true
+          {{- end }}
           {{- with .Values.proxy.resources }}
           resources:
             {{- toYaml . | nindent 12 }}
@@ -68,3 +79,10 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.rootCertificateAuthorities }}
+      volumes:
+      - name: registry-certificate-authorities
+        secret:
+          defaultMode: 420
+          secretName: {{ .secretName }}
+      {{- end }}

helm/kube-image-keeper/values.yaml

@@ -10,6 +10,10 @@ installCRD: true
 architectures: [amd64]
 # -- Insecure registries to allow to cache and proxify images from
 insecureRegistries: []
+# -- Root certificate authorities to trust
+rootCertificateAuthorities: {}
+  # secretName: some-secret
+  # keys: []
 
 controllers:
   # Maximum number of CachedImages that can be handled and reconciled at the same time (put or remove from cache)

internal/proxy/server.go

@@ -3,6 +3,7 @@ package proxy
 import (
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"errors"
 	"fmt"
 	"net/http"
@@ -29,16 +30,18 @@ type Proxy struct {
 	collector          *Collector
 	exporter           *metrics.Exporter
 	insecureRegistries []string
+	rootCAs            *x509.CertPool
 }
 
-func New(k8sClient client.Client, metricsAddr string, insecureRegistries []string) *Proxy {
+func New(k8sClient client.Client, metricsAddr string, insecureRegistries []string, rootCAs *x509.CertPool) *Proxy {
 	collector := NewCollector()
 	return &Proxy{
 		k8sClient:          k8sClient,
 		engine:             gin.Default(),
 		collector:          collector,
 		exporter:           metrics.New(collector, metricsAddr),
 		insecureRegistries: insecureRegistries,
+		rootCAs:            rootCAs,
 	}
 }
 
@@ -144,7 +147,7 @@ func (p *Proxy) routeProxy(c *gin.Context) {
 	if err := p.proxyRegistry(c, registry.Protocol+registry.Endpoint, false, nil); err != nil {
 		klog.InfoS("cached image is not available, proxying origin", "originRegistry", originRegistry, "error", err)
 
-		transport, err := p.getAuthentifiedTransport(originRegistry, repository, p.insecureRegistries)
+		transport, err := p.getAuthentifiedTransport(originRegistry, repository)
 		if err != nil {
 			_ = c.AbortWithError(http.StatusUnauthorized, err)
 			return
@@ -227,7 +230,7 @@ func (p *Proxy) proxyRegistry(c *gin.Context, endpoint string, endpointIsOrigin
 	return proxyError
 }
 
-func (p *Proxy) getAuthentifiedTransport(registryDomain string, repository string, insecureRegistries []string) (http.RoundTripper, error) {
+func (p *Proxy) getAuthentifiedTransport(registryDomain string, repository string) (http.RoundTripper, error) {
 	repositoryLabel := registry.RepositoryLabel(registryDomain + "/" + repository)
 	cachedImages := &kuikenixiov1alpha1.CachedImageList{}
 
@@ -259,11 +262,10 @@ func (p *Proxy) getAuthentifiedTransport(registryDomain string, repository strin
 		return nil, err
 	}
 
-	originalTransport := http.DefaultTransport
-	if slices.Contains(insecureRegistries, ref.Context().Registry.RegistryStr()) {
-		originalTransport = &http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-		}
+	originalTransport := http.DefaultTransport.(*http.Transport).Clone()
+	originalTransport.TLSClientConfig = &tls.Config{RootCAs: p.rootCAs}
+	if slices.Contains(p.insecureRegistries, ref.Context().Registry.RegistryStr()) {
+		originalTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
 	}
 
 	return transport.NewWithContext(context.Background(), ref.Context().Registry, auth, originalTransport, []string{ref.Scope(transport.PullScope)})

internal/proxy/server_test.go

@@ -28,7 +28,7 @@ func init() {
 
 func TestNew(t *testing.T) {
 	g := NewWithT(t)
-	proxy := New(dummyK8sClient, ":8080", []string{})
+	proxy := New(dummyK8sClient, ":8080", []string{}, nil)
 	g.Expect(proxy).To(Not(BeNil()))
 	g.Expect(proxy.engine).To(Not(BeNil()))
 }

internal/registry/certificates.go

@@ -0,0 +1,30 @@
+package registry
+
+import (
+	"crypto/x509"
+	"fmt"
+	"os"
+)
+
+func LoadRootCAPoolFromFiles(certificatePaths []string) (*x509.CertPool, error) {
+	rootCAs, err := x509.SystemCertPool()
+	if err != nil {
+		return nil, err
+	}
+
+	if rootCAs == nil {
+		rootCAs = x509.NewCertPool()
+	}
+
+	for _, path := range certificatePaths {
+		caCert, err := os.ReadFile(path)
+		if err != nil {
+			return nil, err
+		}
+		if !rootCAs.AppendCertsFromPEM(caCert) {
+			return nil, fmt.Errorf("failed to add certificate from %s", path)
+		}
+	}
+
+	return rootCAs, nil
+}

internal/registry/registry.go

@@ -4,6 +4,7 @@ import (
 	"crypto/sha1"
 	"crypto/sha256"
 	"crypto/tls"
+	"crypto/x509"
 	"errors"
 	"fmt"
 	"net/http"
@@ -99,7 +100,7 @@ func DeleteImage(imageName string) error {
 	return remote.Delete(digest)
 }
 
-func CacheImage(imageName string, keychain authn.Keychain, architectures []string, insecureRegistries []string) error {
+func CacheImage(imageName string, keychain authn.Keychain, architectures []string, insecureRegistries []string, rootCAs *x509.CertPool) error {
 	destRef, err := parseLocalReference(imageName)
 	if err != nil {
 		return err
@@ -111,18 +112,18 @@ func CacheImage(imageName string, keychain authn.Keychain, architectures []strin
 
 	auth := remote.WithAuthFromKeychain(keychain)
 	opts := []remote.Option{auth}
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	transport.TLSClientConfig = &tls.Config{RootCAs: rootCAs}
 
 	if slices.Contains(insecureRegistries, sourceRef.Context().Registry.RegistryStr()) {
-		transportOption := remote.WithTransport(&http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-		})
-		opts = append(opts, transportOption)
+		transport.TLSClientConfig.InsecureSkipVerify = true
 	}
 
+	opts = append(opts, remote.WithTransport(transport))
+
 	desc, err := remote.Get(sourceRef, opts...)
 	if err != nil {
 		if errIsImageNotFound(err) {
-
 			return errors.New("could not find source image")
 		}
 		return err

internal/registry/registry_test.go

@@ -325,7 +325,7 @@ func Test_CacheImage(t *testing.T) {
 
 			Endpoint = cacheRegistry.Addr()
 			keychain := NewKubernetesKeychain(nil, "default", []string{})
-			err := CacheImage(originRegistry.Addr()+"/"+tt.image, keychain, []string{"amd64"}, []string{})
+			err := CacheImage(originRegistry.Addr()+"/"+tt.image, keychain, []string{"amd64"}, []string{}, nil)
 			if tt.wantErr != "" {
 				g.Expect(err).To(BeAssignableToTypeOf(tt.errType))
 				g.Expect(err).To(MatchError(ContainSubstring(tt.wantErr)))