feat: private registries

cmd/cache/main.go

@@ -3,8 +3,6 @@ package main
 import (
 	"flag"
 	"os"
-	"regexp"
-	"strings"
 	"time"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
@@ -20,6 +18,7 @@ import (
 
 	kuikenixiov1 "github.com/enix/kube-image-keeper/api/v1"
 	"github.com/enix/kube-image-keeper/controllers"
+	"github.com/enix/kube-image-keeper/internal"
 	"github.com/enix/kube-image-keeper/internal/registry"
 	"github.com/enix/kube-image-keeper/internal/scheme"
 	//+kubebuilder:scaffold:imports
@@ -29,46 +28,17 @@ var (
 	setupLog = ctrl.Log.WithName("setup")
 )
 
-type regexpArrayFlags []*regexp.Regexp
-
-func (re *regexpArrayFlags) String() string {
-	s := []string{}
-	for _, r := range *re {
-		s = append(s, r.String())
-	}
-	return strings.Join(s, ",")
-}
-
-func (re *regexpArrayFlags) Set(value string) error {
-	r, err := regexp.Compile(value)
-	if err != nil {
-		setupLog.Error(err, "unable parse ignored images regex", "controller", "Pod")
-		os.Exit(1)
-	}
-	*re = append(*re, r)
-	return nil
-}
-
-type arrayFlags []string
-
-func (a *arrayFlags) String() string {
-	return strings.Join(*a, ",")
-}
-
-func (a *arrayFlags) Set(value string) error {
-	*a = append(*a, value)
-	return nil
-}
-
 func main() {
 	var metricsAddr string
 	var enableLeaderElection bool
 	var probeAddr string
 	var expiryDelay uint
 	var proxyPort int
-	var ignoreImages regexpArrayFlags
-	var architectures arrayFlags
+	var ignoreImages internal.RegexpArrayFlags
+	var architectures internal.ArrayFlags
 	var maxConcurrentCachedImageReconciles int
+	var insecureRegistries internal.ArrayFlags
+	var rootCAPaths internal.ArrayFlags
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
@@ -80,6 +50,8 @@ func main() {
 	flag.Var(&architectures, "arch", "Architecture of image to put in cache (this flag can be used multiple times).")
 	flag.StringVar(&registry.Endpoint, "registry-endpoint", "kube-image-keeper-registry:5000", "The address of the registry where cached images are stored.")
 	flag.IntVar(&maxConcurrentCachedImageReconciles, "max-concurrent-cached-image-reconciles", 3, "Maximum number of CachedImages that can be handled and reconciled at the same time (put or removed from cache).")
+	flag.Var(&insecureRegistries, "insecure-registries", "Insecure registries to allow to cache and proxify images from (this flag can be used multiple times).")
+	flag.Var(&rootCAPaths, "root-certificate-authorities", "Root certificate authorities to trust.")
 
 	opts := zap.Options{
 		Development:     true,
@@ -104,13 +76,21 @@ func main() {
 		os.Exit(1)
 	}
 
+	rootCAs, err := registry.LoadRootCAPoolFromFiles(rootCAPaths)
+	if err != nil {
+		setupLog.Error(err, "could not load root certificate authorities")
+		os.Exit(1)
+	}
+
 	if err = (&controllers.CachedImageReconciler{
-		Client:        mgr.GetClient(),
-		Scheme:        mgr.GetScheme(),
-		Recorder:      mgr.GetEventRecorderFor("cachedimage-controller"),
-		ApiReader:     mgr.GetAPIReader(),
-		ExpiryDelay:   time.Duration(expiryDelay*24) * time.Hour,
-		Architectures: []string(architectures),
+		Client:             mgr.GetClient(),
+		Scheme:             mgr.GetScheme(),
+		Recorder:           mgr.GetEventRecorderFor("cachedimage-controller"),
+		ApiReader:          mgr.GetAPIReader(),
+		ExpiryDelay:        time.Duration(expiryDelay*24) * time.Hour,
+		Architectures:      []string(architectures),
+		InsecureRegistries: []string(insecureRegistries),
+		RootCAs:            rootCAs,
 	}).SetupWithManager(mgr, maxConcurrentCachedImageReconciles); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "CachedImage")
 		os.Exit(1)

cmd/proxy/main.go

@@ -7,6 +7,7 @@ import (
 
 	_ "go.uber.org/automaxprocs"
 
+	"github.com/enix/kube-image-keeper/internal"
 	"github.com/enix/kube-image-keeper/internal/proxy"
 	"github.com/enix/kube-image-keeper/internal/registry"
 	"github.com/enix/kube-image-keeper/internal/scheme"
@@ -19,10 +20,12 @@ import (
 )
 
 var (
-	kubeconfig     string
-	metricsAddr    string
-	rateLimitQPS   int
-	rateLimitBurst int
+	kubeconfig         string
+	metricsAddr        string
+	rateLimitQPS       int
+	rateLimitBurst     int
+	insecureRegistries internal.ArrayFlags
+	rootCAPaths        internal.ArrayFlags
 )
 
 func initFlags() {
@@ -35,6 +38,8 @@ func initFlags() {
 	flag.StringVar(&registry.Endpoint, "registry-endpoint", "kube-image-keeper-registry:5000", "The address of the registry where cached images are stored.")
 	flag.IntVar(&rateLimitQPS, "kube-api-rate-limit-qps", 0, "Kubernetes API request rate limit")
 	flag.IntVar(&rateLimitBurst, "kube-api-rate-limit-burst", 0, "Kubernetes API request burst")
+	flag.Var(&insecureRegistries, "insecure-registries", "Insecure registries to allow to cache and proxify images from (this flag can be used multiple times).")
+	flag.Var(&rootCAPaths, "root-certificate-authorities", "Root certificate authorities to trust.")
 
 	flag.Parse()
 }
@@ -78,5 +83,10 @@ func main() {
 		panic(err)
 	}
 
-	<-proxy.New(k8sClient, metricsAddr).Run()
+	rootCAs, err := registry.LoadRootCAPoolFromFiles(rootCAPaths)
+	if err != nil {
+		panic(fmt.Errorf("could not load root certificate authorities: %s", err))
+	}
+
+	<-proxy.New(k8sClient, metricsAddr, []string(insecureRegistries), rootCAs).Run()
 }

controllers/cachedimage_controller.go

@@ -2,6 +2,7 @@ package controllers
 
 import (
 	"context"
+	"crypto/x509"
 	"net/http"
 	"time"
 
@@ -36,11 +37,13 @@ const cachedImageFinalizerName = "cachedimage.kuik.enix.io/finalizer"
 // CachedImageReconciler reconciles a CachedImage object
 type CachedImageReconciler struct {
 	client.Client
-	Scheme        *runtime.Scheme
-	Recorder      record.EventRecorder
-	ApiReader     client.Reader
-	ExpiryDelay   time.Duration
-	Architectures []string
+	Scheme             *runtime.Scheme
+	Recorder           record.EventRecorder
+	ApiReader          client.Reader
+	ExpiryDelay        time.Duration
+	Architectures      []string
+	InsecureRegistries []string
+	RootCAs            *x509.CertPool
 }
 
 //+kubebuilder:rbac:groups=kuik.enix.io,resources=cachedimages,verbs=get;list;watch;create;update;patch;delete
@@ -158,7 +161,7 @@ func (r *CachedImageReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	if !isCached {
 		r.Recorder.Eventf(&cachedImage, "Normal", "Caching", "Start caching image %s", cachedImage.Spec.SourceImage)
 		keychain := registry.NewKubernetesKeychain(r.ApiReader, cachedImage.Spec.PullSecretsNamespace, cachedImage.Spec.PullSecretNames)
-		if err := registry.CacheImage(cachedImage.Spec.SourceImage, keychain, r.Architectures); err != nil {
+		if err := registry.CacheImage(cachedImage.Spec.SourceImage, keychain, r.Architectures, r.InsecureRegistries, r.RootCAs); err != nil {
 			log.Error(err, "failed to cache image")
 			r.Recorder.Eventf(&cachedImage, "Warning", "CacheFailed", "Failed to cache image %s, reason: %s", cachedImage.Spec.SourceImage, err)
 			return ctrl.Result{}, err

go.mod

@@ -16,10 +16,12 @@ require (
 	github.com/prometheus/client_golang v1.17.0
 	go.uber.org/automaxprocs v1.5.3
 	go.uber.org/zap v1.26.0
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
 	k8s.io/api v0.23.5
 	k8s.io/apimachinery v0.23.5
 	k8s.io/client-go v0.23.5
 	k8s.io/klog/v2 v2.110.1
+	k8s.io/utils v0.0.0-20211116205334-6203023598ed
 	sigs.k8s.io/controller-runtime v0.11.2
 )
 
@@ -101,15 +103,15 @@ require (
 	go.uber.org/multierr v1.10.0 // indirect
 	golang.org/x/arch v0.3.0 // indirect
 	golang.org/x/crypto v0.14.0 // indirect
-	golang.org/x/mod v0.12.0 // indirect
+	golang.org/x/mod v0.13.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/oauth2 v0.8.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/sync v0.4.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
 	golang.org/x/term v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
-	golang.org/x/tools v0.12.0 // indirect
+	golang.org/x/tools v0.14.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
@@ -120,7 +122,6 @@ require (
 	k8s.io/apiextensions-apiserver v0.23.5 // indirect
 	k8s.io/component-base v0.23.5 // indirect
 	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
-	k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect
 	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect

go.sum

@@ -653,6 +653,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -678,8 +680,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
+golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -756,8 +758,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
+golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -909,8 +911,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM=
-golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss=
-golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
+golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
+golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

helm/kube-image-keeper/templates/controller-deployment.yaml

@@ -50,6 +50,14 @@ spec:
             {{- range .Values.architectures }}
             - -arch={{- . }}
             {{- end }}
+            {{- range .Values.insecureRegistries }}
+            - -insecure-registries={{- . }}
+            {{- end }}
+            {{- with .Values.rootCertificateAuthorities }}
+            {{- range .keys }}
+            - -root-certificate-authorities=/etc/ssl/certs/registry-certificate-authorities/{{- . }}
+            {{- end }}
+            {{- end }}
           ports:
             - containerPort: 9443
               name: webhook-server
@@ -59,8 +67,13 @@ spec:
               protocol: TCP
           volumeMounts:
             - mountPath: /tmp/k8s-webhook-server/serving-certs
-              name: cert
+              name: webhook-cert
+              readOnly: true
+            {{- if .Values.rootCertificateAuthorities }}
+            - mountPath: /etc/ssl/certs/registry-certificate-authorities
+              name: registry-certificate-authorities
               readOnly: true
+            {{- end }}
           {{- with .Values.controllers.resources }}
           resources:
             {{- toYaml . | nindent 12 }}
@@ -78,7 +91,13 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
-      - name: cert
+      - name: webhook-cert
         secret:
           defaultMode: 420
           secretName: {{ include "kube-image-keeper.fullname" . }}-webhook-server-cert
+      {{- with .Values.rootCertificateAuthorities }}
+      - name: registry-certificate-authorities
+        secret:
+          defaultMode: 420
+          secretName: {{ .secretName }}
+      {{- end }}

helm/kube-image-keeper/templates/proxy-daemonset.yaml

@@ -49,6 +49,20 @@ spec:
             - -kube-api-rate-limit-qps={{ .qps }}
             - -kube-api-rate-limit-burst={{ .burst }}
             {{- end }}
+            {{- range .Values.insecureRegistries }}
+            - -insecure-registries={{- . }}
+            {{- end }}
+            {{- with .Values.rootCertificateAuthorities }}
+            {{- range .keys }}
+            - -root-certificate-authorities=/etc/ssl/certs/registry-certificate-authorities/{{- . }}
+            {{- end }}
+            {{- end }}
+          {{- if .Values.rootCertificateAuthorities }}
+          volumeMounts:
+            - mountPath: /etc/ssl/certs/registry-certificate-authorities
+              name: registry-certificate-authorities
+              readOnly: true
+          {{- end }}
           {{- with .Values.proxy.resources }}
           resources:
             {{- toYaml . | nindent 12 }}
@@ -65,3 +79,10 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.rootCertificateAuthorities }}
+      volumes:
+      - name: registry-certificate-authorities
+        secret:
+          defaultMode: 420
+          secretName: {{ .secretName }}
+      {{- end }}

helm/kube-image-keeper/values.yaml

@@ -8,6 +8,12 @@ cachedImagesExpiryDelay: 30
 installCRD: true
 # -- List of architectures to put in cache
 architectures: [amd64]
+# -- Insecure registries to allow to cache and proxify images from
+insecureRegistries: []
+# -- Root certificate authorities to trust
+rootCertificateAuthorities: {}
+  # secretName: some-secret
+  # keys: []
 
 controllers:
   # Maximum number of CachedImages that can be handled and reconciled at the same time (put or remove from cache)

internal/flags.go

@@ -0,0 +1,39 @@
+package internal
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+)
+
+type ArrayFlags []string
+
+func (a *ArrayFlags) String() string {
+	return strings.Join(*a, ",")
+}
+
+func (a *ArrayFlags) Set(value string) error {
+	*a = append(*a, value)
+	return nil
+}
+
+type RegexpArrayFlags []*regexp.Regexp
+
+func (re *RegexpArrayFlags) String() string {
+	s := []string{}
+	for _, r := range *re {
+		s = append(s, r.String())
+	}
+	return strings.Join(s, ",")
+}
+
+func (re *RegexpArrayFlags) Set(value string) error {
+	r, err := regexp.Compile(value)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable parse ignored images regex: %s", err.Error())
+		os.Exit(1)
+	}
+	*re = append(*re, r)
+	return nil
+}