[CLOUD-188] Feature/cloud 188 fully private azure

bin/images/azure/REGISTRY

@@ -1 +1 @@
-eosregistry.azurecr.io/keos/offline
+offlineregistry.azurecr.io/keos

bin/images/azure/imagenes-cloud-controller.txt

@@ -1,2 +1,3 @@
 mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.24.22
-mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.26.14
+mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.26.14
+mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.27.8

bin/images/azure/imagenes-cloud-node.txt

@@ -1,2 +1,3 @@
 mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.24.22
-mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.26.14
+mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.26.14
+mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.27.8

bin/images/commons/REGISTRY

@@ -1,2 +1,3 @@
 #963353511234.dkr.ecr.eu-west-1.amazonaws.com/keos/offline
-eosregistry.azurecr.io/keos/offline
+#eosregistry.azurecr.io/keos/offline
+offlineregistry.azurecr.io/keos

bin/images/commons/imagenes-cert-manager.txt

@@ -7,4 +7,4 @@ quay.io/jetstack/cert-manager-controller:v1.13.1
 quay.io/jetstack/cert-manager-cainjector:v1.13.1
 quay.io/jetstack/cert-manager-webhook:v1.13.1
 quay.io/jetstack/cert-manager-acmesolver:v1.13.1
-quay.io/jetstack/cert-manager-ctl:v1.13.1
+quay.io/jetstack/cert-manager-ctl:v1.13.1

bin/images/commons/imagenes-kind.txt

@@ -1 +1,2 @@
 docker.io/kindest/node:v1.27.0
+docker.io/kindest/kindnetd:v20221004-44d545d1

pkg/cluster/internal/create/actions/createworker/createworker.go

@@ -38,7 +38,7 @@ type action struct {
 	avoidCreation      bool
 	keosCluster        commons.KeosCluster
 	clusterCredentials commons.ClusterCredentials
-	clusterConfig      commons.ClusterConfig
+	clusterConfig      *commons.ClusterConfig
 }
 
 type keosRegistry struct {
@@ -59,6 +59,7 @@ const (
 	storageDefaultPath      = "/kind/manifests/default-storage.yaml"
 	infraGCPVersion         = "v1.4.0"
 	infraAWSVersion         = "v2.2.1"
+	infraAzureVersion       = "v1.11.4"
 )
 
 var PathsToBackupLocally = []string{
@@ -73,7 +74,7 @@ var allowCommonEgressNetPol string
 var rbacInternalLoadBalancing string
 
 // NewAction returns a new action for installing default CAPI
-func NewAction(vaultPassword string, descriptorPath string, moveManagement bool, avoidCreation bool, keosCluster commons.KeosCluster, clusterCredentials commons.ClusterCredentials, clusterConfig commons.ClusterConfig) actions.Action {
+func NewAction(vaultPassword string, descriptorPath string, moveManagement bool, avoidCreation bool, keosCluster commons.KeosCluster, clusterCredentials commons.ClusterCredentials, clusterConfig *commons.ClusterConfig) actions.Action {
 	return &action{
 		vaultPassword:      vaultPassword,
 		descriptorPath:     descriptorPath,
@@ -121,13 +122,22 @@ func (a *action) Execute(ctx *actions.ActionContext) error {
 	awsEKSEnabled := a.keosCluster.Spec.InfraProvider == "aws" && a.keosCluster.Spec.ControlPlane.Managed
 	isMachinePool := a.keosCluster.Spec.InfraProvider != "aws" && a.keosCluster.Spec.ControlPlane.Managed
 
-	privateParams := PrivateParams{
-		KeosCluster: a.keosCluster,
-		KeosRegUrl:  keosRegistry.url,
-		Private:     a.clusterConfig.Spec.Private,
+	var privateParams PrivateParams
+	if a.clusterConfig != nil {
+		privateParams = PrivateParams{
+			KeosCluster: a.keosCluster,
+			KeosRegUrl:  keosRegistry.url,
+			Private:     a.clusterConfig.Spec.Private,
+		}
+	} else {
+		privateParams = PrivateParams{
+			KeosCluster: a.keosCluster,
+			KeosRegUrl:  keosRegistry.url,
+			Private:     false,
+		}
 	}
 
-	if a.clusterConfig.Spec.Private {
+	if privateParams.Private {
 		ctx.Status.Start("Installing Private CNI 🎖️")
 		defer ctx.Status.End(false)
 		c = `sed -i 's/@sha256:[[:alnum:]_-].*$//g' ` + cniDefaultFile
@@ -222,7 +232,7 @@ func (a *action) Execute(ctx *actions.ActionContext) error {
 		}
 	}
 
-	if a.clusterConfig.Spec.Private {
+	if privateParams.Private {
 		err = provider.deployCertManager(n, keosRegistry.url, "")
 		if err != nil {
 			return err
@@ -231,20 +241,36 @@ func (a *action) Execute(ctx *actions.ActionContext) error {
 		c = "echo \"images:\" >> /root/.cluster-api/clusterctl.yaml && " +
 			"echo \"  cluster-api:\" >> /root/.cluster-api/clusterctl.yaml && " +
 			"echo \"    repository: " + keosRegistry.url + "/cluster-api\" >> /root/.cluster-api/clusterctl.yaml && " +
+			"echo \"    timeout: 10m\" >> /root/.cluster-api/clusterctl.yaml && " +
 			"echo \"  bootstrap-kubeadm:\" >> /root/.cluster-api/clusterctl.yaml && " +
 			"echo \"    repository: " + keosRegistry.url + "/cluster-api\" >> /root/.cluster-api/clusterctl.yaml && " +
+			"echo \"    timeout: 10m\" >> /root/.cluster-api/clusterctl.yaml && " +
 			"echo \"  control-plane-kubeadm:\" >> /root/.cluster-api/clusterctl.yaml && " +
 			"echo \"    repository: " + keosRegistry.url + "/cluster-api\" >> /root/.cluster-api/clusterctl.yaml && " +
+			"echo \"    timeout: 10m\" >> /root/.cluster-api/clusterctl.yaml && " +
 			"echo \"  infrastructure-aws:\" >> /root/.cluster-api/clusterctl.yaml && " +
 			"echo \"    repository: " + keosRegistry.url + "/cluster-api-aws\" >> /root/.cluster-api/clusterctl.yaml && " +
 			"echo \"    tag: " + infraAWSVersion + "\" >> /root/.cluster-api/clusterctl.yaml && " +
+			"echo \"    timeout: 10m\" >> /root/.cluster-api/clusterctl.yaml && " +
 			"echo \"  infrastructure-gcp:\" >> /root/.cluster-api/clusterctl.yaml && " +
 			"echo \"    repository: " + keosRegistry.url + "/cluster-api-gcp\" >> /root/.cluster-api/clusterctl.yaml && " +
 			"echo \"    tag: " + infraGCPVersion + "\" >> /root/.cluster-api/clusterctl.yaml && " +
+			"echo \"    timeout: 10m\" >> /root/.cluster-api/clusterctl.yaml && " +
 			"echo \"  infrastructure-azure:\" >> /root/.cluster-api/clusterctl.yaml && " +
 			"echo \"    repository: " + keosRegistry.url + "/cluster-api-azure\" >> /root/.cluster-api/clusterctl.yaml && " +
+			"echo \"    timeout: 10m\" >> /root/.cluster-api/clusterctl.yaml && " +
+			"echo \"  infrastructure-azure/azureserviceoperator:\" >> /root/.cluster-api/clusterctl.yaml && " +
+			"echo \"    repository: " + keosRegistry.url + "/k8s\" >> /root/.cluster-api/clusterctl.yaml && " +
+			"echo \"    timeout: 10m\" >> /root/.cluster-api/clusterctl.yaml && " +
+			"echo \"  infrastructure-azure/kube-rbac-proxy:\" >> /root/.cluster-api/clusterctl.yaml && " +
+			"echo \"    repository: " + keosRegistry.url + "/kubebuilder\" >> /root/.cluster-api/clusterctl.yaml && " +
+			"echo \"    timeout: 10m\" >> /root/.cluster-api/clusterctl.yaml && " +
+			"echo \"  infrastructure-azure/nmi:\" >> /root/.cluster-api/clusterctl.yaml && " +
+			"echo \"    repository: " + keosRegistry.url + "/oss/azure/aad-pod-identity\" >> /root/.cluster-api/clusterctl.yaml && " +
+			"echo \"    timeout: 10m\" >> /root/.cluster-api/clusterctl.yaml && " +
 			"echo \"  cert-manager:\" >> /root/.cluster-api/clusterctl.yaml && " +
-			"echo \"    repository: " + keosRegistry.url + "/cert-manager\" >> /root/.cluster-api/clusterctl.yaml "
+			"echo \"    repository: " + keosRegistry.url + "/cert-manager\" >> /root/.cluster-api/clusterctl.yaml && " +
+			"echo \"    timeout: 10m\" >> /root/.cluster-api/clusterctl.yaml "
 
 		_, err = commons.ExecuteCommand(n, c)
 
@@ -295,7 +321,7 @@ func (a *action) Execute(ctx *actions.ActionContext) error {
 	ctx.Status.Start("Installing keos cluster operator 💻")
 	defer ctx.Status.End(false)
 
-	err = provider.deployClusterOperator(n, privateParams, a.clusterCredentials, keosRegistry, "", true)
+	err = provider.deployClusterOperator(n, privateParams, a.clusterCredentials, keosRegistry, a.clusterConfig, "", true)
 	if err != nil {
 		return errors.Wrap(err, "failed to deploy cluster operator")
 	}
@@ -317,11 +343,20 @@ func (a *action) Execute(ctx *actions.ActionContext) error {
 		ctx.Status.Start("Creating the workload cluster 💥")
 		defer ctx.Status.End(false)
 
+		if a.clusterConfig != nil {
+			// Apply cluster manifests
+			c = "kubectl apply -f " + manifestsPath + "/clusterconfig.yaml"
+			_, err = commons.ExecuteCommand(n, c)
+			if err != nil {
+				return errors.Wrap(err, "failed to apply clusterconfig manifests")
+			}
+		}
+
 		// Apply cluster manifests
 		c = "kubectl apply -f " + manifestsPath + "/keoscluster.yaml"
 		_, err = commons.ExecuteCommand(n, c)
 		if err != nil {
-			return errors.Wrap(err, "failed to apply manifests")
+			return errors.Wrap(err, "failed to apply keoscluster manifests")
 		}
 
 		time.Sleep(20 * time.Second)
@@ -498,7 +533,7 @@ func (a *action) Execute(ctx *actions.ActionContext) error {
 		ctx.Status.Start("Installing CAPx in workload cluster 🎖️")
 		defer ctx.Status.End(false)
 
-		if a.clusterConfig.Spec.Private {
+		if privateParams.Private {
 			err = provider.deployCertManager(n, keosRegistry.url, kubeconfigPath)
 			if err != nil {
 				return err
@@ -592,7 +627,7 @@ func (a *action) Execute(ctx *actions.ActionContext) error {
 				" --set clusterAPIMode=incluster-incluster" +
 				" --set replicaCount=2"
 
-			if a.clusterConfig.Spec.Private {
+			if privateParams.Private {
 				c += " --set image.repository=" + keosRegistry.url + "/autoscaling/cluster-autoscaler"
 			}
 
@@ -607,7 +642,7 @@ func (a *action) Execute(ctx *actions.ActionContext) error {
 		ctx.Status.Start("Installing keos cluster operator in workload cluster 💻")
 		defer ctx.Status.End(false)
 
-		err = provider.deployClusterOperator(n, privateParams, a.clusterCredentials, keosRegistry, kubeconfigPath, true)
+		err = provider.deployClusterOperator(n, privateParams, a.clusterCredentials, keosRegistry, a.clusterConfig, kubeconfigPath, true)
 		if err != nil {
 			return errors.Wrap(err, "failed to deploy cluster operator in workload cluster")
 		}
@@ -690,6 +725,30 @@ func (a *action) Execute(ctx *actions.ActionContext) error {
 				return errors.Wrap(err, "failed to wait for keoscluster controller ready")
 			}
 
+			if a.clusterConfig != nil {
+
+				c = "kubectl -n " + capiClustersNamespace + " patch clusterconfig " + a.clusterConfig.Metadata.Name + " -p '{\"metadata\":{\"ownerReferences\":null,\"finalizers\":null}}' --type=merge"
+				_, err = commons.ExecuteCommand(n, c)
+				if err != nil {
+					return errors.Wrap(err, "failed to remove clusterconfig ownerReferences and finalizers")
+				}
+
+				// Move clusterConfig to workload cluster
+				c = "kubectl -n " + capiClustersNamespace + " get clusterconfig " + a.clusterConfig.Metadata.Name + " -o json | kubectl apply --kubeconfig " + kubeconfigPath + " -f-"
+				_, err = commons.ExecuteCommand(n, c)
+				if err != nil {
+					return errors.Wrap(err, "failed to move clusterconfig to workload cluster")
+				}
+
+				// Delete clusterconfig in management cluster
+				c = "kubectl -n " + capiClustersNamespace + " delete clusterconfig " + a.clusterConfig.Metadata.Name
+				_, err = commons.ExecuteCommand(n, c)
+				if err != nil {
+					return errors.Wrap(err, "failed to delete clusterconfig in management cluster")
+				}
+
+			}
+
 			// Move keoscluster to workload cluster
 			c = "kubectl -n " + capiClustersNamespace + " get keoscluster " + a.keosCluster.Metadata.Name + " -o json | jq 'del(.status)' | kubectl apply --kubeconfig " + kubeconfigPath + " -f-"
 			_, err = commons.ExecuteCommand(n, c)
@@ -710,24 +769,25 @@ func (a *action) Execute(ctx *actions.ActionContext) error {
 				return errors.Wrap(err, "failed to delete keoscluster in management cluster")
 			}
 
-			err = provider.deployClusterOperator(n, privateParams, a.clusterCredentials, keosRegistry, "", false)
+			err = provider.deployClusterOperator(n, privateParams, a.clusterCredentials, keosRegistry, a.clusterConfig, "", false)
 			if err != nil {
 				return errors.Wrap(err, "failed to deploy cluster operator")
 			}
 
 			ctx.Status.End(true) // End Moving the cluster-operator
 		}
-	}
 
-	ctx.Status.Start("Executing post-install steps 🎖️")
-	defer ctx.Status.End(false)
+		ctx.Status.Start("Executing post-install steps 🎖️")
+		defer ctx.Status.End(false)
 
-	err = infra.postInstallPhase(n, kubeconfigPath)
-	if err != nil {
-		return err
-	}
+		err = infra.postInstallPhase(n, kubeconfigPath)
+		if err != nil {
+			return err
+		}
+
+		ctx.Status.End(true)
 
-	ctx.Status.End(true)
+	}
 
 	ctx.Status.Start("Generating the KEOS descriptor 📝")
 	defer ctx.Status.End(false)

pkg/cluster/internal/create/actions/createworker/provider.go

@@ -56,8 +56,8 @@ const (
 	scName = "keos"
 
 	certManagerVersion   = "v1.12.3"
-	clusterOperatorChart = "0.2.0-SNAPSHOT"
-	clusterOperatorImage = "0.2.0-SNAPSHOT"
+	clusterOperatorChart = "0.2.0-PR158-SNAPSHOT"
+	clusterOperatorImage = "0.2.0-PR158-SNAPSHOT"
 
 	postInstallAnnotation = "cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes"
 )
@@ -287,7 +287,7 @@ func (p *Provider) deployCertManager(n nodes.Node, keosRegistryUrl string, kubec
 	return nil
 }
 
-func (p *Provider) deployClusterOperator(n nodes.Node, privateParams PrivateParams, clusterCredentials commons.ClusterCredentials, keosRegistry keosRegistry, kubeconfigPath string, firstInstallation bool) error {
+func (p *Provider) deployClusterOperator(n nodes.Node, privateParams PrivateParams, clusterCredentials commons.ClusterCredentials, keosRegistry keosRegistry, clusterConfig *commons.ClusterConfig, kubeconfigPath string, firstInstallation bool) error {
 	var c string
 	var err error
 	var helmRepository helmRepository
@@ -333,6 +333,20 @@ func (p *Provider) deployClusterOperator(n nodes.Node, privateParams PrivatePara
 			Flavour string `yaml:"flavour,omitempty"`
 			Version string `yaml:"version,omitempty"`
 		}{}
+
+		if clusterConfig != nil {
+			clusterConfigYAML, err := yaml.Marshal(clusterConfig)
+			if err != nil {
+				return err
+			}
+			// Write keoscluster file
+			c = "echo '" + string(clusterConfigYAML) + "' > " + manifestsPath + "/clusterconfig.yaml"
+			_, err = commons.ExecuteCommand(n, c)
+			if err != nil {
+				return errors.Wrap(err, "failed to write the keoscluster file")
+			}
+			keosCluster.Spec.ClusterConfigRef.Name = clusterConfig.Metadata.Name
+		}
 		keosClusterYAML, err := yaml.Marshal(keosCluster)
 		if err != nil {
 			return err

pkg/cluster/internal/create/create.go

@@ -63,7 +63,7 @@ type ClusterOptions struct {
 	MoveManagement     bool
 	AvoidCreation      bool
 	KeosCluster        commons.KeosCluster
-	ClusterConfig      commons.ClusterConfig
+	ClusterConfig      *commons.ClusterConfig
 	ClusterCredentials commons.ClusterCredentials
 	DockerRegUrl       string
 
@@ -121,7 +121,7 @@ func Cluster(logger log.Logger, p providers.Provider, opts *ClusterOptions) erro
 	logger.V(0).Infof("Creating temporary cluster %q ...\n", opts.Config.Name)
 
 	// Create node containers implementing defined config Nodes
-	if err := p.Provision(status, opts.Config, opts.DockerRegUrl); err != nil {
+	if err := p.Provision(status, opts.Config, opts.DockerRegUrl, !opts.KeosCluster.Spec.ControlPlane.Public); err != nil {
 		// In case of errors nodes are deleted (except if retain is explicitly set)
 		if !opts.Retain {
 			_ = delete.Cluster(logger, p, opts.Config.Name, opts.KubeconfigPath)

pkg/cluster/internal/providers/docker/images.go

@@ -40,7 +40,7 @@ var (
 
 // ensureNodeImages ensures that the node images used by the create
 // configuration are present
-func ensureNodeImages(logger log.Logger, status *cli.Status, cfg *config.Cluster, dockerRegUrl string) error {
+func ensureNodeImages(logger log.Logger, status *cli.Status, cfg *config.Cluster, dockerRegUrl string, fullyPrivate bool) error {
 	// pull each required image
 	for _, image := range common.RequiredNodeImages(cfg).List() {
 		// prints user friendly message
@@ -62,17 +62,22 @@ func ensureNodeImages(logger log.Logger, status *cli.Status, cfg *config.Cluster
 			return err
 		}
 		stratioImage := "stratio-capi-image:" + strings.Split(friendlyImageName, ":")[1]
-		// if dockerRegUrl != "" {
-		// 	cmd := exec.Command("docker", "inspect", "--type=image", stratioImage)
-		// 	if err := cmd.Run(); err == nil {
-		// 		logger.V(1).Infof("stratioImage: %s present locally", image)
-		// 	} else {
-		// 		err = buildStratioImage(logger, stratioImage, dockerfileDir)
-		// 	}
-		// } else {
-		// 	err = buildStratioImage(logger, stratioImage, dockerfileDir)
-		// }
-		err = buildStratioImage(logger, stratioImage, dockerfileDir)
+		args := []string{}
+		if dockerRegUrl != "" {
+			args = append(args, "--build-arg=DOCKER_REG="+dockerRegUrl)
+			if fullyPrivate {
+				cmd := exec.Command("docker", "inspect", "--type=image", stratioImage)
+				if err := cmd.Run(); err == nil {
+					logger.V(1).Infof("stratioImage: %s present locally", image)
+				} else {
+					err = buildStratioImage(logger, stratioImage, dockerfileDir, args...)
+				}
+			} else {
+				err = buildStratioImage(logger, stratioImage, dockerfileDir, args...)
+			}
+		} else {
+			err = buildStratioImage(logger, stratioImage, dockerfileDir)
+		}
 		if err != nil {
 			status.End(false)
 			return err
@@ -113,8 +118,13 @@ func ensureStratioImageFiles(logger log.Logger) (dir string, err error) {
 }
 
 // buildStratioImage builds the stratio image
-func buildStratioImage(logger log.Logger, image string, path string) error {
+func buildStratioImage(logger log.Logger, image string, path string, args ...string) error {
 	cmd := exec.Command("docker", "build", "--tag="+image, path)
+	if len(args) != 0 {
+		arg := strings.Join(args, " ")
+		cmd = exec.Command("docker", "build", "--tag="+image, arg, path)
+	}
+
 	if err := cmd.Run(); err != nil {
 		return errors.Wrapf(err, "failed to build image %q", image)
 	}

pkg/cluster/internal/providers/docker/provider.go

@@ -61,10 +61,10 @@ func (p *provider) String() string {
 }
 
 // Provision is part of the providers.Provider interface
-func (p *provider) Provision(status *cli.Status, cfg *config.Cluster, dockerRegUrl string) (err error) {
+func (p *provider) Provision(status *cli.Status, cfg *config.Cluster, dockerRegUrl string, fullyPrivate bool) (err error) {
 	// TODO: validate cfg
 	// ensure node images are pulled before actually provisioning
-	if err := ensureNodeImages(p.logger, status, cfg, dockerRegUrl); err != nil {
+	if err := ensureNodeImages(p.logger, status, cfg, dockerRegUrl, fullyPrivate); err != nil {
 		return err
 	}
 

pkg/cluster/internal/providers/podman/provider.go

@@ -63,7 +63,7 @@ func (p *provider) String() string {
 }
 
 // Provision is part of the providers.Provider interface
-func (p *provider) Provision(status *cli.Status, cfg *config.Cluster, dockerRegUrl string) (err error) {
+func (p *provider) Provision(status *cli.Status, cfg *config.Cluster, dockerRegUrl string, fullyPrivate bool) (err error) {
 	if err := ensureMinVersion(); err != nil {
 		return err
 	}