STONEBLD-2133 Update pooled hosts

deploy/operator/kustomization.yaml

@@ -11,3 +11,4 @@ resources:
   - provision-shared-host.yaml
   - clean-shared-host.yaml
   - openshift-specific-rbac.yaml
+  - update-host.yaml

deploy/operator/update-host.yaml

@@ -0,0 +1,31 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: update-host
+  namespace: multi-platform-controller
+spec:
+  description: >-
+    This task will create a new user on a host, setup ssh keys, and then create the relevant secret.
+  params:
+    - name: HOST
+      type: string
+    - name: USER
+      type: string
+  workspaces:
+    - name: ssh
+  steps:
+    - name: provision
+      image: quay.io/redhat-appstudio/multi-platform-runner:01c7670e81d5120347cf0ad13372742489985e5f@sha256:246adeaaba600e207131d63a7f706cffdcdc37d8f600c56187123ec62823ff44
+      imagePullPolicy: IfNotPresent
+      script: |
+        #!/bin/bash
+        cd /tmp
+        set -o verbose
+        set -eu
+        set -o pipefail
+        mkdir -p /root/.ssh
+        cp $(workspaces.ssh.path)/id_rsa /tmp/master_key
+        chmod 0400 /tmp/master_key
+        export SSH_HOST=$(params.USER)@$(params.HOST)
+        ssh -i /tmp/master_key -o StrictHostKeyChecking=no $SSH_HOST "sudo dnf update -y"
+

deploy/overlays/dev-template/host-config.yaml

@@ -6,7 +6,7 @@ metadata:
   name: host-config
   namespace: multi-platform-controller
 data:
-  dynamic-platforms: linux/arm64,linux/s390x,linux/ppc64le,linux/amd64
+  dynamic-platforms: linux/arm64,linux/amd64
   instance-tag: QUAY_USERNAME-development
 
   dynamic.linux-arm64.type: aws

pkg/controller/controller.go

@@ -90,5 +90,18 @@ func NewManager(cfg *rest.Config, options ctrl.Options) (ctrl.Manager, error) {
 		return nil, err
 	}
 
+	ticker := time.NewTicker(time.Hour * 24)
+	go func() {
+		for range ticker.C {
+			taskrun.UpdateHostPools(operatorNamespace, mgr.GetClient(), &controllerLog)
+		}
+	}()
+	timer := time.NewTimer(time.Minute)
+	go func() {
+		<-timer.C
+		//update the nodes on startup
+		taskrun.UpdateHostPools(operatorNamespace, mgr.GetClient(), &controllerLog)
+	}()
+
 	return mgr, nil
 }

pkg/reconciler/taskrun/taskrun.go

@@ -55,6 +55,7 @@ const (
 
 	TaskTypeLabel     = "build.appstudio.redhat.com/task-type"
 	TaskTypeProvision = "provision"
+	TaskTypeUpdate    = "update"
 	TaskTypeClean     = "clean"
 
 	ServiceAccountName = "multi-platform-controller"
@@ -125,6 +126,10 @@ func (r *ReconcileTaskRun) handleTaskRunReceived(ctx context.Context, log *logr.
 			log.Info("Reconciling provision task")
 			return r.handleProvisionTask(ctx, log, tr)
 		}
+		if taskType == TaskTypeUpdate {
+			// We don't care about these
+			return reconcile.Result{}, nil
+		}
 	}
 	if tr.Spec.Params == nil {
 		return reconcile.Result{}, nil

pkg/reconciler/taskrun/updates.go

@@ -0,0 +1,92 @@
+package taskrun
+
+import (
+	"context"
+	"github.com/go-logr/logr"
+	v1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1"
+	v12 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"strings"
+	"time"
+)
+
+// UpdateHostPools Run the host update task periodically
+func UpdateHostPools(operatorNamespace string, client client.Client, log *logr.Logger) {
+	log.Info("running pooled host update")
+	cm := v12.ConfigMap{}
+	err := client.Get(context.Background(), types.NamespacedName{Namespace: operatorNamespace, Name: HostConfig}, &cm)
+	if err != nil {
+		log.Error(err, "Failed to read config to update hosts", "audit", "true")
+		return
+	}
+
+	hosts := map[string]*Host{}
+	for k, v := range cm.Data {
+		if !strings.HasPrefix(k, "host.") {
+			continue
+		}
+		k = k[len("host."):]
+		pos := strings.LastIndex(k, ".")
+		if pos == -1 {
+			continue
+		}
+		name := k[0:pos]
+		key := k[pos+1:]
+		host := hosts[name]
+		if host == nil {
+			host = &Host{}
+			hosts[name] = host
+			host.Name = name
+		}
+		switch key {
+		case "address":
+			host.Address = v
+		case "user":
+			host.User = v
+		case "platform":
+			host.Platform = v
+		case "secret":
+			host.Secret = v
+		case "concurrency":
+		default:
+			log.Info("unknown key", "key", key)
+		}
+	}
+	delay := 0
+	for hostName := range hosts {
+		log.Info("scheduling host update", "host", hostName)
+		// We don't want to run all updates at once
+		// Stagger all updates by 10 minutes
+		timer := time.NewTimer(time.Minute * time.Duration(delay) * 10)
+		delay++
+		realHostName := hostName
+		host := hosts[realHostName]
+		go func() {
+			<-timer.C
+
+			log.Info("updating host", "host", realHostName)
+			provision := v1.TaskRun{}
+			provision.GenerateName = "update-task"
+			provision.Namespace = operatorNamespace
+			provision.Labels = map[string]string{TaskTypeLabel: TaskTypeUpdate, AssignedHost: realHostName}
+			provision.Spec.TaskRef = &v1.TaskRef{Name: "update-host"}
+			provision.Spec.Workspaces = []v1.WorkspaceBinding{{Name: "ssh", Secret: &v12.SecretVolumeSource{SecretName: host.Secret}}}
+			compute := map[v12.ResourceName]resource.Quantity{v12.ResourceCPU: resource.MustParse("100m"), v12.ResourceMemory: resource.MustParse("256Mi")}
+			provision.Spec.ComputeResources = &v12.ResourceRequirements{Requests: compute, Limits: compute}
+			provision.Spec.ServiceAccountName = ServiceAccountName //TODO: special service account for this
+			provision.Spec.Params = []v1.Param{
+				{
+					Name:  "HOST",
+					Value: *v1.NewStructuredValues(host.Address),
+				},
+				{
+					Name:  "USER",
+					Value: *v1.NewStructuredValues(host.User),
+				},
+			}
+			err = client.Create(context.Background(), &provision)
+		}()
+	}
+}