feat: enable retention policies

controllers/v1beta1/build_helpers.go

@@ -212,7 +212,7 @@ func (r *LagoonBuildReconciler) getOrCreateNamespace(ctx context.Context, namesp
 			return fmt.Errorf("Error getting harbor version, check your harbor configuration. Error was: %v", err)
 		}
 		if lagoonHarbor.UseV2Functions(curVer) {
-			hProject, err := lagoonHarbor.CreateProjectV2(ctx, lagoonBuild.Spec.Project.Name)
+			hProject, err := lagoonHarbor.CreateProjectV2(ctx, *namespace)
 			if err != nil {
 				return fmt.Errorf("Error creating harbor project: %v", err)
 			}

go.mod

@@ -5,6 +5,7 @@ go 1.20
 require (
 	github.com/cheshir/go-mq/v2 v2.0.1
 	github.com/coreos/go-semver v0.3.1
+	github.com/cxmcc/unixsums v0.0.0-20131125091133-89564297d82f
 	github.com/go-logr/logr v1.2.4
 	github.com/google/go-cmp v0.5.9
 	github.com/hashicorp/go-version v1.6.0

go.sum

@@ -334,6 +334,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/cxmcc/unixsums v0.0.0-20131125091133-89564297d82f h1:PkAFGgVtJnasAxOaiEY1RYPx8W+7X7l66vi8T2apKCM=
+github.com/cxmcc/unixsums v0.0.0-20131125091133-89564297d82f/go.mod h1:XJq7OckzkOtlgeEKFwkH2gFbc1+1WRFUBf7QnvfyrzQ=
 github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
 github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=

internal/harbor/harbor.go

@@ -35,6 +35,14 @@ type Harbor struct {
 	WebhookEventTypes     []string
 	LagoonTargetName      string
 	Config                *config.Options
+	TagRetention          TagRetention
+}
+
+type TagRetention struct {
+	Enabled              bool   `json:"enabled"`
+	BranchRetention      int    `json:"branchRetention"`
+	PullRequestRetention int    `json:"pullrequestRetention"`
+	Schedule             string `json:"schedule"`
 }
 
 // New create a new harbor connection.

internal/harbor/harbor22x.go

@@ -16,7 +16,8 @@ import (
 )
 
 // CreateProjectV2 will create a project if one doesn't exist, but will update as required.
-func (h *Harbor) CreateProjectV2(ctx context.Context, projectName string) (*harborclientv5model.Project, error) {
+func (h *Harbor) CreateProjectV2(ctx context.Context, namespace corev1.Namespace) (*harborclientv5model.Project, error) {
+	projectName := namespace.Labels["lagoon.sh/project"]
 	exists, err := h.ClientV5.ProjectExists(ctx, projectName)
 	if err != nil {
 		h.Log.Info(fmt.Sprintf("Error checking project %s exists, err: %v", projectName, err))
@@ -64,6 +65,45 @@ func (h *Harbor) CreateProjectV2(ctx context.Context, projectName string) (*harb
 	// 	fmt.Println(x)
 	// }
 
+	// get the retention policy from the namespace annotations if the annotation exists
+	projectRetention := h.retentionOverrides(namespace)
+	// handle the creation and updating of retention policies as required
+	// generate a somewhat random schedule from the retention schedule template, using the harbor projectname as the seed
+	schedule, err := helpers.ConvertCrontab(projectName, projectRetention.Schedule)
+	if err != nil {
+		h.Log.Info(fmt.Sprintf("Error generating retention schedule %s: %v", project.Name, err))
+	}
+	schedule = fmt.Sprintf("0 %s", schedule) // harbor needs seconds :\
+	// create the retention policy as required
+	retentionPolicy := h.generateEmptyRetentionPolicy(int64(project.ProjectID))
+	if projectRetention.Enabled {
+		// if a retention policy is enabled, configure it here
+		retentionPolicy = h.generateRetentionPolicy(int64(project.ProjectID), projectRetention)
+	}
+	// get the existing one if one exists
+	existingPolicy, err := h.ClientV5.GetRetentionPolicyByProject(ctx, projectName)
+	if err != nil {
+		h.Log.Info(fmt.Sprintf("Error getting retention policy %s: %v", project.Name, err))
+	}
+	if existingPolicy != nil {
+		retentionPolicy.ID = existingPolicy.ID
+		r1, _ := json.Marshal(existingPolicy)
+		r2, _ := json.Marshal(retentionPolicy)
+		// if the policy differs, then we need to update it with our new policy
+		if string(r1) != string(r2) {
+			err := h.ClientV5.UpdateRetentionPolicy(ctx, retentionPolicy)
+			if err != nil {
+				f, _ := json.Marshal(err)
+				h.Log.Info(fmt.Sprintf("Error updating retention policy %s: %v", project.Name, string(f)))
+			}
+		}
+	} else {
+		// create it if it doesn't
+		if err := h.ClientV5.NewRetentionPolicy(ctx, retentionPolicy); err != nil {
+			h.Log.Info(fmt.Sprintf("Error creating retention policy %s: %v", project.Name, err))
+		}
+	}
+
 	if h.WebhookAddition {
 		wps, err := h.ClientV5.ListProjectWebhookPolicies(ctx, int(project.ProjectID))
 		if err != nil {
@@ -278,6 +318,14 @@ func (h *Harbor) DeleteRepository(ctx context.Context, projectName, branch strin
 			if err != nil {
 				h.Log.Info(fmt.Sprintf("Error deleting harbor repository %s", repo.Name))
 			}
+			h.Log.Info(
+				fmt.Sprintf(
+					"Deleted harbor repository %s in  project %s, environment %s",
+					repo.Name,
+					projectName,
+					environmentName,
+				),
+			)
 		}
 	}
 	if len(listRepositories) > 100 {
@@ -293,6 +341,14 @@ func (h *Harbor) DeleteRepository(ctx context.Context, projectName, branch strin
 					if err != nil {
 						h.Log.Info(fmt.Sprintf("Error deleting harbor repository %s", repo.Name))
 					}
+					h.Log.Info(
+						fmt.Sprintf(
+							"Deleted harbor repository %s in  project %s, environment %s",
+							repo.Name,
+							projectName,
+							environmentName,
+						),
+					)
 				}
 			}
 		}
@@ -321,6 +377,14 @@ func (h *Harbor) DeleteRobotAccount(ctx context.Context, projectName, branch str
 				h.Log.Info(fmt.Sprintf("Error deleting project %s robot account %s", projectName, robot.Name))
 				return
 			}
+			h.Log.Info(
+				fmt.Sprintf(
+					"Deleted harbor robot account %s in  project %s, environment %s",
+					robot.Name,
+					projectName,
+					environmentName,
+				),
+			)
 		}
 	}
 }

internal/harbor/harbor_credentialrotation.go

@@ -94,7 +94,7 @@ func (h *Harbor) RotateRobotCredential(ctx context.Context, cl client.Client, ns
 		return false, fmt.Errorf("error checking harbor version: %v", err)
 	}
 	if h.UseV2Functions(curVer) {
-		hProject, err := h.CreateProjectV2(ctx, ns.Labels["lagoon.sh/project"])
+		hProject, err := h.CreateProjectV2(ctx, ns)
 		if err != nil {
 			return false, fmt.Errorf("error getting or creating project: %v", err)
 		}

internal/harbor/harbor_helpers.go

@@ -10,6 +10,7 @@ import (
 	"encoding/json"
 	"time"
 
+	harborclientv5model "github.com/mittwald/goharbor-client/v5/apiv2/model"
 	"github.com/uselagoon/remote-controller/internal/helpers"
 
 	corev1 "k8s.io/api/core/v1"
@@ -204,3 +205,95 @@ func (h *Harbor) UpsertHarborSecret(ctx context.Context, cl client.Client, ns, n
 	}
 	return false, nil
 }
+
+func (h *Harbor) generateRetentionPolicy(projectID int64, policy TagRetention) *harborclientv5model.RetentionPolicy {
+	return &harborclientv5model.RetentionPolicy{
+		Algorithm: "or",
+		Rules: []*harborclientv5model.RetentionRule{
+			{ // create a retention policy for all images
+				Action: "retain",
+				Params: map[string]interface{}{
+					"latestPulledN": policy.BranchRetention,
+				},
+				ScopeSelectors: map[string][]harborclientv5model.RetentionSelector{
+					"repository": {
+						{
+							Decoration: "repoMatches",
+							Kind:       "doublestar",
+							Pattern:    "[^pr\\-]*/*", // exclude pullrequest repository images https://github.com/bmatcuk/doublestar#patterns
+						},
+					},
+				},
+				TagSelectors: []*harborclientv5model.RetentionSelector{
+					{
+						Decoration: "matches",
+						Extras:     "{\"untagged\":true}",
+						Kind:       "doublestar",
+						Pattern:    "**",
+					},
+				},
+				Template: "latestPulledN",
+			},
+			{ // create a retention policy specifically for pullrequests
+				Action: "retain",
+				Params: map[string]interface{}{
+					"latestPulledN": policy.BranchRetention,
+				},
+				ScopeSelectors: map[string][]harborclientv5model.RetentionSelector{
+					"repository": {
+						{
+							Decoration: "repoMatches",
+							Kind:       "doublestar",
+							Pattern:    "pr-*",
+						},
+					},
+				},
+				TagSelectors: []*harborclientv5model.RetentionSelector{
+					{
+						Decoration: "matches",
+						Extras:     "{\"untagged\":true}",
+						Kind:       "doublestar",
+						Pattern:    "**",
+					},
+				},
+				Template: "latestPulledN",
+			},
+		},
+		Scope: &harborclientv5model.RetentionPolicyScope{
+			Level: "project",
+			Ref:   projectID,
+		},
+		Trigger: &harborclientv5model.RetentionRuleTrigger{
+			Kind: "Schedule",
+			Settings: map[string]string{
+				"cron": policy.Schedule,
+			},
+		},
+	}
+}
+
+func (h *Harbor) generateEmptyRetentionPolicy(projectID int64) *harborclientv5model.RetentionPolicy {
+	return &harborclientv5model.RetentionPolicy{
+		Algorithm: "or",
+		Rules:     []*harborclientv5model.RetentionRule{},
+		Scope: &harborclientv5model.RetentionPolicyScope{
+			Level: "project",
+			Ref:   projectID,
+		},
+		Trigger: &harborclientv5model.RetentionRuleTrigger{
+			Kind: "Schedule",
+			Settings: map[string]string{
+				"cron": "",
+			},
+		}}
+}
+
+func (h *Harbor) retentionOverrides(namespace corev1.Namespace) TagRetention {
+	// set the default retention policy
+	retention := h.TagRetention
+	// check if the annotation is set on the namespace
+	if ret, ok := namespace.ObjectMeta.Annotations["harbor.lagoon.sh/retention-policy"]; ok {
+		json.Unmarshal([]byte(ret), &retention)
+	}
+	return retention
+}

internal/harbor/harbor_helpers_test.go

@@ -1,7 +1,11 @@
 package harbor
 
 import (
+	"reflect"
 	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func TestHarbor_matchRobotAccount(t *testing.T) {
@@ -198,3 +202,55 @@ func TestHarbor_generateRobotName(t *testing.T) {
 		})
 	}
 }
+
+func TestHarbor_retentionOverrides(t *testing.T) {
+	type fields struct {
+		TagRetention TagRetention
+	}
+	type args struct {
+		namespace corev1.Namespace
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   TagRetention
+	}{
+		{
+			name: "test1",
+			fields: fields{
+				TagRetention: TagRetention{
+					Enabled:              false,
+					Schedule:             "M H(22-2) D(5-25) * *",
+					PullRequestRetention: 2,
+					BranchRetention:      5,
+				},
+			},
+			args: args{
+				namespace: corev1.Namespace{
+					ObjectMeta: metav1.ObjectMeta{
+						Annotations: map[string]string{
+							"harbor.lagoon.sh/retention-policy": "{\"enabled\":true,\"schedule\":\"M H(2-15) D(5-15) * *\", \"pullrequestRetention\": 3, \"branchRetention\": 6}",
+						},
+					},
+				},
+			},
+			want: TagRetention{
+				Enabled:              true,
+				Schedule:             "M H(2-15) D(5-15) * *",
+				PullRequestRetention: 3,
+				BranchRetention:      6,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			h := &Harbor{
+				TagRetention: tt.fields.TagRetention,
+			}
+			if got := h.retentionOverrides(tt.args.namespace); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Harbor.retentionOverrides() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}