Merge branch 'appset-resource-tree' of https://github.com/alexymantha…

…/argo-cd into alexy-appset-resource-tree

.github/pull_request_template.md

@@ -13,11 +13,12 @@ Checklist:
 * [ ] I've updated both the CLI and UI to expose my feature, or I plan to submit a second PR with them.
 * [ ] Does this PR require documentation updates?
 * [ ] I've updated documentation as required by this PR.
-* [ ] Optional. My organization is added to USERS.md.
 * [ ] I have signed off all my commits as required by [DCO](https://github.com/argoproj/argoproj/blob/master/community/CONTRIBUTING.md#legal)
 * [ ] I have written unit and/or e2e tests for my change. PRs without these are unlikely to be merged.
 * [ ] My build is green ([troubleshooting builds](https://argo-cd.readthedocs.io/en/latest/developer-guide/ci/)).
 * [ ] My new feature complies with the [feature status](https://github.com/argoproj/argoproj/blob/master/community/feature-status.md) guidelines.
 * [ ] I have added a brief description of why this PR is necessary and/or what this PR solves.
+* [ ] Optional. My organization is added to USERS.md.
+* [ ] Optional. For bug fixes, I've indicated what older releases this fix should be cherry-picked into (this may or may not happen depending on risk/complexity).
 
 <!-- Please see [Contribution FAQs](https://argo-cd.readthedocs.io/en/latest/developer-guide/faq/) if you have questions about your pull-request. -->

USERS.md

@@ -148,6 +148,7 @@ Currently, the following organizations are **officially** using Argo CD:
 1. [Kinguin](https://www.kinguin.net/)
 1. [KintoHub](https://www.kintohub.com/)
 1. [KompiTech GmbH](https://www.kompitech.com/)
+1. [Kong Inc.](https://konghq.com/)
 1. [KPMG](https://kpmg.com/uk)
 1. [KubeSphere](https://github.com/kubesphere)
 1. [Kurly](https://www.kurly.com/)

applicationset/controllers/applicationset_controller.go

@@ -108,16 +108,14 @@ func (r *ApplicationSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 
 	// Do not attempt to further reconcile the ApplicationSet if it is being deleted.
 	if applicationSetInfo.ObjectMeta.DeletionTimestamp != nil {
-		if controllerutil.ContainsFinalizer(&applicationSetInfo, argov1alpha1.ResourcesFinalizerName) {
-			deleteAllowed := utils.DefaultPolicy(applicationSetInfo.Spec.SyncPolicy, r.Policy, r.EnablePolicyOverride).AllowDelete()
-			if !deleteAllowed {
-				if err := r.removeOwnerReferencesOnDeleteAppSet(ctx, applicationSetInfo); err != nil {
-					return ctrl.Result{}, err
-				}
-				controllerutil.RemoveFinalizer(&applicationSetInfo, argov1alpha1.ResourcesFinalizerName)
-				if err := r.Update(ctx, &applicationSetInfo); err != nil {
-					return ctrl.Result{}, err
-				}
+		deleteAllowed := utils.DefaultPolicy(applicationSetInfo.Spec.SyncPolicy, r.Policy, r.EnablePolicyOverride).AllowDelete()
+		if !deleteAllowed {
+			if err := r.removeOwnerReferencesOnDeleteAppSet(ctx, applicationSetInfo); err != nil {
+				return ctrl.Result{}, err
+			}
+			controllerutil.RemoveFinalizer(&applicationSetInfo, argov1alpha1.ResourcesFinalizerName)
+			if err := r.Update(ctx, &applicationSetInfo); err != nil {
+				return ctrl.Result{}, err
 			}
 		}
 		return ctrl.Result{}, nil
@@ -165,6 +163,16 @@ func (r *ApplicationSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 		return ctrl.Result{RequeueAfter: ReconcileRequeueOnValidationError}, nil
 	}
 
+	currentApplications, err := r.getCurrentApplications(ctx, applicationSetInfo)
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to get current applications for application set: %w", err)
+	}
+
+  err = r.updateResourcesStatus(ctx, logCtx, &applicationSetInfo, currentApplications)
+  if err != nil {
+    return ctrl.Result{}, fmt.Errorf("failed to get update resources status for application set: %w", err)
+  }
+
 	// appMap is a name->app collection of Applications in this ApplicationSet.
 	appMap := map[string]argov1alpha1.Application{}
 	// appSyncMap tracks which apps will be synced during this reconciliation.
@@ -181,16 +189,11 @@ func (r *ApplicationSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 			}
 		} else if applicationSetInfo.Spec.Strategy != nil {
 			// appset uses progressive sync
-			applications, err := r.getCurrentApplications(ctx, applicationSetInfo)
-			if err != nil {
-				return ctrl.Result{}, fmt.Errorf("failed to get current applications for application set: %w", err)
-			}
-
-			for _, app := range applications {
+			for _, app := range currentApplications {
 				appMap[app.Name] = app
 			}
 
-			appSyncMap, err = r.performProgressiveSyncs(ctx, logCtx, applicationSetInfo, applications, desiredApplications, appMap)
+			appSyncMap, err = r.performProgressiveSyncs(ctx, logCtx, applicationSetInfo, currentApplications, desiredApplications, appMap)
 			if err != nil {
 				return ctrl.Result{}, fmt.Errorf("failed to perform progressive sync reconciliation for application set: %w", err)
 			}
@@ -1351,6 +1354,79 @@ func findApplicationStatusIndex(appStatuses []argov1alpha1.ApplicationSetApplica
 	return -1
 }
 
+func (r *ApplicationSetReconciler) updateResourcesStatus(ctx context.Context, logCtx *log.Entry, appset *argov1alpha1.ApplicationSet, apps []argov1alpha1.Application) error {
+	statusMap := getResourceStatusMap(appset)
+	statusMap = buildResourceStatus(statusMap, apps)
+
+	statuses := []argov1alpha1.ResourceStatus{}
+	for _, status := range statusMap {
+		statuses = append(statuses, status)
+	}
+	appset.Status.Resources = statuses
+
+	namespacedName := types.NamespacedName{Namespace: appset.Namespace, Name: appset.Name}
+	err := r.Client.Status().Update(ctx, appset)
+	if err != nil {
+
+		logCtx.Errorf("unable to set application set status: %v", err)
+		return fmt.Errorf("unable to set application set status: %v", err)
+	}
+
+	if err := r.Get(ctx, namespacedName, appset); err != nil {
+		if client.IgnoreNotFound(err) != nil {
+			return nil
+		}
+		return fmt.Errorf("error fetching updated application set: %v", err)
+	}
+
+	return nil
+}
+
+func buildResourceStatus(statusMap map[string]argov1alpha1.ResourceStatus, apps []argov1alpha1.Application) map[string]argov1alpha1.ResourceStatus {
+	appMap := map[string]argov1alpha1.Application{}
+	for _, app := range apps {
+		appCopy := app
+		appMap[app.Name] = app
+
+		// Create status if it does not exist
+		status, ok := statusMap[app.Name]
+		if !ok {
+			status = argov1alpha1.ResourceStatus{
+				Name:      app.Name,
+				Namespace: app.Namespace,
+				Status:    app.Status.Sync.Status,
+				Health:    &appCopy.Status.Health,
+			}
+		}
+
+		status.Name = app.Name
+		status.Namespace = app.Namespace
+		status.Status = app.Status.Sync.Status
+		status.Health = &appCopy.Status.Health
+
+		statusMap[app.Name] = status
+	}
+	cleanupDeletedApplicationStatuses(statusMap, appMap)
+
+	return statusMap
+}
+
+func getResourceStatusMap(appset *argov1alpha1.ApplicationSet) map[string]argov1alpha1.ResourceStatus {
+	statusMap := map[string]argov1alpha1.ResourceStatus{}
+	for _, status := range appset.Status.Resources {
+		statusMap[status.Name] = status
+	}
+	return statusMap
+}
+
+func cleanupDeletedApplicationStatuses(statusMap map[string]argov1alpha1.ResourceStatus, apps map[string]argov1alpha1.Application) {
+	for name := range statusMap {
+		if _, ok := apps[name]; !ok {
+			delete(statusMap, name)
+		}
+	}
+}
+
 // setApplicationSetApplicationStatus updates the ApplicatonSet's status field
 // with any new/changed Application statuses.
 func (r *ApplicationSetReconciler) setAppSetApplicationStatus(ctx context.Context, logCtx *log.Entry, applicationSet *argov1alpha1.ApplicationSet, applicationStatuses []argov1alpha1.ApplicationSetApplicationStatus) error {

applicationset/generators/git.go

@@ -56,12 +56,14 @@ func (g *GitGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.Applic
 		return nil, EmptyAppSetGeneratorError
 	}
 
+	noRevisionCache := appSet.RefreshRequired()
+
 	var err error
 	var res []map[string]interface{}
 	if len(appSetGenerator.Git.Directories) != 0 {
-		res, err = g.generateParamsForGitDirectories(appSetGenerator, appSet.Spec.GoTemplate, appSet.Spec.GoTemplateOptions)
+		res, err = g.generateParamsForGitDirectories(appSetGenerator, noRevisionCache, appSet.Spec.GoTemplate, appSet.Spec.GoTemplateOptions)
 	} else if len(appSetGenerator.Git.Files) != 0 {
-		res, err = g.generateParamsForGitFiles(appSetGenerator, appSet.Spec.GoTemplate, appSet.Spec.GoTemplateOptions)
+		res, err = g.generateParamsForGitFiles(appSetGenerator, noRevisionCache, appSet.Spec.GoTemplate, appSet.Spec.GoTemplateOptions)
 	} else {
 		return nil, EmptyAppSetGeneratorError
 	}
@@ -72,10 +74,10 @@ func (g *GitGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.Applic
 	return res, nil
 }
 
-func (g *GitGenerator) generateParamsForGitDirectories(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, useGoTemplate bool, goTemplateOptions []string) ([]map[string]interface{}, error) {
+func (g *GitGenerator) generateParamsForGitDirectories(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, noRevisionCache bool, useGoTemplate bool, goTemplateOptions []string) ([]map[string]interface{}, error) {
 
 	// Directories, not files
-	allPaths, err := g.repos.GetDirectories(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision)
+	allPaths, err := g.repos.GetDirectories(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision, noRevisionCache)
 	if err != nil {
 		return nil, fmt.Errorf("error getting directories from repo: %w", err)
 	}
@@ -98,12 +100,12 @@ func (g *GitGenerator) generateParamsForGitDirectories(appSetGenerator *argoproj
 	return res, nil
 }
 
-func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, useGoTemplate bool, goTemplateOptions []string) ([]map[string]interface{}, error) {
+func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, noRevisionCache bool, useGoTemplate bool, goTemplateOptions []string) ([]map[string]interface{}, error) {
 
 	// Get all files that match the requested path string, removing duplicates
 	allFiles := make(map[string][]byte)
 	for _, requestedPath := range appSetGenerator.Git.Files {
-		files, err := g.repos.GetFiles(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision, requestedPath.Path)
+		files, err := g.repos.GetFiles(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision, requestedPath.Path, noRevisionCache)
 		if err != nil {
 			return nil, err
 		}

applicationset/generators/git_test.go

@@ -317,7 +317,7 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) {
 
 			argoCDServiceMock := mocks.Repos{}
 
-			argoCDServiceMock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything).Return(testCaseCopy.repoApps, testCaseCopy.repoError)
+			argoCDServiceMock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(testCaseCopy.repoApps, testCaseCopy.repoError)
 
 			var gitGenerator = NewGitGenerator(&argoCDServiceMock)
 			applicationSetInfo := argoprojiov1alpha1.ApplicationSet{
@@ -613,7 +613,7 @@ func TestGitGenerateParamsFromDirectoriesGoTemplate(t *testing.T) {
 
 			argoCDServiceMock := mocks.Repos{}
 
-			argoCDServiceMock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything).Return(testCaseCopy.repoApps, testCaseCopy.repoError)
+			argoCDServiceMock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(testCaseCopy.repoApps, testCaseCopy.repoError)
 
 			var gitGenerator = NewGitGenerator(&argoCDServiceMock)
 			applicationSetInfo := argoprojiov1alpha1.ApplicationSet{
@@ -972,7 +972,7 @@ cluster:
 			t.Parallel()
 
 			argoCDServiceMock := mocks.Repos{}
-			argoCDServiceMock.On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
+			argoCDServiceMock.On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
 				Return(testCaseCopy.repoFileContents, testCaseCopy.repoPathsError)
 
 			var gitGenerator = NewGitGenerator(&argoCDServiceMock)
@@ -1322,7 +1322,7 @@ cluster:
 			t.Parallel()
 
 			argoCDServiceMock := mocks.Repos{}
-			argoCDServiceMock.On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything).
+			argoCDServiceMock.On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
 				Return(testCaseCopy.repoFileContents, testCaseCopy.repoPathsError)
 
 			var gitGenerator = NewGitGenerator(&argoCDServiceMock)

applicationset/generators/matrix_test.go

@@ -1108,7 +1108,7 @@ func TestGitGenerator_GenerateParams_list_x_git_matrix_generator(t *testing.T) {
 	}
 
 	repoServiceMock := &mocks.Repos{}
-	repoServiceMock.On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(map[string][]byte{
+	repoServiceMock.On("GetFiles", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(map[string][]byte{
 		"some/path.json": []byte("test: content"),
 	}, nil)
 	gitGenerator := NewGitGenerator(repoServiceMock)

applicationset/services/repo_service.go

@@ -11,6 +11,8 @@ import (
 	"github.com/argoproj/argo-cd/v2/util/io"
 )
 
+//go:generate go run github.com/vektra/mockery/[email protected] --name=RepositoryDB
+
 // RepositoryDB Is a lean facade for ArgoDB,
 // Using a lean interface makes it easier to test the functionality of the git generator
 type RepositoryDB interface {
@@ -25,13 +27,15 @@ type argoCDService struct {
 	newFileGlobbingEnabled bool
 }
 
+//go:generate go run github.com/vektra/mockery/[email protected] --name=Repos
+
 type Repos interface {
 
 	// GetFiles returns content of files (not directories) within the target repo
-	GetFiles(ctx context.Context, repoURL string, revision string, pattern string) (map[string][]byte, error)
+	GetFiles(ctx context.Context, repoURL string, revision string, pattern string, noRevisionCache bool) (map[string][]byte, error)
 
 	// GetDirectories returns a list of directories (not files) within the target repo
-	GetDirectories(ctx context.Context, repoURL string, revision string) ([]string, error)
+	GetDirectories(ctx context.Context, repoURL string, revision string, noRevisionCache bool) ([]string, error)
 }
 
 func NewArgoCDService(db db.ArgoDB, submoduleEnabled bool, repoClientset apiclient.Clientset, newFileGlobbingEnabled bool) (Repos, error) {
@@ -43,7 +47,7 @@ func NewArgoCDService(db db.ArgoDB, submoduleEnabled bool, repoClientset apiclie
 	}, nil
 }
 
-func (a *argoCDService) GetFiles(ctx context.Context, repoURL string, revision string, pattern string) (map[string][]byte, error) {
+func (a *argoCDService) GetFiles(ctx context.Context, repoURL string, revision string, pattern string, noRevisionCache bool) (map[string][]byte, error) {
 	repo, err := a.repositoriesDB.GetRepository(ctx, repoURL)
 	if err != nil {
 		return nil, fmt.Errorf("error in GetRepository: %w", err)
@@ -55,6 +59,7 @@ func (a *argoCDService) GetFiles(ctx context.Context, repoURL string, revision s
 		Revision:                  revision,
 		Path:                      pattern,
 		NewGitFileGlobbingEnabled: a.newFileGlobbingEnabled,
+		NoRevisionCache:           noRevisionCache,
 	}
 	closer, client, err := a.repoServerClientSet.NewRepoServerClient()
 	if err != nil {
@@ -69,7 +74,7 @@ func (a *argoCDService) GetFiles(ctx context.Context, repoURL string, revision s
 	return fileResponse.GetMap(), nil
 }
 
-func (a *argoCDService) GetDirectories(ctx context.Context, repoURL string, revision string) ([]string, error) {
+func (a *argoCDService) GetDirectories(ctx context.Context, repoURL string, revision string, noRevisionCache bool) ([]string, error) {
 	repo, err := a.repositoriesDB.GetRepository(ctx, repoURL)
 	if err != nil {
 		return nil, fmt.Errorf("error in GetRepository: %w", err)
@@ -79,6 +84,7 @@ func (a *argoCDService) GetDirectories(ctx context.Context, repoURL string, revi
 		Repo:             repo,
 		SubmoduleEnabled: a.submoduleEnabled,
 		Revision:         revision,
+		NoRevisionCache:  noRevisionCache,
 	}
 
 	closer, client, err := a.repoServerClientSet.NewRepoServerClient()