diff --git a/pkg/clients/repositories/client.go b/pkg/clients/repositories/client.go
index b0977a4..2aff46a 100644
--- a/pkg/clients/repositories/client.go
+++ b/pkg/clients/repositories/client.go
@@ -14,6 +14,7 @@ import (
 
 const (
 	errorRepositoryNotFound = "code = NotFound desc = repo"
+	errorPermissionDenied   = "code = PermissionDenied desc = permission denied"
 )
 
 // RepositoryServiceClient wraps the functions to connect to argocd repositories
@@ -43,3 +44,11 @@ func IsErrorRepositoryNotFound(err error) bool {
 	}
 	return strings.Contains(err.Error(), errorRepositoryNotFound)
 }
+
+// IsErrorPermissionDenied helper function to test for errorPermissionDenied error.
+func IsErrorPermissionDenied(err error) bool {
+	if err == nil {
+		return false
+	}
+	return strings.Contains(err.Error(), errorPermissionDenied)
+}
diff --git a/pkg/controller/repositories/controller.go b/pkg/controller/repositories/controller.go
index 20d0e20..73d1988 100644
--- a/pkg/controller/repositories/controller.go
+++ b/pkg/controller/repositories/controller.go
@@ -114,39 +114,23 @@ func (e *external) Observe(ctx context.Context, mg resource.Managed) (managed.Ex
 	}
 
 	observedRepository, err := e.client.Get(ctx, &repoQuery)
-	if err != nil {
-		return managed.ExternalObservation{}, resource.Ignore(repositories.IsErrorRepositoryNotFound, err)
-	}
 
-	passwordSecretResourceVersion, err := e.getSecretResourceVersion(ctx, cr.Spec.ForProvider.PasswordRef)
-	if err != nil {
-		return managed.ExternalObservation{}, err
-	}
-	sshPrivateKeyResourceVersion, err := e.getSecretResourceVersion(ctx, cr.Spec.ForProvider.SSHPrivateKeyRef)
-	if err != nil {
-		return managed.ExternalObservation{}, err
-	}
-	tlsClientCertDataResourceVersion, err := e.getSecretResourceVersion(ctx, cr.Spec.ForProvider.TLSClientCertDataRef)
-	if err != nil {
-		return managed.ExternalObservation{}, err
+	if err != nil && repositories.IsErrorPermissionDenied(err) || repositories.IsErrorRepositoryNotFound(err) {
+		return managed.ExternalObservation{
+			ResourceExists: false,
+		}, nil
 	}
-	tlsClientCertKeyResourceVersion, err := e.getSecretResourceVersion(ctx, cr.Spec.ForProvider.TLSClientCertKeyRef)
+
 	if err != nil {
 		return managed.ExternalObservation{}, err
 	}
-	githubAppPrivateKeyResourceVersion, err := e.getSecretResourceVersion(ctx, cr.Spec.ForProvider.GithubAppPrivateKeyRef)
+
+	resourceVersions, err := e.getSecretResource(ctx, cr)
+
 	if err != nil {
 		return managed.ExternalObservation{}, err
 	}
 
-	resourceVersions := secretResourceVersion{
-		Password:            passwordSecretResourceVersion,
-		SSHPrivateKey:       sshPrivateKeyResourceVersion,
-		TLSClientCertData:   tlsClientCertDataResourceVersion,
-		TLSClientCertKey:    tlsClientCertKeyResourceVersion,
-		GithubAppPrivateKey: githubAppPrivateKeyResourceVersion,
-	}
-
 	current := cr.Spec.ForProvider.DeepCopy()
 	lateInitializeRepository(&cr.Spec.ForProvider, observedRepository)
 
@@ -536,3 +520,35 @@ func (e *external) getPayload(ctx context.Context, ref *v1alpha1.SecretReference
 
 	return nil, nil
 }
+
+func (e *external) getSecretResource(ctx context.Context, cr *v1alpha1.Repository) (secretResourceVersion, error) {
+	passwordSecretResourceVersion, err := e.getSecretResourceVersion(ctx, cr.Spec.ForProvider.PasswordRef)
+	if err != nil {
+		return secretResourceVersion{}, err
+	}
+	sshPrivateKeyResourceVersion, err := e.getSecretResourceVersion(ctx, cr.Spec.ForProvider.SSHPrivateKeyRef)
+	if err != nil {
+		return secretResourceVersion{}, err
+	}
+	tlsClientCertDataResourceVersion, err := e.getSecretResourceVersion(ctx, cr.Spec.ForProvider.TLSClientCertDataRef)
+	if err != nil {
+		return secretResourceVersion{}, err
+	}
+	tlsClientCertKeyResourceVersion, err := e.getSecretResourceVersion(ctx, cr.Spec.ForProvider.TLSClientCertKeyRef)
+	if err != nil {
+		return secretResourceVersion{}, err
+	}
+	githubAppPrivateKeyResourceVersion, err := e.getSecretResourceVersion(ctx, cr.Spec.ForProvider.GithubAppPrivateKeyRef)
+	if err != nil {
+		return secretResourceVersion{}, err
+	}
+
+	return secretResourceVersion{
+		Password:            passwordSecretResourceVersion,
+		SSHPrivateKey:       sshPrivateKeyResourceVersion,
+		TLSClientCertData:   tlsClientCertDataResourceVersion,
+		TLSClientCertKey:    tlsClientCertKeyResourceVersion,
+		GithubAppPrivateKey: githubAppPrivateKeyResourceVersion,
+	}, nil
+
+}
diff --git a/pkg/controller/repositories/controller_test.go b/pkg/controller/repositories/controller_test.go
index 7eeec6e..2f0a547 100644
--- a/pkg/controller/repositories/controller_test.go
+++ b/pkg/controller/repositories/controller_test.go
@@ -40,8 +40,10 @@ import (
 )
 
 var (
-	errBoom                    = errors.New("boom")
-	errNotFound                = errors.New("code = NotFound desc = repo")
+	errBoom = errors.New("boom")
+	// Unused until issue https://github.com/argoproj/argo-cd/issues/20005 in Argo CD project is resolved
+	// errNotFound                = errors.New("code = NotFound desc = repo")
+	errPermissionDenied        = errors.New("code = PermissionDenied desc = permission denied")
 	testRepositoryExternalName = "testRepo"
 	testRepo                   = "https://gitlab.com/example-group/example-project.git"
 	testUsername               = "testUser"
@@ -206,7 +208,7 @@ func TestObserve(t *testing.T) {
 						&argocdRepository.RepoQuery{
 							Repo: testRepositoryExternalName,
 						},
-					).Return(nil, errNotFound)
+					).Return(nil, errPermissionDenied) // Switch to errNotFound when issue https://github.com/argoproj/argo-cd/issues/20005 in Argo CD is solved
 				}),
 				cr: Repository(
 					withExternalName(testRepositoryExternalName),