199 group flattening can lead to conflicts due to non uniqueness

cicd/account_execution/staging/buildspec.yml

@@ -20,7 +20,7 @@ phases:
       # Update params with the values for this run for a developer account
       - |
         jq -n \
-        --argjson Parameters "{\"AppArn\": \"$AppArn\", \"AppVersion\": \"$AppVersion\", \"GoogleAdminEmailArn\": \"$SecretGoogleAdminEmail\", \"GoogleCredentialsArn\": \"$SecretGoogleCredentials\", \"SCIMEndpointUrlArn\": \"$SecretSCIMEndpoint\", \"SCIMAccessTokenArn\": \"$SecretSCIMAccessToken\", \"RegionArn\": \"$SecretRegion\", \"IdentityStoreIdArn\": \"$SecretIdentityStoreID\", \"GroupMatch\": \"name:AWS*\"}" \
+        --argjson Parameters "{\"AppArn\": \"$AppArn\", \"AppVersion\": \"$AppVersion\", \"GoogleAdminEmailArn\": \"$SecretGoogleAdminEmail\", \"GoogleCredentialsArn\": \"$SecretGoogleCredentials\", \"SCIMEndpointUrlArn\": \"$SecretSCIMEndpoint\", \"SCIMAccessTokenArn\": \"$SecretSCIMAccessToken\", \"RegionArn\": \"$SecretRegion\", \"IdentityStoreIdArn\": \"$SecretIdentityStoreID\", \"GroupMatch\": \"name:AWS*,name=NestedGroups\"}" \
         --argjson StackPolicy "{\"Statement\":[{\"Effect\": \"Allow\", \"NotAction\": \"Update:Delete\", \"Principal\": \"*\", \"Resource\": \"*\"}]}" \
         '$ARGS.named' > ./deploy/developer.json
       - cat ./deploy/developer.json

internal/sync.go

@@ -391,7 +391,7 @@
 		log.Info("creating user")
 		_, err := s.aws.CreateUser(awsUser)
 		if err != nil {
 			errHttp := new(aws.ErrHttpNotOK)
 			if errors.As(err, &errHttp) && errHttp.StatusCode == 409 {
 				log.WithField("user", awsUser.Username).Warn("user already exists")
 				continue
@@ -596,13 +596,25 @@
 		membersUsers := s.getGoogleUsersInGroup(g, gUserDetailCache, gGroupDetailCache)
 
 		// If we've not seen the user email address before add it to the list of unique users
+		// also, we need to deduplicate the list of members.
+		gUniqMembers := make(map[string]*admin.User)
                 for _, m := range membersUsers {
 			_, ok := gUniqUsers[m.PrimaryEmail]
 			if !ok {
 				gUniqUsers[m.PrimaryEmail] = gUserDetailCache[m.PrimaryEmail]
 			}
+
+			_, ok = gUniqMembers[m.PrimaryEmail]
+                        if !ok {
+                                gUniqMembers[m.PrimaryEmail] = gUserDetailCache[m.PrimaryEmail]
+                        }
 		}
-		gGroupsUsers[g.Name] = membersUsers
+
+	        gMembers := make([]*admin.User, 0)
+	        for _, member := range gUniqMembers {
+                        gMembers = append(gMembers, member)
+                }
+		gGroupsUsers[g.Name] = gMembers
 	}
 
 	for _, user := range gUniqUsers {
@@ -778,7 +790,7 @@
 	if err != nil {
 	        log.WithField("error", err).Warn("Problem performing test query against Identity Store")
 		return err
 	} else {
 	        log.WithField("Groups", response).Info("Test call for groups successful")
 
         }
@@ -857,7 +869,7 @@
 	return awsGroups, nil
 }
 
 func ListGroupsPagesCallbackFn(page *identitystore.ListGroupsOutput, lastPage bool) bool {
 	// Loop through each Group returned
 	for _, group := range page.Groups {
 		// Convert to native Group object
@@ -889,7 +901,7 @@
 	return awsUsers, nil
 }
 
 func ListUsersPagesCallbackFn(page *identitystore.ListUsersOutput, lastPage bool) bool {
 	// Loop through each User in ListUsersOutput and convert to native User object
 	for _, user := range page.Users {
 		awsUsers = append(awsUsers, ConvertSdkUserObjToNative(user))
@@ -897,7 +909,7 @@
 	return !lastPage
 }
 
 func ConvertSdkUserObjToNative(user *identitystore.User) *aws.User {
 	// Convert emails into native Email object
 	userEmails := make([]aws.UserEmail, 0)
 
@@ -941,7 +953,7 @@
 	}
 }
 
 func CreateUserIDtoUserObjMap(awsUsers []*aws.User) map[string]*aws.User {
 	awsUsersMap := make(map[string]*aws.User)
 
 	for _, awsUser := range awsUsers {
@@ -951,7 +963,7 @@
 	return awsUsersMap
 }
 
 var ListGroupMembershipPagesCallbackFn func(page *identitystore.ListGroupMembershipsOutput, lastPage bool) bool
 
 func (s *syncGSuite) GetGroupMembershipsLists(awsGroups []*aws.Group, awsUsersMap map[string]*aws.User) (map[string][]*aws.User, error) {
 	awsGroupsUsers := make(map[string][]*aws.User)