Add new integration tests for Azure OIDC for git repositories

Signed-off-by: Dipti Pai <[email protected]>

oci/tests/integration/.env.sample

@@ -11,12 +11,14 @@
 # export TF_VAR_rand=${RANDOM}
 
 ## Azure
+# export ARM_SUBSCRIPTION_ID=
+# export TF_VAR_azuredevops_org=
+# export TF_VAR_azuredevops_pat=
 # export TF_VAR_azure_location=eastus
 ## Set the following only when authenticating using Service Principal (suited
 ## for CI environment).
 # export ARM_CLIENT_ID=
 # export ARM_CLIENT_SECRET=
-# export ARM_SUBSCRIPTION_ID=
 # export ARM_TENANT_ID=
 
 ## GCP
@@ -48,3 +50,7 @@
 # export TF_VAR_wi_k8s_sa_name=test-workload-id
 # export TF_VAR_wi_k8s_sa_ns=default
 # export TF_VAR_enable_wi=true
+
+## Test Configuration variables
+# export TF_VAR_enable_git=true
+# export TF_VAR_enable_oci=true

oci/tests/integration/Makefile

@@ -1,4 +1,5 @@
 GO_TEST_ARGS ?=
+GO_TEST_PREFIX ?=
 PROVIDER_ARG ?=
 TEST_TIMEOUT ?= 30m
 GOARCH ?= amd64
@@ -15,14 +16,20 @@ docker-build: app
 
 test:
 	docker image inspect $(TEST_IMG) >/dev/null
-	TEST_IMG=$(TEST_IMG) go test -timeout $(TEST_TIMEOUT) -v ./ $(GO_TEST_ARGS) $(PROVIDER_ARG) --tags=integration
+	TEST_IMG=$(TEST_IMG) go test -timeout $(TEST_TIMEOUT) -v ./ -run "^$(GO_TEST_PREFIX).*" $(GO_TEST_ARGS) $(PROVIDER_ARG) --tags=integration
 
 test-aws:
 	$(MAKE) test PROVIDER_ARG="-provider aws"
 
 test-azure:
 	$(MAKE) test PROVIDER_ARG="-provider azure"
 
+test-azure-git:
+	$(MAKE) test PROVIDER_ARG="-provider azure" GO_TEST_PREFIX="TestGit"
+
+test-azure-oci:
+	$(MAKE) test PROVIDER_ARG="-provider azure" GO_TEST_PREFIX="TestOci"
+
 test-gcp:
 	$(MAKE) test PROVIDER_ARG="-provider gcp"
 

oci/tests/integration/README.md

@@ -1,7 +1,7 @@
-# OCI integration test
+# Integration tests
 
-OCI integration test uses a test application(`testapp/`) to test the
-oci package against each of the supported cloud providers.
+Integration tests uses a test application(`testapp/`) to test the
+oci and git package against each of the supported cloud providers.
 
 **NOTE:** Tests in this package aren't run automatically by the `test-*` make
 target at the root of `fluxcd/pkg` repo. These tests are more complicated than
@@ -16,7 +16,7 @@ runs the test app as a batch job which tries to log in and list tags from the
 test registry repository. A successful job indicates successful test. If the job
 fails, the test fails.
 
-Logs of a successful job run:
+Logs of a successful job run for oci:
 ```console
 $ kubectl logs test-job-93tbl-4jp2r
 2022/07/28 21:59:06 repo: xxx.dkr.ecr.us-east-2.amazonaws.com/test-repo-flux-test-heroic-ram
@@ -25,6 +25,25 @@ $ kubectl logs test-job-93tbl-4jp2r
 2022/07/28 21:59:06 tags: [v0.1.4 v0.1.3 v0.1.0 v0.1.2]
 ```
 
+Logs of a successful job run for git:
+```console
+$ kubectl logs test-job-dzful-jrcqw
+2024/08/27 22:28:22 Successfully cloned repository
+2024/08/27 22:28:22 apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: foobar
+2024/08/27 22:28:22 Keys in cache  0 [https://dev.azure.com/xxx/fluxProjpopularosheepdog/_git/fluxRepopopularosheepdog]
+2024/08/27 22:28:22 Cache entry expiration  2024-08-28 22:28:21.335223377 +0000 UTC <nil>
+2024/08/27 22:28:22 Successfully cloned repository
+2024/08/27 22:28:22 apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: foobar
+2024/08/27 22:28:22 Keys in cache  1 [https://dev.azure.com/xxx/fluxProjpopularosheepdog/_git/fluxRepopopularosheepdog]
+2024/08/27 22:28:22 Cache entry expiration  2024-08-28 22:28:21.335223377 +0000 UTC <nil>
+```
+
 ## Requirements
 
 ### Amazon Web Services
@@ -316,6 +335,18 @@ module "aws_gh_actions" {
     workloads to access ACR.
 - Azure CLI, need to be logged in using `az login` as a User (not a Service
   Principal).
+- An Azure DevOps organization [connected to Microsoft
+  Entra](https://learn.microsoft.com/en-us/azure/devops/organizations/accounts/connect-organization-to-azure-ad?view=azure-devops),
+  personal access token for accessing repositories within the organization. The
+  scope required for the personal access token is:
+  - Project and Team - read, write and manage access
+  - Member Entitlement Management (Read & Write)
+  - Code - Full
+  - Please take a look at the [terraform
+    provider](https://registry.terraform.io/providers/microsoft/azuredevops/latest/docs/guides/authenticating_using_the_personal_access_token#create-a-personal-access-token)
+    for more explanation.
+  - A valid Azure devops configuration is needed even if git is not being
+    tested. 
 
   **NOTE:** To use Service Principal (for example in CI environment), set the
   `ARM-*` variables in `.env`, source it and authenticate Azure CLI with:
@@ -520,9 +551,10 @@ Run the test with `make test-*`, setting the test app image with variable
 $ make test-azure
 make test PROVIDER_ARG="-provider azure"
 docker image inspect fluxcd/testapp:test >/dev/null
-TEST_IMG=fluxcd/testapp:test go test -timeout 30m -v ./ -verbose -retain -provider azure --tags=integration
-2022/07/29 02:06:51 Terraform binary:  /usr/bin/terraform
-2022/07/29 02:06:51 Init Terraform
+TEST_IMG=fluxcd/testapp:test go test -timeout 30m -v ./ -run "^.*" -provider azure --tags=integration
+2024/08/26 23:39:13 Terraform binary:  /snap/bin/terraform
+2024/08/26 23:39:13 Init Terraform
+2024/08/26 23:39:15 Applying Terraform
 ...
 ```
 
@@ -532,7 +564,10 @@ the resources don't get deleted, the `make destroy-*` commands can be run for
 the respective provider. This will run terraform destroy in the respective
 provider's terraform configuration directory. This can be used to quickly
 destroy the infrastructure without going through the provision-test-destroy
-steps.
+steps. There is a known issue with Azure user not getting cleaned up if the
+infrastructure is retained and destroy is used for cleanup. The workaround is to
+manually delete the user from Azure DevOps Organization
+Settings->Users page. 
 
 ## Workload Identity
 
@@ -547,6 +582,8 @@ export TF_VAR_enable_wi=
 
 They have been included in the `.env.sample` and you can simply uncomment it.
 
+The git integration tests require workload identity to be enabled.
+
 ## Debugging the tests
 
 For debugging environment provisioning, enable verbose output with `-verbose`

oci/tests/integration/aws_test.go

@@ -98,3 +98,18 @@ func getWISAAnnotationsAWS(output map[string]*tfjson.StateOutput) (map[string]st
 		eksRoleArnAnnotation: iamARN,
 	}, nil
 }
+
+// When implemented, getGitTestConfigAws would return the git-specific test config for AWS
+func getGitTestConfigAWS(outputs map[string]*tfjson.StateOutput) (*gitTestConfig, error) {
+	return nil, fmt.Errorf("NotImplemented for AWS")
+}
+
+// When implemented, givePermissionsToRepositoryAWS would grant the required permissions to AWS CodeCommit repository
+func givePermissionsToRepositoryAWS(ctx context.Context, output map[string]*tfjson.StateOutput) (string, error) {
+	return "", fmt.Errorf("NotImplemented for AWS")
+}
+
+// When implemented, revokePermissionsToRepositoryAWS would revoke the permissions granted to AWS CodeCommit repository
+func revokePermissionsToRepositoryAWS(ctx context.Context, gitPermissionID string, outputs map[string]*tfjson.StateOutput) error {
+	return fmt.Errorf("NotImplemented for AWS")
+}

oci/tests/integration/azure_test.go

@@ -22,10 +22,18 @@ package integration
 import (
 	"context"
 	"fmt"
+	"log"
+	"strings"
+	"time"
 
-	tfjson "github.com/hashicorp/terraform-json"
-
+	"github.com/fluxcd/pkg/git"
 	"github.com/fluxcd/test-infra/tftestenv"
+	"github.com/google/uuid"
+	tfjson "github.com/hashicorp/terraform-json"
+	"github.com/microsoft/azure-devops-go-api/azuredevops/v7"
+	"github.com/microsoft/azure-devops-go-api/azuredevops/v7/graph"
+	"github.com/microsoft/azure-devops-go-api/azuredevops/v7/licensing"
+	"github.com/microsoft/azure-devops-go-api/azuredevops/v7/memberentitlementmanagement"
 )
 
 const (
@@ -81,3 +89,147 @@ func getWISAAnnotationsAzure(output map[string]*tfjson.StateOutput) (map[string]
 		azureWIClientIdAnnotation: clientID,
 	}, nil
 }
+
+// Give managed identity permissions on the azure devops project using azure-devops-go-api
+// https://github.com/microsoft/azure-devops-go-api/blob/dev/azuredevops/v7/memberentitlementmanagement/client.go#L147
+// This can be moved to terraform if/when this PR completes -
+// https://github.com/microsoft/terraform-provider-azuredevops/pull/1028
+// Returns a string representing the uuid of the entity that was granted permissions
+func givePermissionsToRepositoryAzure(ctx context.Context, outputs map[string]*tfjson.StateOutput) (string, error) {
+	// Organization, PAT, Project ID and WI ID are availble as terraform output
+	organization := outputs["azure_devops_organization"].Value.(string)
+	projectId := outputs["azure_devops_project_id"].Value.(string)
+	pat := outputs["azure_devops_access_token"].Value.(string)
+	wiObjectId := outputs["workload_identity_object_id"].Value.(string)
+	var servicePrincipalID string
+
+	// Create a connection to the organization and create a new client
+	connection := azuredevops.NewPatConnection(fmt.Sprintf("https://dev.azure.com/%s", organization), pat)
+	client, err := memberentitlementmanagement.NewClient(ctx, connection)
+	if err != nil {
+		return servicePrincipalID, err
+	}
+
+	uuid, err := uuid.Parse(projectId)
+	if err != nil {
+		return servicePrincipalID, err
+	}
+	origin := "AAD"
+	kind := "servicePrincipal"
+	servicePrincipal := memberentitlementmanagement.ServicePrincipalEntitlement{
+		AccessLevel: &licensing.AccessLevel{
+			AccountLicenseType: &licensing.AccountLicenseTypeValues.Express,
+		},
+		ProjectEntitlements: &[]memberentitlementmanagement.ProjectEntitlement{
+			{
+				ProjectRef: &memberentitlementmanagement.ProjectRef{
+					Id: &uuid,
+				},
+				Group: &memberentitlementmanagement.Group{
+					GroupType: &memberentitlementmanagement.GroupTypeValues.ProjectContributor,
+				},
+			},
+		},
+		ServicePrincipal: &graph.GraphServicePrincipal{
+			Origin:      &origin,
+			OriginId:    &wiObjectId,
+			SubjectKind: &kind,
+		},
+	}
+
+	// First request to add new user fails, second request succeeds, add a retry
+	retryAttempts := 2
+	retryDelay := 1 * time.Second // 1 seconds delay
+	attempts := 0
+	for attempts < retryAttempts {
+		attempts++
+		responseValue, err := client.AddServicePrincipalEntitlement(ctx, memberentitlementmanagement.AddServicePrincipalEntitlementArgs{ServicePrincipalEntitlement: &servicePrincipal})
+		if err != nil {
+			return servicePrincipalID, err
+		}
+
+		if !*responseValue.OperationResult.IsSuccess {
+			errMsg := getServicePrincipalEntitlementAPIErrorMessage(*responseValue.OperationResult)
+			if strings.Contains(errMsg, "VS403283: Could not add user") {
+				log.Println("Retryable error encountered", errMsg)
+				time.Sleep(retryDelay)
+				continue
+			} else {
+				return servicePrincipalID, fmt.Errorf(errMsg)
+			}
+		}
+		uuid := responseValue.OperationResult.ServicePrincipalId
+		servicePrincipalID = uuid.String()
+		break
+	}
+
+	log.Println("Added service principal entitlement", servicePrincipalID)
+
+	return servicePrincipalID, nil
+}
+
+func getServicePrincipalEntitlementAPIErrorMessage(operationResult memberentitlementmanagement.ServicePrincipalEntitlementOperationResult) string {
+	errMsg := "Unknown API error"
+	if operationResult.Errors != nil && len(*operationResult.Errors) > 0 {
+		var errorMessages []string
+		for _, err := range *operationResult.Errors {
+			errorMessages = append(errorMessages, fmt.Sprintf("(%v) %s", *err.Key, *err.Value))
+		}
+		errMsg = strings.Join(errorMessages, "\n")
+	}
+	return errMsg
+}
+
+// revokePermissionsToRepositoryAzure deletes the managed identity from users list in the organization using azure-devops-go-api
+// https://github.com/microsoft/azure-devops-go-api/blob/dev/azuredevops/v7/memberentitlementmanagement/client.go#L235
+func revokePermissionsToRepositoryAzure(ctx context.Context, servicePrincipalID string, outputs map[string]*tfjson.StateOutput) error {
+	uuid, err := uuid.Parse(servicePrincipalID)
+	if err != nil {
+		return err
+	}
+	// Organization, PAT, Project ID and WI ID are availble as terraform output
+	organization := outputs["azure_devops_organization"].Value.(string)
+	pat := outputs["azure_devops_access_token"].Value.(string)
+
+	// Create a connection to the organization and create a new client
+	connection := azuredevops.NewPatConnection(fmt.Sprintf("https://dev.azure.com/%s", organization), pat)
+	client, err := memberentitlementmanagement.NewClient(ctx, connection)
+	if err != nil {
+		return err
+	}
+
+	err = client.DeleteServicePrincipalEntitlement(ctx, memberentitlementmanagement.DeleteServicePrincipalEntitlementArgs{ServicePrincipalId: &uuid})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return nil
+}
+
+// getGitTestConfigAzure returns the test config used to setup the git repository
+func getGitTestConfigAzure(outputs map[string]*tfjson.StateOutput) (*gitTestConfig, error) {
+	config := &gitTestConfig{
+		defaultGitTransport:   git.HTTP,
+		gitUsername:           git.DefaultPublicKeyAuthUser,
+		gitPat:                outputs["azure_devops_access_token"].Value.(string),
+		applicationRepository: outputs["git_repo_url"].Value.(string),
+	}
+
+	opts, err := getAuthOpts(config.applicationRepository, map[string][]byte{
+		"password": []byte(config.gitPat),
+		"username": []byte(git.DefaultPublicKeyAuthUser),
+	})
+	if err != nil {
+		return nil, err
+	}
+	config.defaultAuthOpts = opts
+
+	parts := strings.Split(config.applicationRepository, "@")
+	// Check if the URL contains the "@" symbol
+	if len(parts) > 1 {
+		// Reconstruct the URL without the username
+		config.applicationRepositoryWithoutUser = "https://" + parts[1]
+	}
+
+	return config, nil
+}

oci/tests/integration/gcp_test.go

@@ -90,3 +90,18 @@ func getWISAAnnotationsGCP(output map[string]*tfjson.StateOutput) (map[string]st
 		gcpIAMAnnotation: saEmail,
 	}, nil
 }
+
+// When implemented, getGitTestConfigGCP would return the git-specific test config for GCP
+func getGitTestConfigGCP(outputs map[string]*tfjson.StateOutput) (*gitTestConfig, error) {
+	return nil, fmt.Errorf("NotImplemented for GCP")
+}
+
+// When implemented, givePermissionsToRepositoryGCP would grant the required permissions to Google cloud source repositories
+func givePermissionsToRepositoryGCP(ctx context.Context, output map[string]*tfjson.StateOutput) (string, error) {
+	return "", fmt.Errorf("NotImplemented for GCP")
+}
+
+// When implemented, revokePermissionsToRepositoryGCP would revoke the permissions granted to Google cloud source repositories
+func revokePermissionsToRepositoryGCP(ctx context.Context, gitPermissionID string, outputs map[string]*tfjson.StateOutput) error {
+	return fmt.Errorf("NotImplemented for AWS")
+}