Add new integration tests for Azure OIDC for git repositories

Signed-off-by: Dipti Pai <[email protected]>

oci/tests/integration/.env.sample

@@ -11,12 +11,14 @@
 # export TF_VAR_rand=${RANDOM}
 
 ## Azure
+# export ARM_SUBSCRIPTION_ID=
+# export TF_VAR_azuredevops_org=
+# export TF_VAR_azuredevops_pat=
 # export TF_VAR_azure_location=eastus
 ## Set the following only when authenticating using Service Principal (suited
 ## for CI environment).
 # export ARM_CLIENT_ID=
 # export ARM_CLIENT_SECRET=
-# export ARM_SUBSCRIPTION_ID=
 # export ARM_TENANT_ID=
 
 ## GCP
@@ -48,3 +50,7 @@
 # export TF_VAR_wi_k8s_sa_name=test-workload-id
 # export TF_VAR_wi_k8s_sa_ns=default
 # export TF_VAR_enable_wi=true
+
+## Test Configuration variables
+# export TF_VAR_enable_git=true
+# export TF_VAR_enable_oci=true

oci/tests/integration/Makefile

@@ -1,4 +1,5 @@
 GO_TEST_ARGS ?=
+GO_TEST_PREFIX ?=
 PROVIDER_ARG ?=
 TEST_TIMEOUT ?= 30m
 GOARCH ?= amd64
@@ -15,14 +16,20 @@ docker-build: app
 
 test:
 	docker image inspect $(TEST_IMG) >/dev/null
-	TEST_IMG=$(TEST_IMG) go test -timeout $(TEST_TIMEOUT) -v ./ $(GO_TEST_ARGS) $(PROVIDER_ARG) --tags=integration
+	TEST_IMG=$(TEST_IMG) go test -timeout $(TEST_TIMEOUT) -v ./ -run "^$(GO_TEST_PREFIX).*" $(GO_TEST_ARGS) $(PROVIDER_ARG) --tags=integration
 
 test-aws:
 	$(MAKE) test PROVIDER_ARG="-provider aws"
 
 test-azure:
 	$(MAKE) test PROVIDER_ARG="-provider azure"
 
+test-azure-git:
+	$(MAKE) test PROVIDER_ARG="-provider azure" GO_TEST_PREFIX="TestGit"
+
+test-azure-oci:
+	$(MAKE) test PROVIDER_ARG="-provider azure" GO_TEST_PREFIX="TestOci"
+
 test-gcp:
 	$(MAKE) test PROVIDER_ARG="-provider gcp"
 

oci/tests/integration/README.md

@@ -1,7 +1,7 @@
-# OCI integration test
+# Integration tests
 
-OCI integration test uses a test application(`testapp/`) to test the
-oci package against each of the supported cloud providers.
+Integration tests uses a test application(`testapp/`) to test the
+oci and git package against each of the supported cloud providers.
 
 **NOTE:** Tests in this package aren't run automatically by the `test-*` make
 target at the root of `fluxcd/pkg` repo. These tests are more complicated than
@@ -16,7 +16,7 @@ runs the test app as a batch job which tries to log in and list tags from the
 test registry repository. A successful job indicates successful test. If the job
 fails, the test fails.
 
-Logs of a successful job run:
+Logs of a successful job run for oci:
 ```console
 $ kubectl logs test-job-93tbl-4jp2r
 2022/07/28 21:59:06 repo: xxx.dkr.ecr.us-east-2.amazonaws.com/test-repo-flux-test-heroic-ram
@@ -25,6 +25,17 @@ $ kubectl logs test-job-93tbl-4jp2r
 2022/07/28 21:59:06 tags: [v0.1.4 v0.1.3 v0.1.0 v0.1.2]
 ```
 
+Logs of a successful job run for git:
+```console
+$ kubectl logs test-job-dzful-jrcqw
+2024/07/19 19:04:06 Validating git oidc by cloning repo  https://dev.azure.com/xxx/fluxProjtoughowerewolf/_git/fluxRepotoughowerewolf
+2024/07/19 19:04:07 Successfully cloned repository
+2024/07/19 19:04:07 apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: foobar
+```
+
 ## Requirements
 
 ### Amazon Web Services
@@ -42,6 +53,12 @@ $ kubectl logs test-job-93tbl-4jp2r
     workloads to access ACR.
 - Azure CLI, need to be logged in using `az login` as a User (not a Service
   Principal).
+- An Azure DevOps organization, personal access token for accessing repositories within the organization. The scope required for the personal access token is:
+  - Project and Team - read, write and manage access
+  - Member Entitlement Management (Read & Write)
+  - Code - Full
+  - Please take a look at the [terraform provider](https://registry.terraform.io/providers/microsoft/azuredevops/latest/docs/guides/authenticating_using_the_personal_access_token#create-a-personal-access-token) for more explanation.
+  - A valid Azure devops configuration is needed even if git is not being tested. 
 
   **NOTE:** To use Service Principal (for example in CI environment), set the
   `ARM-*` variables in `.env`, source it and authenticate Azure CLI with:
@@ -246,9 +263,10 @@ Run the test with `make test-*`, setting the test app image with variable
 $ make test-azure
 make test PROVIDER_ARG="-provider azure"
 docker image inspect fluxcd/testapp:test >/dev/null
-TEST_IMG=fluxcd/testapp:test go test -timeout 30m -v ./ -verbose -retain -provider azure --tags=integration
-2022/07/29 02:06:51 Terraform binary:  /usr/bin/terraform
-2022/07/29 02:06:51 Init Terraform
+TEST_IMG=fluxcd/testapp:test go test -timeout 30m -v ./ -run "^.*" -provider azure --tags=integration
+2024/08/26 23:39:13 Terraform binary:  /snap/bin/terraform
+2024/08/26 23:39:13 Init Terraform
+2024/08/26 23:39:15 Applying Terraform
 ...
 ```
 
@@ -273,6 +291,8 @@ export TF_VAR_enable_wi=
 
 They have been included in the `.env.sample` and you can simply uncomment it.
 
+The git integration tests require workload identity to be enabled.
+
 ## Debugging the tests
 
 For debugging environment provisioning, enable verbose output with `-verbose`

oci/tests/integration/aws_test.go

@@ -98,3 +98,13 @@ func getWISAAnnotationsAWS(output map[string]*tfjson.StateOutput) (map[string]st
 		eksRoleArnAnnotation: iamARN,
 	}, nil
 }
+
+// When implemented, getGitTestConfigAws would return the git-specific test config for AWS
+func getGitTestConfigAWS(outputs map[string]*tfjson.StateOutput) (*gitTestConfig, error) {
+	return nil, fmt.Errorf("NotImplemented for AWS")
+}
+
+// When implemented, givePermissionsToRepositoryAWS would grant the required permissions to AWS CodeCommit repository
+func givePermissionsToRepositoryAWS(output map[string]*tfjson.StateOutput) error {
+	return fmt.Errorf("NotImplemented for AWS")
+}

oci/tests/integration/azure_test.go

@@ -20,12 +20,20 @@ limitations under the License.
 package integration
 
 import (
+	"bytes"
 	"context"
+	"encoding/base64"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"strings"
+	"time"
 
-	tfjson "github.com/hashicorp/terraform-json"
-
+	"github.com/fluxcd/pkg/git"
 	"github.com/fluxcd/test-infra/tftestenv"
+	tfjson "github.com/hashicorp/terraform-json"
 )
 
 const (
@@ -81,3 +89,154 @@ func getWISAAnnotationsAzure(output map[string]*tfjson.StateOutput) (map[string]
 		azureWIClientIdAnnotation: clientID,
 	}, nil
 }
+
+// Give managed identity permissions on the azure devops project using
+// ServicePrincipalEntitlement REST API
+// https://learn.microsoft.com/en-us/rest/api/azure/devops/memberentitlementmanagement/service-principal-entitlements/add?view=azure-devops-rest-7.1&tabs=HTTP
+// This can be moved to terraform if/when this PR completes -
+// https://github.com/microsoft/terraform-provider-azuredevops/pull/1028
+type ServicePrincipalEntitlement struct {
+	AccessLevel struct {
+		AccountLicenseType string `json:"accountLicenseType"`
+	} `json:"accessLevel"`
+	ProjectEntitlements []struct {
+		Group struct {
+			GroupType string `json:"groupType"`
+		} `json:"group"`
+		ProjectRef struct {
+			ID string `json:"id"`
+		} `json:"projectRef"`
+	} `json:"projectEntitlements"`
+	ServicePrincipal struct {
+		Origin      string `json:"origin"`
+		OriginID    string `json:"originId"`
+		SubjectKind string `json:"subjectKind"`
+	} `json:"servicePrincipal"`
+}
+
+func givePermissionsToRepositoryAzure(outputs map[string]*tfjson.StateOutput) error {
+	// Organization, PAT, Project ID and WI ID are availble as terraform output
+	organization := outputs["azure_devops_organization"].Value.(string)
+	project_id := outputs["azure_devops_project_id"].Value.(string)
+	pat := outputs["azure_devops_access_token"].Value.(string)
+	wi_object_id := outputs["workload_identity_object_id"].Value.(string)
+
+	encodedPat := base64.StdEncoding.EncodeToString([]byte(":" + pat))
+	apiURL := fmt.Sprintf("https://vsaex.dev.azure.com/%s/_apis/serviceprincipalentitlements?api-version=7.1-preview.1", organization)
+
+	// Set up the request payload
+	payload := ServicePrincipalEntitlement{
+		AccessLevel: struct {
+			AccountLicenseType string `json:"accountLicenseType"`
+		}{
+			AccountLicenseType: "express",
+		},
+		ProjectEntitlements: []struct {
+			Group struct {
+				GroupType string `json:"groupType"`
+			} `json:"group"`
+			ProjectRef struct {
+				ID string `json:"id"`
+			} `json:"projectRef"`
+		}{
+			{
+				Group: struct {
+					GroupType string `json:"groupType"`
+				}{
+					GroupType: "projectContributor",
+				},
+				ProjectRef: struct {
+					ID string `json:"id"`
+				}{
+					ID: project_id,
+				},
+			},
+		},
+		ServicePrincipal: struct {
+			Origin      string `json:"origin"`
+			OriginID    string `json:"originId"`
+			SubjectKind string `json:"subjectKind"`
+		}{
+			Origin:      "aad",
+			OriginID:    wi_object_id,
+			SubjectKind: "servicePrincipal",
+		},
+	}
+
+	// Marshal the payload into JSON
+	jsonPayload, err := json.Marshal(payload)
+	if err != nil {
+		log.Printf("Error marshalling the payload:%v", err)
+		return err
+	}
+
+	// First request to add user always fails, second request succeeds, add a
+	// retry
+	retryAttempts := 3
+	retryDelay := 5 * time.Second // 5 seconds delay
+	attempts := 0
+
+	for attempts < retryAttempts {
+		attempts++
+		// Create a new request
+		req, err := http.NewRequest("POST", apiURL, bytes.NewBuffer(jsonPayload))
+		if err != nil {
+			log.Printf("Error creating the request: %v", err)
+			return err
+		}
+
+		// Set the authorization header to use the PAT
+		req.Header.Set("Authorization", "Basic "+strings.TrimSpace(encodedPat))
+		req.Header.Set("Content-Type", "application/json")
+
+		// Send the request
+		client := &http.Client{}
+		resp, err := client.Do(req)
+		if err != nil {
+			log.Printf("Error sending the request: %v", err)
+			return err
+		}
+		defer resp.Body.Close()
+
+		// Read the response body
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil || strings.Contains(string(body), "VS403283: Could not add user") {
+			log.Printf("Encountered error : %v, retrying..", err)
+			time.Sleep(retryDelay)
+			continue
+		}
+
+		log.Printf("Added managed identity to organization:")
+		break
+	}
+	return nil
+}
+
+// getGitTestConfigAzure returns the test config used to setup the git repository
+func getGitTestConfigAzure(outputs map[string]*tfjson.StateOutput) (*gitTestConfig, error) {
+	config := &gitTestConfig{
+		defaultGitTransport:   git.HTTP,
+		gitUsername:           git.DefaultPublicKeyAuthUser,
+		gitPat:                outputs["azure_devops_access_token"].Value.(string),
+		applicationRepository: outputs["git_repo_url"].Value.(string),
+	}
+
+	opts, err := getAuthOpts(config.applicationRepository, map[string][]byte{
+		"password": []byte(config.gitPat),
+		"username": []byte(git.DefaultPublicKeyAuthUser),
+	})
+	if err != nil {
+		return nil, err
+	}
+	config.defaultAuthOpts = opts
+
+	parts := strings.Split(config.applicationRepository, "@")
+	// Check if the URL contains the "@" symbol
+	if len(parts) > 1 {
+		// Reconstruct the URL without the username
+		config.applicationRepositoryWithoutUser = "https://" + parts[1]
+	}
+
+	fmt.Println("URL without username:", config.applicationRepositoryWithoutUser)
+	return config, nil
+}

oci/tests/integration/gcp_test.go

@@ -90,3 +90,13 @@ func getWISAAnnotationsGCP(output map[string]*tfjson.StateOutput) (map[string]st
 		gcpIAMAnnotation: saEmail,
 	}, nil
 }
+
+// When implemented, getGitTestConfigGCP would return the git-specific test config for GCP
+func getGitTestConfigGCP(outputs map[string]*tfjson.StateOutput) (*gitTestConfig, error) {
+	return nil, fmt.Errorf("NotImplemented for GCP")
+}
+
+// When implemented, givePermissionsToRepositoryGCP would grant the required permissions to Google cloud source repositories
+func givePermissionsToRepositoryGCP(output map[string]*tfjson.StateOutput) error {
+	return fmt.Errorf("NotImplemented for GCP")
+}

oci/tests/integration/git_test.go

@@ -0,0 +1,46 @@
+//go:build integration
+// +build integration
+
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"context"
+	"fmt"
+	"testing"
+)
+
+func TestGitCloneUsingProvider(t *testing.T) {
+	if !enableGit {
+		t.Skip("Skipping test, enable git in env, supported providers ", supportedGitProviders)
+	}
+
+	ctx := context.TODO()
+	tmpDir := t.TempDir()
+
+	setupGitRepository(ctx, tmpDir)
+	t.Run("Git oidc credential test", func(t *testing.T) {
+		args := []string{
+			"-category=git",
+			"-oidc-login=true",
+			fmt.Sprintf("-provider=%s", *targetProvider),
+			fmt.Sprintf("-repo=%s", testGitCfg.applicationRepositoryWithoutUser),
+		}
+		testjobExecutionWithArgs(t, args)
+	})
+}