Skip to content

Commit

Permalink
fix: schematics upgrade refactor to tar file only (#916)
Browse files Browse the repository at this point in the history
  • Loading branch information
@toddgiguere
toddgiguere authored Feb 6, 2025
1 parent 8c35ed6 commit 3f7f7e3
Show file tree
Hide file tree
Showing 6 changed files with 189 additions and 192 deletions.
38 changes: 18 additions & 20 deletions .secrets.baseline
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"files": "go.sum|package-lock.json|^.secrets.baseline$",
"lines": null
},
"generated_at": "2025-01-24T16:21:04Z",
"generated_at": "2025-01-29T15:25:24Z",
"plugins_used": [
{
"name": "AWSKeyDetector"
Expand Down Expand Up @@ -164,7 +164,23 @@
"hashed_secret": "bff8f8143a073833d713e3c1821fe97661bc3cef",
"is_secret": false,
"is_verified": false,
"line_number": 318,
"line_number": 320,
"type": "Secret Keyword",
"verified_result": null
},
{
"hashed_secret": "b4e929aa58c928e3e44d12e6f873f39cd8207a25",
"is_secret": false,
"is_verified": false,
"line_number": 465,
"type": "Secret Keyword",
"verified_result": null
},
{
"hashed_secret": "b994b23302ebc7b46888b0f5c623bfc2bcfa2e3f",
"is_secret": false,
"is_verified": false,
"line_number": 478,
"type": "Secret Keyword",
"verified_result": null
}
Expand Down Expand Up @@ -210,24 +226,6 @@
"type": "Secret Keyword",
"verified_result": null
}
],
"testhelper/tests.go": [
{
"hashed_secret": "b4e929aa58c928e3e44d12e6f873f39cd8207a25",
"is_secret": false,
"is_verified": false,
"line_number": 487,
"type": "Secret Keyword",
"verified_result": null
},
{
"hashed_secret": "16282376ddaaaf2bf60be9041a7504280f3f338b",
"is_secret": false,
"is_verified": false,
"line_number": 500,
"type": "Secret Keyword",
"verified_result": null
}
]
},
"version": "0.13.1+ibm.62.dss",
Expand Down
67 changes: 67 additions & 0 deletions common/git.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import (
"strings"
"testing"

"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/transport"
"github.com/go-git/go-git/v5/plumbing/transport/http"
gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh"
Expand Down Expand Up @@ -427,3 +429,68 @@ func SkipUpgradeTest(testing *testing.T, source_repo string, source_branch strin

return doNotRunUpgradeTest
}

func CloneAndCheckoutBranch(testing *testing.T, repoURL string, branch string, cloneDir string) error {

authMethod, authErr := DetermineAuthMethod(repoURL)
if authErr != nil {
logger.Log(testing, "Failed to determine authentication method, trying without authentication...")

// Convert SSH URL to HTTPS URL
if strings.HasPrefix(repoURL, "git@") {
repoURL = strings.Replace(repoURL, ":", "/", 1)
repoURL = strings.Replace(repoURL, "git@", "https://", 1)
repoURL = strings.TrimSuffix(repoURL, ".git") + ".git"
}

// Try to clone without authentication
_, errUnauth := git.PlainClone(cloneDir, false, &git.CloneOptions{
URL: repoURL,
ReferenceName: plumbing.NewBranchReferenceName(branch),
SingleBranch: true,
})

if errUnauth != nil {
// If unauthenticated clone fails and we cannot determine authentication, return the error from the unauthenticated approach
return fmt.Errorf("failed to determine authentication method and clone base repo and branch without authentication: %v", errUnauth)
} else {
logger.Log(testing, "Cloned base repo and branch without authentication")
}
} else {
// Authentication method determined, try with authentication
_, errAuth := git.PlainClone(cloneDir, false, &git.CloneOptions{
URL: repoURL,
ReferenceName: plumbing.NewBranchReferenceName(branch),
SingleBranch: true,
Auth: authMethod,
})

if errAuth != nil {
logger.Log(testing, "Failed to clone base repo and branch with authentication, trying without authentication...")
// Convert SSH URL to HTTPS URL
if strings.HasPrefix(repoURL, "git@") {
repoURL = strings.Replace(repoURL, ":", "/", 1)
repoURL = strings.Replace(repoURL, "git@", "https://", 1)
repoURL = strings.TrimSuffix(repoURL, ".git") + ".git"
}

// Try to clone without authentication
_, errUnauth := git.PlainClone(cloneDir, false, &git.CloneOptions{
URL: repoURL,
ReferenceName: plumbing.NewBranchReferenceName(branch),
SingleBranch: true,
})

if errUnauth != nil {
// If unauthenticated clone also fails, return the error from the authenticated approach
return fmt.Errorf("failed to clone base repo and branch with authentication: %v", errAuth)
} else {
logger.Log(testing, "Cloned base repo and branch without authentication")
}
} else {
logger.Log(testing, "Cloned base repo and branch with authentication")
}
}

return nil
}
63 changes: 3 additions & 60 deletions testhelper/tests.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,8 @@ import (

"github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/cloudinfo"

"github.com/go-git/go-git/v5/plumbing"
"github.com/gruntwork-io/terratest/modules/files"

"github.com/go-git/go-git/v5"
"github.com/gruntwork-io/terratest/modules/logger"
"github.com/gruntwork-io/terratest/modules/terraform"
"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -454,64 +452,9 @@ func (options *TestOptions) RunTestUpgrade() (*terraform.PlanStruct, error) {
}
}

authMethod, err := common.DetermineAuthMethod(baseRepo)
if err != nil {
logger.Log(options.Testing, "Failed to determine authentication method, trying without authentication...")

// Convert SSH URL to HTTPS URL
if strings.HasPrefix(baseRepo, "git@") {
baseRepo = strings.Replace(baseRepo, ":", "/", 1)
baseRepo = strings.Replace(baseRepo, "git@", "https://", 1)
baseRepo = strings.TrimSuffix(baseRepo, ".git") + ".git"
}

// Try to clone without authentication
_, errUnauth := git.PlainClone(baseTempDir, false, &git.CloneOptions{
URL: baseRepo,
ReferenceName: plumbing.NewBranchReferenceName(baseBranch),
SingleBranch: true,
})

if errUnauth != nil {
// If unauthenticated clone fails and we cannot determine authentication, return the error from the unauthenticated approach
return nil, fmt.Errorf("failed to determine authentication method and clone base repo and branch without authentication: %v", errUnauth)
} else {
logger.Log(options.Testing, "Cloned base repo and branch without authentication")
}
} else {
// Authentication method determined, try with authentication
_, errAuth := git.PlainClone(baseTempDir, false, &git.CloneOptions{
URL: baseRepo,
ReferenceName: plumbing.NewBranchReferenceName(baseBranch),
SingleBranch: true,
Auth: authMethod,
})

if errAuth != nil {
logger.Log(options.Testing, "Failed to clone base repo and branch with authentication, trying without authentication...")
// Convert SSH URL to HTTPS URL
if strings.HasPrefix(baseRepo, "git@") {
baseRepo = strings.Replace(baseRepo, ":", "/", 1)
baseRepo = strings.Replace(baseRepo, "git@", "https://", 1)
baseRepo = strings.TrimSuffix(baseRepo, ".git") + ".git"
}

// Try to clone without authentication
_, errUnauth := git.PlainClone(baseTempDir, false, &git.CloneOptions{
URL: baseRepo,
ReferenceName: plumbing.NewBranchReferenceName(baseBranch),
SingleBranch: true,
})

if errUnauth != nil {
// If unauthenticated clone also fails, return the error from the authenticated approach
return nil, fmt.Errorf("failed to clone base repo and branch with authentication: %v", errAuth)
} else {
logger.Log(options.Testing, "Cloned base repo and branch without authentication")
}
} else {
logger.Log(options.Testing, "Cloned base repo and branch with authentication")
}
cloneBaseErr := common.CloneAndCheckoutBranch(options.Testing, baseRepo, baseBranch, baseTempDir)
if cloneBaseErr != nil {
return nil, cloneBaseErr
}

// Set TerraformDir to the appropriate directory within baseTempDir
Expand Down
69 changes: 1 addition & 68 deletions testschematic/schematics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ type SchematicsTestService struct {
BaseTerraformRepoBranch string // the branch name of the main origin branch of the project (main or master), used for upgrade test
TestTerraformRepo string // the URL of the repo for the pull request, will be either origin or a fork
TestTerraformRepoBranch string // the branch of the test, usually the current checked out branch of the test run
BaseTerraformTempDir string // if upgrade test, will contain the temp directory containing clone of base repo
}

// CreateAuthenticator will accept a valid IBM cloud API key, and
Expand Down Expand Up @@ -258,74 +259,6 @@ func (svc *SchematicsTestService) CreateUploadTarFile(projectPath string) (strin
return tarballName, nil
}

// SetTemplateRepoToBranch will call UpdateTestTemplateRepo with the appropriate URLs set at the service level
// for the branch of this test
func (svc *SchematicsTestService) SetTemplateRepoToBranch() error {
return svc.UpdateTestTemplateRepo(svc.TestTerraformRepo, svc.TestTerraformRepoBranch, svc.TestOptions.TemplateGitToken)
}

// SetTemplateRepoToBase will call UpdateTestTemplateRepo with the appropriate URLs set at the service level
// for the base (master or main) branch of this test, used in upgrade tests
func (svc *SchematicsTestService) SetTemplateRepoToBase() error {
return svc.UpdateTestTemplateRepo(svc.BaseTerraformRepo, svc.BaseTerraformRepoBranch, svc.TestOptions.TemplateGitToken)
}

// UpdateTestTemplateRepo will replace the workspace repository with a given github URL and branch
func (svc *SchematicsTestService) UpdateTestTemplateRepo(url string, branch string, token string) error {
svc.TestOptions.Testing.Logf("[SCHEMATICS] Setting template repository to repo %s (%s)", url, branch)
// set up repo update request
repoUpdateRequest := &schematics.TemplateRepoUpdateRequest{
URL: core.StringPtr(url),
Branch: core.StringPtr(branch),
}

// set up overall workspace options
replaceOptions := &schematics.ReplaceWorkspaceOptions{
WID: core.StringPtr(svc.WorkspaceID),
TemplateRepo: repoUpdateRequest,
}

// if supplied set a token for private repo
if len(token) > 0 {
replaceOptions.SetXGithubToken(token)
}

// now update template
var resp *core.DetailedResponse
var updateErr error
retries := 0
for {
_, resp, updateErr = svc.SchematicsApiSvc.ReplaceWorkspace(replaceOptions)
if svc.retryApiCall(updateErr, getDetailedResponseStatusCode(resp), retries) {
retries++
svc.TestOptions.Testing.Logf("[SCHEMATICS] RETRY ReplaceWorkspace, status code: %d", getDetailedResponseStatusCode(resp))
} else {
break
}
}
if updateErr != nil {
return updateErr
}

// -------- SETTING UP WORKSPACE WITH REPO ----------
// find the repository refresh job
replaceJob, replaceJobErr := svc.FindLatestWorkspaceJobByName(SchematicsJobTypeUpdate)
if replaceJobErr != nil {
return fmt.Errorf("error finding the update workspace action: %w - %s", replaceJobErr, svc.WorkspaceNameForLog)
}
// wait for it to finish
replaceJobStatus, replaceJobStatusErr := svc.WaitForFinalJobStatus(*replaceJob.ActionID)
if replaceJobStatusErr != nil {
return fmt.Errorf("error waiting for update of workspace to finish: %w - %s", replaceJobStatusErr, svc.WorkspaceNameForLog)
}
// check if complete
if replaceJobStatus != SchematicsJobStatusCompleted {
return fmt.Errorf("workspace update has failed with status %s - %s", replaceJobStatus, svc.WorkspaceNameForLog)
}

return nil
}

// UpdateTestTemplateVars will update an existing Schematics Workspace terraform template with a
// Variablestore, which will set terraform input variables for test runs.
func (svc *SchematicsTestService) UpdateTestTemplateVars(vars []TestSchematicTerraformVar) error {
Expand Down
14 changes: 6 additions & 8 deletions testschematic/test_options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,12 +80,6 @@ type TestSchematicOptions struct {
// to that value.
TemplateFolder string

// The Git auth token used by schematics to clone initial template source of project.
// If TAR upload option is not used, the default will be to use the Git URL of the test branch for initial schematics workload setup.
// If the test repo is private or protected, set this value to a Git token that is authorized to read/clone the repository.
// NOTE: you may also need to set the same token in a `NetrcSettings` as well for this scenario.
TemplateGitToken string

// Optional list of tags that will be applied to the Schematics Workspace instance
Tags []string

Expand Down Expand Up @@ -165,10 +159,14 @@ type TestSchematicOptions struct {
// Set this value to `true` to have all schematics job logs (plan/apply/destroy) printed to the test log.
PrintAllSchematicsLogs bool

// This property will be set to true by the test when an upgrade test was performed.
// These properties will be set to true by the test when an upgrade test was performed.
// You can then inspect this value after the test run, if needed, to make further code decisions.
// NOTE: this is not an option field that is meant to be set from a unit test, it is informational only
IsUpgradeTest bool
IsUpgradeTest bool
UpgradeTestSkipped bool // Informs the calling test that conditions were met to skip the upgrade test

// Set to true if you wish for an Upgrade test to do a final `terraform apply` after the consistency check on the new (not base) branch.
CheckApplyResultForUpgrade bool
}

type TestSchematicTerraformVar struct {
Expand Down
Loading

0 comments on commit 3f7f7e3

Please sign in to comment.