feat: added Schematic test hooks and outputs (#919) <br> - Added Pre …

…and Post Apply and Destroy hooks to the testschematic options<br> - Added LastTestTerraformOutputs to testschematic options struct and set after each apply step

* fix: schematics upgrade tar file only, remove url options

* refactor: use new common base clone function in  testhelper

* fix: schematic test set upgrade skip

* feat: schematics upgrade test add final apply option

* refactor: secrets baseline update

* feat: add schematics test outputs

* feat: add pre and post hook execution to schematic tests

* fix: add testhelper outputs retrieval for all hooks

* fix: added skip to schematics upgrade if test type is short

testhelper/tests.go

@@ -481,6 +481,17 @@ func (options *TestOptions) RunTestUpgrade() (*terraform.PlanStruct, error) {
 			return nil, resultErr
 		}
 
+		// set outputs after this apply so they are available for hooks
+		var outputErr error
+		// Turn off logging for this step so sensitive data is not logged
+		options.TerraformOptions.Logger = logger.Discard
+		options.LastTestTerraformOutputs, outputErr = terraform.OutputAllE(options.Testing, options.TerraformOptions)
+		options.TerraformOptions.Logger = logger.Default // turn log back on
+
+		if outputErr != nil {
+			logger.Log(options.Testing, "failed to get terraform output: ", outputErr)
+		}
+
 		if options.PostApplyHook != nil {
 			logger.Log(options.Testing, "Running PostApplyHook")
 			hookErr := options.PostApplyHook(options)
@@ -652,6 +663,20 @@ func (options *TestOptions) runTest() (string, error) {
 	assert.Nil(options.Testing, err, "Failed", err)
 	logger.Log(options.Testing, "FINISHED: Init / Apply")
 
+	// set outputs after the apply
+	if err == nil {
+		var outputErr error
+
+		// Turn off logging for this step so sensitive data is not logged
+		options.TerraformOptions.Logger = logger.Discard
+		options.LastTestTerraformOutputs, outputErr = terraform.OutputAllE(options.Testing, options.TerraformOptions)
+		options.TerraformOptions.Logger = logger.Default // turn log back on
+
+		if outputErr != nil {
+			logger.Log(options.Testing, "failed to get terraform output: ", outputErr)
+		}
+	}
+
 	if err == nil && options.PostApplyHook != nil {
 		logger.Log(options.Testing, "Running PostApplyHook")
 		hook_err := options.PostApplyHook(options)

testschematic/mock_test.go

@@ -48,6 +48,7 @@ type schematicServiceMock struct {
 	failPlanWorkspaceCommand     bool
 	failApplyWorkspaceCommand    bool
 	failDestroyWorkspaceCommand  bool
+	failGetOutputsCommand        bool
 	applyComplete                bool
 	destroyComplete              bool
 	workspaceDeleteComplete      bool
@@ -71,6 +72,7 @@ func mockSchematicServiceReset(mock *schematicServiceMock, options *TestSchemati
 	mock.failPlanWorkspaceCommand = false
 	mock.failApplyWorkspaceCommand = false
 	mock.failDestroyWorkspaceCommand = false
+	mock.failGetOutputsCommand = false
 	mock.applyComplete = false
 	mock.destroyComplete = false
 	mock.workspaceDeleteComplete = false
@@ -254,6 +256,23 @@ func (mock *schematicServiceMock) DestroyWorkspaceCommand(destroyWorkspaceComman
 	return result, response, nil
 }
 
+func (mock *schematicServiceMock) GetWorkspaceOutputs(getWorkspaceOutputsOptions *schematics.GetWorkspaceOutputsOptions) ([]schematics.OutputValuesInner, *core.DetailedResponse, error) {
+	if mock.failGetOutputsCommand {
+		return nil, &core.DetailedResponse{StatusCode: 404}, &schematicErrorMock{}
+	}
+
+	result := []schematics.OutputValuesInner{
+		{
+			Folder: core.StringPtr("examples/basic"),
+			OutputValues: []map[string]interface{}{
+				{"mock_output": "the_mock_value"},
+			},
+		},
+	}
+	response := &core.DetailedResponse{StatusCode: 200}
+	return result, response, nil
+}
+
 // IAM AUTHENTIATOR INTERFACE MOCK FUNCTIONS
 func (mock *iamAuthenticatorMock) Authenticate(request *http.Request) error {
 	return nil

testschematic/schematics.go

@@ -6,6 +6,7 @@ import (
 	"archive/tar"
 	"fmt"
 	"io"
+	"maps"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -55,6 +56,7 @@ type SchematicsApiSvcI interface {
 	ApplyWorkspaceCommand(*schematics.ApplyWorkspaceCommandOptions) (*schematics.WorkspaceActivityApplyResult, *core.DetailedResponse, error)
 	DestroyWorkspaceCommand(*schematics.DestroyWorkspaceCommandOptions) (*schematics.WorkspaceActivityDestroyResult, *core.DetailedResponse, error)
 	ReplaceWorkspace(*schematics.ReplaceWorkspaceOptions) (*schematics.WorkspaceResponse, *core.DetailedResponse, error)
+	GetWorkspaceOutputs(*schematics.GetWorkspaceOutputsOptions) ([]schematics.OutputValuesInner, *core.DetailedResponse, error)
 }
 
 // interface for external IBMCloud IAM Authenticator api. Can be mocked for tests
@@ -575,6 +577,43 @@ func (svc *SchematicsTestService) WaitForFinalJobStatus(jobID string) (string, e
 	return status, nil
 }
 
+// GetLatestWorkspaceOutputs will return a map of current terraform outputs stored in the workspace
+func (svc *SchematicsTestService) GetLatestWorkspaceOutputs() (map[string]interface{}, error) {
+
+	var outputResponse []schematics.OutputValuesInner
+	var resp *core.DetailedResponse
+	var err error
+	retries := 0
+	for {
+		outputResponse, resp, err = svc.SchematicsApiSvc.GetWorkspaceOutputs(&schematics.GetWorkspaceOutputsOptions{
+			WID: core.StringPtr(svc.WorkspaceID),
+		})
+		if svc.retryApiCall(err, getDetailedResponseStatusCode(resp), retries) {
+			retries++
+			svc.TestOptions.Testing.Logf("[SCHEMATICS] RETRY GetWorkspaceOutputs, status code: %d", getDetailedResponseStatusCode(resp))
+		} else {
+			break
+		}
+	}
+
+	if err != nil {
+		return make(map[string]interface{}), err
+	}
+
+	// DEV NOTE: the return type from SDK is an array of output wrapper, inside is an array of output maps.
+	// I'm not sure why though, as a schematic workspace would only have one set of outputs?
+	// Through testing I only saw one set of outputs (outputResponse[0].OutputValues[0]),
+	// but just to be safe I'm implementing a loop/merge here, just in case.
+	allOutputs := make(map[string]interface{})
+	for _, outputWrapper := range outputResponse {
+		for _, outputInner := range outputWrapper.OutputValues {
+			maps.Copy(allOutputs, outputInner) // shallow copy into all, from inner (with key merge)
+		}
+	}
+
+	return allOutputs, nil
+}
+
 // DeleteWorkspace will delete the existing workspace created for the test service.
 func (svc *SchematicsTestService) DeleteWorkspace() (string, error) {
 

testschematic/schematics_test.go

@@ -364,6 +364,41 @@ func TestSchematicGetJobDetail(t *testing.T) {
 	})
 }
 
+func TestSchematicGetWorkspaceOutputs(t *testing.T) {
+	zero := 0
+	schematicSvc := new(schematicServiceMock)
+	authSvc := new(iamAuthenticatorMock)
+	svc := &SchematicsTestService{
+		SchematicsApiSvc: schematicSvc,
+		ApiAuthenticator: authSvc,
+		WorkspaceID:      mockWorkspaceID,
+		TemplateID:       mockTemplateID,
+		TestOptions: &TestSchematicOptions{
+			Testing:                      new(testing.T),
+			SchematicSvcRetryCount:       &zero,
+			SchematicSvcRetryWaitSeconds: &zero,
+		},
+	}
+	mockErrorType := new(schematicErrorMock)
+
+	t.Run("OutputsReturned", func(t *testing.T) {
+		result, err := svc.GetLatestWorkspaceOutputs()
+		if assert.NoError(t, err) {
+			if assert.NotNil(t, result) {
+				if assert.Len(t, result, 1) {
+					assert.Equal(t, "the_mock_value", result["mock_output"])
+				}
+			}
+		}
+	})
+
+	t.Run("ServiceError", func(t *testing.T) {
+		schematicSvc.failGetOutputsCommand = true
+		_, err := svc.GetLatestWorkspaceOutputs()
+		assert.ErrorAs(t, err, &mockErrorType)
+	})
+}
+
 func TestSchematicDeleteWorkspace(t *testing.T) {
 	zero := 0
 	schematicSvc := new(schematicServiceMock)

testschematic/test_options.go

@@ -167,6 +167,22 @@ type TestSchematicOptions struct {
 
 	// Set to true if you wish for an Upgrade test to do a final `terraform apply` after the consistency check on the new (not base) branch.
 	CheckApplyResultForUpgrade bool
+
+	// LastTestTerraformOutputs is a map of the last terraform outputs from the last apply of the test.
+	// Note: Plans do not create output. As a side effect of this the upgrade test will have the outputs from the base terraform apply not the upgrade.
+	// Unless the upgrade test is run with the `CheckApplyResultForUpgrade` set to true.
+	LastTestTerraformOutputs map[string]interface{}
+
+	// Hooks These allow us to inject custom code into the test process
+	// example to set a hook:
+	// options.PreApplyHook = func(options *TestSchematicOptions) error {
+	//     // do something
+	//     return nil
+	// }
+	PreApplyHook    func(options *TestSchematicOptions) error // In upgrade tests, this hook will be called before the base apply
+	PostApplyHook   func(options *TestSchematicOptions) error // In upgrade tests, this hook will be called after the base apply
+	PreDestroyHook  func(options *TestSchematicOptions) error // If this fails, the destroy will continue
+	PostDestroyHook func(options *TestSchematicOptions) error
 }
 
 type TestSchematicTerraformVar struct {

testschematic/tests.go

@@ -166,6 +166,20 @@ func executeSchematicTest(options *TestSchematicOptions, performUpgradeTest bool
 	// ------ APPLY ------
 	applySuccess := false // will only flip to true if job completes
 	if !options.Testing.Failed() {
+
+		// PRE-APPLY HOOK
+		if options.PreApplyHook != nil {
+			options.Testing.Log("START: PreApplyHook")
+			preApplyHookErr := options.PreApplyHook(options)
+			if preApplyHookErr != nil {
+				options.Testing.Log("Error running PreApplyHook")
+				options.Testing.Log(preApplyHookErr)
+				options.Testing.Log("END: PreApplyHook")
+			} else {
+				options.Testing.Log("END: PreApplyHook")
+			}
+		}
+
 		applyResponse, applyErr := svc.CreateApplyJob()
 		if assert.NoErrorf(options.Testing, applyErr, "error creating APPLY - %s", svc.WorkspaceNameForLog) {
 
@@ -182,6 +196,27 @@ func executeSchematicTest(options *TestSchematicOptions, performUpgradeTest bool
 				if printApplyLogErr != nil {
 					options.Testing.Logf("Error printing APPLY logs:%s", printApplyLogErr)
 				}
+			} else {
+				// retrieve and store the last set of outputs in case post-commit hook needs it
+				outputs, outputsErr := svc.GetLatestWorkspaceOutputs()
+				if outputsErr != nil {
+					options.Testing.Logf("[SCHEMATICS] There was an error retrieving output values: %s", outputsErr)
+				} else {
+					options.LastTestTerraformOutputs = outputs
+				}
+
+				// POST-APPLY HOOK
+				if options.PostApplyHook != nil {
+					options.Testing.Log("START: PostApplyHook")
+					postApplyHookErr := options.PostApplyHook(options)
+					if postApplyHookErr != nil {
+						options.Testing.Log("Error running PostApplyHook")
+						options.Testing.Log(postApplyHookErr)
+						options.Testing.Log("END: PostApplyHook")
+					} else {
+						options.Testing.Log("END: PostApplyHook")
+					}
+				}
 			}
 		}
 	}
@@ -352,8 +387,29 @@ func testTearDown(svc *SchematicsTestService, options *TestSchematicOptions) {
 		}
 	}()
 
+	// retrieve and store the last set of outputs right before destroy
+	outputs, outputsErr := svc.GetLatestWorkspaceOutputs()
+	if outputsErr != nil {
+		options.Testing.Logf("[SCHEMATICS] There was an error retrieving output values: %s", outputsErr)
+	} else {
+		options.LastTestTerraformOutputs = outputs
+	}
+
 	// only perform if skip is not set
 	if !options.SkipTestTearDown {
+		// PRE-DESTROY HOOK
+		if options.PreDestroyHook != nil {
+			options.Testing.Log("START: PreDestroyHook")
+			preHookErr := options.PreDestroyHook(options)
+			if preHookErr != nil {
+				options.Testing.Log("Error running PreDestroyHook")
+				options.Testing.Log(preHookErr)
+				options.Testing.Log("END: PreDestroyHook, continuing with destroy")
+			} else {
+				options.Testing.Log("END: PreDestroyHook")
+			}
+		}
+
 		// ------ DESTROY RESOURCES ------
 		// only run destroy if we had potentially created resources
 		if svc.TerraformResourcesCreated {
@@ -404,6 +460,19 @@ func testTearDown(svc *SchematicsTestService, options *TestSchematicOptions) {
 			}
 		}
 
+		// POST-DESTROY HOOK
+		if options.PostDestroyHook != nil {
+			options.Testing.Log("START: PostDestroyHook")
+			postHookErr := options.PostDestroyHook(options)
+			if postHookErr != nil {
+				options.Testing.Log("Error running PostDestroyHook")
+				options.Testing.Log(postHookErr)
+				options.Testing.Log("END: PostDestroyHook")
+			} else {
+				options.Testing.Log("END: PostDestroyHook")
+			}
+		}
+
 		// clean up any temp directories that were created
 		if len(svc.BaseTerraformTempDir) > 0 {
 			options.Testing.Logf("[SCHEMATICS] Removing temp directory for upgrade test: %s", svc.BaseTerraformTempDir)