Add cleanup validation

commands/cleanup.go

@@ -5,11 +5,15 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/mitchellh/cli"
+	"github.com/ms-henglu/armstrong/report"
 	"github.com/ms-henglu/armstrong/tf"
+	"github.com/ms-henglu/armstrong/types"
 )
 
 type CleanupCommand struct {
@@ -48,6 +52,11 @@ func (c CleanupCommand) Run(args []string) int {
 }
 
 func (c CleanupCommand) Execute() int {
+	const (
+		allPassedReportFileName     = "cleanup_all_passed_report.md"
+		partialPassedReportFileName = "cleanup_partial_passed_report.md"
+	)
+
 	log.Println("[INFO] ----------- cleanup resources ---------")
 	wd, err := os.Getwd()
 	if err != nil {
@@ -65,13 +74,99 @@ func (c CleanupCommand) Execute() int {
 	if err != nil {
 		log.Fatalf("[ERROR] error creating terraform executable: %+v\n", err)
 	}
+
+	state, err := terraform.Show()
+	if err != nil {
+		log.Fatalf("[ERROR] error getting state: %+v\n", err)
+	}
+
+	passReport := tf.NewPassReportFromState(state)
+	idAddressMap := tf.NewIdAdressFromState(state)
+
+	reportDir := fmt.Sprintf("armstrong_cleanup_reports_%s", time.Now().Format(time.Stamp))
+	reportDir = strings.ReplaceAll(reportDir, ":", "")
+	reportDir = strings.ReplaceAll(reportDir, " ", "_")
+	reportDir = path.Join(wd, reportDir)
+	err = os.Mkdir(reportDir, 0755)
+	if err != nil {
+		log.Fatalf("[ERROR] error creating report dir %s: %+v", reportDir, err)
+	}
+
 	log.Println("[INFO] prepare working directory")
 	_ = terraform.Init()
 	log.Println("[INFO] running destroy command to cleanup resources...")
-	err = terraform.Destroy()
+	destroyErr := terraform.Destroy()
+	if destroyErr != nil {
+		log.Printf("[ERROR] error cleaning up resources: %+v\n", destroyErr)
+	} else {
+		log.Println("[INFO] all resources are cleaned up")
+		storeCleanupReport(passReport, reportDir, allPassedReportFileName)
+	}
+
+	logs, err := report.ParseLogs(path.Join(wd, "log.txt"))
 	if err != nil {
-		log.Fatalf("[ERROR] error cleaning up resources: %+v\n", err)
+		log.Printf("[ERROR] parsing log.txt: %+v", err)
+	}
+
+	errorReport := types.ErrorReport{}
+	if destroyErr != nil {
+		errorReport := tf.NewCleanupErrorReport(destroyErr, logs)
+		for i := range errorReport.Errors {
+			if address, ok := idAddressMap[errorReport.Errors[i].Id]; ok {
+				errorReport.Errors[i].Label = address
+			}
+		}
+		storeCleanupErrorReport(errorReport, reportDir)
 	}
-	log.Println("[INFO] all resources have been deleted")
+
+	resources := make([]types.Resource, 0)
+	if state, err := terraform.Show(); err == nil && state != nil && state.Values != nil && state.Values.RootModule != nil && state.Values.RootModule.Resources != nil {
+		for _, passRes := range passReport.Resources {
+			isDeleted := true
+			for _, res := range state.Values.RootModule.Resources {
+				if passRes.Address == res.Address {
+					isDeleted = false
+					break
+				}
+			}
+			if isDeleted {
+				resources = append(resources, passRes)
+			}
+		}
+	}
+
+	passReport.Resources = resources
+	storeCleanupReport(passReport, reportDir, partialPassedReportFileName)
+
+	log.Println("[INFO] ---------------- Summary ----------------")
+	log.Printf("[INFO] %d resources passed the cleanup tests.", len(passReport.Resources))
+	if len(errorReport.Errors) != 0 {
+		log.Printf("[INFO] %d errors when cleanup the testing resources.", len(errorReport.Errors))
+	}
+
 	return 0
 }
+
+func storeCleanupReport(passReport types.PassReport, reportDir string, reportName string) {
+	if len(passReport.Resources) != 0 {
+		err := os.WriteFile(path.Join(reportDir, reportName), []byte(report.CleanupMarkdownReport(passReport)), 0644)
+		if err != nil {
+			log.Printf("[WARN] failed to save passed markdown report to %s: %+v", reportName, err)
+		} else {
+			log.Printf("[INFO] markdown report saved to %s", reportName)
+		}
+	}
+}
+
+func storeCleanupErrorReport(errorReport types.ErrorReport, reportDir string) {
+	for _, r := range errorReport.Errors {
+		log.Printf("[WARN] found an error when deleting %s, address: %s\n", r.Type, r.Label)
+		markdownFilename := fmt.Sprintf("%s_%s.md", strings.ReplaceAll(r.Type, "/", "_"), r.Label)
+		err := os.WriteFile(path.Join(reportDir, markdownFilename), []byte(report.CleanupErrorMarkdownReport(r, errorReport.Logs)), 0644)
+		if err != nil {
+			log.Printf("[WARN] failed to save markdown report to %s: %+v", markdownFilename, err)
+		} else {
+			log.Printf("[INFO] markdown report saved to %s", markdownFilename)
+		}
+	}
+}

commands/test.go

@@ -187,7 +187,7 @@ func storePassReport(passReport types.PassReport, coverageReport coverage.Covera
 
 func storeErrorReport(errorReport types.ErrorReport, reportDir string) {
 	for _, r := range errorReport.Errors {
-		log.Printf("[WARN] found an error when create %s, address: azapi_resource.%s\n", r.Type, r.Label)
+		log.Printf("[WARN] found an error when creating %s, address: azapi_resource.%s\n", r.Type, r.Label)
 		markdownFilename := fmt.Sprintf("%s_%s.md", strings.ReplaceAll(r.Type, "/", "_"), r.Label)
 		err := os.WriteFile(path.Join(reportDir, markdownFilename), []byte(report.ErrorMarkdownReport(r, errorReport.Logs)), 0644)
 		if err != nil {

report/cleanup_error_report.go

@@ -0,0 +1,67 @@
+package report
+
+import (
+	_ "embed"
+	"strings"
+
+	"github.com/ms-henglu/armstrong/types"
+)
+
+//go:embed cleanup_error_report.md
+var cleanupErrorReportTemplate string
+
+func CleanupErrorMarkdownReport(report types.Error, logs []types.RequestTrace) string {
+	parts := strings.Split(report.Type, "@")
+	resourceType := ""
+	apiVersion := ""
+	if len(parts) == 2 {
+		resourceType = parts[0]
+		apiVersion = parts[1]
+	}
+	requestTraces := CleanupAllRequestTracesContent(report.Id, logs)
+	content := cleanupErrorReportTemplate
+	content = strings.ReplaceAll(content, "${resource_type}", resourceType)
+	content = strings.ReplaceAll(content, "${api_version}", apiVersion)
+	content = strings.ReplaceAll(content, "${request_traces}", requestTraces)
+	content = strings.ReplaceAll(content, "${error_message}", report.Message)
+	return content
+}
+
+func CleanupAllRequestTracesContent(id string, logs []types.RequestTrace) string {
+	content := ""
+	for i := len(logs) - 1; i >= 0; i-- {
+		if !strings.EqualFold(id, logs[i].ID) {
+			continue
+		}
+		log := logs[i]
+		if log.HttpMethod == "GET" && strings.Contains(log.Content, "REQUEST/RESPONSE") {
+			st := strings.Index(log.Content, "GET https")
+			ed := strings.Index(log.Content, ": timestamp=")
+			trimContent := log.Content
+			if st < ed {
+				trimContent = log.Content[st:ed]
+			}
+			content = trimContent + "\n\n\n" + content
+		} else if log.HttpMethod == "DELETE" {
+			if strings.Contains(log.Content, "REQUEST/RESPONSE") {
+				st := strings.Index(log.Content, "RESPONSE Status")
+				ed := strings.Index(log.Content, ": timestamp=")
+				trimContent := log.Content
+				if st < ed {
+					trimContent = log.Content[st:ed]
+				}
+				content = trimContent + "\n\n\n" + content
+			} else if strings.Contains(log.Content, "OUTGOING REQUEST") {
+				st := strings.Index(log.Content, "DELETE https")
+				ed := strings.Index(log.Content, ": timestamp=")
+				trimContent := log.Content
+				if st < ed {
+					trimContent = log.Content[st:ed]
+				}
+				content = trimContent + "\n\n" + content
+			}
+		}
+	}
+
+	return content
+}

report/cleanup_error_report.md

@@ -0,0 +1,52 @@
+## ${resource_type}@${api_version} - Error
+
+### Description
+
+I found an error when deleting this resource:
+
+```bash
+${error_message}
+```
+
+### Details
+
+1. ARM Fully-Qualified Resource Type
+```
+${resource_type}
+```
+
+2. API Version
+```
+${api_version}
+```
+
+3. Swagger issue type
+```
+Other
+```
+
+4. OperationId
+```
+TODO
+```
+
+5. Swagger GitHub permalink
+```
+TODO, 
+e.g., https://github.com/Azure/azure-rest-api-specs/blob/60723d13309c8f8060d020a7f3dd9d6e380f0bbd
+/specification/compute/resource-manager/Microsoft.Compute/stable/2020-06-01/compute.json#L9065-L9101
+```
+
+6. Error code
+```
+TODO
+```
+
+7. Request traces
+```
+${request_traces}
+```
+
+### Links
+1. [Semantic and Model Violations Reference](https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/Semantic-and-Model-Violations-Reference.md)
+2. [S360 action item generator for Swagger issues](https://aka.ms/swaggers360)

report/cleanup_passed_report.go

@@ -0,0 +1,26 @@
+package report
+
+import (
+	_ "embed"
+	"fmt"
+	"strings"
+
+	"github.com/ms-henglu/armstrong/types"
+)
+
+//go:embed cleanup_passed_report.md
+var cleanupReportTemplate string
+
+// for now, we don't display bool and enum detail in coverage detail
+
+func CleanupMarkdownReport(passReport types.PassReport) string {
+	resourceTypes := make([]string, 0)
+	for _, resource := range passReport.Resources {
+		resourceTypes = append(resourceTypes, fmt.Sprintf("%s (%s)", resource.Type, resource.Address))
+	}
+
+	content := cleanupReportTemplate
+	content = strings.ReplaceAll(content, "${resource_type}", strings.Join(resourceTypes, "\n"))
+
+	return content
+}

report/cleanup_passed_report.md

@@ -0,0 +1,9 @@
+## Armstrong Cleanup Passed
+
+__This file is automatically generated, please do not edit it directly.__
+
+### Deleted resource types and addresses
+
+```
+${resource_type}
+```

tf/utils.go

@@ -306,3 +306,47 @@ func NewErrorReport(applyErr error, logs []types.RequestTrace) types.ErrorReport
 	}
 	return out
 }
+
+func NewCleanupErrorReport(applyErr error, logs []types.RequestTrace) types.ErrorReport {
+	out := types.ErrorReport{
+		Errors: make([]types.Error, 0),
+		Logs:   logs,
+	}
+	res := strings.Split(applyErr.Error(), "Error: deleting")
+	for _, e := range res {
+		var id, apiVersion string
+		errorMessage := e
+		if lastIndex := strings.LastIndex(e, "------"); lastIndex != -1 {
+			errorMessage = errorMessage[0:lastIndex]
+		}
+		if matches := regexp.MustCompile(`ResourceId \\"(.+)\\" / Api Version \\"(.+)\\"\)`).FindAllStringSubmatch(e, -1); len(matches) == 1 {
+			id = matches[0][1]
+			apiVersion = matches[0][2]
+		} else {
+			continue
+		}
+
+		out.Errors = append(out.Errors, types.Error{
+			Id:      id,
+			Type:    fmt.Sprintf("%s@%s", utils.GetResourceType(id), apiVersion),
+			Message: errorMessage,
+		})
+	}
+	return out
+}
+
+func NewIdAdressFromState(state *tfjson.State) map[string]string {
+	out := map[string]string{}
+	if state == nil || state.Values == nil || state.Values.RootModule == nil || state.Values.RootModule.Resources == nil {
+		log.Printf("[WARN] new id address mapping from state: state is nil")
+		return out
+	}
+	for _, res := range state.Values.RootModule.Resources {
+		id := ""
+		if v, ok := res.AttributeValues["id"]; ok {
+			id = v.(string)
+		}
+		out[id] = res.Address
+	}
+	return out
+}