Merge pull request #71 from ms-henglu/branch-201016

use pal package

commands/cleanup.go

@@ -12,6 +12,7 @@ import (
 	"github.com/ms-henglu/armstrong/report"
 	"github.com/ms-henglu/armstrong/tf"
 	"github.com/ms-henglu/armstrong/types"
+	"github.com/ms-henglu/pal/trace"
 	"github.com/sirupsen/logrus"
 )
 
@@ -83,7 +84,7 @@ func (c CleanupCommand) Execute() int {
 	}
 
 	passReport := tf.NewPassReportFromState(state)
-	idAddressMap := tf.NewIdAdressFromState(state)
+	idAddressMap := tf.NewIdAddressFromState(state)
 
 	reportDir := fmt.Sprintf("armstrong_cleanup_reports_%s", time.Now().Format(time.Stamp))
 	reportDir = strings.ReplaceAll(reportDir, ":", "")
@@ -98,48 +99,44 @@ func (c CleanupCommand) Execute() int {
 	_ = terraform.Init()
 	logrus.Infof("running terraform destroy...")
 	destroyErr := terraform.Destroy()
+
+	errorReport := types.ErrorReport{}
 	if destroyErr != nil {
 		logrus.Errorf("failed to destroy resources: %+v", destroyErr)
-	} else {
-		logrus.Infof("all resources are cleaned up")
-		storeCleanupReport(passReport, reportDir, allPassedReportFileName)
-	}
 
-	logs, err := report.ParseLogs(path.Join(wd, "log.txt"))
-	if err != nil {
-		logrus.Errorf("failed to parse log.txt: %+v", err)
-	}
+		logs, err := trace.RequestTracesFromFile(path.Join(wd, "log.txt"))
+		if err != nil {
+			logrus.Errorf("failed to parse log.txt: %+v", err)
+		}
 
-	errorReport := types.ErrorReport{}
-	if destroyErr != nil {
-		errorReport := tf.NewCleanupErrorReport(destroyErr, logs)
+		errorReport = tf.NewCleanupErrorReport(destroyErr, logs)
 		for i := range errorReport.Errors {
 			if address, ok := idAddressMap[errorReport.Errors[i].Id]; ok {
 				errorReport.Errors[i].Label = address
 			}
 		}
 		storeCleanupErrorReport(errorReport, reportDir)
-	}
 
-	resources := make([]types.Resource, 0)
-	if state, err := terraform.Show(); err == nil && state != nil && state.Values != nil && state.Values.RootModule != nil && state.Values.RootModule.Resources != nil {
-		for _, passRes := range passReport.Resources {
-			isDeleted := true
-			for _, res := range state.Values.RootModule.Resources {
-				if passRes.Address == res.Address {
-					isDeleted = false
-					break
+		resources := make([]types.Resource, 0)
+		if state, err := terraform.Show(); err == nil && state != nil && state.Values != nil && state.Values.RootModule != nil && state.Values.RootModule.Resources != nil {
+			for _, passRes := range passReport.Resources {
+				isDeleted := true
+				for _, res := range state.Values.RootModule.Resources {
+					if passRes.Address == res.Address {
+						isDeleted = false
+						break
+					}
+				}
+				if isDeleted {
+					resources = append(resources, passRes)
 				}
-			}
-			if isDeleted {
-				resources = append(resources, passRes)
 			}
 		}
-	}
-
-	if len(resources) > 0 {
 		passReport.Resources = resources
 		storeCleanupReport(passReport, reportDir, partialPassedReportFileName)
+	} else {
+		logrus.Infof("all resources are cleaned up")
+		storeCleanupReport(passReport, reportDir, allPassedReportFileName)
 	}
 
 	logrus.Infof("---------------- Summary ----------------")

commands/test.go

@@ -4,7 +4,6 @@ import (
 	"flag"
 	"fmt"
 	"os"
-	"os/exec"
 	"path"
 	"path/filepath"
 	"strings"
@@ -15,6 +14,9 @@ import (
 	"github.com/ms-henglu/armstrong/tf"
 	"github.com/ms-henglu/armstrong/types"
 	"github.com/ms-henglu/armstrong/utils"
+	"github.com/ms-henglu/pal/formatter"
+	"github.com/ms-henglu/pal/trace"
+	paltypes "github.com/ms-henglu/pal/types"
 	"github.com/sirupsen/logrus"
 )
 
@@ -132,7 +134,7 @@ func (c TestCommand) Execute() int {
 	}
 
 	logrus.Infof("parsing log.txt...")
-	logs, err := report.ParseLogs(path.Join(wd, "log.txt"))
+	logs, err := trace.RequestTracesFromFile(path.Join(wd, "log.txt"))
 	if err != nil {
 		logrus.Errorf("parsing log.txt: %+v", err)
 	}
@@ -185,12 +187,8 @@ func (c TestCommand) Execute() int {
 			logrus.Errorf("error creating trace dir %s: %+v", traceDir, err)
 		}
 	}
-	cmd := exec.Command("pal", "-i", path.Join(wd, "log.txt"), "-m", "oav", "-o", traceDir)
-	err = cmd.Run()
-	if err != nil {
-		logrus.Errorf("error running pal: %+v", err)
-	}
 
+	storeOavTraffic(logs, traceDir)
 	logrus.Infof("copying traces to report directory...")
 	if err := utils.Copy(traceDir, path.Join(reportDir, "traces")); err != nil {
 		logrus.Errorf("error copying traces: %+v", err)
@@ -254,3 +252,22 @@ func storeDiffReport(diffReport types.DiffReport, reportDir string) {
 		}
 	}
 }
+
+func storeOavTraffic(traces []paltypes.RequestTrace, output string) {
+	format := formatter.OavTrafficFormatter{}
+	files, err := os.ReadDir(output)
+	if err != nil {
+		logrus.Warnf("failed to read trace output directory: %v", err)
+	}
+	index := len(files)
+	for _, t := range traces {
+		out := format.Format(t)
+		index = index + 1
+		outputPath := path.Join(output, fmt.Sprintf("trace-%d.json", index))
+		if err := os.WriteFile(outputPath, []byte(out), 0644); err != nil {
+			logrus.Warnf("failed to write file: %v", err)
+		} else {
+			logrus.Debugf("trace saved to %s", outputPath)
+		}
+	}
+}

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/hashicorp/terraform-json v0.13.0
 	github.com/magodo/azure-rest-api-index v0.0.0-20230522080218-497fe558c02f
 	github.com/mitchellh/cli v1.1.2
+	github.com/ms-henglu/pal v0.4.0
 	github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249
 	github.com/sirupsen/logrus v1.9.3
 	github.com/zclconf/go-cty v1.9.1

go.sum

@@ -522,6 +522,8 @@ github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zx
 github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/ms-henglu/pal v0.4.0 h1:6rzFslYszURQxH2sv0XTTtoLAr/kM2nGDR4iaN9T9k8=
+github.com/ms-henglu/pal v0.4.0/go.mod h1:PunQwlMaYBFFPv1uhjqXCKm8YPDohtuTRWktvTrMPps=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 h1:NHrXEjTNQY7P0Zfx1aMrNhpgxHmow66XQtm0aQLY0AE=
 github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249/go.mod h1:mpRZBD8SJ55OIICQ3iWH0Yz3cjzA61JdqMLoWXeB2+8=

report/cleanup_error_report.go

@@ -5,12 +5,13 @@ import (
 	"strings"
 
 	"github.com/ms-henglu/armstrong/types"
+	paltypes "github.com/ms-henglu/pal/types"
 )
 
 //go:embed cleanup_error_report.md
 var cleanupErrorReportTemplate string
 
-func CleanupErrorMarkdownReport(report types.Error, logs []types.RequestTrace) string {
+func CleanupErrorMarkdownReport(report types.Error, logs []paltypes.RequestTrace) string {
 	parts := strings.Split(report.Type, "@")
 	resourceType := ""
 	apiVersion := ""
@@ -27,41 +28,20 @@ func CleanupErrorMarkdownReport(report types.Error, logs []types.RequestTrace) s
 	return content
 }
 
-func CleanupAllRequestTracesContent(id string, logs []types.RequestTrace) string {
+func CleanupAllRequestTracesContent(id string, logs []paltypes.RequestTrace) string {
 	content := ""
-	for i := len(logs) - 1; i >= 0; i-- {
-		if !strings.EqualFold(id, logs[i].ID) {
-			continue
+	index := len(logs) - 1
+	for ; index >= 0; index-- {
+		if IsUrlMatchWithId(logs[index].Url, id) && logs[index].Method == "DELETE" {
+			content = RequestTraceToString(logs[index])
+			break
 		}
-		log := logs[i]
-		if log.HttpMethod == "GET" && strings.Contains(log.Content, "REQUEST/RESPONSE") {
-			st := strings.Index(log.Content, "GET https")
-			ed := strings.Index(log.Content, ": timestamp=")
-			trimContent := log.Content
-			if st < ed {
-				trimContent = log.Content[st:ed]
-			}
-			content = trimContent + "\n\n\n" + content
-		} else if log.HttpMethod == "DELETE" {
-			if strings.Contains(log.Content, "REQUEST/RESPONSE") {
-				st := strings.Index(log.Content, "RESPONSE Status")
-				ed := strings.Index(log.Content, ": timestamp=")
-				trimContent := log.Content
-				if st < ed {
-					trimContent = log.Content[st:ed]
-				}
-				content = trimContent + "\n\n\n" + content
-			} else if strings.Contains(log.Content, "OUTGOING REQUEST") {
-				st := strings.Index(log.Content, "DELETE https")
-				ed := strings.Index(log.Content, ": timestamp=")
-				trimContent := log.Content
-				if st < ed {
-					trimContent = log.Content[st:ed]
-				}
-				content = trimContent + "\n\n" + content
-			}
+	}
+	for ; index >= 0; index-- {
+		if IsUrlMatchWithId(logs[index].Url, id) && logs[index].Method == "GET" {
+			content = RequestTraceToString(logs[index]) + "\n\n\n" + content
+			break
 		}
 	}
-
 	return content
 }

report/diff_report.go

@@ -5,12 +5,13 @@ import (
 	"strings"
 
 	"github.com/ms-henglu/armstrong/types"
+	paltypes "github.com/ms-henglu/pal/types"
 )
 
 //go:embed diff_report.md
 var diffReportTemplate string
 
-func DiffMarkdownReport(report types.Diff, logs []types.RequestTrace) string {
+func DiffMarkdownReport(report types.Diff, logs []paltypes.RequestTrace) string {
 	parts := strings.Split(report.Type, "@")
 	resourceType := ""
 	apiVersion := ""
@@ -46,47 +47,20 @@ func DiffMarkdownReport(report types.Diff, logs []types.RequestTrace) string {
 	return content
 }
 
-func RequestTracesContent(id string, logs []types.RequestTrace) string {
+func RequestTracesContent(id string, logs []paltypes.RequestTrace) string {
 	content := ""
 	index := len(logs) - 1
-	if log, i := findLastLog(logs, id, "GET", "REQUEST/RESPONSE", index); i != -1 {
-		st := strings.Index(log.Content, "GET https")
-		ed := strings.Index(log.Content, ": timestamp=")
-		trimContent := log.Content
-		if st < ed {
-			trimContent = log.Content[st:ed]
+	for ; index >= 0; index-- {
+		if IsUrlMatchWithId(logs[index].Url, id) && logs[index].Method == "GET" {
+			content = RequestTraceToString(logs[index])
+			break
 		}
-		content = trimContent
-		index = i
 	}
-	if log, i := findLastLog(logs, id, "PUT", "REQUEST/RESPONSE", index); i != -1 {
-		st := strings.Index(log.Content, "RESPONSE Status")
-		ed := strings.Index(log.Content, ": timestamp=")
-		trimContent := log.Content
-		if st < ed {
-			trimContent = log.Content[st:ed]
+	for ; index >= 0; index-- {
+		if IsUrlMatchWithId(logs[index].Url, id) && logs[index].Method == "PUT" {
+			content = RequestTraceToString(logs[index]) + "\n\n\n" + content
+			break
 		}
-		content = trimContent + "\n\n\n" + content
-		index = i
-	}
-	if log, i := findLastLog(logs, id, "PUT", "OUTGOING REQUEST", index); i != -1 {
-		st := strings.Index(log.Content, "PUT https")
-		ed := strings.Index(log.Content, ": timestamp=")
-		trimContent := log.Content
-		if st < ed {
-			trimContent = log.Content[st:ed]
-		}
-		content = trimContent + "\n\n" + content
 	}
 	return content
 }
-
-func findLastLog(logs []types.RequestTrace, id string, method string, substr string, index int) (types.RequestTrace, int) {
-	for i := index; i >= 0; i-- {
-		log := logs[i]
-		if log.ID == id && log.HttpMethod == method && strings.Contains(log.Content, substr) {
-			return log, i
-		}
-	}
-	return types.RequestTrace{}, -1
-}

report/error_report.go

@@ -2,6 +2,7 @@ package report
 
 import (
 	_ "embed"
+	paltypes "github.com/ms-henglu/pal/types"
 	"strings"
 
 	"github.com/ms-henglu/armstrong/types"
@@ -10,7 +11,7 @@ import (
 //go:embed error_report.md
 var errorReportTemplate string
 
-func ErrorMarkdownReport(report types.Error, logs []types.RequestTrace) string {
+func ErrorMarkdownReport(report types.Error, logs []paltypes.RequestTrace) string {
 	parts := strings.Split(report.Type, "@")
 	resourceType := ""
 	apiVersion := ""
@@ -27,39 +28,11 @@ func ErrorMarkdownReport(report types.Error, logs []types.RequestTrace) string {
 	return content
 }
 
-func AllRequestTracesContent(id string, logs []types.RequestTrace) string {
+func AllRequestTracesContent(id string, logs []paltypes.RequestTrace) string {
 	content := ""
 	for i := len(logs) - 1; i >= 0; i-- {
-		if !strings.EqualFold(id, logs[i].ID) {
-			continue
-		}
-		log := logs[i]
-		if log.HttpMethod == "GET" && strings.Contains(log.Content, "REQUEST/RESPONSE") {
-			st := strings.Index(log.Content, "GET https")
-			ed := strings.Index(log.Content, ": timestamp=")
-			trimContent := log.Content
-			if st < ed {
-				trimContent = log.Content[st:ed]
-			}
-			content = trimContent + "\n\n\n" + content
-		} else if log.HttpMethod == "PUT" {
-			if strings.Contains(log.Content, "REQUEST/RESPONSE") {
-				st := strings.Index(log.Content, "RESPONSE Status")
-				ed := strings.Index(log.Content, ": timestamp=")
-				trimContent := log.Content
-				if st < ed {
-					trimContent = log.Content[st:ed]
-				}
-				content = trimContent + "\n\n\n" + content
-			} else if strings.Contains(log.Content, "OUTGOING REQUEST") {
-				st := strings.Index(log.Content, "PUT https")
-				ed := strings.Index(log.Content, ": timestamp=")
-				trimContent := log.Content
-				if st < ed {
-					trimContent = log.Content[st:ed]
-				}
-				content = trimContent + "\n\n" + content
-			}
+		if IsUrlMatchWithId(logs[i].Url, id) {
+			content += RequestTraceToString(logs[i]) + "\n\n\n"
 		}
 	}