From f2c548ba36ace6dd69a244af512235c6091f4324 Mon Sep 17 00:00:00 2001 From: Heng Lu Date: Tue, 17 Oct 2023 15:59:06 +0800 Subject: [PATCH 1/2] use pal package --- commands/cleanup.go | 51 ++- commands/test.go | 31 +- go.mod | 1 + go.sum | 2 + report/cleanup_error_report.go | 46 +-- report/diff_report.go | 48 +-- report/error_report.go | 37 +- report/log.go | 86 +++-- tf/utils.go | 9 +- types/log.go | 8 - types/report.go | 6 +- vendor/github.com/ms-henglu/pal/.gitignore | 3 + .../github.com/ms-henglu/pal/.goreleaser.yml | 54 +++ vendor/github.com/ms-henglu/pal/CHANGELOG.md | 40 ++ vendor/github.com/ms-henglu/pal/GNUmakefile | 19 + vendor/github.com/ms-henglu/pal/README.md | 60 +++ .../pal/formatter/azapi/formatter.go | 348 ++++++++++++++++++ .../pal/formatter/azapi/hcl/marshal.go | 64 ++++ .../ms-henglu/pal/formatter/azapi/types.go | 101 +++++ .../ms-henglu/pal/formatter/azapi/utils.go | 121 ++++++ .../ms-henglu/pal/formatter/base.go | 9 + .../ms-henglu/pal/formatter/markdown.go | 94 +++++ .../ms-henglu/pal/formatter/oav_traffic.go | 73 ++++ vendor/github.com/ms-henglu/pal/main.go | 140 +++++++ .../ms-henglu/pal/provider/azapi.go | 98 +++++ .../ms-henglu/pal/provider/azuread.go | 121 ++++++ .../ms-henglu/pal/provider/azurerm.go | 166 +++++++++ .../github.com/ms-henglu/pal/provider/base.go | 15 + .../ms-henglu/pal/rawlog/raw_log.go | 41 +++ .../github.com/ms-henglu/pal/trace/trace.go | 165 +++++++++ .../github.com/ms-henglu/pal/types/types.go | 24 ++ .../github.com/ms-henglu/pal/utils/utils.go | 69 ++++ vendor/modules.txt | 11 + 33 files changed, 1973 insertions(+), 188 deletions(-) delete mode 100644 types/log.go create mode 100644 vendor/github.com/ms-henglu/pal/.gitignore create mode 100644 vendor/github.com/ms-henglu/pal/.goreleaser.yml create mode 100644 vendor/github.com/ms-henglu/pal/CHANGELOG.md create mode 100644 vendor/github.com/ms-henglu/pal/GNUmakefile create mode 100644 vendor/github.com/ms-henglu/pal/README.md create mode 100644 vendor/github.com/ms-henglu/pal/formatter/azapi/formatter.go create mode 100644 vendor/github.com/ms-henglu/pal/formatter/azapi/hcl/marshal.go create mode 100644 vendor/github.com/ms-henglu/pal/formatter/azapi/types.go create mode 100644 vendor/github.com/ms-henglu/pal/formatter/azapi/utils.go create mode 100644 vendor/github.com/ms-henglu/pal/formatter/base.go create mode 100644 vendor/github.com/ms-henglu/pal/formatter/markdown.go create mode 100644 vendor/github.com/ms-henglu/pal/formatter/oav_traffic.go create mode 100644 vendor/github.com/ms-henglu/pal/main.go create mode 100644 vendor/github.com/ms-henglu/pal/provider/azapi.go create mode 100644 vendor/github.com/ms-henglu/pal/provider/azuread.go create mode 100644 vendor/github.com/ms-henglu/pal/provider/azurerm.go create mode 100644 vendor/github.com/ms-henglu/pal/provider/base.go create mode 100644 vendor/github.com/ms-henglu/pal/rawlog/raw_log.go create mode 100644 vendor/github.com/ms-henglu/pal/trace/trace.go create mode 100644 vendor/github.com/ms-henglu/pal/types/types.go create mode 100644 vendor/github.com/ms-henglu/pal/utils/utils.go diff --git a/commands/cleanup.go b/commands/cleanup.go index 4f3d3150..74472570 100644 --- a/commands/cleanup.go +++ b/commands/cleanup.go @@ -12,6 +12,7 @@ import ( "github.com/ms-henglu/armstrong/report" "github.com/ms-henglu/armstrong/tf" "github.com/ms-henglu/armstrong/types" + "github.com/ms-henglu/pal/trace" "github.com/sirupsen/logrus" ) @@ -83,7 +84,7 @@ func (c CleanupCommand) Execute() int { } passReport := tf.NewPassReportFromState(state) - idAddressMap := tf.NewIdAdressFromState(state) + idAddressMap := tf.NewIdAddressFromState(state) reportDir := fmt.Sprintf("armstrong_cleanup_reports_%s", time.Now().Format(time.Stamp)) reportDir = strings.ReplaceAll(reportDir, ":", "") @@ -98,48 +99,44 @@ func (c CleanupCommand) Execute() int { _ = terraform.Init() logrus.Infof("running terraform destroy...") destroyErr := terraform.Destroy() + + errorReport := types.ErrorReport{} if destroyErr != nil { logrus.Errorf("failed to destroy resources: %+v", destroyErr) - } else { - logrus.Infof("all resources are cleaned up") - storeCleanupReport(passReport, reportDir, allPassedReportFileName) - } - logs, err := report.ParseLogs(path.Join(wd, "log.txt")) - if err != nil { - logrus.Errorf("failed to parse log.txt: %+v", err) - } + logs, err := trace.RequestTracesFromFile(path.Join(wd, "log.txt")) + if err != nil { + logrus.Errorf("failed to parse log.txt: %+v", err) + } - errorReport := types.ErrorReport{} - if destroyErr != nil { - errorReport := tf.NewCleanupErrorReport(destroyErr, logs) + errorReport = tf.NewCleanupErrorReport(destroyErr, logs) for i := range errorReport.Errors { if address, ok := idAddressMap[errorReport.Errors[i].Id]; ok { errorReport.Errors[i].Label = address } } storeCleanupErrorReport(errorReport, reportDir) - } - resources := make([]types.Resource, 0) - if state, err := terraform.Show(); err == nil && state != nil && state.Values != nil && state.Values.RootModule != nil && state.Values.RootModule.Resources != nil { - for _, passRes := range passReport.Resources { - isDeleted := true - for _, res := range state.Values.RootModule.Resources { - if passRes.Address == res.Address { - isDeleted = false - break + resources := make([]types.Resource, 0) + if state, err := terraform.Show(); err == nil && state != nil && state.Values != nil && state.Values.RootModule != nil && state.Values.RootModule.Resources != nil { + for _, passRes := range passReport.Resources { + isDeleted := true + for _, res := range state.Values.RootModule.Resources { + if passRes.Address == res.Address { + isDeleted = false + break + } + } + if isDeleted { + resources = append(resources, passRes) } - } - if isDeleted { - resources = append(resources, passRes) } } - } - - if len(resources) > 0 { passReport.Resources = resources storeCleanupReport(passReport, reportDir, partialPassedReportFileName) + } else { + logrus.Infof("all resources are cleaned up") + storeCleanupReport(passReport, reportDir, allPassedReportFileName) } logrus.Infof("---------------- Summary ----------------") diff --git a/commands/test.go b/commands/test.go index 5ff6da84..fdb5cf28 100644 --- a/commands/test.go +++ b/commands/test.go @@ -4,7 +4,6 @@ import ( "flag" "fmt" "os" - "os/exec" "path" "path/filepath" "strings" @@ -15,6 +14,9 @@ import ( "github.com/ms-henglu/armstrong/tf" "github.com/ms-henglu/armstrong/types" "github.com/ms-henglu/armstrong/utils" + "github.com/ms-henglu/pal/formatter" + "github.com/ms-henglu/pal/trace" + paltypes "github.com/ms-henglu/pal/types" "github.com/sirupsen/logrus" ) @@ -132,7 +134,7 @@ func (c TestCommand) Execute() int { } logrus.Infof("parsing log.txt...") - logs, err := report.ParseLogs(path.Join(wd, "log.txt")) + logs, err := trace.RequestTracesFromFile(path.Join(wd, "log.txt")) if err != nil { logrus.Errorf("parsing log.txt: %+v", err) } @@ -185,12 +187,8 @@ func (c TestCommand) Execute() int { logrus.Errorf("error creating trace dir %s: %+v", traceDir, err) } } - cmd := exec.Command("pal", "-i", path.Join(wd, "log.txt"), "-m", "oav", "-o", traceDir) - err = cmd.Run() - if err != nil { - logrus.Errorf("error running pal: %+v", err) - } + storeOavTraffic(logs, traceDir) logrus.Infof("copying traces to report directory...") if err := utils.Copy(traceDir, path.Join(reportDir, "traces")); err != nil { logrus.Errorf("error copying traces: %+v", err) @@ -254,3 +252,22 @@ func storeDiffReport(diffReport types.DiffReport, reportDir string) { } } } + +func storeOavTraffic(traces []paltypes.RequestTrace, output string) { + format := formatter.OavTrafficFormatter{} + files, err := os.ReadDir(output) + if err != nil { + logrus.Warnf("failed to read trace output directory: %v", err) + } + index := len(files) + for _, t := range traces { + out := format.Format(t) + index = index + 1 + outputPath := path.Join(output, fmt.Sprintf("trace-%d.json", index)) + if err := os.WriteFile(outputPath, []byte(out), 0644); err != nil { + logrus.Warnf("failed to write file: %v", err) + } else { + logrus.Debugf("trace saved to %s", outputPath) + } + } +} diff --git a/go.mod b/go.mod index 6f83b442..ba1619c6 100644 --- a/go.mod +++ b/go.mod @@ -85,6 +85,7 @@ require ( github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/ms-henglu/pal v0.4.0 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/posener/complete v1.1.1 // indirect diff --git a/go.sum b/go.sum index 796cfb44..9c0787ee 100644 --- a/go.sum +++ b/go.sum @@ -522,6 +522,8 @@ github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zx github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/ms-henglu/pal v0.4.0 h1:6rzFslYszURQxH2sv0XTTtoLAr/kM2nGDR4iaN9T9k8= +github.com/ms-henglu/pal v0.4.0/go.mod h1:PunQwlMaYBFFPv1uhjqXCKm8YPDohtuTRWktvTrMPps= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 h1:NHrXEjTNQY7P0Zfx1aMrNhpgxHmow66XQtm0aQLY0AE= github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249/go.mod h1:mpRZBD8SJ55OIICQ3iWH0Yz3cjzA61JdqMLoWXeB2+8= diff --git a/report/cleanup_error_report.go b/report/cleanup_error_report.go index 14d1f7e8..4d6fe8eb 100644 --- a/report/cleanup_error_report.go +++ b/report/cleanup_error_report.go @@ -5,12 +5,13 @@ import ( "strings" "github.com/ms-henglu/armstrong/types" + paltypes "github.com/ms-henglu/pal/types" ) //go:embed cleanup_error_report.md var cleanupErrorReportTemplate string -func CleanupErrorMarkdownReport(report types.Error, logs []types.RequestTrace) string { +func CleanupErrorMarkdownReport(report types.Error, logs []paltypes.RequestTrace) string { parts := strings.Split(report.Type, "@") resourceType := "" apiVersion := "" @@ -27,41 +28,20 @@ func CleanupErrorMarkdownReport(report types.Error, logs []types.RequestTrace) s return content } -func CleanupAllRequestTracesContent(id string, logs []types.RequestTrace) string { +func CleanupAllRequestTracesContent(id string, logs []paltypes.RequestTrace) string { content := "" - for i := len(logs) - 1; i >= 0; i-- { - if !strings.EqualFold(id, logs[i].ID) { - continue + index := len(logs) - 1 + for ; index >= 0; index-- { + if IsUrlMatchWithId(logs[index].Url, id) && logs[index].Method == "DELETE" { + content = RequestTraceToString(logs[index]) + break } - log := logs[i] - if log.HttpMethod == "GET" && strings.Contains(log.Content, "REQUEST/RESPONSE") { - st := strings.Index(log.Content, "GET https") - ed := strings.Index(log.Content, ": timestamp=") - trimContent := log.Content - if st < ed { - trimContent = log.Content[st:ed] - } - content = trimContent + "\n\n\n" + content - } else if log.HttpMethod == "DELETE" { - if strings.Contains(log.Content, "REQUEST/RESPONSE") { - st := strings.Index(log.Content, "RESPONSE Status") - ed := strings.Index(log.Content, ": timestamp=") - trimContent := log.Content - if st < ed { - trimContent = log.Content[st:ed] - } - content = trimContent + "\n\n\n" + content - } else if strings.Contains(log.Content, "OUTGOING REQUEST") { - st := strings.Index(log.Content, "DELETE https") - ed := strings.Index(log.Content, ": timestamp=") - trimContent := log.Content - if st < ed { - trimContent = log.Content[st:ed] - } - content = trimContent + "\n\n" + content - } + } + for ; index >= 0; index-- { + if IsUrlMatchWithId(logs[index].Url, id) && logs[index].Method == "GET" { + content = RequestTraceToString(logs[index]) + "\n\n\n" + content + break } } - return content } diff --git a/report/diff_report.go b/report/diff_report.go index 26d695c1..18f0d0a4 100644 --- a/report/diff_report.go +++ b/report/diff_report.go @@ -5,12 +5,13 @@ import ( "strings" "github.com/ms-henglu/armstrong/types" + paltypes "github.com/ms-henglu/pal/types" ) //go:embed diff_report.md var diffReportTemplate string -func DiffMarkdownReport(report types.Diff, logs []types.RequestTrace) string { +func DiffMarkdownReport(report types.Diff, logs []paltypes.RequestTrace) string { parts := strings.Split(report.Type, "@") resourceType := "" apiVersion := "" @@ -46,47 +47,20 @@ func DiffMarkdownReport(report types.Diff, logs []types.RequestTrace) string { return content } -func RequestTracesContent(id string, logs []types.RequestTrace) string { +func RequestTracesContent(id string, logs []paltypes.RequestTrace) string { content := "" index := len(logs) - 1 - if log, i := findLastLog(logs, id, "GET", "REQUEST/RESPONSE", index); i != -1 { - st := strings.Index(log.Content, "GET https") - ed := strings.Index(log.Content, ": timestamp=") - trimContent := log.Content - if st < ed { - trimContent = log.Content[st:ed] + for ; index >= 0; index-- { + if IsUrlMatchWithId(logs[index].Url, id) && logs[index].Method == "GET" { + content = RequestTraceToString(logs[index]) + break } - content = trimContent - index = i } - if log, i := findLastLog(logs, id, "PUT", "REQUEST/RESPONSE", index); i != -1 { - st := strings.Index(log.Content, "RESPONSE Status") - ed := strings.Index(log.Content, ": timestamp=") - trimContent := log.Content - if st < ed { - trimContent = log.Content[st:ed] + for ; index >= 0; index-- { + if IsUrlMatchWithId(logs[index].Url, id) && logs[index].Method == "PUT" { + content = RequestTraceToString(logs[index]) + "\n\n\n" + content + break } - content = trimContent + "\n\n\n" + content - index = i - } - if log, i := findLastLog(logs, id, "PUT", "OUTGOING REQUEST", index); i != -1 { - st := strings.Index(log.Content, "PUT https") - ed := strings.Index(log.Content, ": timestamp=") - trimContent := log.Content - if st < ed { - trimContent = log.Content[st:ed] - } - content = trimContent + "\n\n" + content } return content } - -func findLastLog(logs []types.RequestTrace, id string, method string, substr string, index int) (types.RequestTrace, int) { - for i := index; i >= 0; i-- { - log := logs[i] - if log.ID == id && log.HttpMethod == method && strings.Contains(log.Content, substr) { - return log, i - } - } - return types.RequestTrace{}, -1 -} diff --git a/report/error_report.go b/report/error_report.go index 45bec86d..69360050 100644 --- a/report/error_report.go +++ b/report/error_report.go @@ -2,6 +2,7 @@ package report import ( _ "embed" + paltypes "github.com/ms-henglu/pal/types" "strings" "github.com/ms-henglu/armstrong/types" @@ -10,7 +11,7 @@ import ( //go:embed error_report.md var errorReportTemplate string -func ErrorMarkdownReport(report types.Error, logs []types.RequestTrace) string { +func ErrorMarkdownReport(report types.Error, logs []paltypes.RequestTrace) string { parts := strings.Split(report.Type, "@") resourceType := "" apiVersion := "" @@ -27,39 +28,11 @@ func ErrorMarkdownReport(report types.Error, logs []types.RequestTrace) string { return content } -func AllRequestTracesContent(id string, logs []types.RequestTrace) string { +func AllRequestTracesContent(id string, logs []paltypes.RequestTrace) string { content := "" for i := len(logs) - 1; i >= 0; i-- { - if !strings.EqualFold(id, logs[i].ID) { - continue - } - log := logs[i] - if log.HttpMethod == "GET" && strings.Contains(log.Content, "REQUEST/RESPONSE") { - st := strings.Index(log.Content, "GET https") - ed := strings.Index(log.Content, ": timestamp=") - trimContent := log.Content - if st < ed { - trimContent = log.Content[st:ed] - } - content = trimContent + "\n\n\n" + content - } else if log.HttpMethod == "PUT" { - if strings.Contains(log.Content, "REQUEST/RESPONSE") { - st := strings.Index(log.Content, "RESPONSE Status") - ed := strings.Index(log.Content, ": timestamp=") - trimContent := log.Content - if st < ed { - trimContent = log.Content[st:ed] - } - content = trimContent + "\n\n\n" + content - } else if strings.Contains(log.Content, "OUTGOING REQUEST") { - st := strings.Index(log.Content, "PUT https") - ed := strings.Index(log.Content, ": timestamp=") - trimContent := log.Content - if st < ed { - trimContent = log.Content[st:ed] - } - content = trimContent + "\n\n" + content - } + if IsUrlMatchWithId(logs[i].Url, id) { + content += RequestTraceToString(logs[i]) + "\n\n\n" } } diff --git a/report/log.go b/report/log.go index 731f630d..7457ba67 100644 --- a/report/log.go +++ b/report/log.go @@ -1,54 +1,64 @@ package report import ( - "os" - "regexp" - "strconv" + "encoding/json" + "fmt" "strings" - "github.com/ms-henglu/armstrong/types" + paltypes "github.com/ms-henglu/pal/types" ) -func ParseLogs(filepath string) ([]types.RequestTrace, error) { - data, err := os.ReadFile(filepath) - if err != nil { - return nil, err +func IsUrlMatchWithId(url string, id string) bool { + return strings.HasPrefix(url, id+"?") +} + +func RequestTraceToString(r paltypes.RequestTrace) string { + return fmt.Sprintf(`%s %s +Status Code: %d +------------ Request ------------ +%s +------------ Response ------------ +%s + +`, r.Method, r.Url, r.StatusCode, HttpRequestToString(r.Request), HttpResponseToString(r.Response)) +} + +func HttpRequestToString(r *paltypes.HttpRequest) string { + if r == nil { + return "" + } + headers := "" + for k, v := range r.Headers { + headers += fmt.Sprintf("%s: %s\n", k, v) } - logs := make([]types.RequestTrace, 0) - logPrefixReg, _ := regexp.Compile(`^\d{4}-\d{2}-\d{2}`) - temp := "" - lines := strings.Split(string(data), "\n") - for _, line := range lines { - if logPrefixReg.MatchString(line) { - if (strings.Contains(temp, "OUTGOING REQUEST") || strings.Contains(temp, "REQUEST/RESPONSE")) && strings.Contains(temp, "management.azure.com") { - logs = append(logs, NewRequestTrace(temp)) - } - temp = "" + bodyContent := r.Body + var body interface{} + if err := json.Unmarshal([]byte(bodyContent), &body); err == nil { + if data, err := json.MarshalIndent(body, "", " "); err == nil { + bodyContent = string(data) } - temp += line + "\n" } - - return logs, nil + return fmt.Sprintf(`%s +--- +%s +`, headers, bodyContent) } -func NewRequestTrace(raw string) types.RequestTrace { - trace := types.RequestTrace{} - - methodReg, _ := regexp.Compile(`([A-Z]+)\shttps`) - if matches := methodReg.FindAllStringSubmatch(raw, -1); len(matches) > 0 && len(matches[0]) == 2 { - trace.HttpMethod = matches[0][1] +func HttpResponseToString(r *paltypes.HttpResponse) string { + if r == nil { + return "" } - - statusCodeReg, _ := regexp.Compile(`RESPONSE\sStatus:\s(\d+)`) - if matches := statusCodeReg.FindAllStringSubmatch(raw, -1); len(matches) > 0 && len(matches[0]) == 2 { - trace.StatusCode, _ = strconv.ParseInt(matches[0][1], 10, 32) + headers := "" + for k, v := range r.Headers { + headers += fmt.Sprintf("%s: %s\n", k, v) } - - idReg, _ := regexp.Compile(`management\.azure\.com(.+)\?api-version`) - if matches := idReg.FindAllStringSubmatch(raw, -1); len(matches) > 0 && len(matches[0]) == 2 { - trace.ID = matches[0][1] + bodyContent := r.Body + var body interface{} + if err := json.Unmarshal([]byte(bodyContent), &body); err == nil { + if data, err := json.MarshalIndent(body, "", " "); err == nil { + bodyContent = string(data) + } } - - trace.Content = raw - return trace + return fmt.Sprintf(`%s------ +%s`, headers, bodyContent) } diff --git a/tf/utils.go b/tf/utils.go index 7bc4dcff..7413ee16 100644 --- a/tf/utils.go +++ b/tf/utils.go @@ -10,6 +10,7 @@ import ( "github.com/ms-henglu/armstrong/coverage" "github.com/ms-henglu/armstrong/types" "github.com/ms-henglu/armstrong/utils" + paltypes "github.com/ms-henglu/pal/types" "github.com/sirupsen/logrus" ) @@ -54,7 +55,7 @@ func GetChanges(plan *tfjson.Plan) []Action { return actions } -func NewDiffReport(plan *tfjson.Plan, logs []types.RequestTrace) types.DiffReport { +func NewDiffReport(plan *tfjson.Plan, logs []paltypes.RequestTrace) types.DiffReport { out := types.DiffReport{ Diffs: make([]types.Diff, 0), Logs: logs, @@ -287,7 +288,7 @@ func expandIdentity(input []interface{}) map[string]interface{} { return config } -func NewErrorReport(applyErr error, logs []types.RequestTrace) types.ErrorReport { +func NewErrorReport(applyErr error, logs []paltypes.RequestTrace) types.ErrorReport { out := types.ErrorReport{ Errors: make([]types.Error, 0), Logs: logs, @@ -322,7 +323,7 @@ func NewErrorReport(applyErr error, logs []types.RequestTrace) types.ErrorReport return out } -func NewCleanupErrorReport(applyErr error, logs []types.RequestTrace) types.ErrorReport { +func NewCleanupErrorReport(applyErr error, logs []paltypes.RequestTrace) types.ErrorReport { out := types.ErrorReport{ Errors: make([]types.Error, 0), Logs: logs, @@ -350,7 +351,7 @@ func NewCleanupErrorReport(applyErr error, logs []types.RequestTrace) types.Erro return out } -func NewIdAdressFromState(state *tfjson.State) map[string]string { +func NewIdAddressFromState(state *tfjson.State) map[string]string { out := map[string]string{} if state == nil || state.Values == nil || state.Values.RootModule == nil || state.Values.RootModule.Resources == nil { logrus.Warnf("new id address mapping from state: state is nil") diff --git a/types/log.go b/types/log.go deleted file mode 100644 index 3e244680..00000000 --- a/types/log.go +++ /dev/null @@ -1,8 +0,0 @@ -package types - -type RequestTrace struct { - HttpMethod string - StatusCode int64 - ID string - Content string -} diff --git a/types/report.go b/types/report.go index 65e34a4d..f2adf06e 100644 --- a/types/report.go +++ b/types/report.go @@ -1,5 +1,7 @@ package types +import paltypes "github.com/ms-henglu/pal/types" + type PassReport struct { Resources []Resource } @@ -11,7 +13,7 @@ type Resource struct { type DiffReport struct { Diffs []Diff - Logs []RequestTrace + Logs []paltypes.RequestTrace } type Diff struct { @@ -28,7 +30,7 @@ type Change struct { type ErrorReport struct { Errors []Error - Logs []RequestTrace + Logs []paltypes.RequestTrace } type Error struct { diff --git a/vendor/github.com/ms-henglu/pal/.gitignore b/vendor/github.com/ms-henglu/pal/.gitignore new file mode 100644 index 00000000..14bc234b --- /dev/null +++ b/vendor/github.com/ms-henglu/pal/.gitignore @@ -0,0 +1,3 @@ +.idea/* +.idea +*/.DS_Store diff --git a/vendor/github.com/ms-henglu/pal/.goreleaser.yml b/vendor/github.com/ms-henglu/pal/.goreleaser.yml new file mode 100644 index 00000000..7740d89c --- /dev/null +++ b/vendor/github.com/ms-henglu/pal/.goreleaser.yml @@ -0,0 +1,54 @@ +# Visit https://goreleaser.com for documentation on how to customize this +# behavior. +before: + hooks: + # this is just an example and not a requirement for provider building/publishing + - go mod tidy +builds: + - env: + # goreleaser does not work with CGO, it could also complicate + # usage by users in CI/CD systems like Terraform Cloud where + # they are unable to install libraries. + - CGO_ENABLED=0 + mod_timestamp: '{{ .CommitTimestamp }}' + flags: + - -trimpath + ldflags: + - '-s -w -X main.version={{.Version}} -X main.commit={{.Commit}}' + goos: + - freebsd + - windows + - linux + - darwin + goarch: + - amd64 + - '386' + - arm + - arm64 + ignore: + - goos: darwin + goarch: '386' + binary: '{{ .ProjectName }}' +archives: + - format: zip + name_template: '{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}' +checksum: + name_template: '{{ .ProjectName }}_{{ .Version }}_SHA256SUMS' + algorithm: sha256 +signs: + - artifacts: checksum + args: + # if you are using this in a GitHub action or some other automated pipeline, you + # need to pass the batch flag to indicate its not interactive. + - "--batch" + - "--local-user" + - "{{ .Env.GPG_FINGERPRINT }}" # set this environment variable for your signing key + - "--output" + - "${signature}" + - "--detach-sign" + - "${artifact}" +release: + # If you want to manually examine the release before its live, uncomment this line: + draft: true +changelog: + skip: true diff --git a/vendor/github.com/ms-henglu/pal/CHANGELOG.md b/vendor/github.com/ms-henglu/pal/CHANGELOG.md new file mode 100644 index 00000000..ab400882 --- /dev/null +++ b/vendor/github.com/ms-henglu/pal/CHANGELOG.md @@ -0,0 +1,40 @@ +## v0.4.0 +BREAKING CHANGES: +- Only support logs from terraform-provider-azapi v1.10.0 or above. + +ENHANCEMENTS: +- The redundant query parameters are removed in the `markdown` format. +- Remove the `/providers` API from parsed logs, because its response couldn't be parsed. + +BUG FIXES: +- Fix the issue that some resources may not be outputted to `azapi` format. + +## v0.3.0 + +FEATURES: +- Support parsing terraform logs to `azapi` format. + +BUG FIXES: +- Fix the issue that the parsed URL paths are not normalized. +- Fix the issue that the request body from azurerm provider may not be parsed correctly, when the request body is pretty printed JSON. + +## v0.2.0 + +FEATURES: +- Support parsing terraform logs to `oav` traffic format. +- Support `-version` option to show the version information. +- Support `-help` option to show the help information. +- Support `-o` option to specify the output directory. +- Support `-i` option to specify the input file path. +- Support `-m` option to specify the output format. + +BUG FIXES: +- Fix the issue that response headers may contain duplicated values. +- Fix the issue that logs from released `azurerm` provider may not be parsed correctly. + +## v0.1.0 + +FEATURES: + +- Support parsing terraform logs to markdown format. +- Support `azurerm`, `azuread` and `azapi` providers. \ No newline at end of file diff --git a/vendor/github.com/ms-henglu/pal/GNUmakefile b/vendor/github.com/ms-henglu/pal/GNUmakefile new file mode 100644 index 00000000..84b8d1ea --- /dev/null +++ b/vendor/github.com/ms-henglu/pal/GNUmakefile @@ -0,0 +1,19 @@ +fmt: + find . -name '*.go' | grep -v vendor | xargs gofmt -s -w + +depscheck: + @echo "==> Checking source code with go mod tidy..." + @go mod tidy + @git diff --exit-code -- go.mod go.sum || \ + (echo; echo "Unexpected difference in go.mod/go.sum files. Run 'go mod tidy' command or revert any go.mod/go.sum changes and commit."; exit 1) + @echo "==> Checking source code with go mod vendor..." + @go mod vendor + @git diff --compact-summary --ignore-space-at-eol --exit-code -- vendor || \ + (echo; echo "Unexpected difference in vendor/ directory. Run 'go mod vendor' command or revert any go.mod/go.sum/vendor changes and commit."; exit 1) + +test: + @go test ./... + +lint: + @echo "==> Checking source code against linters..." + @if command -v golangci-lint; then (golangci-lint run ./...); else ($(GOPATH)/bin/golangci-lint run ./...); fi \ No newline at end of file diff --git a/vendor/github.com/ms-henglu/pal/README.md b/vendor/github.com/ms-henglu/pal/README.md new file mode 100644 index 00000000..529ba4a2 --- /dev/null +++ b/vendor/github.com/ms-henglu/pal/README.md @@ -0,0 +1,60 @@ +# Parsing Azure's Logs - Pal + +---- + +## Introduction + +Pal is a simple tool to parse Terraform Azure provider logs. + +It can output different formats of the request traces, including: + +- Markdown: + + It can output the request traces in Markdown format, which can be used to create a GitHub issue or a forum post. + + The Markdown output supports expanding/collapsing the request traces. This is useful when the request trace is very long. + +- OAV Traffic: + + It can output the request traces in OAV traffic format, which can be used to validate the request traces with [OAV](https://github.com/Azure/oav). + +- AzAPI config: + + It can output the request traces in AzAPI config format, which can be used to reproduce the deployed resources with [AzAPI](https://registry.terraform.io/providers/Azure/azapi/latest). + +## Usage + +```bash +$ pal {path to terraform_log_file} +``` + +Full usage: + +```bash +Usage of pal: + -help + Show help + -i string + Input terraform log file + -m markdown + Output format, allowed values are markdown, `oav` and `azapi` (default "markdown") + -o string + Output directory + -version + Show version +``` + +## Example + +```bash +$ cd ./testdata +$ pal ./input.txt +``` + +Above command will generate a [markdown file named "output.md"](https://github.com/ms-henglu/pal/tree/main/testdata/output.md) in the same working directory. + +## How to install? + +```bash +$ go install github.com/ms-henglu/pal@latest +``` \ No newline at end of file diff --git a/vendor/github.com/ms-henglu/pal/formatter/azapi/formatter.go b/vendor/github.com/ms-henglu/pal/formatter/azapi/formatter.go new file mode 100644 index 00000000..01e3bd98 --- /dev/null +++ b/vendor/github.com/ms-henglu/pal/formatter/azapi/formatter.go @@ -0,0 +1,348 @@ +package azapi + +import ( + "encoding/json" + "fmt" + "log" + "strings" + + pluralize "github.com/gertd/go-pluralize" + "github.com/ms-henglu/pal/formatter" + "github.com/ms-henglu/pal/types" +) + +var _ formatter.Formatter = &AzapiFormatter{} + +var pluralizeClient = pluralize.NewClient() + +type AzapiFormatter struct { + existingResourceSet map[string]bool + labels map[string]bool + valueToKeyMap map[string]string + dataSourceActionUrls map[string]bool +} + +func ignoreKeywords() []string { + return []string{ + "operationresults", + "asyncoperations", + "operationstatuses", + "operationsStatus", + "operations", + } +} + +func (formatter *AzapiFormatter) Format(r types.RequestTrace) string { + if formatter.existingResourceSet == nil { + formatter.existingResourceSet = make(map[string]bool) + formatter.labels = make(map[string]bool) + formatter.valueToKeyMap = make(map[string]string) + formatter.dataSourceActionUrls = make(map[string]bool) + } + + if r.Host != "management.azure.com" { + // ignore the request to other hosts + return "" + } + + if shouldIgnore(r.Url) { + return "" + } + + resourceId := GetId(r.Url) + resourceType, apiVersion := parseResourceTypeApiVersion(r.Url) + + // ignore the request to fetch the resource group's resources + if resourceType == "" && strings.Contains(resourceId, "/resources") { + return "" + } + + var requestBody interface{} + _ = json.Unmarshal([]byte(r.Request.Body), &requestBody) + var responseBody interface{} + if r.Response != nil && r.Response.Body != "" { + _ = json.Unmarshal([]byte(r.Response.Body), &responseBody) + } + def := AzapiDefinition{ + AzureResourceType: resourceType, + ApiVersion: apiVersion, + Body: requestBody, + Output: responseBody, + ResourceId: resourceId, + Method: r.Method, + AdditionalFields: make(map[string]Value), + } + + switch r.Method { + case "PUT": + switch { + case strings.EqualFold(def.AzureResourceType, "Microsoft.KeyVault/vaults/accessPolicies"): + def = formatter.formatAsAzapiActionResource(def) + case IsResourceAction(def.ResourceId): + def = formatter.formatAsAzapiActionResource(def) + case formatter.existingResourceSet[def.ResourceId]: + def = formatter.formatAsAzapiUpdateResource(def) + default: + def = formatter.formatAsAzapiResource(def) + } + case "GET": + if r.StatusCode == 200 && !formatter.existingResourceSet[def.ResourceId] { + if IsResourceAction(def.ResourceId) { + // resource action data source + check := fmt.Sprintf("%s.%s", def.Method, def.ResourceId) + if _, ok := formatter.dataSourceActionUrls[check]; !ok { + def = formatter.formatAsAzapiActionDataSource(def) + formatter.dataSourceActionUrls[check] = true + } else { + return "" + } + } else { + // reading a resource which is created by the service + check := fmt.Sprintf("%s.%s", def.Method, def.ResourceId) + if _, ok := formatter.dataSourceActionUrls[check]; !ok { + def = formatter.formatAsAzapiDataSource(def) + formatter.dataSourceActionUrls[check] = true + formatter.existingResourceSet[def.ResourceId] = true + } else { + return "" + } + } + } else { + return "" + } + case "POST": + if r.Request.Body == "" { + check := fmt.Sprintf("%s.%s", def.Method, def.ResourceId) + if _, ok := formatter.dataSourceActionUrls[check]; !ok { + def = formatter.formatAsAzapiActionDataSource(def) + formatter.dataSourceActionUrls[check] = true + } else { + return "" + } + } else { + def = formatter.formatAsAzapiActionResource(def) + } + case "PATCH": + if !formatter.existingResourceSet[def.ResourceId] { + log.Printf("[WARN] PATCH %s is not a created resource", def.ResourceId) + return "" + } else { + def = formatter.formatAsAzapiActionResource(def) + } + case "DELETE": + if !formatter.existingResourceSet[def.ResourceId] { + log.Printf("[WARN] DELETE %s is not a created resource", def.ResourceId) + } + return "" + default: + return "" + } + + def.Body = formatter.injectReference(def.Body) + + address := fmt.Sprintf("%s.%s.%s", def.Kind, def.ResourceName, def.Label) + if def.Kind == "resource" { + address = fmt.Sprintf("%s.%s", def.ResourceName, def.Label) + } + prefix := fmt.Sprintf(`jsondecode(%s.output)`, address) + formatter.populateReference(prefix, def.Output) + + return def.String() +} + +func shouldIgnore(url string) bool { + resourceType := GetResourceType(url) + if strings.EqualFold(resourceType, "Microsoft.ApiManagement/service/apis/operations") || strings.EqualFold(resourceType, "Microsoft.ApiManagement/service/apis/operations/tags") { + return false + } + for _, v := range ignoreKeywords() { + if strings.Contains(url, v) { + return true + } + } + return false +} + +func (formatter *AzapiFormatter) formatAsAzapiResource(def AzapiDefinition) AzapiDefinition { + formatter.existingResourceSet[def.ResourceId] = true + def.Kind = "resource" + def.ResourceName = "azapi_resource" + def.Label = newUniqueLabel(def.ResourceName, defaultLabel(def.AzureResourceType), &formatter.labels) + + def.AdditionalFields["parent_id"] = formatter.tryAddressOrLiteral(GetParentId(def.ResourceId)) + def.AdditionalFields["name"] = NewLiteralValue(GetName(def.ResourceId)) + if def.Body != nil { + if requestBody, ok := def.Body.(map[string]interface{}); ok && requestBody != nil { + if location := requestBody["location"]; location != nil { + def.AdditionalFields["location"] = NewLiteralValue(location.(string)) + delete(requestBody, "location") + } + if name := requestBody["name"]; name != nil { + delete(requestBody, "name") + } + if strings.EqualFold(def.AzureResourceType, "Microsoft.ManagedIdentity/userAssignedIdentities") { + delete(requestBody, "properties") + } + def.Body = requestBody + } + } + formatter.valueToKeyMap[strings.ToUpper(def.ResourceId)] = fmt.Sprintf("%s.%s.id", def.ResourceName, def.Label) + + return def +} + +func (formatter *AzapiFormatter) formatAsAzapiDataSource(def AzapiDefinition) AzapiDefinition { + def.Kind = "data" + def.ResourceName = "azapi_resource" + def.Label = newUniqueLabel(def.ResourceName, defaultLabel(def.AzureResourceType), &formatter.labels) + + parentId := GetParentId(def.ResourceId) + if parentAddress := formatter.valueToKeyMap[strings.ToUpper(parentId)]; parentAddress != "" { + def.AdditionalFields["parent_id"] = NewReferenceValue(parentAddress) + def.AdditionalFields["name"] = NewLiteralValue(GetName(def.ResourceId)) + } else { + def.AdditionalFields["resource_id"] = NewLiteralValue(def.ResourceId) + } + def.AdditionalFields["response_export_values"] = NewRawValue(`["*"]`) + formatter.valueToKeyMap[strings.ToUpper(def.ResourceId)] = fmt.Sprintf("data.%s.%s.id", def.ResourceName, def.Label) + + return def +} + +func (formatter *AzapiFormatter) formatAsAzapiActionResource(def AzapiDefinition) AzapiDefinition { + def.Kind = "resource" + def.ResourceName = "azapi_resource_action" + + action := "" + if parts := strings.Split(def.ResourceId, "/"); len(parts)%2 == 0 { + action = parts[len(parts)-1] + def.ResourceId = strings.Join(parts[:len(parts)-1], "/") + def.AdditionalFields["action"] = NewLiteralValue(action) + } + + label := action + if label == "" { + label = fmt.Sprintf("%s_%s", strings.ToLower(def.Method), defaultLabel(def.AzureResourceType)) + } + def.Label = newUniqueLabel(def.ResourceName, label, &formatter.labels) + + def.AdditionalFields["resource_id"] = formatter.tryAddressOrLiteral(def.ResourceId) + if def.Method != "POST" { + def.AdditionalFields["method"] = NewLiteralValue(def.Method) + } + + return def +} + +func (formatter *AzapiFormatter) formatAsAzapiActionDataSource(def AzapiDefinition) AzapiDefinition { + def.Kind = "data" + def.ResourceName = "azapi_resource_action" + + action := "" + if parts := strings.Split(def.ResourceId, "/"); len(parts)%2 == 0 { + action = parts[len(parts)-1] + def.ResourceId = strings.Join(parts[:len(parts)-1], "/") + def.AdditionalFields["action"] = NewLiteralValue(action) + } + + label := action + if label == "" { + label = fmt.Sprintf("%s_%s", strings.ToLower(def.Method), defaultLabel(def.AzureResourceType)) + } + def.Label = newUniqueLabel(def.ResourceName, label, &formatter.labels) + + def.AdditionalFields["resource_id"] = formatter.tryAddressOrLiteral(def.ResourceId) + if def.Method != "POST" { + def.AdditionalFields["method"] = NewLiteralValue(def.Method) + } + + return def +} + +func (formatter *AzapiFormatter) formatAsAzapiUpdateResource(def AzapiDefinition) AzapiDefinition { + def.Kind = "resource" + def.ResourceName = "azapi_update_resource" + + def.Label = newUniqueLabel(def.ResourceName, fmt.Sprintf("update_%s", defaultLabel(def.AzureResourceType)), &formatter.labels) + + parentId := GetParentId(def.ResourceId) + if resourceIdAddress := formatter.valueToKeyMap[strings.ToUpper(def.ResourceId)]; resourceIdAddress != "" { + def.AdditionalFields["resource_id"] = NewReferenceValue(resourceIdAddress) + } else if parentAddress := formatter.valueToKeyMap[strings.ToUpper(parentId)]; parentAddress != "" { + def.AdditionalFields["parent_id"] = NewReferenceValue(parentAddress) + def.AdditionalFields["name"] = NewLiteralValue(GetName(def.ResourceId)) + } else { + def.AdditionalFields["resource_id"] = NewLiteralValue(def.ResourceId) + } + + if def.Body != nil { + bodyMap, ok := def.Body.(map[string]interface{}) + if ok && bodyMap != nil { + delete(bodyMap, "id") + delete(bodyMap, "name") + delete(bodyMap, "type") + } + } + + return def +} + +func (formatter *AzapiFormatter) tryAddressOrLiteral(resourceId string) Value { + if address := formatter.valueToKeyMap[strings.ToUpper(resourceId)]; address != "" { + return NewReferenceValue(address) + } + return NewLiteralValue(resourceId) +} + +func (formatter *AzapiFormatter) injectReference(raw interface{}) interface{} { + if raw == nil { + return nil + } + switch value := raw.(type) { + case map[string]interface{}: + out := make(map[string]interface{}) + for k, v := range value { + key := formatter.injectReference(k) + out[key.(string)] = formatter.injectReference(v) + } + return out + case []interface{}: + for i, v := range value { + value[i] = formatter.injectReference(v) + } + return value + case string: + if address := formatter.valueToKeyMap[strings.ToUpper(value)]; address != "" { + return fmt.Sprintf("${%s}", address) + } else { + return value + } + default: + return value + } +} + +func (formatter *AzapiFormatter) populateReference(prefix string, raw interface{}) { + if raw == nil { + return + } + switch value := raw.(type) { + case map[string]interface{}: + for k, v := range value { + formatter.populateReference(fmt.Sprintf("%s.%s", prefix, k), v) + } + case []interface{}: + for i, v := range value { + formatter.populateReference(fmt.Sprintf("%s[%d]", prefix, i), v) + } + case string: + if len(value) < 32 { + // if the string is too short, it's probably not a resource id + return + } + if _, ok := formatter.valueToKeyMap[strings.ToUpper(value)]; !ok { + formatter.valueToKeyMap[strings.ToUpper(value)] = prefix + } + } + +} diff --git a/vendor/github.com/ms-henglu/pal/formatter/azapi/hcl/marshal.go b/vendor/github.com/ms-henglu/pal/formatter/azapi/hcl/marshal.go new file mode 100644 index 00000000..7d25674e --- /dev/null +++ b/vendor/github.com/ms-henglu/pal/formatter/azapi/hcl/marshal.go @@ -0,0 +1,64 @@ +package hcl + +import ( + "fmt" + "sort" + "strings" +) + +func MarshalIndent(input interface{}, prefix, indent string) string { + if input == nil { + return "null" + } + switch i := input.(type) { + case map[string]interface{}: + keys := make([]string, 0) + for key := range i { + keys = append(keys, key) + } + sort.Strings(keys) + content := "" + for _, key := range keys { + value := i[key] + wrapKey := key + if strings.HasPrefix(key, "${") && strings.HasSuffix(key, "}") { + wrapKey = fmt.Sprintf("(%s)", key[2:len(key)-1]) + } else if needWrapKey(key) { + wrapKey = fmt.Sprintf(`"%s"`, key) + } + content += fmt.Sprintf("%s%s = %s\n", prefix+indent, wrapKey, MarshalIndent(value, prefix+indent, indent)) + } + return fmt.Sprintf("{\n%s%s}", content, prefix) + case []interface{}: + content := "" + for _, value := range i { + content += fmt.Sprintf("%s%s,\n", prefix+indent, MarshalIndent(value, prefix+indent, indent)) + } + return fmt.Sprintf("[\n%s%s]", content, prefix) + case string: + if strings.HasPrefix(i, "${") && strings.HasSuffix(i, "}") { + return i[2 : len(i)-1] + } + i = strings.ReplaceAll(i, "\\", "\\\\") // escape backslashes + i = strings.ReplaceAll(i, "\n", "\\n") + i = strings.ReplaceAll(i, "\t", "\\t") + i = strings.ReplaceAll(i, "\r", "\\r") + i = strings.ReplaceAll(i, "\"", "\\\"") + return fmt.Sprintf(`"%s"`, i) + default: + return fmt.Sprintf("%v", i) + } +} + +func needWrapKey(input string) bool { + if len(input) == 0 { + return false + } + if strings.Contains(input, ".") || strings.Contains(input, "/") { + return true + } + if input[0] == '$' { + return true + } + return input[0] >= '0' && input[0] <= '9' +} diff --git a/vendor/github.com/ms-henglu/pal/formatter/azapi/types.go b/vendor/github.com/ms-henglu/pal/formatter/azapi/types.go new file mode 100644 index 00000000..7a567174 --- /dev/null +++ b/vendor/github.com/ms-henglu/pal/formatter/azapi/types.go @@ -0,0 +1,101 @@ +package azapi + +import ( + "fmt" + + "github.com/ms-henglu/pal/formatter/azapi/hcl" +) + +type AzapiDefinition struct { + Kind string // resource or data + ResourceName string // azapi_resource, azapi_update_resource, azapi_resource_action + Label string // example: test + AzureResourceType string // example: Microsoft.Network/virtualNetworks + ApiVersion string // example: 2020-06-01 + Body interface{} + Output interface{} + ResourceId string + Method string + AdditionalFields map[string]Value +} + +func (def AzapiDefinition) String() string { + expressions := fmt.Sprintf(` type = "%[1]s@%[2]s"`, def.AzureResourceType, def.ApiVersion) + fields := []string{"resource_id", "parent_id", "name", "location", "action", "method"} + for _, field := range fields { + if value, ok := def.AdditionalFields[field]; ok { + expressions += fmt.Sprintf(` + %[1]s = %[2]s`, field, value) + } + } + if def.Body != nil { + bodyMap, ok := def.Body.(map[string]interface{}) + if ok { + tagRaw, ok := bodyMap["tags"] + if ok && tagRaw != nil { + if tagMap, ok := tagRaw.(map[string]interface{}); ok && len(tagMap) == 0 { + delete(bodyMap, "tags") + } + } + } + if len(bodyMap) > 0 { + expressions += fmt.Sprintf(` + body = jsonencode(%[1]s)`, hcl.MarshalIndent(bodyMap, " ", " ")) + } + } + return fmt.Sprintf( + `%[1]s "%[2]s" "%[3]s" { +%[4]s +} +`, def.Kind, def.ResourceName, def.Label, expressions) +} + +type Value interface { + String() string +} + +type RawValue struct { + Raw string +} + +func (v RawValue) String() string { + return v.Raw +} + +func NewRawValue(raw string) RawValue { + return RawValue{ + Raw: raw, + } +} + +type ReferenceValue struct { + Reference string +} + +func (v ReferenceValue) String() string { + return v.Reference +} + +func NewReferenceValue(reference string) ReferenceValue { + return ReferenceValue{ + Reference: reference, + } +} + +type LiteralValue struct { + Literal string +} + +func (v LiteralValue) String() string { + return fmt.Sprintf(`"%s"`, v.Literal) +} + +func NewLiteralValue(literal string) LiteralValue { + return LiteralValue{ + Literal: literal, + } +} + +var _ Value = &RawValue{} +var _ Value = &ReferenceValue{} +var _ Value = &LiteralValue{} diff --git a/vendor/github.com/ms-henglu/pal/formatter/azapi/utils.go b/vendor/github.com/ms-henglu/pal/formatter/azapi/utils.go new file mode 100644 index 00000000..8954f741 --- /dev/null +++ b/vendor/github.com/ms-henglu/pal/formatter/azapi/utils.go @@ -0,0 +1,121 @@ +package azapi + +import ( + "fmt" + "net/url" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" +) + +func parseResourceTypeApiVersion(input string) (string, string) { + idUrl, err := url.Parse(input) + if err != nil { + return "", "" + } + apiVersion := idUrl.Query().Get("api-version") + resourceType := GetResourceType(idUrl.Path) + return resourceType, apiVersion +} + +func GetResourceType(id string) string { + idURL, err := url.ParseRequestURI(id) + if err != nil { + return "" + } + + path := idURL.Path + + path = strings.TrimPrefix(path, "/") + path = strings.TrimSuffix(path, "/") + + components := strings.Split(path, "/") + resourceType := "" + provider := "" + for current := 0; current < len(components)-1; current += 2 { + key := components[current] + value := "" + value = components[current+1] + + // Check key/value for empty strings. + if key == "" || value == "" { + return "" + } + + if key == "providers" { + provider = value + resourceType = provider + } else if len(provider) > 0 { + resourceType += "/" + key + } + } + if resourceType == "" { + resourceId, err := arm.ParseResourceID(id) + if err != nil { + return "" + } + return resourceId.ResourceType.String() + } + return resourceType +} + +// newLabel returns an unique label for a resource type +func newUniqueLabel(prefix string, label string, labels *map[string]bool) string { + check := fmt.Sprintf("%s.%s", prefix, label) + _, ok := (*labels)[check] + if !ok { + (*labels)[check] = true + return label + } + for i := 2; i <= 100; i++ { + newLabel := fmt.Sprintf("%s%d", label, i) + check = fmt.Sprintf("%s.%s", prefix, newLabel) + _, ok := (*labels)[check] + if !ok { + (*labels)[check] = true + return newLabel + } + } + return label +} + +func defaultLabel(resourceType string) string { + parts := strings.Split(resourceType, "/") + label := "test" + if len(parts) != 0 { + label = parts[len(parts)-1] + label = pluralizeClient.Singular(label) + } + return label +} + +func GetName(id string) string { + resourceId, err := arm.ParseResourceID(id) + if err != nil { + return "" + } + return resourceId.Name +} + +func GetParentId(id string) string { + resourceId, err := arm.ParseResourceID(id) + if err != nil { + return "" + } + if resourceId.Parent.ResourceType.String() == arm.TenantResourceType.String() { + return "/" + } + return resourceId.Parent.String() +} + +func GetId(input string) string { + idUrl, err := url.Parse(input) + if err != nil { + return "" + } + return idUrl.Path +} + +func IsResourceAction(resourceId string) bool { + return len(strings.Split(resourceId, "/"))%2 == 0 +} diff --git a/vendor/github.com/ms-henglu/pal/formatter/base.go b/vendor/github.com/ms-henglu/pal/formatter/base.go new file mode 100644 index 00000000..5d822db8 --- /dev/null +++ b/vendor/github.com/ms-henglu/pal/formatter/base.go @@ -0,0 +1,9 @@ +package formatter + +import ( + "github.com/ms-henglu/pal/types" +) + +type Formatter interface { + Format(r types.RequestTrace) string +} diff --git a/vendor/github.com/ms-henglu/pal/formatter/markdown.go b/vendor/github.com/ms-henglu/pal/formatter/markdown.go new file mode 100644 index 00000000..f1f85e08 --- /dev/null +++ b/vendor/github.com/ms-henglu/pal/formatter/markdown.go @@ -0,0 +1,94 @@ +package formatter + +import ( + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/ms-henglu/pal/types" + "github.com/ms-henglu/pal/utils" +) + +var _ Formatter = MarkdownFormatter{} + +type MarkdownFormatter struct { +} + +func (m MarkdownFormatter) Format(r types.RequestTrace) string { + content := markdownTemplate + content = strings.ReplaceAll(content, "{Time}", r.TimeStamp.Format("15:04:05")) + content = strings.ReplaceAll(content, "{Method}", r.Method) + content = strings.ReplaceAll(content, "{Host}", r.Host) + urlStr := r.Url + parsedUrl, err := url.Parse(r.Url) + if err == nil { + urlStr = parsedUrl.Path + if value := parsedUrl.Query()["api-version"]; len(value) > 0 { + urlStr += "?api-version=" + value[0] + } + } + content = strings.ReplaceAll(content, "{Url}", urlStr) + content = strings.ReplaceAll(content, "{StatusCode}", fmt.Sprintf("%d", r.StatusCode)) + content = strings.ReplaceAll(content, "{StatusMessage}", http.StatusText(r.StatusCode)) + content = strings.ReplaceAll(content, "{RequestHeaders}", m.formatHeaders(r.Request.Headers)) + content = strings.ReplaceAll(content, "{RequestBody}", utils.JsonPretty(r.Request.Body)) + if r.Response == nil { + content = strings.ReplaceAll(content, "{ResponseHeaders}", "") + content = strings.ReplaceAll(content, "{ResponseBody}", "") + return content + } + content = strings.ReplaceAll(content, "{ResponseHeaders}", m.formatHeaders(r.Response.Headers)) + content = strings.ReplaceAll(content, "{ResponseBody}", utils.JsonPretty(r.Response.Body)) + return content +} + +func (m MarkdownFormatter) formatHeaders(headers map[string]string) string { + var content string + for k, v := range headers { + content += fmt.Sprintf("| %s | %s |\n", k, v) + } + return content +} + +const markdownTemplate = ` +##### +
+ + {Time} {Method} {Host} {Url} {StatusCode} + +
+
+ Request + +| Header | Value | +| ----- | ----- | +{RequestHeaders} + +Request Body: +` + "```" + `json +{RequestBody} +` + "```" + ` + +
+
+ Response + + **Response Status: {StatusCode} {StatusMessage}** + +| Header | Value | +| ----- | ----- | +{ResponseHeaders} + +Response Body: +` + "```" + `json +{ResponseBody} +` + "```" + ` + +
+
+
+ +----- + +` diff --git a/vendor/github.com/ms-henglu/pal/formatter/oav_traffic.go b/vendor/github.com/ms-henglu/pal/formatter/oav_traffic.go new file mode 100644 index 00000000..73609a32 --- /dev/null +++ b/vendor/github.com/ms-henglu/pal/formatter/oav_traffic.go @@ -0,0 +1,73 @@ +package formatter + +import ( + "encoding/json" + "fmt" + + "github.com/ms-henglu/pal/types" +) + +var _ Formatter = OavTrafficFormatter{} + +type OavTrafficFormatter struct { +} + +type OavTraffic struct { + LiveRequest LiveRequest `json:"liveRequest"` + LiveResponse LiveResponse `json:"liveResponse"` +} + +type LiveRequest struct { + Headers map[string]string `json:"headers"` + Method string `json:"method"` + Url string `json:"url"` + Body interface{} `json:"body"` +} + +type LiveResponse struct { + StatusCode string `json:"statusCode"` + Headers map[string]string `json:"headers"` + Body interface{} `json:"body"` +} + +func (o OavTrafficFormatter) Format(r types.RequestTrace) string { + var requestBody interface{} + requestHeaders := make(map[string]string) + if r.Request != nil { + err := json.Unmarshal([]byte(r.Request.Body), &requestBody) + if err != nil { + requestBody = nil + } + requestHeaders = r.Request.Headers + } + + var responseBody interface{} + responseHeaders := make(map[string]string) + if r.Response != nil { + err := json.Unmarshal([]byte(r.Response.Body), &responseBody) + if err != nil { + responseBody = nil + } + responseHeaders = r.Response.Headers + } + + out := OavTraffic{ + LiveRequest: LiveRequest{ + Headers: requestHeaders, + Method: r.Method, + Url: r.Url, + Body: requestBody, + }, + LiveResponse: LiveResponse{ + StatusCode: fmt.Sprintf("%d", r.StatusCode), + Headers: responseHeaders, + Body: responseBody, + }, + } + + content, err := json.MarshalIndent(out, "", " ") + if err != nil { + return "" + } + return string(content) +} diff --git a/vendor/github.com/ms-henglu/pal/main.go b/vendor/github.com/ms-henglu/pal/main.go new file mode 100644 index 00000000..84701361 --- /dev/null +++ b/vendor/github.com/ms-henglu/pal/main.go @@ -0,0 +1,140 @@ +package main + +import ( + "flag" + "fmt" + "log" + "os" + "path" + "strings" + + "github.com/ms-henglu/pal/formatter" + "github.com/ms-henglu/pal/formatter/azapi" + "github.com/ms-henglu/pal/trace" +) + +const version = "0.4.0" + +var showHelp = flag.Bool("help", false, "Show help") +var showVersion = flag.Bool("version", false, "Show version") + +func main() { + input := "" + output := "" + mode := "" + + flag.StringVar(&input, "i", "", "Input terraform log file") + flag.StringVar(&output, "o", "", "Output directory") + flag.StringVar(&mode, "m", "markdown", "Output format, allowed values are `markdown`, `oav` and `azapi`") + + // backward compatibility, the first argument is the input file + if len(os.Args) == 2 { + if _, err := os.Stat(os.Args[1]); err == nil { + input = os.Args[1] + mode = "markdown" + } + } + if input == "" { + flag.Parse() + if *showHelp { + flag.Usage() + os.Exit(0) + } + if *showVersion { + fmt.Println(version) + os.Exit(0) + } + } + if input == "" { + flag.Usage() + log.Fatalf("[ERROR] input file is required") + } + + if output == "" { + output = path.Dir(input) + } + + log.Printf("[INFO] input file: %s", input) + log.Printf("[INFO] output directory: %s", output) + log.Printf("[INFO] output format: %s", mode) + + traces, err := trace.RequestTracesFromFile(input) + if err != nil { + log.Fatalf("[ERROR] failed to parse request traces: %v", err) + } + + for _, t := range traces { + out := trace.VerifyRequestTrace(t) + if len(out) > 0 { + log.Printf("[WARN] verification failed: url %s\n%s", t.Url, strings.Join(out, "\n")) + } + } + + switch mode { + case "oav": + format := formatter.OavTrafficFormatter{} + index := 0 + for _, t := range traces { + out := format.Format(t) + index = index + 1 + outputPath := path.Join(output, fmt.Sprintf("trace-%d.json", index)) + if err := os.WriteFile(outputPath, []byte(out), 0644); err != nil { + log.Fatalf("[ERROR] failed to write file: %v", err) + } + log.Printf("[INFO] output file: %s", outputPath) + } + case "markdown": + content := markdownPrefix + format := formatter.MarkdownFormatter{} + for _, t := range traces { + content += format.Format(t) + } + outputPath := path.Clean(path.Join(output, "output.md")) + if err := os.WriteFile(outputPath, []byte(content), 0644); err != nil { + log.Fatalf("[ERROR] failed to write file: %v", err) + } + log.Printf("[INFO] output file: %s", outputPath) + case "azapi": + content := azapiPrefix + format := azapi.AzapiFormatter{} + for _, t := range traces { + if res := format.Format(t); res != "" { + content += res + content += "\n" + } + } + outputPath := path.Clean(path.Join(output, "pal-main.tf")) + if err := os.WriteFile(outputPath, []byte(content), 0644); err != nil { + log.Fatalf("[ERROR] failed to write file: %v", err) + } + log.Printf("[INFO] output file: %s", outputPath) + default: + log.Fatalf("[ERROR] unsupported output format: %s", mode) + } + +} + +const markdownPrefix = ` + +` + +const azapiPrefix = ` +terraform { + required_providers { + azapi = { + source = "Azure/azapi" + } + } +} + +provider "azapi" { + skip_provider_registration = false +} + +` diff --git a/vendor/github.com/ms-henglu/pal/provider/azapi.go b/vendor/github.com/ms-henglu/pal/provider/azapi.go new file mode 100644 index 00000000..c474b180 --- /dev/null +++ b/vendor/github.com/ms-henglu/pal/provider/azapi.go @@ -0,0 +1,98 @@ +package provider + +import ( + "encoding/json" + "fmt" + "net/url" + "regexp" + "strings" + + "github.com/ms-henglu/pal/rawlog" + "github.com/ms-henglu/pal/types" +) + +var _ Provider = AzAPIProvider{} + +var r = regexp.MustCompile(`Live traffic: (.+): timestamp`) + +type AzAPIProvider struct { +} + +func (a AzAPIProvider) IsTrafficTrace(l rawlog.RawLog) bool { + return l.Level == "DEBUG" && strings.Contains(l.Message, "Live traffic:") +} + +func (a AzAPIProvider) ParseTraffic(l rawlog.RawLog) (*types.RequestTrace, error) { + matches := r.FindAllStringSubmatch(l.Message, -1) + if len(matches) == 0 || len(matches[0]) != 2 { + return nil, fmt.Errorf("failed to parse request trace, no matches found") + } + trafficJson := matches[0][1] + var liveTraffic traffic + err := json.Unmarshal([]byte(trafficJson), &liveTraffic) + if err != nil { + return nil, fmt.Errorf("failed to parse request trace, %v", err) + } + parsedUrl, err := url.Parse(liveTraffic.LiveRequest.Url) + if err != nil { + return nil, fmt.Errorf("failed to parse request trace, %v", err) + } + + if liveTraffic.LiveRequest.Headers == nil { + liveTraffic.LiveRequest.Headers = map[string]string{} + } + if liveTraffic.LiveResponse.Headers == nil { + liveTraffic.LiveResponse.Headers = map[string]string{} + } + + return &types.RequestTrace{ + TimeStamp: l.TimeStamp, + Method: liveTraffic.LiveRequest.Method, + Host: parsedUrl.Host, + Url: parsedUrl.Path + "?" + parsedUrl.RawQuery, + StatusCode: liveTraffic.LiveResponse.StatusCode, + Provider: "azapi", + Request: &types.HttpRequest{ + Headers: liveTraffic.LiveRequest.Headers, + Body: liveTraffic.LiveRequest.Body, + }, + Response: &types.HttpResponse{ + Headers: liveTraffic.LiveResponse.Headers, + Body: liveTraffic.LiveResponse.Body, + }, + }, nil +} + +func (a AzAPIProvider) IsRequestTrace(l rawlog.RawLog) bool { + return false +} + +func (a AzAPIProvider) IsResponseTrace(l rawlog.RawLog) bool { + return false +} + +func (a AzAPIProvider) ParseRequest(l rawlog.RawLog) (*types.RequestTrace, error) { + return nil, fmt.Errorf("not implemented") +} + +func (a AzAPIProvider) ParseResponse(l rawlog.RawLog) (*types.RequestTrace, error) { + return nil, fmt.Errorf("not implemented") +} + +type traffic struct { + LiveRequest liveRequest `json:"request"` + LiveResponse liveResponse `json:"response"` +} + +type liveRequest struct { + Headers map[string]string `json:"headers"` + Method string `json:"method"` + Url string `json:"url"` + Body string `json:"body"` +} + +type liveResponse struct { + StatusCode int `json:"statusCode"` + Headers map[string]string `json:"headers"` + Body string `json:"body"` +} diff --git a/vendor/github.com/ms-henglu/pal/provider/azuread.go b/vendor/github.com/ms-henglu/pal/provider/azuread.go new file mode 100644 index 00000000..7384ed90 --- /dev/null +++ b/vendor/github.com/ms-henglu/pal/provider/azuread.go @@ -0,0 +1,121 @@ +package provider + +import ( + "fmt" + "net/url" + "regexp" + "strings" + + "github.com/ms-henglu/pal/rawlog" + "github.com/ms-henglu/pal/types" + "github.com/ms-henglu/pal/utils" +) + +var _ Provider = AzureADProvider{} +var statusCodeRegex = regexp.MustCompile(`HTTP/\d.\d\s(\d{3})\s.+`) + +type AzureADProvider struct { +} + +func (a AzureADProvider) IsTrafficTrace(l rawlog.RawLog) bool { + return false +} + +func (a AzureADProvider) ParseTraffic(l rawlog.RawLog) (*types.RequestTrace, error) { + return nil, fmt.Errorf("not implemented") +} + +func (a AzureADProvider) IsRequestTrace(l rawlog.RawLog) bool { + return l.Level == "INFO" && strings.Contains(l.Message, "============================ Begin AzureAD Request") +} + +func (a AzureADProvider) IsResponseTrace(l rawlog.RawLog) bool { + return l.Level == "INFO" && strings.Contains(l.Message, "============================ Begin AzureAD Response") +} + +func (a AzureADProvider) ParseRequest(l rawlog.RawLog) (*types.RequestTrace, error) { + urlLine := "" + headers := make(map[string]string) + method := "" + url := "" + for _, line := range strings.Split(l.Message, "\n") { + switch { + case line == "" || strings.Contains(line, "======"): + continue + case strings.Contains(line, ": "): + key, value, err := utils.ParseHeader(line) + if err != nil { + return nil, err + } + headers[key] = value + default: + urlLine = line + if parts := strings.Split(urlLine, " "); len(parts) == 3 { + method = parts[0] + url = parts[1] + } + } + } + + return &types.RequestTrace{ + TimeStamp: l.TimeStamp, + Url: utils.NormalizeUrlPath(url), + Method: method, + Host: headers["Host"], + Provider: "azuread", + Request: &types.HttpRequest{ + Headers: headers, + }, + }, nil +} + +func (a AzureADProvider) ParseResponse(l rawlog.RawLog) (*types.RequestTrace, error) { + headers := make(map[string]string) + statusCode := 0 + method := "" + rawUrl := "" + host := "" + body := "" + for _, line := range strings.Split(l.Message, "\n") { + switch { + case line == "" || strings.Contains(line, "======"): + continue + case statusCodeRegex.FindAllStringSubmatch(line, -1) != nil: + matches := statusCodeRegex.FindAllStringSubmatch(line, -1) + if len(matches) > 0 && len(matches[0]) == 2 { + fmt.Sscanf(matches[0][1], "%d", &statusCode) + } + case utils.IsJson(line): + body = line + case strings.Contains(line, ": "): + key, value, err := utils.ParseHeader(line) + if err != nil { + return nil, err + } + headers[key] = value + default: + parts := strings.Split(line, " ") + if len(parts) == 2 { + method = parts[0] + parsedUrl, err := url.Parse(parts[1]) + if err == nil { + host = parsedUrl.Host + rawUrl = fmt.Sprintf("%s?%s", parsedUrl.Path, parsedUrl.RawQuery) + } + } + } + } + + return &types.RequestTrace{ + TimeStamp: l.TimeStamp, + Url: utils.NormalizeUrlPath(rawUrl), + Method: method, + Host: host, + StatusCode: statusCode, + Provider: "azuread", + Response: &types.HttpResponse{ + Headers: headers, + Body: body, + }, + }, nil +} diff --git a/vendor/github.com/ms-henglu/pal/provider/azurerm.go b/vendor/github.com/ms-henglu/pal/provider/azurerm.go new file mode 100644 index 00000000..4cb74979 --- /dev/null +++ b/vendor/github.com/ms-henglu/pal/provider/azurerm.go @@ -0,0 +1,166 @@ +package provider + +import ( + "fmt" + "net/url" + "strings" + + "github.com/ms-henglu/pal/rawlog" + "github.com/ms-henglu/pal/types" + "github.com/ms-henglu/pal/utils" +) + +var _ Provider = AzureRMProvider{} + +type AzureRMProvider struct { +} + +func (a AzureRMProvider) IsTrafficTrace(l rawlog.RawLog) bool { + return false +} + +func (a AzureRMProvider) ParseTraffic(l rawlog.RawLog) (*types.RequestTrace, error) { + return nil, fmt.Errorf("not implemented") +} + +func (a AzureRMProvider) IsRequestTrace(l rawlog.RawLog) bool { + return l.Level == "DEBUG" && strings.Contains(l.Message, "AzureRM Request:") +} + +func (a AzureRMProvider) IsResponseTrace(l rawlog.RawLog) bool { + return l.Level == "DEBUG" && strings.Contains(l.Message, "AzureRM Response for") +} + +func (a AzureRMProvider) ParseRequest(l rawlog.RawLog) (*types.RequestTrace, error) { + urlPath := "" + method := "" + headers := make(map[string]string) + body := "" + + lines := strings.Split(l.Message, "\n") + i := 0 + foundBodySegment := false + for ; i < len(lines); i++ { + line := lines[i] + switch { + case strings.TrimSpace(line) == "": + foundBodySegment = true + case strings.Contains(line, ": "): + key, value, err := utils.ParseHeader(line) + if strings.HasPrefix(key, "provider.terraform-provider-azurerm") { + continue + } + if key == "AzureRM Request" { + continue + } + if err != nil { + return nil, err + } + headers[key] = value + default: + if parts := strings.Split(line, " "); len(parts) == 3 { + method = parts[0] + urlPath = parts[1] + } + } + if foundBodySegment { + break + } + } + + if i+1 < len(lines) { + line := strings.Join(lines[i+1:], "\n") + if strings.Contains(line, ": timestamp") { + index := strings.LastIndex(line, ": timestamp") + if utils.IsJson(line[0:index]) { + body = line[0:index] + } else { + lineTrimTimestamp := line[0:index] + key, value, err := utils.ParseHeader(lineTrimTimestamp) + if err == nil { + headers[key] = value + } + } + } else { + body = line + } + } + return &types.RequestTrace{ + TimeStamp: l.TimeStamp, + Url: utils.NormalizeUrlPath(urlPath), + Method: method, + Host: headers["Host"], + Provider: "azurerm", + Request: &types.HttpRequest{ + Headers: headers, + Body: body, + }, + }, nil +} + +func (a AzureRMProvider) ParseResponse(l rawlog.RawLog) (*types.RequestTrace, error) { + urlPath := "" + host := "" + method := "" // TODO: this is not available in the response + body := "" + headers := make(map[string]string) + statusCode := 0 + + lines := strings.Split(l.Message, "\n") + i := 0 + foundBodySegment := false + for ; i < len(lines); i++ { + line := lines[i] + switch { + case strings.TrimSpace(line) == "": + foundBodySegment = true + case strings.Contains(line, "AzureRM Response for "): + urlLine := line[strings.Index(line, "AzureRM Response for ")+len("AzureRM Response for "):] + urlLine = strings.Trim(urlLine, " \n\r:") + parsedUrl, err := url.Parse(urlLine) + if err != nil { + return nil, err + } + host = parsedUrl.Host + urlPath = fmt.Sprintf("%s?%s", parsedUrl.Path, parsedUrl.RawQuery) + case strings.Contains(line, ": "): + key, value, err := utils.ParseHeader(line) + if err != nil { + return nil, err + } + headers[key] = value + default: + if matches := statusCodeRegex.FindAllStringSubmatch(line, -1); len(matches) > 0 && len(matches[0]) == 2 { + fmt.Sscanf(matches[0][1], "%d", &statusCode) + } + } + if foundBodySegment { + break + } + } + + if i+1 < len(lines) { + line := strings.Join(lines[i+1:], "\n") + if strings.Contains(line, ": timestamp") { + index := strings.LastIndex(line, ": timestamp") + if utils.IsJson(line[0:index]) { + body = line[0:index] + } + } else { + body = line + } + } + + return &types.RequestTrace{ + TimeStamp: l.TimeStamp, + Url: utils.NormalizeUrlPath(urlPath), + Host: host, + Method: method, + StatusCode: statusCode, + Provider: "azurerm", + Response: &types.HttpResponse{ + Headers: headers, + Body: body, + }, + }, nil +} diff --git a/vendor/github.com/ms-henglu/pal/provider/base.go b/vendor/github.com/ms-henglu/pal/provider/base.go new file mode 100644 index 00000000..34e82457 --- /dev/null +++ b/vendor/github.com/ms-henglu/pal/provider/base.go @@ -0,0 +1,15 @@ +package provider + +import ( + "github.com/ms-henglu/pal/rawlog" + "github.com/ms-henglu/pal/types" +) + +type Provider interface { + IsTrafficTrace(l rawlog.RawLog) bool + IsRequestTrace(l rawlog.RawLog) bool + IsResponseTrace(l rawlog.RawLog) bool + ParseTraffic(l rawlog.RawLog) (*types.RequestTrace, error) + ParseRequest(l rawlog.RawLog) (*types.RequestTrace, error) + ParseResponse(l rawlog.RawLog) (*types.RequestTrace, error) +} diff --git a/vendor/github.com/ms-henglu/pal/rawlog/raw_log.go b/vendor/github.com/ms-henglu/pal/rawlog/raw_log.go new file mode 100644 index 00000000..72beb99d --- /dev/null +++ b/vendor/github.com/ms-henglu/pal/rawlog/raw_log.go @@ -0,0 +1,41 @@ +package rawlog + +import ( + "fmt" + "regexp" + "strings" + "time" +) + +type RawLog struct { + TimeStamp time.Time + Level string + Message string +} + +var regLayoutMap = map[*regexp.Regexp]string{ + regexp.MustCompile(`([\d+.:T\-]{28})\s\[([A-Z]+)]`): "2006-01-02T15:04:05.999-0700", + regexp.MustCompile(`([\d+.:T\- ]{19})\s\[([A-Z]+)]`): "2006-01-02 15:04:05", + regexp.MustCompile(`([\d+.:T/ ]{19})\s\[([A-Z]+)]`): "2006/01/02 15:04:05", +} + +func NewRawLog(message string) (*RawLog, error) { + for reg, layout := range regLayoutMap { + matches := reg.FindAllStringSubmatch(message, -1) + if len(matches) == 0 || len(matches[0]) != 3 { + continue + } + t, err := time.Parse(layout, matches[0][1]) + if err != nil { + continue + } + m := message[len(matches[0][0]):] + m = strings.Trim(m, " \n") + return &RawLog{ + TimeStamp: t, + Level: matches[0][2], + Message: m, + }, nil + } + return nil, fmt.Errorf("failed to parse log message: %s", message) +} diff --git a/vendor/github.com/ms-henglu/pal/trace/trace.go b/vendor/github.com/ms-henglu/pal/trace/trace.go new file mode 100644 index 00000000..d2cc339e --- /dev/null +++ b/vendor/github.com/ms-henglu/pal/trace/trace.go @@ -0,0 +1,165 @@ +package trace + +import ( + "fmt" + "log" + "os" + "regexp" + "strconv" + + "github.com/ms-henglu/pal/provider" + "github.com/ms-henglu/pal/rawlog" + "github.com/ms-henglu/pal/types" + "github.com/ms-henglu/pal/utils" +) + +var providers = []provider.Provider{ + provider.AzureADProvider{}, + provider.AzureRMProvider{}, + provider.AzAPIProvider{}, +} + +var providerUrlRegex = regexp.MustCompile(`/subscriptions/[a-zA-Z\d\-]+/providers`) + +func RequestTracesFromFile(input string) ([]types.RequestTrace, error) { + data, err := os.ReadFile(input) + if err != nil { + return nil, fmt.Errorf("failed to read input file: %v", err) + } + logRegex := regexp.MustCompile(`([\d+.:T\-/ ]{19,28})\s\[([A-Z]+)]`) + lines := utils.SplitBefore(string(data), logRegex) + log.Printf("[INFO] total lines: %d", len(lines)) + + traces := make([]types.RequestTrace, 0) + for _, line := range lines { + l, err := rawlog.NewRawLog(line) + if err != nil { + log.Printf("[WARN] failed to parse log: %v", err) + } + if l == nil { + continue + } + t, err := NewRequestTrace(*l) + if err == nil { + traces = append(traces, *t) + } + } + requestCount, responseCount := 0, 0 + for _, t := range traces { + if t.Request != nil { + requestCount++ + } + if t.Response != nil { + responseCount++ + } + } + log.Printf("[INFO] total traces: %d", len(traces)) + log.Printf("[INFO] request count: %d", requestCount) + log.Printf("[INFO] response count: %d", responseCount) + + mergedTraces := make([]types.RequestTrace, 0) + for i := 0; i < len(traces); i++ { + // skip GET /subscriptions/******/providers + if traces[i].Method == "GET" && providerUrlRegex.MatchString(traces[i].Url) { + continue + } + + if traces[i].Request != nil && traces[i].Response != nil { + mergedTraces = append(mergedTraces, traces[i]) + continue + } + + if traces[i].Request != nil { + found := false + for j := i + 1; j < len(traces); j++ { + if traces[j].Response == nil || traces[i].Url != traces[j].Url || traces[i].Host != traces[j].Host { + continue + } + found = true + mergedTraces = append(mergedTraces, types.RequestTrace{ + TimeStamp: traces[i].TimeStamp, + Url: traces[i].Url, + Method: traces[i].Method, + Host: traces[i].Host, + StatusCode: traces[j].StatusCode, + Request: traces[i].Request, + Response: traces[j].Response, + }) + break + } + if !found { + log.Printf("[WARN] failed to find response for request: url %s, method %s", traces[i].Url, traces[i].Method) + mergedTraces = append(mergedTraces, traces[i]) + } + } + } + log.Printf("[INFO] merged traces: %d", len(mergedTraces)) + return mergedTraces, nil +} + +func VerifyRequestTrace(t types.RequestTrace) []string { + out := make([]string, 0) + if len(t.Url) == 0 { + out = append(out, "[ERROR] url is empty") + } + if len(t.Host) == 0 { + out = append(out, "[ERROR] host is empty") + } + if len(t.Method) == 0 { + out = append(out, "[ERROR] method is empty") + } + if t.StatusCode == 0 { + out = append(out, "[ERROR] status code is empty") + } + if t.TimeStamp.IsZero() { + out = append(out, "[ERROR] timestamp is empty") + } + switch { + case t.Request == nil: + out = append(out, "[ERROR] request is nil") + case t.Request.Headers == nil: + out = append(out, "[ERROR] request headers is nil") + default: + if contentLength, ok := t.Request.Headers["Content-Length"]; ok { + length, err := strconv.ParseInt(contentLength, 10, 64) + if err != nil { + out = append(out, fmt.Sprintf("[ERROR] failed to parse content length: %v", err)) + } + if length != 0 && len(t.Request.Body) == 0 { + out = append(out, "[ERROR] request body is empty") + } + } + } + switch { + case t.Response == nil: + out = append(out, "[ERROR] response is nil") + case t.Response.Headers == nil: + out = append(out, "[ERROR] response headers is nil") + default: + if contentLength, ok := t.Response.Headers["Content-Length"]; ok { + length, err := strconv.ParseInt(contentLength, 10, 64) + if err != nil { + out = append(out, fmt.Sprintf("[ERROR] failed to parse content length: %v", err)) + } + if length != 0 && len(t.Response.Body) == 0 { + out = append(out, "[ERROR] response body is empty") + } + } + } + return out +} + +func NewRequestTrace(l rawlog.RawLog) (*types.RequestTrace, error) { + for _, p := range providers { + if p.IsTrafficTrace(l) { + return p.ParseTraffic(l) + } + if p.IsRequestTrace(l) { + return p.ParseRequest(l) + } + if p.IsResponseTrace(l) { + return p.ParseResponse(l) + } + } + return nil, fmt.Errorf("TODO: implement other providers") +} diff --git a/vendor/github.com/ms-henglu/pal/types/types.go b/vendor/github.com/ms-henglu/pal/types/types.go new file mode 100644 index 00000000..f4e29b7b --- /dev/null +++ b/vendor/github.com/ms-henglu/pal/types/types.go @@ -0,0 +1,24 @@ +package types + +import "time" + +type RequestTrace struct { + Url string + Method string + Host string + StatusCode int + Provider string + TimeStamp time.Time + Request *HttpRequest + Response *HttpResponse +} + +type HttpRequest struct { + Headers map[string]string + Body string +} + +type HttpResponse struct { + Headers map[string]string + Body string +} diff --git a/vendor/github.com/ms-henglu/pal/utils/utils.go b/vendor/github.com/ms-henglu/pal/utils/utils.go new file mode 100644 index 00000000..6fcc79fc --- /dev/null +++ b/vendor/github.com/ms-henglu/pal/utils/utils.go @@ -0,0 +1,69 @@ +package utils + +import ( + "encoding/json" + "fmt" + "regexp" + "strings" +) + +func IsJson(input string) bool { + var out interface{} + err := json.Unmarshal([]byte(input), &out) + return err == nil +} + +func JsonPretty(input string) string { + var out interface{} + err := json.Unmarshal([]byte(input), &out) + if err != nil { + return input + } + b, err := json.MarshalIndent(out, "", " ") + if err != nil { + return input + } + return string(b) +} + +func SplitBefore(s string, re *regexp.Regexp) []string { + out := make([]string, 0) + is := re.FindAllStringIndex(s, -1) + if len(is) == 0 { + return append(out, s) + } + for i := 0; i < len(is)-1; i++ { + out = append(out, s[is[i][0]:is[i+1][0]]) + } + return append(out, s[is[len(is)-1][0]:]) +} + +func ParseHeader(input string) (string, string, error) { + deliminatorIndex := strings.Index(input, ":") + if deliminatorIndex == -1 { + return "", "", fmt.Errorf("failed to parse header, `:` is not found: %s", input) + } + key := strings.Trim(input[0:deliminatorIndex], " ") + value := input[deliminatorIndex+1:] + if index := strings.LastIndex(value, ": timestamp"); index != -1 { + value = value[0:index] + } + value = strings.Trim(value, " \n\r") + return key, value, nil +} + +func LineAt(input string, index int) string { + lines := strings.Split(input, "\n") + if len(lines) > index { + return lines[index] + } + return input +} + +func NormalizeUrlPath(input string) string { + if !strings.HasPrefix(input, "/") { + input = "/" + input + } + input = strings.ReplaceAll(input, "//", "/") + return input +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 07ea4be6..58feb28b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -419,6 +419,17 @@ github.com/mitchellh/mapstructure # github.com/mitchellh/reflectwalk v1.0.2 ## explicit github.com/mitchellh/reflectwalk +# github.com/ms-henglu/pal v0.4.0 +## explicit; go 1.19 +github.com/ms-henglu/pal +github.com/ms-henglu/pal/formatter +github.com/ms-henglu/pal/formatter/azapi +github.com/ms-henglu/pal/formatter/azapi/hcl +github.com/ms-henglu/pal/provider +github.com/ms-henglu/pal/rawlog +github.com/ms-henglu/pal/trace +github.com/ms-henglu/pal/types +github.com/ms-henglu/pal/utils # github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 ## explicit; go 1.16 github.com/nsf/jsondiff From dd73b06800f96bfbb737a316456312d9ce37ab86 Mon Sep 17 00:00:00 2001 From: Heng Lu Date: Tue, 17 Oct 2023 16:04:48 +0800 Subject: [PATCH 2/2] go mod vendor --- go.mod | 2 +- vendor/github.com/ms-henglu/pal/.gitignore | 3 - .../github.com/ms-henglu/pal/.goreleaser.yml | 54 --- vendor/github.com/ms-henglu/pal/CHANGELOG.md | 40 -- vendor/github.com/ms-henglu/pal/GNUmakefile | 19 - vendor/github.com/ms-henglu/pal/README.md | 60 --- .../pal/formatter/azapi/formatter.go | 348 ------------------ .../pal/formatter/azapi/hcl/marshal.go | 64 ---- .../ms-henglu/pal/formatter/azapi/types.go | 101 ----- .../ms-henglu/pal/formatter/azapi/utils.go | 121 ------ vendor/github.com/ms-henglu/pal/main.go | 140 ------- vendor/modules.txt | 3 - 12 files changed, 1 insertion(+), 954 deletions(-) delete mode 100644 vendor/github.com/ms-henglu/pal/.gitignore delete mode 100644 vendor/github.com/ms-henglu/pal/.goreleaser.yml delete mode 100644 vendor/github.com/ms-henglu/pal/CHANGELOG.md delete mode 100644 vendor/github.com/ms-henglu/pal/GNUmakefile delete mode 100644 vendor/github.com/ms-henglu/pal/README.md delete mode 100644 vendor/github.com/ms-henglu/pal/formatter/azapi/formatter.go delete mode 100644 vendor/github.com/ms-henglu/pal/formatter/azapi/hcl/marshal.go delete mode 100644 vendor/github.com/ms-henglu/pal/formatter/azapi/types.go delete mode 100644 vendor/github.com/ms-henglu/pal/formatter/azapi/utils.go delete mode 100644 vendor/github.com/ms-henglu/pal/main.go diff --git a/go.mod b/go.mod index ba1619c6..ca4cd861 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/hashicorp/terraform-json v0.13.0 github.com/magodo/azure-rest-api-index v0.0.0-20230522080218-497fe558c02f github.com/mitchellh/cli v1.1.2 + github.com/ms-henglu/pal v0.4.0 github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 github.com/sirupsen/logrus v1.9.3 github.com/zclconf/go-cty v1.9.1 @@ -85,7 +86,6 @@ require ( github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/ms-henglu/pal v0.4.0 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/posener/complete v1.1.1 // indirect diff --git a/vendor/github.com/ms-henglu/pal/.gitignore b/vendor/github.com/ms-henglu/pal/.gitignore deleted file mode 100644 index 14bc234b..00000000 --- a/vendor/github.com/ms-henglu/pal/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -.idea/* -.idea -*/.DS_Store diff --git a/vendor/github.com/ms-henglu/pal/.goreleaser.yml b/vendor/github.com/ms-henglu/pal/.goreleaser.yml deleted file mode 100644 index 7740d89c..00000000 --- a/vendor/github.com/ms-henglu/pal/.goreleaser.yml +++ /dev/null @@ -1,54 +0,0 @@ -# Visit https://goreleaser.com for documentation on how to customize this -# behavior. -before: - hooks: - # this is just an example and not a requirement for provider building/publishing - - go mod tidy -builds: - - env: - # goreleaser does not work with CGO, it could also complicate - # usage by users in CI/CD systems like Terraform Cloud where - # they are unable to install libraries. - - CGO_ENABLED=0 - mod_timestamp: '{{ .CommitTimestamp }}' - flags: - - -trimpath - ldflags: - - '-s -w -X main.version={{.Version}} -X main.commit={{.Commit}}' - goos: - - freebsd - - windows - - linux - - darwin - goarch: - - amd64 - - '386' - - arm - - arm64 - ignore: - - goos: darwin - goarch: '386' - binary: '{{ .ProjectName }}' -archives: - - format: zip - name_template: '{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}' -checksum: - name_template: '{{ .ProjectName }}_{{ .Version }}_SHA256SUMS' - algorithm: sha256 -signs: - - artifacts: checksum - args: - # if you are using this in a GitHub action or some other automated pipeline, you - # need to pass the batch flag to indicate its not interactive. - - "--batch" - - "--local-user" - - "{{ .Env.GPG_FINGERPRINT }}" # set this environment variable for your signing key - - "--output" - - "${signature}" - - "--detach-sign" - - "${artifact}" -release: - # If you want to manually examine the release before its live, uncomment this line: - draft: true -changelog: - skip: true diff --git a/vendor/github.com/ms-henglu/pal/CHANGELOG.md b/vendor/github.com/ms-henglu/pal/CHANGELOG.md deleted file mode 100644 index ab400882..00000000 --- a/vendor/github.com/ms-henglu/pal/CHANGELOG.md +++ /dev/null @@ -1,40 +0,0 @@ -## v0.4.0 -BREAKING CHANGES: -- Only support logs from terraform-provider-azapi v1.10.0 or above. - -ENHANCEMENTS: -- The redundant query parameters are removed in the `markdown` format. -- Remove the `/providers` API from parsed logs, because its response couldn't be parsed. - -BUG FIXES: -- Fix the issue that some resources may not be outputted to `azapi` format. - -## v0.3.0 - -FEATURES: -- Support parsing terraform logs to `azapi` format. - -BUG FIXES: -- Fix the issue that the parsed URL paths are not normalized. -- Fix the issue that the request body from azurerm provider may not be parsed correctly, when the request body is pretty printed JSON. - -## v0.2.0 - -FEATURES: -- Support parsing terraform logs to `oav` traffic format. -- Support `-version` option to show the version information. -- Support `-help` option to show the help information. -- Support `-o` option to specify the output directory. -- Support `-i` option to specify the input file path. -- Support `-m` option to specify the output format. - -BUG FIXES: -- Fix the issue that response headers may contain duplicated values. -- Fix the issue that logs from released `azurerm` provider may not be parsed correctly. - -## v0.1.0 - -FEATURES: - -- Support parsing terraform logs to markdown format. -- Support `azurerm`, `azuread` and `azapi` providers. \ No newline at end of file diff --git a/vendor/github.com/ms-henglu/pal/GNUmakefile b/vendor/github.com/ms-henglu/pal/GNUmakefile deleted file mode 100644 index 84b8d1ea..00000000 --- a/vendor/github.com/ms-henglu/pal/GNUmakefile +++ /dev/null @@ -1,19 +0,0 @@ -fmt: - find . -name '*.go' | grep -v vendor | xargs gofmt -s -w - -depscheck: - @echo "==> Checking source code with go mod tidy..." - @go mod tidy - @git diff --exit-code -- go.mod go.sum || \ - (echo; echo "Unexpected difference in go.mod/go.sum files. Run 'go mod tidy' command or revert any go.mod/go.sum changes and commit."; exit 1) - @echo "==> Checking source code with go mod vendor..." - @go mod vendor - @git diff --compact-summary --ignore-space-at-eol --exit-code -- vendor || \ - (echo; echo "Unexpected difference in vendor/ directory. Run 'go mod vendor' command or revert any go.mod/go.sum/vendor changes and commit."; exit 1) - -test: - @go test ./... - -lint: - @echo "==> Checking source code against linters..." - @if command -v golangci-lint; then (golangci-lint run ./...); else ($(GOPATH)/bin/golangci-lint run ./...); fi \ No newline at end of file diff --git a/vendor/github.com/ms-henglu/pal/README.md b/vendor/github.com/ms-henglu/pal/README.md deleted file mode 100644 index 529ba4a2..00000000 --- a/vendor/github.com/ms-henglu/pal/README.md +++ /dev/null @@ -1,60 +0,0 @@ -# Parsing Azure's Logs - Pal - ----- - -## Introduction - -Pal is a simple tool to parse Terraform Azure provider logs. - -It can output different formats of the request traces, including: - -- Markdown: - - It can output the request traces in Markdown format, which can be used to create a GitHub issue or a forum post. - - The Markdown output supports expanding/collapsing the request traces. This is useful when the request trace is very long. - -- OAV Traffic: - - It can output the request traces in OAV traffic format, which can be used to validate the request traces with [OAV](https://github.com/Azure/oav). - -- AzAPI config: - - It can output the request traces in AzAPI config format, which can be used to reproduce the deployed resources with [AzAPI](https://registry.terraform.io/providers/Azure/azapi/latest). - -## Usage - -```bash -$ pal {path to terraform_log_file} -``` - -Full usage: - -```bash -Usage of pal: - -help - Show help - -i string - Input terraform log file - -m markdown - Output format, allowed values are markdown, `oav` and `azapi` (default "markdown") - -o string - Output directory - -version - Show version -``` - -## Example - -```bash -$ cd ./testdata -$ pal ./input.txt -``` - -Above command will generate a [markdown file named "output.md"](https://github.com/ms-henglu/pal/tree/main/testdata/output.md) in the same working directory. - -## How to install? - -```bash -$ go install github.com/ms-henglu/pal@latest -``` \ No newline at end of file diff --git a/vendor/github.com/ms-henglu/pal/formatter/azapi/formatter.go b/vendor/github.com/ms-henglu/pal/formatter/azapi/formatter.go deleted file mode 100644 index 01e3bd98..00000000 --- a/vendor/github.com/ms-henglu/pal/formatter/azapi/formatter.go +++ /dev/null @@ -1,348 +0,0 @@ -package azapi - -import ( - "encoding/json" - "fmt" - "log" - "strings" - - pluralize "github.com/gertd/go-pluralize" - "github.com/ms-henglu/pal/formatter" - "github.com/ms-henglu/pal/types" -) - -var _ formatter.Formatter = &AzapiFormatter{} - -var pluralizeClient = pluralize.NewClient() - -type AzapiFormatter struct { - existingResourceSet map[string]bool - labels map[string]bool - valueToKeyMap map[string]string - dataSourceActionUrls map[string]bool -} - -func ignoreKeywords() []string { - return []string{ - "operationresults", - "asyncoperations", - "operationstatuses", - "operationsStatus", - "operations", - } -} - -func (formatter *AzapiFormatter) Format(r types.RequestTrace) string { - if formatter.existingResourceSet == nil { - formatter.existingResourceSet = make(map[string]bool) - formatter.labels = make(map[string]bool) - formatter.valueToKeyMap = make(map[string]string) - formatter.dataSourceActionUrls = make(map[string]bool) - } - - if r.Host != "management.azure.com" { - // ignore the request to other hosts - return "" - } - - if shouldIgnore(r.Url) { - return "" - } - - resourceId := GetId(r.Url) - resourceType, apiVersion := parseResourceTypeApiVersion(r.Url) - - // ignore the request to fetch the resource group's resources - if resourceType == "" && strings.Contains(resourceId, "/resources") { - return "" - } - - var requestBody interface{} - _ = json.Unmarshal([]byte(r.Request.Body), &requestBody) - var responseBody interface{} - if r.Response != nil && r.Response.Body != "" { - _ = json.Unmarshal([]byte(r.Response.Body), &responseBody) - } - def := AzapiDefinition{ - AzureResourceType: resourceType, - ApiVersion: apiVersion, - Body: requestBody, - Output: responseBody, - ResourceId: resourceId, - Method: r.Method, - AdditionalFields: make(map[string]Value), - } - - switch r.Method { - case "PUT": - switch { - case strings.EqualFold(def.AzureResourceType, "Microsoft.KeyVault/vaults/accessPolicies"): - def = formatter.formatAsAzapiActionResource(def) - case IsResourceAction(def.ResourceId): - def = formatter.formatAsAzapiActionResource(def) - case formatter.existingResourceSet[def.ResourceId]: - def = formatter.formatAsAzapiUpdateResource(def) - default: - def = formatter.formatAsAzapiResource(def) - } - case "GET": - if r.StatusCode == 200 && !formatter.existingResourceSet[def.ResourceId] { - if IsResourceAction(def.ResourceId) { - // resource action data source - check := fmt.Sprintf("%s.%s", def.Method, def.ResourceId) - if _, ok := formatter.dataSourceActionUrls[check]; !ok { - def = formatter.formatAsAzapiActionDataSource(def) - formatter.dataSourceActionUrls[check] = true - } else { - return "" - } - } else { - // reading a resource which is created by the service - check := fmt.Sprintf("%s.%s", def.Method, def.ResourceId) - if _, ok := formatter.dataSourceActionUrls[check]; !ok { - def = formatter.formatAsAzapiDataSource(def) - formatter.dataSourceActionUrls[check] = true - formatter.existingResourceSet[def.ResourceId] = true - } else { - return "" - } - } - } else { - return "" - } - case "POST": - if r.Request.Body == "" { - check := fmt.Sprintf("%s.%s", def.Method, def.ResourceId) - if _, ok := formatter.dataSourceActionUrls[check]; !ok { - def = formatter.formatAsAzapiActionDataSource(def) - formatter.dataSourceActionUrls[check] = true - } else { - return "" - } - } else { - def = formatter.formatAsAzapiActionResource(def) - } - case "PATCH": - if !formatter.existingResourceSet[def.ResourceId] { - log.Printf("[WARN] PATCH %s is not a created resource", def.ResourceId) - return "" - } else { - def = formatter.formatAsAzapiActionResource(def) - } - case "DELETE": - if !formatter.existingResourceSet[def.ResourceId] { - log.Printf("[WARN] DELETE %s is not a created resource", def.ResourceId) - } - return "" - default: - return "" - } - - def.Body = formatter.injectReference(def.Body) - - address := fmt.Sprintf("%s.%s.%s", def.Kind, def.ResourceName, def.Label) - if def.Kind == "resource" { - address = fmt.Sprintf("%s.%s", def.ResourceName, def.Label) - } - prefix := fmt.Sprintf(`jsondecode(%s.output)`, address) - formatter.populateReference(prefix, def.Output) - - return def.String() -} - -func shouldIgnore(url string) bool { - resourceType := GetResourceType(url) - if strings.EqualFold(resourceType, "Microsoft.ApiManagement/service/apis/operations") || strings.EqualFold(resourceType, "Microsoft.ApiManagement/service/apis/operations/tags") { - return false - } - for _, v := range ignoreKeywords() { - if strings.Contains(url, v) { - return true - } - } - return false -} - -func (formatter *AzapiFormatter) formatAsAzapiResource(def AzapiDefinition) AzapiDefinition { - formatter.existingResourceSet[def.ResourceId] = true - def.Kind = "resource" - def.ResourceName = "azapi_resource" - def.Label = newUniqueLabel(def.ResourceName, defaultLabel(def.AzureResourceType), &formatter.labels) - - def.AdditionalFields["parent_id"] = formatter.tryAddressOrLiteral(GetParentId(def.ResourceId)) - def.AdditionalFields["name"] = NewLiteralValue(GetName(def.ResourceId)) - if def.Body != nil { - if requestBody, ok := def.Body.(map[string]interface{}); ok && requestBody != nil { - if location := requestBody["location"]; location != nil { - def.AdditionalFields["location"] = NewLiteralValue(location.(string)) - delete(requestBody, "location") - } - if name := requestBody["name"]; name != nil { - delete(requestBody, "name") - } - if strings.EqualFold(def.AzureResourceType, "Microsoft.ManagedIdentity/userAssignedIdentities") { - delete(requestBody, "properties") - } - def.Body = requestBody - } - } - formatter.valueToKeyMap[strings.ToUpper(def.ResourceId)] = fmt.Sprintf("%s.%s.id", def.ResourceName, def.Label) - - return def -} - -func (formatter *AzapiFormatter) formatAsAzapiDataSource(def AzapiDefinition) AzapiDefinition { - def.Kind = "data" - def.ResourceName = "azapi_resource" - def.Label = newUniqueLabel(def.ResourceName, defaultLabel(def.AzureResourceType), &formatter.labels) - - parentId := GetParentId(def.ResourceId) - if parentAddress := formatter.valueToKeyMap[strings.ToUpper(parentId)]; parentAddress != "" { - def.AdditionalFields["parent_id"] = NewReferenceValue(parentAddress) - def.AdditionalFields["name"] = NewLiteralValue(GetName(def.ResourceId)) - } else { - def.AdditionalFields["resource_id"] = NewLiteralValue(def.ResourceId) - } - def.AdditionalFields["response_export_values"] = NewRawValue(`["*"]`) - formatter.valueToKeyMap[strings.ToUpper(def.ResourceId)] = fmt.Sprintf("data.%s.%s.id", def.ResourceName, def.Label) - - return def -} - -func (formatter *AzapiFormatter) formatAsAzapiActionResource(def AzapiDefinition) AzapiDefinition { - def.Kind = "resource" - def.ResourceName = "azapi_resource_action" - - action := "" - if parts := strings.Split(def.ResourceId, "/"); len(parts)%2 == 0 { - action = parts[len(parts)-1] - def.ResourceId = strings.Join(parts[:len(parts)-1], "/") - def.AdditionalFields["action"] = NewLiteralValue(action) - } - - label := action - if label == "" { - label = fmt.Sprintf("%s_%s", strings.ToLower(def.Method), defaultLabel(def.AzureResourceType)) - } - def.Label = newUniqueLabel(def.ResourceName, label, &formatter.labels) - - def.AdditionalFields["resource_id"] = formatter.tryAddressOrLiteral(def.ResourceId) - if def.Method != "POST" { - def.AdditionalFields["method"] = NewLiteralValue(def.Method) - } - - return def -} - -func (formatter *AzapiFormatter) formatAsAzapiActionDataSource(def AzapiDefinition) AzapiDefinition { - def.Kind = "data" - def.ResourceName = "azapi_resource_action" - - action := "" - if parts := strings.Split(def.ResourceId, "/"); len(parts)%2 == 0 { - action = parts[len(parts)-1] - def.ResourceId = strings.Join(parts[:len(parts)-1], "/") - def.AdditionalFields["action"] = NewLiteralValue(action) - } - - label := action - if label == "" { - label = fmt.Sprintf("%s_%s", strings.ToLower(def.Method), defaultLabel(def.AzureResourceType)) - } - def.Label = newUniqueLabel(def.ResourceName, label, &formatter.labels) - - def.AdditionalFields["resource_id"] = formatter.tryAddressOrLiteral(def.ResourceId) - if def.Method != "POST" { - def.AdditionalFields["method"] = NewLiteralValue(def.Method) - } - - return def -} - -func (formatter *AzapiFormatter) formatAsAzapiUpdateResource(def AzapiDefinition) AzapiDefinition { - def.Kind = "resource" - def.ResourceName = "azapi_update_resource" - - def.Label = newUniqueLabel(def.ResourceName, fmt.Sprintf("update_%s", defaultLabel(def.AzureResourceType)), &formatter.labels) - - parentId := GetParentId(def.ResourceId) - if resourceIdAddress := formatter.valueToKeyMap[strings.ToUpper(def.ResourceId)]; resourceIdAddress != "" { - def.AdditionalFields["resource_id"] = NewReferenceValue(resourceIdAddress) - } else if parentAddress := formatter.valueToKeyMap[strings.ToUpper(parentId)]; parentAddress != "" { - def.AdditionalFields["parent_id"] = NewReferenceValue(parentAddress) - def.AdditionalFields["name"] = NewLiteralValue(GetName(def.ResourceId)) - } else { - def.AdditionalFields["resource_id"] = NewLiteralValue(def.ResourceId) - } - - if def.Body != nil { - bodyMap, ok := def.Body.(map[string]interface{}) - if ok && bodyMap != nil { - delete(bodyMap, "id") - delete(bodyMap, "name") - delete(bodyMap, "type") - } - } - - return def -} - -func (formatter *AzapiFormatter) tryAddressOrLiteral(resourceId string) Value { - if address := formatter.valueToKeyMap[strings.ToUpper(resourceId)]; address != "" { - return NewReferenceValue(address) - } - return NewLiteralValue(resourceId) -} - -func (formatter *AzapiFormatter) injectReference(raw interface{}) interface{} { - if raw == nil { - return nil - } - switch value := raw.(type) { - case map[string]interface{}: - out := make(map[string]interface{}) - for k, v := range value { - key := formatter.injectReference(k) - out[key.(string)] = formatter.injectReference(v) - } - return out - case []interface{}: - for i, v := range value { - value[i] = formatter.injectReference(v) - } - return value - case string: - if address := formatter.valueToKeyMap[strings.ToUpper(value)]; address != "" { - return fmt.Sprintf("${%s}", address) - } else { - return value - } - default: - return value - } -} - -func (formatter *AzapiFormatter) populateReference(prefix string, raw interface{}) { - if raw == nil { - return - } - switch value := raw.(type) { - case map[string]interface{}: - for k, v := range value { - formatter.populateReference(fmt.Sprintf("%s.%s", prefix, k), v) - } - case []interface{}: - for i, v := range value { - formatter.populateReference(fmt.Sprintf("%s[%d]", prefix, i), v) - } - case string: - if len(value) < 32 { - // if the string is too short, it's probably not a resource id - return - } - if _, ok := formatter.valueToKeyMap[strings.ToUpper(value)]; !ok { - formatter.valueToKeyMap[strings.ToUpper(value)] = prefix - } - } - -} diff --git a/vendor/github.com/ms-henglu/pal/formatter/azapi/hcl/marshal.go b/vendor/github.com/ms-henglu/pal/formatter/azapi/hcl/marshal.go deleted file mode 100644 index 7d25674e..00000000 --- a/vendor/github.com/ms-henglu/pal/formatter/azapi/hcl/marshal.go +++ /dev/null @@ -1,64 +0,0 @@ -package hcl - -import ( - "fmt" - "sort" - "strings" -) - -func MarshalIndent(input interface{}, prefix, indent string) string { - if input == nil { - return "null" - } - switch i := input.(type) { - case map[string]interface{}: - keys := make([]string, 0) - for key := range i { - keys = append(keys, key) - } - sort.Strings(keys) - content := "" - for _, key := range keys { - value := i[key] - wrapKey := key - if strings.HasPrefix(key, "${") && strings.HasSuffix(key, "}") { - wrapKey = fmt.Sprintf("(%s)", key[2:len(key)-1]) - } else if needWrapKey(key) { - wrapKey = fmt.Sprintf(`"%s"`, key) - } - content += fmt.Sprintf("%s%s = %s\n", prefix+indent, wrapKey, MarshalIndent(value, prefix+indent, indent)) - } - return fmt.Sprintf("{\n%s%s}", content, prefix) - case []interface{}: - content := "" - for _, value := range i { - content += fmt.Sprintf("%s%s,\n", prefix+indent, MarshalIndent(value, prefix+indent, indent)) - } - return fmt.Sprintf("[\n%s%s]", content, prefix) - case string: - if strings.HasPrefix(i, "${") && strings.HasSuffix(i, "}") { - return i[2 : len(i)-1] - } - i = strings.ReplaceAll(i, "\\", "\\\\") // escape backslashes - i = strings.ReplaceAll(i, "\n", "\\n") - i = strings.ReplaceAll(i, "\t", "\\t") - i = strings.ReplaceAll(i, "\r", "\\r") - i = strings.ReplaceAll(i, "\"", "\\\"") - return fmt.Sprintf(`"%s"`, i) - default: - return fmt.Sprintf("%v", i) - } -} - -func needWrapKey(input string) bool { - if len(input) == 0 { - return false - } - if strings.Contains(input, ".") || strings.Contains(input, "/") { - return true - } - if input[0] == '$' { - return true - } - return input[0] >= '0' && input[0] <= '9' -} diff --git a/vendor/github.com/ms-henglu/pal/formatter/azapi/types.go b/vendor/github.com/ms-henglu/pal/formatter/azapi/types.go deleted file mode 100644 index 7a567174..00000000 --- a/vendor/github.com/ms-henglu/pal/formatter/azapi/types.go +++ /dev/null @@ -1,101 +0,0 @@ -package azapi - -import ( - "fmt" - - "github.com/ms-henglu/pal/formatter/azapi/hcl" -) - -type AzapiDefinition struct { - Kind string // resource or data - ResourceName string // azapi_resource, azapi_update_resource, azapi_resource_action - Label string // example: test - AzureResourceType string // example: Microsoft.Network/virtualNetworks - ApiVersion string // example: 2020-06-01 - Body interface{} - Output interface{} - ResourceId string - Method string - AdditionalFields map[string]Value -} - -func (def AzapiDefinition) String() string { - expressions := fmt.Sprintf(` type = "%[1]s@%[2]s"`, def.AzureResourceType, def.ApiVersion) - fields := []string{"resource_id", "parent_id", "name", "location", "action", "method"} - for _, field := range fields { - if value, ok := def.AdditionalFields[field]; ok { - expressions += fmt.Sprintf(` - %[1]s = %[2]s`, field, value) - } - } - if def.Body != nil { - bodyMap, ok := def.Body.(map[string]interface{}) - if ok { - tagRaw, ok := bodyMap["tags"] - if ok && tagRaw != nil { - if tagMap, ok := tagRaw.(map[string]interface{}); ok && len(tagMap) == 0 { - delete(bodyMap, "tags") - } - } - } - if len(bodyMap) > 0 { - expressions += fmt.Sprintf(` - body = jsonencode(%[1]s)`, hcl.MarshalIndent(bodyMap, " ", " ")) - } - } - return fmt.Sprintf( - `%[1]s "%[2]s" "%[3]s" { -%[4]s -} -`, def.Kind, def.ResourceName, def.Label, expressions) -} - -type Value interface { - String() string -} - -type RawValue struct { - Raw string -} - -func (v RawValue) String() string { - return v.Raw -} - -func NewRawValue(raw string) RawValue { - return RawValue{ - Raw: raw, - } -} - -type ReferenceValue struct { - Reference string -} - -func (v ReferenceValue) String() string { - return v.Reference -} - -func NewReferenceValue(reference string) ReferenceValue { - return ReferenceValue{ - Reference: reference, - } -} - -type LiteralValue struct { - Literal string -} - -func (v LiteralValue) String() string { - return fmt.Sprintf(`"%s"`, v.Literal) -} - -func NewLiteralValue(literal string) LiteralValue { - return LiteralValue{ - Literal: literal, - } -} - -var _ Value = &RawValue{} -var _ Value = &ReferenceValue{} -var _ Value = &LiteralValue{} diff --git a/vendor/github.com/ms-henglu/pal/formatter/azapi/utils.go b/vendor/github.com/ms-henglu/pal/formatter/azapi/utils.go deleted file mode 100644 index 8954f741..00000000 --- a/vendor/github.com/ms-henglu/pal/formatter/azapi/utils.go +++ /dev/null @@ -1,121 +0,0 @@ -package azapi - -import ( - "fmt" - "net/url" - "strings" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" -) - -func parseResourceTypeApiVersion(input string) (string, string) { - idUrl, err := url.Parse(input) - if err != nil { - return "", "" - } - apiVersion := idUrl.Query().Get("api-version") - resourceType := GetResourceType(idUrl.Path) - return resourceType, apiVersion -} - -func GetResourceType(id string) string { - idURL, err := url.ParseRequestURI(id) - if err != nil { - return "" - } - - path := idURL.Path - - path = strings.TrimPrefix(path, "/") - path = strings.TrimSuffix(path, "/") - - components := strings.Split(path, "/") - resourceType := "" - provider := "" - for current := 0; current < len(components)-1; current += 2 { - key := components[current] - value := "" - value = components[current+1] - - // Check key/value for empty strings. - if key == "" || value == "" { - return "" - } - - if key == "providers" { - provider = value - resourceType = provider - } else if len(provider) > 0 { - resourceType += "/" + key - } - } - if resourceType == "" { - resourceId, err := arm.ParseResourceID(id) - if err != nil { - return "" - } - return resourceId.ResourceType.String() - } - return resourceType -} - -// newLabel returns an unique label for a resource type -func newUniqueLabel(prefix string, label string, labels *map[string]bool) string { - check := fmt.Sprintf("%s.%s", prefix, label) - _, ok := (*labels)[check] - if !ok { - (*labels)[check] = true - return label - } - for i := 2; i <= 100; i++ { - newLabel := fmt.Sprintf("%s%d", label, i) - check = fmt.Sprintf("%s.%s", prefix, newLabel) - _, ok := (*labels)[check] - if !ok { - (*labels)[check] = true - return newLabel - } - } - return label -} - -func defaultLabel(resourceType string) string { - parts := strings.Split(resourceType, "/") - label := "test" - if len(parts) != 0 { - label = parts[len(parts)-1] - label = pluralizeClient.Singular(label) - } - return label -} - -func GetName(id string) string { - resourceId, err := arm.ParseResourceID(id) - if err != nil { - return "" - } - return resourceId.Name -} - -func GetParentId(id string) string { - resourceId, err := arm.ParseResourceID(id) - if err != nil { - return "" - } - if resourceId.Parent.ResourceType.String() == arm.TenantResourceType.String() { - return "/" - } - return resourceId.Parent.String() -} - -func GetId(input string) string { - idUrl, err := url.Parse(input) - if err != nil { - return "" - } - return idUrl.Path -} - -func IsResourceAction(resourceId string) bool { - return len(strings.Split(resourceId, "/"))%2 == 0 -} diff --git a/vendor/github.com/ms-henglu/pal/main.go b/vendor/github.com/ms-henglu/pal/main.go deleted file mode 100644 index 84701361..00000000 --- a/vendor/github.com/ms-henglu/pal/main.go +++ /dev/null @@ -1,140 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "log" - "os" - "path" - "strings" - - "github.com/ms-henglu/pal/formatter" - "github.com/ms-henglu/pal/formatter/azapi" - "github.com/ms-henglu/pal/trace" -) - -const version = "0.4.0" - -var showHelp = flag.Bool("help", false, "Show help") -var showVersion = flag.Bool("version", false, "Show version") - -func main() { - input := "" - output := "" - mode := "" - - flag.StringVar(&input, "i", "", "Input terraform log file") - flag.StringVar(&output, "o", "", "Output directory") - flag.StringVar(&mode, "m", "markdown", "Output format, allowed values are `markdown`, `oav` and `azapi`") - - // backward compatibility, the first argument is the input file - if len(os.Args) == 2 { - if _, err := os.Stat(os.Args[1]); err == nil { - input = os.Args[1] - mode = "markdown" - } - } - if input == "" { - flag.Parse() - if *showHelp { - flag.Usage() - os.Exit(0) - } - if *showVersion { - fmt.Println(version) - os.Exit(0) - } - } - if input == "" { - flag.Usage() - log.Fatalf("[ERROR] input file is required") - } - - if output == "" { - output = path.Dir(input) - } - - log.Printf("[INFO] input file: %s", input) - log.Printf("[INFO] output directory: %s", output) - log.Printf("[INFO] output format: %s", mode) - - traces, err := trace.RequestTracesFromFile(input) - if err != nil { - log.Fatalf("[ERROR] failed to parse request traces: %v", err) - } - - for _, t := range traces { - out := trace.VerifyRequestTrace(t) - if len(out) > 0 { - log.Printf("[WARN] verification failed: url %s\n%s", t.Url, strings.Join(out, "\n")) - } - } - - switch mode { - case "oav": - format := formatter.OavTrafficFormatter{} - index := 0 - for _, t := range traces { - out := format.Format(t) - index = index + 1 - outputPath := path.Join(output, fmt.Sprintf("trace-%d.json", index)) - if err := os.WriteFile(outputPath, []byte(out), 0644); err != nil { - log.Fatalf("[ERROR] failed to write file: %v", err) - } - log.Printf("[INFO] output file: %s", outputPath) - } - case "markdown": - content := markdownPrefix - format := formatter.MarkdownFormatter{} - for _, t := range traces { - content += format.Format(t) - } - outputPath := path.Clean(path.Join(output, "output.md")) - if err := os.WriteFile(outputPath, []byte(content), 0644); err != nil { - log.Fatalf("[ERROR] failed to write file: %v", err) - } - log.Printf("[INFO] output file: %s", outputPath) - case "azapi": - content := azapiPrefix - format := azapi.AzapiFormatter{} - for _, t := range traces { - if res := format.Format(t); res != "" { - content += res - content += "\n" - } - } - outputPath := path.Clean(path.Join(output, "pal-main.tf")) - if err := os.WriteFile(outputPath, []byte(content), 0644); err != nil { - log.Fatalf("[ERROR] failed to write file: %v", err) - } - log.Printf("[INFO] output file: %s", outputPath) - default: - log.Fatalf("[ERROR] unsupported output format: %s", mode) - } - -} - -const markdownPrefix = ` - -` - -const azapiPrefix = ` -terraform { - required_providers { - azapi = { - source = "Azure/azapi" - } - } -} - -provider "azapi" { - skip_provider_registration = false -} - -` diff --git a/vendor/modules.txt b/vendor/modules.txt index 58feb28b..b4118cb2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -421,10 +421,7 @@ github.com/mitchellh/mapstructure github.com/mitchellh/reflectwalk # github.com/ms-henglu/pal v0.4.0 ## explicit; go 1.19 -github.com/ms-henglu/pal github.com/ms-henglu/pal/formatter -github.com/ms-henglu/pal/formatter/azapi -github.com/ms-henglu/pal/formatter/azapi/hcl github.com/ms-henglu/pal/provider github.com/ms-henglu/pal/rawlog github.com/ms-henglu/pal/trace