From 7d9a6c3d2fb60d6e45913bc61102e67e4bde8545 Mon Sep 17 00:00:00 2001 From: Zhenhua Hu Date: Mon, 18 Sep 2023 16:36:52 +0800 Subject: [PATCH] add code --- .gitignore | 1 + commands/api_test_generate_report.go | 69 +++++ commands/test.go | 26 +- main.go | 3 + report/api_test_report.go | 420 +++++++++++++++++++++++++++ 5 files changed, 517 insertions(+), 2 deletions(-) create mode 100644 commands/api_test_generate_report.go create mode 100644 report/api_test_report.go diff --git a/.gitignore b/.gitignore index 03014b95..8bc6a60a 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ commands/.temp # never upload the build to git armstrong +armstrong.exe # test output coverage/test_coverage_report*.md diff --git a/commands/api_test_generate_report.go b/commands/api_test_generate_report.go new file mode 100644 index 00000000..d8c3adba --- /dev/null +++ b/commands/api_test_generate_report.go @@ -0,0 +1,69 @@ +package commands + +import ( + "flag" + "fmt" + "github.com/ms-henglu/armstrong/report" + "github.com/sirupsen/logrus" + "log" + "os" + "path/filepath" + "strings" +) + +type ApiTestGenerateReportCommand struct { + workingDir string + swaggerPath string +} + +func (c *ApiTestGenerateReportCommand) flags() *flag.FlagSet { + fs := defaultFlagSet("api-test-generate-report") + fs.StringVar(&c.workingDir, "working-dir", "", "path that contains all the test cases") + fs.StringVar(&c.swaggerPath, "swagger", "", "path to the .json swagger which is being test") + fs.Usage = func() { logrus.Error(c.Help()) } + return fs +} + +func (c ApiTestGenerateReportCommand) Help() string { + helpText := ` +Usage: armstrong api-test-generate-report +` + c.Synopsis() + "\n\n" + helpForFlags(c.flags()) + + return strings.TrimSpace(helpText) +} + +func (c ApiTestGenerateReportCommand) Synopsis() string { + return "Generate test report for a set of test results" +} + +func (c ApiTestGenerateReportCommand) Run(args []string) int { + f := c.flags() + if err := f.Parse(args); err != nil { + logrus.Error(fmt.Sprintf("Error parsing command-line flags: %s", err)) + return 1 + } + return c.Execute() +} + +func (c ApiTestGenerateReportCommand) Execute() int { + log.Println("[INFO] ----------- generate API Test Report ---------") + wd, err := os.Getwd() + if err != nil { + logrus.Error(fmt.Sprintf("failed to get working directory: %+v", err)) + return 1 + } + + if c.workingDir != "" { + wd, err = filepath.Abs(c.workingDir) + if err != nil { + logrus.Error(fmt.Sprintf("working directory is invalid: %+v", err)) + return 1 + } + } + + if report.GenerateApiTestReports(wd, c.swaggerPath) != nil { + log.Fatalf("[ERROR] failed to generate API Test Report: %+v", err) + } + + return 0 +} diff --git a/commands/test.go b/commands/test.go index 44b84b3c..fe115cdf 100644 --- a/commands/test.go +++ b/commands/test.go @@ -18,14 +18,18 @@ import ( ) type TestCommand struct { - verbose bool - workingDir string + verbose bool + workingDir string + destroyAfterTest bool + swaggerPath string } func (c *TestCommand) flags() *flag.FlagSet { fs := defaultFlagSet("test") fs.BoolVar(&c.verbose, "v", false, "whether show terraform logs") fs.StringVar(&c.workingDir, "working-dir", "", "path to Terraform configuration files") + fs.BoolVar(&c.destroyAfterTest, "destroy-after-test", false, "whether to destroy the created resources after each test") + fs.StringVar(&c.swaggerPath, "swagger", "", "path to the .json swagger which is being test") fs.Usage = func() { logrus.Error(c.Help()) } return fs } @@ -140,9 +144,27 @@ func (c TestCommand) Execute() int { } else { log.Fatalf("[ERROR] error showing terraform state: %+v", err) } + return 0 } + if applyErr == nil { + if c.destroyAfterTest { + destroyErr := terraform.Destroy() + if destroyErr != nil { + log.Printf("[ERROR] error running terraform destroy: %+v\n", destroyErr) + } else { + log.Println("[INFO] test resource has been deleted") + } + } + } + + if c.swaggerPath != "" { + if report.StoreApiTestReport(wd, c.swaggerPath) != nil { + log.Fatalf("[ERROR] error storing api test report: %+v", err) + } + } + logs, err := report.ParseLogs(path.Join(wd, "log.txt")) if err != nil { log.Printf("[ERROR] parsing log.txt: %+v", err) diff --git a/main.go b/main.go index 8c09240b..6388e16d 100644 --- a/main.go +++ b/main.go @@ -31,6 +31,9 @@ func main() { "cleanup": func() (cli.Command, error) { return &commands.CleanupCommand{}, nil }, + "api-test-generate-report": func() (cli.Command, error) { + return &commands.ApiTestGenerateReportCommand{}, nil + }, } exitStatus, err := c.Run() diff --git a/report/api_test_report.go b/report/api_test_report.go new file mode 100644 index 00000000..9ffe8b51 --- /dev/null +++ b/report/api_test_report.go @@ -0,0 +1,420 @@ +package report + +import ( + "encoding/json" + "fmt" + "github.com/sirupsen/logrus" + "io" + "io/fs" + "os" + "os/exec" + "path" + "path/filepath" + "sort" + "strings" +) + +const ( + TestReportDirName = "ApiTestReport" + TestResultDirName = "ApiTestResult" + TraceLogDirName = "ApiTestTraces" + ApiTestReportFileName = "ApiTestReport" + ApiTestConfigFileName = "ApiTestConfig.json" +) + +type ApiTestReport struct { + ApiVersion string `json:"apiVersion"` + CoverageResults []CoverageResult `json:"coverageResultsForRendering"` +} + +type CoverageResult struct { + GeneralErrorsInnerList []GeneralErrorsInner `json:"generalErrorsInnerList"` + UnCoveredOperationsList []UnCoveredOperation `json:"unCoveredOperationsList"` +} + +type GeneralErrorsInner struct { + ErrorsForRendering []ErrorForRendering `json:"errorsForRendering"` + OperationInfo OperationInfo `json:"operationInfo"` +} + +type OperationInfo struct { + OperationId string `json:"operationId"` +} + +type ErrorForRendering struct { + Code string `json:"code"` + Message string `json:"message"` + Link string `json:"link"` + SchemaPathWithPosition string `json:"schemaPathWithPosition"` +} + +type UnCoveredOperation struct { + OperationId string `json:"operationId"` +} + +type ApiTestConfig struct { + SuppressionList []Suppression `json:"suppressionList"` +} + +type Suppression struct { + Code string `json:"rule"` + File string `json:"file"` + Operation string `json:"operation"` + Reason string `json:"reason"` +} + +func StoreApiTestReport(wd string, swaggerPath string) error { + wd = filepath.ToSlash(wd) + swaggerPath = filepath.ToSlash(swaggerPath) + testResultPath := path.Join(wd, TestResultDirName) + traceLogPath := path.Join(testResultPath, TraceLogDirName) + if err := os.RemoveAll(testResultPath); err != nil { + return fmt.Errorf("[ERROR] error removing trace log dir %s: %+v", testResultPath, err) + } + + if err := os.MkdirAll(traceLogPath, 0755); err != nil { + return fmt.Errorf("[ERROR] error creating trace log dir %s: %+v", testResultPath, err) + } + + cmd := exec.Command("pal", "-i", path.Join(wd, "log.txt"), "-m", "oav", "-o", traceLogPath) + err := cmd.Run() + if err != nil { + return fmt.Errorf("failed to run `pal` command with %+v", err) + } + + reportFileName := fmt.Sprintf("%s.html", ApiTestReportFileName) + cmd = exec.Command("oav", "validate-traffic", traceLogPath, swaggerPath, "--report", path.Join(testResultPath, reportFileName)) + cmd.Run() + if _, err = os.Stat(path.Join(testResultPath, reportFileName)); os.IsNotExist(err) { + return fmt.Errorf("failed to generate test report") + } + + return nil +} + +func GenerateApiTestReports(wd string, swaggerPath string) error { + wd = filepath.ToSlash(wd) + testReportPath := path.Join(wd, TestReportDirName) + traceLogPath := path.Join(testReportPath, TraceLogDirName) + apiTestReportFilePath := path.Join(testReportPath, fmt.Sprintf("%s.json", ApiTestReportFileName)) + + if err := mergeApiTestTraceFiles(wd, traceLogPath); err != nil { + return fmt.Errorf("[ERROR] failed to merge trace files: %+v", err) + } + + swaggerFilePaths, err := getApiTestSwaggerPaths(swaggerPath) + if err != nil { + return fmt.Errorf("[ERROR] failed to get swagger paths: %+v", err) + } + + result, err := readApiTestHistoryReport(swaggerPath, apiTestReportFilePath) + if err != nil { + return err + } + + if err = generateApiTestJsonReport(result, swaggerFilePaths, testReportPath); err != nil { + return fmt.Errorf("[ERROR] failed to generate oav reports: %+v", err) + } + + if err = generateApiTestMarkdownReport(result, testReportPath, path.Join(wd, ApiTestConfigFileName)); err != nil { + return fmt.Errorf("[ERROR] failed to generate markdown report: %+v", err) + } + + return nil +} + +func isSuppressedInApiTest(suppressionList []Suppression, rule string, filePath string, operation string) bool { + segments := strings.Split(filepath.ToSlash(filePath), "/") + file := segments[len(segments)-1] + + for _, suppression := range suppressionList { + if strings.EqualFold(suppression.Code, rule) && strings.EqualFold(suppression.File, file) && (strings.EqualFold(suppression.Code, "SWAGGER_NOT_TEST") || strings.EqualFold(suppression.Operation, operation)) { + return true + } + } + + return false +} + +func generateApiTestMarkdownReport(result map[string]*ApiTestReport, testReportPath string, apiTestConfigFilePath string) error { + var config ApiTestConfig + + if _, err := os.Stat(apiTestConfigFilePath); err == nil { + contentBytes, err := os.ReadFile(apiTestConfigFilePath) + if err != nil { + logrus.Errorf("error when opening file(%s): %+v", apiTestConfigFilePath, err) + } + + err = json.Unmarshal(contentBytes, &config) + if err != nil { + logrus.Errorf("error during Unmarshal() for file(%s): %+v", apiTestConfigFilePath, err) + } + } else { + logrus.Infof("no config file found") + } + + mdTitle := "## API TEST ERROR REPORT
\n|Rule|Message|\n|---|---|" + mdTable := make([]string, 0) + + for rk, rv := range result { + if rv == nil { + if !isSuppressedInApiTest(config.SuppressionList, "SWAGGER_NOT_TEST", rk, "") { + mdTable = append(mdTable, fmt.Sprintf("|[SWAGGER_NOT_TEST](about:blank)|**message**: No operations in swagger is test.
**location**: %s", rk[strings.Index(rk, "/specification/"):])) + } + + continue + } + + for _, coverageResult := range rv.CoverageResults { + for _, operation := range coverageResult.UnCoveredOperationsList { + if !isSuppressedInApiTest(config.SuppressionList, "OPERATION_NOT_TEST", rk, operation.OperationId) { + mdTable = append(mdTable, fmt.Sprintf("|[OPERATION_NOT_TEST](about:blank)|**message**: **%s** opeartion is not test.
**opeartion**: %s
**location**: %s", operation.OperationId, operation.OperationId, rk[strings.Index(rk, "/specification/"):])) + } + + } + + for _, item := range coverageResult.GeneralErrorsInnerList { + for _, errItem := range item.ErrorsForRendering { + location := rk[strings.Index(rk, "/specification/"):] + normalizedPath := filepath.ToSlash(errItem.SchemaPathWithPosition) + if subIndex := strings.Index(normalizedPath, "/specification/"); subIndex != -1 { + location = normalizedPath[subIndex:] + } + + if !isSuppressedInApiTest(config.SuppressionList, errItem.Code, rk, item.OperationInfo.OperationId) { + mdTable = append(mdTable, fmt.Sprintf("|[%s](%s)|**message**: %s.
**opeartion**: %s
**location**: %s", errItem.Code, errItem.Link, errItem.Message, item.OperationInfo.OperationId, location)) + } + } + } + } + } + + sort.Strings(mdTable) + + mdReportFilePath := path.Join(testReportPath, fmt.Sprintf("%s.md", ApiTestReportFileName)) + if err := os.WriteFile(mdReportFilePath, []byte(mdTitle+"\n"+strings.Join(mdTable, "\n")), 0644); err != nil { + return fmt.Errorf("error when writing file(%s): %+v", mdReportFilePath, err) + } + + return nil +} + +func generateApiTestJsonReport(result map[string]*ApiTestReport, swaggerFilePaths []string, testReportPath string) error { + for idx, filePath := range swaggerFilePaths { + logrus.Infof("generating oav report for %d: %s", idx, filePath) + if report, err := retrieveOavReport(filePath, testReportPath); err != nil { + return err + } else { + result[filePath] = report + } + } + + jsonStr, err := json.Marshal(result) + if err != nil { + return err + } + + jsonReportFilePath := path.Join(testReportPath, fmt.Sprintf("%s.json", ApiTestReportFileName)) + if err = os.WriteFile(jsonReportFilePath, jsonStr, 0644); err != nil { + return fmt.Errorf("error when writing file(%s): %+v", jsonReportFilePath, err) + } + + return nil +} + +func readApiTestHistoryReport(swaggerPath string, apiTestReportFilePath string) (map[string]*ApiTestReport, error) { + swaggerPathInfo, err := os.Stat(swaggerPath) + if err != nil { + return nil, err + } + + if !swaggerPathInfo.IsDir() { + swaggerPath = path.Dir(swaggerPath) + } + + swaggerFiles, err := getApiTestSwaggerPathsFromDirectory(swaggerPath) + if err != nil { + return nil, err + } + + result := make(map[string]*ApiTestReport) + for _, v := range swaggerFiles { + result[v] = nil + } + + if _, err = os.Stat(apiTestReportFilePath); os.IsNotExist(err) { + logrus.Infof("no history report found") + return result, nil + } + + contentBytes, err := os.ReadFile(apiTestReportFilePath) + if err != nil { + return nil, fmt.Errorf("error when opening file(%s): %+v", apiTestReportFilePath, err) + } + + var payload map[string]*ApiTestReport + err = json.Unmarshal(contentBytes, &payload) + if err != nil { + return nil, fmt.Errorf("error during Unmarshal() for file(%s): %+v", apiTestReportFilePath, err) + } + + for k, _ := range result { + if v, exists := payload[k]; exists { + result[k] = v + } + } + + return result, nil +} + +func retrieveOavReport(filePath string, testReportPath string) (*ApiTestReport, error) { + traceLogPath := path.Join(testReportPath, TraceLogDirName) + prefix := strings.Split(path.Base(filePath), ".")[0] + "_report" + htmlReportFilePath := path.Join(testReportPath, fmt.Sprintf("%s.html", prefix)) + jsonReportFilePath := path.Join(testReportPath, fmt.Sprintf("%s.json", prefix)) + + if err := os.RemoveAll(htmlReportFilePath); err != nil { + return nil, fmt.Errorf("[ERROR] error removing test report file %s: %+v", htmlReportFilePath, err) + } + + if err := os.RemoveAll(jsonReportFilePath); err != nil { + return nil, fmt.Errorf("[ERROR] error removing test report file %s: %+v", jsonReportFilePath, err) + } + + cmd := exec.Command("oav", "validate-traffic", traceLogPath, filePath, "--report", htmlReportFilePath) + cmd.Run() + if _, err := os.Stat(path.Join(testReportPath, jsonReportFilePath)); os.IsNotExist(err) { + return nil, fmt.Errorf("[ERROR] failed to generate report for: %s", testReportPath) + } + + contentBytes, err := os.ReadFile(jsonReportFilePath) + if err != nil { + return nil, fmt.Errorf("error when opening file(%s): %+v", jsonReportFilePath, err) + } + + var payload *ApiTestReport + err = json.Unmarshal(contentBytes, &payload) + if err != nil { + return nil, fmt.Errorf("error during Unmarshal() for file(%s): %+v", jsonReportFilePath, err) + } + + if err = os.RemoveAll(jsonReportFilePath); err != nil { + return nil, fmt.Errorf("[ERROR] error removing test report file %s: %+v", jsonReportFilePath, err) + } + + if payload != nil && payload.ApiVersion != "unknown" { + return payload, nil + } + + return nil, nil +} + +func getApiTestSwaggerPaths(swaggerPath string) ([]string, error) { + logrus.Infof("loading swagger spec: %s...", swaggerPath) + file, err := os.Stat(swaggerPath) + if err != nil { + return nil, fmt.Errorf("loading swagger spec: %+v", err) + } + + apiPathsAll := make([]string, 0) + if file.IsDir() { + if apiPathsAll, err = getApiTestSwaggerPathsFromDirectory(swaggerPath); err != nil { + return nil, err + } + + } else { + logrus.Infof("parsing swagger spec: %s...", swaggerPath) + apiPathsAll = append(apiPathsAll, swaggerPath) + } + + logrus.Infof("found %d api paths", len(apiPathsAll)) + return apiPathsAll, nil +} + +func getApiTestSwaggerPathsFromDirectory(swaggerPath string) ([]string, error) { + logrus.Infof("swagger spec is a directory") + logrus.Infof("loading swagger spec directory: %s...", swaggerPath) + files, err := os.ReadDir(swaggerPath) + if err != nil { + return nil, fmt.Errorf("reading swagger spec directory: %+v", err) + } + + apiPathsAll := make([]string, 0) + for _, file := range files { + if !strings.HasSuffix(file.Name(), ".json") || file.IsDir() { + continue + } + + filePath := path.Join(swaggerPath, file.Name()) + apiPathsAll = append(apiPathsAll, filePath) + } + + return apiPathsAll, nil +} + +func mergeApiTestTraceFiles(wd string, traceLogPath string) error { + if err := os.RemoveAll(traceLogPath); err != nil { + return fmt.Errorf("[ERROR] error removing test trace dir %s: %+v", traceLogPath, err) + } + + if err := os.MkdirAll(traceLogPath, 0755); err != nil { + return fmt.Errorf("[ERROR] error creating test report dir %s: %+v", traceLogPath, err) + } + + destIndex := 1 + err := filepath.WalkDir(wd, func(walkPath string, d fs.DirEntry, err error) error { + if d.IsDir() { + traceDir := filepath.Join(walkPath, TestResultDirName, TraceLogDirName) + if _, err = os.Stat(traceDir); !os.IsNotExist(err) { + destIndex, err = copyApiTestTraceFiles(traceDir, traceLogPath, destIndex) + if err != nil { + return fmt.Errorf("failed to copy trace files: %+v", err) + } + } + } + + return nil + }) + + if err != nil { + return fmt.Errorf("impossible to walk directories: %s", err) + } + + return nil +} + +func copyApiTestTraceFiles(src string, dest string, destIndex int) (int, error) { + err := filepath.WalkDir(src, func(walkPath string, d fs.DirEntry, err error) error { + if d.IsDir() || !strings.HasSuffix(d.Name(), ".json") { + return nil + } + + srcFile, err := os.Open(walkPath) + if err != nil { + return err + } + defer srcFile.Close() + + destPath := path.Join(dest, fmt.Sprintf("trace-%d.json", destIndex)) + destFile, err := os.Create(destPath) + if err != nil { + return err + } + + defer destFile.Close() + + _, err = io.Copy(destFile, srcFile) + if err != nil { + return err + } + + destIndex++ + return nil + }) + + if err != nil { + return destIndex, err + } + + return destIndex, nil +}