Add operation properties coverage report for API test scenarios

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## v0.15.0
 
 ENHANCEMENTS:
+- Add operation properties coverage report for API test scenarios
 - Hide readonly array item in coverage report
 - Update mapping of azurerm v4.0.1
 

commands/test.go

@@ -216,6 +216,20 @@ func (c TestCommand) Execute() int {
 		if _, err = report.OavValidateTraffic(traceDir, c.swaggerPath, reportDir); err != nil {
 			logrus.Errorf("error storing swagger accuracy report: %+v", err)
 		}
+
+		logrus.Infof("generating operation properties coverage report...")
+		if covReport, err := coverage.NewOperationPropertiesCoverageReport(traceDir, c.swaggerPath); err == nil {
+			reportContent := covReport.MarkdownContent()
+			outputPath := path.Join(reportDir, report.CoverageReportFileName+".md")
+			err := os.WriteFile(outputPath, []byte(reportContent), 0644)
+			if err != nil {
+				logrus.Warnf("failed to save operation properties coverage report to %s: %+v", outputPath, err)
+			} else {
+				logrus.Infof("operation properties coverage report saved to %s", outputPath)
+			}
+		} else {
+			logrus.Warnf("failed to generate operation properties coverage report: %+v", err)
+		}
 	} else {
 		logrus.Warnf("no swagger file provided, swagger accuracy report will not be generated")
 	}

coverage/coverage_test.go

@@ -3,7 +3,6 @@ package coverage_test
 import (
 	"encoding/json"
 	"fmt"
-	"log"
 	"os"
 	"path"
 	"strings"
@@ -12,6 +11,7 @@ import (
 	"github.com/azure/armstrong/coverage"
 	"github.com/azure/armstrong/report"
 	"github.com/azure/armstrong/types"
+	"github.com/sirupsen/logrus"
 )
 
 type testCase struct {
@@ -1543,11 +1543,12 @@ func testCoverage(t *testing.T, tc testCase) (*coverage.Model, error) {
 	//t.Logf("coverage model %s", string(out))
 
 	coverageReport := coverage.CoverageReport{
-		Coverages: map[coverage.ArmResource]*coverage.Model{
-			{
-				ApiPath: swaggerModel.ApiPath,
-				Type:    tc.resourceType,
-			}: model,
+		Coverages: map[string]*coverage.CoverageItem{
+			swaggerModel.ApiPath: {
+				ApiPath:     swaggerModel.ApiPath,
+				DisplayName: tc.resourceType,
+				Model:       model,
+			},
 		},
 	}
 
@@ -1569,9 +1570,9 @@ func storeCoverageReport(passReport types.PassReport, coverageReport coverage.Co
 	if len(coverageReport.Coverages) != 0 {
 		err := os.WriteFile(path.Join(reportDir, reportName), []byte(report.PassedMarkdownReport(passReport, coverageReport)), 0644)
 		if err != nil {
-			log.Printf("[WARN] failed to save passed markdown report to %s: %+v", reportName, err)
+			logrus.Warnf("failed to save passed markdown report to %s: %+v", reportName, err)
 		} else {
-			log.Printf("[INFO] markdown report saved to %s", reportName)
+			logrus.Infof("markdown report saved to %s", reportName)
 		}
 	}
 }

coverage/from_local_spec.go

@@ -11,7 +11,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func GetModelInfoFromLocalDir(resourceId, apiVersion, swaggerPath string) (*SwaggerModel, error) {
+func GetModelInfoFromLocalDir(resourceId, swaggerPath string, method string) (*SwaggerModel, error) {
 	swaggerPath, err := filepath.Abs(swaggerPath)
 	if err != nil {
 		return nil, err
@@ -21,14 +21,14 @@ func GetModelInfoFromLocalDir(resourceId, apiVersion, swaggerPath string) (*Swag
 		return nil, err
 	}
 	if !file.IsDir() {
-		return GetModelInfoFromLocalSpecFile(resourceId, apiVersion, swaggerPath)
+		return GetModelInfoFromLocalSpecFile(resourceId, swaggerPath, method)
 	}
 	files, err := utils.ListFiles(swaggerPath, ".json", 1)
 	if err != nil {
 		return nil, err
 	}
 	for _, filename := range files {
-		model, err := GetModelInfoFromLocalSpecFile(resourceId, apiVersion, filename)
+		model, err := GetModelInfoFromLocalSpecFile(resourceId, filename, method)
 		if err != nil {
 			logrus.Warnf("failed to get model info from local spec file %v: %+v", filename, err)
 		}
@@ -39,7 +39,7 @@ func GetModelInfoFromLocalDir(resourceId, apiVersion, swaggerPath string) (*Swag
 	return nil, nil
 }
 
-func GetModelInfoFromLocalSpecFile(resourceId, apiVersion, swaggerPath string) (*SwaggerModel, error) {
+func GetModelInfoFromLocalSpecFile(resourceId, swaggerPath string, method string) (*SwaggerModel, error) {
 	doc, err := loadSwagger(swaggerPath)
 	if err != nil {
 		return nil, err
@@ -55,7 +55,25 @@ func GetModelInfoFromLocalSpecFile(resourceId, apiVersion, swaggerPath string) (
 			continue
 		}
 
-		operation := pathItem.Put
+		var operation *openapispec.Operation
+		switch strings.ToUpper(method) {
+		case "GET":
+			operation = pathItem.Get
+		case "PUT":
+			operation = pathItem.Put
+		case "POST":
+			operation = pathItem.Post
+		case "DELETE":
+			operation = pathItem.Delete
+		case "OPTIONS":
+			operation = pathItem.Options
+		case "HEAD":
+			operation = pathItem.Head
+		case "PATCH":
+			operation = pathItem.Patch
+		default:
+			logrus.Warnf("unsupported method %v", method)
+		}
 		if operation == nil {
 			// should not happen
 			logrus.Warnf("no PUT operation found for path %v", pathKey)
@@ -90,6 +108,7 @@ func GetModelInfoFromLocalSpecFile(resourceId, apiVersion, swaggerPath string) (
 			ApiPath:     pathKey,
 			ModelName:   modelName,
 			SwaggerPath: swaggerPath,
+			OperationID: operation.ID,
 		}, nil
 	}
 	return nil, nil

coverage/from_local_spec_test.go

@@ -77,7 +77,7 @@ func Test_GetModelInfoFromLocalDir(t *testing.T) {
 
 	for _, testcase := range testcases {
 		t.Logf("testcase: %+v", testcase.ResourceId)
-		actual, err := coverage.GetModelInfoFromLocalDir(testcase.ResourceId, testcase.ApiVersion, swaggerPath)
+		actual, err := coverage.GetModelInfoFromLocalDir(testcase.ResourceId, swaggerPath, "PUT")
 		if err != nil {
 			t.Fatalf("get model info from local dir error: %+v", err)
 		}

coverage/index.go

@@ -160,6 +160,7 @@ type SwaggerModel struct {
 	ApiPath     string
 	ModelName   string
 	SwaggerPath string
+	OperationID string
 }
 
 // GetModelInfoFromIndex will try to download online index from https://github.com/teowa/azure-rest-api-index-file, and get model info from it

coverage/operation_properties_coverage.go

@@ -0,0 +1,86 @@
+package coverage
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/ms-henglu/pal/formatter"
+	"github.com/sirupsen/logrus"
+)
+
+func NewOperationPropertiesCoverageReport(traceDir string, swaggerPath string) (*CoverageReport, error) {
+	files, err := os.ReadDir(traceDir)
+	if err != nil {
+		return nil, err
+	}
+	report := &CoverageReport{
+		Coverages: make(map[string]*CoverageItem),
+	}
+
+	for _, file := range files {
+		if !strings.HasSuffix(file.Name(), ".json") {
+			continue
+		}
+
+		data, err := os.ReadFile(path.Join(traceDir, file.Name()))
+		if err != nil {
+			logrus.Warnf("failed to read file %s: %+v", file.Name(), err)
+			continue
+		}
+
+		var trace formatter.OavTraffic
+		if err := json.Unmarshal(data, &trace); err != nil {
+			logrus.Warnf("failed to unmarshal file %s: %+v", file.Name(), err)
+			continue
+		}
+
+		swaggerModel, err := GetModelInfoFromLocalDir(removeQueryParameters(trace.LiveRequest.Url), swaggerPath, trace.LiveRequest.Method)
+		if err != nil {
+			logrus.Warnf("failed to get model info from local dir: %+v", err)
+			continue
+		}
+
+		if swaggerModel == nil {
+			// the API is not in the swagger file, usually it's an API that out of the testing scope
+			continue
+		}
+
+		index := fmt.Sprintf("%s-%s", trace.LiveRequest.Method, swaggerModel.ApiPath)
+		if swaggerModel.ModelName == "" {
+			// the API has no request body, mark it as fully covered
+			report.Coverages[index] = &CoverageItem{
+				ApiPath:     swaggerModel.ApiPath,
+				DisplayName: swaggerModel.OperationID,
+				Model: &Model{
+					IsFullyCovered: true,
+				},
+			}
+			continue
+		}
+
+		if _, ok := report.Coverages[index]; !ok {
+			expanded, err := Expand(swaggerModel.ModelName, swaggerModel.SwaggerPath)
+			if err != nil {
+				logrus.Warnf("failed to expand model %s property: %+v", swaggerModel.ModelName, err)
+				continue
+			}
+			report.Coverages[index] = &CoverageItem{
+				ApiPath:     swaggerModel.ApiPath,
+				DisplayName: swaggerModel.OperationID,
+				Model:       expanded,
+			}
+		}
+
+		report.Coverages[index].Model.MarkCovered(trace.LiveRequest.Body)
+		report.Coverages[index].Model.CountCoverage()
+	}
+
+	return report, nil
+}
+
+func removeQueryParameters(url string) string {
+	return strings.Split(url, "?")[0]
+}