From e58a82f25fe2994be4afc90b494ab78e09cd9033 Mon Sep 17 00:00:00 2001 From: teowa <104055472+teowa@users.noreply.github.com> Date: Mon, 15 Apr 2024 18:19:50 +0800 Subject: [PATCH] support add cache for credscan index --- .gitignore | 2 +- commands/credential_scan.go | 27 +++++-- coverage/coverage_test.go | 7 ++ coverage/expand.go | 3 +- coverage/expand_test.go | 4 +- coverage/from_local_spec.go | 4 +- coverage/from_local_spec_test.go | 16 +++-- coverage/index.go | 95 +++++++++++++++++++++--- coverage/index_test.go | 119 ++++++++++++++++++++++++++++++- coverage/report.go | 2 +- readme.md | 3 +- 11 files changed, 246 insertions(+), 36 deletions(-) diff --git a/.gitignore b/.gitignore index 03b8b3fd..e87a764b 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,4 @@ armstrong.exe # test output coverage/test_coverage_report*.md - +coverage/testdata/index.json diff --git a/commands/credential_scan.go b/commands/credential_scan.go index 67519329..e47c002a 100644 --- a/commands/credential_scan.go +++ b/commands/credential_scan.go @@ -17,23 +17,25 @@ import ( ) type CredentialScanCommand struct { - workingDir string - swaggerRepoPath string - verbose bool + workingDir string + swaggerRepoPath string + swaggerIndexFile string + verbose bool } func (c *CredentialScanCommand) flags() *flag.FlagSet { fs := defaultFlagSet("test") fs.BoolVar(&c.verbose, "v", false, "whether show terraform logs") - fs.StringVar(&c.workingDir, "working-dir", "", "path to Terraform configuration files") + fs.StringVar(&c.workingDir, "working-dir", "", "path to directory containing Terraform configuration files") fs.StringVar(&c.swaggerRepoPath, "swagger-repo", "", "path to the swagger repo specification directory") + fs.StringVar(&c.swaggerIndexFile, "swagger-index-file", "", "path to the swagger index file, omit this will use the online swagger index file or locally build index") fs.Usage = func() { logrus.Error(c.Help()) } return fs } func (c CredentialScanCommand) Help() string { helpText := ` -Usage: armstrong credscan [-v] [-working-dir ] [-swagger-repo ] +Usage: armstrong credscan [-v] [-working-dir ] [-swagger-repo ] [-swagger-index-file ] ` + c.Synopsis() + "\n\n" + helpForFlags(c.flags()) return strings.TrimSpace(helpText) @@ -91,6 +93,17 @@ func (c CredentialScanCommand) Execute() int { c.swaggerRepoPath += "/" } + if c.swaggerIndexFile != "" { + c.swaggerIndexFile, err = filepath.Abs(c.swaggerIndexFile) + if err != nil { + logrus.Errorf("swagger index file path %q is invalid: %+v", c.swaggerIndexFile, err) + return 1 + } + + if _, err := os.Stat(c.swaggerIndexFile); os.IsNotExist(err) { + logrus.Infof("swagger index file %q does not exist, will try to build or download index", c.swaggerIndexFile) + } + } tfFiles, err := hcl.FindTfFiles(wd) if err != nil { @@ -210,7 +223,7 @@ func (c CredentialScanCommand) Execute() int { var swaggerModel *coverage.SwaggerModel if c.swaggerRepoPath != "" { logrus.Infof("scan based on local swagger repo: %s", c.swaggerRepoPath) - swaggerModel, err = coverage.GetModelInfoFromLocalIndex(mockedResourceId, apiVersion, c.swaggerRepoPath) + swaggerModel, err = coverage.GetModelInfoFromLocalIndex(mockedResourceId, apiVersion, c.swaggerRepoPath, c.swaggerIndexFile) if err != nil { credScanErr := makeCredScanError( azapiResource, @@ -223,7 +236,7 @@ func (c CredentialScanCommand) Execute() int { continue } } else { - swaggerModel, err = coverage.GetModelInfoFromIndex(mockedResourceId, apiVersion) + swaggerModel, err = coverage.GetModelInfoFromIndex(mockedResourceId, apiVersion, c.swaggerIndexFile) if err != nil { credScanErr := makeCredScanError( azapiResource, diff --git a/coverage/coverage_test.go b/coverage/coverage_test.go index d3dda7b0..0d0561bc 100644 --- a/coverage/coverage_test.go +++ b/coverage/coverage_test.go @@ -6,6 +6,7 @@ import ( "log" "os" "path" + "strings" "testing" "github.com/azure/armstrong/coverage" @@ -21,6 +22,10 @@ type testCase struct { resourceType string } +func normarlizePath(path string) string { + return strings.ReplaceAll(path, string(os.PathSeparator), "/") +} + func TestCoverage_ResourceGroup(t *testing.T) { tc := testCase{ name: "ResourceGroup", @@ -1402,6 +1407,7 @@ func testCoverage(t *testing.T, tc testCase) (*coverage.Model, error) { swaggerModel, err := coverage.GetModelInfoFromIndex( tc.apiPath, tc.apiVersion, + "", ) t.Logf("swaggerModel: %+v", swaggerModel) @@ -1477,6 +1483,7 @@ func testCredScan(t *testing.T, tc testCase) (*map[string]string, error) { swaggerModel, err := coverage.GetModelInfoFromIndex( tc.apiPath, tc.apiVersion, + "", ) t.Logf("swaggerModel: %+v", swaggerModel) diff --git a/coverage/expand.go b/coverage/expand.go index fe59f720..18b3d50e 100644 --- a/coverage/expand.go +++ b/coverage/expand.go @@ -2,7 +2,6 @@ package coverage import ( "fmt" - "net/url" "path/filepath" "strings" @@ -327,7 +326,7 @@ func SchemaNamePathFromRef(swaggerPath string, ref openapiSpec.Ref) (schemaName schemaPath = swaggerPath } else { swaggerPath, _ := filepath.Split(swaggerPath) - schemaPath, _ = url.JoinPath(swaggerPath, schemaPath) + schemaPath = swaggerPath + schemaPath } fragments := strings.Split(refUrl.Fragment, "/") diff --git a/coverage/expand_test.go b/coverage/expand_test.go index a44618f8..e84f72ba 100644 --- a/coverage/expand_test.go +++ b/coverage/expand_test.go @@ -119,7 +119,7 @@ func TestExpandAll(t *testing.T) { if strings.HasSuffix(azureRepoDir, "specification") { azureRepoDir += "/" } - if !strings.HasSuffix(azureRepoDir, "specification/") { + if !strings.HasSuffix(normarlizePath(azureRepoDir), "specification/") { t.Fatalf("AZURE_REST_REPO_DIR must specify the specification folder, e.g., AZURE_REST_REPO_DIR=\"/home/test/go/src/github.com/azure/azure-rest-api-specs/specification/\"") } @@ -207,7 +207,7 @@ func testExpandAll(t *testing.T, azureRepoDir, testResultPath string) result { } func getRefList(t *testing.T) []jsonreference.Ref { - index, err := coverage.GetIndex() + index, err := coverage.GetIndex("") if err != nil { t.Fatal(err) } diff --git a/coverage/from_local_spec.go b/coverage/from_local_spec.go index 1889af5a..2777f420 100644 --- a/coverage/from_local_spec.go +++ b/coverage/from_local_spec.go @@ -51,7 +51,7 @@ func GetModelInfoFromLocalSpecFile(resourceId, apiVersion, swaggerPath string) ( } for pathKey, pathItem := range paths.Paths { - if !isPathKeyMatchWithResourceId(pathKey, resourceId) { + if !IsPathKeyMatchWithResourceId(pathKey, resourceId) { continue } @@ -95,7 +95,7 @@ func GetModelInfoFromLocalSpecFile(resourceId, apiVersion, swaggerPath string) ( return nil, nil } -func isPathKeyMatchWithResourceId(pathKey, resourceId string) bool { +func IsPathKeyMatchWithResourceId(pathKey, resourceId string) bool { pathParts := strings.Split(strings.Trim(pathKey, "/"), "/") resourceIdParts := strings.Split(strings.Trim(resourceId, "/"), "/") i := len(pathParts) - 1 diff --git a/coverage/from_local_spec_test.go b/coverage/from_local_spec_test.go index e263598b..8d889b02 100644 --- a/coverage/from_local_spec_test.go +++ b/coverage/from_local_spec_test.go @@ -1,9 +1,11 @@ -package coverage +package coverage_test import ( "os" "path" "testing" + + "github.com/azure/armstrong/coverage" ) func Test_isPathKeyMatchWithResourceId(t *testing.T) { @@ -35,7 +37,7 @@ func Test_isPathKeyMatchWithResourceId(t *testing.T) { } for _, testcase := range testcases { t.Logf("testcase: %+v", testcase) - actual := isPathKeyMatchWithResourceId(testcase.PathKey, testcase.ResourceId) + actual := coverage.IsPathKeyMatchWithResourceId(testcase.PathKey, testcase.ResourceId) if actual != testcase.Expected { t.Fatalf("expected %v, got %v", testcase.Expected, actual) } @@ -51,12 +53,12 @@ func Test_GetModelInfoFromLocalDir(t *testing.T) { testcases := []struct { ResourceId string ApiVersion string - Expected *SwaggerModel + Expected *coverage.SwaggerModel }{ { ResourceId: "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/test-resources/providers/Microsoft.Automation/automationAccounts/test-automation-account", ApiVersion: "2022-08-08", - Expected: &SwaggerModel{ + Expected: &coverage.SwaggerModel{ ApiPath: "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Automation/automationAccounts/{automationAccountName}", ModelName: "AutomationAccountCreateOrUpdateParameters", SwaggerPath: path.Join(swaggerPath, "account.json"), @@ -65,7 +67,7 @@ func Test_GetModelInfoFromLocalDir(t *testing.T) { { ResourceId: "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/test-resources/providers/Microsoft.Automation/automationAccounts/test-automation-account/certificates/test-certificate", ApiVersion: "2022-08-08", - Expected: &SwaggerModel{ + Expected: &coverage.SwaggerModel{ ApiPath: "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Automation/automationAccounts/{automationAccountName}/certificates/{certificateName}", ModelName: "CertificateCreateOrUpdateParameters", SwaggerPath: path.Join(swaggerPath, "certificate.json"), @@ -75,7 +77,7 @@ func Test_GetModelInfoFromLocalDir(t *testing.T) { for _, testcase := range testcases { t.Logf("testcase: %+v", testcase.ResourceId) - actual, err := GetModelInfoFromLocalDir(testcase.ResourceId, testcase.ApiVersion, swaggerPath) + actual, err := coverage.GetModelInfoFromLocalDir(testcase.ResourceId, testcase.ApiVersion, swaggerPath) if err != nil { t.Fatalf("get model info from local dir error: %+v", err) } @@ -88,7 +90,7 @@ func Test_GetModelInfoFromLocalDir(t *testing.T) { if actual.ModelName != testcase.Expected.ModelName { t.Fatalf("expected modelName %s, got %s", testcase.Expected.ModelName, actual.ModelName) } - if actual.SwaggerPath != testcase.Expected.SwaggerPath { + if normarlizePath(actual.SwaggerPath) != normarlizePath(testcase.Expected.SwaggerPath) { t.Fatalf("expected swaggerPath %s, got %s", testcase.Expected.SwaggerPath, actual.SwaggerPath) } } diff --git a/coverage/index.go b/coverage/index.go index a3a973f5..66d36de7 100644 --- a/coverage/index.go +++ b/coverage/index.go @@ -24,12 +24,28 @@ const ( var indexCache *azidx.Index -func GetIndexFromLocalDir(swaggerRepo string) (*azidx.Index, error) { +func GetIndexFromLocalDir(swaggerRepo, indexFilePath string) (*azidx.Index, error) { if indexCache != nil { return indexCache, nil } - logrus.Infof("building index from from local swagger %s", swaggerRepo) + if indexFilePath != "" { + if _, err := os.Stat(indexFilePath); err == nil { + byteValue, _ := os.ReadFile(indexFilePath) + + var index azidx.Index + if err := json.Unmarshal(byteValue, &index); err != nil { + return nil, fmt.Errorf("unmarshal index file: %+v", err) + } + indexCache = &index + + logrus.Infof("load index from cache file %s", indexFilePath) + + return indexCache, nil + } + } + + logrus.Infof("building index from from local swagger %s, it might take several minutes", swaggerRepo) index, err := azidx.BuildIndex(swaggerRepo, "") if err != nil { logrus.Error(fmt.Sprintf("failed to build index: %+v", err)) @@ -39,19 +55,53 @@ func GetIndexFromLocalDir(swaggerRepo string) (*azidx.Index, error) { indexCache = index + if indexFilePath != "" { + jsonBytes, err := json.Marshal(&index) + if err != nil { + logrus.Warningf("failed to marshal index: %+v", err) + return index, nil + } + + err = os.WriteFile(indexFilePath, jsonBytes, 0644) + if err != nil { + logrus.Warningf("failed to write index cache file %s: %+v", indexFilePath, err) + return index, nil + } + + logrus.Infof("index successfully saved to cache file %s", indexFilePath) + } + return index, nil } -func GetIndex() (*azidx.Index, error) { +func GetIndex(indexFilePath string) (*azidx.Index, error) { if indexCache != nil { return indexCache, nil } + if indexFilePath != "" { + if _, err := os.Stat(indexFilePath); err == nil { + byteValue, _ := os.ReadFile(indexFilePath) + + var index azidx.Index + if err := json.Unmarshal(byteValue, &index); err != nil { + return nil, fmt.Errorf("unmarshal index file: %+v", err) + } + indexCache = &index + + logrus.Infof("load index from cache file %s", indexFilePath) + + return indexCache, nil + } + } + resp, err := http.Get(indexFileURL) if err != nil { return nil, fmt.Errorf("get index file from %v: %+v", indexFileURL, err) } + logrus.Infof("downloading index file from %s", indexFileURL) + defer resp.Body.Close() b, err := io.ReadAll(resp.Body) @@ -86,6 +136,23 @@ func GetIndex() (*azidx.Index, error) { indexCache = &index logrus.Infof("load index based commit: https://github.com/Azure/azure-rest-api-specs/tree/%s", index.Commit) + + if indexFilePath != "" { + jsonBytes, err := json.Marshal(&index) + if err != nil { + logrus.Warningf("failed to marshal index: %+v", err) + return indexCache, nil + } + + err = os.WriteFile(indexFilePath, jsonBytes, 0644) + if err != nil { + logrus.Warningf("failed to write index cache file %s: %+v", indexFilePath, err) + return indexCache, nil + } + + logrus.Infof("index successfully saved to cache file %s", indexFilePath) + } + return indexCache, nil } @@ -95,8 +162,10 @@ type SwaggerModel struct { SwaggerPath string } -func GetModelInfoFromIndex(resourceId, apiVersion string) (*SwaggerModel, error) { - index, err := GetIndex() +// GetModelInfoFromIndex will try to download online index from https://github.com/teowa/azure-rest-api-index-file, and get model info from it +// if the index is already downloaded as in {indexFilePath}, it will use the cached index +func GetModelInfoFromIndex(resourceId, apiVersion, indexFilePath string) (*SwaggerModel, error) { + index, err := GetIndex(indexFilePath) if err != nil { return nil, err } @@ -123,7 +192,7 @@ func GetModelInfoFromIndex(resourceId, apiVersion string) (*SwaggerModel, error) } // GetModelInfoFromLocalIndex tries to build index from local swagger repo and get model info from it -func GetModelInfoFromLocalIndex(resourceId, apiVersion, swaggerRepo string) (*SwaggerModel, error) { +func GetModelInfoFromLocalIndex(resourceId, apiVersion, swaggerRepo, indexCacheFile string) (*SwaggerModel, error) { swaggerRepo, err := filepath.Abs(swaggerRepo) if err != nil { return nil, fmt.Errorf("swagger repo path %q is invalid: %+v", swaggerRepo, err) @@ -141,7 +210,7 @@ func GetModelInfoFromLocalIndex(resourceId, apiVersion, swaggerRepo string) (*Sw swaggerRepo += "/" - index, err := GetIndexFromLocalDir(swaggerRepo) + index, err := GetIndexFromLocalDir(swaggerRepo, indexCacheFile) if err != nil { return nil, fmt.Errorf("build index from local dir %s: %+v", swaggerRepo, err) } @@ -170,7 +239,13 @@ func GetModelInfoFromLocalIndex(resourceId, apiVersion, swaggerRepo string) (*Sw func GetModelInfoFromIndexRef(ref openapispec.Ref, swaggerRepo string) (*SwaggerModel, error) { _, swaggerPath := SchemaNamePathFromRef(swaggerRepo, ref) - relativeBase := swaggerRepo + strings.Split(ref.GetURL().Path, "/")[0] + seperator := "/" + // in windows the ref might use backslashes + if strings.Contains(ref.GetURL().Path, string(os.PathSeparator)) { + seperator = string(os.PathSeparator) + } + + relativeBase := swaggerRepo + strings.Split(ref.GetURL().Path, seperator)[0] operation, err := openapispec.ResolvePathItemWithBase(nil, ref, &openapispec.ExpandOptions{RelativeBase: relativeBase}) if err != nil { return nil, err @@ -233,7 +308,7 @@ func MockResourceIDFromType(azapiResourceType string) (string, string) { return fmt.Sprintf("%s/%s/providers/%s/%s", subscritionSeg, resourceGroupSeg, resourceProvider, typeIds), apiVersion } -func GetModelInfoFromIndexWithType(azapiResourceType string) (*SwaggerModel, error) { +func GetModelInfoFromIndexWithType(azapiResourceType, indexCacheFile string) (*SwaggerModel, error) { resourceId, apiVersion := MockResourceIDFromType(azapiResourceType) - return GetModelInfoFromIndex(resourceId, apiVersion) + return GetModelInfoFromIndex(resourceId, apiVersion, indexCacheFile) } diff --git a/coverage/index_test.go b/coverage/index_test.go index 0817a4b0..a70a1a03 100644 --- a/coverage/index_test.go +++ b/coverage/index_test.go @@ -6,13 +6,45 @@ import ( "testing" "github.com/azure/armstrong/coverage" + "github.com/go-openapi/jsonreference" + openapispec "github.com/go-openapi/spec" ) +const indexFilePath = "testdata/index.json" + func TestGetModelInfoFromIndex_DataCollectionRule(t *testing.T) { apiVersion := "2022-06-01" swaggerModel, err := coverage.GetModelInfoFromIndex( "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/test-resources/providers/Microsoft.Insights/dataCollectionRules/testDCR", apiVersion, + "", + ) + if err != nil { + t.Fatalf("get model info from index error: %+v", err) + } + + expectedApiPath := "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Insights/dataCollectionRules/{dataCollectionRuleName}" + if swaggerModel.ApiPath != expectedApiPath { + t.Fatalf("expected apiPath %s, got %s", expectedApiPath, swaggerModel.ApiPath) + } + + expectedModelName := "DataCollectionRuleResource" + if swaggerModel.ModelName != expectedModelName { + t.Fatalf("expected modelName %s, got %s", expectedModelName, swaggerModel.ModelName) + } + + expectedModelSwaggerPath := "https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/monitor/resource-manager/Microsoft.Insights/stable/2022-06-01/dataCollectionRules_API.json" + if swaggerModel.SwaggerPath != expectedModelSwaggerPath { + t.Fatalf("expected modelSwaggerPath %s, got %s", expectedModelSwaggerPath, swaggerModel.SwaggerPath) + } +} + +func TestGetModelInfoFromIndexWithCache_DataCollectionRule(t *testing.T) { + apiVersion := "2022-06-01" + swaggerModel, err := coverage.GetModelInfoFromIndex( + "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/test-resources/providers/Microsoft.Insights/dataCollectionRules/testDCR", + apiVersion, + indexFilePath, ) if err != nil { t.Fatalf("get model info from index error: %+v", err) @@ -39,6 +71,7 @@ func TestGetModelInfoFromIndex_DeviceSecurityGroups(t *testing.T) { swaggerModel, err := coverage.GetModelInfoFromIndex( "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/SampleRG/providers/Microsoft.Devices/iotHubs/sampleiothub/providers/Microsoft.Security/deviceSecurityGroups/samplesecuritygroup", apiVersion, + "", ) if err != nil { t.Fatalf("get model info from index error: %+v", err) @@ -62,7 +95,7 @@ func TestGetModelInfoFromIndex_DeviceSecurityGroups(t *testing.T) { func TestGetModelInfoFromIndexWithType_DataCollectionRule(t *testing.T) { azapiResourceType := "Microsoft.Insights/dataCollectionRules@2022-06-01" - swaggerModel, err := coverage.GetModelInfoFromIndexWithType(azapiResourceType) + swaggerModel, err := coverage.GetModelInfoFromIndexWithType(azapiResourceType, "") if err != nil { t.Fatalf("get model info from index error: %+v", err) } @@ -73,9 +106,27 @@ func TestGetModelInfoFromIndexWithType_DataCollectionRule(t *testing.T) { } } +func TestGetModelInfoFromIndexWithTypeWithCache_DataCollectionRule(t *testing.T) { + azapiResourceType := "Microsoft.Insights/dataCollectionRules@2022-06-01" + swaggerModel, err := coverage.GetModelInfoFromIndexWithType(azapiResourceType, indexFilePath) + if err != nil { + t.Fatalf("get model info from index error: %+v", err) + } + + expectedApiPath := "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Insights/dataCollectionRules/{dataCollectionRuleName}" + if swaggerModel.ApiPath != expectedApiPath { + t.Fatalf("expected apiPath %s, got %s", expectedApiPath, swaggerModel.ApiPath) + } + + _, err = coverage.GetModelInfoFromIndexWithType(azapiResourceType, indexFilePath) + if err != nil { + t.Fatalf("get model info from index error: %+v", err) + } +} + func TestGetModelInfoFromIndexWithType_DeviceSecurityGroups(t *testing.T) { azapiResourceType := "Microsoft.Security/deviceSecurityGroups@2019-08-01" - swaggerModel, err := coverage.GetModelInfoFromIndexWithType(azapiResourceType) + swaggerModel, err := coverage.GetModelInfoFromIndexWithType(azapiResourceType, "") if err != nil { t.Fatalf("get model info from index error: %+v", err) } @@ -97,6 +148,7 @@ func TestGetModelInfoFromLocalIndex_DataCollectionRule(t *testing.T) { "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/test-resources/providers/Microsoft.Insights/dataCollectionRules/testDCR", apiVersion, azureRepoDir, + "", ) if err != nil { t.Fatalf("get model info from index error: %+v", err) @@ -113,7 +165,68 @@ func TestGetModelInfoFromLocalIndex_DataCollectionRule(t *testing.T) { } expectedModelSwaggerPathSuffix := "/monitor/resource-manager/Microsoft.Insights/stable/2022-06-01/dataCollectionRules_API.json" - if !strings.HasSuffix(swaggerModel.SwaggerPath, expectedModelSwaggerPathSuffix) { + if !strings.HasSuffix(normarlizePath(swaggerModel.SwaggerPath), expectedModelSwaggerPathSuffix) { t.Fatalf("expected modelSwaggerPath has suffix %s, got %s", expectedModelSwaggerPathSuffix, swaggerModel.SwaggerPath) } } + +func TestGetModelInfoFromLocalIndexWithCache_DataCollectionRule(t *testing.T) { + azureRepoDir := os.Getenv("AZURE_REST_REPO_DIR") + if azureRepoDir == "" { + t.Skip("AZURE_REST_REPO_DIR is not set") + } + + apiVersion := "2022-06-01" + swaggerModel, err := coverage.GetModelInfoFromLocalIndex( + "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/test-resources/providers/Microsoft.Insights/dataCollectionRules/testDCR", + apiVersion, + azureRepoDir, + indexFilePath, + ) + if err != nil { + t.Fatalf("get model info from index error: %+v", err) + } + + expectedApiPath := "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Insights/dataCollectionRules/{dataCollectionRuleName}" + if swaggerModel.ApiPath != expectedApiPath { + t.Fatalf("expected apiPath %s, got %s", expectedApiPath, swaggerModel.ApiPath) + } + + expectedModelName := "DataCollectionRuleResource" + if swaggerModel.ModelName != expectedModelName { + t.Fatalf("expected modelName %s, got %s", expectedModelName, swaggerModel.ModelName) + } + + expectedModelSwaggerPathSuffix := "/monitor/resource-manager/Microsoft.Insights/stable/2022-06-01/dataCollectionRules_API.json" + if !strings.HasSuffix(normarlizePath(swaggerModel.SwaggerPath), expectedModelSwaggerPathSuffix) { + t.Fatalf("expected modelSwaggerPath has suffix %s, got %s", expectedModelSwaggerPathSuffix, swaggerModel.SwaggerPath) + } + + _, err = coverage.GetModelInfoFromLocalIndex( + "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/test-resources/providers/Microsoft.Insights/dataCollectionRules/testDCR", + apiVersion, + azureRepoDir, + indexFilePath, + ) + if err != nil { + t.Fatalf("get model info from index error: %+v", err) + } + +} + +func TestGetModelInfoFromIndexRef(t *testing.T) { + if os.PathSeparator != '\\' { + t.Skip("this test is only for windows with backslashes(\\) as the file path seperator") + } + + azureRepoDir := os.Getenv("AZURE_REST_REPO_DIR") + if azureRepoDir == "" { + t.Skip("AZURE_REST_REPO_DIR is not set") + } + + pathRef := jsonreference.MustCreateRef("monitor%5Cresource-manager%5CMicrosoft.Insights%5Cstable%5C2021-08-01%5CscheduledQueryRule_API.json#/paths/~1subscriptions~1%7BsubscriptionId%7D~1resourceGroups~1%7BresourceGroupName%7D~1providers~1Microsoft.Insights~1scheduledQueryRules~1%7BruleName%7D/put") + _, err := coverage.GetModelInfoFromIndexRef(openapispec.Ref{Ref: pathRef}, azureRepoDir) + if err != nil { + t.Fatal(err) + } +} diff --git a/coverage/report.go b/coverage/report.go index ad915ba3..d97f47f8 100644 --- a/coverage/report.go +++ b/coverage/report.go @@ -28,7 +28,7 @@ func (c *CoverageReport) AddCoverageFromState(resourceId, resourceType string, j swaggerModel = swaggerModelFromLocal } if swaggerModel == nil { - swaggerModelFromIndex, err := GetModelInfoFromIndex(resourceId, apiVersion) + swaggerModelFromIndex, err := GetModelInfoFromIndex(resourceId, apiVersion, "") if err != nil { return fmt.Errorf("error find the path for %s from index: %+v", resourceId, err) } diff --git a/readme.md b/readme.md index b2a86648..1be88005 100644 --- a/readme.md +++ b/readme.md @@ -147,7 +147,8 @@ armstrong credscan Supported options: 1. `-working-dir`: Specify the working directory containing Terraform config files, default is current directory. 2. `-swagger-repo`: Specify the swagger repo path used to match credentials, omit this will use the online swagger repo. -2. `-v`: Enable verbose mode, default is false. +3. `-swagger-index-file`: Specify the path to the swagger index file, omit this will use the online swagger index file or locally build index. If the specified file is not found, the downloaded or built index will be saved in the provided file. +4. `-v`: Enable verbose mode, default is false. Armstrong also output different kinds of reports: 1. `errors.json`: A json report which contains scan errors.