diff --git a/CHANGELOG.md b/CHANGELOG.md index 894149f..ab40088 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,18 @@ +## v0.4.0 +BREAKING CHANGES: +- Only support logs from terraform-provider-azapi v1.10.0 or above. + +ENHANCEMENTS: +- The redundant query parameters are removed in the `markdown` format. +- Remove the `/providers` API from parsed logs, because its response couldn't be parsed. + +BUG FIXES: +- Fix the issue that some resources may not be outputted to `azapi` format. + ## v0.3.0 FEATURES: -- Support parsing terraform logs to `azapi` traffic format. +- Support parsing terraform logs to `azapi` format. BUG FIXES: - Fix the issue that the parsed URL paths are not normalized. diff --git a/formatter/azapi/formatter.go b/formatter/azapi/formatter.go index 4b813ab..01e3bd9 100644 --- a/formatter/azapi/formatter.go +++ b/formatter/azapi/formatter.go @@ -44,10 +44,9 @@ func (formatter *AzapiFormatter) Format(r types.RequestTrace) string { // ignore the request to other hosts return "" } - for _, v := range ignoreKeywords() { - if strings.Contains(r.Url, v) { - return "" - } + + if shouldIgnore(r.Url) { + return "" } resourceId := GetId(r.Url) @@ -151,6 +150,19 @@ func (formatter *AzapiFormatter) Format(r types.RequestTrace) string { return def.String() } +func shouldIgnore(url string) bool { + resourceType := GetResourceType(url) + if strings.EqualFold(resourceType, "Microsoft.ApiManagement/service/apis/operations") || strings.EqualFold(resourceType, "Microsoft.ApiManagement/service/apis/operations/tags") { + return false + } + for _, v := range ignoreKeywords() { + if strings.Contains(url, v) { + return true + } + } + return false +} + func (formatter *AzapiFormatter) formatAsAzapiResource(def AzapiDefinition) AzapiDefinition { formatter.existingResourceSet[def.ResourceId] = true def.Kind = "resource" diff --git a/formatter/markdown.go b/formatter/markdown.go index a62ef4b..f1f85e0 100644 --- a/formatter/markdown.go +++ b/formatter/markdown.go @@ -3,6 +3,7 @@ package formatter import ( "fmt" "net/http" + "net/url" "strings" "github.com/ms-henglu/pal/types" @@ -19,7 +20,15 @@ func (m MarkdownFormatter) Format(r types.RequestTrace) string { content = strings.ReplaceAll(content, "{Time}", r.TimeStamp.Format("15:04:05")) content = strings.ReplaceAll(content, "{Method}", r.Method) content = strings.ReplaceAll(content, "{Host}", r.Host) - content = strings.ReplaceAll(content, "{Url}", r.Url) + urlStr := r.Url + parsedUrl, err := url.Parse(r.Url) + if err == nil { + urlStr = parsedUrl.Path + if value := parsedUrl.Query()["api-version"]; len(value) > 0 { + urlStr += "?api-version=" + value[0] + } + } + content = strings.ReplaceAll(content, "{Url}", urlStr) content = strings.ReplaceAll(content, "{StatusCode}", fmt.Sprintf("%d", r.StatusCode)) content = strings.ReplaceAll(content, "{StatusMessage}", http.StatusText(r.StatusCode)) content = strings.ReplaceAll(content, "{RequestHeaders}", m.formatHeaders(r.Request.Headers)) diff --git a/main.go b/main.go index f2cc2f5..8470136 100644 --- a/main.go +++ b/main.go @@ -13,7 +13,7 @@ import ( "github.com/ms-henglu/pal/trace" ) -const version = "0.3.0" +const version = "0.4.0" var showHelp = flag.Bool("help", false, "Show help") var showVersion = flag.Bool("version", false, "Show version") diff --git a/provider/azapi.go b/provider/azapi.go index a628dfd..c474b18 100644 --- a/provider/azapi.go +++ b/provider/azapi.go @@ -1,129 +1,98 @@ package provider import ( + "encoding/json" "fmt" "net/url" + "regexp" "strings" "github.com/ms-henglu/pal/rawlog" "github.com/ms-henglu/pal/types" - "github.com/ms-henglu/pal/utils" ) var _ Provider = AzAPIProvider{} -type AzAPIProvider struct { -} +var r = regexp.MustCompile(`Live traffic: (.+): timestamp`) -func (a AzAPIProvider) IsRequestTrace(l rawlog.RawLog) bool { - return l.Level == "DEBUG" && strings.Contains(l.Message, "Request: ==> OUTGOING REQUEST") +type AzAPIProvider struct { } -func (a AzAPIProvider) IsResponseTrace(l rawlog.RawLog) bool { - return l.Level == "DEBUG" && strings.Contains(l.Message, "Response: ==> REQUEST/RESPONSE") +func (a AzAPIProvider) IsTrafficTrace(l rawlog.RawLog) bool { + return l.Level == "DEBUG" && strings.Contains(l.Message, "Live traffic:") } -func (a AzAPIProvider) ParseRequest(l rawlog.RawLog) (*types.RequestTrace, error) { - method := "" - host := "" - uriPath := "" - body := "" - headers := make(map[string]string) - for _, line := range strings.Split(l.Message, "\n") { - line = strings.Trim(line, " ") - switch { - case line == "" || strings.Contains(line, "==>") || - strings.Contains(line, "Request contained no body") || - strings.Contains(line, "-----"): - continue - case strings.Contains(line, ": "): - key, value, err := utils.ParseHeader(line) - if err != nil { - return nil, err - } - headers[key] = value - case utils.IsJson(line): - body = line - default: - if parts := strings.Split(line, " "); len(parts) == 2 { - method = parts[0] - parsedUrl, err := url.Parse(parts[1]) - if err == nil { - host = parsedUrl.Host - uriPath = fmt.Sprintf("%s?%s", parsedUrl.Path, parsedUrl.RawQuery) - } - } - } +func (a AzAPIProvider) ParseTraffic(l rawlog.RawLog) (*types.RequestTrace, error) { + matches := r.FindAllStringSubmatch(l.Message, -1) + if len(matches) == 0 || len(matches[0]) != 2 { + return nil, fmt.Errorf("failed to parse request trace, no matches found") } - return &types.RequestTrace{ - TimeStamp: l.TimeStamp, - Method: method, - Host: host, - Url: utils.NormalizeUrlPath(uriPath), - Provider: "azapi", - Request: &types.HttpRequest{ - Headers: headers, - Body: body, - }, - }, nil -} - -func (a AzAPIProvider) ParseResponse(l rawlog.RawLog) (*types.RequestTrace, error) { - method := "" - host := "" - uriPath := "" - body := "" - headers := make(map[string]string) - - sections := strings.Split(l.Message, strings.Repeat("-", 80)) - message := l.Message - if len(sections) == 4 { - body = sections[2] - message = utils.LineAt(sections[0], 1) + sections[1] + trafficJson := matches[0][1] + var liveTraffic traffic + err := json.Unmarshal([]byte(trafficJson), &liveTraffic) + if err != nil { + return nil, fmt.Errorf("failed to parse request trace, %v", err) } - - for _, line := range strings.Split(message, "\n") { - line = strings.Trim(line, " ") - switch { - case line == "" || strings.Contains(line, "==>") || - strings.Contains(line, "contained no body") || - strings.Contains(line, "-----"): - continue - case strings.Contains(line, ": "): - key, value, err := utils.ParseHeader(line) - if err != nil { - return nil, err - } - headers[key] = value - case utils.IsJson(line): - body = line - default: - if parts := strings.Split(line, " "); len(parts) == 2 { - method = parts[0] - parsedUrl, err := url.Parse(parts[1]) - if err == nil { - host = parsedUrl.Host - uriPath = fmt.Sprintf("%s?%s", parsedUrl.Path, parsedUrl.RawQuery) - } - } - } + parsedUrl, err := url.Parse(liveTraffic.LiveRequest.Url) + if err != nil { + return nil, fmt.Errorf("failed to parse request trace, %v", err) } - statusCode := 0 - if v := headers["RESPONSE Status"]; v != "" { - delete(headers, "RESPONSE Status") - fmt.Sscanf(v, "%d", &statusCode) + if liveTraffic.LiveRequest.Headers == nil { + liveTraffic.LiveRequest.Headers = map[string]string{} } + if liveTraffic.LiveResponse.Headers == nil { + liveTraffic.LiveResponse.Headers = map[string]string{} + } + return &types.RequestTrace{ TimeStamp: l.TimeStamp, - Method: method, - Host: host, - Url: utils.NormalizeUrlPath(uriPath), - StatusCode: statusCode, + Method: liveTraffic.LiveRequest.Method, + Host: parsedUrl.Host, + Url: parsedUrl.Path + "?" + parsedUrl.RawQuery, + StatusCode: liveTraffic.LiveResponse.StatusCode, Provider: "azapi", + Request: &types.HttpRequest{ + Headers: liveTraffic.LiveRequest.Headers, + Body: liveTraffic.LiveRequest.Body, + }, Response: &types.HttpResponse{ - Headers: headers, - Body: body, + Headers: liveTraffic.LiveResponse.Headers, + Body: liveTraffic.LiveResponse.Body, }, }, nil } + +func (a AzAPIProvider) IsRequestTrace(l rawlog.RawLog) bool { + return false +} + +func (a AzAPIProvider) IsResponseTrace(l rawlog.RawLog) bool { + return false +} + +func (a AzAPIProvider) ParseRequest(l rawlog.RawLog) (*types.RequestTrace, error) { + return nil, fmt.Errorf("not implemented") +} + +func (a AzAPIProvider) ParseResponse(l rawlog.RawLog) (*types.RequestTrace, error) { + return nil, fmt.Errorf("not implemented") +} + +type traffic struct { + LiveRequest liveRequest `json:"request"` + LiveResponse liveResponse `json:"response"` +} + +type liveRequest struct { + Headers map[string]string `json:"headers"` + Method string `json:"method"` + Url string `json:"url"` + Body string `json:"body"` +} + +type liveResponse struct { + StatusCode int `json:"statusCode"` + Headers map[string]string `json:"headers"` + Body string `json:"body"` +} diff --git a/provider/azapi_test.go b/provider/azapi_test.go index 3ae3373..4824212 100644 --- a/provider/azapi_test.go +++ b/provider/azapi_test.go @@ -1,8 +1,6 @@ package provider_test import ( - "encoding/json" - "reflect" "testing" "github.com/ms-henglu/pal/provider" @@ -10,64 +8,17 @@ import ( "github.com/ms-henglu/pal/types" ) -func TestAzAPIProvider_IsRequestTrace(t *testing.T) { +func TestAzAPIProvider_IsTrafficTrace(t *testing.T) { testcases := []struct { name string log string want bool }{ { - name: "azapi request trace", - log: "2023-04-28T13:13:16.092+0800 [DEBUG] provider.terraform-provider-azapi: Apr 28 13:13:16.092051 Request: ==> OUTGOING REQUEST (Try=1)", + name: "azapi GET request trace", + log: "2023-09-15T13:40:31.447+0800 [DEBUG] provider.terraform-provider-azapi: Live traffic: {}: timestamp=2023-09-15T13:40:31.447+0800", want: true, }, - { - name: "azapi response trace", - log: "2023-04-28T13:13:18.304+0800 [DEBUG] provider.terraform-provider-azapi: Apr 28 13:13:18.304139 Response: ==> REQUEST/RESPONSE (Try=1/2.211800791s, OpTime=2.211881666s) -- RESPONSE RECEIVED", - want: false, - }, - { - name: "azurerm request trace", - log: "2023-04-28T13:12:48.330+0800 [DEBUG] provider.terraform-provider-azurerm: AzureRM Request:", - want: false, - }, - { - name: "azurerm response trace", - log: "2023-04-28T13:12:48.908+0800 [DEBUG] provider.terraform-provider-azurerm: AzureRM Response for https://management.azure.com/subscriptions/******/resourcegroups/henglu1114?api-version=2020-06-01:", - want: false, - }, - { - name: "azuread request trace", - log: "2023-04-14T15:14:53.530+0800 [INFO] provider.terraform-provider-azuread: 2023/04/14 15:14:53 [DEBUG] ============================ Begin AzureAD Request ============================", - want: false, - }, - { - name: "azuread response trace", - log: "2023-04-14T15:14:54.084+0800 [INFO] provider.terraform-provider-azuread: 2023/04/14 15:14:54 [DEBUG] ============================ Begin AzureAD Response ===========================", - want: false, - }, - } - - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - l, err := rawlog.NewRawLog(tc.log) - if err != nil { - t.Fatalf("failed to parse log: %v", err) - } - got := provider.AzAPIProvider{}.IsRequestTrace(*l) - if got != tc.want { - t.Errorf("IsRequestTrace() = %v, want %v", got, tc.want) - } - }) - } -} - -func TestAzAPIProvider_IsResponseTrace(t *testing.T) { - testcases := []struct { - name string - log string - want bool - }{ { name: "azapi request trace", log: "2023-04-28T13:13:16.092+0800 [DEBUG] provider.terraform-provider-azapi: Apr 28 13:13:16.092051 Request: ==> OUTGOING REQUEST (Try=1)", @@ -76,7 +27,7 @@ func TestAzAPIProvider_IsResponseTrace(t *testing.T) { { name: "azapi response trace", log: "2023-04-28T13:13:18.304+0800 [DEBUG] provider.terraform-provider-azapi: Apr 28 13:13:18.304139 Response: ==> REQUEST/RESPONSE (Try=1/2.211800791s, OpTime=2.211881666s) -- RESPONSE RECEIVED", - want: true, + want: false, }, { name: "azurerm request trace", @@ -106,15 +57,15 @@ func TestAzAPIProvider_IsResponseTrace(t *testing.T) { if err != nil { t.Fatalf("failed to parse log: %v", err) } - got := provider.AzAPIProvider{}.IsResponseTrace(*l) + got := provider.AzAPIProvider{}.IsTrafficTrace(*l) if got != tc.want { - t.Errorf("IsResponseTrace() = %v, want %v", got, tc.want) + t.Errorf("IsRequestTrace() = %v, want %v", got, tc.want) } }) } } -func TestAzAPIProvider_ParseRequest(t *testing.T) { +func TestAzAPIProvider_ParseTraffic(t *testing.T) { testcases := []struct { name string log string @@ -122,278 +73,132 @@ func TestAzAPIProvider_ParseRequest(t *testing.T) { }{ { name: "azapi GET request trace", - log: `2023-04-28T13:12:52.862+0800 [DEBUG] provider.terraform-provider-azapi: Apr 28 13:12:52.862113 Request: ==> OUTGOING REQUEST (Try=1) - GET https://management.azure.com/subscriptions/******/resourceGroups/henglu1114/providers/Microsoft.Automation/automationAccounts/henglu1?api-version=2020-01-13-preview - Accept: application/json - Authorization: REDACTED - User-Agent: HashiCorp Terraform/1.4.5 (+https://www.terraform.io) Terraform Plugin SDK/2.8.0 terraform-provider-azapi/dev pid-222c6c49-1b0a-5959-a213-6608f9eb8820 - X-Ms-Correlation-Request-Id: 8817767b-435f-9298-42f8-534407f68afb - Request contained no body: timestamp=2023-04-28T13:12:52.862+0800`, + log: `2023-09-15T13:40:31.447+0800 [DEBUG] provider.terraform-provider-azapi: Live traffic: {"request":{"headers":{"Accept":"application/json","User-Agent":"HashiCorp Terraform/1.5.2 (+https://www.terraform.io) Terraform Plugin SDK/2.8.0 terraform-provider-azapi/dev pid-222c6c49-1b0a-5959-a213-6608f9eb8820","X-Ms-Correlation-Request-Id":"ea90befd-41e6-0f04-a1e5-3be4d56632cc"},"method":"GET","url":"https://management.azure.com/subscriptions/******/resourceGroups/henglu915?api-version=2023-07-01","body":""},"response":{"statusCode":200,"headers":{"Cache-Control":"no-cache","Content-Type":"application/json; charset=utf-8","Date":"Fri, 15 Sep 2023 05:40:30 GMT","Expires":"-1","Pragma":"no-cache","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Vary":"Accept-Encoding","X-Content-Type-Options":"nosniff","X-Ms-Correlation-Request-Id":"ea90befd-41e6-0f04-a1e5-3be4d56632cc","X-Ms-Ratelimit-Remaining-Subscription-Reads":"11999","X-Ms-Request-Id":"04ae0566-5067-4114-816c-1dace25a7b65","X-Ms-Routing-Request-Id":"SOUTHEASTASIA:20230915T054031Z:04ae0566-5067-4114-816c-1dace25a7b65"},"body":"{\"id\":\"/subscriptions/******/resourceGroups/henglu915\",\"name\":\"henglu915\",\"type\":\"Microsoft.Resources/resourceGroups\",\"location\":\"westus\",\"properties\":{\"provisioningState\":\"Succeeded\"}}"}}: timestamp=2023-09-15T13:40:31.447+0800`, want: types.RequestTrace{ - Provider: "azapi", - Method: "GET", - Host: "management.azure.com", - Url: "/subscriptions/******/resourceGroups/henglu1114/providers/Microsoft.Automation/automationAccounts/henglu1?api-version=2020-01-13-preview", - Request: &types.HttpRequest{ - Headers: map[string]string{ - "Accept": "application/json", - "Authorization": "REDACTED", - "User-Agent": "HashiCorp Terraform/1.4.5 (+https://www.terraform.io) Terraform Plugin SDK/2.8.0 terraform-provider-azapi/dev pid-222c6c49-1b0a-5959-a213-6608f9eb8820", - "X-Ms-Correlation-Request-Id": "8817767b-435f-9298-42f8-534407f68afb", - }, - }, - }, - }, - { - name: "azapi PUT request trace", - log: `2023-04-28T13:12:54.170+0800 [DEBUG] provider.terraform-provider-azapi: Apr 28 13:12:54.169864 Request: ==> OUTGOING REQUEST (Try=1) - PUT https://management.azure.com/subscriptions/******/resourceGroups/henglu1114/providers/Microsoft.Automation/automationAccounts/henglu1?api-version=2020-01-13-preview - Accept: application/json - Authorization: REDACTED - Content-Length: 80 - Content-Type: application/json - User-Agent: HashiCorp Terraform/1.4.5 (+https://www.terraform.io) Terraform Plugin SDK/2.8.0 terraform-provider-azapi/dev pid-222c6c49-1b0a-5959-a213-6608f9eb8820 - X-Ms-Correlation-Request-Id: 8817767b-435f-9298-42f8-534407f68afb - -------------------------------------------------------------------------------- -{"location":"westeurope","name":"henglu1","properties":{"sku":{"name":"Basic"}}} - --------------------------------------------------------------------------------: timestamp=2023-04-28T13:12:54.169+0800`, - want: types.RequestTrace{ - Provider: "azapi", - Method: "PUT", - Host: "management.azure.com", - Url: "/subscriptions/******/resourceGroups/henglu1114/providers/Microsoft.Automation/automationAccounts/henglu1?api-version=2020-01-13-preview", + Provider: "azapi", + Method: "GET", + Host: "management.azure.com", + Url: "/subscriptions/******/resourceGroups/henglu915?api-version=2023-07-01", + StatusCode: 200, Request: &types.HttpRequest{ Headers: map[string]string{ "Accept": "application/json", - "Authorization": "REDACTED", - "Content-Length": "80", - "Content-Type": "application/json", - "User-Agent": "HashiCorp Terraform/1.4.5 (+https://www.terraform.io) Terraform Plugin SDK/2.8.0 terraform-provider-azapi/dev pid-222c6c49-1b0a-5959-a213-6608f9eb8820", - "X-Ms-Correlation-Request-Id": "8817767b-435f-9298-42f8-534407f68afb", + "User-Agent": "HashiCorp Terraform/1.5.2 (+https://www.terraform.io) Terraform Plugin SDK/2.8.0 terraform-provider-azapi/dev pid-222c6c49-1b0a-5959-a213-6608f9eb8820", + "X-Ms-Correlation-Request-Id": "ea90befd-41e6-0f04-a1e5-3be4d56632cc", }, - Body: `{"location":"westeurope","name":"henglu1","properties":{"sku":{"name":"Basic"}}}`, + Body: "", }, - }, - }, - } - - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - l, err := rawlog.NewRawLog(tc.log) - if err != nil { - t.Fatalf("failed to parse log: %v", err) - } - got, err := provider.AzAPIProvider{}.ParseRequest(*l) - if err != nil { - t.Fatalf("failed to parse request: %v", err) - } - if got.Host != tc.want.Host { - t.Errorf("ParseRequest() host = %v, want %v", got.Host, tc.want.Host) - } - if got.Method != tc.want.Method { - t.Errorf("ParseRequest() method = %v, want %v", got.Method, tc.want.Method) - } - if got.Url != tc.want.Url { - t.Errorf("ParseRequest() url = %v, want %v", got.Url, tc.want.Url) - } - if got.Request == nil { - t.Errorf("ParseRequest() request = nil, want not nil") - } else { - if len(got.Request.Headers) != len(tc.want.Request.Headers) { - t.Errorf("ParseRequest() request headers length = %v, want %v", len(got.Request.Headers), len(tc.want.Request.Headers)) - } - for k, v := range got.Request.Headers { - if v != tc.want.Request.Headers[k] { - t.Errorf("ParseRequest() request header %v = %v, want %v", k, v, tc.want.Request.Headers[k]) - } - } - if len(tc.want.Request.Body) == 0 { - if len(got.Request.Body) != 0 { - t.Errorf("ParseRequest() request body = %v, want %v", got.Request.Body, tc.want.Request.Body) - } - return - } - - var gotBody, wantBody interface{} - err = json.Unmarshal([]byte(got.Request.Body), &gotBody) - if err != nil { - t.Errorf("ParseRequest() request body unmarshal error = %v", err) - } - err = json.Unmarshal([]byte(tc.want.Request.Body), &wantBody) - if err != nil { - t.Errorf("ParseRequest() request body unmarshal error = %v", err) - } - - if !reflect.DeepEqual(gotBody, wantBody) { - t.Errorf("ParseRequest() request body = %v, want %v", gotBody, wantBody) - } - } - - }) - } -} - -func TestAzAPIProvider_ParseResponse(t *testing.T) { - testcases := []struct { - name string - log string - want types.RequestTrace - }{ - { - name: "azapi GET response trace", - log: `2023-04-28T13:12:54.167+0800 [DEBUG] provider.terraform-provider-azapi: Apr 28 13:12:54.166603 Response: ==> REQUEST/RESPONSE (Try=1/1.304370833s, OpTime=1.304446958s) -- RESPONSE RECEIVED - GET https://management.azure.com/subscriptions/******/resourceGroups/henglu1114/providers/Microsoft.Automation/automationAccounts/henglu1?api-version=2020-01-13-preview - Accept: application/json - Authorization: REDACTED - User-Agent: HashiCorp Terraform/1.4.5 (+https://www.terraform.io) Terraform Plugin SDK/2.8.0 terraform-provider-azapi/dev pid-222c6c49-1b0a-5959-a213-6608f9eb8820 - X-Ms-Correlation-Request-Id: 8817767b-435f-9298-42f8-534407f68afb - -------------------------------------------------------------------------------- - RESPONSE Status: 404 Not Found - Cache-Control: no-cache - Content-Length: 229 - Content-Type: application/json; charset=utf-8 - Date: Fri, 28 Apr 2023 05:12:53 GMT - Expires: -1 - Pragma: no-cache - Strict-Transport-Security: max-age=31536000; includeSubDomains - X-Content-Type-Options: nosniff - X-Ms-Correlation-Request-Id: 8817767b-435f-9298-42f8-534407f68afb - X-Ms-Failure-Cause: REDACTED - X-Ms-Request-Id: e718dd71-7703-4ca0-a461-9cf510071ede - X-Ms-Routing-Request-Id: SOUTHEASTASIA:20230428T051254Z:e718dd71-7703-4ca0-a461-9cf510071ede - -------------------------------------------------------------------------------- -{"error":{"code":"ResourceNotFound","message":"The Resource 'Microsoft.Automation/automationAccounts/henglu1' under resource group 'henglu1114' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix"}} - --------------------------------------------------------------------------------: timestamp=2023-04-28T13:12:54.166+0800 -`, - want: types.RequestTrace{ - Provider: "azapi", - Method: "GET", - StatusCode: 404, - Host: "management.azure.com", - Url: "/subscriptions/******/resourceGroups/henglu1114/providers/Microsoft.Automation/automationAccounts/henglu1?api-version=2020-01-13-preview", Response: &types.HttpResponse{ Headers: map[string]string{ "Cache-Control": "no-cache", - "Content-Length": "229", "Content-Type": "application/json; charset=utf-8", - "Date": "Fri, 28 Apr 2023 05:12:53 GMT", + "Date": "Fri, 15 Sep 2023 05:40:30 GMT", "Expires": "-1", "Pragma": "no-cache", "Strict-Transport-Security": "max-age=31536000; includeSubDomains", + "Vary": "Accept-Encoding", "X-Content-Type-Options": "nosniff", - "X-Ms-Correlation-Request-Id": "8817767b-435f-9298-42f8-534407f68afb", - "X-Ms-Failure-Cause": "REDACTED", - "X-Ms-Request-Id": "e718dd71-7703-4ca0-a461-9cf510071ede", - "X-Ms-Routing-Request-Id": "SOUTHEASTASIA:20230428T051254Z:e718dd71-7703-4ca0-a461-9cf510071ede", + "X-Ms-Correlation-Request-Id": "ea90befd-41e6-0f04-a1e5-3be4d56632cc", + "X-Ms-Ratelimit-Remaining-Subscription-Reads": "11999", + "X-Ms-Request-Id": "04ae0566-5067-4114-816c-1dace25a7b65", + "X-Ms-Routing-Request-Id": "SOUTHEASTASIA:20230915T054031Z:04ae0566-5067-4114-816c-1dace25a7b65", }, - Body: `{"error":{"code":"ResourceNotFound","message":"The Resource 'Microsoft.Automation/automationAccounts/henglu1' under resource group 'henglu1114' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix"}}`, + Body: "{\"id\":\"/subscriptions/******/resourceGroups/henglu915\",\"name\":\"henglu915\",\"type\":\"Microsoft.Resources/resourceGroups\",\"location\":\"westus\",\"properties\":{\"provisioningState\":\"Succeeded\"}}", }, }, }, { - name: "azapi PUT response trace", - log: `2023-04-28T13:13:00.563+0800 [DEBUG] provider.terraform-provider-azapi: Apr 28 13:13:00.563408 Response: ==> REQUEST/RESPONSE (Try=1/6.393429583s, OpTime=6.393500375s) -- RESPONSE RECEIVED - PUT https://management.azure.com/subscriptions/******/resourceGroups/henglu1114/providers/Microsoft.Automation/automationAccounts/henglu1?api-version=2020-01-13-preview - Accept: application/json - Authorization: REDACTED - Content-Length: 80 - Content-Type: application/json - User-Agent: HashiCorp Terraform/1.4.5 (+https://www.terraform.io) Terraform Plugin SDK/2.8.0 terraform-provider-azapi/dev pid-222c6c49-1b0a-5959-a213-6608f9eb8820 - X-Ms-Correlation-Request-Id: 8817767b-435f-9298-42f8-534407f68afb - -------------------------------------------------------------------------------- - RESPONSE Status: 201 Created - Cache-Control: no-cache - Content-Length: 812 - Content-Type: application/json; charset=utf-8 - Date: Fri, 28 Apr 2023 05:13:00 GMT - Expires: -1 - Location: REDACTED - Pragma: no-cache - Server: Microsoft-HTTPAPI/2.0 - Strict-Transport-Security: max-age=31536000; includeSubDomains - X-Content-Type-Options: nosniff - X-Ms-Correlation-Request-Id: 8817767b-435f-9298-42f8-534407f68afb - X-Ms-Ratelimit-Remaining-Subscription-Writes: 1198 - X-Ms-Request-Id: 5e488609-4d2d-4c51-9efe-ee293d97ddd5 - X-Ms-Routing-Request-Id: SOUTHEASTASIA:20230428T051300Z:8cca4a82-b200-4546-9114-96bd6a3b9dfa - -------------------------------------------------------------------------------- -{"name":"henglu1","id":"/subscriptions/******/resourceGroups/henglu1114/providers/Microsoft.Automation/automationAccounts/henglu1","type":"Microsoft.Automation/AutomationAccounts","location":"westeurope","tags":{},"etag":null,"properties":{"sku":{"name":"Basic","family":null,"capacity":null},"state":"Ok","RegistrationUrl":"https://ff3a2b15-73eb-4d82-af7b-b399b3560822.agentsvc.we.azure-automation.net/accounts/ff3a2b15-73eb-4d82-af7b-b399b3560822","encryption":{"keySource":"Microsoft.Automation","identity":{"userAssignedIdentity":null}},"RuntimeConfiguration":{"powershell":{"builtinModules":{"Az":"8.0.0"}},"powershell7":{"builtinModules":{"Az":"8.0.0"}}},"creationTime":"2023-04-28T05:12:58.247+00:00","lastModifiedBy":null,"lastModifiedTime":"2023-04-28T05:12:58.247+00:00"}} - --------------------------------------------------------------------------------: timestamp=2023-04-28T13:13:00.563+0800 -`, + name: "azapi PUT request trace", + log: `2023-09-15T14:33:49.909+0800 [DEBUG] provider.terraform-provider-azapi: Live traffic: {"request":{"headers":{"Accept":"application/json","Content-Length":"40","Content-Type":"application/json","User-Agent":"HashiCorp Terraform/1.5.2 (+https://www.terraform.io) Terraform Plugin SDK/2.8.0 terraform-provider-azapi/dev pid-222c6c49-1b0a-5959-a213-6608f9eb8820","X-Ms-Correlation-Request-Id":"6b6ed2c7-055a-3986-5d5f-4c04a4b78193"},"method":"PUT","url":"https://management.azure.com/subscriptions/******/resourceGroups/henglu915?api-version=2023-07-01","body":"{\"location\":\"westus\",\"name\":\"henglu915\"}"},"response":{"statusCode":201,"headers":{"Cache-Control":"no-cache","Content-Length":"215","Content-Type":"application/json; charset=utf-8","Date":"Fri, 15 Sep 2023 06:33:49 GMT","Expires":"-1","Pragma":"no-cache","Strict-Transport-Security":"max-age=31536000; includeSubDomains","X-Content-Type-Options":"nosniff","X-Ms-Correlation-Request-Id":"6b6ed2c7-055a-3986-5d5f-4c04a4b78193","X-Ms-Ratelimit-Remaining-Subscription-Writes":"1199","X-Ms-Request-Id":"e1eef6e2-b814-4f69-82b6-4c117d33cc13","X-Ms-Routing-Request-Id":"SOUTHEASTASIA:20230915T063350Z:e1eef6e2-b814-4f69-82b6-4c117d33cc13"},"body":"{\"id\":\"/subscriptions/******/resourceGroups/henglu915\",\"name\":\"henglu915\",\"type\":\"Microsoft.Resources/resourceGroups\",\"location\":\"westus\",\"properties\":{\"provisioningState\":\"Succeeded\"}}"}}: timestamp=2023-09-15T14:33:49.908+0800`, want: types.RequestTrace{ Provider: "azapi", - StatusCode: 201, - Host: "management.azure.com", - Url: "/subscriptions/******/resourceGroups/henglu1114/providers/Microsoft.Automation/automationAccounts/henglu1?api-version=2020-01-13-preview", Method: "PUT", + Host: "management.azure.com", + Url: "/subscriptions/******/resourceGroups/henglu915?api-version=2023-07-01", + StatusCode: 201, + Request: &types.HttpRequest{ + Headers: map[string]string{ + "Accept": "application/json", + "Content-Length": "40", + "Content-Type": "application/json", + "User-Agent": "HashiCorp Terraform/1.5.2 (+https://www.terraform.io) Terraform Plugin SDK/2.8.0 terraform-provider-azapi/dev pid-222c6c49-1b0a-5959-a213-6608f9eb8820", + "X-Ms-Correlation-Request-Id": "6b6ed2c7-055a-3986-5d5f-4c04a4b78193", + }, + Body: "{\"location\":\"westus\",\"name\":\"henglu915\"}", + }, Response: &types.HttpResponse{ Headers: map[string]string{ "Cache-Control": "no-cache", - "Content-Length": "812", + "Content-Length": "215", "Content-Type": "application/json; charset=utf-8", - "Date": "Fri, 28 Apr 2023 05:13:00 GMT", + "Date": "Fri, 15 Sep 2023 06:33:49 GMT", "Expires": "-1", - "Location": "REDACTED", "Pragma": "no-cache", - "Server": "Microsoft-HTTPAPI/2.0", "Strict-Transport-Security": "max-age=31536000; includeSubDomains", "X-Content-Type-Options": "nosniff", - "X-Ms-Correlation-Request-Id": "8817767b-435f-9298-42f8-534407f68afb", - "X-Ms-Ratelimit-Remaining-Subscription-Writes": "1198", - "X-Ms-Request-Id": "5e488609-4d2d-4c51-9efe-ee293d97ddd5", - "X-Ms-Routing-Request-Id": "SOUTHEASTASIA:20230428T051300Z:8cca4a82-b200-4546-9114-96bd6a3b9dfa", + "X-Ms-Correlation-Request-Id": "6b6ed2c7-055a-3986-5d5f-4c04a4b78193", + "X-Ms-Ratelimit-Remaining-Subscription-Writes": "1199", + "X-Ms-Request-Id": "e1eef6e2-b814-4f69-82b6-4c117d33cc13", + "X-Ms-Routing-Request-Id": "SOUTHEASTASIA:20230915T063350Z:e1eef6e2-b814-4f69-82b6-4c117d33cc13", }, - Body: `{"name":"henglu1","id":"/subscriptions/******/resourceGroups/henglu1114/providers/Microsoft.Automation/automationAccounts/henglu1","type":"Microsoft.Automation/AutomationAccounts","location":"westeurope","tags":{},"etag":null,"properties":{"sku":{"name":"Basic","family":null,"capacity":null},"state":"Ok","RegistrationUrl":"https://ff3a2b15-73eb-4d82-af7b-b399b3560822.agentsvc.we.azure-automation.net/accounts/ff3a2b15-73eb-4d82-af7b-b399b3560822","encryption":{"keySource":"Microsoft.Automation","identity":{"userAssignedIdentity":null}},"RuntimeConfiguration":{"powershell":{"builtinModules":{"Az":"8.0.0"}},"powershell7":{"builtinModules":{"Az":"8.0.0"}}},"creationTime":"2023-04-28T05:12:58.247+00:00","lastModifiedBy":null,"lastModifiedTime":"2023-04-28T05:12:58.247+00:00"}}`, + Body: "{\"id\":\"/subscriptions/******/resourceGroups/henglu915\",\"name\":\"henglu915\",\"type\":\"Microsoft.Resources/resourceGroups\",\"location\":\"westus\",\"properties\":{\"provisioningState\":\"Succeeded\"}}", }, }, }, } for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - l, err := rawlog.NewRawLog(tc.log) - if err != nil { - t.Fatalf("failed to parse log: %v", err) - } - got, err := provider.AzAPIProvider{}.ParseResponse(*l) - if err != nil { - t.Fatalf("ParseResponse() error = %v", err) - } - if got.Host != tc.want.Host { - t.Errorf("ParseResponse() host = %v, want %v", got.Host, tc.want.Host) - } - if got.Method != tc.want.Method { - t.Errorf("ParseResponse() method = %v, want %v", got.Method, tc.want.Method) - } - if got.Url != tc.want.Url { - t.Errorf("ParseResponse() url = %v, want %v", got.Url, tc.want.Url) - } - if got.Response == nil { - t.Errorf("ParseResponse() response = nil, want not nil") - } else { - if len(got.Response.Headers) != len(tc.want.Response.Headers) { - t.Errorf("ParseResponse() response headers length = %v, want %v", len(got.Response.Headers), len(tc.want.Response.Headers)) - } - for k, v := range got.Response.Headers { - if v != tc.want.Response.Headers[k] { - t.Errorf("ParseResponse() response header %v = %v, want %v", k, v, tc.want.Response.Headers[k]) - } - } - var gotBody, wantBody interface{} - err = json.Unmarshal([]byte(got.Response.Body), &gotBody) - if err != nil { - t.Errorf("ParseResponse() response body unmarshal error = %v", err) - } - err = json.Unmarshal([]byte(tc.want.Response.Body), &wantBody) - if err != nil { - t.Errorf("ParseResponse() response body unmarshal error = %v", err) - } - - if !reflect.DeepEqual(gotBody, wantBody) { - t.Errorf("ParseResponse() response body = %v, want %v", gotBody, wantBody) - } - } - - }) + l, err := rawlog.NewRawLog(tc.log) + if err != nil { + t.Fatalf("failed to parse log: %v", err) + } + got, err := provider.AzAPIProvider{}.ParseTraffic(*l) + if err != nil { + t.Fatalf("failed to parse request: %v", err) + } + if got.Host != tc.want.Host { + t.Errorf("ParseRequest() host = %v, want %v", got.Host, tc.want.Host) + } + if got.Method != tc.want.Method { + t.Errorf("ParseRequest() method = %v, want %v", got.Method, tc.want.Method) + } + if got.Url != tc.want.Url { + t.Errorf("ParseRequest() url = %v, want %v", got.Url, tc.want.Url) + } + if got.StatusCode != tc.want.StatusCode { + t.Errorf("ParseRequest() status code = %v, want %v", got.StatusCode, tc.want.StatusCode) + } + if got.Request == nil { + t.Errorf("ParseRequest() request is nil") + continue + } + if got.Response == nil { + t.Errorf("ParseRequest() response is nil") + continue + } + if got.Request.Body != tc.want.Request.Body { + t.Errorf("ParseRequest() request body = %v, want %v", got.Request.Body, tc.want.Request.Body) + } + if got.Response.Body != tc.want.Response.Body { + t.Errorf("ParseRequest() response body = %v, want %v", got.Response.Body, tc.want.Response.Body) + } + if len(got.Request.Headers) != len(tc.want.Request.Headers) { + t.Errorf("ParseRequest() request headers length = %v, want %v", len(got.Request.Headers), len(tc.want.Request.Headers)) + continue + } + for k, v := range got.Request.Headers { + if tc.want.Request.Headers[k] != v { + t.Errorf("ParseRequest() request header %v = %v, want %v", k, v, tc.want.Request.Headers[k]) + } + } + if len(got.Response.Headers) != len(tc.want.Response.Headers) { + t.Errorf("ParseRequest() response headers length = %v, want %v", len(got.Response.Headers), len(tc.want.Response.Headers)) + continue + } + for k, v := range got.Response.Headers { + if tc.want.Response.Headers[k] != v { + t.Errorf("ParseRequest() response header %v = %v, want %v", k, v, tc.want.Response.Headers[k]) + } + } } } diff --git a/provider/azuread.go b/provider/azuread.go index 566de7b..7384ed9 100644 --- a/provider/azuread.go +++ b/provider/azuread.go @@ -17,6 +17,14 @@ var statusCodeRegex = regexp.MustCompile(`HTTP/\d.\d\s(\d{3})\s.+`) type AzureADProvider struct { } +func (a AzureADProvider) IsTrafficTrace(l rawlog.RawLog) bool { + return false +} + +func (a AzureADProvider) ParseTraffic(l rawlog.RawLog) (*types.RequestTrace, error) { + return nil, fmt.Errorf("not implemented") +} + func (a AzureADProvider) IsRequestTrace(l rawlog.RawLog) bool { return l.Level == "INFO" && strings.Contains(l.Message, "============================ Begin AzureAD Request") } diff --git a/provider/azurerm.go b/provider/azurerm.go index f900ebf..4cb7497 100644 --- a/provider/azurerm.go +++ b/provider/azurerm.go @@ -15,6 +15,14 @@ var _ Provider = AzureRMProvider{} type AzureRMProvider struct { } +func (a AzureRMProvider) IsTrafficTrace(l rawlog.RawLog) bool { + return false +} + +func (a AzureRMProvider) ParseTraffic(l rawlog.RawLog) (*types.RequestTrace, error) { + return nil, fmt.Errorf("not implemented") +} + func (a AzureRMProvider) IsRequestTrace(l rawlog.RawLog) bool { return l.Level == "DEBUG" && strings.Contains(l.Message, "AzureRM Request:") } diff --git a/provider/base.go b/provider/base.go index 082dd38..34e8245 100644 --- a/provider/base.go +++ b/provider/base.go @@ -6,8 +6,10 @@ import ( ) type Provider interface { + IsTrafficTrace(l rawlog.RawLog) bool IsRequestTrace(l rawlog.RawLog) bool IsResponseTrace(l rawlog.RawLog) bool + ParseTraffic(l rawlog.RawLog) (*types.RequestTrace, error) ParseRequest(l rawlog.RawLog) (*types.RequestTrace, error) ParseResponse(l rawlog.RawLog) (*types.RequestTrace, error) } diff --git a/rawlog/raw_log_test.go b/rawlog/raw_log_test.go index ae25d23..2ff112d 100644 --- a/rawlog/raw_log_test.go +++ b/rawlog/raw_log_test.go @@ -63,8 +63,8 @@ func Test_NewRawLog(t *testing.T) { t.Errorf("want not nil, got nil") continue } - if got.TimeStamp != tc.want.TimeStamp { - t.Errorf("want timestamp %v, got %v", tc.want.TimeStamp, got.TimeStamp) + if got.TimeStamp.Unix() != tc.want.TimeStamp.Unix() { + t.Errorf("want timestamp %v, got %v", tc.want.TimeStamp.Unix(), got.TimeStamp.Unix()) } if got.Level != tc.want.Level { t.Errorf("want level %s, got %s", tc.want.Level, got.Level) diff --git a/trace/trace.go b/trace/trace.go index 4580ea3..d2cc339 100644 --- a/trace/trace.go +++ b/trace/trace.go @@ -19,6 +19,8 @@ var providers = []provider.Provider{ provider.AzAPIProvider{}, } +var providerUrlRegex = regexp.MustCompile(`/subscriptions/[a-zA-Z\d\-]+/providers`) + func RequestTracesFromFile(input string) ([]types.RequestTrace, error) { data, err := os.ReadFile(input) if err != nil { @@ -57,6 +59,16 @@ func RequestTracesFromFile(input string) ([]types.RequestTrace, error) { mergedTraces := make([]types.RequestTrace, 0) for i := 0; i < len(traces); i++ { + // skip GET /subscriptions/******/providers + if traces[i].Method == "GET" && providerUrlRegex.MatchString(traces[i].Url) { + continue + } + + if traces[i].Request != nil && traces[i].Response != nil { + mergedTraces = append(mergedTraces, traces[i]) + continue + } + if traces[i].Request != nil { found := false for j := i + 1; j < len(traces); j++ { @@ -139,6 +151,9 @@ func VerifyRequestTrace(t types.RequestTrace) []string { func NewRequestTrace(l rawlog.RawLog) (*types.RequestTrace, error) { for _, p := range providers { + if p.IsTrafficTrace(l) { + return p.ParseTraffic(l) + } if p.IsRequestTrace(l) { return p.ParseRequest(l) }