From f65499b64057520bb2d6c82b24f4a0bad21030f9 Mon Sep 17 00:00:00 2001 From: Rob Cresswell Date: Wed, 16 Aug 2023 11:03:28 +0100 Subject: [PATCH] feat: use v2 API if available This patch adds support for using the new v2 API by default, and falling back to v1. This shouldn't be a breaking change until we remove support for the v1 API Signed-off-by: Rob Cresswell --- go.mod | 2 + go.sum | 6 + pkg/reporter/reporter.go | 148 ++++++++++++++++++++----- pkg/reporter/reporter_test.go | 202 ++++++++++++++++++++++++++++++++-- 4 files changed, 323 insertions(+), 35 deletions(-) diff --git a/go.mod b/go.mod index f683c2b..a74ff69 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.20 require ( github.com/adrg/xdg v0.4.0 github.com/aws/aws-sdk-go v1.44.163 + github.com/h2non/gock v1.2.0 github.com/mitchellh/go-homedir v1.1.0 github.com/spf13/cobra v1.6.1 github.com/spf13/viper v1.14.0 @@ -16,6 +17,7 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect diff --git a/go.sum b/go.sum index 0380489..342de8a 100644 --- a/go.sum +++ b/go.sum @@ -122,6 +122,10 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/h2non/gock v1.2.0 h1:K6ol8rfrRkUOefooBC8elXoaNGYkpp7y2qcxGG6BzUE= +github.com/h2non/gock v1.2.0/go.mod h1:tNhoxHYW2W42cYkYb1WqzdbYIieALC99kpYr7rH/BQk= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -149,6 +153,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= diff --git a/pkg/reporter/reporter.go b/pkg/reporter/reporter.go index 2ed04d8..93fa635 100644 --- a/pkg/reporter/reporter.go +++ b/pkg/reporter/reporter.go @@ -6,72 +6,164 @@ import ( "crypto/tls" "encoding/json" "fmt" + "io" "net/http" "net/url" "time" + "github.com/h2non/gock" + "github.com/anchore/ecs-inventory/internal/logger" "github.com/anchore/ecs-inventory/internal/tracker" "github.com/anchore/ecs-inventory/pkg/connection" ) -const ReportAPIPath = "v1/enterprise/ecs-inventory" +const v1ReportAPIPath = "v1/enterprise/ecs-inventory" +const v2ReportAPIPath = "v2/ecs-inventory" + +var apiPath = v2ReportAPIPath -// This method does the actual Reporting (via HTTP) to Anchore -// -//nolint:gosec func Post(report Report, anchoreDetails connection.AnchoreInfo) error { + logger.Log.Info("Reporting results to Anchore") defer tracker.TrackFunctionTime(time.Now(), fmt.Sprintf("Posting Inventory Report for cluster %s", report.ClusterARN)) - logger.Log.Info("Reporting results to Anchore", "Account", anchoreDetails.Account) tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: anchoreDetails.HTTP.Insecure}, - } + } // #nosec G402 client := &http.Client{ Transport: tr, Timeout: time.Duration(anchoreDetails.HTTP.TimeoutSeconds) * time.Second, } + gock.InterceptClient(client) + + req, err := prepareRequest(report, anchoreDetails) + + if err != nil { + return err + } + + resp, err := client.Do(req) + if err != nil { + if resp != nil && resp.StatusCode == 401 { + return fmt.Errorf("failed to report data to Anchore, check credentials: %w", err) + } + return fmt.Errorf("failed to report data to Anchore: %w", err) + } + defer resp.Body.Close() + + // If we get a 404, make an assumption that the backend API support may have + // changed, either because our default v2 is too new or because the API + // service has been upgraded. Check the version, and if the version changes, + // cache it and retry the request + if resp.StatusCode == 404 { + previousAPIPath := apiPath + apiPath, err = fetchVersionedAPIPath(anchoreDetails) + if err != nil { + return fmt.Errorf("failed to validate Enterprise API: %w", err) + } + apiEndpoint, err := url.JoinPath(anchoreDetails.URL, apiPath) + if err != nil { + return fmt.Errorf("failed to parse API URL: %w", err) + } + + if apiPath != previousAPIPath { + logger.Log.Info("Retrying inventory report with new endpoint", "apiEndpoint", apiEndpoint) + return Post(report, anchoreDetails) + } + + return fmt.Errorf("failed to report data to Anchore: %+v", resp) + } + + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return fmt.Errorf("failed to report data to Anchore: %+v", resp) + } + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read response from Anchore: %w", err) + } + if len(respBody) > 0 && !json.Valid(respBody) { + logger.Log.Debug("Anchore response body: ", string(respBody)) + return fmt.Errorf("failed to report data to Anchore not a valid json response: %+v", resp) + } + logger.Log.Debug("Successfully reported results to Anchore") + return nil +} - anchoreURL, err := buildURL(anchoreDetails) +func prepareRequest(report Report, anchoreDetails connection.AnchoreInfo) (*http.Request, error) { + apiEndpoint, err := url.JoinPath(anchoreDetails.URL, apiPath) if err != nil { - return fmt.Errorf("failed to build url: %w", err) + return nil, fmt.Errorf("failed to parse API URL: %w", err) } + logger.Log.Debug("Reporting results to Anchore", "Endpoint", apiEndpoint) reqBody, err := json.Marshal(report) if err != nil { - return fmt.Errorf("failed to serialize results as JSON: %w", err) + return nil, fmt.Errorf("failed to serialize results as JSON: %w", err) } - req, err := http.NewRequest("POST", anchoreURL, bytes.NewBuffer(reqBody)) + req, err := http.NewRequest("POST", apiEndpoint, bytes.NewBuffer(reqBody)) if err != nil { - return fmt.Errorf("failed to build request to report data to Anchore: %w", err) + return nil, fmt.Errorf("failed to build request to report data to Anchore: %w", err) } req.SetBasicAuth(anchoreDetails.User, anchoreDetails.Password) req.Header.Set("Content-Type", "application/json") req.Header.Set("x-anchore-account", anchoreDetails.Account) - resp, err := client.Do(req) + + return req, nil +} + +type AnchoreVersion struct { + API struct { + Version string `json:"version"` + } `json:"api"` + DB struct { + SchemaVersion string `json:"schema_version"` + } `json:"db"` + Service struct { + Version string `json:"version"` + } `json:"service"` +} + +func fetchVersionedAPIPath(anchoreDetails connection.AnchoreInfo) (string, error) { + logger.Log.Debug("Detecting Anchore API version") + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: anchoreDetails.HTTP.Insecure}, + } // #nosec G402 + client := &http.Client{ + Transport: tr, + Timeout: time.Duration(anchoreDetails.HTTP.TimeoutSeconds) * time.Second, + } + gock.InterceptClient(client) // Required to use gock for testing custom client + + versionEndpoint, err := url.JoinPath(anchoreDetails.URL, "version") if err != nil { - if resp != nil { - if resp.StatusCode == 401 { - return fmt.Errorf("failed to report data to Anchore, check credentials: %w", err) - } - } - return fmt.Errorf("failed to report data to Anchore: %w", err) + return v1ReportAPIPath, fmt.Errorf("failed to parse API URL: %w", err) + } + + resp, err := client.Get(versionEndpoint) + if err != nil { + return v1ReportAPIPath, fmt.Errorf("failed to contact Anchore API: %w", err) } defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode > 299 { - return fmt.Errorf("failed to report data to Anchore: %+v", resp) + if resp.StatusCode != 200 { + return v1ReportAPIPath, fmt.Errorf("failed to retrieve Anchore API version: %+v", resp) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return v1ReportAPIPath, fmt.Errorf("failed to read Anchore API version: %w", err) } - logger.Log.Debug("Successfully reported results to Anchore", "Account", anchoreDetails.Account) - return nil -} -func buildURL(anchoreDetails connection.AnchoreInfo) (string, error) { - anchoreURL, err := url.Parse(anchoreDetails.URL) + ver := AnchoreVersion{} + err = json.Unmarshal(body, &ver) if err != nil { - return "", err + return v1ReportAPIPath, fmt.Errorf("failed to parse API version: %w", err) } - anchoreURL.Path += ReportAPIPath + logger.Log.Debugf("Anchore API version: %v", ver) + + if ver.API.Version == "2" { + return v2ReportAPIPath, nil + } - return anchoreURL.String(), nil + return v1ReportAPIPath, nil } diff --git a/pkg/reporter/reporter_test.go b/pkg/reporter/reporter_test.go index 0281cba..c7ebc61 100644 --- a/pkg/reporter/reporter_test.go +++ b/pkg/reporter/reporter_test.go @@ -3,19 +3,207 @@ package reporter import ( "testing" + "github.com/h2non/gock" + "github.com/stretchr/testify/assert" + "github.com/anchore/ecs-inventory/pkg/connection" ) -func TestBuildUrl(t *testing.T) { - anchoreDetails := connection.AnchoreInfo{ +func TestPost(t *testing.T) { + defer gock.Off() + + type args struct { + report Report + anchoreDetails connection.AnchoreInfo + } + tests := []struct { + name string + args args + wantErr bool + expectedAPIPath string + }{ + { + name: "default post to v2", + args: args{ + report: Report{}, + anchoreDetails: connection.AnchoreInfo{ + URL: "https://ancho.re", + User: "admin", + Password: "foobar", + Account: "test", + HTTP: connection.HTTPConfig{ + TimeoutSeconds: 10, + Insecure: true, + }, + }, + }, + wantErr: false, + expectedAPIPath: v2ReportAPIPath, + }, + { + name: "post to v1 when v2 is not found", + args: args{ + report: Report{}, + anchoreDetails: connection.AnchoreInfo{ + URL: "https://ancho.re", + User: "admin", + Password: "foobar", + Account: "test", + HTTP: connection.HTTPConfig{ + TimeoutSeconds: 10, + Insecure: true, + }, + }, + }, + wantErr: false, + expectedAPIPath: v1ReportAPIPath, + }, + { + name: "error when v1 and v2 are not found", + args: args{ + report: Report{}, + anchoreDetails: connection.AnchoreInfo{ + URL: "https://ancho.re", + User: "admin", + Password: "foobar", + Account: "test", + HTTP: connection.HTTPConfig{ + TimeoutSeconds: 10, + Insecure: true, + }, + }, + }, + wantErr: true, + expectedAPIPath: v1ReportAPIPath, + }, + { + name: "error when api response is not JSON", + args: args{ + report: Report{}, + anchoreDetails: connection.AnchoreInfo{ + URL: "https://ancho.re", + User: "admin", + Password: "foobar", + Account: "test", + HTTP: connection.HTTPConfig{ + TimeoutSeconds: 10, + Insecure: true, + }, + }, + }, + wantErr: true, + expectedAPIPath: v2ReportAPIPath, + }, + } + for _, tt := range tests { + switch tt.name { + case "default post to v2": + gock.New("https://ancho.re"). + Post(v2ReportAPIPath). + Reply(201). + JSON(map[string]interface{}{}) + case "post to v1 when v2 is not found": + gock.New("https://ancho.re"). + Post(v2ReportAPIPath). + Reply(404) + gock.New("https://ancho.re"). + Post(v1ReportAPIPath). + Reply(201). + JSON(map[string]interface{}{}) + gock.New("https://ancho.re"). + Get("/version"). + Reply(200). + JSON(map[string]interface{}{ + "api": map[string]interface{}{}, + "db": map[string]interface{}{"schema_version": "400"}, + "service": map[string]interface{}{"version": "4.8.0"}, + }) + case "error when v1 and v2 are not found": + gock.New("https://ancho.re"). + Post(v2ReportAPIPath). + Reply(404) + gock.New("https://ancho.re"). + Get("/version"). + Reply(404) + case "error when api response is not JSON": + gock.New("https://ancho.re"). + Post(v2ReportAPIPath). + Reply(200). + BodyString("not json") + } + + t.Run(tt.name, func(t *testing.T) { + // Reset apiPath to the default each test run + apiPath = v2ReportAPIPath + + err := Post(tt.args.report, tt.args.anchoreDetails) + + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expectedAPIPath, apiPath) + } + }) + } +} + +// Simulate a handover from Enterprise 4.x to 5.x +// In this case v1 should be used initially instead of v2 then when v1 is no longer available v2 should be used +func TestPostSimulateV1ToV2HandoverFromEnterprise4Xto5X(t *testing.T) { + defer gock.Off() + + testReport := Report{} + testAnchoreDetails := connection.AnchoreInfo{ URL: "https://ancho.re", User: "admin", Password: "foobar", + Account: "test", + HTTP: connection.HTTPConfig{ + TimeoutSeconds: 10, + Insecure: true, + }, } - expectedURL := "https://ancho.re/v1/enterprise/ecs-inventory" - actualURL, err := buildURL(anchoreDetails) - if err != nil || expectedURL != actualURL { - t.Errorf("Failed to build URL:\nexpected=%s\nactual=%s", expectedURL, actualURL) - } + apiPath = v2ReportAPIPath + + // After the first post to default v2, the apiPath should be set to v1 + gock.New("https://ancho.re"). + Post(v2ReportAPIPath). + Reply(404) + gock.New("https://ancho.re"). + Get("/version"). + Reply(200). + JSON(map[string]interface{}{ + "api": map[string]interface{}{}, + "db": map[string]interface{}{"schema_version": "400"}, + "service": map[string]interface{}{"version": "4.8.0"}, + }) + gock.New("https://ancho.re"). + Post(v1ReportAPIPath). + Reply(201). + JSON(map[string]interface{}{}) + err := Post(testReport, testAnchoreDetails) + assert.NoError(t, err) + assert.Equal(t, v1ReportAPIPath, apiPath) + + // Simulate upgrade to Enterprise 5.x, v1 should no longer be available + gock.New("https://ancho.re"). + Post(v1ReportAPIPath). + Reply(404) + gock.New("https://ancho.re"). + Get("/version"). + Reply(200). + JSON(map[string]interface{}{ + "api": map[string]interface{}{"version": "2"}, + "db": map[string]interface{}{"schema_version": "400"}, + "service": map[string]interface{}{"version": "4.8.0"}, + }) + gock.New("https://ancho.re"). + Post(v2ReportAPIPath). + Reply(201). + JSON(map[string]interface{}{}) + err = Post(testReport, testAnchoreDetails) + assert.NoError(t, err) + assert.Equal(t, v2ReportAPIPath, apiPath) }