Add changes from review

Remove the UTC() time conversion Add a max retry limit on maven feed on failures Signed-off-by: Yolanda Robla <[email protected]> Signed-off-by: Yolanda Robla <[email protected]>
ossf · Feb 21, 2024 · 02fa827 · 02fa827
1 parent 130c7cd
commit 02fa827
Show file tree

Hide file tree

Showing 2 changed files with 46 additions and 29 deletions.
diff --git a/pkg/feeds/feed.go b/pkg/feeds/feed.go
@@ -70,7 +70,7 @@ func NewArtifact(created time.Time, name, version, artifactID, feed string) *Pac
 func ApplyCutoff(pkgs []*Package, cutoff time.Time) []*Package {
 	filteredPackages := []*Package{}
 	for _, pkg := range pkgs {
-		if pkg.CreatedDate.UTC().After(cutoff) {
+		if pkg.CreatedDate.After(cutoff) {
 			filteredPackages = append(filteredPackages, pkg)
 		}
 	}

diff --git a/pkg/feeds/maven/maven.go b/pkg/feeds/maven/maven.go
@@ -3,6 +3,7 @@ package maven
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 	"time"
@@ -52,39 +53,55 @@ type Response struct {
 
 // fetchPackages fetches packages from Sonatype API for the given page.
 func (feed Feed) fetchPackages(page int) ([]Package, error) {
-	// Define the request payload
-	payload := map[string]interface{}{
-		"page":          page,
-		"size":          20,
-		"sortField":     "publishedDate",
-		"sortDirection": "desc",
-	}
+	maxRetries := 5
+	retryDelay := 5 * time.Second
+	ErrMaxRetriesReached := errors.New("maximum retries reached due to rate limiting")
+
+	for attempt := 0; attempt <= maxRetries; attempt++ {
+		// Define the request payload
+		payload := map[string]interface{}{
+			"page":          page,
+			"size":          20,
+			"sortField":     "publishedDate",
+			"sortDirection": "desc",
+		}
 
-	jsonPayload, err := json.Marshal(payload)
-	if err != nil {
-		return nil, fmt.Errorf("error encoding JSON: %w", err)
-	}
+		jsonPayload, err := json.Marshal(payload)
+		if err != nil {
+			return nil, fmt.Errorf("error encoding JSON: %w", err)
+		}
 
-	// Send POST request to Sonatype API.
-	resp, err := http.Post(feed.baseURL+"?repository=maven-central", "application/json", bytes.NewBuffer(jsonPayload))
-	if err != nil {
-		return nil, fmt.Errorf("error sending request: %w", err)
-	}
-	defer resp.Body.Close()
+		// Send POST request to Sonatype API.
+		resp, err := http.Post(feed.baseURL+"?repository=maven-central", "application/json", bytes.NewBuffer(jsonPayload))
+		if err != nil {
+			// Check if maximum retries have been reached
+			if attempt == maxRetries {
+				return nil, fmt.Errorf("error sending request: %w", err)
+			}
+			time.Sleep(retryDelay) // Wait before retrying
+			continue
+		}
+		defer resp.Body.Close()
 
-	// Handle rate limiting (HTTP status code 429).
-	if resp.StatusCode == http.StatusTooManyRequests {
-		time.Sleep(5 * time.Second)
-		return feed.fetchPackages(page) // Retry the request
-	}
+		// Handle rate limiting (HTTP status code 429).
+		if resp.StatusCode == http.StatusTooManyRequests {
+			// Check if maximum retries have been reached
+			if attempt == maxRetries {
+				return nil, ErrMaxRetriesReached
+			}
+			time.Sleep(retryDelay) // Wait before retrying
+			continue
+		}
 
-	// Decode response.
-	var response Response
-	err = json.NewDecoder(resp.Body).Decode(&response)
-	if err != nil {
-		return nil, fmt.Errorf("error decoding response: %w", err)
+		// Decode response.
+		var response Response
+		err = json.NewDecoder(resp.Body).Decode(&response)
+		if err != nil {
+			return nil, fmt.Errorf("error decoding response: %w", err)
+		}
+		return response.Components, nil
 	}
-	return response.Components, nil
+	return nil, ErrMaxRetriesReached
 }
 
 func (feed Feed) Latest(cutoff time.Time) ([]*feeds.Package, time.Time, []error) {