From 130c7cd65a065b5ce4a45a9942849573a357a25e Mon Sep 17 00:00:00 2001 From: Yolanda Robla Date: Mon, 19 Feb 2024 14:53:18 +0100 Subject: [PATCH 1/4] feat: add support for java Use sonatype API to query for updates in maven central repo and publish those into feeds Signed-off-by: Yolanda Robla Signed-off-by: Yolanda Robla --- pkg/config/scheduledfeed.go | 7 ++ pkg/feeds/feed.go | 2 +- pkg/feeds/maven/README.md | 13 ++++ pkg/feeds/maven/maven.go | 141 ++++++++++++++++++++++++++++++++++ pkg/feeds/maven/maven_test.go | 113 +++++++++++++++++++++++++++ 5 files changed, 275 insertions(+), 1 deletion(-) create mode 100644 pkg/feeds/maven/README.md create mode 100644 pkg/feeds/maven/maven.go create mode 100644 pkg/feeds/maven/maven_test.go diff --git a/pkg/config/scheduledfeed.go b/pkg/config/scheduledfeed.go index f6659828..59d35cf4 100644 --- a/pkg/config/scheduledfeed.go +++ b/pkg/config/scheduledfeed.go @@ -17,6 +17,7 @@ import ( "github.com/ossf/package-feeds/pkg/feeds" "github.com/ossf/package-feeds/pkg/feeds/crates" "github.com/ossf/package-feeds/pkg/feeds/goproxy" + "github.com/ossf/package-feeds/pkg/feeds/maven" "github.com/ossf/package-feeds/pkg/feeds/npm" "github.com/ossf/package-feeds/pkg/feeds/nuget" "github.com/ossf/package-feeds/pkg/feeds/packagist" @@ -171,6 +172,8 @@ func (fc FeedConfig) ToFeed(eventHandler *events.Handler) (feeds.ScheduledFeed, return npm.New(fc.Options, eventHandler) case nuget.FeedName: return nuget.New(fc.Options) + case maven.FeedName: + return maven.New(fc.Options) case pypi.FeedName: return pypi.New(fc.Options, eventHandler) case packagist.FeedName: @@ -214,6 +217,10 @@ func Default() *ScheduledFeedConfig { Type: nuget.FeedName, Options: defaultFeedOptions, }, + { + Type: maven.FeedName, + Options: defaultFeedOptions, + }, { Type: packagist.FeedName, Options: defaultFeedOptions, diff --git a/pkg/feeds/feed.go b/pkg/feeds/feed.go index 690d59e4..be1869fc 100644 --- a/pkg/feeds/feed.go +++ b/pkg/feeds/feed.go @@ -70,7 +70,7 @@ func NewArtifact(created time.Time, name, version, artifactID, feed string) *Pac func ApplyCutoff(pkgs []*Package, cutoff time.Time) []*Package { filteredPackages := []*Package{} for _, pkg := range pkgs { - if pkg.CreatedDate.After(cutoff) { + if pkg.CreatedDate.UTC().After(cutoff) { filteredPackages = append(filteredPackages, pkg) } } diff --git a/pkg/feeds/maven/README.md b/pkg/feeds/maven/README.md new file mode 100644 index 00000000..bc81b87d --- /dev/null +++ b/pkg/feeds/maven/README.md @@ -0,0 +1,13 @@ +# maven Feed + +This feed allows polling of package updates from central.sonatype, polling Maven central repository. + +## Configuration options + +The `packages` field is not supported by the maven feed. + + +``` +feeds: +- type: maven +``` \ No newline at end of file diff --git a/pkg/feeds/maven/maven.go b/pkg/feeds/maven/maven.go new file mode 100644 index 00000000..0f1026d9 --- /dev/null +++ b/pkg/feeds/maven/maven.go @@ -0,0 +1,141 @@ +package maven + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/ossf/package-feeds/pkg/feeds" +) + +const ( + FeedName = "maven" + indexPath = "/api/internal/browse/components" +) + +type Feed struct { + baseURL string + options feeds.FeedOptions +} + +func New(feedOptions feeds.FeedOptions) (*Feed, error) { + if feedOptions.Packages != nil { + return nil, feeds.UnsupportedOptionError{ + Feed: FeedName, + Option: "packages", + } + } + return &Feed{ + baseURL: "https://central.sonatype.com/" + indexPath, + options: feedOptions, + }, nil +} + +// Package represents package information. +type LatestVersionInfo struct { + Version string `json:"version"` + TimestampUnixWithMS int64 `json:"timestampUnixWithMS"` +} + +type Package struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + LatestVersionInfo LatestVersionInfo `json:"latestVersionInfo"` +} + +// Response represents the response structure from Sonatype API. +type Response struct { + Components []Package `json:"components"` +} + +// fetchPackages fetches packages from Sonatype API for the given page. +func (feed Feed) fetchPackages(page int) ([]Package, error) { + // Define the request payload + payload := map[string]interface{}{ + "page": page, + "size": 20, + "sortField": "publishedDate", + "sortDirection": "desc", + } + + jsonPayload, err := json.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("error encoding JSON: %w", err) + } + + // Send POST request to Sonatype API. + resp, err := http.Post(feed.baseURL+"?repository=maven-central", "application/json", bytes.NewBuffer(jsonPayload)) + if err != nil { + return nil, fmt.Errorf("error sending request: %w", err) + } + defer resp.Body.Close() + + // Handle rate limiting (HTTP status code 429). + if resp.StatusCode == http.StatusTooManyRequests { + time.Sleep(5 * time.Second) + return feed.fetchPackages(page) // Retry the request + } + + // Decode response. + var response Response + err = json.NewDecoder(resp.Body).Decode(&response) + if err != nil { + return nil, fmt.Errorf("error decoding response: %w", err) + } + return response.Components, nil +} + +func (feed Feed) Latest(cutoff time.Time) ([]*feeds.Package, time.Time, []error) { + pkgs := []*feeds.Package{} + var errs []error + + page := 0 + for { + // Fetch packages from Sonatype API for the current page. + packages, err := feed.fetchPackages(page) + if err != nil { + errs = append(errs, err) + break + } + + // Iterate over packages + hasToCut := false + for _, pkg := range packages { + // convert published to date to compare with cutoff + if pkg.LatestVersionInfo.TimestampUnixWithMS > cutoff.UnixMilli() { + // Append package to pkgs + timestamp := time.Unix(pkg.LatestVersionInfo.TimestampUnixWithMS/1000, 0) + packageName := pkg.Namespace + ":" + pkg.Name + + newPkg := feeds.NewPackage(timestamp, packageName, pkg.LatestVersionInfo.Version, FeedName) + pkgs = append(pkgs, newPkg) + } else { + // Break the loop if the cutoff date is reached + hasToCut = true + } + } + + // Move to the next page + page++ + + // Check if the loop should be terminated + if len(pkgs) == 0 || hasToCut { + break + } + } + + newCutoff := feeds.FindCutoff(cutoff, pkgs) + pkgs = feeds.ApplyCutoff(pkgs, cutoff) + + return pkgs, newCutoff, errs +} + +func (feed Feed) GetName() string { + return FeedName +} + +func (feed Feed) GetFeedOptions() feeds.FeedOptions { + return feed.options +} diff --git a/pkg/feeds/maven/maven_test.go b/pkg/feeds/maven/maven_test.go new file mode 100644 index 00000000..5fdee128 --- /dev/null +++ b/pkg/feeds/maven/maven_test.go @@ -0,0 +1,113 @@ +package maven + +import ( + "net/http" + "testing" + "time" + + "github.com/ossf/package-feeds/pkg/feeds" + testutils "github.com/ossf/package-feeds/pkg/utils/test" +) + +func TestMavenLatest(t *testing.T) { + t.Parallel() + + handlers := map[string]testutils.HTTPHandlerFunc{ + indexPath: mavenPackageResponse, + } + srv := testutils.HTTPServerMock(handlers) + + feed, err := New(feeds.FeedOptions{}) + if err != nil { + t.Fatalf("Failed to create Maven feed: %v", err) + } + feed.baseURL = srv.URL + "/api/internal/browse/components" + + cutoff := time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC) + pkgs, gotCutoff, errs := feed.Latest(cutoff) + + if len(errs) != 0 { + t.Fatalf("feed.Latest returned error: %v", err) + } + + // Returned cutoff should match the newest package creation time of packages retrieved. + wantCutoff := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) + if gotCutoff.UTC().Sub(wantCutoff).Abs() > time.Second { + t.Errorf("Latest() cutoff %v, want %v", gotCutoff, wantCutoff) + } + if pkgs[0].Name != "com.github.example:project" { + t.Errorf("Unexpected package `%s` found in place of expected `com.github.example:project`", pkgs[0].Name) + } + if pkgs[0].Version != "1.0.0" { + t.Errorf("Unexpected version `%s` found in place of expected `1.0.0`", pkgs[0].Version) + } + + for _, p := range pkgs { + if p.Type != FeedName { + t.Errorf("Feed type not set correctly in goproxy package following Latest()") + } + } +} + +func TestMavenNotFound(t *testing.T) { + t.Parallel() + + handlers := map[string]testutils.HTTPHandlerFunc{ + indexPath: testutils.NotFoundHandlerFunc, + } + srv := testutils.HTTPServerMock(handlers) + + feed, err := New(feeds.FeedOptions{}) + if err != nil { + t.Fatalf("Failed to create Maven feed: %v", err) + } + feed.baseURL = srv.URL + "/api/internal/browse/components" + + cutoff := time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC) + + _, gotCutoff, errs := feed.Latest(cutoff) + if cutoff != gotCutoff { + t.Error("feed.Latest() cutoff should be unchanged if an error is returned") + } + if len(errs) == 0 { + t.Fatalf("feed.Latest() was successful when an error was expected") + } +} + +func mavenPackageResponse(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + responseJSON := ` + { + "components": [ + { + "id": "pkg:maven/com.github.example/project", + "type": "COMPONENT", + "namespace": "com.github.example", + "name": "project", + "version": "1.0.0", + "publishedEpochMillis": 946684800000, + "latestVersionInfo": { + "version": "1.0.0", + "timestampUnixWithMS": 946684800000 + } + }, + { + "id": "pkg:maven/com.github.example/project1", + "type": "COMPONENT", + "namespace": "com.github.example", + "name": "project", + "version": "1.0.0", + "publishedEpochMillis": null, + "latestVersionInfo": { + "version": "1.0.0", + "timestampUnixWithMS": 0 + } + } + ] + } + ` + _, err := w.Write([]byte(responseJSON)) + if err != nil { + http.Error(w, testutils.UnexpectedWriteError(err), http.StatusInternalServerError) + } +} From 5f3b09bc40cc94d49488a1f77ca92c6f6227f690 Mon Sep 17 00:00:00 2001 From: Yolanda Robla Date: Wed, 21 Feb 2024 11:25:49 +0100 Subject: [PATCH 2/4] Add changes from review Remove the UTC() time conversion Add a max retry limit on maven feed on failures Signed-off-by: Yolanda Robla Signed-off-by: Yolanda Robla --- pkg/feeds/feed.go | 2 +- pkg/feeds/maven/maven.go | 74 +++++++++++++++++++++++++--------------- 2 files changed, 47 insertions(+), 29 deletions(-) diff --git a/pkg/feeds/feed.go b/pkg/feeds/feed.go index be1869fc..690d59e4 100644 --- a/pkg/feeds/feed.go +++ b/pkg/feeds/feed.go @@ -70,7 +70,7 @@ func NewArtifact(created time.Time, name, version, artifactID, feed string) *Pac func ApplyCutoff(pkgs []*Package, cutoff time.Time) []*Package { filteredPackages := []*Package{} for _, pkg := range pkgs { - if pkg.CreatedDate.UTC().After(cutoff) { + if pkg.CreatedDate.After(cutoff) { filteredPackages = append(filteredPackages, pkg) } } diff --git a/pkg/feeds/maven/maven.go b/pkg/feeds/maven/maven.go index 0f1026d9..e62cb2fb 100644 --- a/pkg/feeds/maven/maven.go +++ b/pkg/feeds/maven/maven.go @@ -3,6 +3,7 @@ package maven import ( "bytes" "encoding/json" + "errors" "fmt" "net/http" "time" @@ -20,6 +21,8 @@ type Feed struct { options feeds.FeedOptions } +var ErrMaxRetriesReached = errors.New("maximum retries reached due to rate limiting") + func New(feedOptions feeds.FeedOptions) (*Feed, error) { if feedOptions.Packages != nil { return nil, feeds.UnsupportedOptionError{ @@ -52,39 +55,54 @@ type Response struct { // fetchPackages fetches packages from Sonatype API for the given page. func (feed Feed) fetchPackages(page int) ([]Package, error) { - // Define the request payload - payload := map[string]interface{}{ - "page": page, - "size": 20, - "sortField": "publishedDate", - "sortDirection": "desc", - } + maxRetries := 5 + retryDelay := 5 * time.Second + + for attempt := 0; attempt <= maxRetries; attempt++ { + // Define the request payload + payload := map[string]interface{}{ + "page": page, + "size": 20, + "sortField": "publishedDate", + "sortDirection": "desc", + } - jsonPayload, err := json.Marshal(payload) - if err != nil { - return nil, fmt.Errorf("error encoding JSON: %w", err) - } + jsonPayload, err := json.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("error encoding JSON: %w", err) + } - // Send POST request to Sonatype API. - resp, err := http.Post(feed.baseURL+"?repository=maven-central", "application/json", bytes.NewBuffer(jsonPayload)) - if err != nil { - return nil, fmt.Errorf("error sending request: %w", err) - } - defer resp.Body.Close() + // Send POST request to Sonatype API. + resp, err := http.Post(feed.baseURL+"?repository=maven-central", "application/json", bytes.NewBuffer(jsonPayload)) + if err != nil { + // Check if maximum retries have been reached + if attempt == maxRetries { + return nil, fmt.Errorf("error sending request: %w", err) + } + time.Sleep(retryDelay) // Wait before retrying + continue + } + defer resp.Body.Close() - // Handle rate limiting (HTTP status code 429). - if resp.StatusCode == http.StatusTooManyRequests { - time.Sleep(5 * time.Second) - return feed.fetchPackages(page) // Retry the request - } + // Handle rate limiting (HTTP status code 429). + if resp.StatusCode == http.StatusTooManyRequests { + // Check if maximum retries have been reached + if attempt == maxRetries { + return nil, ErrMaxRetriesReached + } + time.Sleep(retryDelay) // Wait before retrying + continue + } - // Decode response. - var response Response - err = json.NewDecoder(resp.Body).Decode(&response) - if err != nil { - return nil, fmt.Errorf("error decoding response: %w", err) + // Decode response. + var response Response + err = json.NewDecoder(resp.Body).Decode(&response) + if err != nil { + return nil, fmt.Errorf("error decoding response: %w", err) + } + return response.Components, nil } - return response.Components, nil + return nil, ErrMaxRetriesReached } func (feed Feed) Latest(cutoff time.Time) ([]*feeds.Package, time.Time, []error) { From 8a01ad6c0acaf6151e22bff77580ec200f334f2c Mon Sep 17 00:00:00 2001 From: Yolanda Robla Mota Date: Thu, 22 Feb 2024 09:46:43 +0100 Subject: [PATCH 3/4] Update maven.go Signed-off-by: Yolanda Robla Mota --- pkg/feeds/maven/maven.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/feeds/maven/maven.go b/pkg/feeds/maven/maven.go index e62cb2fb..f0a28f0c 100644 --- a/pkg/feeds/maven/maven.go +++ b/pkg/feeds/maven/maven.go @@ -12,7 +12,7 @@ import ( ) const ( - FeedName = "maven" + FeedName = "maven-central" indexPath = "/api/internal/browse/components" ) From afb09cfa55c179a2830602475a065f785fd18f29 Mon Sep 17 00:00:00 2001 From: Yolanda Robla Mota Date: Thu, 22 Feb 2024 09:47:07 +0100 Subject: [PATCH 4/4] Update README.md Signed-off-by: Yolanda Robla Mota --- pkg/feeds/maven/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/feeds/maven/README.md b/pkg/feeds/maven/README.md index bc81b87d..09bc8c88 100644 --- a/pkg/feeds/maven/README.md +++ b/pkg/feeds/maven/README.md @@ -9,5 +9,5 @@ The `packages` field is not supported by the maven feed. ``` feeds: -- type: maven -``` \ No newline at end of file +- type: maven-central +```