diff --git a/pkg/config/scheduledfeed.go b/pkg/config/scheduledfeed.go index f6659828..59d35cf4 100644 --- a/pkg/config/scheduledfeed.go +++ b/pkg/config/scheduledfeed.go @@ -17,6 +17,7 @@ import ( "github.com/ossf/package-feeds/pkg/feeds" "github.com/ossf/package-feeds/pkg/feeds/crates" "github.com/ossf/package-feeds/pkg/feeds/goproxy" + "github.com/ossf/package-feeds/pkg/feeds/maven" "github.com/ossf/package-feeds/pkg/feeds/npm" "github.com/ossf/package-feeds/pkg/feeds/nuget" "github.com/ossf/package-feeds/pkg/feeds/packagist" @@ -171,6 +172,8 @@ func (fc FeedConfig) ToFeed(eventHandler *events.Handler) (feeds.ScheduledFeed, return npm.New(fc.Options, eventHandler) case nuget.FeedName: return nuget.New(fc.Options) + case maven.FeedName: + return maven.New(fc.Options) case pypi.FeedName: return pypi.New(fc.Options, eventHandler) case packagist.FeedName: @@ -214,6 +217,10 @@ func Default() *ScheduledFeedConfig { Type: nuget.FeedName, Options: defaultFeedOptions, }, + { + Type: maven.FeedName, + Options: defaultFeedOptions, + }, { Type: packagist.FeedName, Options: defaultFeedOptions, diff --git a/pkg/feeds/maven/README.md b/pkg/feeds/maven/README.md new file mode 100644 index 00000000..09bc8c88 --- /dev/null +++ b/pkg/feeds/maven/README.md @@ -0,0 +1,13 @@ +# maven Feed + +This feed allows polling of package updates from central.sonatype, polling Maven central repository. + +## Configuration options + +The `packages` field is not supported by the maven feed. + + +``` +feeds: +- type: maven-central +``` diff --git a/pkg/feeds/maven/maven.go b/pkg/feeds/maven/maven.go new file mode 100644 index 00000000..f0a28f0c --- /dev/null +++ b/pkg/feeds/maven/maven.go @@ -0,0 +1,159 @@ +package maven + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "net/http" + "time" + + "github.com/ossf/package-feeds/pkg/feeds" +) + +const ( + FeedName = "maven-central" + indexPath = "/api/internal/browse/components" +) + +type Feed struct { + baseURL string + options feeds.FeedOptions +} + +var ErrMaxRetriesReached = errors.New("maximum retries reached due to rate limiting") + +func New(feedOptions feeds.FeedOptions) (*Feed, error) { + if feedOptions.Packages != nil { + return nil, feeds.UnsupportedOptionError{ + Feed: FeedName, + Option: "packages", + } + } + return &Feed{ + baseURL: "https://central.sonatype.com/" + indexPath, + options: feedOptions, + }, nil +} + +// Package represents package information. +type LatestVersionInfo struct { + Version string `json:"version"` + TimestampUnixWithMS int64 `json:"timestampUnixWithMS"` +} + +type Package struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + LatestVersionInfo LatestVersionInfo `json:"latestVersionInfo"` +} + +// Response represents the response structure from Sonatype API. +type Response struct { + Components []Package `json:"components"` +} + +// fetchPackages fetches packages from Sonatype API for the given page. +func (feed Feed) fetchPackages(page int) ([]Package, error) { + maxRetries := 5 + retryDelay := 5 * time.Second + + for attempt := 0; attempt <= maxRetries; attempt++ { + // Define the request payload + payload := map[string]interface{}{ + "page": page, + "size": 20, + "sortField": "publishedDate", + "sortDirection": "desc", + } + + jsonPayload, err := json.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("error encoding JSON: %w", err) + } + + // Send POST request to Sonatype API. + resp, err := http.Post(feed.baseURL+"?repository=maven-central", "application/json", bytes.NewBuffer(jsonPayload)) + if err != nil { + // Check if maximum retries have been reached + if attempt == maxRetries { + return nil, fmt.Errorf("error sending request: %w", err) + } + time.Sleep(retryDelay) // Wait before retrying + continue + } + defer resp.Body.Close() + + // Handle rate limiting (HTTP status code 429). + if resp.StatusCode == http.StatusTooManyRequests { + // Check if maximum retries have been reached + if attempt == maxRetries { + return nil, ErrMaxRetriesReached + } + time.Sleep(retryDelay) // Wait before retrying + continue + } + + // Decode response. + var response Response + err = json.NewDecoder(resp.Body).Decode(&response) + if err != nil { + return nil, fmt.Errorf("error decoding response: %w", err) + } + return response.Components, nil + } + return nil, ErrMaxRetriesReached +} + +func (feed Feed) Latest(cutoff time.Time) ([]*feeds.Package, time.Time, []error) { + pkgs := []*feeds.Package{} + var errs []error + + page := 0 + for { + // Fetch packages from Sonatype API for the current page. + packages, err := feed.fetchPackages(page) + if err != nil { + errs = append(errs, err) + break + } + + // Iterate over packages + hasToCut := false + for _, pkg := range packages { + // convert published to date to compare with cutoff + if pkg.LatestVersionInfo.TimestampUnixWithMS > cutoff.UnixMilli() { + // Append package to pkgs + timestamp := time.Unix(pkg.LatestVersionInfo.TimestampUnixWithMS/1000, 0) + packageName := pkg.Namespace + ":" + pkg.Name + + newPkg := feeds.NewPackage(timestamp, packageName, pkg.LatestVersionInfo.Version, FeedName) + pkgs = append(pkgs, newPkg) + } else { + // Break the loop if the cutoff date is reached + hasToCut = true + } + } + + // Move to the next page + page++ + + // Check if the loop should be terminated + if len(pkgs) == 0 || hasToCut { + break + } + } + + newCutoff := feeds.FindCutoff(cutoff, pkgs) + pkgs = feeds.ApplyCutoff(pkgs, cutoff) + + return pkgs, newCutoff, errs +} + +func (feed Feed) GetName() string { + return FeedName +} + +func (feed Feed) GetFeedOptions() feeds.FeedOptions { + return feed.options +} diff --git a/pkg/feeds/maven/maven_test.go b/pkg/feeds/maven/maven_test.go new file mode 100644 index 00000000..5fdee128 --- /dev/null +++ b/pkg/feeds/maven/maven_test.go @@ -0,0 +1,113 @@ +package maven + +import ( + "net/http" + "testing" + "time" + + "github.com/ossf/package-feeds/pkg/feeds" + testutils "github.com/ossf/package-feeds/pkg/utils/test" +) + +func TestMavenLatest(t *testing.T) { + t.Parallel() + + handlers := map[string]testutils.HTTPHandlerFunc{ + indexPath: mavenPackageResponse, + } + srv := testutils.HTTPServerMock(handlers) + + feed, err := New(feeds.FeedOptions{}) + if err != nil { + t.Fatalf("Failed to create Maven feed: %v", err) + } + feed.baseURL = srv.URL + "/api/internal/browse/components" + + cutoff := time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC) + pkgs, gotCutoff, errs := feed.Latest(cutoff) + + if len(errs) != 0 { + t.Fatalf("feed.Latest returned error: %v", err) + } + + // Returned cutoff should match the newest package creation time of packages retrieved. + wantCutoff := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) + if gotCutoff.UTC().Sub(wantCutoff).Abs() > time.Second { + t.Errorf("Latest() cutoff %v, want %v", gotCutoff, wantCutoff) + } + if pkgs[0].Name != "com.github.example:project" { + t.Errorf("Unexpected package `%s` found in place of expected `com.github.example:project`", pkgs[0].Name) + } + if pkgs[0].Version != "1.0.0" { + t.Errorf("Unexpected version `%s` found in place of expected `1.0.0`", pkgs[0].Version) + } + + for _, p := range pkgs { + if p.Type != FeedName { + t.Errorf("Feed type not set correctly in goproxy package following Latest()") + } + } +} + +func TestMavenNotFound(t *testing.T) { + t.Parallel() + + handlers := map[string]testutils.HTTPHandlerFunc{ + indexPath: testutils.NotFoundHandlerFunc, + } + srv := testutils.HTTPServerMock(handlers) + + feed, err := New(feeds.FeedOptions{}) + if err != nil { + t.Fatalf("Failed to create Maven feed: %v", err) + } + feed.baseURL = srv.URL + "/api/internal/browse/components" + + cutoff := time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC) + + _, gotCutoff, errs := feed.Latest(cutoff) + if cutoff != gotCutoff { + t.Error("feed.Latest() cutoff should be unchanged if an error is returned") + } + if len(errs) == 0 { + t.Fatalf("feed.Latest() was successful when an error was expected") + } +} + +func mavenPackageResponse(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + responseJSON := ` + { + "components": [ + { + "id": "pkg:maven/com.github.example/project", + "type": "COMPONENT", + "namespace": "com.github.example", + "name": "project", + "version": "1.0.0", + "publishedEpochMillis": 946684800000, + "latestVersionInfo": { + "version": "1.0.0", + "timestampUnixWithMS": 946684800000 + } + }, + { + "id": "pkg:maven/com.github.example/project1", + "type": "COMPONENT", + "namespace": "com.github.example", + "name": "project", + "version": "1.0.0", + "publishedEpochMillis": null, + "latestVersionInfo": { + "version": "1.0.0", + "timestampUnixWithMS": 0 + } + } + ] + } + ` + _, err := w.Write([]byte(responseJSON)) + if err != nil { + http.Error(w, testutils.UnexpectedWriteError(err), http.StatusInternalServerError) + } +}