feat: add HTTP support for downloading DBs

internal/dbtest/fake.go

@@ -13,8 +13,8 @@ import (
 	"github.com/samber/lo"
 	"github.com/stretchr/testify/require"
 
+	"github.com/aquasecurity/trivy/pkg/asset"
 	ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
-	"github.com/aquasecurity/trivy/pkg/oci"
 )
 
 const defaultMediaType = "application/vnd.aquasec.trivy.db.layer.v1.tar+gzip"
@@ -38,7 +38,7 @@ type FakeDBOptions struct {
 	MediaType types.MediaType
 }
 
-func NewFakeDB(t *testing.T, dbPath string, opts FakeDBOptions) *oci.Artifact {
+func NewFakeDB(t *testing.T, dbPath string, opts FakeDBOptions) *asset.OCI {
 	mediaType := lo.Ternary(opts.MediaType != "", opts.MediaType, defaultMediaType)
 	img := new(fakei.FakeImage)
 	img.LayersReturns([]v1.Layer{NewFakeLayer(t, dbPath, mediaType)}, nil)
@@ -59,10 +59,13 @@ func NewFakeDB(t *testing.T, dbPath string, opts FakeDBOptions) *oci.Artifact {
 	}, nil)
 
 	// Mock OCI artifact
-	opt := ftypes.RegistryOptions{
-		Insecure: false,
+	assetOpts := asset.Options{
+		MediaType: defaultMediaType,
+		RegistryOptions: ftypes.RegistryOptions{
+			Insecure: false,
+		},
 	}
-	return oci.NewArtifact("dummy", opt, oci.WithImage(img))
+	return asset.NewOCI("dummy", assetOpts, asset.WithImage(img))
 }
 
 func ArchiveDir(t *testing.T, dir string) string {

pkg/asset/asset.go

@@ -0,0 +1,91 @@
+package asset
+
+import (
+	"context"
+	"errors"
+	"strings"
+
+	"github.com/google/go-containerregistry/pkg/v1/remote/transport"
+	"github.com/hashicorp/go-multierror"
+	"golang.org/x/xerrors"
+
+	"github.com/aquasecurity/trivy/pkg/fanal/types"
+	"github.com/aquasecurity/trivy/pkg/log"
+	"github.com/aquasecurity/trivy/pkg/version/doc"
+)
+
+type Options struct {
+	// For OCI
+	MediaType string // Accept any media type if not specified
+
+	// Common
+	Filename string // Use the annotation if not specified
+	Quiet    bool
+
+	types.RegistryOptions
+}
+
+type Assets []Asset
+
+type Asset interface {
+	Location() string
+	Download(ctx context.Context, dst string) error
+}
+
+func NewAssets(locations []string, assetOpts Options, opts ...Option) Assets {
+	var assets Assets
+	for _, location := range locations {
+		switch {
+		case strings.HasPrefix(location, "https://"):
+			assets = append(assets, NewHTTP(location, assetOpts))
+		default:
+			assets = append(assets, NewOCI(location, assetOpts, opts...))
+		}
+	}
+	return assets
+}
+
+// Download downloads artifacts until one of them succeeds.
+// Attempts to download next artifact if the first one fails due to a temporary error.
+func (a Assets) Download(ctx context.Context, dst string) error {
+	var errs error
+	for i, art := range a {
+		logger := log.With("location", art.Location())
+		logger.InfoContext(ctx, "Downloading artifact...")
+		err := art.Download(ctx, dst)
+		if err == nil {
+			logger.InfoContext(ctx, "OCI successfully downloaded")
+			return nil
+		}
+
+		if !shouldTryOtherRepo(err) {
+			return xerrors.Errorf("failed to download artifact from %s: %w", art.Location(), err)
+		}
+		logger.ErrorContext(ctx, "Failed to download artifact", log.Err(err))
+		if i < len(a)-1 {
+			log.InfoContext(ctx, "Trying to download artifact from other repository...") // Use the default logger
+		}
+		errs = multierror.Append(errs, err)
+	}
+
+	return xerrors.Errorf("failed to download artifact from any source: %w", errs)
+}
+
+func shouldTryOtherRepo(err error) bool {
+	var terr *transport.Error
+	if !errors.As(err, &terr) {
+		return false
+	}
+
+	for _, diagnostic := range terr.Errors {
+		// For better user experience
+		if diagnostic.Code == transport.DeniedErrorCode || diagnostic.Code == transport.UnauthorizedErrorCode {
+			// e.g. https://aquasecurity.github.io/trivy/latest/docs/references/troubleshooting/#db
+			log.Warnf("See %s", doc.URL("/docs/references/troubleshooting/", "db"))
+			break
+		}
+	}
+
+	// try the following artifact only if a temporary error occurs
+	return terr.Temporary()
+}

pkg/asset/http.go

@@ -0,0 +1,35 @@
+package asset
+
+import (
+	"context"
+
+	"golang.org/x/xerrors"
+
+	"github.com/aquasecurity/trivy/pkg/downloader"
+)
+
+type HTTP struct {
+	url  string
+	opts Options
+}
+
+func NewHTTP(location string, assetOpts Options) *HTTP {
+	return &HTTP{
+		url:  location,
+		opts: assetOpts,
+	}
+}
+
+func (h *HTTP) Location() string {
+	return h.url
+}
+
+func (h *HTTP) Download(ctx context.Context, dir string) error {
+	_, err := downloader.Download(ctx, h.url, dir, ".", downloader.Options{
+		Insecure: h.opts.Insecure,
+	})
+	if err != nil {
+		return xerrors.Errorf("failed to download artifact via HTTP: %w", err)
+	}
+	return nil
+}

pkg/asset/oci.go

@@ -0,0 +1,203 @@
+package asset
+
+import (
+	"context"
+	"io"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/cheggaaa/pb/v3"
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"golang.org/x/xerrors"
+
+	"github.com/aquasecurity/trivy/pkg/downloader"
+	"github.com/aquasecurity/trivy/pkg/remote"
+)
+
+const (
+	// Artifact types
+	CycloneDXArtifactType = "application/vnd.cyclonedx+json"
+	SPDXArtifactType      = "application/spdx+json"
+
+	// Media types
+	OCIImageManifest = "application/vnd.oci.image.manifest.v1+json"
+
+	// Annotations
+	titleAnnotation = "org.opencontainers.image.title"
+)
+
+var SupportedSBOMArtifactTypes = []string{
+	CycloneDXArtifactType,
+	SPDXArtifactType,
+}
+
+// Option is a functional option
+type Option func(*OCI)
+
+// WithImage takes an OCI v1 Image
+func WithImage(img v1.Image) Option {
+	return func(a *OCI) {
+		a.image = img
+	}
+}
+
+// OCI is used to download OCI artifacts such as vulnerability database and policies from OCI registries.
+type OCI struct {
+	m          sync.Mutex
+	repository string
+	opts       Options
+
+	image v1.Image // For testing
+}
+
+// NewOCI returns a new instance of the OCI artifact
+func NewOCI(repo string, assetOpts Options, opts ...Option) *OCI {
+	art := &OCI{
+		repository: repo,
+		opts:       assetOpts,
+	}
+
+	for _, o := range opts {
+		o(art)
+	}
+	return art
+}
+
+func (o *OCI) populate(ctx context.Context) error {
+	if o.image != nil {
+		return nil
+	}
+
+	o.m.Lock()
+	defer o.m.Unlock()
+
+	var nameOpts []name.Option
+	if o.opts.Insecure {
+		nameOpts = append(nameOpts, name.Insecure)
+	}
+
+	ref, err := name.ParseReference(o.repository, nameOpts...)
+	if err != nil {
+		return xerrors.Errorf("repository name error (%s): %w", o.repository, err)
+	}
+
+	o.image, err = remote.Image(ctx, ref, o.opts.RegistryOptions)
+	if err != nil {
+		return xerrors.Errorf("OCI repository error: %w", err)
+	}
+	return nil
+}
+
+func (o *OCI) Location() string {
+	return o.repository
+}
+
+func (o *OCI) Download(ctx context.Context, dir string) error {
+	if err := o.populate(ctx); err != nil {
+		return err
+	}
+
+	layers, err := o.image.Layers()
+	if err != nil {
+		return xerrors.Errorf("OCI layer error: %w", err)
+	}
+
+	manifest, err := o.image.Manifest()
+	if err != nil {
+		return xerrors.Errorf("OCI manifest error: %w", err)
+	}
+
+	// A single layer is only supported now.
+	if len(layers) != 1 || len(manifest.Layers) != 1 {
+		return xerrors.Errorf("OCI artifact must be a single layer")
+	}
+
+	// Take the first layer
+	layer := layers[0]
+
+	// Take the file name of the first layer if not specified
+	fileName := o.opts.Filename
+	if fileName == "" {
+		if v, ok := manifest.Layers[0].Annotations[titleAnnotation]; !ok {
+			return xerrors.Errorf("annotation %s is missing", titleAnnotation)
+		} else {
+			fileName = v
+		}
+	}
+
+	layerMediaType, err := layer.MediaType()
+	if err != nil {
+		return xerrors.Errorf("media type error: %w", err)
+	} else if o.opts.MediaType != "" && o.opts.MediaType != string(layerMediaType) {
+		return xerrors.Errorf("unacceptable media type: %s", string(layerMediaType))
+	}
+
+	if err = o.download(ctx, layer, fileName, dir, o.opts.Quiet); err != nil {
+		return xerrors.Errorf("oci download error: %w", err)
+	}
+
+	return nil
+}
+
+func (o *OCI) download(ctx context.Context, layer v1.Layer, fileName, dir string, quiet bool) error {
+	size, err := layer.Size()
+	if err != nil {
+		return xerrors.Errorf("size error: %w", err)
+	}
+
+	rc, err := layer.Compressed()
+	if err != nil {
+		return xerrors.Errorf("failed to fetch the layer: %w", err)
+	}
+	defer rc.Close()
+
+	// Show progress bar
+	bar := pb.Full.Start64(size)
+	if quiet {
+		bar.SetWriter(io.Discard)
+	}
+	pr := bar.NewProxyReader(rc)
+	defer bar.Finish()
+
+	// https://github.com/hashicorp/go-getter/issues/326
+	tempDir, err := os.MkdirTemp("", "trivy")
+	if err != nil {
+		return xerrors.Errorf("failed to create o temp dir: %w", err)
+	}
+
+	f, err := os.Create(filepath.Join(tempDir, fileName))
+	if err != nil {
+		return xerrors.Errorf("failed to create o temp file: %w", err)
+	}
+	defer func() {
+		_ = f.Close()
+		_ = os.RemoveAll(tempDir)
+	}()
+
+	// Download the layer content into o temporal file
+	if _, err = io.Copy(f, pr); err != nil {
+		return xerrors.Errorf("copy error: %w", err)
+	}
+
+	// Decompress the downloaded file if it is compressed and copy it into the dst
+	// NOTE: it's local copying, the insecure option doesn't matter.
+	if _, err = downloader.Download(ctx, f.Name(), dir, dir, downloader.Options{}); err != nil {
+		return xerrors.Errorf("download error: %w", err)
+	}
+
+	return nil
+}
+
+func (o *OCI) Digest(ctx context.Context) (string, error) {
+	if err := o.populate(ctx); err != nil {
+		return "", err
+	}
+
+	digest, err := o.image.Digest()
+	if err != nil {
+		return "", xerrors.Errorf("digest error: %w", err)
+	}
+	return digest.String(), nil
+}

pkg/oci/artifact_test.go → pkg/asset/oci_test.go

@@ -1,4 +1,4 @@
-package oci_test
+package asset_test
 
 import (
 	"context"
@@ -14,8 +14,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
-	"github.com/aquasecurity/trivy/pkg/oci"
+	"github.com/aquasecurity/trivy/pkg/asset"
 )
 
 type fakeLayer struct {
@@ -116,11 +115,12 @@ func TestArtifact_Download(t *testing.T) {
 				},
 			}, nil)
 
-			artifact := oci.NewArtifact("repo", ftypes.RegistryOptions{}, oci.WithImage(img))
-			err = artifact.Download(context.Background(), tempDir, oci.DownloadOption{
+			artifact := asset.NewOCI("repo", asset.Options{
 				MediaType: tt.mediaType,
 				Quiet:     true,
-			})
+			}, asset.WithImage(img))
+
+			err = artifact.Download(context.Background(), tempDir)
 			if tt.wantErr != "" {
 				assert.ErrorContains(t, err, tt.wantErr)
 				return