Merge pull request #189 from maysunfaisal/1221-1

Implement Interface definition for DownloadInMemory

.codecov.yaml

@@ -29,6 +29,8 @@ coverage:
     - "vendor/*"
     - "Makefile"
     - ".travis.yml"
+    - "pkg/devfile/parser/util/mock.go"
+    - "pkg/util/mock.go"
 
 # See http://docs.codecov.io/docs/pull-request-comments-1
 comment:

README.md

@@ -226,6 +226,29 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g
    }
    ```
 
+11. To download/access files from a private repository like a private GitHub use the `Token` property
+   ```go
+   parserArgs := parser.ParserArgs{
+		Token: "my-PAT",
+   }
+   ```
+
+   ```go
+   src: YamlSrc{
+		URL: "http://github.com/my-private-repo",
+		Token: "my-PAT",
+   }
+   values, err := ReadKubernetesYaml(src, fs, nil)
+   ```
+
+   If you would like to use the mock implementation for the `DevfileUtils` interface method defined in [pkg/devfile/parser/util/interface.go](pkg/devfile/parser/util/interface.go), then use 
+   ```go
+   var devfileUtilsClient DevfileUtils
+   devfileUtilsClient = NewMockDevfileUtilsClient()
+   devfileUtilsClient.DownloadInMemory(params)
+   ```
+
+
 ## Projects using devfile/library
 
 The following projects are consuming this library as a Golang dependency

pkg/devfile/parse_test.go

@@ -230,8 +230,7 @@ spec:
   type: LoadBalancer
 `
 	uri := "127.0.0.1:8080"
-	var testServer *httptest.Server
-	testServer = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	testServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		var err error
 		if strings.Contains(r.URL.Path, "/outerloop-deploy.yaml") {
 			_, err = w.Write([]byte(outerloopDeployContent))
@@ -252,7 +251,7 @@ spec:
 	// create a listener with the desired port.
 	l, err := net.Listen("tcp", uri)
 	if err != nil {
-		t.Errorf("Test_parseParentAndPluginFromURI() unexpected error while creating listener: %v", err)
+		t.Errorf("TestParseDevfileAndValidate() unexpected error while creating listener: %v", err)
 	}
 
 	// NewUnstartedServer creates a listener. Close that listener and replace

pkg/devfile/parser/context/content.go

@@ -19,6 +19,7 @@ import (
 	"bytes"
 	"unicode"
 
+	parserUtil "github.com/devfile/library/v2/pkg/devfile/parser/util"
 	"github.com/devfile/library/v2/pkg/util"
 	"github.com/pkg/errors"
 	"k8s.io/klog"
@@ -62,7 +63,7 @@ func hasPrefix(buf []byte, prefix []byte) bool {
 }
 
 // SetDevfileContent reads devfile and if devfile is in YAML format converts it to JSON
-func (d *DevfileCtx) SetDevfileContent() error {
+func (d *DevfileCtx) SetDevfileContent(devfileUtilsClient parserUtil.DevfileUtils) error {
 
 	var err error
 	var data []byte
@@ -72,7 +73,7 @@ func (d *DevfileCtx) SetDevfileContent() error {
 		if d.token != "" {
 			params.Token = d.token
 		}
-		data, err = util.DownloadInMemory(params)
+		data, err = devfileUtilsClient.DownloadInMemory(params)
 		if err != nil {
 			return errors.Wrap(err, "error getting devfile info from url")
 		}

pkg/devfile/parser/context/content_test.go

@@ -19,6 +19,7 @@ import (
 	"os"
 	"testing"
 
+	parserUtil "github.com/devfile/library/v2/pkg/devfile/parser/util"
 	"github.com/devfile/library/v2/pkg/testingutil/filesystem"
 )
 
@@ -67,7 +68,7 @@ func TestSetDevfileContent(t *testing.T) {
 		)
 		defer os.Remove(tempDevfile.Name())
 
-		err := d.SetDevfileContent()
+		err := d.SetDevfileContent(parserUtil.NewDevfileUtilsClient())
 
 		if err != nil {
 			t.Errorf("unexpected error '%v'", err)
@@ -90,7 +91,7 @@ func TestSetDevfileContent(t *testing.T) {
 		)
 		defer os.Remove(tempDevfile.Name())
 
-		err := d.SetDevfileContent()
+		err := d.SetDevfileContent(parserUtil.NewDevfileUtilsClient())
 
 		if err == nil {
 			t.Errorf("expected error, didn't get one ")
@@ -111,7 +112,7 @@ func TestSetDevfileContent(t *testing.T) {
 			}
 		)
 
-		err := d.SetDevfileContent()
+		err := d.SetDevfileContent(parserUtil.NewDevfileUtilsClient())
 
 		if err == nil {
 			t.Errorf("expected an error, didn't get one")

pkg/devfile/parser/context/context.go

@@ -18,6 +18,7 @@ package parser
 import (
 	"net/url"
 
+	parserUtil "github.com/devfile/library/v2/pkg/devfile/parser/util"
 	"github.com/devfile/library/v2/pkg/testingutil/filesystem"
 	"github.com/devfile/library/v2/pkg/util"
 	"k8s.io/klog"
@@ -94,7 +95,7 @@ func (d *DevfileCtx) populateDevfile() (err error) {
 }
 
 // Populate fills the DevfileCtx struct with relevant context info
-func (d *DevfileCtx) Populate() (err error) {
+func (d *DevfileCtx) Populate(devfileUtilsClient parserUtil.DevfileUtils) (err error) {
 	d.relPath, err = lookupDevfileFromPath(d.fs, d.relPath)
 	if err != nil {
 		return err
@@ -104,20 +105,20 @@ func (d *DevfileCtx) Populate() (err error) {
 	}
 	klog.V(4).Infof("absolute devfile path: '%s'", d.absPath)
 	// Read and save devfile content
-	if err = d.SetDevfileContent(); err != nil {
+	if err = d.SetDevfileContent(devfileUtilsClient); err != nil {
 		return err
 	}
 	return d.populateDevfile()
 }
 
 // PopulateFromURL fills the DevfileCtx struct with relevant context info
-func (d *DevfileCtx) PopulateFromURL() (err error) {
+func (d *DevfileCtx) PopulateFromURL(devfileUtilsClient parserUtil.DevfileUtils) (err error) {
 	_, err = url.ParseRequestURI(d.url)
 	if err != nil {
 		return err
 	}
 	// Read and save devfile content
-	if err := d.SetDevfileContent(); err != nil {
+	if err := d.SetDevfileContent(devfileUtilsClient); err != nil {
 		return err
 	}
 	return d.populateDevfile()

pkg/devfile/parser/context/context_test.go

@@ -17,6 +17,9 @@ package parser
 
 import (
 	"github.com/stretchr/testify/assert"
+
+	parserUtil "github.com/devfile/library/v2/pkg/devfile/parser/util"
+
 	"net/http"
 	"net/http/httptest"
 	"testing"
@@ -54,7 +57,7 @@ func TestPopulateFromBytes(t *testing.T) {
 				}
 			)
 			defer testServer.Close()
-			err := d.PopulateFromURL()
+			err := d.PopulateFromURL(parserUtil.NewDevfileUtilsClient())
 			if (tt.expectError != nil) != (err != nil) {
 				t.Errorf("TestPopulateFromBytes(): unexpected error: %v, wantErr: %v", err, tt.expectError)
 			} else if tt.expectError != nil {
@@ -73,7 +76,7 @@ func TestPopulateFromInvalidURL(t *testing.T) {
 			}
 		)
 
-		err := d.PopulateFromURL()
+		err := d.PopulateFromURL(parserUtil.NewDevfileUtilsClient())
 
 		if err == nil {
 			t.Errorf("TestPopulateFromInvalidURL(): expected an error, didn't get one")

pkg/devfile/parser/parse.go

@@ -37,6 +37,8 @@ import (
 	"k8s.io/klog"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	parserUtil "github.com/devfile/library/v2/pkg/devfile/parser/util"
+
 	v1 "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
 	apiOverride "github.com/devfile/api/v2/pkg/utils/overriding"
 	"github.com/devfile/api/v2/pkg/validation"
@@ -120,7 +122,7 @@ type ParserArgs struct {
 	// DownloadGitResources downloads the resources from Git repository if true
 	DownloadGitResources *bool
 	// DevfileUtilsClient exposes the interface for mock implementation.
-	DevfileUtilsClient DevfileUtils
+	DevfileUtilsClient parserUtil.DevfileUtils
 }
 
 // ImageSelectorArgs defines the structure to leverage for using image names as selectors after parsing the Devfile.
@@ -169,7 +171,7 @@ func ParseDevfile(args ParserArgs) (d DevfileObj, err error) {
 	}
 
 	if args.DevfileUtilsClient == nil {
-		args.DevfileUtilsClient = NewDevfileUtilsClient()
+		args.DevfileUtilsClient = parserUtil.NewDevfileUtilsClient()
 	}
 
 	downloadGitResources := true
@@ -216,7 +218,7 @@ func ParseDevfile(args ParserArgs) (d DevfileObj, err error) {
 
 	if convertUriToInlined {
 		d.Ctx.SetConvertUriToInlined(true)
-		err = parseKubeResourceFromURI(d)
+		err = parseKubeResourceFromURI(d, tool.devfileUtilsClient)
 		if err != nil {
 			return d, err
 		}
@@ -241,7 +243,7 @@ type resolverTools struct {
 	// downloadGitResources downloads the resources from Git repository if true
 	downloadGitResources bool
 	// devfileUtilsClient exposes the Git Interface to be able to use mock implementation.
-	devfileUtilsClient DevfileUtils
+	devfileUtilsClient parserUtil.DevfileUtils
 }
 
 func populateAndParseDevfile(d DevfileObj, resolveCtx *resolutionContextTree, tool resolverTools, flattenedDevfile bool) (DevfileObj, error) {
@@ -251,11 +253,11 @@ func populateAndParseDevfile(d DevfileObj, resolveCtx *resolutionContextTree, to
 	}
 	// Fill the fields of DevfileCtx struct
 	if d.Ctx.GetURL() != "" {
-		err = d.Ctx.PopulateFromURL()
+		err = d.Ctx.PopulateFromURL(tool.devfileUtilsClient)
 	} else if d.Ctx.GetDevfileContent() != nil {
 		err = d.Ctx.PopulateFromRaw()
 	} else {
-		err = d.Ctx.Populate()
+		err = d.Ctx.Populate(tool.devfileUtilsClient)
 	}
 	if err != nil {
 		return d, err
@@ -343,6 +345,7 @@ func parseParentAndPlugin(d DevfileObj, resolveCtx *resolutionContextTree, tool
 				if err != nil {
 					return fmt.Errorf("fail to parse version of parent devfile from: %v", resolveImportReference(parent.ImportReference))
 				}
+
 				if parentDevfileVerson.GreaterThan(mainDevfileVersion) {
 					return fmt.Errorf("the parent devfile version from %v is greater than the child devfile version from %v", resolveImportReference(parent.ImportReference), resolveImportReference(resolveCtx.importReference))
 				}
@@ -765,7 +768,7 @@ func setEndpoints(endpoints []v1.Endpoint) {
 }
 
 // parseKubeResourceFromURI iterate through all kubernetes & openshift components, and parse from uri and update the content to inlined field in devfileObj
-func parseKubeResourceFromURI(devObj DevfileObj) error {
+func parseKubeResourceFromURI(devObj DevfileObj, devfileUtilsClient parserUtil.DevfileUtils) error {
 	getKubeCompOptions := common.DevfileOptions{
 		ComponentOptions: common.ComponentOptions{
 			ComponentType: v1.KubernetesComponentType,
@@ -787,7 +790,7 @@ func parseKubeResourceFromURI(devObj DevfileObj) error {
 	for _, kubeComp := range kubeComponents {
 		if kubeComp.Kubernetes != nil && kubeComp.Kubernetes.Uri != "" {
 			/* #nosec G601 -- not an issue, kubeComp is de-referenced in sequence*/
-			err := convertK8sLikeCompUriToInlined(&kubeComp, devObj.Ctx)
+			err := convertK8sLikeCompUriToInlined(&kubeComp, devObj.Ctx, devfileUtilsClient)
 			if err != nil {
 				return errors.Wrapf(err, "failed to convert kubernetes uri to inlined for component '%s'", kubeComp.Name)
 			}
@@ -800,7 +803,7 @@ func parseKubeResourceFromURI(devObj DevfileObj) error {
 	for _, openshiftComp := range openshiftComponents {
 		if openshiftComp.Openshift != nil && openshiftComp.Openshift.Uri != "" {
 			/* #nosec G601 -- not an issue, openshiftComp is de-referenced in sequence*/
-			err := convertK8sLikeCompUriToInlined(&openshiftComp, devObj.Ctx)
+			err := convertK8sLikeCompUriToInlined(&openshiftComp, devObj.Ctx, devfileUtilsClient)
 			if err != nil {
 				return errors.Wrapf(err, "failed to convert openshift uri to inlined for component '%s'", openshiftComp.Name)
 			}
@@ -814,14 +817,14 @@ func parseKubeResourceFromURI(devObj DevfileObj) error {
 }
 
 // convertK8sLikeCompUriToInlined read in kubernetes resources definition from uri and converts to kubernetest inlined field
-func convertK8sLikeCompUriToInlined(component *v1.Component, d devfileCtx.DevfileCtx) error {
+func convertK8sLikeCompUriToInlined(component *v1.Component, d devfileCtx.DevfileCtx, devfileUtilsClient parserUtil.DevfileUtils) error {
 	var uri string
 	if component.Kubernetes != nil {
 		uri = component.Kubernetes.Uri
 	} else if component.Openshift != nil {
 		uri = component.Openshift.Uri
 	}
-	data, err := getKubernetesDefinitionFromUri(uri, d)
+	data, err := getKubernetesDefinitionFromUri(uri, d, devfileUtilsClient)
 	if err != nil {
 		return err
 	}
@@ -841,7 +844,7 @@ func convertK8sLikeCompUriToInlined(component *v1.Component, d devfileCtx.Devfil
 }
 
 // getKubernetesDefinitionFromUri read in kubernetes resources definition from uri and returns the raw content
-func getKubernetesDefinitionFromUri(uri string, d devfileCtx.DevfileCtx) ([]byte, error) {
+func getKubernetesDefinitionFromUri(uri string, d devfileCtx.DevfileCtx, devfileUtilsClient parserUtil.DevfileUtils) ([]byte, error) {
 	// validate URI
 	err := validation.ValidateURI(uri)
 	if err != nil {
@@ -876,7 +879,7 @@ func getKubernetesDefinitionFromUri(uri string, d devfileCtx.DevfileCtx) ([]byte
 		if d.GetToken() != "" {
 			params.Token = d.GetToken()
 		}
-		data, err = util.DownloadInMemory(params)
+		data, err = devfileUtilsClient.DownloadInMemory(params)
 		if err != nil {
 			return nil, errors.Wrapf(err, "error getting kubernetes resources definition information")
 		}