From 208f9d60ebc3160a044101a87c3188e802534424 Mon Sep 17 00:00:00 2001 From: Maysun J Faisal Date: Fri, 17 Nov 2023 18:46:57 -0500 Subject: [PATCH 1/5] Implement Interface definition for DownloadInMemory Signed-off-by: Maysun J Faisal --- pkg/devfile/parser/context/content.go | 5 +- pkg/devfile/parser/context/content_test.go | 7 +- pkg/devfile/parser/context/context.go | 9 +- pkg/devfile/parser/context/context_test.go | 7 +- pkg/devfile/parser/parse.go | 29 ++-- pkg/devfile/parser/parse_test.go | 56 ++++++-- pkg/devfile/parser/reader.go | 2 +- pkg/devfile/parser/util/interface.go | 23 ++++ .../parser/{parser_mock.go => util/mock.go} | 41 ++++-- pkg/devfile/parser/util/utils.go | 90 +++++++++++++ pkg/devfile/parser/utils.go | 62 --------- pkg/devfile/parser/utils_test.go | 12 +- pkg/util/mock.go | 124 ++++++++++++++++++ pkg/util/util.go | 21 +-- 14 files changed, 361 insertions(+), 127 deletions(-) create mode 100644 pkg/devfile/parser/util/interface.go rename pkg/devfile/parser/{parser_mock.go => util/mock.go} (64%) create mode 100644 pkg/devfile/parser/util/utils.go diff --git a/pkg/devfile/parser/context/content.go b/pkg/devfile/parser/context/content.go index 52b3a45c..6f755877 100644 --- a/pkg/devfile/parser/context/content.go +++ b/pkg/devfile/parser/context/content.go @@ -19,6 +19,7 @@ import ( "bytes" "unicode" + parserUtil "github.com/devfile/library/v2/pkg/devfile/parser/util" "github.com/devfile/library/v2/pkg/util" "github.com/pkg/errors" "k8s.io/klog" @@ -62,7 +63,7 @@ func hasPrefix(buf []byte, prefix []byte) bool { } // SetDevfileContent reads devfile and if devfile is in YAML format converts it to JSON -func (d *DevfileCtx) SetDevfileContent() error { +func (d *DevfileCtx) SetDevfileContent(devfileUtilsClient parserUtil.DevfileUtils) error { var err error var data []byte @@ -72,7 +73,7 @@ func (d *DevfileCtx) SetDevfileContent() error { if d.token != "" { params.Token = d.token } - data, err = util.DownloadInMemory(params) + data, err = devfileUtilsClient.DownloadInMemory(params) if err != nil { return errors.Wrap(err, "error getting devfile info from url") } diff --git a/pkg/devfile/parser/context/content_test.go b/pkg/devfile/parser/context/content_test.go index d0b4509f..9f6ee3fa 100644 --- a/pkg/devfile/parser/context/content_test.go +++ b/pkg/devfile/parser/context/content_test.go @@ -19,6 +19,7 @@ import ( "os" "testing" + parserUtil "github.com/devfile/library/v2/pkg/devfile/parser/util" "github.com/devfile/library/v2/pkg/testingutil/filesystem" ) @@ -67,7 +68,7 @@ func TestSetDevfileContent(t *testing.T) { ) defer os.Remove(tempDevfile.Name()) - err := d.SetDevfileContent() + err := d.SetDevfileContent(parserUtil.NewDevfileUtilsClient()) if err != nil { t.Errorf("unexpected error '%v'", err) @@ -90,7 +91,7 @@ func TestSetDevfileContent(t *testing.T) { ) defer os.Remove(tempDevfile.Name()) - err := d.SetDevfileContent() + err := d.SetDevfileContent(parserUtil.NewDevfileUtilsClient()) if err == nil { t.Errorf("expected error, didn't get one ") @@ -111,7 +112,7 @@ func TestSetDevfileContent(t *testing.T) { } ) - err := d.SetDevfileContent() + err := d.SetDevfileContent(parserUtil.NewDevfileUtilsClient()) if err == nil { t.Errorf("expected an error, didn't get one") diff --git a/pkg/devfile/parser/context/context.go b/pkg/devfile/parser/context/context.go index 0a283121..413b6b46 100644 --- a/pkg/devfile/parser/context/context.go +++ b/pkg/devfile/parser/context/context.go @@ -18,6 +18,7 @@ package parser import ( "net/url" + parserUtil "github.com/devfile/library/v2/pkg/devfile/parser/util" "github.com/devfile/library/v2/pkg/testingutil/filesystem" "github.com/devfile/library/v2/pkg/util" "k8s.io/klog" @@ -94,7 +95,7 @@ func (d *DevfileCtx) populateDevfile() (err error) { } // Populate fills the DevfileCtx struct with relevant context info -func (d *DevfileCtx) Populate() (err error) { +func (d *DevfileCtx) Populate(devfileUtilsClient parserUtil.DevfileUtils) (err error) { d.relPath, err = lookupDevfileFromPath(d.fs, d.relPath) if err != nil { return err @@ -104,20 +105,20 @@ func (d *DevfileCtx) Populate() (err error) { } klog.V(4).Infof("absolute devfile path: '%s'", d.absPath) // Read and save devfile content - if err = d.SetDevfileContent(); err != nil { + if err = d.SetDevfileContent(devfileUtilsClient); err != nil { return err } return d.populateDevfile() } // PopulateFromURL fills the DevfileCtx struct with relevant context info -func (d *DevfileCtx) PopulateFromURL() (err error) { +func (d *DevfileCtx) PopulateFromURL(devfileUtilsClient parserUtil.DevfileUtils) (err error) { _, err = url.ParseRequestURI(d.url) if err != nil { return err } // Read and save devfile content - if err := d.SetDevfileContent(); err != nil { + if err := d.SetDevfileContent(devfileUtilsClient); err != nil { return err } return d.populateDevfile() diff --git a/pkg/devfile/parser/context/context_test.go b/pkg/devfile/parser/context/context_test.go index 3f332e88..9f2dc4d1 100644 --- a/pkg/devfile/parser/context/context_test.go +++ b/pkg/devfile/parser/context/context_test.go @@ -17,6 +17,9 @@ package parser import ( "github.com/stretchr/testify/assert" + + parserUtil "github.com/devfile/library/v2/pkg/devfile/parser/util" + "net/http" "net/http/httptest" "testing" @@ -54,7 +57,7 @@ func TestPopulateFromBytes(t *testing.T) { } ) defer testServer.Close() - err := d.PopulateFromURL() + err := d.PopulateFromURL(parserUtil.NewDevfileUtilsClient()) if (tt.expectError != nil) != (err != nil) { t.Errorf("TestPopulateFromBytes(): unexpected error: %v, wantErr: %v", err, tt.expectError) } else if tt.expectError != nil { @@ -73,7 +76,7 @@ func TestPopulateFromInvalidURL(t *testing.T) { } ) - err := d.PopulateFromURL() + err := d.PopulateFromURL(parserUtil.NewDevfileUtilsClient()) if err == nil { t.Errorf("TestPopulateFromInvalidURL(): expected an error, didn't get one") diff --git a/pkg/devfile/parser/parse.go b/pkg/devfile/parser/parse.go index 29bf13d9..33d555df 100644 --- a/pkg/devfile/parser/parse.go +++ b/pkg/devfile/parser/parse.go @@ -37,6 +37,8 @@ import ( "k8s.io/klog" "sigs.k8s.io/controller-runtime/pkg/client" + parserUtil "github.com/devfile/library/v2/pkg/devfile/parser/util" + v1 "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2" apiOverride "github.com/devfile/api/v2/pkg/utils/overriding" "github.com/devfile/api/v2/pkg/validation" @@ -120,7 +122,7 @@ type ParserArgs struct { // DownloadGitResources downloads the resources from Git repository if true DownloadGitResources *bool // DevfileUtilsClient exposes the interface for mock implementation. - DevfileUtilsClient DevfileUtils + DevfileUtilsClient parserUtil.DevfileUtils } // ImageSelectorArgs defines the structure to leverage for using image names as selectors after parsing the Devfile. @@ -169,7 +171,7 @@ func ParseDevfile(args ParserArgs) (d DevfileObj, err error) { } if args.DevfileUtilsClient == nil { - args.DevfileUtilsClient = NewDevfileUtilsClient() + args.DevfileUtilsClient = parserUtil.NewDevfileUtilsClient() } downloadGitResources := true @@ -216,7 +218,7 @@ func ParseDevfile(args ParserArgs) (d DevfileObj, err error) { if convertUriToInlined { d.Ctx.SetConvertUriToInlined(true) - err = parseKubeResourceFromURI(d) + err = parseKubeResourceFromURI(d, tool.devfileUtilsClient) if err != nil { return d, err } @@ -241,7 +243,7 @@ type resolverTools struct { // downloadGitResources downloads the resources from Git repository if true downloadGitResources bool // devfileUtilsClient exposes the Git Interface to be able to use mock implementation. - devfileUtilsClient DevfileUtils + devfileUtilsClient parserUtil.DevfileUtils } func populateAndParseDevfile(d DevfileObj, resolveCtx *resolutionContextTree, tool resolverTools, flattenedDevfile bool) (DevfileObj, error) { @@ -251,11 +253,11 @@ func populateAndParseDevfile(d DevfileObj, resolveCtx *resolutionContextTree, to } // Fill the fields of DevfileCtx struct if d.Ctx.GetURL() != "" { - err = d.Ctx.PopulateFromURL() + err = d.Ctx.PopulateFromURL(tool.devfileUtilsClient) } else if d.Ctx.GetDevfileContent() != nil { err = d.Ctx.PopulateFromRaw() } else { - err = d.Ctx.Populate() + err = d.Ctx.Populate(tool.devfileUtilsClient) } if err != nil { return d, err @@ -343,6 +345,7 @@ func parseParentAndPlugin(d DevfileObj, resolveCtx *resolutionContextTree, tool if err != nil { return fmt.Errorf("fail to parse version of parent devfile from: %v", resolveImportReference(parent.ImportReference)) } + if parentDevfileVerson.GreaterThan(mainDevfileVersion) { return fmt.Errorf("the parent devfile version from %v is greater than the child devfile version from %v", resolveImportReference(parent.ImportReference), resolveImportReference(resolveCtx.importReference)) } @@ -765,7 +768,7 @@ func setEndpoints(endpoints []v1.Endpoint) { } // parseKubeResourceFromURI iterate through all kubernetes & openshift components, and parse from uri and update the content to inlined field in devfileObj -func parseKubeResourceFromURI(devObj DevfileObj) error { +func parseKubeResourceFromURI(devObj DevfileObj, devfileUtilsClient parserUtil.DevfileUtils) error { getKubeCompOptions := common.DevfileOptions{ ComponentOptions: common.ComponentOptions{ ComponentType: v1.KubernetesComponentType, @@ -787,7 +790,7 @@ func parseKubeResourceFromURI(devObj DevfileObj) error { for _, kubeComp := range kubeComponents { if kubeComp.Kubernetes != nil && kubeComp.Kubernetes.Uri != "" { /* #nosec G601 -- not an issue, kubeComp is de-referenced in sequence*/ - err := convertK8sLikeCompUriToInlined(&kubeComp, devObj.Ctx) + err := convertK8sLikeCompUriToInlined(&kubeComp, devObj.Ctx, devfileUtilsClient) if err != nil { return errors.Wrapf(err, "failed to convert kubernetes uri to inlined for component '%s'", kubeComp.Name) } @@ -800,7 +803,7 @@ func parseKubeResourceFromURI(devObj DevfileObj) error { for _, openshiftComp := range openshiftComponents { if openshiftComp.Openshift != nil && openshiftComp.Openshift.Uri != "" { /* #nosec G601 -- not an issue, openshiftComp is de-referenced in sequence*/ - err := convertK8sLikeCompUriToInlined(&openshiftComp, devObj.Ctx) + err := convertK8sLikeCompUriToInlined(&openshiftComp, devObj.Ctx, devfileUtilsClient) if err != nil { return errors.Wrapf(err, "failed to convert openshift uri to inlined for component '%s'", openshiftComp.Name) } @@ -814,14 +817,14 @@ func parseKubeResourceFromURI(devObj DevfileObj) error { } // convertK8sLikeCompUriToInlined read in kubernetes resources definition from uri and converts to kubernetest inlined field -func convertK8sLikeCompUriToInlined(component *v1.Component, d devfileCtx.DevfileCtx) error { +func convertK8sLikeCompUriToInlined(component *v1.Component, d devfileCtx.DevfileCtx, devfileUtilsClient parserUtil.DevfileUtils) error { var uri string if component.Kubernetes != nil { uri = component.Kubernetes.Uri } else if component.Openshift != nil { uri = component.Openshift.Uri } - data, err := getKubernetesDefinitionFromUri(uri, d) + data, err := getKubernetesDefinitionFromUri(uri, d, devfileUtilsClient) if err != nil { return err } @@ -841,7 +844,7 @@ func convertK8sLikeCompUriToInlined(component *v1.Component, d devfileCtx.Devfil } // getKubernetesDefinitionFromUri read in kubernetes resources definition from uri and returns the raw content -func getKubernetesDefinitionFromUri(uri string, d devfileCtx.DevfileCtx) ([]byte, error) { +func getKubernetesDefinitionFromUri(uri string, d devfileCtx.DevfileCtx, devfileUtilsClient parserUtil.DevfileUtils) ([]byte, error) { // validate URI err := validation.ValidateURI(uri) if err != nil { @@ -876,7 +879,7 @@ func getKubernetesDefinitionFromUri(uri string, d devfileCtx.DevfileCtx) ([]byte if d.GetToken() != "" { params.Token = d.GetToken() } - data, err = util.DownloadInMemory(params) + data, err = devfileUtilsClient.DownloadInMemory(params) if err != nil { return nil, errors.Wrapf(err, "error getting kubernetes resources definition information") } diff --git a/pkg/devfile/parser/parse_test.go b/pkg/devfile/parser/parse_test.go index f4e214af..5e4d83d0 100644 --- a/pkg/devfile/parser/parse_test.go +++ b/pkg/devfile/parser/parse_test.go @@ -39,6 +39,7 @@ import ( "github.com/devfile/library/v2/pkg/devfile/parser/data" v2 "github.com/devfile/library/v2/pkg/devfile/parser/data/v2" "github.com/devfile/library/v2/pkg/devfile/parser/data/v2/common" + parserUtil "github.com/devfile/library/v2/pkg/devfile/parser/util" "github.com/devfile/library/v2/pkg/testingutil" "github.com/kylelemons/godebug/pretty" "github.com/stretchr/testify/assert" @@ -2862,7 +2863,7 @@ func Test_parseParentAndPluginFromURI(t *testing.T) { tt.args.devFileObj.Data.AddComponents(plugincomp) } - err := parseParentAndPlugin(tt.args.devFileObj, &resolutionContextTree{}, resolverTools{devfileUtilsClient: NewDevfileUtilsClient()}) + err := parseParentAndPlugin(tt.args.devFileObj, &resolutionContextTree{}, resolverTools{devfileUtilsClient: parserUtil.NewDevfileUtilsClient()}) // Unexpected error if (err != nil) != (tt.wantErr != nil) { @@ -2889,6 +2890,11 @@ func Test_parseParentAndPlugin_RecursivelyReference(t *testing.T) { Ctx: devfileCtx.NewDevfileCtx(OutputDevfileYamlPath), Data: &v2.DevfileV2{ Devfile: v1.Devfile{ + DevfileHeader: devfilepkg.DevfileHeader{ + Metadata: devfilepkg.DevfileMetadata{ + Name: "main-devfile", + }, + }, DevWorkspaceTemplateSpec: v1.DevWorkspaceTemplateSpec{ Parent: &v1.Parent{ ImportReference: v1.ImportReference{ @@ -2922,6 +2928,9 @@ func Test_parseParentAndPlugin_RecursivelyReference(t *testing.T) { Devfile: v1.Devfile{ DevfileHeader: devfilepkg.DevfileHeader{ SchemaVersion: schemaVersion, + Metadata: devfilepkg.DevfileMetadata{ + Name: "parent-devfile-1", + }, }, DevWorkspaceTemplateSpec: v1.DevWorkspaceTemplateSpec{ Parent: &v1.Parent{ @@ -2959,6 +2968,9 @@ func Test_parseParentAndPlugin_RecursivelyReference(t *testing.T) { Devfile: v1.Devfile{ DevfileHeader: devfilepkg.DevfileHeader{ SchemaVersion: schemaVersion, + Metadata: devfilepkg.DevfileMetadata{ + Name: "parent-devfile-2", + }, }, DevWorkspaceTemplateSpec: v1.DevWorkspaceTemplateSpec{ Parent: &v1.Parent{ @@ -3064,6 +3076,9 @@ func Test_parseParentAndPlugin_RecursivelyReference(t *testing.T) { Kind: "DevWorkspaceTemplate", APIVersion: "testgroup/v1alpha2", }, + ObjectMeta: kubev1.ObjectMeta{ + Name: "dwtemplate", + }, Spec: parentSpec, }, } @@ -3079,7 +3094,7 @@ func Test_parseParentAndPlugin_RecursivelyReference(t *testing.T) { k8sClient: testK8sClient, context: context.Background(), httpTimeout: &httpTimeout, - devfileUtilsClient: NewMockDevfileUtilsClient(), + devfileUtilsClient: parserUtil.NewDevfileUtilsClient(), } err := parseParentAndPlugin(devFileObj, &resolutionContextTree{}, tool) @@ -4173,7 +4188,7 @@ func Test_parseFromURI(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parseFromURI(tt.importReference, tt.curDevfileCtx, &resolutionContextTree{}, resolverTools{devfileUtilsClient: NewDevfileUtilsClient()}) + got, err := parseFromURI(tt.importReference, tt.curDevfileCtx, &resolutionContextTree{}, resolverTools{devfileUtilsClient: parserUtil.NewDevfileUtilsClient()}) if (err != nil) != (tt.wantErr != nil) { t.Errorf("Test_parseFromURI() unexpected error: %v, wantErr %v", err, tt.wantErr) } else if err == nil && !reflect.DeepEqual(got.Data, tt.wantDevFile.Data) { @@ -4230,12 +4245,13 @@ func Test_parseFromURI_GitProviders(t *testing.T) { invalidTokenError := "failed to clone repo with token, ensure that the url and token is correct" invalidGitSwitchError := "failed to switch repo to revision*" invalidDevfilePathError := "error getting devfile from url: failed to retrieve*" - invalidGitProviderError := "Failed to download resources from parent devfile. Unsupported Git Provider for %s " + invalidGitProviderError := "failed to download resources from parent devfile. Unsupported Git Provider for %s " tests := []struct { name string url string // alias for parent devfile URL gitUrl *util.GitUrl + devfileUtilsClient parserUtil.MockDevfileUtilsClient token string destDir string importReference v1.ImportReference @@ -4249,7 +4265,12 @@ func Test_parseFromURI_GitProviders(t *testing.T) { name: "private parent devfile", url: validUrl, gitUrl: validGitUrl, - token: validToken, + devfileUtilsClient: parserUtil.MockDevfileUtilsClient{ + DownloadOptions: util.MockDownloadOptions{ + MockFile: minimalDevfileContent, + }, + }, + token: validToken, importReference: v1.ImportReference{ ImportReferenceUnion: v1.ImportReferenceUnion{ Uri: server.URL, @@ -4264,7 +4285,12 @@ func Test_parseFromURI_GitProviders(t *testing.T) { name: "public parent devfile", url: validUrl, gitUrl: validGitUrl, - token: "", + devfileUtilsClient: parserUtil.MockDevfileUtilsClient{ + DownloadOptions: util.MockDownloadOptions{ + MockFile: minimalDevfileContent, + }, + }, + token: "", importReference: v1.ImportReference{ ImportReferenceUnion: v1.ImportReferenceUnion{ Uri: server.URL, @@ -4279,6 +4305,11 @@ func Test_parseFromURI_GitProviders(t *testing.T) { name: "public parent devfile with download turned off", url: validUrl, gitUrl: validGitUrl, + devfileUtilsClient: parserUtil.MockDevfileUtilsClient{ + DownloadOptions: util.MockDownloadOptions{ + MockFile: minimalDevfileContent, + }, + }, importReference: v1.ImportReference{ ImportReferenceUnion: v1.ImportReferenceUnion{ Uri: server.URL, @@ -4413,12 +4444,11 @@ func Test_parseFromURI_GitProviders(t *testing.T) { t.Errorf("Unexpected err: %+v", err) } - mockDC := NewMockDevfileUtilsClient() - mockDC.ParentURLAlias = tt.url - mockDC.GitTestToken = tt.token - mockDC.MockGitURL = util.MockGitUrl(*tt.gitUrl) + tt.devfileUtilsClient.ParentURLAlias = tt.url + tt.devfileUtilsClient.GitTestToken = tt.token + tt.devfileUtilsClient.MockGitURL = util.MockGitUrl(*tt.gitUrl) - got, err := parseFromURI(tt.importReference, curDevfileContext, &resolutionContextTree{}, resolverTools{downloadGitResources: tt.downloadGitResources, devfileUtilsClient: mockDC}) + got, err := parseFromURI(tt.importReference, curDevfileContext, &resolutionContextTree{}, resolverTools{downloadGitResources: tt.downloadGitResources, devfileUtilsClient: tt.devfileUtilsClient}) // validate even if we want an error; check that no files are copied to destDir validateGitResourceFunctions(t, tt.wantResources, tt.wantResourceContent, destDir) @@ -4854,7 +4884,7 @@ func Test_DownloadGitRepoResources(t *testing.T) { }, } - mockDC := NewMockDevfileUtilsClient() + mockDC := parserUtil.NewMockDevfileUtilsClient() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { destDir := t.TempDir() @@ -5121,7 +5151,7 @@ commands: } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := getKubernetesDefinitionFromUri(tt.uri, tt.devfileCtx) + got, err := getKubernetesDefinitionFromUri(tt.uri, tt.devfileCtx, parserUtil.NewDevfileUtilsClient()) if (err != nil) != (tt.wantErr != nil) { t.Errorf("Test_getKubernetesDefinitionFromUri() unexpected error: %v, wantErr %v", err, *tt.wantErr) } else if err == nil { diff --git a/pkg/devfile/parser/reader.go b/pkg/devfile/parser/reader.go index a9c27d46..9728ad8c 100644 --- a/pkg/devfile/parser/reader.go +++ b/pkg/devfile/parser/reader.go @@ -64,7 +64,7 @@ func ReadKubernetesYaml(src YamlSrc, fs *afero.Afero) ([]interface{}, error) { if src.URL != "" { params := util.HTTPRequestParams{URL: src.URL} - data, err = util.DownloadInMemory(params) + data, err = util.DownloadInMemory(params) // TODO: call with Token if err != nil { return nil, errors.Wrapf(err, "failed to download file %q", src.URL) } diff --git a/pkg/devfile/parser/util/interface.go b/pkg/devfile/parser/util/interface.go new file mode 100644 index 00000000..c1d64092 --- /dev/null +++ b/pkg/devfile/parser/util/interface.go @@ -0,0 +1,23 @@ +// +// Copyright 2023 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import "github.com/devfile/library/v2/pkg/util" + +type DevfileUtils interface { + DownloadGitRepoResources(url string, destDir string, token string) error + DownloadInMemory(params util.HTTPRequestParams) ([]byte, error) +} diff --git a/pkg/devfile/parser/parser_mock.go b/pkg/devfile/parser/util/mock.go similarity index 64% rename from pkg/devfile/parser/parser_mock.go rename to pkg/devfile/parser/util/mock.go index aa67db92..1978ce7e 100644 --- a/pkg/devfile/parser/parser_mock.go +++ b/pkg/devfile/parser/util/mock.go @@ -13,25 +13,50 @@ // See the License for the specific language governing permissions and // limitations under the License. -package parser +package util import ( "fmt" - "github.com/devfile/library/v2/pkg/util" + "net/http" "os" "strings" + + "github.com/devfile/library/v2/pkg/util" ) type MockDevfileUtilsClient struct { - ParentURLAlias string // Specify a valid git URL as an alias if using a localhost HTTP server in order to pass validation. - MockGitURL util.MockGitUrl - GitTestToken string // Mock Git token. Specify the string "valid-token" for the mock CloneGitRepo to pass + // Specify a valid git URL as an alias if using a localhost HTTP server in order to pass validation. + ParentURLAlias string + + // MockGitUrl struct for mocking git related ops + MockGitURL util.MockGitUrl + + // Mock Git token. Specify the string "valid-token" for the mock CloneGitRepo to pass + GitTestToken string + + // Options to specify what file download needs to be mocked + DownloadOptions util.MockDownloadOptions } func NewMockDevfileUtilsClient() MockDevfileUtilsClient { return MockDevfileUtilsClient{} } +func (gc MockDevfileUtilsClient) DownloadInMemory(params util.HTTPRequestParams) ([]byte, error) { + var httpClient = &http.Client{Transport: &http.Transport{ + ResponseHeaderTimeout: util.HTTPRequestResponseTimeout, + }, Timeout: util.HTTPRequestResponseTimeout} + + var mockGitUrl util.MockGitUrl + + if util.IsGitProviderRepo(gc.MockGitURL.Host) { + mockGitUrl = gc.MockGitURL + mockGitUrl.Token = gc.GitTestToken + } + + return mockGitUrl.DownloadInMemoryWithClient(params, httpClient, gc.DownloadOptions) +} + func (gc MockDevfileUtilsClient) DownloadGitRepoResources(url string, destDir string, token string) error { //the url parameter that gets passed in will be the localhost IP of the test server, so it will fail all the validation checks. We will use the global testURL variable instead @@ -45,13 +70,13 @@ func (gc MockDevfileUtilsClient) DownloadGitRepoResources(url string, destDir st return fmt.Errorf("error getting devfile from url: failed to retrieve %s", url+"/"+mockGitUrl.Path) } - stackDir, err := os.MkdirTemp("", fmt.Sprintf("git-resources")) + stackDir, err := os.MkdirTemp("", "git-resources") if err != nil { return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) } defer func(path string) { - err := os.RemoveAll(path) + err = os.RemoveAll(path) if err != nil { err = fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) } @@ -68,7 +93,7 @@ func (gc MockDevfileUtilsClient) DownloadGitRepoResources(url string, destDir st } } else { - return fmt.Errorf("Failed to download resources from parent devfile. Unsupported Git Provider for %s ", gc.ParentURLAlias) + return fmt.Errorf("failed to download resources from parent devfile. Unsupported Git Provider for %s ", gc.ParentURLAlias) } return nil diff --git a/pkg/devfile/parser/util/utils.go b/pkg/devfile/parser/util/utils.go new file mode 100644 index 00000000..4f1ada5e --- /dev/null +++ b/pkg/devfile/parser/util/utils.go @@ -0,0 +1,90 @@ +// +// Copyright 2022-2023 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "fmt" + "os" + "path" + "strings" + + "github.com/devfile/library/v2/pkg/util" + "github.com/hashicorp/go-multierror" +) + +// Default filenames for create devfile +const ( + OutputDevfileYamlPath = "devfile.yaml" +) + +type DevfileUtilsClient struct { +} + +func NewDevfileUtilsClient() DevfileUtilsClient { + return DevfileUtilsClient{} +} + +// DownloadInMemory is a wrapper to the util.DownloadInMemory() call. +// This is done to help devfile/library clients invoke this function with a client. +func (c DevfileUtilsClient) DownloadInMemory(params util.HTTPRequestParams) ([]byte, error) { + return util.DownloadInMemory(params) +} + +// DownloadGitRepoResources downloads the git repository resources +func (c DevfileUtilsClient) DownloadGitRepoResources(url string, destDir string, token string) error { + var returnedErr error + if util.IsGitProviderRepo(url) { + gitUrl, err := util.NewGitURL(url, token) + if err != nil { + return err + } + + if !gitUrl.IsFile || gitUrl.Revision == "" || !strings.Contains(gitUrl.Path, OutputDevfileYamlPath) { + return fmt.Errorf("error getting devfile from url: failed to retrieve %s", url) + } + + stackDir, err := os.MkdirTemp("", "git-resources") + if err != nil { + return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) + } + + defer func(path string) { + err := os.RemoveAll(path) + if err != nil { + returnedErr = multierror.Append(returnedErr, err) + } + }(stackDir) + + gitUrl.Token = token + + err = gitUrl.CloneGitRepo(stackDir) + if err != nil { + returnedErr = multierror.Append(returnedErr, err) + return returnedErr + } + + dir := path.Dir(path.Join(stackDir, gitUrl.Path)) + err = util.CopyAllDirFiles(dir, destDir) + if err != nil { + returnedErr = multierror.Append(returnedErr, err) + return returnedErr + } + } else { + return fmt.Errorf("failed to download resources from parent devfile. Unsupported Git Provider for %s ", url) + } + + return nil +} diff --git a/pkg/devfile/parser/utils.go b/pkg/devfile/parser/utils.go index 477e3903..96a75bf3 100644 --- a/pkg/devfile/parser/utils.go +++ b/pkg/devfile/parser/utils.go @@ -17,75 +17,13 @@ package parser import ( "fmt" - "github.com/devfile/library/v2/pkg/util" - "github.com/hashicorp/go-multierror" - "os" - "path" "reflect" - "strings" devfilev1 "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2" "github.com/devfile/library/v2/pkg/devfile/parser/data" "github.com/devfile/library/v2/pkg/devfile/parser/data/v2/common" ) -type DevfileUtilsClient struct { -} - -func NewDevfileUtilsClient() DevfileUtilsClient { - return DevfileUtilsClient{} -} - -type DevfileUtils interface { - DownloadGitRepoResources(url string, destDir string, token string) error -} - -// DownloadGitRepoResources mock implementation of the real method. -func (gc DevfileUtilsClient) DownloadGitRepoResources(url string, destDir string, token string) error { - var returnedErr error - if util.IsGitProviderRepo(url) { - gitUrl, err := util.NewGitURL(url, token) - if err != nil { - return err - } - - if !gitUrl.IsFile || gitUrl.Revision == "" || !strings.Contains(gitUrl.Path, OutputDevfileYamlPath) { - return fmt.Errorf("error getting devfile from url: failed to retrieve %s", url) - } - - stackDir, err := os.MkdirTemp("", fmt.Sprintf("git-resources")) - if err != nil { - return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) - } - - defer func(path string) { - err := os.RemoveAll(path) - if err != nil { - returnedErr = multierror.Append(returnedErr, err) - } - }(stackDir) - - gitUrl.Token = token - - err = gitUrl.CloneGitRepo(stackDir) - if err != nil { - returnedErr = multierror.Append(returnedErr, err) - return returnedErr - } - - dir := path.Dir(path.Join(stackDir, gitUrl.Path)) - err = util.CopyAllDirFiles(dir, destDir) - if err != nil { - returnedErr = multierror.Append(returnedErr, err) - return returnedErr - } - } else { - return fmt.Errorf("Failed to download resources from parent devfile. Unsupported Git Provider for %s ", url) - } - - return nil -} - // GetDeployComponents gets the default deploy command associated components func GetDeployComponents(devfileData data.DevfileData) (map[string]string, error) { deployCommandFilter := common.DevfileOptions{ diff --git a/pkg/devfile/parser/utils_test.go b/pkg/devfile/parser/utils_test.go index 56c7ea58..8ccb9b9f 100644 --- a/pkg/devfile/parser/utils_test.go +++ b/pkg/devfile/parser/utils_test.go @@ -192,11 +192,7 @@ func TestGetDeployComponents(t *testing.T) { mockApplyCommands.Return(nil, fmt.Errorf(*tt.wantMockErr2)) } - devObj := DevfileObj{ - Data: mockDevfileData, - } - - componentMap, err := GetDeployComponents(devObj.Data) + componentMap, err := GetDeployComponents(mockDevfileData) // Unexpected error if (err != nil) != (tt.wantErr != nil) { t.Errorf("TestGetDeployComponents() error: %v, wantErr %v", err, tt.wantErr) @@ -368,11 +364,7 @@ func TestGetImageBuildComponent(t *testing.T) { mockGetComponents.Return(nil, fmt.Errorf(*tt.wantMockErr)) } - devObj := DevfileObj{ - Data: mockDevfileData, - } - - component, err := GetImageBuildComponent(devObj.Data, tt.deployAssociatedComponents) + component, err := GetImageBuildComponent(mockDevfileData, tt.deployAssociatedComponents) // Unexpected error if (err != nil) != (tt.wantErr != nil) { t.Errorf("TestGetImageBuildComponent() error: %v, wantErr %v", err, tt.wantErr) diff --git a/pkg/util/mock.go b/pkg/util/mock.go index bdfc4110..a45833c8 100644 --- a/pkg/util/mock.go +++ b/pkg/util/mock.go @@ -33,6 +33,12 @@ type MockGitUrl struct { IsFile bool // defines if the URL points to a file in the repo } +type MockDownloadOptions struct { + MockDevfile bool + MockDockerfile bool + MockFile string +} + func (m *MockGitUrl) GetToken() string { return m.Token } @@ -130,6 +136,124 @@ func (m *MockGitUrl) CloneGitRepo(destDir string) error { return nil } +var mockDevfile = ` +schemaVersion: 2.2.0 +metadata: + displayName: Go Mock Runtime + icon: https://raw.githubusercontent.com/devfile-samples/devfile-stack-icons/main/golang.svg + language: go + name: go + projectType: go + tags: + - Go + version: 1.0.0 +starterProjects: + - name: go-starter + git: + checkoutFrom: + revision: main + remotes: + origin: https://github.com/devfile-samples/devfile-stack-go.git +components: + - container: + endpoints: + - name: http + targetPort: 8080 + image: golang:latest + memoryLimit: 1024Mi + mountSources: true + sourceMapping: /project + name: runtime + - name: image-build + image: + imageName: go-image:latest + dockerfile: + uri: docker/Dockerfile + buildContext: . + rootRequired: false + - name: kubernetes-deploy + kubernetes: + inlined: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + creationTimestamp: null + labels: + test: test + name: deploy-sample + endpoints: + - name: http-8081 + targetPort: 8081 + path: / +commands: + - exec: + commandLine: GOCACHE=/project/.cache go build main.go + component: runtime + group: + isDefault: true + kind: build + workingDir: /project + id: build + - exec: + commandLine: ./main + component: runtime + group: + isDefault: true + kind: run + workingDir: /project + id: run + - id: build-image + apply: + component: image-build + - id: deployk8s + apply: + component: kubernetes-deploy + - id: deploy + composite: + commands: + - build-image + - deployk8s + group: + kind: deploy + isDefault: true +` + +var mockDockerfile = ` +FROM python:slim + +WORKDIR /projects + +RUN python3 -m venv venv +RUN . venv/bin/activate + +# optimize image caching +COPY requirements.txt . +RUN pip install -r requirements.txt + +COPY . . + +EXPOSE 8081 +CMD [ "waitress-serve", "--port=8081", "app:app"] +` + +func (m MockGitUrl) DownloadInMemoryWithClient(params HTTPRequestParams, httpClient HTTPClient, options MockDownloadOptions) ([]byte, error) { + + if m.GetToken() == "valid-token" || m.GetToken() == "" { + switch { + case options.MockDevfile: + return []byte(mockDevfile), nil + case options.MockDockerfile: + return []byte(mockDockerfile), nil + case len(options.MockFile) > 0: + return []byte(options.MockFile), nil + default: + return []byte(mockDevfile), nil + } + } + + return nil, fmt.Errorf("failed to retrieve %s", params.URL) +} + func (m *MockGitUrl) SetToken(token string) error { m.Token = token return nil diff --git a/pkg/util/util.go b/pkg/util/util.go index 6ce8a6d6..e4c2eda6 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -21,10 +21,6 @@ import ( "bytes" "crypto/rand" "fmt" - gitpkg "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/gregjones/httpcache" - "github.com/gregjones/httpcache/diskcache" "io" "io/ioutil" "math/big" @@ -45,6 +41,11 @@ import ( "syscall" "time" + gitpkg "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/gregjones/httpcache" + "github.com/gregjones/httpcache/diskcache" + "github.com/devfile/library/v2/pkg/testingutil/filesystem" "github.com/fatih/color" "github.com/gobwas/glob" @@ -1093,26 +1094,28 @@ func DownloadFileInMemory(url string) ([]byte, error) { return ioutil.ReadAll(resp.Body) } -// DownloadInMemory uses HTTPRequestParams to download the file and return bytes +// DownloadInMemory uses HTTPRequestParams to download the file and return bytes. +// Use the pkg/devfile/parser/utils.go DownloadInMemory() invocation if you want to +// call with a client instead. func DownloadInMemory(params HTTPRequestParams) ([]byte, error) { var httpClient = &http.Client{Transport: &http.Transport{ ResponseHeaderTimeout: HTTPRequestResponseTimeout, }, Timeout: HTTPRequestResponseTimeout} - var g GitUrl + var g *GitUrl var err error if IsGitProviderRepo(params.URL) { - g, err = NewGitUrlWithURL(params.URL) + g, err = NewGitURL(params.URL, params.Token) if err != nil { return nil, errors.Errorf("failed to parse git repo. error: %v", err) } } - return downloadInMemoryWithClient(params, httpClient, g) + return g.downloadInMemoryWithClient(params, httpClient) } -func downloadInMemoryWithClient(params HTTPRequestParams, httpClient HTTPClient, g GitUrl) ([]byte, error) { +func (g *GitUrl) downloadInMemoryWithClient(params HTTPRequestParams, httpClient HTTPClient) ([]byte, error) { var url string url = params.URL req, err := http.NewRequest("GET", url, nil) From ab4c336ba882ccf675f690ed5435b2a236cdb9dc Mon Sep 17 00:00:00 2001 From: Maysun J Faisal Date: Mon, 20 Nov 2023 16:40:38 -0500 Subject: [PATCH 2/5] Add tests and token support for YAML parser from URL Signed-off-by: Maysun J Faisal --- .codecov.yaml | 2 + pkg/devfile/parser/reader.go | 16 +++- pkg/devfile/parser/reader_test.go | 91 ++++++++++++++++--- pkg/devfile/parser/util/utils_test.go | 123 ++++++++++++++++++++++++++ 4 files changed, 214 insertions(+), 18 deletions(-) create mode 100644 pkg/devfile/parser/util/utils_test.go diff --git a/.codecov.yaml b/.codecov.yaml index 8739c455..4bff7045 100644 --- a/.codecov.yaml +++ b/.codecov.yaml @@ -29,6 +29,8 @@ coverage: - "vendor/*" - "Makefile" - ".travis.yml" + - "pkg/devfile/parser/util/mock.go" + - "pkg/util/mock.go" # See http://docs.codecov.io/docs/pull-request-comments-1 comment: diff --git a/pkg/devfile/parser/reader.go b/pkg/devfile/parser/reader.go index 9728ad8c..e974ed26 100644 --- a/pkg/devfile/parser/reader.go +++ b/pkg/devfile/parser/reader.go @@ -20,6 +20,7 @@ import ( "fmt" "io" + parserUtil "github.com/devfile/library/v2/pkg/devfile/parser/util" "github.com/devfile/library/v2/pkg/util" "github.com/pkg/errors" "github.com/spf13/afero" @@ -38,6 +39,8 @@ type YamlSrc struct { Path string // URL of the yaml file URL string + // Token to access a private URL like a private repository + Token string // Data is the yaml content in []byte format Data []byte } @@ -56,15 +59,20 @@ type KubernetesResources struct { // It returns all the parsed Kubernetes objects as an array of interface. // Consumers interested in the Kubernetes resources are expected to Unmarshal // it to the struct of the respective Kubernetes resource. If a Path is being passed, -// provide a filesystem, otherwise nil can be passed in -func ReadKubernetesYaml(src YamlSrc, fs *afero.Afero) ([]interface{}, error) { +// provide a filesystem, otherwise nil can be passed in. +// Pass in an optional client to use either the actual implementation or a mock implementation of the interface. +func ReadKubernetesYaml(src YamlSrc, fs *afero.Afero, devfileUtilsClient parserUtil.DevfileUtils) ([]interface{}, error) { var data []byte var err error if src.URL != "" { - params := util.HTTPRequestParams{URL: src.URL} - data, err = util.DownloadInMemory(params) // TODO: call with Token + if devfileUtilsClient == nil { + devfileUtilsClient = parserUtil.NewDevfileUtilsClient() + } + + params := util.HTTPRequestParams{URL: src.URL, Token: src.Token} + data, err = devfileUtilsClient.DownloadInMemory(params) if err != nil { return nil, errors.Wrapf(err, "failed to download file %q", src.URL) } diff --git a/pkg/devfile/parser/reader_test.go b/pkg/devfile/parser/reader_test.go index 508b0e58..2000154d 100644 --- a/pkg/devfile/parser/reader_test.go +++ b/pkg/devfile/parser/reader_test.go @@ -22,6 +22,7 @@ import ( "reflect" "testing" + parserUtil "github.com/devfile/library/v2/pkg/devfile/parser/util" "github.com/devfile/library/v2/pkg/util" "github.com/spf13/afero" "github.com/stretchr/testify/assert" @@ -68,10 +69,13 @@ func TestReadAndParseKubernetesYaml(t *testing.T) { badData := append(data, 59) + devfileUtilsClient := parserUtil.NewDevfileUtilsClient() + tests := []struct { name string src YamlSrc fs *afero.Afero + devfileUtilsClient parserUtil.DevfileUtils testParseYamlOnly bool wantErr bool wantParserErr bool @@ -87,6 +91,20 @@ func TestReadAndParseKubernetesYaml(t *testing.T) { URL: "http://" + serverIP, }, fs: nil, + devfileUtilsClient: devfileUtilsClient, + wantDeploymentNames: []string{"deploy-sample", "deploy-sample-2"}, + wantServiceNames: []string{"service-sample", "service-sample-2"}, + wantRouteNames: []string{"route-sample", "route-sample-2"}, + wantIngressNames: []string{"ingress-sample", "ingress-sample-2"}, + wantOtherNames: []string{"pvc-sample", "pvc-sample-2"}, + }, + { + name: "Read the YAML from the URL with no devfile utility client", + src: YamlSrc{ + URL: "http://" + serverIP, + }, + fs: nil, + devfileUtilsClient: nil, wantDeploymentNames: []string{"deploy-sample", "deploy-sample-2"}, wantServiceNames: []string{"service-sample", "service-sample-2"}, wantRouteNames: []string{"route-sample", "route-sample-2"}, @@ -99,6 +117,7 @@ func TestReadAndParseKubernetesYaml(t *testing.T) { Path: "../../../tests/yamls/resources.yaml", }, fs: &fs, + devfileUtilsClient: devfileUtilsClient, wantDeploymentNames: []string{"deploy-sample", "deploy-sample-2"}, wantServiceNames: []string{"service-sample", "service-sample-2"}, wantRouteNames: []string{"route-sample", "route-sample-2"}, @@ -110,16 +129,18 @@ func TestReadAndParseKubernetesYaml(t *testing.T) { src: YamlSrc{ Path: "../../../tests/yamls/resources.yaml", }, - fs: nil, - wantErr: true, + fs: nil, + devfileUtilsClient: devfileUtilsClient, + wantErr: true, }, { name: "Bad Path", src: YamlSrc{ Path: "$%^&", }, - fs: &fs, - wantErr: true, + fs: &fs, + devfileUtilsClient: devfileUtilsClient, + wantErr: true, }, { name: "Read the YAML from the Data", @@ -127,6 +148,7 @@ func TestReadAndParseKubernetesYaml(t *testing.T) { Data: data, }, fs: nil, + devfileUtilsClient: devfileUtilsClient, wantDeploymentNames: []string{"deploy-sample", "deploy-sample-2"}, wantServiceNames: []string{"service-sample", "service-sample-2"}, wantRouteNames: []string{"route-sample", "route-sample-2"}, @@ -138,39 +160,80 @@ func TestReadAndParseKubernetesYaml(t *testing.T) { src: YamlSrc{ URL: "http://badurl", }, - fs: nil, - wantErr: true, + fs: nil, + devfileUtilsClient: devfileUtilsClient, + wantErr: true, }, { name: "Bad Path", src: YamlSrc{ Path: "$%^&", }, - fs: &fs, - wantErr: true, + fs: &fs, + devfileUtilsClient: devfileUtilsClient, + wantErr: true, }, { name: "Bad Data", src: YamlSrc{ Data: badData, }, - fs: nil, - wantErr: true, + fs: nil, + devfileUtilsClient: devfileUtilsClient, + wantErr: true, }, { name: "Invalid kube yaml Data", src: YamlSrc{ Data: []byte("invalidyaml"), }, - fs: nil, - testParseYamlOnly: true, - wantParserErr: true, + fs: nil, + devfileUtilsClient: devfileUtilsClient, + testParseYamlOnly: true, + wantParserErr: true, + }, + { + name: "Read the YAML from the URL with mock client", + src: YamlSrc{ + URL: "http://" + serverIP, + }, + fs: nil, + devfileUtilsClient: parserUtil.MockDevfileUtilsClient{DownloadOptions: util.MockDownloadOptions{MockFile: string(data)}, MockGitURL: util.MockGitUrl{Host: "http://github.com"}}, + wantDeploymentNames: []string{"deploy-sample", "deploy-sample-2"}, + wantServiceNames: []string{"service-sample", "service-sample-2"}, + wantRouteNames: []string{"route-sample", "route-sample-2"}, + wantIngressNames: []string{"ingress-sample", "ingress-sample-2"}, + wantOtherNames: []string{"pvc-sample", "pvc-sample-2"}, + }, + { + name: "Read the YAML from the URL with mock client and mock token", + src: YamlSrc{ + URL: "http://" + serverIP, + Token: "valid-token", + }, + fs: nil, + devfileUtilsClient: parserUtil.MockDevfileUtilsClient{DownloadOptions: util.MockDownloadOptions{MockFile: string(data)}, MockGitURL: util.MockGitUrl{Host: "http://github.com"}, GitTestToken: "valid-token"}, + wantDeploymentNames: []string{"deploy-sample", "deploy-sample-2"}, + wantServiceNames: []string{"service-sample", "service-sample-2"}, + wantRouteNames: []string{"route-sample", "route-sample-2"}, + wantIngressNames: []string{"ingress-sample", "ingress-sample-2"}, + wantOtherNames: []string{"pvc-sample", "pvc-sample-2"}, + }, + { + name: "Bad token with mock client", + src: YamlSrc{ + URL: "http://badurl", + Token: "invalid-token", + }, + fs: nil, + devfileUtilsClient: parserUtil.MockDevfileUtilsClient{DownloadOptions: util.MockDownloadOptions{MockFile: string(data)}, MockGitURL: util.MockGitUrl{Host: "http://github.com"}, GitTestToken: "invalid-token"}, + wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - values, err := ReadKubernetesYaml(tt.src, tt.fs) + values, err := ReadKubernetesYaml(tt.src, tt.fs, tt.devfileUtilsClient) if (err != nil) != tt.wantErr { t.Errorf("unexpected error: %v, wantErr: %v", err, tt.wantErr) return diff --git a/pkg/devfile/parser/util/utils_test.go b/pkg/devfile/parser/util/utils_test.go new file mode 100644 index 00000000..a2419b09 --- /dev/null +++ b/pkg/devfile/parser/util/utils_test.go @@ -0,0 +1,123 @@ +// +// Copyright 2023 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/devfile/library/v2/pkg/util" + "github.com/kylelemons/godebug/pretty" + "github.com/stretchr/testify/assert" +) + +const ( + RawGitHubHost string = "raw.githubusercontent.com" +) + +func TestDownloadInMemoryClient(t *testing.T) { + const downloadErr = "failed to retrieve %s, 404: Not Found" + // Start a local HTTP server + server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + // Send response to be tested + _, err := rw.Write([]byte("OK")) + if err != nil { + t.Error(err) + } + })) + + // Close the server when test finishes + defer server.Close() + + devfileUtilsClient := NewDevfileUtilsClient() + + tests := []struct { + name string + url string + token string + client DevfileUtils + want []byte + wantErr string + }{ + { + name: "Case 1: Input url is valid", + client: devfileUtilsClient, + url: server.URL, + want: []byte{79, 75}, + }, + { + name: "Case 2: Input url is invalid", + client: devfileUtilsClient, + url: "invalid", + wantErr: "unsupported protocol scheme", + }, + { + name: "Case 3: Git provider with invalid url", + client: devfileUtilsClient, + url: "github.com/mike-hoang/invalid-repo", + token: "", + want: []byte(nil), + wantErr: "failed to parse git repo. error:*", + }, + { + name: "Case 4: Public Github repo with missing blob", + client: devfileUtilsClient, + url: "https://github.com/devfile/library/main/README.md", + wantErr: "failed to parse git repo. error: url path to directory or file should contain 'tree' or 'blob'*", + }, + { + name: "Case 5: Public Github repo, with invalid token ", + client: devfileUtilsClient, + url: "https://github.com/devfile/library/blob/main/devfile.yaml", + token: "fake-token", + wantErr: fmt.Sprintf(downloadErr, "https://"+RawGitHubHost+"/devfile/library/main/devfile.yaml"), + }, + { + name: "Case 6: Input url is valid with a mock client", + client: MockDevfileUtilsClient{MockGitURL: util.MockGitUrl{Host: "https://github.com/devfile/library/blob/main/devfile.yaml"}, DownloadOptions: util.MockDownloadOptions{MockFile: "OK"}}, + url: "https://github.com/devfile/library/blob/main/devfile.yaml", + want: []byte{79, 75}, + }, + { + name: "Case 7: Input url is valid with a mock client and mock token", + client: MockDevfileUtilsClient{MockGitURL: util.MockGitUrl{Host: "https://github.com/devfile/library/blob/main/devfile.yaml"}, GitTestToken: "valid-token", DownloadOptions: util.MockDownloadOptions{MockFile: "OK"}}, + url: "https://github.com/devfile/library/blob/main/devfile.yaml", + want: []byte{79, 75}, + }, + { + name: "Case 8: Public Github repo, with invalid token ", + client: MockDevfileUtilsClient{MockGitURL: util.MockGitUrl{Host: "https://github.com/devfile/library/blob/main/devfile.yaml"}, GitTestToken: "invalid-token"}, + url: "https://github.com/devfile/library/blob/main/devfile.yaml", + wantErr: "failed to retrieve https://github.com/devfile/library/blob/main/devfile.yaml", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := tt.client.DownloadInMemory(util.HTTPRequestParams{URL: tt.url, Token: tt.token}) + if (err != nil) != (tt.wantErr != "") { + t.Errorf("Failed to download file with error: %s", err) + } else if err == nil && !reflect.DeepEqual(data, tt.want) { + t.Errorf("Expected: %v, received: %v, difference at %v", tt.want, string(data[:]), pretty.Compare(tt.want, data)) + } else if err != nil { + assert.Regexp(t, tt.wantErr, err.Error(), "Error message should match") + } + }) + } +} From f5ed8c8aeecdfa240c535e3392845659dd0dc69a Mon Sep 17 00:00:00 2001 From: Maysun J Faisal Date: Mon, 20 Nov 2023 16:58:31 -0500 Subject: [PATCH 3/5] Update README and document how to use devfile utils client Signed-off-by: Maysun J Faisal --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index 6ab17c23..48bcb7df 100644 --- a/README.md +++ b/README.md @@ -226,6 +226,29 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g } ``` +11. To download/access files from a private repository like a private GitHub use the `Token` property + ```go + parserArgs := parser.ParserArgs{ + Token: "my-PAT", + } + ``` + + ```go + src: YamlSrc{ + URL: "http://github.com/my-private-repo", + Token: "my-PAT", + } + values, err := ReadKubernetesYaml(src, fs, nil) + ``` + + If you would like to use the mock implementation for the `DevfileUtils` interface method defined in `pkg/devfile/parser/util/interface.go`, then use + ```go + var devfileUtilsClient DevfileUtils + devfileUtilsClient = NewMockDevfileUtilsClient() + devfileUtilsClient.DownloadInMemory(params) + ``` + + ## Projects using devfile/library The following projects are consuming this library as a Golang dependency From f978ca4c4396fb8a8faf5be9e9b324f62d1836b1 Mon Sep 17 00:00:00 2001 From: Maysun J Faisal Date: Mon, 20 Nov 2023 17:31:01 -0500 Subject: [PATCH 4/5] Update Test log Signed-off-by: Maysun J Faisal --- pkg/devfile/parse_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/devfile/parse_test.go b/pkg/devfile/parse_test.go index 3213518a..68daae17 100644 --- a/pkg/devfile/parse_test.go +++ b/pkg/devfile/parse_test.go @@ -230,8 +230,7 @@ spec: type: LoadBalancer ` uri := "127.0.0.1:8080" - var testServer *httptest.Server - testServer = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + testServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var err error if strings.Contains(r.URL.Path, "/outerloop-deploy.yaml") { _, err = w.Write([]byte(outerloopDeployContent)) @@ -252,7 +251,7 @@ spec: // create a listener with the desired port. l, err := net.Listen("tcp", uri) if err != nil { - t.Errorf("Test_parseParentAndPluginFromURI() unexpected error while creating listener: %v", err) + t.Errorf("TestParseDevfileAndValidate() unexpected error while creating listener: %v", err) } // NewUnstartedServer creates a listener. Close that listener and replace From 59df52404a08ffd14e5626155043075f4a64134a Mon Sep 17 00:00:00 2001 From: Maysun J Faisal Date: Mon, 27 Nov 2023 11:30:39 -0500 Subject: [PATCH 5/5] Format README Signed-off-by: Maysun J Faisal --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 48bcb7df..a7e9b9a9 100644 --- a/README.md +++ b/README.md @@ -235,13 +235,13 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g ```go src: YamlSrc{ - URL: "http://github.com/my-private-repo", - Token: "my-PAT", - } + URL: "http://github.com/my-private-repo", + Token: "my-PAT", + } values, err := ReadKubernetesYaml(src, fs, nil) ``` - If you would like to use the mock implementation for the `DevfileUtils` interface method defined in `pkg/devfile/parser/util/interface.go`, then use + If you would like to use the mock implementation for the `DevfileUtils` interface method defined in [pkg/devfile/parser/util/interface.go](pkg/devfile/parser/util/interface.go), then use ```go var devfileUtilsClient DevfileUtils devfileUtilsClient = NewMockDevfileUtilsClient()