From f67db1b477d9cd1a4637d5101e011dac2ec0fcaa Mon Sep 17 00:00:00 2001 From: Akash Kumar <91385321+AkashKumar7902@users.noreply.github.com> Date: Sun, 18 Feb 2024 08:33:34 +0530 Subject: [PATCH] feat: run remote repo in kpm run (#251) * feat: run remote repo in kpm run Signed-off-by: Akash Kumar * address review comments Signed-off-by: Akash Kumar * test: Add unit test Signed-off-by: Akash Kumar --------- Signed-off-by: Akash Kumar --- pkg/client/client.go | 27 +++++++++++++++++++++++++++ pkg/client/client_test.go | 24 ++++++++++++++++++++++++ pkg/cmd/cmd_run.go | 9 +++++++-- pkg/constants/constants.go | 1 + pkg/git/git.go | 11 +++++++++++ pkg/git/git_test.go | 10 ++++++++++ pkg/runner/entry.go | 6 ++++++ pkg/runner/entry_test.go | 1 + pkg/utils/utils.go | 7 +++++++ pkg/utils/utils_test.go | 34 ++++++++++++++++++++++++++++++++++ 10 files changed, 128 insertions(+), 2 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index 62833d0c..1e94cdad 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -439,6 +439,33 @@ func (c *KpmClient) CompileTarPkg(tarPath string, opts *opt.CompileOptions) (*kc return c.CompileWithOpts(opts) } +// CompileGitPkg will compile the kcl package from the git url. +func (c *KpmClient) CompileGitPkg(gitOpts *git.CloneOptions, compileOpts *opt.CompileOptions) (*kcl.KCLResultList, error) { + // 1. Create the temporary directory to pull the tar. + tmpDir, err := os.MkdirTemp("", "") + if err != nil { + return nil, reporter.NewErrorEvent(reporter.Bug, err, "internal bugs, please contact us to fix it.") + } + // clean the temp dir. + defer os.RemoveAll(tmpDir) + + // 2. clone the git repo + _, err = git.CloneWithOpts( + git.WithCommit(gitOpts.Commit), + git.WithBranch(gitOpts.Branch), + git.WithTag(gitOpts.Tag), + git.WithRepoURL(gitOpts.RepoURL), + git.WithLocalPath(tmpDir), + ) + if err != nil { + return nil, reporter.NewErrorEvent(reporter.FailedGetPkg, err, "failed to get the git repository") + } + + compileOpts.SetPkgPath(tmpDir) + + return c.CompileWithOpts(compileOpts) +} + // CompileOciPkg will compile the kcl package from the OCI reference or url. func (c *KpmClient) CompileOciPkg(ociSource, version string, opts *opt.CompileOptions) (*kcl.KCLResultList, error) { ociOpts, err := c.ParseOciOptionFromString(ociSource, version) diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index c741e185..2950108c 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -15,6 +15,7 @@ import ( "github.com/otiai10/copy" "github.com/stretchr/testify/assert" "kcl-lang.io/kpm/pkg/env" + "kcl-lang.io/kpm/pkg/git" "kcl-lang.io/kpm/pkg/opt" pkg "kcl-lang.io/kpm/pkg/package" "kcl-lang.io/kpm/pkg/runner" @@ -844,6 +845,29 @@ func TestRunWithNoSumCheck(t *testing.T) { }() } +func TestRemoteRun(t *testing.T) { + kpmcli, err := NewKpmClient() + assert.Equal(t, err, nil) + + opts := opt.DefaultCompileOptions() + gitOpts := git.NewCloneOptions("https://github.com/KusionStack/catalog", "", "0.1.2", "", "", nil) + + opts.SetEntries([]string{"models/samples/hellocollaset/prod/main.k"}) + result, err := kpmcli.CompileGitPkg(gitOpts, opts) + assert.Equal(t, err, nil) + assert.Equal(t, result.GetRawJsonResult(), "[{\"hellocollaset\": {\"workload\": {\"containers\": {\"nginx\": {\"image\": \"nginx:v2\"}}}}}]") + + opts.SetEntries([]string{"models/samples/pgadmin/base/base.k"}) + result, err = kpmcli.CompileGitPkg(gitOpts, opts) + assert.Equal(t, err, nil) + assert.Equal(t, result.GetRawJsonResult(), "[{\"pgadmin\": {\"workload\": {\"containers\": {\"pgadmin\": {\"image\": \"dpage/pgadmin4:latest\", \"env\": {\"PGADMIN_DEFAULT_EMAIL\": \"admin@admin.com\", \"PGADMIN_DEFAULT_PASSWORD\": \"secret://pgadmin-secret/pgadmin-default-password\", \"PGADMIN_PORT\": \"80\"}, \"resources\": {\"cpu\": \"500m\", \"memory\": \"512Mi\"}}}, \"secrets\": {\"pgadmin-secret\": {\"type\": \"opaque\", \"data\": {\"pgadmin-default-password\": \"*******\"}}}, \"replicas\": 1, \"ports\": [{\"port\": 80, \"protocol\": \"TCP\", \"public\": false}]}, \"database\": {\"pgadmin\": {\"type\": \"cloud\", \"version\": \"14.0\"}}}}]") + + opts.SetEntries([]string{"models/samples/wordpress/prod/main.k"}) + result, err = kpmcli.CompileGitPkg(gitOpts, opts) + assert.Equal(t, err, nil) + assert.Equal(t, result.GetRawJsonResult(), "[{\"wordpress\": {\"workload\": {\"containers\": {\"wordpress\": {\"image\": \"wordpress:6.3\", \"env\": {\"WORDPRESS_DB_HOST\": \"$(KUSION_DB_HOST_WORDPRESS)\", \"WORDPRESS_DB_USER\": \"$(KUSION_DB_USERNAME_WORDPRESS)\", \"WORDPRESS_DB_PASSWORD\": \"$(KUSION_DB_PASSWORD_WORDPRESS)\", \"WORDPRESS_DB_NAME\": \"mysql\"}, \"resources\": {\"cpu\": \"500m\", \"memory\": \"512Mi\"}}}, \"replicas\": 1, \"ports\": [{\"port\": 80, \"protocol\": \"TCP\", \"public\": false}]}, \"database\": {\"wordpress\": {\"type\": \"cloud\", \"version\": \"8.0\"}}}}]") +} + func TestUpdateWithNoSumCheck(t *testing.T) { pkgPath := getTestDir("test_update_no_sum_check") kpmcli, err := NewKpmClient() diff --git a/pkg/cmd/cmd_run.go b/pkg/cmd/cmd_run.go index bd621030..2ad84567 100644 --- a/pkg/cmd/cmd_run.go +++ b/pkg/cmd/cmd_run.go @@ -12,6 +12,7 @@ import ( "kcl-lang.io/kcl-go/pkg/kcl" "kcl-lang.io/kpm/pkg/api" "kcl-lang.io/kpm/pkg/client" + "kcl-lang.io/kpm/pkg/git" "kcl-lang.io/kpm/pkg/opt" "kcl-lang.io/kpm/pkg/reporter" "kcl-lang.io/kpm/pkg/runner" @@ -110,7 +111,7 @@ func KpmRun(c *cli.Context, kpmcli *client.KpmClient) error { return errEvent } - // 'kpm run' compile the current package undor '$pwd'. + // 'kpm run' compile the current package under '$pwd'. if runEntry.IsEmpty() { pwd, err := os.Getwd() kclOpts.SetPkgPath(pwd) @@ -140,8 +141,12 @@ func KpmRun(c *cli.Context, kpmcli *client.KpmClient) error { compileResult, err = kpmcli.CompileWithOpts(kclOpts) } } else if runEntry.IsTar() { - // 'kpm run' compile the package from the kcl pakcage tar. + // 'kpm run' compile the package from the kcl package tar. compileResult, err = kpmcli.CompileTarPkg(runEntry.PackageSource(), kclOpts) + } else if runEntry.IsGit() { + gitOpts := git.NewCloneOptions(runEntry.PackageSource(), "", c.String(FLAG_TAG), "", "", nil) + // 'kpm run' compile the package from the git url + compileResult, err = kpmcli.CompileGitPkg(gitOpts, kclOpts) } else { // 'kpm run' compile the package from the OCI reference or url. compileResult, err = kpmcli.CompileOciPkg(runEntry.PackageSource(), c.String(FLAG_TAG), kclOpts) diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 621e5ea1..a9b3faaa 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -9,6 +9,7 @@ const ( UrlEntry = "url" RefEntry = "ref" TarEntry = "tar" + GitEntry = "git" KCL_MOD = "kcl.mod" OCI_SEPARATOR = ":" KCL_PKG_TAR = "*.tar" diff --git a/pkg/git/git.go b/pkg/git/git.go index d7483583..0ae258f2 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -24,6 +24,17 @@ type CloneOptions struct { // CloneOption is a function that modifies CloneOptions type CloneOption func(*CloneOptions) +func NewCloneOptions(repoUrl, commit, tag, branch, localpath string, Writer io.Writer) *CloneOptions { + return &CloneOptions{ + RepoURL: repoUrl, + Commit: commit, + Tag: tag, + Branch: branch, + LocalPath: localpath, + Writer: Writer, + } +} + // WithRepoURL sets the repo URL for CloneOptions func WithRepoURL(repoURL string) CloneOption { return func(o *CloneOptions) { diff --git a/pkg/git/git_test.go b/pkg/git/git_test.go index a82c04c4..41adcaad 100644 --- a/pkg/git/git_test.go +++ b/pkg/git/git_test.go @@ -24,6 +24,16 @@ func TestWithGitOptions(t *testing.T) { assert.Equal(t, cloneOpts.Writer, nil) } +func TestNewCloneOptions(t *testing.T) { + cloneOpts := NewCloneOptions("https://github.com/kcl-lang/kcl", "", "v1.0.0", "", "", nil) + assert.Equal(t, cloneOpts.RepoURL, "https://github.com/kcl-lang/kcl") + assert.Equal(t, cloneOpts.Tag, "v1.0.0") + assert.Equal(t, cloneOpts.Commit, "") + assert.Equal(t, cloneOpts.Branch, "") + assert.Equal(t, cloneOpts.LocalPath, "") + assert.Equal(t, cloneOpts.Writer, nil) +} + func TestValidateGitOptions(t *testing.T) { cloneOpts := &CloneOptions{} WithBranch("test_branch")(cloneOpts) diff --git a/pkg/runner/entry.go b/pkg/runner/entry.go index ea69eeef..64686a21 100644 --- a/pkg/runner/entry.go +++ b/pkg/runner/entry.go @@ -67,6 +67,10 @@ func (e *Entry) IsTar() bool { return e.kind == constants.TarEntry } +func (e *Entry) IsGit() bool { + return e.kind == constants.GitEntry +} + // IsEmpty will return true if the entry is empty. func (e *Entry) IsEmpty() bool { return len(e.packageSource) == 0 @@ -166,6 +170,8 @@ func GetSourceKindFrom(source string) EntryKind { return constants.FileEntry } else if utils.IsTar(source) { return constants.TarEntry + } else if utils.IsGitRepoUrl(source) { + return constants.GitEntry } else if utils.IsURL(source) { return constants.UrlEntry } else if utils.IsRef(source) { diff --git a/pkg/runner/entry_test.go b/pkg/runner/entry_test.go index 2548668a..7362fc9c 100644 --- a/pkg/runner/entry_test.go +++ b/pkg/runner/entry_test.go @@ -44,6 +44,7 @@ func TestGetSourceKindFrom(t *testing.T) { assert.Equal(t, string(GetSourceKindFrom("./testdata_external/external/main.k")), constants.FileEntry) assert.Equal(t, string(GetSourceKindFrom("main.tar")), constants.TarEntry) assert.Equal(t, string(GetSourceKindFrom("oci://test_url")), constants.UrlEntry) + assert.Equal(t, string(GetSourceKindFrom("https://github.com/test_org/test")), constants.GitEntry) assert.Equal(t, string(GetSourceKindFrom("test_ref:0.0.1")), constants.RefEntry) assert.Equal(t, string(GetSourceKindFrom("invalid input")), "") } diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index da4f316e..083fba28 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -11,6 +11,7 @@ import ( "net/url" "os" "path/filepath" + "regexp" "strings" goerrors "errors" @@ -378,6 +379,12 @@ func IsURL(str string) bool { return err == nil && u.Scheme != "" && u.Host != "" } +// IsGitRepoUrl will check whether the string 'str' is a git repo url +func IsGitRepoUrl(str string) bool { + r := regexp.MustCompile(`((git|ssh|http(s)?)|(git@[\w\.]+))(:(//)?)([\w\.@\:/\-~]+)(\.git)?(/)?`) + return r.MatchString(str) +} + // IsRef will check whether the string 'str' is a reference. func IsRef(str string) bool { _, err := reference.ParseNormalizedNamed(str) diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go index d88d63cb..019827b7 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/utils_test.go @@ -138,6 +138,40 @@ func TestIsUrl(t *testing.T) { assert.Equal(t, IsURL("https://"), false) } +func TestIsGitRepoUrl(t *testing.T) { + assert.Equal(t, IsGitRepoUrl("invalid url"), false) + assert.Equal(t, IsGitRepoUrl("ftp://github.com/user/project.git"), false) + assert.Equal(t, IsGitRepoUrl("file:///path/to/repo.git/"), false) + assert.Equal(t, IsGitRepoUrl("file://~/path/to/repo.git/"), false) + assert.Equal(t, IsGitRepoUrl("path/to/repo.git/"), false) + assert.Equal(t, IsGitRepoUrl("~/path/to/repo.git"), false) + assert.Equal(t, IsGitRepoUrl("rsync://host.xz/path/to/repo.git/"), false) + assert.Equal(t, IsGitRepoUrl("host.xz:path/to/repo.git"), false) + assert.Equal(t, IsGitRepoUrl("user@host.xz:path/to/repo.git"), false) + assert.Equal(t, IsGitRepoUrl("C:\\path\\to\\repo.git"), false) + assert.Equal(t, IsGitRepoUrl("/path/to/repo.git"), false) + assert.Equal(t, IsGitRepoUrl("./path/to/repo.git"), false) + assert.Equal(t, IsGitRepoUrl("oci://host.xz/path/to/repo.git/"), false) + assert.Equal(t, IsGitRepoUrl("https://github.com/user/project"), true) + assert.Equal(t, IsGitRepoUrl("git@github.com:user/project.git"), true) + assert.Equal(t, IsGitRepoUrl("https://github.com/user/project.git"), true) + assert.Equal(t, IsGitRepoUrl("https://github.com/user/project.git"), true) + assert.Equal(t, IsGitRepoUrl("git@192.168.101.127:user/project.git"), true) + assert.Equal(t, IsGitRepoUrl("https://192.168.101.127/user/project.git"), true) + assert.Equal(t, IsGitRepoUrl("http://192.168.101.127/user/project.git"), true) + assert.Equal(t, IsGitRepoUrl("ssh://user@host.xz:port/path/to/repo.git/"), true) + assert.Equal(t, IsGitRepoUrl("ssh://user@host.xz/path/to/repo.git/"), true) + assert.Equal(t, IsGitRepoUrl("ssh://host.xz:port/path/to/repo.git/"), true) + assert.Equal(t, IsGitRepoUrl("ssh://host.xz/path/to/repo.git/"), true) + assert.Equal(t, IsGitRepoUrl("ssh://user@host.xz/path/to/repo.git/"), true) + assert.Equal(t, IsGitRepoUrl("ssh://user@host.xz/~user/path/to/repo.git/"), true) + assert.Equal(t, IsGitRepoUrl("ssh://host.xz/~user/path/to/repo.git/"), true) + assert.Equal(t, IsGitRepoUrl("ssh://user@host.xz/~/path/to/repo.git"), true) + assert.Equal(t, IsGitRepoUrl("git://host.xz/path/to/repo.git/"), true) + assert.Equal(t, IsGitRepoUrl("http://host.xz/path/to/repo.git/"), true) + assert.Equal(t, IsGitRepoUrl("https://host.xz/path/to/repo.git/"), true) +} + func TestIsRef(t *testing.T) { assert.Equal(t, IsRef("invalid ref"), false) assert.Equal(t, IsRef("ghcr.io/xxx/xxx"), true)