From d13144d7247336b23fbecc444c29a23b4abadd19 Mon Sep 17 00:00:00 2001 From: zongzhe Date: Wed, 6 Nov 2024 10:04:27 +0800 Subject: [PATCH 1/5] fix: add missing ModSpec Signed-off-by: zongz --- pkg/client/pull.go | 13 ++++++++++++- pkg/client/run.go | 11 +++++++++++ pkg/downloader/source.go | 2 +- pkg/visitor/visitor.go | 19 ++++++++++++++++++- 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/pkg/client/pull.go b/pkg/client/pull.go index c54cd0bd..78dbf9b7 100644 --- a/pkg/client/pull.go +++ b/pkg/client/pull.go @@ -3,9 +3,11 @@ package client import ( "errors" "fmt" + "net/url" "os" "path/filepath" + "kcl-lang.io/kpm/pkg/constants" "kcl-lang.io/kpm/pkg/downloader" pkg "kcl-lang.io/kpm/pkg/package" "kcl-lang.io/kpm/pkg/reporter" @@ -76,8 +78,17 @@ func (c *KpmClient) Pull(options ...PullOption) (*pkg.KclPkg, error) { if err != nil { return nil, err } + + sourceUrl, err := url.Parse(sourceStr) + if err != nil { + return nil, err + } + queryParams := sourceUrl.Query() + queryParams.Del(constants.Mod) + sourceUrl.RawQuery = queryParams.Encode() + reporter.ReportMsgTo( - fmt.Sprintf("start to pull %s", sourceStr), + fmt.Sprintf("start to pull %s", sourceUrl.String()), c.GetLogWriter(), ) diff --git a/pkg/client/run.go b/pkg/client/run.go index e7352db8..1b59c32c 100644 --- a/pkg/client/run.go +++ b/pkg/client/run.go @@ -401,6 +401,14 @@ func (o *RunOptions) applyCompileOptions(source downloader.Source, kclPkg *pkg.K sourcePath = filepath.Join(workDir, sourcePath) } + var err error + if pkgSource.ModSpec != nil && pkgSource.ModSpec.Name != "" { + sourcePath, err = utils.FindPackage(sourcePath, pkgSource.ModSpec.Name) + if err != nil { + return false + } + } + pkgHome := kclPkg.HomePath if !filepath.IsAbs(pkgHome) && !utils.IsModRelativePath(sourcePath) { @@ -479,6 +487,7 @@ func (o *RunOptions) getPkgSource() (*downloader.Source, error) { workDir := o.WorkDir var pkgSource *downloader.Source + var modSpec *downloader.ModSpec if len(o.Sources) == 0 { workDir, err := filepath.Abs(workDir) if err != nil { @@ -497,6 +506,7 @@ func (o *RunOptions) getPkgSource() (*downloader.Source, error) { for _, source := range o.Sources { if pkgSource == nil { pkgSource = source + modSpec = source.ModSpec rootPath, err = source.FindRootPath() if err != nil { return nil, err @@ -548,6 +558,7 @@ func (o *RunOptions) getPkgSource() (*downloader.Source, error) { if err != nil { return nil, err } + pkgSource.ModSpec = modSpec } } diff --git a/pkg/downloader/source.go b/pkg/downloader/source.go index c8696799..3d40b397 100644 --- a/pkg/downloader/source.go +++ b/pkg/downloader/source.go @@ -443,7 +443,7 @@ func (source *Source) FromString(sourceStr string) error { } else if sourceUrl.Scheme == constants.DefaultOciScheme { source.ModSpec = &ModSpec{} source.ModSpec.FromString(sourceUrl.String()) - } else { + } else if sourceUrl.Path != "" { source.Local = &Local{} source.Local.FromString(sourceUrl.String()) } diff --git a/pkg/visitor/visitor.go b/pkg/visitor/visitor.go index be686f6f..e7b4ebb0 100644 --- a/pkg/visitor/visitor.go +++ b/pkg/visitor/visitor.go @@ -41,7 +41,7 @@ func (pv *PkgVisitor) Visit(s *downloader.Source, v visitFunc) error { return err } - if !s.ModSpec.IsNil() { + if s.ModSpec != nil && s.ModSpec.Name != "" { modPath, err = utils.FindPackage(modPath, s.ModSpec.Name) if err != nil { return err @@ -120,6 +120,14 @@ func (rv *RemoteVisitor) Visit(s *downloader.Source, v visitFunc) error { return fmt.Errorf("source is not remote") } + if s.SpecOnly() { + s.Oci = &downloader.Oci{ + Reg: rv.Settings.DefaultOciRegistry(), + Repo: utils.JoinPath(rv.Settings.DefaultOciRepo(), s.ModSpec.Name), + Tag: s.ModSpec.Version, + } + } + var cacheFullPath string var modFullPath string @@ -134,6 +142,15 @@ func (rv *RemoteVisitor) Visit(s *downloader.Source, v visitFunc) error { } } + if !rv.EnableCache { + cacheFullPath, err = os.MkdirTemp("", "") + if err != nil { + return err + } + + defer os.RemoveAll(cacheFullPath) + } + // 1. Load the credential file. credCli, err := downloader.LoadCredentialFile(rv.Settings.CredentialsFile) if err != nil { From 5f3251d7593f29009867390485fd067f9296856d Mon Sep 17 00:00:00 2001 From: zongz Date: Wed, 6 Nov 2024 15:32:18 +0800 Subject: [PATCH 2/5] feat: add some comments Signed-off-by: zongz --- pkg/client/pull.go | 2 ++ pkg/client/run.go | 4 ++++ pkg/downloader/source.go | 1 + pkg/visitor/visitor.go | 3 +++ 4 files changed, 10 insertions(+) diff --git a/pkg/client/pull.go b/pkg/client/pull.go index 78dbf9b7..979c66e5 100644 --- a/pkg/client/pull.go +++ b/pkg/client/pull.go @@ -79,6 +79,8 @@ func (c *KpmClient) Pull(options ...PullOption) (*pkg.KclPkg, error) { return nil, err } + // Remove information about modspec from message + // to avoid escape symbols ':' in modspec. sourceUrl, err := url.Parse(sourceStr) if err != nil { return nil, err diff --git a/pkg/client/run.go b/pkg/client/run.go index 1b59c32c..a22aa4a9 100644 --- a/pkg/client/run.go +++ b/pkg/client/run.go @@ -401,6 +401,8 @@ func (o *RunOptions) applyCompileOptions(source downloader.Source, kclPkg *pkg.K sourcePath = filepath.Join(workDir, sourcePath) } + // When determining whether the path in 'Source' is the same as homepath of 'KclPkg', + // use the subdirectories specified by 'ModSpec' var err error if pkgSource.ModSpec != nil && pkgSource.ModSpec.Name != "" { sourcePath, err = utils.FindPackage(sourcePath, pkgSource.ModSpec.Name) @@ -506,6 +508,8 @@ func (o *RunOptions) getPkgSource() (*downloader.Source, error) { for _, source := range o.Sources { if pkgSource == nil { pkgSource = source + // If the root source is found, + // update the modSpec and rootPath. modSpec = source.ModSpec rootPath, err = source.FindRootPath() if err != nil { diff --git a/pkg/downloader/source.go b/pkg/downloader/source.go index 3d40b397..0c0bfc2a 100644 --- a/pkg/downloader/source.go +++ b/pkg/downloader/source.go @@ -443,6 +443,7 @@ func (source *Source) FromString(sourceStr string) error { } else if sourceUrl.Scheme == constants.DefaultOciScheme { source.ModSpec = &ModSpec{} source.ModSpec.FromString(sourceUrl.String()) + // There is a case where there is only 'ModSpec' } else if sourceUrl.Path != "" { source.Local = &Local{} source.Local.FromString(sourceUrl.String()) diff --git a/pkg/visitor/visitor.go b/pkg/visitor/visitor.go index e7b4ebb0..4b91fe43 100644 --- a/pkg/visitor/visitor.go +++ b/pkg/visitor/visitor.go @@ -120,6 +120,7 @@ func (rv *RemoteVisitor) Visit(s *downloader.Source, v visitFunc) error { return fmt.Errorf("source is not remote") } + // For some sources with only the spec, the default registry and repo will be used. if s.SpecOnly() { s.Oci = &downloader.Oci{ Reg: rv.Settings.DefaultOciRegistry(), @@ -142,6 +143,8 @@ func (rv *RemoteVisitor) Visit(s *downloader.Source, v visitFunc) error { } } + // If the cache is not enabled, + // create a temporary directory to get the latest commit of git repo if !rv.EnableCache { cacheFullPath, err = os.MkdirTemp("", "") if err != nil { From 636b1e2d2e1f7a0582d463642ca4c2870656521c Mon Sep 17 00:00:00 2001 From: zongz Date: Wed, 6 Nov 2024 15:38:14 +0800 Subject: [PATCH 3/5] fix: supports the default version for git repo Signed-off-by: zongz --- pkg/downloader/source.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/downloader/source.go b/pkg/downloader/source.go index 0c0bfc2a..dd462ca9 100644 --- a/pkg/downloader/source.go +++ b/pkg/downloader/source.go @@ -557,7 +557,8 @@ func (git *Git) GetValidGitReference() (string, error) { nonEmptyRef = git.Branch } - if nonEmptyFields != 1 { + // The latest commit id for git repo is supported. + if nonEmptyFields > 1 { return "", errors.New("only one of branch, tag or commit is allowed") } From 43c507c30d62c075c7ce8a1a95b9bbd978475266 Mon Sep 17 00:00:00 2001 From: zongz Date: Thu, 7 Nov 2024 08:41:39 +0800 Subject: [PATCH 4/5] fix: fix windows path Signed-off-by: zongz --- pkg/downloader/source.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/downloader/source.go b/pkg/downloader/source.go index dd462ca9..d32abb35 100644 --- a/pkg/downloader/source.go +++ b/pkg/downloader/source.go @@ -444,7 +444,9 @@ func (source *Source) FromString(sourceStr string) error { source.ModSpec = &ModSpec{} source.ModSpec.FromString(sourceUrl.String()) // There is a case where there is only 'ModSpec' - } else if sourceUrl.Path != "" { + // On winodws, the path url will be parsed as 'Opaque' + // On linux, the path url will be parsed as 'Path' + } else if sourceUrl.Path != "" || sourceUrl.Opaque != "" { source.Local = &Local{} source.Local.FromString(sourceUrl.String()) } From 657392ded25d82607aa9a92ae45fa98c318dce4e Mon Sep 17 00:00:00 2001 From: zongz Date: Thu, 7 Nov 2024 11:18:47 +0800 Subject: [PATCH 5/5] fix: add 'WithModSpec' for api 'Pull', 'Add' and 'Run' Signed-off-by: zongz --- pkg/client/add.go | 153 +++++++++++++++++++++------------------------ pkg/client/pull.go | 31 +++++---- pkg/client/run.go | 23 +++++++ 3 files changed, 113 insertions(+), 94 deletions(-) diff --git a/pkg/client/add.go b/pkg/client/add.go index 07d913d1..05ad039e 100644 --- a/pkg/client/add.go +++ b/pkg/client/add.go @@ -13,54 +13,46 @@ import ( type AddOptions struct { // Source is the source of the package to be pulled. // Including git, oci, local. - Sources []*downloader.Source - KclPkg *pkg.KclPkg + Source *downloader.Source + KclPkg *pkg.KclPkg } type AddOption func(*AddOptions) error -func WithAddSource(source *downloader.Source) AddOption { +func WithAddModSpec(modSpec *downloader.ModSpec) AddOption { return func(opts *AddOptions) error { - if opts.Sources == nil { - opts.Sources = make([]*downloader.Source, 0) + if modSpec == nil { + return fmt.Errorf("modSpec cannot be nil") } - opts.Sources = append(opts.Sources, source) + if opts.Source == nil { + opts.Source = &downloader.Source{ + ModSpec: modSpec, + } + } else { + opts.Source.ModSpec = modSpec + } + return nil } } -func WithAddSources(sources []*downloader.Source) AddOption { - return func(ro *AddOptions) error { - ro.Sources = sources +func WithAddSource(source *downloader.Source) AddOption { + return func(opts *AddOptions) error { + if source == nil { + return fmt.Errorf("source cannot be nil") + } + opts.Source = source return nil } } func WithAddSourceUrl(sourceUrl string) AddOption { return func(opts *AddOptions) error { - if opts.Sources == nil { - opts.Sources = make([]*downloader.Source, 0) - } source, err := downloader.NewSourceFromStr(sourceUrl) if err != nil { return err } - opts.Sources = append(opts.Sources, source) - return nil - } -} - -func WithAddSourceUrls(sourceUrls []string) AddOption { - return func(opts *AddOptions) error { - var sources []*downloader.Source - for _, sourceUrl := range sourceUrls { - source, err := downloader.NewSourceFromStr(sourceUrl) - if err != nil { - return err - } - sources = append(sources, source) - } - opts.Sources = sources + opts.Source = source return nil } } @@ -88,6 +80,7 @@ func (c *KpmClient) Add(options ...AddOption) error { } } addedPkg := opts.KclPkg + depSource := opts.Source visitorSelector := func(source *downloader.Source) (visitor.Visitor, error) { pkgVisitor := &visitor.PkgVisitor{ @@ -113,73 +106,71 @@ func (c *KpmClient) Add(options ...AddOption) error { } } - for _, depSource := range opts.Sources { - // Set the default OCI registry and repo if the source is nil and the package spec is not nil. - if depSource.IsNilSource() && !depSource.ModSpec.IsNil() { - depSource.Oci = &downloader.Oci{ - Reg: c.GetSettings().Conf.DefaultOciRegistry, - Repo: utils.JoinPath(c.GetSettings().Conf.DefaultOciRepo, depSource.ModSpec.Name), - Tag: depSource.ModSpec.Version, - } + // Set the default OCI registry and repo if the source is nil and the package spec is not nil. + if depSource.IsNilSource() && !depSource.ModSpec.IsNil() { + depSource.Oci = &downloader.Oci{ + Reg: c.GetSettings().Conf.DefaultOciRegistry, + Repo: utils.JoinPath(c.GetSettings().Conf.DefaultOciRepo, depSource.ModSpec.Name), + Tag: depSource.ModSpec.Version, } + } - var fullSouce *downloader.Source - // Transform the relative path to the full path. - if depSource.IsLocalPath() && !filepath.IsAbs(depSource.Path) { - fullSouce = &downloader.Source{ - ModSpec: depSource.ModSpec, - Local: &downloader.Local{ - Path: filepath.Join(addedPkg.HomePath, depSource.Path), - }, - } - } else { - fullSouce = depSource + var fullSouce *downloader.Source + // Transform the relative path to the full path. + if depSource.IsLocalPath() && !filepath.IsAbs(depSource.Path) { + fullSouce = &downloader.Source{ + ModSpec: depSource.ModSpec, + Local: &downloader.Local{ + Path: filepath.Join(addedPkg.HomePath, depSource.Path), + }, } + } else { + fullSouce = depSource + } - visitor, err := visitorSelector(fullSouce) - if err != nil { - return err - } + visitor, err := visitorSelector(fullSouce) + if err != nil { + return err + } - // Visit the dependency source - // If the dependency is remote, the visitor will download it to the local. - // If the dependency is already in local cache, the visitor will not download it again. - err = visitor.Visit(fullSouce, func(depPkg *pkg.KclPkg) error { - var modSpec *downloader.ModSpec - if depSource.ModSpec.IsNil() { - modSpec = &downloader.ModSpec{ - Name: depPkg.ModFile.Pkg.Name, - Version: depPkg.ModFile.Pkg.Version, - } - depSource.ModSpec = modSpec + // Visit the dependency source + // If the dependency is remote, the visitor will download it to the local. + // If the dependency is already in local cache, the visitor will not download it again. + err = visitor.Visit(fullSouce, func(depPkg *pkg.KclPkg) error { + var modSpec *downloader.ModSpec + if depSource.ModSpec.IsNil() { + modSpec = &downloader.ModSpec{ + Name: depPkg.ModFile.Pkg.Name, + Version: depPkg.ModFile.Pkg.Version, } + depSource.ModSpec = modSpec + } - dep := pkg.Dependency{ - Name: depPkg.ModFile.Pkg.Name, - FullName: depPkg.GetPkgFullName(), - Version: depPkg.ModFile.Pkg.Version, - LocalFullPath: depPkg.HomePath, - Source: *depSource, - } + dep := pkg.Dependency{ + Name: depPkg.ModFile.Pkg.Name, + FullName: depPkg.GetPkgFullName(), + Version: depPkg.ModFile.Pkg.Version, + LocalFullPath: depPkg.HomePath, + Source: *depSource, + } - // Add the dependency to the kcl.mod file. - if modExistDep, ok := addedPkg.ModFile.Dependencies.Deps.Get(dep.Name); ok { - if less, err := modExistDep.VersionLessThan(&dep); less && err == nil { - addedPkg.ModFile.Dependencies.Deps.Set(dep.Name, dep) - } - } else { + // Add the dependency to the kcl.mod file. + if modExistDep, ok := addedPkg.ModFile.Dependencies.Deps.Get(dep.Name); ok { + if less, err := modExistDep.VersionLessThan(&dep); less && err == nil { addedPkg.ModFile.Dependencies.Deps.Set(dep.Name, dep) } - - return nil - }) - if err != nil { - return err + } else { + addedPkg.ModFile.Dependencies.Deps.Set(dep.Name, dep) } + + return nil + }) + if err != nil { + return err } // Iterate the dependencies and update the kcl.mod and kcl.mod.lock respectively. - _, err := c.Update( + _, err = c.Update( WithUpdatedKclPkg(addedPkg), ) diff --git a/pkg/client/pull.go b/pkg/client/pull.go index 979c66e5..f580954c 100644 --- a/pkg/client/pull.go +++ b/pkg/client/pull.go @@ -3,11 +3,9 @@ package client import ( "errors" "fmt" - "net/url" "os" "path/filepath" - "kcl-lang.io/kpm/pkg/constants" "kcl-lang.io/kpm/pkg/downloader" pkg "kcl-lang.io/kpm/pkg/package" "kcl-lang.io/kpm/pkg/reporter" @@ -25,6 +23,23 @@ type PullOptions struct { type PullOption func(*PullOptions) error +func WithPullModSpec(modSpec *downloader.ModSpec) PullOption { + return func(opts *PullOptions) error { + if modSpec == nil { + return errors.New("modSpec cannot be nil") + } + if opts.Source == nil { + opts.Source = &downloader.Source{ + ModSpec: modSpec, + } + } else { + opts.Source.ModSpec = modSpec + } + + return nil + } +} + func WithPullSourceUrl(sourceUrl string) PullOption { return func(opts *PullOptions) error { source, err := downloader.NewSourceFromStr(sourceUrl) @@ -79,18 +94,8 @@ func (c *KpmClient) Pull(options ...PullOption) (*pkg.KclPkg, error) { return nil, err } - // Remove information about modspec from message - // to avoid escape symbols ':' in modspec. - sourceUrl, err := url.Parse(sourceStr) - if err != nil { - return nil, err - } - queryParams := sourceUrl.Query() - queryParams.Del(constants.Mod) - sourceUrl.RawQuery = queryParams.Encode() - reporter.ReportMsgTo( - fmt.Sprintf("start to pull %s", sourceUrl.String()), + fmt.Sprintf("start to pull %s", sourceStr), c.GetLogWriter(), ) diff --git a/pkg/client/run.go b/pkg/client/run.go index a22aa4a9..23866d11 100644 --- a/pkg/client/run.go +++ b/pkg/client/run.go @@ -89,6 +89,29 @@ type RunOptions struct { type RunOption func(*RunOptions) error +func WithRunModSpec(modSpec *downloader.ModSpec) RunOption { + return func(ro *RunOptions) error { + if modSpec == nil { + return errors.New("modSpec cannot be nil") + } + if ro.Sources == nil { + ro.Sources = make([]*downloader.Source, 0) + } + if len(ro.Sources) > 1 { + return errors.New("only allows one package to be compiled at a time") + } + if len(ro.Sources) == 0 { + ro.Sources = append(ro.Sources, &downloader.Source{ + ModSpec: modSpec, + }) + } else { + ro.Sources[0].ModSpec = modSpec + } + + return nil + } +} + // Use the another RunOptions to override the current RunOptions. func WithRunOptions(runOpts *RunOptions) RunOption { return func(ro *RunOptions) error {