diff --git a/go.mod b/go.mod index 05c1ecc4..70a2ab66 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.3 // indirect github.com/google/go-cmp v0.5.9 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/iancoleman/strcase v0.3.0 // indirect @@ -81,11 +82,13 @@ require ( golang.org/x/crypto v0.7.0 // indirect golang.org/x/mod v0.9.0 // indirect golang.org/x/net v0.9.0 // indirect + golang.org/x/oauth2 v0.6.0 // indirect golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.7.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.7.0 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect google.golang.org/grpc v1.53.0 // indirect google.golang.org/protobuf v1.28.1 // indirect @@ -99,6 +102,7 @@ require ( github.com/go-git/go-git/v5 v5.6.1 github.com/gofrs/flock v0.8.1 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 + github.com/google/go-github/v50 v50.2.0 github.com/hashicorp/go-version v1.6.0 github.com/moby/term v0.0.0-20221205130635-1aeaba878587 github.com/onsi/ginkgo/v2 v2.9.2 diff --git a/go.sum b/go.sum index 0b6f51c2..e3d3d94f 100644 --- a/go.sum +++ b/go.sum @@ -210,10 +210,15 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-github/v50 v50.2.0 h1:j2FyongEHlO9nxXLc+LP3wuBSVU9mVxfpdYUexMpIfk= +github.com/google/go-github/v50 v50.2.0/go.mod h1:VBY8FB6yPIjrtKhozXv4FQupxKLS6H4m6xFZlT43q8Q= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -511,6 +516,8 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -674,6 +681,8 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= diff --git a/kpm.go b/kpm.go index f6958d23..65ae09d2 100644 --- a/kpm.go +++ b/kpm.go @@ -37,6 +37,7 @@ func main() { cmd.NewLogoutCmd(kpmcli), cmd.NewPushCmd(kpmcli), cmd.NewPullCmd(kpmcli), + cmd.NewUpdateCmd(kpmcli), } app.Flags = []cli.Flag{ &cli.BoolFlag{ diff --git a/pkg/api/test_data/test_run_pkg_in_path/test_kcl/kcl.mod b/pkg/api/test_data/test_run_pkg_in_path/test_kcl/kcl.mod index 8a37fb17..d62b7d10 100644 --- a/pkg/api/test_data/test_run_pkg_in_path/test_kcl/kcl.mod +++ b/pkg/api/test_data/test_run_pkg_in_path/test_kcl/kcl.mod @@ -4,4 +4,4 @@ edition = "0.0.1" version = "0.0.1" [dependencies] -konfig = { git = "https://github.com/awesome-kusion/konfig.git", tag = "v0.0.1" } \ No newline at end of file +konfig = { git = "https://github.com/awesome-kusion/konfig.git", tag = "v0.0.1" } diff --git a/pkg/client/client.go b/pkg/client/client.go index ae15f965..777b1371 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -9,6 +9,7 @@ import ( "reflect" "strings" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/otiai10/copy" "kcl-lang.io/kcl-go/pkg/kcl" "kcl-lang.io/kpm/pkg/constants" @@ -22,6 +23,7 @@ import ( "kcl-lang.io/kpm/pkg/runner" "kcl-lang.io/kpm/pkg/settings" "kcl-lang.io/kpm/pkg/utils" + "oras.land/oras-go/v2" ) // KpmClient is the client of kpm. @@ -82,6 +84,50 @@ func (c *KpmClient) GetSettings() *settings.Settings { return &c.settings } +func (c *KpmClient) LoadPkgFromPath(pkgPath string) (*pkg.KclPkg, error) { + modFile, err := c.LoadModFile(pkgPath) + if err != nil { + return nil, reporter.NewErrorEvent(reporter.FailedLoadKclMod, err, fmt.Sprintf("could not load 'kcl.mod' in '%s'.", pkgPath)) + } + + // Get dependencies from kcl.mod.lock. + deps, err := c.LoadLockDeps(pkgPath) + + if err != nil { + return nil, reporter.NewErrorEvent(reporter.FailedLoadKclMod, err, fmt.Sprintf("could not load 'kcl.mod.lock' in '%s'.", pkgPath)) + } + + return &pkg.KclPkg{ + ModFile: *modFile, + HomePath: pkgPath, + Dependencies: *deps, + }, nil +} + +func (c *KpmClient) LoadModFile(pkgPath string) (*pkg.ModFile, error) { + modFile := new(pkg.ModFile) + err := modFile.LoadModFile(filepath.Join(pkgPath, pkg.MOD_FILE)) + if err != nil { + return nil, err + } + + modFile.HomePath = pkgPath + + if modFile.Dependencies.Deps == nil { + modFile.Dependencies.Deps = make(map[string]pkg.Dependency) + } + err = c.FillDependenciesInfo(modFile) + if err != nil { + return nil, err + } + + return modFile, nil +} + +func (c *KpmClient) LoadLockDeps(pkgPath string) (*pkg.Dependencies, error) { + return pkg.LoadLockDeps(pkgPath) +} + // ResolveDepsIntoMap will calculate the map of kcl package name and local storage path of the external packages. func (c *KpmClient) ResolveDepsIntoMap(kclPkg *pkg.KclPkg) (map[string]string, error) { err := c.ResolvePkgDepsMetadata(kclPkg, true) @@ -115,8 +161,29 @@ func (c *KpmClient) ResolvePkgDepsMetadata(kclPkg *pkg.KclPkg, update bool) erro } // alian the dependencies between kcl.mod and kcl.mod.lock + // clean the dependencies in kcl.mod.lock which not in kcl.mod + for name := range kclPkg.Dependencies.Deps { + if _, ok := kclPkg.ModFile.Dependencies.Deps[name]; !ok { + reporter.ReportEventTo( + reporter.NewEvent( + reporter.RemoveDep, + fmt.Sprintf("removing '%s'", name), + ), + c.logWriter, + ) + delete(kclPkg.Dependencies.Deps, name) + } + } + // add the dependencies in kcl.mod which not in kcl.mod.lock for name, d := range kclPkg.ModFile.Dependencies.Deps { if _, ok := kclPkg.Dependencies.Deps[name]; !ok { + reporter.ReportEventTo( + reporter.NewEvent( + reporter.AddDep, + fmt.Sprintf("adding '%s'", name), + ), + c.logWriter, + ) kclPkg.Dependencies.Deps[name] = d } } @@ -176,6 +243,20 @@ func (c *KpmClient) ResolvePkgDepsMetadata(kclPkg *pkg.KclPkg, update bool) erro return nil } +// UpdateDeps will update the dependencies. +func (c *KpmClient) UpdateDeps(kclPkg *pkg.KclPkg) error { + _, err := c.ResolveDepsMetadataInJsonStr(kclPkg, true) + if err != nil { + return err + } + + err = kclPkg.UpdateModAndLockFile() + if err != nil { + return err + } + return nil +} + // ResolveDepsMetadataInJsonStr will calculate the local storage path of the external package, // and check whether the package exists locally. If the package does not exist, it will re-download to the local. // Finally, the calculated metadata of the dependent packages is serialized into a json string and returned. @@ -564,10 +645,33 @@ func (c *KpmClient) VendorDeps(kclPkg *pkg.KclPkg) error { // FillDepInfo will fill registry information for a dependency. func (c *KpmClient) FillDepInfo(dep *pkg.Dependency) error { - if dep.Source.Oci != nil { + if dep.Source.Git == nil { dep.Source.Oci.Reg = c.GetSettings().DefaultOciRegistry() urlpath := utils.JoinPath(c.GetSettings().DefaultOciRepo(), dep.Name) dep.Source.Oci.Repo = urlpath + manifest := ocispec.Manifest{} + jsonDesc, err := c.FetchOciManifestIntoJsonStr(opt.OciFetchOptions{ + FetchBytesOptions: oras.DefaultFetchBytesOptions, + OciOptions: opt.OciOptions{ + Reg: c.GetSettings().DefaultOciRegistry(), + Repo: fmt.Sprintf("%s/%s", c.GetSettings().DefaultOciRepo(), dep.Name), + Tag: dep.Version, + }, + }) + + if err != nil { + return err + } + + err = json.Unmarshal([]byte(jsonDesc), &manifest) + if err != nil { + return err + } + + if value, ok := manifest.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_SUM]; ok { + dep.Sum = value + } + return nil } return nil } diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 6f04d29b..e70723d3 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -8,8 +8,10 @@ import ( "log" "os" "path/filepath" + "strings" "testing" + "github.com/otiai10/copy" "github.com/stretchr/testify/assert" "kcl-lang.io/kpm/pkg/env" "kcl-lang.io/kpm/pkg/opt" @@ -687,3 +689,64 @@ func TestParseOciOptionFromString(t *testing.T) { assert.Equal(t, ociOption.Repo, "/test_oci_repo") assert.Equal(t, ociOption.Tag, "test_tag") } + +func TestUpdateWithKclMod(t *testing.T) { + kpmcli, err := NewKpmClient() + assert.Equal(t, err, nil) + + testDir := getTestDir("test_update") + src_testDir := filepath.Join(testDir, "test_update_kcl_mod") + dest_testDir := filepath.Join(testDir, "test_update_kcl_mod_tmp") + err = copy.Copy(src_testDir, dest_testDir) + assert.Equal(t, err, nil) + + kclPkg, err := pkg.LoadKclPkg(dest_testDir) + assert.Equal(t, err, nil) + err = kpmcli.UpdateDeps(kclPkg) + assert.Equal(t, err, nil) + got_lock_file := filepath.Join(dest_testDir, "kcl.mod.lock") + got_content, err := os.ReadFile(got_lock_file) + assert.Equal(t, err, nil) + + expected_path := filepath.Join(dest_testDir, "expected") + expected_content, err := os.ReadFile(expected_path) + + assert.Equal(t, err, nil) + expect := strings.ReplaceAll(string(expected_content), "\r\n", "\n") + assert.Equal(t, string(got_content), expect) + + defer func() { + err := os.RemoveAll(dest_testDir) + assert.Equal(t, err, nil) + }() +} + +func TestUpdateWithKclModlock(t *testing.T) { + kpmcli, err := NewKpmClient() + assert.Equal(t, err, nil) + + testDir := getTestDir("test_update") + src_testDir := filepath.Join(testDir, "test_update_kcl_mod_lock") + dest_testDir := filepath.Join(testDir, "test_update_kcl_mod_lock_tmp") + err = copy.Copy(src_testDir, dest_testDir) + assert.Equal(t, err, nil) + + kclPkg, err := pkg.LoadKclPkg(dest_testDir) + assert.Equal(t, err, nil) + err = kpmcli.UpdateDeps(kclPkg) + assert.Equal(t, err, nil) + got_lock_file := filepath.Join(dest_testDir, "kcl.mod.lock") + got_content, err := os.ReadFile(got_lock_file) + assert.Equal(t, err, nil) + + expected_path := filepath.Join(dest_testDir, "expected") + expected_content, err := os.ReadFile(expected_path) + + assert.Equal(t, err, nil) + assert.Equal(t, string(got_content), string(expected_content)) + + defer func() { + err := os.RemoveAll(dest_testDir) + assert.Equal(t, err, nil) + }() +} diff --git a/pkg/client/test_data/test_update/test_update_kcl_mod/expected b/pkg/client/test_data/test_update/test_update_kcl_mod/expected new file mode 100644 index 00000000..6270d4f4 --- /dev/null +++ b/pkg/client/test_data/test_update/test_update_kcl_mod/expected @@ -0,0 +1,9 @@ +[dependencies] + [dependencies.helloworld] + name = "helloworld" + full_name = "helloworld_0.1.1" + version = "0.1.1" + sum = "7OO4YK2QuRWPq9C7KTzcWcti5yUnueCjptT3OXiPVeQ=" + reg = "ghcr.io" + repo = "kcl-lang/helloworld" + oci_tag = "0.1.1" diff --git a/pkg/client/test_data/test_update/test_update_kcl_mod/kcl.mod b/pkg/client/test_data/test_update/test_update_kcl_mod/kcl.mod new file mode 100644 index 00000000..d3a99616 --- /dev/null +++ b/pkg/client/test_data/test_update/test_update_kcl_mod/kcl.mod @@ -0,0 +1,7 @@ +[package] +name = "test_update" +edition = "0.0.1" +version = "0.0.1" + +[dependencies] +helloworld = "0.1.1" diff --git a/pkg/client/test_data/test_update/test_update_kcl_mod/kcl.mod.lock b/pkg/client/test_data/test_update/test_update_kcl_mod/kcl.mod.lock new file mode 100644 index 00000000..e69de29b diff --git a/pkg/client/test_data/test_update/test_update_kcl_mod/main.k b/pkg/client/test_data/test_update/test_update_kcl_mod/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/test_update/test_update_kcl_mod/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/client/test_data/test_update/test_update_kcl_mod_lock/expected b/pkg/client/test_data/test_update/test_update_kcl_mod_lock/expected new file mode 100644 index 00000000..e69de29b diff --git a/pkg/client/test_data/test_update/test_update_kcl_mod_lock/kcl.mod b/pkg/client/test_data/test_update/test_update_kcl_mod_lock/kcl.mod new file mode 100644 index 00000000..f43948ad --- /dev/null +++ b/pkg/client/test_data/test_update/test_update_kcl_mod_lock/kcl.mod @@ -0,0 +1,4 @@ +[package] +name = "test_update" +edition = "0.0.1" +version = "0.0.1" diff --git a/pkg/client/test_data/test_update/test_update_kcl_mod_lock/kcl.mod.lock b/pkg/client/test_data/test_update/test_update_kcl_mod_lock/kcl.mod.lock new file mode 100644 index 00000000..6270d4f4 --- /dev/null +++ b/pkg/client/test_data/test_update/test_update_kcl_mod_lock/kcl.mod.lock @@ -0,0 +1,9 @@ +[dependencies] + [dependencies.helloworld] + name = "helloworld" + full_name = "helloworld_0.1.1" + version = "0.1.1" + sum = "7OO4YK2QuRWPq9C7KTzcWcti5yUnueCjptT3OXiPVeQ=" + reg = "ghcr.io" + repo = "kcl-lang/helloworld" + oci_tag = "0.1.1" diff --git a/pkg/client/test_data/test_update/test_update_kcl_mod_lock/main.k b/pkg/client/test_data/test_update/test_update_kcl_mod_lock/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/test_update/test_update_kcl_mod_lock/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/cmd/cmd_push.go b/pkg/cmd/cmd_push.go index 05f3c4cc..a7477fe3 100644 --- a/pkg/cmd/cmd_push.go +++ b/pkg/cmd/cmd_push.go @@ -167,7 +167,11 @@ func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, kpmcli *cli "only support url scheme 'oci://'.", ) } - ociOpts.Annotations = oci.GenOciManifestFromPkg(kclPkg) + + ociOpts.Annotations, err = oci.GenOciManifestFromPkg(kclPkg) + if err != nil { + return err + } reporter.ReportMsgTo(fmt.Sprintf("kpm: package '%s' will be pushed", kclPkg.GetPkgName()), kpmcli.GetLogWriter()) // 4. Push it. diff --git a/pkg/cmd/cmd_update.go b/pkg/cmd/cmd_update.go new file mode 100644 index 00000000..80d9d7c8 --- /dev/null +++ b/pkg/cmd/cmd_update.go @@ -0,0 +1,68 @@ +// Copyright 2023 The KCL Authors. All rights reserved. + +package cmd + +import ( + "os" + + "github.com/urfave/cli/v2" + "kcl-lang.io/kpm/pkg/client" + "kcl-lang.io/kpm/pkg/env" + "kcl-lang.io/kpm/pkg/errors" + "kcl-lang.io/kpm/pkg/reporter" +) + +// NewUpdateCmd new a Command for `kpm update`. +func NewUpdateCmd(kpmcli *client.KpmClient) *cli.Command { + return &cli.Command{ + Hidden: false, + Name: "update", + Usage: "Update dependencies listed in kcl.mod.lock based on kcl.mod", + Action: func(c *cli.Context) error { + return KpmUpdate(c, kpmcli) + }, + } +} + +func KpmUpdate(c *cli.Context, kpmcli *client.KpmClient) error { + input_paths := c.Args().Slice() + + pkg_paths := []string{} + if len(input_paths) == 0 { + pwd, err := os.Getwd() + if err != nil { + return errors.InternalBug + } + pkg_paths = append(pkg_paths, pwd) + } else { + pkg_paths = input_paths + } + + for _, pkg_path := range pkg_paths { + kclPkg, err := kpmcli.LoadPkgFromPath(pkg_path) + if err != nil { + return err + } + + globalPkgPath, err := env.GetAbsPkgPath() + if err != nil { + return err + } + + err = kclPkg.ValidateKpmHome(globalPkgPath) + if err != (*reporter.KpmEvent)(nil) { + return err + } + + _, err = kpmcli.ResolveDepsMetadataInJsonStr(kclPkg, true) + if err != nil { + return err + } + + err = kclPkg.UpdateModAndLockFile() + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index cbd41662..c8cd3fcd 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -17,5 +17,6 @@ const ( DEFAULT_KCL_OCI_MANIFEST_NAME = "org.kcllang.package.name" DEFAULT_KCL_OCI_MANIFEST_VERSION = "org.kcllang.package.version" DEFAULT_KCL_OCI_MANIFEST_DESCRIPTION = "org.kcllang.package.description" + DEFAULT_KCL_OCI_MANIFEST_SUM = "org.kcllang.package.sum" DEFAULT_CREATE_OCI_MANIFEST_TIME = "org.opencontainers.image.created" ) diff --git a/pkg/oci/oci.go b/pkg/oci/oci.go index 24e042fa..1618c222 100644 --- a/pkg/oci/oci.go +++ b/pkg/oci/oci.go @@ -338,10 +338,16 @@ func Push(localPath, hostName, repoName, tag string, settings *settings.Settings return ociClient.Push(localPath, tag) } -func GenOciManifestFromPkg(kclPkg *pkg.KclPkg) map[string]string { +// GenOciManifestFromPkg will generate the oci manifest from the kcl package. +func GenOciManifestFromPkg(kclPkg *pkg.KclPkg) (map[string]string, error) { res := make(map[string]string) res[constants.DEFAULT_KCL_OCI_MANIFEST_NAME] = kclPkg.GetPkgName() res[constants.DEFAULT_KCL_OCI_MANIFEST_VERSION] = kclPkg.GetPkgVersion() res[constants.DEFAULT_KCL_OCI_MANIFEST_DESCRIPTION] = kclPkg.GetPkgDescription() - return res + sum, err := kclPkg.GenCheckSum() + if err != nil { + return nil, err + } + res[constants.DEFAULT_KCL_OCI_MANIFEST_SUM] = sum + return res, nil } diff --git a/pkg/package/modfile.go b/pkg/package/modfile.go index 569ba89a..4ce754ea 100644 --- a/pkg/package/modfile.go +++ b/pkg/package/modfile.go @@ -220,7 +220,7 @@ func NewModFile(opts *opt.InitOptions) *ModFile { } // Load the kcl.mod file. -func (mod *ModFile) loadModFile(filepath string) error { +func (mod *ModFile) LoadModFile(filepath string) error { modData, err := os.ReadFile(filepath) if err != nil { @@ -239,7 +239,7 @@ func (mod *ModFile) loadModFile(filepath string) error { // LoadModFile load the contents of the 'kcl.mod' file in the path. func LoadModFile(homePath string) (*ModFile, error) { modFile := new(ModFile) - err := modFile.loadModFile(filepath.Join(homePath, MOD_FILE)) + err := modFile.LoadModFile(filepath.Join(homePath, MOD_FILE)) if err != nil { return nil, err } diff --git a/pkg/package/package.go b/pkg/package/package.go index cc5270c6..de762f08 100644 --- a/pkg/package/package.go +++ b/pkg/package/package.go @@ -194,3 +194,8 @@ func (KclPkg *KclPkg) GetPkgVersion() string { func (KclPkg *KclPkg) GetPkgDescription() string { return KclPkg.ModFile.Pkg.Description } + +// GenCheckSum generates the checksum of the current kcl package. +func (KclPkg *KclPkg) GenCheckSum() (string, error) { + return utils.HashDir(KclPkg.HomePath) +} diff --git a/pkg/reporter/reporter.go b/pkg/reporter/reporter.go index ce8d667e..95b21a36 100644 --- a/pkg/reporter/reporter.go +++ b/pkg/reporter/reporter.go @@ -98,6 +98,8 @@ const ( AddItselfAsDep PkgTagExists DependencyNotFound + RemoveDep + AddDep KclModNotFound CompileFailed FailedParseVersion diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 6511ad19..49e88bb6 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -38,8 +38,10 @@ func HashDir(dir string) (string, error) { // files in the ".git "directory will cause the same repository, cloned at different times, // has different checksum. - if strings.Contains(path, ".git") { - return nil + for _, ignore := range ignores { + if strings.Contains(path, ignore) { + return nil + } } f, err := os.Open(path) diff --git a/test/e2e/kpm_test.go b/test/e2e/kpm_test.go index ffa0c354..49c42b22 100644 --- a/test/e2e/kpm_test.go +++ b/test/e2e/kpm_test.go @@ -176,7 +176,7 @@ var _ = ginkgo.Describe("Kpm CLI Testing", func() { gomega.Expect(stderr).To(gomega.ContainSubstring(expectedStderr)) } - bytes, err := os.ReadFile(filepath.Join(testDataRoot, ts.Name, "expected_oci_manifest.json")) + bytes, err := os.ReadFile(filepath.Join(testDataRoot, "expected_oci_manifest.json")) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) // 2. fetch the metadata in OCI manifest to check if the metadata is correct @@ -209,6 +209,8 @@ var _ = ginkgo.Describe("Kpm CLI Testing", func() { To(gomega.Equal(manifest_got.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_VERSION])) gomega.Expect(manifest_expect.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_DESCRIPTION]). To(gomega.Equal(manifest_got.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_DESCRIPTION])) + gomega.Expect(manifest_expect.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_SUM]). + To(gomega.Equal(manifest_got.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_SUM])) }) ginkgo.It("testing 'fetch api '", func() { @@ -226,10 +228,11 @@ var _ = ginkgo.Describe("Kpm CLI Testing", func() { var manifest_expect v1.Manifest err = json.Unmarshal([]byte(jsonstr), &manifest_expect) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - gomega.Expect(len(manifest_expect.Annotations)).To(gomega.Equal(4)) + gomega.Expect(len(manifest_expect.Annotations)).To(gomega.Equal(5)) gomega.Expect(manifest_expect.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_NAME]).To(gomega.Equal("kcl2")) gomega.Expect(manifest_expect.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_VERSION]).To(gomega.Equal("0.0.1")) gomega.Expect(manifest_expect.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_DESCRIPTION]).To(gomega.Equal("This is the kcl package named kcl2")) + gomega.Expect(manifest_expect.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_SUM]).To(gomega.Equal("Y/QXruiaxcJcmOnKWl4UEFuUqKTtbi4jTTeuEjeGV8s=")) }) } }) diff --git a/test/e2e/test_suites/kpm/exec_outside_pkg/help_msg/test_suite.stdout b/test/e2e/test_suites/kpm/exec_outside_pkg/help_msg/test_suite.stdout index 5d96e56c..f58a3cb7 100644 --- a/test/e2e/test_suites/kpm/exec_outside_pkg/help_msg/test_suite.stdout +++ b/test/e2e/test_suites/kpm/exec_outside_pkg/help_msg/test_suite.stdout @@ -9,4 +9,5 @@ COMMANDS: logout logout from a registry push push kcl package to OCI registry. pull pull kcl package from OCI registry. + update Update dependencies listed in kcl.mod.lock based on kcl.mod help, h Shows a list of commands or help for one command \ No newline at end of file diff --git a/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_run_oci_url/test_suite.stdout b/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_run_oci_url/test_suite.stdout index 04e751a8..f4059255 100644 --- a/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_run_oci_url/test_suite.stdout +++ b/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_run_oci_url/test_suite.stdout @@ -1,4 +1 @@ -kpm: the lastest version '0.0.1' will be pulled. -kpm: pulling '/test/kcl1:0.0.1' from 'localhost:5001/test/kcl1'. -kpm: downloading 'test/k8s:1.27' from 'localhost:5001/test/k8s:1.27'. The_first_kcl_program: Hello World! \ No newline at end of file diff --git a/test/e2e/test_suites/kpm/kpm_run/test_kpm_run_with_only_kcl_mod/test_suite.stdout b/test/e2e/test_suites/kpm/kpm_run/test_kpm_run_with_only_kcl_mod/test_suite.stdout index 52b79fa3..c52b846e 100644 --- a/test/e2e/test_suites/kpm/kpm_run/test_kpm_run_with_only_kcl_mod/test_suite.stdout +++ b/test/e2e/test_suites/kpm/kpm_run/test_kpm_run_with_only_kcl_mod/test_suite.stdout @@ -1,3 +1,4 @@ +kpm: adding 'k8s' kpm: downloading 'test/k8s:1.27' from 'localhost:5001/test/k8s:1.27'. apiVersion: v1 kind: Pod diff --git a/test/e2e/test_suites/test_data/test_push_with_oci_manifest/expected_oci_manifest.json b/test/e2e/test_suites/test_data/expected_oci_manifest.json similarity index 81% rename from test/e2e/test_suites/test_data/test_push_with_oci_manifest/expected_oci_manifest.json rename to test/e2e/test_suites/test_data/expected_oci_manifest.json index 1c966fc2..719a659f 100644 --- a/test/e2e/test_suites/test_data/test_push_with_oci_manifest/expected_oci_manifest.json +++ b/test/e2e/test_suites/test_data/expected_oci_manifest.json @@ -1 +1 @@ -{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","artifactType":"application/vnd.oci.image.layer.v1.tar","config":{"mediaType":"application/vnd.oci.empty.v1+json","digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2,"data":"e30="},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar","digest":"sha256:1d0b4bf7380ee67c23d48a16ebfbcc0720e9d0c59a1fe0f603e452ee8e99068f","size":4608,"annotations":{"org.opencontainers.image.title":"test_push_with_oci_manifest-0.0.1.tar"}}],"annotations":{"org.kcllang.package.description":"This is the description of the package","org.kcllang.package.name":"test_push_with_oci_manifest","org.kcllang.package.version":"0.0.1","org.opencontainers.image.created":"2023-10-23T06:24:49Z"}} \ No newline at end of file +{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","artifactType":"application/vnd.oci.image.layer.v1.tar","config":{"mediaType":"application/vnd.oci.empty.v1+json","digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2,"data":"e30="},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar","digest":"sha256:1d0b4bf7380ee67c23d48a16ebfbcc0720e9d0c59a1fe0f603e452ee8e99068f","size":4608,"annotations":{"org.opencontainers.image.title":"test_push_with_oci_manifest-0.0.1.tar"}}],"annotations":{"org.kcllang.package.description":"This is the description of the package","org.kcllang.package.name":"test_push_with_oci_manifest","org.kcllang.package.version":"0.0.1","org.kcllang.package.sum":"ZxaNe8Na988Tnyh8hTInPhPYPWShgJInsgGYpxqxfe4=","org.opencontainers.image.created":"2023-10-23T06:24:49Z"}} \ No newline at end of file