feat: add 'kpm update' to update the dependencies

Signed-off-by: zongz <[email protected]>

kpm.go

@@ -37,6 +37,7 @@ func main() {
 		cmd.NewLogoutCmd(kpmcli),
 		cmd.NewPushCmd(kpmcli),
 		cmd.NewPullCmd(kpmcli),
+		cmd.NewUpdateCmd(kpmcli),
 	}
 	app.Flags = []cli.Flag{
 		&cli.BoolFlag{

pkg/client/client.go

@@ -9,6 +9,7 @@ import (
 	"reflect"
 	"strings"
 
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/otiai10/copy"
 	"kcl-lang.io/kcl-go/pkg/kcl"
 	"kcl-lang.io/kpm/pkg/constants"
@@ -22,6 +23,7 @@ import (
 	"kcl-lang.io/kpm/pkg/runner"
 	"kcl-lang.io/kpm/pkg/settings"
 	"kcl-lang.io/kpm/pkg/utils"
+	"oras.land/oras-go/v2"
 )
 
 // KpmClient is the client of kpm.
@@ -97,6 +99,33 @@ func (c *KpmClient) ResolveDepsIntoMap(kclPkg *pkg.KclPkg) (map[string]string, e
 	return pkgMap, nil
 }
 
+// / fillPkgInfoFromOciMenifast will fill the package information from the oci manifest.
+func (c *KpmClient) fillPkgInfoFromOciMenifast(name, tag string, dep pkg.Dependency) (pkg.Dependency, error) {
+	manifest := ocispec.Manifest{}
+	jsonDesc, err := c.FetchOciManifestIntoJsonStr(opt.OciFetchOptions{
+		FetchBytesOptions: oras.DefaultFetchBytesOptions,
+		OciOptions: opt.OciOptions{
+			Reg:  c.GetSettings().DefaultOciRegistry(),
+			Repo: fmt.Sprintf("%s/%s", c.GetSettings().DefaultOciRepo(), name),
+			Tag:  tag,
+		},
+	})
+
+	if err != nil {
+		return dep, err
+	}
+
+	err = json.Unmarshal([]byte(jsonDesc), &manifest)
+	if err != nil {
+		return dep, err
+	}
+
+	if value, ok := manifest.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_SUM]; ok {
+		dep.Sum = value
+	}
+	return dep, nil
+}
+
 // ResolveDepsMetadata will calculate the local storage path of the external package,
 // and check whether the package exists locally.
 // If the package does not exist, it will re-download to the local.
@@ -115,8 +144,33 @@ func (c *KpmClient) ResolvePkgDepsMetadata(kclPkg *pkg.KclPkg, update bool) erro
 	}
 
 	// alian the dependencies between kcl.mod and kcl.mod.lock
+	// clean the dependencies in kcl.mod.lock which not in kcl.mod
+	for name := range kclPkg.Dependencies.Deps {
+		if _, ok := kclPkg.ModFile.Dependencies.Deps[name]; !ok {
+			reporter.ReportEventTo(
+				reporter.NewEvent(
+					reporter.RemoveDep,
+					fmt.Sprintf("removing '%s'", name),
+				),
+				c.logWriter,
+			)
+			delete(kclPkg.Dependencies.Deps, name)
+		}
+	}
+	// add the dependencies in kcl.mod which not in kcl.mod.lock
 	for name, d := range kclPkg.ModFile.Dependencies.Deps {
 		if _, ok := kclPkg.Dependencies.Deps[name]; !ok {
+			reporter.ReportEventTo(
+				reporter.NewEvent(
+					reporter.AddDep,
+					fmt.Sprintf("adding '%s'", name),
+				),
+				c.logWriter,
+			)
+			d, err := c.fillPkgInfoFromOciMenifast(name, d.Version, d)
+			if err != nil {
+				return err
+			}
 			kclPkg.Dependencies.Deps[name] = d
 		}
 	}
@@ -176,6 +230,20 @@ func (c *KpmClient) ResolvePkgDepsMetadata(kclPkg *pkg.KclPkg, update bool) erro
 	return nil
 }
 
+// / UpdateDeps will update the dependencies.
+func (c *KpmClient) UpdateDeps(kclPkg *pkg.KclPkg) error {
+	_, err := c.ResolveDepsMetadataInJsonStr(kclPkg, true)
+	if err != nil {
+		return err
+	}
+
+	err = kclPkg.UpdateModAndLockFile()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 // ResolveDepsMetadataInJsonStr will calculate the local storage path of the external package,
 // and check whether the package exists locally. If the package does not exist, it will re-download to the local.
 // Finally, the calculated metadata of the dependent packages is serialized into a json string and returned.

pkg/client/client_test.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/otiai10/copy"
 	"github.com/stretchr/testify/assert"
 	"kcl-lang.io/kpm/pkg/env"
 	"kcl-lang.io/kpm/pkg/opt"
@@ -687,3 +688,64 @@ func TestParseOciOptionFromString(t *testing.T) {
 	assert.Equal(t, ociOption.Repo, "/test_oci_repo")
 	assert.Equal(t, ociOption.Tag, "test_tag")
 }
+
+func TestUpdateWithKclMod(t *testing.T) {
+	kpmcli, err := NewKpmClient()
+	assert.Equal(t, err, nil)
+
+	testDir := getTestDir("test_update")
+	src_testDir := filepath.Join(testDir, "test_update_kcl_mod")
+	dest_testDir := filepath.Join(testDir, "test_update_kcl_mod_tmp")
+	err = copy.Copy(src_testDir, dest_testDir)
+	assert.Equal(t, err, nil)
+
+	kclPkg, err := pkg.LoadKclPkg(dest_testDir)
+	assert.Equal(t, err, nil)
+	err = kpmcli.UpdateDeps(kclPkg)
+	assert.Equal(t, err, nil)
+	got_lock_file := filepath.Join(dest_testDir, "kcl.mod.lock")
+	got_content, err := os.ReadFile(got_lock_file)
+	assert.Equal(t, err, nil)
+
+	expected_path := filepath.Join(dest_testDir, "expected")
+	expected_content, err := os.ReadFile(expected_path)
+
+	assert.Equal(t, err, nil)
+	assert.Equal(t, string(got_content), string(expected_content))
+
+	defer func() {
+		err := os.RemoveAll(dest_testDir)
+		assert.Equal(t, err, nil)
+	}()
+}
+
+
+func TestUpdateWithKclModlock(t *testing.T) {
+	kpmcli, err := NewKpmClient()
+	assert.Equal(t, err, nil)
+
+	testDir := getTestDir("test_update")
+	src_testDir := filepath.Join(testDir, "test_update_kcl_mod_lock")
+	dest_testDir := filepath.Join(testDir, "test_update_kcl_mod_lock_tmp")
+	err = copy.Copy(src_testDir, dest_testDir)
+	assert.Equal(t, err, nil)
+
+	kclPkg, err := pkg.LoadKclPkg(dest_testDir)
+	assert.Equal(t, err, nil)
+	err = kpmcli.UpdateDeps(kclPkg)
+	assert.Equal(t, err, nil)
+	got_lock_file := filepath.Join(dest_testDir, "kcl.mod.lock")
+	got_content, err := os.ReadFile(got_lock_file)
+	assert.Equal(t, err, nil)
+
+	expected_path := filepath.Join(dest_testDir, "expected")
+	expected_content, err := os.ReadFile(expected_path)
+
+	assert.Equal(t, err, nil)
+	assert.Equal(t, string(got_content), string(expected_content))
+
+	defer func() {
+		err := os.RemoveAll(dest_testDir)
+		assert.Equal(t, err, nil)
+	}()
+}

pkg/client/test_data/test_update/test_update_kcl_mod/expected

@@ -0,0 +1,9 @@
+[dependencies]
+  [dependencies.helloworld]
+    name = "helloworld"
+    full_name = "helloworld_0.1.1"
+    version = "0.1.1"
+    sum = "7OO4YK2QuRWPq9C7KTzcWcti5yUnueCjptT3OXiPVeQ="
+    reg = "ghcr.io"
+    repo = "kcl-lang/helloworld"
+    oci_tag = "0.1.1"

pkg/client/test_data/test_update/test_update_kcl_mod/kcl.mod

@@ -0,0 +1,7 @@
+[package]
+name = "test_update"
+edition = "0.0.1"
+version = "0.0.1"
+
+[dependencies]
+helloworld = "0.1.1"

pkg/client/test_data/test_update/test_update_kcl_mod/main.k

@@ -0,0 +1 @@
+The_first_kcl_program = 'Hello World!'

pkg/client/test_data/test_update/test_update_kcl_mod_lock/kcl.mod

@@ -0,0 +1,4 @@
+[package]
+name = "test_update"
+edition = "0.0.1"
+version = "0.0.1"

pkg/client/test_data/test_update/test_update_kcl_mod_lock/kcl.mod.lock

@@ -0,0 +1,9 @@
+[dependencies]
+  [dependencies.helloworld]
+    name = "helloworld"
+    full_name = "helloworld_0.1.1"
+    version = "0.1.1"
+    sum = "7OO4YK2QuRWPq9C7KTzcWcti5yUnueCjptT3OXiPVeQ="
+    reg = "ghcr.io"
+    repo = "kcl-lang/helloworld"
+    oci_tag = "0.1.1"

pkg/client/test_data/test_update/test_update_kcl_mod_lock/main.k

@@ -0,0 +1 @@
+The_first_kcl_program = 'Hello World!'

pkg/cmd/cmd_push.go

@@ -167,7 +167,11 @@ func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, kpmcli *cli
 			"only support url scheme 'oci://'.",
 		)
 	}
-	ociOpts.Annotations = oci.GenOciManifestFromPkg(kclPkg)
+
+	ociOpts.Annotations, err = oci.GenOciManifestFromPkg(kclPkg)
+	if err != nil {
+		return err
+	}
 
 	reporter.ReportMsgTo(fmt.Sprintf("kpm: package '%s' will be pushed", kclPkg.GetPkgName()), kpmcli.GetLogWriter())
 	// 4. Push it.

pkg/cmd/cmd_update.go

@@ -0,0 +1,69 @@
+// Copyright 2023 The KCL Authors. All rights reserved.
+
+package cmd
+
+import (
+	"os"
+
+	"github.com/urfave/cli/v2"
+	"kcl-lang.io/kpm/pkg/client"
+	"kcl-lang.io/kpm/pkg/env"
+	"kcl-lang.io/kpm/pkg/errors"
+	pkg "kcl-lang.io/kpm/pkg/package"
+	"kcl-lang.io/kpm/pkg/reporter"
+)
+
+// NewUpdateCmd new a Command for `kpm update`.
+func NewUpdateCmd(kpmcli *client.KpmClient) *cli.Command {
+	return &cli.Command{
+		Hidden: false,
+		Name:   "update",
+		Usage:  "Update dependencies listed in kcl.mod.lock",
+		Action: func(c *cli.Context) error {
+			return KpmUpdate(c, kpmcli)
+		},
+	}
+}
+
+func KpmUpdate(c *cli.Context, kpmcli *client.KpmClient) error {
+	input_paths := c.Args().Slice()
+
+	pkg_paths := []string{}
+	if len(input_paths) == 0 {
+		pwd, err := os.Getwd()
+		if err != nil {
+			return errors.InternalBug
+		}
+		pkg_paths = append(pkg_paths, pwd)
+	} else {
+		pkg_paths = input_paths
+	}
+
+	for _, pkg_path := range pkg_paths {
+		kclPkg, err := pkg.LoadKclPkg(pkg_path)
+		if err != nil {
+			return err
+		}
+
+		globalPkgPath, err := env.GetAbsPkgPath()
+		if err != nil {
+			return err
+		}
+
+		err = kclPkg.ValidateKpmHome(globalPkgPath)
+		if err != (*reporter.KpmEvent)(nil) {
+			return err
+		}
+
+		_, err = kpmcli.ResolveDepsMetadataInJsonStr(kclPkg, true)
+		if err != nil {
+			return err
+		}
+
+		err = kclPkg.UpdateModAndLockFile()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/constants/constants.go

@@ -17,5 +17,6 @@ const (
 	DEFAULT_KCL_OCI_MANIFEST_NAME        = "org.kcllang.package.name"
 	DEFAULT_KCL_OCI_MANIFEST_VERSION     = "org.kcllang.package.version"
 	DEFAULT_KCL_OCI_MANIFEST_DESCRIPTION = "org.kcllang.package.description"
+	DEFAULT_KCL_OCI_MANIFEST_SUM         = "org.kcllang.package.sum"
 	DEFAULT_CREATE_OCI_MANIFEST_TIME     = "org.opencontainers.image.created"
 )

pkg/oci/oci.go

@@ -338,10 +338,16 @@ func Push(localPath, hostName, repoName, tag string, settings *settings.Settings
 	return ociClient.Push(localPath, tag)
 }
 
-func GenOciManifestFromPkg(kclPkg *pkg.KclPkg) map[string]string {
+// GenOciManifestFromPkg will generate the oci manifest from the kcl package.
+func GenOciManifestFromPkg(kclPkg *pkg.KclPkg) (map[string]string, error) {
 	res := make(map[string]string)
 	res[constants.DEFAULT_KCL_OCI_MANIFEST_NAME] = kclPkg.GetPkgName()
 	res[constants.DEFAULT_KCL_OCI_MANIFEST_VERSION] = kclPkg.GetPkgVersion()
 	res[constants.DEFAULT_KCL_OCI_MANIFEST_DESCRIPTION] = kclPkg.GetPkgDescription()
-	return res
+	sum, err := kclPkg.GenCheckSum()
+	if err != nil {
+		return nil, err
+	}
+	res[constants.DEFAULT_KCL_OCI_MANIFEST_SUM] = sum
+	return res, nil
 }

pkg/package/package.go

@@ -194,3 +194,8 @@ func (KclPkg *KclPkg) GetPkgVersion() string {
 func (KclPkg *KclPkg) GetPkgDescription() string {
 	return KclPkg.ModFile.Pkg.Description
 }
+
+// GenCheckSum generates the checksum of the current kcl package.
+func (KclPkg *KclPkg) GenCheckSum() (string, error) {
+	return utils.HashDir(KclPkg.HomePath)
+}

pkg/reporter/reporter.go

@@ -98,6 +98,8 @@ const (
 	AddItselfAsDep
 	PkgTagExists
 	DependencyNotFound
+	RemoveDep
+	AddDep
 	KclModNotFound
 	CompileFailed
 	FailedParseVersion

pkg/utils/utils.go

@@ -38,8 +38,10 @@ func HashDir(dir string) (string, error) {
 
 		// files in the ".git "directory will cause the same repository, cloned at different times,
 		// has different checksum.
-		if strings.Contains(path, ".git") {
-			return nil
+		for _, ignore := range ignores {
+			if strings.Contains(path, ignore) {
+				return nil
+			}
 		}
 
 		f, err := os.Open(path)

test/e2e/kpm_test.go

@@ -176,7 +176,7 @@ var _ = ginkgo.Describe("Kpm CLI Testing", func() {
 					gomega.Expect(stderr).To(gomega.ContainSubstring(expectedStderr))
 				}
 
-				bytes, err := os.ReadFile(filepath.Join(testDataRoot, ts.Name, "expected_oci_manifest.json"))
+				bytes, err := os.ReadFile(filepath.Join(testDataRoot, "expected_oci_manifest.json"))
 				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
 
 				// 2. fetch the metadata in OCI manifest to check if the metadata is correct
@@ -209,6 +209,8 @@ var _ = ginkgo.Describe("Kpm CLI Testing", func() {
 					To(gomega.Equal(manifest_got.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_VERSION]))
 				gomega.Expect(manifest_expect.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_DESCRIPTION]).
 					To(gomega.Equal(manifest_got.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_DESCRIPTION]))
+				gomega.Expect(manifest_expect.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_SUM]).
+					To(gomega.Equal(manifest_got.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_SUM]))
 			})
 
 			ginkgo.It("testing 'fetch api '", func() {
@@ -226,10 +228,11 @@ var _ = ginkgo.Describe("Kpm CLI Testing", func() {
 				var manifest_expect v1.Manifest
 				err = json.Unmarshal([]byte(jsonstr), &manifest_expect)
 				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
-				gomega.Expect(len(manifest_expect.Annotations)).To(gomega.Equal(4))
+				gomega.Expect(len(manifest_expect.Annotations)).To(gomega.Equal(5))
 				gomega.Expect(manifest_expect.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_NAME]).To(gomega.Equal("kcl2"))
 				gomega.Expect(manifest_expect.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_VERSION]).To(gomega.Equal("0.0.1"))
 				gomega.Expect(manifest_expect.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_DESCRIPTION]).To(gomega.Equal("This is the kcl package named kcl2"))
+				gomega.Expect(manifest_expect.Annotations[constants.DEFAULT_KCL_OCI_MANIFEST_SUM]).To(gomega.Equal("zQ7PTOcJi4gzXqypCWML6bsjToQU+E9Q2WZw/N3WnNY="))
 			})
 		}
 	})