cmd/cue: cue mod tidy command

This implements a first cut of a `cue mod tidy` command,
according to the proposal at #2330.

This is all experimental for now, guarded by `CUEEXPERIMENT=modules`,
and is subject to change.

Not yet done:
- support for running `cue mod tidy` not in the module root
- default major versions to allow package imports without
an explicit major version
- caching of module contents

Signed-off-by: Roger Peppe <[email protected]>
Change-Id: Id288405bde9c85b9d16a41602f5e02898c57d13e
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1173171
Unity-Result: CUE porcuepine <[email protected]>
TryBot-Result: CUEcueckoo <[email protected]>
Reviewed-by: Daniel Martí <[email protected]>

cmd/cue/cmd/common.go

@@ -1,4 +1,4 @@
-// Copyright 2018 The CUE Authors
+// Copyright 2018 CUE Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

cmd/cue/cmd/mod.go

@@ -46,6 +46,7 @@ func newModCmd(c *Command) *cobra.Command {
 
 	cmd.AddCommand(newModInitCmd(c))
 	cmd.AddCommand(newModUploadCmd(c))
+	cmd.AddCommand(newModTidyCmd(c))
 	return cmd
 }
 

cmd/cue/cmd/modtidy.go

@@ -0,0 +1,136 @@
+// Copyright 2023 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"archive/zip"
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+
+	"cuelang.org/go/internal/mod/modfile"
+	"cuelang.org/go/internal/mod/modload"
+	"cuelang.org/go/internal/mod/modpkgload"
+	"cuelang.org/go/internal/mod/modregistry"
+	"cuelang.org/go/internal/mod/modrequirements"
+	"cuelang.org/go/internal/mod/module"
+)
+
+func newModTidyCmd(c *Command) *cobra.Command {
+	cmd := &cobra.Command{
+		// TODO: this command is still experimental, don't show it in
+		// the documentation just yet.
+		Hidden: true,
+
+		Use:   "tidy",
+		Short: "download and tidy module dependencies",
+		Long: `WARNING: THIS COMMAND IS EXPERIMENTAL.
+
+Currently this command must be run in the module's root directory.
+`,
+		RunE: mkRunE(c, runModTidy),
+		Args: cobra.ExactArgs(0),
+	}
+
+	return cmd
+}
+
+func runModTidy(cmd *Command, args []string) error {
+	reg, err := getRegistry()
+	if err != nil {
+		return err
+	}
+	if reg == nil {
+		return fmt.Errorf("no registry configured to upload to")
+	}
+	ctx := context.Background()
+	// TODO don't assume we're running in the module's root directory.
+	wd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+	loadReg := &modloadRegistry{modregistry.NewClient(reg)}
+	mf, err := modload.Load(ctx, os.DirFS(wd), ".", loadReg)
+	if err != nil {
+		return err
+	}
+	// TODO check whether it's changed or not.
+	data, err := mf.Format()
+	if err != nil {
+		return fmt.Errorf("internal error: invalid module.cue file generated: %v", err)
+	}
+	if err := os.WriteFile(filepath.Join("cue.mod", "module.cue"), data, 0o666); err != nil {
+		return err
+	}
+	return nil
+}
+
+type modloadRegistry struct {
+	reg *modregistry.Client
+}
+
+func (r *modloadRegistry) CUEModSummary(ctx context.Context, mv module.Version) (*modrequirements.ModFileSummary, error) {
+	m, err := r.reg.GetModule(ctx, mv)
+	if err != nil {
+		return nil, err
+	}
+	data, err := m.ModuleFile(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("cannot get module file from %v: %v", m, err)
+	}
+	mf, err := modfile.Parse(data, mv.String())
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse module file from %v: %v", m, err)
+	}
+	return &modrequirements.ModFileSummary{
+		Require: mf.DepVersions(),
+		Module:  mv,
+	}, nil
+}
+
+// getModContents downloads the module with the given version
+// and returns the directory where it's stored.
+func (c *modloadRegistry) Fetch(ctx context.Context, mv module.Version) (modpkgload.SourceLoc, error) {
+	m, err := c.reg.GetModule(ctx, mv)
+	if err != nil {
+		return modpkgload.SourceLoc{}, err
+	}
+	r, err := m.GetZip(ctx)
+	if err != nil {
+		return modpkgload.SourceLoc{}, err
+	}
+	defer r.Close()
+	zipData, err := io.ReadAll(r)
+	if err != nil {
+		return modpkgload.SourceLoc{}, err
+	}
+	zipr, err := zip.NewReader(bytes.NewReader(zipData), int64(len(zipData)))
+	if err != nil {
+		return modpkgload.SourceLoc{}, err
+	}
+	return modpkgload.SourceLoc{
+		FS:  zipr,
+		Dir: ".",
+	}, nil
+}
+
+func (r *modloadRegistry) ModuleVersions(ctx context.Context, mpath string) ([]string, error) {
+	return r.reg.ModuleVersions(ctx, mpath)
+}

cmd/cue/cmd/registry.go

@@ -14,6 +14,9 @@ import (
 	"cuelang.org/go/internal/mod/modresolve"
 )
 
+// getRegistry returns the registry to pull modules from.
+// If external modules are disabled and there's no other issue,
+// it returns (nil, nil).
 func getRegistry() (ociregistry.Interface, error) {
 	// TODO document CUE_REGISTRY via a new "cue help environment" subcommand.
 	env := os.Getenv("CUE_REGISTRY")

cmd/cue/cmd/testdata/script/modtidy_ambiguous_import.txtar

@@ -0,0 +1,29 @@
+# Test the errors resulting there's an "ambiguous import" error.
+# The errors _should_ report the full directory name for each module,
+# but currently only report the relative name.
+
+! exec cue mod tidy
+cmp stderr want-stderr
+
+-- want-stderr --
+failed to resolve "example.com/foo/bar@v0": ambiguous import: found package example.com/foo/bar@v0 in multiple modules:
+	example.com@v0 v0.0.1 (foo/bar)
+	example.com/foo@v0 v0.1.0 (bar)
+-- cue.mod/module.cue --
+module: "main.org@v0"
+
+-- main.cue --
+package main
+import "example.com/foo/bar@v0"
+
+-- _registry/example.com_v0.0.1/cue.mod/module.cue --
+module: "example.com@v0"
+
+-- _registry/example.com_v0.0.1/foo/bar/x.cue --
+package bar
+
+-- _registry/example.com_foo_v0.1.0/cue.mod/module.cue --
+module: "example.com/foo@v0"
+
+-- _registry/example.com_foo_v0.1.0/bar/x.cue --
+package bar

cmd/cue/cmd/testdata/script/modtidy_initial.txtar

@@ -0,0 +1,151 @@
+# Check that cue mod tidy can add dependencies by
+# querying the registry, that it doesn't upgrade existing
+# dependencies, and that it removes dependencies that
+# aren't needed any more.
+
+exec cue mod tidy
+cmp cue.mod/module.cue want-module
+
+# Check that the resulting module evaluates as expected.
+exec cue export .
+cmp stdout want-stdout
+-- want-module --
+{
+	module: "main.org@v0"
+	deps: {
+		"bar.com@v0": {
+			v: "v0.5.0"
+		}
+		"baz.org@v0": {
+			v: "v0.10.1"
+		}
+		"example.com@v0": {
+			v: "v0.0.1"
+		}
+		"foo.com/bar/hello@v0": {
+			v: "v0.2.3"
+		}
+	}
+}
+-- want-stdout --
+{
+    "main": "main",
+    "foo.com/bar/hello@v0": "v0.2.3",
+    "bar.com@v0": "v0.5.0",
+    "baz.org@v0": "v0.10.1",
+    "example.com@v0": "v0.0.1"
+}
+-- cue.mod/module.cue --
+module: "main.org@v0"
+
+deps: "example.com@v0": v: "v0.0.1"
+deps: "unused.com@v0": v: "v0.2.4"
+
+-- main.cue --
+package main
+import "example.com@v0:main"
+
+main
+
+-- _registry/example.com_v0.0.1/cue.mod/module.cue --
+module: "example.com@v0"
+deps: {
+	"foo.com/bar/hello@v0": v: "v0.2.3"
+	"bar.com@v0": v: "v0.5.0"
+}
+
+-- _registry/example.com_v0.0.1/top.cue --
+package main
+
+// TODO: import without a major version should
+// the major version from the module.cue file.
+import a "foo.com/bar/hello@v0"
+a
+main: "main"
+"example.com@v0": "v0.0.1"
+
+-- _registry/unused.com_v0.2.4/cue.mod/module.cue --
+module: "unused.com@v0"
+
+-- _registry/example.com_v0.1.2/cue.mod/module.cue --
+module: "example.com@v0"
+
+-- _registry/example.com_v0.1.2/top.cue --
+package main
+"example.com@v0": "v0.1.2"
+
+// TODO: import without a major version should
+// the major version from the module.cue file.
+main: "main"
+"example.com@v0": "v0.0.1"
+
+-- _registry/foo.com_bar_hello_v0.2.3/cue.mod/module.cue --
+module: "foo.com/bar/hello@v0"
+deps: {
+	"bar.com@v0": v: "v0.0.2"
+	"baz.org@v0": v: "v0.10.1"
+}
+
+-- _registry/foo.com_bar_hello_v0.2.3/x.cue --
+package hello
+import (
+	a "bar.com/bar@v0"
+	b "baz.org@v0:baz"
+)
+"foo.com/bar/hello@v0": "v0.2.3"
+a
+b
+
+
+-- _registry/bar.com_v0.0.2/cue.mod/module.cue --
+module: "bar.com@v0"
+deps: "baz.org@v0": v: "v0.0.2"
+
+-- _registry/bar.com_v0.0.2/bar/x.cue --
+package bar
+import a "baz.org@v0:baz"
+"bar.com@v0": "v0.0.2"
+a
+
+
+-- _registry/bar.com_v0.5.0/cue.mod/module.cue --
+module: "bar.com@v0"
+deps: "baz.org@v0": v: "v0.5.0"
+
+-- _registry/bar.com_v0.5.0/bar/x.cue --
+package bar
+import a "baz.org@v0:baz"
+"bar.com@v0": "v0.5.0"
+a
+
+
+-- _registry/baz.org_v0.0.2/cue.mod/module.cue --
+module: "baz.org@v0"
+
+-- _registry/baz.org_v0.0.2/baz.cue --
+package baz
+"baz.org@v0": "v0.0.2"
+
+
+-- _registry/baz.org_v0.1.2/cue.mod/module.cue --
+module: "baz.org@v0"
+
+-- _registry/baz.org_v0.1.2/baz.cue --
+package baz
+"baz.org@v0": "v0.1.2"
+
+
+-- _registry/baz.org_v0.5.0/cue.mod/module.cue --
+module: "baz.org@v0"
+
+-- _registry/baz.org_v0.5.0/baz.cue --
+package baz
+"baz.org@v0": "v0.5.0"
+
+
+-- _registry/baz.org_v0.10.1/cue.mod/module.cue --
+module: "baz.org@v0"
+
+-- _registry/baz.org_v0.10.1/baz.cue --
+package baz
+"baz.org@v0": "v0.10.1"

internal/mod/modload/load.go

@@ -42,7 +42,7 @@ func Load(ctx context.Context, fsys fs.FS, modRoot string, reg Registry) (*modfi
 	modFilePath := path.Join(modRoot, "cue.mod/module.cue")
 	data, err := fs.ReadFile(fsys, modFilePath)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("cannot read cue.mod file: %v", err)
 	}
 	mf, err := modfile.ParseNonStrict(data, modFilePath)
 	if err != nil {

internal/mod/modpkgload/import.go

@@ -105,10 +105,10 @@ func (pkgs *Packages) importFromModules(ctx context.Context, pkgPath string) (m
 				// continue the loop and find the package in some other module,
 				// we need to look at this module to make sure the import is
 				// not ambiguous.
-				return fail(err)
+				return fail(fmt.Errorf("cannot fetch %v: %v", m, err))
 			}
 			if loc, ok, err := locInModule(pkgPathOnly, prefix, mloc, isLocal); err != nil {
-				return fail(err)
+				return fail(fmt.Errorf("cannot find package: %v", err))
 			} else if ok {
 				mods = append(mods, m)
 				locs = append(locs, loc)
@@ -145,7 +145,7 @@ func (pkgs *Packages) importFromModules(ctx context.Context, pkgPath string) (m
 			// the module graph, so we can't return an ImportMissingError here — one
 			// of the missing modules might actually contain the package in question,
 			// in which case we shouldn't go looking for it in some new dependency.
-			return fail(err)
+			return fail(fmt.Errorf("cannot expand module graph: %v", err))
 		}
 	}
 }

internal/mod/modpkgload/pkgload.go

@@ -253,7 +253,7 @@ func (pkgs *Packages) load(ctx context.Context, pkg *Package) {
 	}
 	imports, err := modimports.AllImports(modimports.PackageFiles(pkg.loc.FS, pkg.loc.Dir))
 	if err != nil {
-		pkg.err = err
+		pkg.err = fmt.Errorf("cannot get imports: %v", err)
 		return
 	}
 

internal/mod/modpkgload/testdata/simple.txtar

@@ -33,7 +33,7 @@ example.com/blah@v0
 		foo.com/bar/hello/goodbye@v0
 foo.com/bar/hello/goodbye@v0
 	flags: inAll,isRoot,fromRoot
-	error: module foo.com/bar/[email protected] not found at _registry/foo.com_bar_hello_v0.2.3
+	error: cannot fetch foo.com/bar/[email protected]: module foo.com/bar/[email protected] not found at _registry/foo.com_bar_hello_v0.2.3
 	missing: false
 -- main.cue --
 package main