From 26351196e988c652b8fcef574a3100b5a34c2d7b Mon Sep 17 00:00:00 2001 From: Roger Peppe Date: Tue, 5 Dec 2023 10:53:12 +0000 Subject: [PATCH] cmd/cue: cue mod tidy command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This implements a first cut of a `cue mod tidy` command, according to the proposal at #2330. This is all experimental for now, guarded by `CUEEXPERIMENT=modules`, and is subject to change. Not yet done: - support for running `cue mod tidy` not in the module root - default major versions to allow package imports without an explicit major version - caching of module contents Signed-off-by: Roger Peppe Change-Id: Id288405bde9c85b9d16a41602f5e02898c57d13e Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1173171 Unity-Result: CUE porcuepine TryBot-Result: CUEcueckoo Reviewed-by: Daniel Martí --- cmd/cue/cmd/common.go | 2 +- cmd/cue/cmd/mod.go | 1 + cmd/cue/cmd/modtidy.go | 136 ++++++++++++++++ cmd/cue/cmd/registry.go | 3 + .../script/modtidy_ambiguous_import.txtar | 29 ++++ .../cmd/testdata/script/modtidy_initial.txtar | 151 ++++++++++++++++++ internal/mod/modload/load.go | 2 +- internal/mod/modpkgload/import.go | 6 +- internal/mod/modpkgload/pkgload.go | 2 +- internal/mod/modpkgload/testdata/simple.txtar | 2 +- 10 files changed, 327 insertions(+), 7 deletions(-) create mode 100644 cmd/cue/cmd/modtidy.go create mode 100644 cmd/cue/cmd/testdata/script/modtidy_ambiguous_import.txtar create mode 100644 cmd/cue/cmd/testdata/script/modtidy_initial.txtar diff --git a/cmd/cue/cmd/common.go b/cmd/cue/cmd/common.go index 9937c81cb57..bbd61d2dd4a 100644 --- a/cmd/cue/cmd/common.go +++ b/cmd/cue/cmd/common.go @@ -1,4 +1,4 @@ -// Copyright 2018 The CUE Authors +// Copyright 2018 CUE Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cmd/cue/cmd/mod.go b/cmd/cue/cmd/mod.go index fb8626b53e2..eff5b04cd1b 100644 --- a/cmd/cue/cmd/mod.go +++ b/cmd/cue/cmd/mod.go @@ -46,6 +46,7 @@ func newModCmd(c *Command) *cobra.Command { cmd.AddCommand(newModInitCmd(c)) cmd.AddCommand(newModUploadCmd(c)) + cmd.AddCommand(newModTidyCmd(c)) return cmd } diff --git a/cmd/cue/cmd/modtidy.go b/cmd/cue/cmd/modtidy.go new file mode 100644 index 00000000000..9a713855983 --- /dev/null +++ b/cmd/cue/cmd/modtidy.go @@ -0,0 +1,136 @@ +// Copyright 2023 The CUE Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "archive/zip" + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/spf13/cobra" + + "cuelang.org/go/internal/mod/modfile" + "cuelang.org/go/internal/mod/modload" + "cuelang.org/go/internal/mod/modpkgload" + "cuelang.org/go/internal/mod/modregistry" + "cuelang.org/go/internal/mod/modrequirements" + "cuelang.org/go/internal/mod/module" +) + +func newModTidyCmd(c *Command) *cobra.Command { + cmd := &cobra.Command{ + // TODO: this command is still experimental, don't show it in + // the documentation just yet. + Hidden: true, + + Use: "tidy", + Short: "download and tidy module dependencies", + Long: `WARNING: THIS COMMAND IS EXPERIMENTAL. + +Currently this command must be run in the module's root directory. +`, + RunE: mkRunE(c, runModTidy), + Args: cobra.ExactArgs(0), + } + + return cmd +} + +func runModTidy(cmd *Command, args []string) error { + reg, err := getRegistry() + if err != nil { + return err + } + if reg == nil { + return fmt.Errorf("no registry configured to upload to") + } + ctx := context.Background() + // TODO don't assume we're running in the module's root directory. + wd, err := os.Getwd() + if err != nil { + return err + } + loadReg := &modloadRegistry{modregistry.NewClient(reg)} + mf, err := modload.Load(ctx, os.DirFS(wd), ".", loadReg) + if err != nil { + return err + } + // TODO check whether it's changed or not. + data, err := mf.Format() + if err != nil { + return fmt.Errorf("internal error: invalid module.cue file generated: %v", err) + } + if err := os.WriteFile(filepath.Join("cue.mod", "module.cue"), data, 0o666); err != nil { + return err + } + return nil +} + +type modloadRegistry struct { + reg *modregistry.Client +} + +func (r *modloadRegistry) CUEModSummary(ctx context.Context, mv module.Version) (*modrequirements.ModFileSummary, error) { + m, err := r.reg.GetModule(ctx, mv) + if err != nil { + return nil, err + } + data, err := m.ModuleFile(ctx) + if err != nil { + return nil, fmt.Errorf("cannot get module file from %v: %v", m, err) + } + mf, err := modfile.Parse(data, mv.String()) + if err != nil { + return nil, fmt.Errorf("cannot parse module file from %v: %v", m, err) + } + return &modrequirements.ModFileSummary{ + Require: mf.DepVersions(), + Module: mv, + }, nil +} + +// getModContents downloads the module with the given version +// and returns the directory where it's stored. +func (c *modloadRegistry) Fetch(ctx context.Context, mv module.Version) (modpkgload.SourceLoc, error) { + m, err := c.reg.GetModule(ctx, mv) + if err != nil { + return modpkgload.SourceLoc{}, err + } + r, err := m.GetZip(ctx) + if err != nil { + return modpkgload.SourceLoc{}, err + } + defer r.Close() + zipData, err := io.ReadAll(r) + if err != nil { + return modpkgload.SourceLoc{}, err + } + zipr, err := zip.NewReader(bytes.NewReader(zipData), int64(len(zipData))) + if err != nil { + return modpkgload.SourceLoc{}, err + } + return modpkgload.SourceLoc{ + FS: zipr, + Dir: ".", + }, nil +} + +func (r *modloadRegistry) ModuleVersions(ctx context.Context, mpath string) ([]string, error) { + return r.reg.ModuleVersions(ctx, mpath) +} diff --git a/cmd/cue/cmd/registry.go b/cmd/cue/cmd/registry.go index 630fa79e295..a85bddd61a7 100644 --- a/cmd/cue/cmd/registry.go +++ b/cmd/cue/cmd/registry.go @@ -14,6 +14,9 @@ import ( "cuelang.org/go/internal/mod/modresolve" ) +// getRegistry returns the registry to pull modules from. +// If external modules are disabled and there's no other issue, +// it returns (nil, nil). func getRegistry() (ociregistry.Interface, error) { // TODO document CUE_REGISTRY via a new "cue help environment" subcommand. env := os.Getenv("CUE_REGISTRY") diff --git a/cmd/cue/cmd/testdata/script/modtidy_ambiguous_import.txtar b/cmd/cue/cmd/testdata/script/modtidy_ambiguous_import.txtar new file mode 100644 index 00000000000..a948a6220db --- /dev/null +++ b/cmd/cue/cmd/testdata/script/modtidy_ambiguous_import.txtar @@ -0,0 +1,29 @@ +# Test the errors resulting there's an "ambiguous import" error. +# The errors _should_ report the full directory name for each module, +# but currently only report the relative name. + +! exec cue mod tidy +cmp stderr want-stderr + +-- want-stderr -- +failed to resolve "example.com/foo/bar@v0": ambiguous import: found package example.com/foo/bar@v0 in multiple modules: + example.com@v0 v0.0.1 (foo/bar) + example.com/foo@v0 v0.1.0 (bar) +-- cue.mod/module.cue -- +module: "main.org@v0" + +-- main.cue -- +package main +import "example.com/foo/bar@v0" + +-- _registry/example.com_v0.0.1/cue.mod/module.cue -- +module: "example.com@v0" + +-- _registry/example.com_v0.0.1/foo/bar/x.cue -- +package bar + +-- _registry/example.com_foo_v0.1.0/cue.mod/module.cue -- +module: "example.com/foo@v0" + +-- _registry/example.com_foo_v0.1.0/bar/x.cue -- +package bar diff --git a/cmd/cue/cmd/testdata/script/modtidy_initial.txtar b/cmd/cue/cmd/testdata/script/modtidy_initial.txtar new file mode 100644 index 00000000000..446515ac122 --- /dev/null +++ b/cmd/cue/cmd/testdata/script/modtidy_initial.txtar @@ -0,0 +1,151 @@ +# Check that cue mod tidy can add dependencies by +# querying the registry, that it doesn't upgrade existing +# dependencies, and that it removes dependencies that +# aren't needed any more. + +exec cue mod tidy +cmp cue.mod/module.cue want-module + +# Check that the resulting module evaluates as expected. +exec cue export . +cmp stdout want-stdout +-- want-module -- +{ + module: "main.org@v0" + deps: { + "bar.com@v0": { + v: "v0.5.0" + } + "baz.org@v0": { + v: "v0.10.1" + } + "example.com@v0": { + v: "v0.0.1" + } + "foo.com/bar/hello@v0": { + v: "v0.2.3" + } + } +} +-- want-stdout -- +{ + "main": "main", + "foo.com/bar/hello@v0": "v0.2.3", + "bar.com@v0": "v0.5.0", + "baz.org@v0": "v0.10.1", + "example.com@v0": "v0.0.1" +} +-- cue.mod/module.cue -- +module: "main.org@v0" + +deps: "example.com@v0": v: "v0.0.1" +deps: "unused.com@v0": v: "v0.2.4" + +-- main.cue -- +package main +import "example.com@v0:main" + +main + +-- _registry/example.com_v0.0.1/cue.mod/module.cue -- +module: "example.com@v0" +deps: { + "foo.com/bar/hello@v0": v: "v0.2.3" + "bar.com@v0": v: "v0.5.0" +} + +-- _registry/example.com_v0.0.1/top.cue -- +package main + +// TODO: import without a major version should +// the major version from the module.cue file. +import a "foo.com/bar/hello@v0" +a +main: "main" +"example.com@v0": "v0.0.1" + +-- _registry/unused.com_v0.2.4/cue.mod/module.cue -- +module: "unused.com@v0" + +-- _registry/example.com_v0.1.2/cue.mod/module.cue -- +module: "example.com@v0" + +-- _registry/example.com_v0.1.2/top.cue -- +package main +"example.com@v0": "v0.1.2" + +// TODO: import without a major version should +// the major version from the module.cue file. +main: "main" +"example.com@v0": "v0.0.1" + +-- _registry/foo.com_bar_hello_v0.2.3/cue.mod/module.cue -- +module: "foo.com/bar/hello@v0" +deps: { + "bar.com@v0": v: "v0.0.2" + "baz.org@v0": v: "v0.10.1" +} + +-- _registry/foo.com_bar_hello_v0.2.3/x.cue -- +package hello +import ( + a "bar.com/bar@v0" + b "baz.org@v0:baz" +) +"foo.com/bar/hello@v0": "v0.2.3" +a +b + + +-- _registry/bar.com_v0.0.2/cue.mod/module.cue -- +module: "bar.com@v0" +deps: "baz.org@v0": v: "v0.0.2" + +-- _registry/bar.com_v0.0.2/bar/x.cue -- +package bar +import a "baz.org@v0:baz" +"bar.com@v0": "v0.0.2" +a + + +-- _registry/bar.com_v0.5.0/cue.mod/module.cue -- +module: "bar.com@v0" +deps: "baz.org@v0": v: "v0.5.0" + +-- _registry/bar.com_v0.5.0/bar/x.cue -- +package bar +import a "baz.org@v0:baz" +"bar.com@v0": "v0.5.0" +a + + +-- _registry/baz.org_v0.0.2/cue.mod/module.cue -- +module: "baz.org@v0" + +-- _registry/baz.org_v0.0.2/baz.cue -- +package baz +"baz.org@v0": "v0.0.2" + + +-- _registry/baz.org_v0.1.2/cue.mod/module.cue -- +module: "baz.org@v0" + +-- _registry/baz.org_v0.1.2/baz.cue -- +package baz +"baz.org@v0": "v0.1.2" + + +-- _registry/baz.org_v0.5.0/cue.mod/module.cue -- +module: "baz.org@v0" + +-- _registry/baz.org_v0.5.0/baz.cue -- +package baz +"baz.org@v0": "v0.5.0" + + +-- _registry/baz.org_v0.10.1/cue.mod/module.cue -- +module: "baz.org@v0" + +-- _registry/baz.org_v0.10.1/baz.cue -- +package baz +"baz.org@v0": "v0.10.1" diff --git a/internal/mod/modload/load.go b/internal/mod/modload/load.go index 6e3456f347d..39038b525b5 100644 --- a/internal/mod/modload/load.go +++ b/internal/mod/modload/load.go @@ -42,7 +42,7 @@ func Load(ctx context.Context, fsys fs.FS, modRoot string, reg Registry) (*modfi modFilePath := path.Join(modRoot, "cue.mod/module.cue") data, err := fs.ReadFile(fsys, modFilePath) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot read cue.mod file: %v", err) } mf, err := modfile.ParseNonStrict(data, modFilePath) if err != nil { diff --git a/internal/mod/modpkgload/import.go b/internal/mod/modpkgload/import.go index 0e7cbfdc947..96f1f09213b 100644 --- a/internal/mod/modpkgload/import.go +++ b/internal/mod/modpkgload/import.go @@ -105,10 +105,10 @@ func (pkgs *Packages) importFromModules(ctx context.Context, pkgPath string) (m // continue the loop and find the package in some other module, // we need to look at this module to make sure the import is // not ambiguous. - return fail(err) + return fail(fmt.Errorf("cannot fetch %v: %v", m, err)) } if loc, ok, err := locInModule(pkgPathOnly, prefix, mloc, isLocal); err != nil { - return fail(err) + return fail(fmt.Errorf("cannot find package: %v", err)) } else if ok { mods = append(mods, m) locs = append(locs, loc) @@ -145,7 +145,7 @@ func (pkgs *Packages) importFromModules(ctx context.Context, pkgPath string) (m // the module graph, so we can't return an ImportMissingError here — one // of the missing modules might actually contain the package in question, // in which case we shouldn't go looking for it in some new dependency. - return fail(err) + return fail(fmt.Errorf("cannot expand module graph: %v", err)) } } } diff --git a/internal/mod/modpkgload/pkgload.go b/internal/mod/modpkgload/pkgload.go index c283b0860ee..a0411f51a08 100644 --- a/internal/mod/modpkgload/pkgload.go +++ b/internal/mod/modpkgload/pkgload.go @@ -253,7 +253,7 @@ func (pkgs *Packages) load(ctx context.Context, pkg *Package) { } imports, err := modimports.AllImports(modimports.PackageFiles(pkg.loc.FS, pkg.loc.Dir)) if err != nil { - pkg.err = err + pkg.err = fmt.Errorf("cannot get imports: %v", err) return } diff --git a/internal/mod/modpkgload/testdata/simple.txtar b/internal/mod/modpkgload/testdata/simple.txtar index f68d100f7e0..de99bd95f40 100644 --- a/internal/mod/modpkgload/testdata/simple.txtar +++ b/internal/mod/modpkgload/testdata/simple.txtar @@ -33,7 +33,7 @@ example.com/blah@v0 foo.com/bar/hello/goodbye@v0 foo.com/bar/hello/goodbye@v0 flags: inAll,isRoot,fromRoot - error: module foo.com/bar/hello@v0.2.3 not found at _registry/foo.com_bar_hello_v0.2.3 + error: cannot fetch foo.com/bar/hello@v0.2.3: module foo.com/bar/hello@v0.2.3 not found at _registry/foo.com_bar_hello_v0.2.3 missing: false -- main.cue -- package main