internal/mod/modimports: new package

This encapsulates the functionality for walking all CUE files in a
module, which will be used when finding out module dependencies in
`cue mod tidy`.

Signed-off-by: Roger Peppe <[email protected]>
Change-Id: Idc84401ea06ae0bcc26b6a3a15445701c9f52fb1
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1172704
TryBot-Result: CUEcueckoo <[email protected]>
Reviewed-by: Daniel Martí <[email protected]>
Unity-Result: CUE porcuepine <[email protected]>

internal/mod/modimports/modimports.go

@@ -0,0 +1,96 @@
+package modimports
+
+import (
+	"errors"
+	"io/fs"
+	"path"
+	"strings"
+
+	"cuelang.org/go/cue/ast"
+	"cuelang.org/go/cue/parser"
+	"cuelang.org/go/internal/cueimports"
+)
+
+type ModuleFile struct {
+	// FilePath holds the path of the module file
+	// relative to the root of the fs. This will be
+	// valid even if there's an associated error.
+	//
+	// If there's an error, it might not a be CUE file.
+	FilePath string
+
+	// Syntax includes only the portion of the file up to and including
+	// the imports. It will be nil if there was an error reading the file.
+	Syntax *ast.File
+}
+
+// AllModuleFiles returns an iterator that produces all the CUE files inside the
+// module at the given root.
+func AllModuleFiles(fsys fs.FS, root string) func(func(ModuleFile, error) bool) {
+	return func(yield func(ModuleFile, error) bool) {
+		fs.WalkDir(fsys, root, func(fpath string, d fs.DirEntry, err error) (_err error) {
+			if err != nil {
+				if !yield(ModuleFile{
+					FilePath: fpath,
+				}, err) {
+					return fs.SkipAll
+				}
+				return nil
+			}
+			if path.Base(fpath) == "cue.mod" {
+				return fs.SkipDir
+			}
+			if d.IsDir() {
+				if fpath == root {
+					return nil
+				}
+				base := path.Base(fpath)
+				if strings.HasPrefix(base, ".") || strings.HasPrefix(base, "_") {
+					return fs.SkipDir
+				}
+				_, err := fs.Stat(fsys, path.Join(fpath, "cue.mod"))
+				if err == nil {
+					// TODO is it enough to have a cue.mod directory
+					// or should we look for cue.mod/module.cue too?
+					return fs.SkipDir
+				}
+				if !errors.Is(err, fs.ErrNotExist) {
+					// We haven't got a package file to produce with the
+					// error here. Should we just ignore the error or produce
+					// a ModuleFile with an empty path?
+					yield(ModuleFile{}, err)
+					return fs.SkipAll
+				}
+				return nil
+			}
+			if !strings.HasSuffix(fpath, ".cue") {
+				return nil
+			}
+			if !yieldPackageFile(fsys, fpath, yield) {
+				return fs.SkipAll
+			}
+			return nil
+		})
+	}
+}
+
+func yieldPackageFile(fsys fs.FS, path string, yield func(ModuleFile, error) bool) bool {
+	pf := ModuleFile{
+		FilePath: path,
+	}
+	f, err := fsys.Open(path)
+	if err != nil {
+		return yield(pf, err)
+	}
+	defer f.Close()
+	data, err := cueimports.Read(f)
+	if err != nil {
+		return yield(pf, err)
+	}
+	syntax, err := parser.ParseFile(path, data, parser.ParseComments)
+	if err != nil {
+		return yield(pf, err)
+	}
+	pf.Syntax = syntax
+	return yield(pf, nil)
+}

internal/mod/modimports/modimports_test.go

@@ -0,0 +1,45 @@
+package modimports
+
+import (
+	"fmt"
+	"io/fs"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"cuelang.org/go/internal/txtarfs"
+	"github.com/go-quicktest/qt"
+	"github.com/google/go-cmp/cmp"
+	"golang.org/x/tools/txtar"
+)
+
+func TestAllPackageFiles(t *testing.T) {
+	files, err := filepath.Glob("testdata/*.txtar")
+	qt.Assert(t, qt.IsNil(err))
+	for _, f := range files {
+		t.Run(f, func(t *testing.T) {
+			ar, err := txtar.ParseFile(f)
+			qt.Assert(t, qt.IsNil(err))
+			tfs := txtarfs.FS(ar)
+			want, err := fs.ReadFile(tfs, "want")
+			qt.Assert(t, qt.IsNil(err))
+			iter := AllModuleFiles(tfs, ".")
+			var out strings.Builder
+			iter(func(pf ModuleFile, err error) bool {
+				out.WriteString(pf.FilePath)
+				if err != nil {
+					fmt.Fprintf(&out, ": error: %v\n", err)
+					return true
+				}
+				for _, imp := range pf.Syntax.Imports {
+					fmt.Fprintf(&out, " %s", imp.Path.Value)
+				}
+				out.WriteString("\n")
+				return true
+			})
+			if diff := cmp.Diff(string(want), out.String()); diff != "" {
+				t.Fatalf("unexpected results (-want +got):\n%s", diff)
+			}
+		})
+	}
+}

internal/mod/modimports/testdata/parseerror.txtar

@@ -0,0 +1,4 @@
+-- want --
+x.cue: error: expected label or ':', found 'IDENT' contents
+-- x.cue --
+bogus contents

internal/mod/modimports/testdata/simple.txtar

@@ -0,0 +1,43 @@
+-- want --
+x.cue "dep1" "dep2" "dep3"
+y/y1.cue
+y/y2.cue
+y/z1.cue
+y/z2.cue
+-- cue.mod/module.cue --
+module: "example.com"
+
+-- x.cue --
+package example
+
+import (
+	"dep1"
+	"dep2"
+)
+import "dep3"
+
+x: true
+-- y/y1.cue --
+package y
+
+
+-- y/y2.cue --
+package y
+
+-- y/z1.cue --
+package z
+
+-- y/z2.cue --
+package z
+
+-- _omitted1/foo.cue --
+not even looked at
+
+-- .omitted2/foo.cue --
+not looked at either
+
+-- z/cue.mod/module.cue --
+module "other.com"
+
+-- z/z.cue --
+package z