Skip to content

Commit

Permalink
internal/registrytest: rewrite in terms of io/fs
Browse files Browse the repository at this point in the history
This way, we can use it in the context of a testscript in cmd/cue;
testscript.Setup doesn't give us direct access to the txtar.Archive,
but it does extract it to the working directory, so we can use os.DirFS.

Signed-off-by: Daniel Martí <[email protected]>
Change-Id: I04fe783d6d16e802c1bd62f413877055bd11655b
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1169976
Reviewed-by: Roger Peppe <[email protected]>
Unity-Result: CUE porcuepine <[email protected]>
TryBot-Result: CUEcueckoo <[email protected]>
  • Loading branch information
mvdan committed Sep 26, 2023
1 parent 633874c commit 5f19042
Show file tree
Hide file tree
Showing 4 changed files with 237 additions and 12 deletions.
2 changes: 1 addition & 1 deletion cue/load/module_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ func TestModuleFetch(t *testing.T) {
Name: "modfetch",
}
test.Run(t, func(t *cuetxtar.Test) {
r, err := registrytest.New(t.Archive)
r, err := registrytest.New(registrytest.TxtarFS(t.Archive))
if err != nil {
t.Fatal(err)
}
Expand Down
36 changes: 26 additions & 10 deletions internal/registrytest/registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@ package registrytest
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"io/fs"
"net/http/httptest"
"strings"

Expand All @@ -28,13 +30,13 @@ import (
// contain a cue.mod/module.cue file holding the module info.
//
// The Registry should be closed after use.
func New(ar *txtar.Archive) (*Registry, error) {
func New(fsys fs.FS) (*Registry, error) {
srv := httptest.NewServer(ociserver.New(ocimem.New(), nil))
client, err := modregistry.NewClient(srv.URL, "cue/")
if err != nil {
return nil, fmt.Errorf("cannot make client: %v", err)
}
mods, err := getModules(ar)
mods, err := getModules(fsys)
if err != nil {
return nil, fmt.Errorf("invalid modules: %v", err)
}
Expand Down Expand Up @@ -100,27 +102,41 @@ type handler struct {
modules []*moduleContent
}

func getModules(ar *txtar.Archive) (map[module.Version]*moduleContent, error) {
func getModules(fsys fs.FS) (map[module.Version]*moduleContent, error) {
ctx := cuecontext.New()
modules := make(map[string]*moduleContent)
for _, f := range ar.Files {
path := strings.TrimPrefix(f.Name, "_registry/")
if len(path) == len(f.Name) {
continue
if err := fs.WalkDir(fsys, "_registry", func(path string, d fs.DirEntry, err error) error {
if err != nil {
// If a filesystem has no _registry directory at all,
// return zero modules without an error.
if path == "_registry" && errors.Is(err, fs.ErrNotExist) {
return fs.SkipAll
}
return err
}
modver, rest, ok := strings.Cut(path, "/")
if d.IsDir() {
return nil // we're only interested in regular files, not their parent directories
}
modver, rest, ok := strings.Cut(strings.TrimPrefix(path, "_registry/"), "/")
if !ok {
return nil, fmt.Errorf("_registry should only contain directories, but found regular file %q", path)
return fmt.Errorf("_registry should only contain directories, but found regular file %q", path)
}
content := modules[modver]
if content == nil {
content = &moduleContent{}
modules[modver] = content
}
data, err := fs.ReadFile(fsys, path)
if err != nil {
return err
}
content.files = append(content.files, txtar.File{
Name: rest,
Data: f.Data,
Data: data,
})
return nil
}); err != nil {
return nil, err
}
for modver, content := range modules {
if err := content.init(ctx, modver); err != nil {
Expand Down
2 changes: 1 addition & 1 deletion internal/registrytest/registry_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func TestRegistry(t *testing.T) {
t.Fatal(err)
}
t.Run(strings.TrimSuffix(name, ".txtar"), func(t *testing.T) {
r, err := New(ar)
r, err := New(TxtarFS(ar))
if err != nil {
t.Fatal(err)
}
Expand Down
209 changes: 209 additions & 0 deletions internal/registrytest/txtar_fs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,209 @@
// Borrowed from https://github.com/golang/tools/pull/289.
// TODO(mvdan): replace by x/tools/txtar once the proposal at
// https://github.com/golang/go/issues/44158 is accepted and implemented.

// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package registrytest

import (
"bytes"
"errors"
"io"
"io/fs"
"path"
"strings"
"time"

"golang.org/x/tools/txtar"
)

// TxtarFS returns an fs.FS which reads from a txtar.Archive.
func TxtarFS(ar *txtar.Archive) fs.FS {
return archiveFS{ar}
}

type archiveFS struct {
a *txtar.Archive
}

// Open implements fs.FS.
func (fsys archiveFS) Open(name string) (fs.File, error) {
if !fs.ValidPath(name) {
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
}

for _, f := range fsys.a.Files {
// In case the txtar has weird filenames
cleanName := path.Clean(f.Name)
if name == cleanName {
return newOpenFile(f), nil
}
}
var entries []fileInfo
dirs := make(map[string]bool)
prefix := name + "/"
if name == "." {
prefix = ""
}

for _, f := range fsys.a.Files {
cleanName := path.Clean(f.Name)
if !strings.HasPrefix(cleanName, prefix) {
continue
}
felem := cleanName[len(prefix):]
i := strings.Index(felem, "/")
if i < 0 {
entries = append(entries, newFileInfo(f))
} else {
dirs[felem[:i]] = true
}
}
// If there are no children of the name,
// then the directory is treated as not existing
// unless the directory is "."
if len(entries) == 0 && len(dirs) == 0 && name != "." {
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
}

for name := range dirs {
entries = append(entries, newDirInfo(name))
}

return &openDir{newDirInfo(name), entries, 0}, nil
}

var _ fs.ReadFileFS = archiveFS{}

// ReadFile implements fs.ReadFileFS.
func (fsys archiveFS) ReadFile(name string) ([]byte, error) {
if !fs.ValidPath(name) {
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
}
if name == "." {
return nil, &fs.PathError{Op: "read", Path: name, Err: errors.New("is a directory")}
}
prefix := name + "/"
for _, f := range fsys.a.Files {
if cleanName := path.Clean(f.Name); name == cleanName {
return append(([]byte)(nil), f.Data...), nil
}
// It's a directory
if strings.HasPrefix(f.Name, prefix) {
return nil, &fs.PathError{Op: "read", Path: name, Err: errors.New("is a directory")}
}
}
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
}

var (
_ fs.File = (*openFile)(nil)
_ io.ReadSeekCloser = (*openFile)(nil)
_ io.ReaderAt = (*openFile)(nil)
_ io.WriterTo = (*openFile)(nil)
)

type openFile struct {
bytes.Reader
fi fileInfo
}

func newOpenFile(f txtar.File) *openFile {
var o openFile
o.Reader.Reset(f.Data)
o.fi = fileInfo{f, 0444}
return &o
}

func (o *openFile) Stat() (fs.FileInfo, error) { return o.fi, nil }

func (o *openFile) Close() error { return nil }

var _ fs.FileInfo = fileInfo{}

type fileInfo struct {
f txtar.File
m fs.FileMode
}

func newFileInfo(f txtar.File) fileInfo {
return fileInfo{f, 0444}
}

func newDirInfo(name string) fileInfo {
return fileInfo{txtar.File{Name: name}, fs.ModeDir | 0555}
}

func (f fileInfo) Name() string { return path.Base(f.f.Name) }
func (f fileInfo) Size() int64 { return int64(len(f.f.Data)) }
func (f fileInfo) Mode() fs.FileMode { return f.m }
func (f fileInfo) Type() fs.FileMode { return f.m.Type() }
func (f fileInfo) ModTime() time.Time { return time.Time{} }
func (f fileInfo) IsDir() bool { return f.m.IsDir() }
func (f fileInfo) Sys() interface{} { return f.f }
func (f fileInfo) Info() (fs.FileInfo, error) { return f, nil }

type openDir struct {
dirInfo fileInfo
entries []fileInfo
offset int
}

func (d *openDir) Stat() (fs.FileInfo, error) { return &d.dirInfo, nil }
func (d *openDir) Close() error { return nil }
func (d *openDir) Read(b []byte) (int, error) {
return 0, &fs.PathError{Op: "read", Path: d.dirInfo.f.Name, Err: errors.New("is a directory")}
}

func (d *openDir) ReadDir(count int) ([]fs.DirEntry, error) {
n := len(d.entries) - d.offset
if count > 0 && n > count {
n = count
}
if n == 0 && count > 0 {
return nil, io.EOF
}
entries := make([]fs.DirEntry, n)
for i := range entries {
entries[i] = &d.entries[d.offset+i]
}
d.offset += n
return entries, nil
}

// From constructs an Archive with the contents of fsys and an empty Comment.
// Subsequent changes to fsys are not reflected in the returned archive.
//
// The transformation is lossy.
// For example, because directories are implicit in txtar archives,
// empty directories in fsys will be lost,
// and txtar does not represent file mode, mtime, or other file metadata.
// From does not guarantee that a.File[i].Data contains no file marker lines.
// See also warnings on Format.
// In short, it is unwise to use txtar as a generic filesystem serialization mechanism.
func From(fsys fs.FS) (*txtar.Archive, error) {
ar := new(txtar.Archive)
walkfn := func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
// Directories in txtar are implicit.
return nil
}
data, err := fs.ReadFile(fsys, path)
if err != nil {
return err
}
ar.Files = append(ar.Files, txtar.File{Name: path, Data: data})
return nil
}

if err := fs.WalkDir(fsys, ".", walkfn); err != nil {
return nil, err
}
return ar, nil
}

0 comments on commit 5f19042

Please sign in to comment.