Vendor standard packages (#63)

To be able to compile coroutines in the standard library, we must first
move the package(s) to a location where we have write access and where
mutations aren't visible to other modules.

It's not possible to vendor standard packages under the standard
`./vendor` prefix. The Go compiler complains that it locates the
packages in two locations (`./vendor` and `$GOROOT`) and refuses to
choose one.

This PR updates our compiler to first copy standard library packages and
their dependencies into a `$MODULEDIR/goroot` directory, and also
organizes the directory so that it can be used as a Go root. The caller
can then compile their coroutines using standard Go tooling, they just
need to override the root, e.g.

```
GOROOT=$(pwd)/goroot go build ...
```

compiler/color.go

@@ -27,6 +27,13 @@ func colorFunctions(cg *callgraph.Graph, yieldInstances functionColors) (functio
 type functionColors map[*ssa.Function]*types.Signature
 
 func colorFunctions0(cg *callgraph.Graph, colors functionColors, fn *ssa.Function, color *types.Signature) error {
+	if origin := fn.Origin(); origin != nil && origin.Pkg != nil {
+		// Don't follow edges into and through the coroutine package.
+		if pkgPath := origin.Pkg.Pkg.Path(); pkgPath == coroutinePackage {
+			return nil
+		}
+	}
+
 	existing, ok := colors[fn]
 	if ok {
 		if !types.Identical(existing, color) {

compiler/compile.go

@@ -110,11 +110,16 @@ func (c *compiler) compile(path string) error {
 			return fmt.Errorf("pattern more than one module (%s + %s)", moduleDir, p.Module.Dir)
 		}
 	}
-	flatpkgs := flattenPackages(pkgs)
-	for _, p := range flatpkgs {
-		for _, err := range p.Errors {
-			return err
+	err = nil
+	packages.Visit(pkgs, func(p *packages.Package) bool {
+		for _, e := range p.Errors {
+			err = e
+			break
 		}
+		return err == nil
+	}, nil)
+	if err != nil {
+		return err
 	}
 
 	log.Printf("building SSA program")
@@ -126,12 +131,12 @@ func (c *compiler) compile(path string) error {
 
 	log.Printf("finding generic yield instantiations")
 	var coroutinePkg *packages.Package
-	for _, p := range flatpkgs {
+	packages.Visit(pkgs, func(p *packages.Package) bool {
 		if p.PkgPath == coroutinePackage {
 			coroutinePkg = p
-			break
 		}
-	}
+		return coroutinePkg == nil
+	}, nil)
 	if coroutinePkg == nil {
 		log.Printf("%s not imported by the module. Nothing to do", coroutinePackage)
 		return nil
@@ -150,9 +155,10 @@ func (c *compiler) compile(path string) error {
 		return err
 	}
 	pkgsByTypes := map[*types.Package]*packages.Package{}
-	for _, p := range flatpkgs {
+	packages.Visit(pkgs, func(p *packages.Package) bool {
 		pkgsByTypes[p.Types] = p
-	}
+		return true
+	}, nil)
 	colorsByPkg := map[*packages.Package]functionColors{}
 	for fn, color := range colors {
 		if fn.Pkg == nil {
@@ -168,10 +174,22 @@ func (c *compiler) compile(path string) error {
 		pkgColors[fn] = color
 	}
 
-	for p, colors := range colorsByPkg {
+	var needVendoring []*packages.Package
+	for p := range colorsByPkg {
 		if p.Module == nil || p.Module.Dir != moduleDir {
-			return fmt.Errorf("not implemented: compilation for packages outside module (need to compile %s)", p.PkgPath)
+			needVendoring = append(needVendoring, p)
+			break
+		}
+	}
+	if len(needVendoring) > 0 {
+		log.Printf("vendoring packages")
+		newRoot := filepath.Join(moduleDir, "goroot")
+		if err := vendor(newRoot, needVendoring); err != nil {
+			return err
 		}
+	}
+
+	for p, colors := range colorsByPkg {
 		if err := c.compilePackage(p, colors, prog); err != nil {
 			return err
 		}

compiler/go.mod

@@ -5,6 +5,7 @@ go 1.21.0
 require (
 	github.com/google/go-cmp v0.5.9
 	github.com/stealthrocket/coroutine v0.0.0-20230906012022-7474cda88ddc
+	golang.org/x/sync v0.3.0
 	golang.org/x/tools v0.13.0
 )
 

compiler/testdata/http/.gitignore

@@ -0,0 +1 @@
+/goroot