Remove YB CLI container logic to use Narwhal; go 1.13 (#63)

* Remove YB CLI container logic to use Narwhal -- updates in Narwhal support Darwin port checks properly

.yourbase.yml

@@ -1,6 +1,6 @@
 dependencies:
   build:
-    - go:1.12.10
+    - go:1.13
     - python:3.6
 
 build_targets:

cli/build.go

@@ -19,14 +19,14 @@ import (
 	"time"
 
 	"github.com/johnewart/archiver"
+	"github.com/johnewart/narwhal"
 	"github.com/johnewart/subcommands"
 
 	ybconfig "github.com/yourbase/yb/config"
 	. "github.com/yourbase/yb/packages"
 	. "github.com/yourbase/yb/plumbing"
 	"github.com/yourbase/yb/plumbing/log"
 	. "github.com/yourbase/yb/types"
-	. "github.com/yourbase/yb/workspace"
 )
 
 const TIME_FORMAT = "15:04:05 MST"
@@ -93,6 +93,7 @@ func (b *BuildCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
 	}
 
 	manifest := targetPackage.Manifest
+	workDir := targetPackage.BuildRoot()
 
 	// Determine build target
 	buildTargetName := "default"
@@ -123,17 +124,17 @@ func (b *BuildCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
 		// Setup dependencies
 		containers := target.Dependencies.Containers
 
-		if len(containers) > 0 && !b.NoSideContainer {
-			contextId := fmt.Sprintf("%s-%s", targetPackage.Name, target.Name)
-			log.Infof("Starting %d containers with context id %s...", len(containers), contextId)
-			sc, err := NewServiceContextWithId(contextId, targetPackage, target.Dependencies.ContainerList())
-			if err != nil {
-				log.Errorf("Couldn't create service context for dependencies: %v", err)
-				return subcommands.ExitFailure
-			}
+		contextId := fmt.Sprintf("%s-%s", targetPackage.Name, target.Name)
+		sc, err := narwhal.NewServiceContextWithId(contextId, workDir)
+		if err != nil {
+			log.Errorf("Couldn't create service context for dependencies: %v", err)
+			return subcommands.ExitFailure
+		}
 
-			buildData.Containers.ServiceContext = sc
+		buildData.Containers.ServiceContext = sc
 
+		if len(containers) > 0 && !b.NoSideContainer {
+			log.Infof("Starting %d containers with context id %s...", len(containers), contextId)
 			if !b.ReuseContainers {
 				log.Infof("Not reusing containers, will teardown existing ones and clean up after ourselves")
 				defer Cleanup(buildData)
@@ -143,9 +144,12 @@ func (b *BuildCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
 				}
 			}
 
-			if err = sc.StandUp(); err != nil {
-				log.Errorf("Couldn't start dependencies: %v", err)
-				return subcommands.ExitFailure
+			for _, c := range containers {
+				_, err := sc.StartContainer(c)
+				if err != nil {
+					log.Errorf("Couldn't start dependencies: %v", err)
+					return subcommands.ExitFailure
+				}
 			}
 
 		}
@@ -171,29 +175,30 @@ func (b *BuildCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
 		log.Infof("Executing build steps in container")
 
 		target := primaryTarget
-		container := target.Container
-		container.Command = "/usr/bin/tail -f /dev/null"
+		containerDef := target.Container
+		containerDef.Command = "/usr/bin/tail -f /dev/null"
 
 		// Append build environment variables
 		//container.Environment = append(container.Environment, buildData.EnvironmentVariables()...)
-		container.Environment = []string{}
+		containerDef.Environment = []string{}
+
+		// Add package to mounts @ /workspace
+		sourceMapDir := "/workspace"
+		if containerDef.WorkDir != "" {
+			sourceMapDir = containerDef.WorkDir
+		}
+
+		log.Infof("Will mount package %s at %s in container", targetPackage.Path, sourceMapDir)
+		mount := fmt.Sprintf("%s:%s", targetPackage.Path, sourceMapDir)
+		containerDef.Mounts = append(containerDef.Mounts, mount)
 
 		if false {
 			u, _ := user.Current()
 			fmt.Printf("U! %s", u.Uid)
 		}
 
-		containerOpts := BuildContainerOpts{
-			ContainerOpts: container,
-			Package:       targetPackage,
-			Target:        target.Name,
-			//ExecUserId:    u.Uid,
-			//ExecGroupId:   u.Gid,
-			MountPackage: true,
-		}
-
 		// Do our build in a container - don't do the build locally
-		buildErr := b.BuildInsideContainer(target, containerOpts, buildData, b.ExecPrefix, b.ReuseContainers)
+		buildErr := b.BuildInsideContainer(target, containerDef, buildData, b.ExecPrefix, b.ReuseContainers)
 		if buildErr != nil {
 			log.Errorf("Unable to build %s inside container: %v", target.Name, buildErr)
 			return subcommands.ExitFailure
@@ -354,11 +359,11 @@ func Cleanup(b BuildData) {
 	}
 }
 
-func (b *BuildCmd) BuildInsideContainer(target BuildTarget, containerOpts BuildContainerOpts, buildData BuildData, execPrefix string, reuseOldContainer bool) error {
+func (b *BuildCmd) BuildInsideContainer(target BuildTarget, containerDef narwhal.ContainerDefinition, buildData BuildData, execPrefix string, reuseOldContainer bool) error {
 	// Perform build inside a container
 	image := target.Container.Image
 	log.Infof("Using container image: %s", image)
-	buildContainer, err := FindContainer(containerOpts)
+	buildContainer, err := buildData.Containers.ServiceContext.StartContainer(containerDef)
 
 	if err != nil {
 		return fmt.Errorf("Failed trying to find container: %v", err)
@@ -372,7 +377,7 @@ func (b *BuildCmd) BuildInsideContainer(target BuildTarget, containerOpts BuildC
 		if !reuseOldContainer {
 			log.Infof("Elected to not re-use containers, will remove the container")
 			// Try stopping it first if needed
-			if err = RemoveContainerById(buildContainer.Id); err != nil {
+			if err = narwhal.RemoveContainerById(buildContainer.Id); err != nil {
 				return fmt.Errorf("Unable to remove existing container: %v", err)
 			}
 		}
@@ -382,10 +387,10 @@ func (b *BuildCmd) BuildInsideContainer(target BuildTarget, containerOpts BuildC
 		log.Infof("Reusing existing build container %s", buildContainer.Id[0:12])
 	} else {
 		log.Infof("Creating a new build container...")
-		if c, err := NewContainer(containerOpts); err != nil {
+		if c, err := buildData.Containers.ServiceContext.StartContainer(containerDef); err != nil {
 			return fmt.Errorf("Couldn't create a new build container: %v", err)
 		} else {
-			buildContainer = &c
+			buildContainer = c
 		}
 	}
 
@@ -409,8 +414,8 @@ func (b *BuildCmd) BuildInsideContainer(target BuildTarget, containerOpts BuildC
 
 	// Set container work dir
 	containerWorkDir := "/workspace"
-	if containerOpts.ContainerOpts.WorkDir != "" {
-		containerWorkDir = containerOpts.ContainerOpts.WorkDir
+	if containerDef.WorkDir != "" {
+		containerWorkDir = containerDef.WorkDir
 	}
 
 	log.Infof("Building from %s in container: %s", containerWorkDir, buildContainer.Id)
@@ -618,13 +623,13 @@ func SetupBuildDependencies(pkg Package) (TargetTimer, error) {
 
 // Build metadata; used for things like interpolation in environment variables
 type ContainerData struct {
-	ServiceContext *ServiceContext
+	ServiceContext *narwhal.ServiceContext
 }
 
 func (c ContainerData) IP(label string) string {
 	// Check service context
 	if c.ServiceContext != nil {
-		if buildContainer, ok := c.ServiceContext.BuildContainers[label]; ok {
+		if buildContainer, ok := c.ServiceContext.Containers[label]; ok {
 			if ipv4, err := buildContainer.IPv4Address(); err == nil {
 				return ipv4
 			}
@@ -643,7 +648,7 @@ func (c ContainerData) IP(label string) string {
 func (c ContainerData) Environment() map[string]string {
 	result := make(map[string]string)
 	if c.ServiceContext != nil {
-		for label, container := range c.ServiceContext.BuildContainers {
+		for label, container := range c.ServiceContext.Containers {
 			if ipv4, err := container.IPv4Address(); err == nil {
 				key := fmt.Sprintf("YB_CONTAINER_%s_IP", strings.ToUpper(label))
 				result[key] = ipv4

cli/exec.go

@@ -3,13 +3,14 @@ package cli
 import (
 	"context"
 	"flag"
+	"path/filepath"
 	"strings"
 
+	"github.com/johnewart/narwhal"
 	"github.com/johnewart/subcommands"
 
 	. "github.com/yourbase/yb/plumbing"
 	"github.com/yourbase/yb/plumbing/log"
-	. "github.com/yourbase/yb/workspace"
 )
 
 type ExecCmd struct {
@@ -55,16 +56,24 @@ func (b *ExecCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
 
 	if len(containers) > 0 {
 		log.ActiveSection("Containers")
+		localContainerWorkDir := filepath.Join(targetPackage.BuildRoot(), "containers")
+		MkdirAsNeeded(localContainerWorkDir)
+
+		log.Infof("Will use %s as the dependency work dir", localContainerWorkDir)
 		log.Infof("Starting %d dependencies...", len(containers))
-		sc, err := NewServiceContextWithId("exec", targetPackage, containers)
+		sc, err := narwhal.NewServiceContextWithId("exec", targetPackage.BuildRoot())
 		if err != nil {
 			log.Errorf("Couldn't create service context for dependencies: %v", err)
 			return subcommands.ExitFailure
 		}
 
-		if err = sc.StandUp(); err != nil {
-			log.Infof("Couldn't start dependencies: %v\n", err)
-			return subcommands.ExitFailure
+		for _, c := range containers {
+			// TODO: avoid setting these here
+			c.LocalWorkDir = localContainerWorkDir
+			if _, err = sc.StartContainer(c); err != nil {
+				log.Infof("Couldn't start dependencies: %v\n", err)
+				return subcommands.ExitFailure
+			}
 		}
 
 		buildData.Containers.ServiceContext = sc

go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f // indirect
 	github.com/johnewart/archiver v3.1.4+incompatible
 	github.com/johnewart/go-dockerclient v1.3.7
+	github.com/johnewart/narwhal v0.0.0-20191030181313-9828f6c923b0
 	github.com/johnewart/subcommands v0.0.0-20181012225330-46f0354f6315
 	github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c // indirect
 	github.com/juju/errors v0.0.0-20190930114154-d42613fe1ab9 // indirect
@@ -41,7 +42,6 @@ require (
 	golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271 // indirect
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
 	golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c // indirect
-	google.golang.org/appengine v1.4.0 // indirect
 	google.golang.org/genproto v0.0.0-20191028173616-919d9bdd9fe6 // indirect
 	google.golang.org/grpc v1.24.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
@@ -51,3 +51,5 @@ require (
 	gopkg.in/src-d/go-git.v4 v4.13.1
 	gopkg.in/yaml.v2 v2.2.4
 )
+
+go 1.13

go.sum

@@ -98,6 +98,10 @@ github.com/johnewart/archiver v3.1.4+incompatible h1:509GRUpPDmAvOLIeqRa1LnbhGvz
 github.com/johnewart/archiver v3.1.4+incompatible/go.mod h1:RZZSHcya+2fNoD31xh/kmPmwCa7hDaxCm3dFNlPDmaY=
 github.com/johnewart/go-dockerclient v1.3.7 h1:oyTFh8XsnQ/XdnK8D6RGHQYQSZmZ05sg2BbHbVzwIKQ=
 github.com/johnewart/go-dockerclient v1.3.7/go.mod h1:FCtHjONdIZH2BKTKbXFJ2Qp8h31iQkiXNboLI3M/TYg=
+github.com/johnewart/narwhal v0.0.0-20191030172955-5a51cfc5963a h1:CMttmUMYzeqbuex8Zk+hGIzJ1qbeEFfI54Z/f4ahsYM=
+github.com/johnewart/narwhal v0.0.0-20191030172955-5a51cfc5963a/go.mod h1:EsUUI8gi1HFwS6NoGEocjpErZEK2ER+b+D0b1qxKM/g=
+github.com/johnewart/narwhal v0.0.0-20191030181313-9828f6c923b0 h1:Hrt19BwfJaNsEK29v32epT4+ZPDfL7dZ2vHbkAKvBIQ=
+github.com/johnewart/narwhal v0.0.0-20191030181313-9828f6c923b0/go.mod h1:EsUUI8gi1HFwS6NoGEocjpErZEK2ER+b+D0b1qxKM/g=
 github.com/johnewart/subcommands v0.0.0-20181012225330-46f0354f6315 h1:IHd01G1+fdwW9G/AraiEZzx6GNbGdntaICZjP5PI+oM=
 github.com/johnewart/subcommands v0.0.0-20181012225330-46f0354f6315/go.mod h1:P9HfiwoEAZ8bQktefnfAuKGlAYDuwfmBqt/cgIBGt+M=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=