From f6ce92478d9c62719370db6bc1ba0aca0b1e2f25 Mon Sep 17 00:00:00 2001 From: Ridai Govinda Pombo Date: Fri, 15 May 2020 10:46:04 -0300 Subject: [PATCH] New runtime context --- .gitignore | 2 + .yourbase.yml | 2 +- buildpacks/anaconda.go | 8 +- buildpacks/android_ndk.go | 11 +- buildpacks/android_sdk.go | 27 +- buildpacks/ant.go | 13 +- buildpacks/dart.go | 5 +- buildpacks/flutter.go | 13 +- buildpacks/glide.go | 15 +- buildpacks/golang.go | 49 +- buildpacks/gradle.go | 18 +- buildpacks/heroku.go | 18 +- buildpacks/homebrew.go | 11 +- buildpacks/maven.go | 16 +- buildpacks/nodejs.go | 17 +- buildpacks/openjdk.go | 11 +- buildpacks/protoc.go | 6 +- buildpacks/python.go | 53 +- buildpacks/rlang.go | 19 +- buildpacks/ruby.go | 44 +- buildpacks/rust.go | 21 +- buildpacks/tool_spec.go | 32 + buildpacks/yarn.go | 5 +- cli/build.go | 571 +----------------- cli/exec.go | 84 +-- cli/helpers.go | 1 - cli/ls.go | 38 ++ cli/package.go | 139 ++++- cli/remote_build.go | 16 +- cli/run.go | 65 +- cli/workspace.go | 3 +- config/settings.go | 12 + go.mod | 45 +- go.sum | 166 +++-- packages/package.go | 91 --- plumbing/util.go | 264 +------- runtime/common.go | 167 +++++ runtime/containers.go | 333 ++++++++++ runtime/metal.go | 387 ++++++++++++ runtime/runtime.go | 189 ++++++ {plumbing => runtime}/sandbox_darwin.go | 4 +- {plumbing => runtime}/sandbox_linux.go | 2 +- {plumbing => runtime}/sandbox_windows.go | 2 +- types/build_manifest.go | 75 --- types/types.go | 99 +-- workspace/build_manifest.go | 202 +++++++ workspace/build_target.go | 216 +++++++ {buildpacks => workspace}/buildpack_loader.go | 68 +-- workspace/package.go | 236 ++++++++ workspace/workspace.go | 16 +- 50 files changed, 2390 insertions(+), 1517 deletions(-) create mode 100644 buildpacks/tool_spec.go create mode 100644 cli/ls.go delete mode 100644 packages/package.go create mode 100644 runtime/common.go create mode 100644 runtime/containers.go create mode 100644 runtime/metal.go create mode 100644 runtime/runtime.go rename {plumbing => runtime}/sandbox_darwin.go (95%) rename {plumbing => runtime}/sandbox_linux.go (89%) rename {plumbing => runtime}/sandbox_windows.go (89%) delete mode 100644 types/build_manifest.go create mode 100644 workspace/build_manifest.go create mode 100644 workspace/build_target.go rename {buildpacks => workspace}/buildpack_loader.go (54%) create mode 100644 workspace/package.go diff --git a/.gitignore b/.gitignore index 0385beb8..be6807d8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ yb .DS_Store release .vscode +.env +.vscode diff --git a/.yourbase.yml b/.yourbase.yml index 38b16a1e..d925973c 100644 --- a/.yourbase.yml +++ b/.yourbase.yml @@ -1,6 +1,6 @@ dependencies: build: - - go:1.13.4 + - go:1.14.2 - python:3.6 build_targets: diff --git a/buildpacks/anaconda.go b/buildpacks/anaconda.go index 30ef49a3..0e0d162c 100644 --- a/buildpacks/anaconda.go +++ b/buildpacks/anaconda.go @@ -7,6 +7,7 @@ import ( . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" + "github.com/yourbase/yb/runtime" . "github.com/yourbase/yb/types" ) @@ -50,6 +51,7 @@ func (bt AnacondaBuildTool) InstallDir() string { func (bt AnacondaBuildTool) Install() error { anacondaDir := bt.InstallDir() setupDir := bt.spec.PackageDir + t := bt.spec.InstallTarget if _, err := os.Stat(anacondaDir); err == nil { log.Infof("anaconda installed in %s", anacondaDir) @@ -59,7 +61,7 @@ func (bt AnacondaBuildTool) Install() error { downloadUrl := bt.DownloadUrl() log.Infof("Downloading Miniconda from URL %s...", downloadUrl) - localFile, err := DownloadFileWithCache(downloadUrl) + localFile, err := t.DownloadFile(downloadUrl) if err != nil { log.Errorf("Unable to download: %v\n", err) return err @@ -69,7 +71,7 @@ func (bt AnacondaBuildTool) Install() error { fmt.Sprintf("bash %s -b -p %s", localFile, anacondaDir), } { log.Infof("Running: '%v' ", cmd) - ExecToStdout(cmd, setupDir) + runtime.ExecToStdout(cmd, setupDir) } } @@ -133,7 +135,7 @@ func (bt AnacondaBuildTool) Setup() error { fmt.Sprintf("conda update -q conda"), } { log.Infof("Running: '%v' ", cmd) - ExecToStdout(cmd, setupDir) + runtime.ExecToStdout(cmd, setupDir) } return nil diff --git a/buildpacks/android_ndk.go b/buildpacks/android_ndk.go index 002e9dd2..976a79a3 100644 --- a/buildpacks/android_ndk.go +++ b/buildpacks/android_ndk.go @@ -2,11 +2,10 @@ package buildpacks import ( "fmt" + "github.com/yourbase/yb/runtime" "os" "path/filepath" - "github.com/johnewart/archiver" - . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" . "github.com/yourbase/yb/types" ) @@ -63,7 +62,7 @@ func (bt AndroidNdkBuildTool) Version() string { } func (bt AndroidNdkBuildTool) InstallDir() string { - return filepath.Join(ToolsDir(), "android-ndk") + return filepath.Join(bt.spec.InstallTarget.ToolsDir(), "android-ndk") } func (bt AndroidNdkBuildTool) NdkDir() string { @@ -74,7 +73,7 @@ func (bt AndroidNdkBuildTool) Setup() error { ndkDir := bt.NdkDir() log.Infof("Setting ANDROID_NDK_HOME to %s", ndkDir) - os.Setenv("ANDROID_NDK_HOME", ndkDir) + runtime.SetEnv("ANDROID_NDK_HOME", ndkDir) return nil } @@ -91,12 +90,12 @@ func (bt AndroidNdkBuildTool) Install() error { downloadUrl := bt.DownloadUrl() log.Infof("Downloading Android NDK v%s from URL %s...", bt.Version(), downloadUrl) - localFile, err := DownloadFileWithCache(downloadUrl) + localFile, err := bt.spec.InstallTarget.DownloadFile(downloadUrl) if err != nil { log.Errorf("Unable to download: %v", err) return err } - err = archiver.Unarchive(localFile, installDir) + err = bt.spec.InstallTarget.Unarchive(localFile, installDir) if err != nil { log.Errorf("Unable to decompress: %v", err) return err diff --git a/buildpacks/android_sdk.go b/buildpacks/android_sdk.go index 640a722c..ba825753 100644 --- a/buildpacks/android_sdk.go +++ b/buildpacks/android_sdk.go @@ -2,11 +2,11 @@ package buildpacks import ( "fmt" + "github.com/yourbase/yb/runtime" "os" "path/filepath" "strings" - "github.com/johnewart/archiver" . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" . "github.com/yourbase/yb/types" @@ -74,7 +74,7 @@ func (bt AndroidBuildTool) Version() string { } func (bt AndroidBuildTool) InstallDir() string { - return filepath.Join(ToolsDir(), "android", fmt.Sprintf("android-%s", bt.Version())) + return filepath.Join(bt.spec.InstallTarget.ToolsDir(), "android", fmt.Sprintf("android-%s", bt.Version())) } func (bt AndroidBuildTool) AndroidDir() string { @@ -121,25 +121,18 @@ func (bt AndroidBuildTool) WriteAgreements() bool { func (bt AndroidBuildTool) Setup() error { androidDir := bt.AndroidDir() - - binPath := fmt.Sprintf("%s/tools/bin", androidDir) - toolsPath := fmt.Sprintf("%s/tools", androidDir) - currentPath := os.Getenv("PATH") - newPath := fmt.Sprintf("%s:%s", binPath, currentPath) - newPath = fmt.Sprintf("%s:%s", toolsPath, newPath) - log.Infof("Setting PATH to %s", newPath) - os.Setenv("PATH", newPath) - - //fmt.Printf("Setting ANDROID_HOME to %s\n", androidHomeDir) - //os.Setenv("ANDROID_HOME", androidHomeDir) + t := bt.spec.InstallTarget log.Infof("Setting ANDROID_SDK_ROOT to %s", androidDir) - os.Setenv("ANDROID_SDK_ROOT", androidDir) - os.Setenv("ANDROID_HOME", androidDir) + runtime.SetEnv("ANDROID_SDK_ROOT", androidDir) + runtime.SetEnv("ANDROID_HOME", androidDir) log.Infof("Writing agreement hashes...") bt.WriteAgreements() + t.PrependToPath(filepath.Join(androidDir, "tools")) + t.PrependToPath(filepath.Join(androidDir, "tools", "bin")) + return nil } @@ -155,12 +148,12 @@ func (bt AndroidBuildTool) Install() error { downloadUrl := bt.DownloadUrl() log.Infof("Downloading Android from URL %s...", downloadUrl) - localFile, err := DownloadFileWithCache(downloadUrl) + localFile, err := bt.spec.InstallTarget.DownloadFile(downloadUrl) if err != nil { log.Errorf("Unable to download: %v", err) return err } - err = archiver.Unarchive(localFile, installDir) + err = bt.spec.InstallTarget.Unarchive(localFile, installDir) if err != nil { log.Errorf("Unable to decompress: %v", err) return err diff --git a/buildpacks/ant.go b/buildpacks/ant.go index b48b6221..d08eefe8 100644 --- a/buildpacks/ant.go +++ b/buildpacks/ant.go @@ -6,8 +6,6 @@ import ( "path/filepath" "strings" - "github.com/johnewart/archiver" - . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" . "github.com/yourbase/yb/types" ) @@ -66,12 +64,9 @@ func (bt AntBuildTool) InstallDir() string { func (bt AntBuildTool) Setup() error { antDir := bt.AntDir() + t := bt.spec.InstallTarget - cmdPath := filepath.Join(antDir, "bin") - currentPath := os.Getenv("PATH") - newPath := fmt.Sprintf("%s:%s", cmdPath, currentPath) - log.Infof("Setting PATH to %s", newPath) - os.Setenv("PATH", newPath) + t.PrependToPath(filepath.Join(antDir, "bin")) return nil } @@ -88,12 +83,12 @@ func (bt AntBuildTool) Install() error { downloadUrl := bt.DownloadUrl() log.Infof("Downloading Ant from URL %s...", downloadUrl) - localFile, err := DownloadFileWithCache(downloadUrl) + localFile, err := bt.spec.InstallTarget.DownloadFile(downloadUrl) if err != nil { log.Errorf("Unable to download: %v", err) return err } - err = archiver.Unarchive(localFile, installDir) + err = bt.spec.InstallTarget.Unarchive(localFile, installDir) if err != nil { log.Errorf("Unable to decompress: %v", err) return err diff --git a/buildpacks/dart.go b/buildpacks/dart.go index 5ea84b24..67c20e3e 100644 --- a/buildpacks/dart.go +++ b/buildpacks/dart.go @@ -5,7 +5,6 @@ import ( "path/filepath" "strings" - "github.com/johnewart/archiver" . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" . "github.com/yourbase/yb/types" @@ -98,12 +97,12 @@ func (bt DartBuildTool) Install() error { downloadUrl := bt.DownloadUrl() log.Infof("Downloading Dart from URL %s...", downloadUrl) - localFile, err := DownloadFileWithCache(downloadUrl) + localFile, err := bt.spec.InstallTarget.DownloadFile(downloadUrl) if err != nil { log.Errorf("Unable to download: %v", err) return err } - err = archiver.Unarchive(localFile, installDir) + err = bt.spec.InstallTarget.Unarchive(localFile, installDir) if err != nil { log.Errorf("Unable to decompress: %v", err) return err diff --git a/buildpacks/flutter.go b/buildpacks/flutter.go index a980c68a..32a25fa7 100644 --- a/buildpacks/flutter.go +++ b/buildpacks/flutter.go @@ -6,10 +6,8 @@ import ( "path/filepath" "strings" - "github.com/johnewart/archiver" "golang.org/x/mod/semver" - . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" . "github.com/yourbase/yb/types" ) @@ -96,12 +94,9 @@ func (bt FlutterBuildTool) FlutterDir() string { func (bt FlutterBuildTool) Setup() error { flutterDir := bt.FlutterDir() + t := bt.spec.InstallTarget - cmdPath := filepath.Join(flutterDir, "bin") - currentPath := os.Getenv("PATH") - newPath := fmt.Sprintf("%s:%s", cmdPath, currentPath) - log.Infof("Setting PATH to %s", newPath) - os.Setenv("PATH", newPath) + t.PrependToPath(filepath.Join(flutterDir, "bin")) return nil } @@ -118,12 +113,12 @@ func (bt FlutterBuildTool) Install() error { downloadUrl := bt.DownloadUrl() log.Infof("Downloading Flutter from URL %s...", downloadUrl) - localFile, err := DownloadFileWithCache(downloadUrl) + localFile, err := bt.spec.InstallTarget.DownloadFile(downloadUrl) if err != nil { log.Errorf("Unable to download: %v", err) return err } - err = archiver.Unarchive(localFile, installDir) + err = bt.spec.InstallTarget.Unarchive(localFile, installDir) if err != nil { log.Errorf("Unable to decompress: %v", err) return err diff --git a/buildpacks/glide.go b/buildpacks/glide.go index 63327d78..44be169f 100644 --- a/buildpacks/glide.go +++ b/buildpacks/glide.go @@ -5,7 +5,6 @@ import ( "os" "path/filepath" - "github.com/johnewart/archiver" . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" . "github.com/yourbase/yb/types" @@ -44,11 +43,11 @@ func (bt GlideBuildTool) GlideDir() string { func (bt GlideBuildTool) Setup() error { glideDir := bt.GlideDir() - cmdPath := fmt.Sprintf("%s/%s-%s", glideDir, OS(), Arch()) - currentPath := os.Getenv("PATH") - newPath := fmt.Sprintf("%s:%s", cmdPath, currentPath) - log.Infof("Setting PATH to %s", newPath) - os.Setenv("PATH", newPath) + + t := bt.spec.InstallTarget + cmdPath := filepath.Join(glideDir, fmt.Sprintf("%s-%s", OS(), Arch())) + + t.PrependToPath(cmdPath) return nil } @@ -74,7 +73,7 @@ func (bt GlideBuildTool) Install() error { } log.Infof("Downloading from URL %s ...", downloadUrl) - localFile, err := DownloadFileWithCache(downloadUrl) + localFile, err := bt.spec.InstallTarget.DownloadFile(downloadUrl) if err != nil { log.Errorf("Unable to download: %v", err) return err @@ -83,7 +82,7 @@ func (bt GlideBuildTool) Install() error { extractDir := bt.GlideDir() MkdirAsNeeded(extractDir) log.Infof("Extracting glide %s to %s...", bt.Version(), extractDir) - err = archiver.Unarchive(localFile, extractDir) + err = bt.spec.InstallTarget.Unarchive(localFile, extractDir) if err != nil { log.Errorf("Unable to decompress: %v", err) return err diff --git a/buildpacks/golang.go b/buildpacks/golang.go index 515cced2..d11cdab4 100644 --- a/buildpacks/golang.go +++ b/buildpacks/golang.go @@ -2,12 +2,10 @@ package buildpacks import ( "fmt" - "os" + "github.com/yourbase/yb/runtime" "path/filepath" "strings" - "github.com/johnewart/archiver" - . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" . "github.com/yourbase/yb/types" ) @@ -31,9 +29,22 @@ func NewGolangBuildTool(toolSpec BuildToolSpec) GolangBuildTool { } func (bt GolangBuildTool) ArchiveFile() string { - operatingSystem := OS() - arch := Arch() - return fmt.Sprintf("go%s.%s-%s.tar.gz", bt.Version(), operatingSystem, arch) + operatingSystem := bt.spec.InstallTarget.OS() + arch := bt.spec.InstallTarget.Architecture() + os := "linux" + architecture := "amd64" + + if operatingSystem == runtime.Linux { + os = "linux" + } else { + os = "darwin" + } + + if arch == runtime.Amd64 { + architecture = "amd64" + } + + return fmt.Sprintf("go%s.%s-%s.tar.gz", bt.Version(), os, architecture) } func (bt GolangBuildTool) DownloadUrl() string { @@ -63,6 +74,8 @@ func (bt GolangBuildTool) GolangDir() string { // TODO: handle multiple packages, for now this is ok func (bt GolangBuildTool) Setup() error { + t := bt.spec.InstallTarget + golangDir := bt.GolangDir() goPath := bt.spec.PackageCacheDir pkgPath := bt.spec.PackageDir @@ -71,45 +84,41 @@ func (bt GolangBuildTool) Setup() error { goPathVar := strings.Join(goPathElements, ":") cmdPath := filepath.Join(golangDir, "bin") - PrependToPath(cmdPath) + t.PrependToPath(cmdPath) for _, pathElement := range goPathElements { pathBinDir := filepath.Join(pathElement, "bin") - PrependToPath(pathBinDir) + t.PrependToPath(pathBinDir) } log.Infof("Setting GOROOT to %s", golangDir) - os.Setenv("GOROOT", golangDir) + t.SetEnv("GOROOT", golangDir) log.Infof("Setting GOPATH to %s", goPath) - os.Setenv("GOPATH", goPathVar) + t.SetEnv("GOPATH", goPathVar) return nil } // TODO, generalize downloader func (bt GolangBuildTool) Install() error { + t := bt.spec.InstallTarget + installDir := bt.InstallDir() golangDir := bt.GolangDir() - if _, err := os.Stat(golangDir); err == nil { + if t.PathExists(golangDir) { log.Infof("Golang v%s located in %s!", bt.Version(), golangDir) } else { log.Infof("Will install Golang v%s into %s", bt.Version(), golangDir) downloadUrl := bt.DownloadUrl() log.Infof("Downloading from URL %s ...", downloadUrl) - localFile, err := DownloadFileWithCache(downloadUrl) - if err != nil { - log.Errorf("Unable to download: %v", err) - return err - } - err = archiver.Unarchive(localFile, installDir) + localFile, err := t.DownloadFile(downloadUrl) + + err = t.Unarchive(localFile, installDir) if err != nil { log.Errorf("Unable to decompress: %v", err) return err } - - log.Infof("Making go installation in %s read-only", golangDir) - RemoveWritePermissionRecursively(golangDir) } return nil diff --git a/buildpacks/gradle.go b/buildpacks/gradle.go index 7039241b..66ea20e5 100644 --- a/buildpacks/gradle.go +++ b/buildpacks/gradle.go @@ -6,9 +6,8 @@ import ( "path/filepath" "strings" - "github.com/johnewart/archiver" - . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" + "github.com/yourbase/yb/runtime" . "github.com/yourbase/yb/types" ) @@ -72,15 +71,12 @@ func (bt GradleBuildTool) InstallDir() string { func (bt GradleBuildTool) Setup() error { gradleDir := bt.GradleDir() gradleHome := filepath.Join(bt.spec.PackageCacheDir, "gradle-home", bt.Version()) - - cmdPath := filepath.Join(gradleDir, "bin") - currentPath := os.Getenv("PATH") - newPath := fmt.Sprintf("%s:%s", cmdPath, currentPath) - log.Infof("Setting PATH to %s", newPath) - os.Setenv("PATH", newPath) + t := bt.spec.InstallTarget log.Infof("Setting GRADLE_USER_HOME to %s", gradleHome) - os.Setenv("GRADLE_USER_HOME", gradleHome) + runtime.SetEnv("GRADLE_USER_HOME", gradleHome) + + t.PrependToPath(filepath.Join(gradleDir, "bin")) return nil } @@ -97,12 +93,12 @@ func (bt GradleBuildTool) Install() error { downloadUrl := bt.DownloadUrl() log.Infof("Downloading Gradle from URL %s...", downloadUrl) - localFile, err := DownloadFileWithCache(downloadUrl) + localFile, err := bt.spec.InstallTarget.DownloadFile(downloadUrl) if err != nil { log.Errorf("Unable to download: %v", err) return err } - err = archiver.Unarchive(localFile, installDir) + err = bt.spec.InstallTarget.Unarchive(localFile, installDir) if err != nil { log.Errorf("Unable to decompress: %v", err) return err diff --git a/buildpacks/heroku.go b/buildpacks/heroku.go index 328d6c6a..391b0d06 100644 --- a/buildpacks/heroku.go +++ b/buildpacks/heroku.go @@ -2,11 +2,9 @@ package buildpacks import ( "fmt" - "os" + "path/filepath" "strings" - "github.com/johnewart/archiver" - . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" . "github.com/yourbase/yb/types" ) @@ -69,11 +67,9 @@ func (bt HerokuBuildTool) HerokuDir() string { func (bt HerokuBuildTool) Setup() error { herokuDir := bt.HerokuDir() - cmdPath := fmt.Sprintf("%s/heroku/bin", herokuDir) - currentPath := os.Getenv("PATH") - newPath := fmt.Sprintf("%s:%s", cmdPath, currentPath) - log.Infof("Setting PATH to %s", newPath) - os.Setenv("PATH", newPath) + + t := bt.spec.InstallTarget + t.PrependToPath(filepath.Join(herokuDir, "heroku", "bin")) return nil } @@ -82,19 +78,19 @@ func (bt HerokuBuildTool) Setup() error { func (bt HerokuBuildTool) Install() error { herokuDir := bt.HerokuDir() - if _, err := os.Stat(herokuDir); err == nil { + if bt.spec.InstallTarget.PathExists(herokuDir) { log.Infof("Heroku v%s located in %s!", bt.Version(), herokuDir) } else { log.Infof("Will install Heroku v%s into %s", bt.Version(), herokuDir) downloadUrl := bt.DownloadUrl() log.Infof("Downloading Heroku from URL %s...", downloadUrl) - localFile, err := DownloadFileWithCache(downloadUrl) + localFile, err := bt.spec.InstallTarget.DownloadFile(downloadUrl) if err != nil { log.Errorf("Unable to download: %v", err) return err } - err = archiver.Unarchive(localFile, herokuDir) + err = bt.spec.InstallTarget.Unarchive(localFile, herokuDir) if err != nil { log.Errorf("Unable to decompress: %v", err) return err diff --git a/buildpacks/homebrew.go b/buildpacks/homebrew.go index b9610742..fbc5350b 100644 --- a/buildpacks/homebrew.go +++ b/buildpacks/homebrew.go @@ -10,6 +10,7 @@ import ( "github.com/matishsiao/goInfo" . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" + "github.com/yourbase/yb/runtime" . "github.com/yourbase/yb/types" "gopkg.in/src-d/go-git.v4" ) @@ -143,7 +144,7 @@ func (bt HomebrewBuildTool) InstallPackage() error { brewDir := bt.HomebrewDir() updateCmd := "brew update" - err := ExecToStdout(updateCmd, brewDir) + err := runtime.ExecToStdout(updateCmd, brewDir) if err != nil { return fmt.Errorf("Couldn't update brew: %v", err) } @@ -155,7 +156,7 @@ func (bt HomebrewBuildTool) InstallPackage() error { log.Infof("Going to install %s%s from Homebrew...", bt.pkgName, pkgVersion) installCmd := fmt.Sprintf("brew install %s%s", bt.pkgName, pkgVersion) - err = ExecToStdout(installCmd, brewDir) + err = runtime.ExecToStdout(installCmd, brewDir) if err != nil { return fmt.Errorf("Couldn't intall %s@%s from Homebrew: %v", bt.pkgName, bt.version, err) @@ -189,7 +190,7 @@ func (bt HomebrewBuildTool) InstallDarwin() error { } log.Infof("Updating brew") updateCmd := "brew update" - ExecToStdout(updateCmd, brewDir) + runtime.ExecToStdout(updateCmd, brewDir) return nil } @@ -221,7 +222,7 @@ func (bt HomebrewBuildTool) InstallLinux() error { log.Infof("Updating brew") updateCmd := "brew update" - ExecToStdout(updateCmd, brewDir) + runtime.ExecToStdout(updateCmd, brewDir) } return nil } @@ -242,7 +243,7 @@ func (bt HomebrewBuildTool) Setup() error { brewBinDir := filepath.Join(brewDir, "bin") PrependToPath(brewBinDir) brewLibDir := filepath.Join(brewDir, "lib") - os.Setenv("LD_LIBRARY_PATH", brewLibDir) + runtime.SetEnv("LD_LIBRARY_PATH", brewLibDir) } return nil } diff --git a/buildpacks/maven.go b/buildpacks/maven.go index d7a5bc15..587359e7 100644 --- a/buildpacks/maven.go +++ b/buildpacks/maven.go @@ -6,8 +6,6 @@ import ( "path/filepath" "strings" - "github.com/johnewart/archiver" - . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" . "github.com/yourbase/yb/types" ) @@ -54,7 +52,7 @@ func (bt MavenBuildTool) Version() string { } func (bt MavenBuildTool) InstallDir() string { - return filepath.Join(ToolsDir(), "maven") + return filepath.Join(bt.spec.InstallTarget.ToolsDir(), "maven") } func (bt MavenBuildTool) MavenDir() string { @@ -63,11 +61,9 @@ func (bt MavenBuildTool) MavenDir() string { func (bt MavenBuildTool) Setup() error { mavenDir := bt.MavenDir() - cmdPath := fmt.Sprintf("%s/bin", mavenDir) - currentPath := os.Getenv("PATH") - newPath := fmt.Sprintf("%s:%s", cmdPath, currentPath) - log.Infof("Setting PATH to %s", newPath) - os.Setenv("PATH", newPath) + t := bt.spec.InstallTarget + + t.PrependToPath(filepath.Join(mavenDir, "bin")) return nil } @@ -83,12 +79,12 @@ func (bt MavenBuildTool) Install() error { downloadUrl := bt.DownloadUrl() log.Infof("Downloading Maven from URL %s...", downloadUrl) - localFile, err := DownloadFileWithCache(downloadUrl) + localFile, err := bt.spec.InstallTarget.DownloadFile(downloadUrl) if err != nil { log.Errorf("Unable to download: %v", err) return err } - err = archiver.Unarchive(localFile, bt.InstallDir()) + err = bt.spec.InstallTarget.Unarchive(localFile, bt.InstallDir()) if err != nil { log.Errorf("Unable to decompress: %v", err) return err diff --git a/buildpacks/nodejs.go b/buildpacks/nodejs.go index ebcc5829..bd1bb780 100644 --- a/buildpacks/nodejs.go +++ b/buildpacks/nodejs.go @@ -2,11 +2,8 @@ package buildpacks import ( "fmt" - "os" "path/filepath" - "github.com/johnewart/archiver" - . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" . "github.com/yourbase/yb/types" ) @@ -57,21 +54,22 @@ func (bt NodeBuildTool) Install() error { nodeDir := bt.NodeDir() installDir := bt.InstallDir() nodePkgString := bt.PackageString() + t := bt.spec.InstallTarget - if _, err := os.Stat(nodeDir); err == nil { + if t.PathExists(nodeDir) { log.Infof("Node v%s located in %s!", bt.Version(), nodeDir) } else { log.Infof("Would install Node v%s into %s", bt.Version(), installDir) archiveFile := fmt.Sprintf("%s.tar.gz", nodePkgString) downloadUrl := fmt.Sprintf("%s/v%s/%s", NODE_DIST_MIRROR, bt.Version(), archiveFile) log.Infof("Downloading from URL %s...", downloadUrl) - localFile, err := DownloadFileWithCache(downloadUrl) + localFile, err := bt.spec.InstallTarget.DownloadFile(downloadUrl) if err != nil { log.Errorf("Unable to download: %v", err) return err } - err = archiver.Unarchive(localFile, installDir) + err = bt.spec.InstallTarget.Unarchive(localFile, installDir) if err != nil { log.Errorf("Unable to decompress: %v", err) return err @@ -82,16 +80,17 @@ func (bt NodeBuildTool) Install() error { } func (bt NodeBuildTool) Setup() error { + t := bt.spec.InstallTarget nodeDir := bt.NodeDir() cmdPath := filepath.Join(nodeDir, "bin") - PrependToPath(cmdPath) + t.PrependToPath(cmdPath) // TODO: Fix this to be the package cache? nodePath := bt.spec.PackageDir log.Infof("Setting NODE_PATH to %s", nodePath) - os.Setenv("NODE_PATH", nodePath) + t.SetEnv("NODE_PATH", nodePath) npmBinPath := filepath.Join(nodePath, "node_modules", ".bin") - PrependToPath(npmBinPath) + t.PrependToPath(npmBinPath) return nil } diff --git a/buildpacks/openjdk.go b/buildpacks/openjdk.go index 143d928b..cbbbf987 100644 --- a/buildpacks/openjdk.go +++ b/buildpacks/openjdk.go @@ -2,13 +2,12 @@ package buildpacks import ( "fmt" + "github.com/yourbase/yb/runtime" "os" "path/filepath" "strings" "strconv" - "github.com/johnewart/archiver" - . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" . "github.com/yourbase/yb/types" @@ -121,9 +120,9 @@ func (bt JavaBuildTool) Setup() error { currentPath := os.Getenv("PATH") newPath := fmt.Sprintf("%s:%s", cmdPath, currentPath) log.Infof("Setting PATH to %s", newPath) - os.Setenv("PATH", newPath) + runtime.SetEnv("PATH", newPath) log.Infof("Setting JAVA_HOME to %s", javaDir) - os.Setenv("JAVA_HOME", javaDir) + runtime.SetEnv("JAVA_HOME", javaDir) return nil } @@ -198,12 +197,12 @@ func (bt JavaBuildTool) Install() error { downloadUrl := bt.DownloadUrl() log.Infof("Downloading from URL %s ", downloadUrl) - localFile, err := DownloadFileWithCache(downloadUrl) + localFile, err := bt.spec.InstallTarget.DownloadFile(downloadUrl) if err != nil { log.Errorf("Unable to download: %v", err) return err } - err = archiver.Unarchive(localFile, javaInstallDir) + err = bt.spec.InstallTarget.Unarchive(localFile, javaInstallDir) if err != nil { log.Errorf("Unable to decompress: %v", err) return err diff --git a/buildpacks/protoc.go b/buildpacks/protoc.go index c89ca723..feb4c49b 100644 --- a/buildpacks/protoc.go +++ b/buildpacks/protoc.go @@ -6,8 +6,6 @@ import ( "path/filepath" "strings" - "github.com/johnewart/archiver" - . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" . "github.com/yourbase/yb/types" @@ -102,12 +100,12 @@ func (bt ProtocBuildTool) Install() error { downloadUrl := bt.DownloadUrl() log.Infof("Downloading Protoc from URL %s...", downloadUrl) - localFile, err := DownloadFileWithCache(downloadUrl) + localFile, err := bt.spec.InstallTarget.DownloadFile(downloadUrl) if err != nil { log.Errorf("Unable to download: %v", err) return err } - err = archiver.Unarchive(localFile, installDir) + err = bt.spec.InstallTarget.Unarchive(localFile, installDir) if err != nil { log.Errorf("Unable to decompress: %v", err) return err diff --git a/buildpacks/python.go b/buildpacks/python.go index df5c18d0..b1187069 100644 --- a/buildpacks/python.go +++ b/buildpacks/python.go @@ -2,11 +2,10 @@ package buildpacks import ( "fmt" - "os" "path/filepath" - . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" + "github.com/yourbase/yb/runtime" . "github.com/yourbase/yb/types" ) @@ -44,8 +43,9 @@ func (bt PythonBuildTool) EnvironmentDir() string { func (bt PythonBuildTool) Install() error { anacondaDir := bt.AnacondaInstallDir() setupDir := bt.spec.PackageDir + t := bt.spec.InstallTarget - if _, err := os.Stat(anacondaDir); err == nil { + if bt.spec.InstallTarget.PathExists(anacondaDir) { log.Infof("anaconda installed in %s", anacondaDir) } else { log.Infof("Installing anaconda") @@ -53,7 +53,7 @@ func (bt PythonBuildTool) Install() error { downloadUrl := bt.DownloadUrl() log.Infof("Downloading Miniconda from URL %s...", downloadUrl) - localFile, err := DownloadFileWithCache(downloadUrl) + localFile, err := t.DownloadFile(downloadUrl) if err != nil { log.Errorf("Unable to download: %v", err) return err @@ -65,7 +65,13 @@ func (bt PythonBuildTool) Install() error { fmt.Sprintf("bash %s -b -p %s", localFile, anacondaDir), } { log.Infof("Running: '%v' ", cmd) - ExecToStdout(cmd, setupDir) + p := runtime.Process{ + Command: cmd, + Directory: setupDir, + } + if err := t.Run(p); err != nil { + return fmt.Errorf("Couldn't install python: %v", err) + } } } @@ -74,8 +80,8 @@ func (bt PythonBuildTool) Install() error { } func (bt PythonBuildTool) DownloadUrl() string { - opsys := OS() - arch := Arch() + opsys := "" + arch := "" extension := "sh" version := bt.Version() @@ -83,19 +89,17 @@ func (bt PythonBuildTool) DownloadUrl() string { version = "latest" } - if arch == "amd64" { + switch bt.spec.InstallTarget.Architecture() { + case runtime.Amd64: arch = "x86_64" } - if opsys == "darwin" { - opsys = "MacOSX" - } - - if opsys == "linux" { + switch bt.spec.InstallTarget.OS() { + case runtime.Linux: opsys = "Linux" - } - - if opsys == "windows" { + case runtime.Darwin: + opsys = "MacOSX" + case runtime.Windows: opsys = "Windows" extension = "exe" } @@ -122,12 +126,12 @@ func (bt PythonBuildTool) DownloadUrl() string { func (bt PythonBuildTool) Setup() error { condaDir := bt.AnacondaInstallDir() envDir := bt.EnvironmentDir() + t := bt.spec.InstallTarget - if _, err := os.Stat(envDir); err == nil { + if t.PathExists(envDir) { log.Infof("environment installed in %s", envDir) } else { - currentPath := os.Getenv("PATH") - newPath := fmt.Sprintf("PATH=%s:%s", filepath.Join(condaDir, "bin"), currentPath) + newPath := fmt.Sprintf("PATH=%s:%s", filepath.Join(condaDir, "bin"), t.GetDefaultPath()) setupDir := bt.spec.PackageDir condaBin := filepath.Join(condaDir, "bin", "conda") @@ -138,7 +142,14 @@ func (bt PythonBuildTool) Setup() error { fmt.Sprintf("%s create --prefix %s python=%s", condaBin, envDir, bt.Version()), } { log.Infof("Running: '%v' ", cmd) - if err := ExecToStdoutWithEnv(cmd, setupDir, []string{newPath}); err != nil { + p := runtime.Process{ + Command: cmd, + Interactive: false, + Directory: setupDir, + Environment: []string{newPath}, + } + + if err := t.Run(p); err != nil { log.Errorf("Unable to run setup command: %s", cmd) return fmt.Errorf("Unable to run '%s': %v", cmd, err) } @@ -146,7 +157,7 @@ func (bt PythonBuildTool) Setup() error { } // Add new env to path - PrependToPath(filepath.Join(envDir, "bin")) + t.PrependToPath(filepath.Join(envDir, "bin")) return nil diff --git a/buildpacks/rlang.go b/buildpacks/rlang.go index 78a073a8..22763c14 100644 --- a/buildpacks/rlang.go +++ b/buildpacks/rlang.go @@ -6,9 +6,9 @@ import ( "path/filepath" "strings" - "github.com/johnewart/archiver" . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" + "github.com/yourbase/yb/runtime" . "github.com/yourbase/yb/types" ) @@ -61,12 +61,9 @@ func (bt RLangBuildTool) RLangDir() string { func (bt RLangBuildTool) Setup() error { rlangDir := bt.RLangDir() + t := bt.spec.InstallTarget - cmdPath := fmt.Sprintf("%s/bin", rlangDir) - currentPath := os.Getenv("PATH") - newPath := fmt.Sprintf("%s:%s", cmdPath, currentPath) - log.Infof("Setting PATH to %s", newPath) - os.Setenv("PATH", newPath) + t.PrependToPath(filepath.Join(rlangDir, "bin")) return nil } @@ -83,7 +80,7 @@ func (bt RLangBuildTool) Install() error { downloadUrl := bt.DownloadUrl() log.Infof("Downloading from URL %s ...", downloadUrl) - localFile, err := DownloadFileWithCache(downloadUrl) + localFile, err := bt.spec.InstallTarget.DownloadFile(downloadUrl) if err != nil { log.Errorf("Unable to download: %v", err) return err @@ -93,7 +90,7 @@ func (bt RLangBuildTool) Install() error { srcDir := filepath.Join(tmpDir, fmt.Sprintf("R-%s", bt.Version())) if !DirectoryExists(srcDir) { - err = archiver.Unarchive(localFile, tmpDir) + err = bt.spec.InstallTarget.Unarchive(localFile, tmpDir) if err != nil { log.Errorf("Unable to decompress: %v", err) return err @@ -102,10 +99,10 @@ func (bt RLangBuildTool) Install() error { MkdirAsNeeded(rlangDir) configCmd := fmt.Sprintf("./configure --with-x=no --prefix=%s", rlangDir) - ExecToStdout(configCmd, srcDir) + runtime.ExecToStdout(configCmd, srcDir) - ExecToStdout("make", srcDir) - ExecToStdout("make install", srcDir) + runtime.ExecToStdout("make", srcDir) + runtime.ExecToStdout("make install", srcDir) } return nil diff --git a/buildpacks/ruby.go b/buildpacks/ruby.go index 71a6faa8..16c419d6 100644 --- a/buildpacks/ruby.go +++ b/buildpacks/ruby.go @@ -8,11 +8,11 @@ import ( "path/filepath" "strings" - "github.com/johnewart/archiver" "github.com/matishsiao/goInfo" . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" + "github.com/yourbase/yb/runtime" . "github.com/yourbase/yb/types" "gopkg.in/src-d/go-git.v4" ) @@ -21,23 +21,21 @@ const YBRubyDownloadTemplate = "https://yourbase-build-tools.s3-us-west-2.amazon func (bt RubyBuildTool) DownloadUrl() string { extension := "tar.bz2" - osVersion := OSVersion() + operatingSystem := "unknown" + osVersion := bt.spec.InstallTarget.OSVersion() arch := Arch() if arch == "amd64" { arch = "x86_64" } - operatingSystem := OS() - if operatingSystem == "darwin" { - operatingSystem = "Darwin" - } - - if operatingSystem == "linux" { + switch bt.spec.InstallTarget.OS() { + case runtime.Linux: operatingSystem = "Linux" - } - - if operatingSystem == "windows" { + case runtime.Darwin: + operatingSystem = "Darwin" + case runtime.Windows: + operatingSystem = "windows" extension = "zip" } @@ -116,8 +114,9 @@ TODO: Install libssl-dev (or equivalent / warn) and zlib-dev based on platform func (bt RubyBuildTool) Install() error { rubyVersionDir := bt.RubyDir() + t := bt.spec.InstallTarget - if _, err := os.Stat(rubyVersionDir); err == nil { + if t.PathExists(rubyVersionDir) { log.Infof("Ruby %s installed in %s", bt.Version(), rubyVersionDir) } else { @@ -126,12 +125,12 @@ func (bt RubyBuildTool) Install() error { downloadUrl := bt.DownloadUrl() log.Infof("Will download pre-built Ruby from %s", downloadUrl) - localFile, err := DownloadFileWithCache(downloadUrl) + localFile, err := t.DownloadFile(downloadUrl) if err != nil { log.Errorf("Unable to download: %v", err) return err } - err = archiver.Unarchive(localFile, rubyVersionsDir) + err = t.Unarchive(localFile, rubyVersionsDir) if err != nil { log.Errorf("Unable to decompress: %v", err) return err @@ -185,11 +184,11 @@ func (bt RubyBuildTool) Install() error { } } - os.Setenv("RBENV_ROOT", rbenvDir) - PrependToPath(filepath.Join(rbenvDir, "bin")) + runtime.SetEnv("RBENV_ROOT", rbenvDir) + t.PrependToPath(filepath.Join(rbenvDir, "bin")) installCmd := fmt.Sprintf("rbenv install %s", bt.Version()) - ExecToStdout(installCmd, rbenvDir) + runtime.ExecToStdout(installCmd, rbenvDir) } return nil @@ -197,16 +196,19 @@ func (bt RubyBuildTool) Install() error { func (bt RubyBuildTool) Setup() error { gemsDir := filepath.Join(bt.spec.PackageCacheDir, "rubygems") - MkdirAsNeeded(gemsDir) + + //MkdirAsNeeded(gemsDir) + + t := bt.spec.InstallTarget log.Infof("Setting GEM_HOME to %s", gemsDir) - os.Setenv("GEM_HOME", gemsDir) + t.SetEnv("GEM_HOME", gemsDir) gemBinDir := filepath.Join(gemsDir, "bin") rubyDir := bt.RubyDir() - PrependToPath(filepath.Join(rubyDir, "bin")) - PrependToPath(gemBinDir) + t.PrependToPath(filepath.Join(rubyDir, "bin")) + t.PrependToPath(gemBinDir) return nil } diff --git a/buildpacks/rust.go b/buildpacks/rust.go index 94f1e698..e32c5738 100644 --- a/buildpacks/rust.go +++ b/buildpacks/rust.go @@ -7,6 +7,7 @@ import ( . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" + "github.com/yourbase/yb/runtime" . "github.com/yourbase/yb/types" ) @@ -41,14 +42,12 @@ func (bt RustBuildTool) InstallDir() string { func (bt RustBuildTool) Setup() error { rustDir := bt.RustDir() - cmdPath := fmt.Sprintf("%s/bin", rustDir) - currentPath := os.Getenv("PATH") - newPath := fmt.Sprintf("%s:%s", cmdPath, currentPath) - log.Infof("Setting PATH to %s", newPath) - os.Setenv("PATH", newPath) + t := bt.spec.InstallTarget - os.Setenv("CARGO_HOME", rustDir) - os.Setenv("RUSTUP_HOME", rustDir) + t.PrependToPath(filepath.Join(rustDir, "bin")) + + runtime.SetEnv("CARGO_HOME", rustDir) + runtime.SetEnv("RUSTUP_HOME", rustDir) return nil } @@ -73,18 +72,18 @@ func (bt RustBuildTool) Install() error { downloadDir := bt.spec.PackageCacheDir localFile := filepath.Join(downloadDir, installerFile) log.Infof("Downloading from URL %s to local file %s", downloadUrl, localFile) - err := DownloadFile(localFile, downloadUrl) + localFile, err := bt.spec.InstallTarget.DownloadFile(downloadUrl) if err != nil { log.Errorf("Unable to download: %v", err) return err } os.Chmod(localFile, 0700) - os.Setenv("CARGO_HOME", rustDir) - os.Setenv("RUSTUP_HOME", rustDir) + runtime.SetEnv("CARGO_HOME", rustDir) + runtime.SetEnv("RUSTUP_HOME", rustDir) installCmd := fmt.Sprintf("%s -y", localFile) - ExecToStdout(installCmd, downloadDir) + runtime.ExecToStdout(installCmd, downloadDir) } return nil diff --git a/buildpacks/tool_spec.go b/buildpacks/tool_spec.go new file mode 100644 index 00000000..92044166 --- /dev/null +++ b/buildpacks/tool_spec.go @@ -0,0 +1,32 @@ +package buildpacks + +import ( + "bytes" + "github.com/yourbase/yb/plumbing/log" + "github.com/yourbase/yb/runtime" + "text/template" +) + +type BuildToolSpec struct { + Tool string + Version string + SharedCacheDir string + PackageCacheDir string + PackageDir string + InstallTarget runtime.Target +} + +func TemplateToString(templateText string, data interface{}) (string, error) { + t, err := template.New("generic").Parse(templateText) + if err != nil { + return "", err + } + var tpl bytes.Buffer + if err := t.Execute(&tpl, data); err != nil { + log.Errorf("Can't render template:: %v", err) + return "", err + } + + result := tpl.String() + return result, nil +} \ No newline at end of file diff --git a/buildpacks/yarn.go b/buildpacks/yarn.go index 122752eb..cf42aa11 100644 --- a/buildpacks/yarn.go +++ b/buildpacks/yarn.go @@ -5,7 +5,6 @@ import ( "os" "path/filepath" - "github.com/johnewart/archiver" . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" . "github.com/yourbase/yb/types" @@ -62,12 +61,12 @@ func (bt YarnBuildTool) Install() error { log.Infof("Will install Yarn v%s into %s", bt.Version(), installDir) downloadUrl := bt.DownloadUrl() log.Infof("Downloading from URL %s...", downloadUrl) - localFile, err := DownloadFileWithCache(downloadUrl) + localFile, err := bt.spec.InstallTarget.DownloadFile(downloadUrl) if err != nil { return fmt.Errorf("Unable to download %s: %v", downloadUrl, err) } - if err := archiver.Unarchive(localFile, installDir); err != nil { + if err := bt.spec.InstallTarget.Unarchive(localFile, installDir); err != nil { return fmt.Errorf("Unable to decompress archive: %v", err) } } diff --git a/cli/build.go b/cli/build.go index bc99bc01..b70f8bfb 100644 --- a/cli/build.go +++ b/cli/build.go @@ -3,30 +3,21 @@ package cli import ( "bytes" "context" - "crypto/sha256" "encoding/json" "flag" "fmt" - "github.com/matishsiao/goInfo" "io" "io/ioutil" "os" - "os/signal" - "os/user" - "path/filepath" - "runtime" - "strings" "time" - "github.com/johnewart/archiver" - "github.com/johnewart/narwhal" - "github.com/johnewart/subcommands" + "github.com/matishsiao/goInfo" + "github.com/johnewart/subcommands" ybconfig "github.com/yourbase/yb/config" - . "github.com/yourbase/yb/packages" . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" - . "github.com/yourbase/yb/types" + "github.com/yourbase/yb/workspace" ) const TIME_FORMAT = "15:04:05 MST" @@ -36,20 +27,8 @@ type BuildCmd struct { Version string ExecPrefix string NoContainer bool - NoSideContainer bool DependenciesOnly bool - ReuseContainers bool -} - -type BuildConfiguration struct { - Target BuildTarget - TargetDir string - Sandboxed bool - ExecPrefix string - ForceNoContainer bool - TargetPackage Package - BuildData BuildData - ReuseContainers bool + CleanBuild bool } type BuildLog struct { @@ -67,8 +46,7 @@ func (b *BuildCmd) SetFlags(f *flag.FlagSet) { f.BoolVar(&b.NoContainer, "no-container", false, "Bypass container even if specified") f.BoolVar(&b.DependenciesOnly, "deps-only", false, "Install only dependencies, don't do anything else") f.StringVar(&b.ExecPrefix, "exec-prefix", "", "Add a prefix to all executed commands (useful for timing or wrapping things)") - f.BoolVar(&b.ReuseContainers, "reuse-containers", true, "Reuse the container for building") - f.BoolVar(&b.NoSideContainer, "no-side-container", false, "Don't run/create any side container") + f.BoolVar(&b.CleanBuild, "clean", false, "Perform a completely clean build -- don't reuse anything when building") } func (b *BuildCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { @@ -91,10 +69,6 @@ func (b *BuildCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) log.Errorf("%v", err) return subcommands.ExitFailure } - - manifest := targetPackage.Manifest - workDir := targetPackage.BuildRoot() - // Determine build target buildTargetName := "default" @@ -102,176 +76,20 @@ func (b *BuildCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) buildTargetName = f.Args()[0] } - // Named target, look for that and resolve it - buildTargets, err := manifest.ResolveBuildTargets(buildTargetName) - - if err != nil { - log.Errorf("Could not compute build target '%s': %v", buildTargetName, err) - log.Infof("Valid build targets: %s", strings.Join(manifest.BuildTargetList(), ", ")) - return subcommands.ExitFailure - } - - primaryTarget, err := manifest.BuildTarget(buildTargetName) - if err != nil { - log.Errorf("Couldn't get primary build target '%s' specs: %v", buildTargetName, err) - return subcommands.ExitFailure - } - - buildData := NewBuildData() - - if !b.NoContainer { - target := primaryTarget - // Setup dependencies - containers := target.Dependencies.Containers - - if err := narwhal.DockerClient().Ping(); err != nil { - log.Error("Couldn't connect to Docker daemon. Try installing Docker Desktop: https://hub.docker.com/search/?type=edition&offering=community") - return subcommands.ExitFailure - } - contextId := fmt.Sprintf("%s-%s", targetPackage.Name, target.Name) - sc, err := narwhal.NewServiceContextWithId(contextId, workDir) - if err != nil { - log.Errorf("Couldn't create service context for dependencies: %v", err) - return subcommands.ExitFailure - } - - buildData.Containers.ServiceContext = sc - - if len(containers) > 0 && !b.NoSideContainer { - log.Infof("Starting %d containers with context id %s...", len(containers), contextId) - if !b.ReuseContainers { - log.Infof("Not reusing containers, will teardown existing ones and clean up after ourselves") - defer Cleanup(buildData) - if err := sc.TearDown(); err != nil { - log.Errorf("Couldn't terminate existing containers: %v", err) - // FAIL? - } - } - - for _, c := range containers { - _, err := sc.StartContainer(c) - if err != nil { - log.Errorf("Couldn't start dependencies: %v", err) - return subcommands.ExitFailure - } - } - - } - } - - // Do this after the containers are up - for _, envString := range primaryTarget.Environment { - parts := strings.SplitN(envString, "=", 2) - if len(parts) != 2 { - log.Warnf("'%s' doesn't look like an environment variable", envString) - continue - } - buildData.SetEnv(parts[0], parts[1]) - } - - _, exists := os.LookupEnv("YB_NO_CONTAINER_BUILDS") - // Should we build in a container? - if !b.NoContainer && !primaryTarget.HostOnly && !exists { - log.StartSection("BUILD IN CONTAINER", "BCONTAINER") - log.Infof("Executing build steps in container") - - target := primaryTarget - containerDef := target.Container - containerDef.Command = "/usr/bin/tail -f /dev/null" - - // Append build environment variables - //container.Environment = append(container.Environment, buildData.EnvironmentVariables()...) - containerDef.Environment = []string{} - - // Add package to mounts @ /workspace - sourceMapDir := "/workspace" - if containerDef.WorkDir != "" { - sourceMapDir = containerDef.WorkDir - } + var targetTimers []workspace.TargetTimer - log.Infof("Will mount package %s at %s in container", targetPackage.Path, sourceMapDir) - mount := fmt.Sprintf("%s:%s", targetPackage.Path, sourceMapDir) - containerDef.Mounts = append(containerDef.Mounts, mount) - - if false { - u, _ := user.Current() - fmt.Printf("U! %s", u.Uid) - } - - // Do our build in a container - don't do the build locally - buildErr := b.BuildInsideContainer(target, containerDef, buildData, b.ExecPrefix, b.ReuseContainers) - if buildErr != nil { - log.Errorf("Unable to build %s inside container: %v", target.Name, buildErr) - return subcommands.ExitFailure - } - - return subcommands.ExitSuccess + buildFlags := workspace.BuildFlags{ + HostOnly: b.NoContainer, + CleanBuild: b.CleanBuild, } - - // Do the build! - targetDir := targetPackage.Path - - log.Infof("Building package %s in %s...", targetPackage.Name, targetDir) - log.Infof("Checksum of dependencies: %s", manifest.BuildDependenciesChecksum()) - - log.EndSection() - - log.StartSection("ENVIRONMENT SETUP", "ENVIRONMENT") - setupTimer, err := SetupBuildDependencies(targetPackage) + stepTimers, buildError := targetPackage.BuildTarget(buildTargetName, buildFlags) if err != nil { - log.Infof("Error setting up dependencies: %v", err) + log.Errorf("Failed to build target package: %v\n", err) return subcommands.ExitFailure } - if b.DependenciesOnly { - log.Infof("Only installing dependencies; done!") - return subcommands.ExitSuccess - } - - if len(primaryTarget.Dependencies.Containers) > 0 { - log.Info("") - log.Info("") - log.Infof("Available side containers:") - for label, c := range primaryTarget.Dependencies.Containers { - ipv4 := buildData.Containers.IP(label) - log.Infof(" * %s (using %s) has IP address %s", label, c.ImageNameWithTag(), ipv4) - } - } - log.EndSection() - - var targetTimers []TargetTimer - var buildError error - - log.StartSection("BUILD", "BUILD") - targetTimers = append(targetTimers, setupTimer) - - config := BuildConfiguration{ - ExecPrefix: b.ExecPrefix, - TargetDir: targetDir, - ForceNoContainer: b.NoContainer, - TargetPackage: targetPackage, - ReuseContainers: b.ReuseContainers, - BuildData: buildData, - } - - log.Infof("Going to build targets in the following order: ") - for _, target := range buildTargets { - log.Infof(" - %s", target.Name) - } - - var buildStepTimers []CommandTimer - for _, target := range buildTargets { - if buildError == nil { - log.SubSection(fmt.Sprintf("Build target: %s", target.Name)) - config.Target = target - config.Sandboxed = manifest.IsTargetSandboxed(target) - buildData.ExportEnvironmentPublicly() - log.Infof("Executing build steps...") - buildStepTimers, buildError = RunCommands(config) - targetTimers = append(targetTimers, TargetTimer{Name: target.Name, Timers: buildStepTimers}) - } - } + targetTimers = append(targetTimers, workspace.TargetTimer{Name: buildTargetName, Timers: stepTimers}) endTime := time.Now() buildTime := endTime.Sub(startTime) @@ -351,216 +169,6 @@ func (b *BuildCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) return subcommands.ExitSuccess } -func Cleanup(b BuildData) { - if b.Containers.ServiceContext != nil { - log.Infof("Cleaning up containers...") - if err := b.Containers.ServiceContext.TearDown(); err != nil { - log.Warnf("Problem tearing down containers: %v", err) - } - } -} - -func (b *BuildCmd) BuildInsideContainer(target BuildTarget, containerDef narwhal.ContainerDefinition, buildData BuildData, execPrefix string, reuseOldContainer bool) error { - // Perform build inside a container - image := target.Container.Image - log.Infof("Using container image: %s", image) - buildContainer, err := buildData.Containers.ServiceContext.StartContainer(containerDef) - - if err != nil { - return fmt.Errorf("Failed trying to find container: %v", err) - } - - if buildContainer != nil { - log.Infof("Found existing container %s", buildContainer.Id[0:12]) - - _ = buildContainer.Stop(0) - - if !reuseOldContainer { - log.Infof("Elected to not re-use containers, will remove the container") - // Try stopping it first if needed - if err = narwhal.RemoveContainerById(buildContainer.Id); err != nil { - return fmt.Errorf("Unable to remove existing container: %v", err) - } - } - } - - if buildContainer != nil && reuseOldContainer { - log.Infof("Reusing existing build container %s", buildContainer.Id[0:12]) - } else { - log.Infof("Creating a new build container...") - if c, err := buildData.Containers.ServiceContext.StartContainer(containerDef); err != nil { - return fmt.Errorf("Couldn't create a new build container: %v", err) - } else { - buildContainer = c - } - } - - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt) - go func() { - <-c - buildContainer.Stop(0) - os.Exit(0) - }() - - defer buildContainer.Stop(0) - - if err != nil { - return fmt.Errorf("Error creating build container: %v", err) - } - - if err := buildContainer.Start(); err != nil { - return fmt.Errorf("Unable to start container %s: %v", buildContainer.Id, err) - } - - // Set container work dir - containerWorkDir := "/workspace" - if containerDef.WorkDir != "" { - containerWorkDir = containerDef.WorkDir - } - - log.Infof("Building from %s in container: %s", containerWorkDir, buildContainer.Id) - - // Local path to binary that we want to inject - // TODO: this only supports Linux containers - localYbPath := "" - devMode := false - - // If this is development mode, use the YB binary currently running - // We can't do this by default because this only works if the host is - // Linux so we might as well behave the same on all platforms - // If not development mode, download the binary from the distribution channel - if b.Version == "DEVELOPMENT" { - if runtime.GOOS == "linux" { - // TODO: If we support building inside a container that's not Linux we will want to do something different - log.Infof("Development version in use, will upload development binary to the container") - devMode = true - } - } - - if devMode { - if p, err := os.Executable(); err != nil { - return fmt.Errorf("Can't determine local path to YB: %v", err) - } else { - localYbPath = p - } - } else { - if p, err := DownloadYB(); err != nil { - return fmt.Errorf("Couldn't download YB: %v", err) - } else { - localYbPath = p - } - } - - // Upload and update CLI - log.Infof("Uploading YB from %s to /yb", localYbPath) - if err := buildContainer.UploadFile(localYbPath, "yb", "/"); err != nil { - return fmt.Errorf("Unable to upload YB to container: %v", err) - } - - if !devMode { - // Default to whatever channel being used, unless told otherwise - // TODO: Make the download URL used for downloading track the latest stable - ybChannel := b.Channel - - // Overwrites local configuration - if envChannel, exists := os.LookupEnv("YB_UPDATE_CHANNEL"); exists { - ybChannel = envChannel - } - - log.Infof("Updating YB in container from channel %s", ybChannel) - updateCmd := fmt.Sprintf("/yb update -channel=%s", ybChannel) - if err := buildContainer.ExecToStdoutWithEnv(updateCmd, containerWorkDir, buildData.EnvironmentVariables()); err != nil { - return fmt.Errorf("Aborting build, unable to run %s: %v", updateCmd, err) - } - } - - cmdString := "/yb build" - - if len(execPrefix) > 0 { - cmdString = fmt.Sprintf("%s -exec-prefix %s", cmdString, execPrefix) - } - - cmdString = fmt.Sprintf("%s -no-container %s", cmdString, target.Name) - - log.Infof("Running %s in the container in directory %s", cmdString, containerWorkDir) - - if err := buildContainer.ExecToStdoutWithEnv(cmdString, containerWorkDir, buildData.EnvironmentVariables()); err != nil { - log.Errorf("Failed to run %s: %v", cmdString, err) - return fmt.Errorf("Aborting build, unable to run %s: %v", cmdString, err) - } - - // Make sure our goroutine gets this from stdout - // TODO: There must be a better way... - time.Sleep(10 * time.Millisecond) - - return nil -} - -func RunCommands(config BuildConfiguration) ([]CommandTimer, error) { - - stepTimes := make([]CommandTimer, 0) - - target := config.Target - sandboxed := config.Sandboxed - targetDir := config.TargetDir - if target.Root != "" { - log.Infof("Build root is %s", target.Root) - targetDir = filepath.Join(targetDir, target.Root) - } - - for _, cmdString := range target.Commands { - var stepError error - - stepStartTime := time.Now() - if strings.HasPrefix(cmdString, "cd ") { - parts := strings.SplitN(cmdString, " ", 2) - dir := filepath.Join(targetDir, parts[1]) - //log.Infof("Chdir to %s", dir) - //os.Chdir(dir) - targetDir = dir - } else { - if len(config.ExecPrefix) > 0 { - cmdString = fmt.Sprintf("%s %s", config.ExecPrefix, cmdString) - } - - if sandboxed { - log.Infoln("Running build in a sandbox!") - if err := ExecInSandbox(cmdString, targetDir); err != nil { - log.Errorf("Failed to run %s: %v", cmdString, err) - stepError = err - } - } else { - if err := ExecToStdout(cmdString, targetDir); err != nil { - log.Errorf("Failed to run %s: %v", cmdString, err) - stepError = err - } - } - } - - stepEndTime := time.Now() - stepTotalTime := stepEndTime.Sub(stepStartTime) - - log.Infof("Completed '%s' in %s", cmdString, stepTotalTime) - - cmdTimer := CommandTimer{ - Command: cmdString, - StartTime: stepStartTime, - EndTime: stepEndTime, - } - - stepTimes = append(stepTimes, cmdTimer) - // Make sure our goroutine gets this from stdout - // TODO: There must be a better way... - time.Sleep(10 * time.Millisecond) - if stepError != nil { - return stepTimes, stepError - } - } - - return stepTimes, nil -} - func UploadBuildLogsToAPI(buf *bytes.Buffer) { log.Infof("Uploading build logs...") buildLog := BuildLog{ @@ -602,158 +210,3 @@ func UploadBuildLogsToAPI(buf *bytes.Buffer) { } } - -func SetupBuildDependencies(pkg Package) (TargetTimer, error) { - - startTime := time.Now() - // Ensure build deps are :+1: - stepTimers, err := pkg.SetupBuildDependencies() - - endTime := time.Now() - elapsedTime := endTime.Sub(startTime) - - log.Infof("Dependency setup happened in %s", elapsedTime) - - setupTimer := TargetTimer{ - Name: "dependency_setup", - Timers: stepTimers, - } - - return setupTimer, err -} - -// Build metadata; used for things like interpolation in environment variables -type ContainerData struct { - ServiceContext *narwhal.ServiceContext -} - -func (c ContainerData) IP(label string) string { - // Check service context - if c.ServiceContext != nil { - if buildContainer, ok := c.ServiceContext.Containers[label]; ok { - if ipv4, err := buildContainer.IPv4Address(); err == nil { - return ipv4 - } - } - } - - // Look for environment variable (injected into containers) - envKey := fmt.Sprintf("YB_CONTAINER_%s_IP", strings.ToUpper(label)) - if ip, exists := os.LookupEnv(envKey); exists { - return ip - } - - return "" -} - -func (c ContainerData) Environment() map[string]string { - result := make(map[string]string) - if c.ServiceContext != nil { - for label, container := range c.ServiceContext.Containers { - if ipv4, err := container.IPv4Address(); err == nil { - key := fmt.Sprintf("YB_CONTAINER_%s_IP", strings.ToUpper(label)) - result[key] = ipv4 - } - } - } - return result -} - -type BuildData struct { - Containers ContainerData - Environment map[string]string -} - -func NewBuildData() BuildData { - return BuildData{ - Containers: ContainerData{}, - Environment: make(map[string]string), - } -} - -func (b BuildData) SetEnv(key string, value string) { - interpolated, err := TemplateToString(value, b) - if err != nil { - b.Environment[key] = value - } else { - b.Environment[key] = interpolated - } -} - -func (b BuildData) mergedEnvironment() map[string]string { - result := make(map[string]string) - for k, v := range b.Containers.Environment() { - result[k] = v - } - for k, v := range b.Environment { - result[k] = v - } - return result -} - -func (b BuildData) ExportEnvironmentPublicly() { - log.Infof("Exporting environment") - for k, v := range b.mergedEnvironment() { - log.Infof(" * %s = %s", k, v) - os.Setenv(k, v) - } -} - -func (b BuildData) EnvironmentVariables() []string { - result := make([]string, 0) - for k, v := range b.mergedEnvironment() { - result = append(result, fmt.Sprintf("%s=%s", k, v)) - } - return result -} - -func sha256File(path string) (string, error) { - f, err := os.Open(path) - if err != nil { - return "", err - } - defer f.Close() - - h := sha256.New() - if _, err := io.Copy(h, f); err != nil { - return "", err - } - - return fmt.Sprintf("%x", h.Sum(nil)), nil -} - -// TODO: non-linux things too if we ever support non-Linux containers -// TODO: on non-Linux platforms we shouldn't constantly try to re-download it -func DownloadYB() (string, error) { - // Stick with this version, we can track some relatively recent version because - // we will just update anyway so it doesn't need to be super-new unless we broke something - downloadUrl := "https://bin.equinox.io/a/7G9uDXWDjh8/yb-0.0.39-linux-amd64.tar.gz" - binarySha256 := "3e21a9c98daa168ea95a5be45d14408c18688b5eea211d7936f6cd013bd23210" - cachePath := CacheDir() - tmpPath := filepath.Join(cachePath, ".yb-tmp") - ybPath := filepath.Join(tmpPath, "yb") - MkdirAsNeeded(tmpPath) - - if PathExists(ybPath) { - checksum, err := sha256File(ybPath) - // Checksum match? We're good to go - if err == nil && checksum == binarySha256 { - return ybPath, nil - } - - log.Infof("Local binary checksum mis-match, re-downloading YB") - } - - // Couldn't tell, check if we need to and download the archive - localFile, err := DownloadFileWithCache(downloadUrl) - if err != nil { - return "", err - } - - err = archiver.Unarchive(localFile, tmpPath) - if err != nil { - return "", fmt.Errorf("Couldn't decompress: %v", err) - } - - return ybPath, nil -} diff --git a/cli/exec.go b/cli/exec.go index 4b6572ce..47d84ba8 100644 --- a/cli/exec.go +++ b/cli/exec.go @@ -3,13 +3,13 @@ package cli import ( "context" "flag" - "path/filepath" - "strings" + "fmt" + "os" + "os/signal" + "syscall" - "github.com/johnewart/narwhal" "github.com/johnewart/subcommands" - . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" ) @@ -49,71 +49,23 @@ func (b *ExecCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) return subcommands.ExitFailure } - instructions := targetPackage.Manifest - containers := instructions.Exec.Dependencies.ContainerList() + execRuntime, err := targetPackage.ExecutionRuntime(b.environment) - buildData := NewBuildData() + c := make(chan os.Signal, 2) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + <-c + fmt.Println("\r- Ctrl+C pressed in Terminal") + execRuntime.Shutdown() + os.Exit(0) + }() - if len(containers) > 0 { - log.ActiveSection("Containers") - localContainerWorkDir := filepath.Join(targetPackage.BuildRoot(), "containers") - MkdirAsNeeded(localContainerWorkDir) + log.Infof("Executing package '%s'...\n", targetPackage.Name) - log.Infof("Will use %s as the dependency work dir", localContainerWorkDir) - log.Infof("Starting %d dependencies...", len(containers)) - sc, err := narwhal.NewServiceContextWithId("exec", targetPackage.BuildRoot()) - if err != nil { - log.Errorf("Couldn't create service context for dependencies: %v", err) - return subcommands.ExitFailure - } - - for _, c := range containers { - // TODO: avoid setting these here - c.LocalWorkDir = localContainerWorkDir - if _, err = sc.StartContainer(c); err != nil { - log.Infof("Couldn't start dependencies: %v\n", err) - return subcommands.ExitFailure - } - } - - buildData.Containers.ServiceContext = sc - } - - log.ActiveSection("Environment") - log.Infof("Setting environment variables...") - for _, property := range instructions.Exec.Environment["default"] { - s := strings.SplitN(property, "=", 2) - if len(s) == 2 { - buildData.SetEnv(s[0], s[1]) - } - } - - if b.environment != "default" { - for _, property := range instructions.Exec.Environment[b.environment] { - s := strings.Split(property, "=") - if len(s) == 2 { - buildData.SetEnv(s[0], s[1]) - } - } - } - - buildData.ExportEnvironmentPublicly() - - log.ActiveSection("Exec") - - log.Infof("Execing target package %s...\n", targetPackage.Name) - - execDir := targetPackage.Path - - for _, logFile := range instructions.Exec.LogFiles { - log.Infof("Will tail %s...\n", logFile) - } - - for _, cmdString := range instructions.Exec.Commands { - if err := ExecToStdout(cmdString, execDir); err != nil { - log.Errorf("Failed to run %s: %v", cmdString, err) - return subcommands.ExitFailure - } + err = targetPackage.Execute(execRuntime) + if err != nil { + log.Errorf("Unable to run command: %v", err) + return subcommands.ExitFailure } return subcommands.ExitSuccess diff --git a/cli/helpers.go b/cli/helpers.go index e2d64d3e..9e81c1fd 100644 --- a/cli/helpers.go +++ b/cli/helpers.go @@ -4,7 +4,6 @@ import ( "fmt" "path/filepath" - . "github.com/yourbase/yb/packages" . "github.com/yourbase/yb/plumbing" . "github.com/yourbase/yb/types" . "github.com/yourbase/yb/workspace" diff --git a/cli/ls.go b/cli/ls.go new file mode 100644 index 00000000..f56e9942 --- /dev/null +++ b/cli/ls.go @@ -0,0 +1,38 @@ +package cli + +import ( + "context" + "flag" + "fmt" + + "github.com/johnewart/subcommands" + + "github.com/yourbase/yb/plumbing/log" +) + +type LsCmd struct { + file string +} + +func (*LsCmd) Name() string { return "ls" } +func (*LsCmd) Synopsis() string { return "List the metadata - build targets, containers, etc." } +func (*LsCmd) Usage() string { + return `ls` +} + +func (b *LsCmd) SetFlags(f *flag.FlagSet) { +} + +func (b *LsCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { + + targetPackage, err := GetTargetPackage() + if err != nil { + log.Errorf("%v", err) + return subcommands.ExitFailure + } + + m := targetPackage.Manifest + + fmt.Printf("Dependencies: %s\n", m.BuildDependenciesChecksum()) + return subcommands.ExitSuccess +} diff --git a/cli/package.go b/cli/package.go index 637f9616..d60111ee 100644 --- a/cli/package.go +++ b/cli/package.go @@ -4,11 +4,14 @@ import ( "context" "flag" "fmt" - "github.com/johnewart/archiver" - "github.com/johnewart/subcommands" "os" "path/filepath" + "strings" + + "github.com/johnewart/archiver" + "github.com/johnewart/subcommands" + "github.com/yourbase/narwhal" . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" . "github.com/yourbase/yb/workspace" @@ -21,7 +24,7 @@ type PackageCmd struct { func (*PackageCmd) Name() string { return "package" } func (*PackageCmd) Synopsis() string { return "Create a package artifact" } func (*PackageCmd) Usage() string { - return `package [--target pkg]` + return `` } func (p *PackageCmd) SetFlags(f *flag.FlagSet) { @@ -34,46 +37,140 @@ Executing the target involves: 2. Run any dependent components 3. Start target */ -func (b *PackageCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { - return subcommands.ExitFailure -} +func (b *PackageCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { + + cmdr := subcommands.NewCommander(f, "packages") -func (b *PackageCmd) ArchiveWorkspace() subcommands.ExitStatus { + targetPackage := Package{} workspace, err := LoadWorkspace() + if err == nil { + if b.target != "" { + targetPackage, _ = workspace.PackageByName(b.target) + } else { + targetPackage, _ = workspace.TargetPackage() + b.target = targetPackage.Name + } + } + + // Fallback for no workspace + if targetPackage.Name == "" { + targetPackage, err = GetTargetPackage() + if err != nil { + log.Errorf("Unable to load package: %v\n", err) + return subcommands.ExitFailure + } + } + + cmdr.Register(&DockerArchiveCmd{targetPackage: targetPackage}, "") + cmdr.Register(&TarArchiveCmd{targetPackage: targetPackage}, "") + + return (cmdr.Execute(ctx)) +} + +type DockerArchiveCmd struct { + targetPackage Package + dockerRegistry string + dockerTag string +} + +func (*DockerArchiveCmd) Name() string { return "docker" } +func (*DockerArchiveCmd) Synopsis() string { return "Create a docker image" } +func (*DockerArchiveCmd) Usage() string { + return `` +} + +func (d *DockerArchiveCmd) SetFlags(f *flag.FlagSet) { + f.StringVar(&d.dockerRegistry, "registry", "docker-registry.yourbase.io", "Registry to upload image") + f.StringVar(&d.dockerTag, "tag", "latest", "Container tag") +} + +//Execute will create a container image matching the file format below +// +// package: +// docker: +// base_image: ubuntu +// image: dispatcher +// working_dir: / +// exec: /dispatcher +// artifacts: +// - dispatcher +func (d *DockerArchiveCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { + + instructions := d.targetPackage.Manifest.Package + if instructions.Docker.BaseImage == "" { + log.Errorf("BaseImage is a required field in the configuration") + return subcommands.ExitFailure + } + cd := narwhal.ContainerDefinition{ + Image: instructions.Docker.BaseImage, + WorkDir: instructions.Docker.WorkingDir, + Command: instructions.Docker.Exec, + } + + tmpArchiveFile := filepath.Join(d.targetPackage.Path(), fmt.Sprintf("%s-package.tar", d.targetPackage.Name)) + tar := archiver.Tar{MkdirAll: true} + err := tar.Archive(instructions.Artifacts, tmpArchiveFile) if err != nil { + log.Errorf("Could not create archive: %v\n", err) return subcommands.ExitFailure } + defer os.Remove(tmpArchiveFile) - targetPackage, _ := workspace.TargetPackage() + // Default to yourbase container registry. User can pass in an empty string for local registry + registry := d.targetPackage.Name + if d.dockerRegistry != "" { + registry = fmt.Sprintf("%s/%s", strings.TrimRight(d.dockerRegistry, "/"), d.targetPackage.Name) + } - if b.target != "" { - targetPackage, _ = workspace.PackageByName(b.target) + err = narwhal.BuildImageWithArchive(cd, registry, d.dockerTag, tmpArchiveFile, "tmp.tar", instructions.Docker.WorkingDir) + if err != nil { + log.Error(err) + return subcommands.ExitFailure } - instructions := targetPackage.Manifest + return subcommands.ExitSuccess +} + +type TarArchiveCmd struct { + targetPackage Package + tar_file string +} + +func (*TarArchiveCmd) Name() string { return "tar" } +func (*TarArchiveCmd) Synopsis() string { return "Create a tar file" } +func (*TarArchiveCmd) Usage() string { + return `` +} - buildDir := workspace.BuildRoot() - outputDir := filepath.Join(buildDir, "output") - MkdirAsNeeded(outputDir) - archiveFile := fmt.Sprintf("%s-package.tar", targetPackage.Name) - pkgFile := filepath.Join(outputDir, archiveFile) +func (t *TarArchiveCmd) SetFlags(f *flag.FlagSet) { + f.StringVar(&t.tar_file, "tar_file", "", "tar file path and name") +} + +func (t *TarArchiveCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { + + if t.tar_file == "" { + outputDir := filepath.Join(t.targetPackage.BuildRoot(), "output") + MkdirAsNeeded(outputDir) + archiveFile := fmt.Sprintf("%s-package.tar", t.targetPackage.Name) + t.tar_file = filepath.Join(outputDir, archiveFile) + } - if PathExists(pkgFile) { - os.Remove(pkgFile) + if PathExists(t.tar_file) { + os.Remove(t.tar_file) } - log.Infof("Generating package file %s...\n", pkgFile) + log.Infof("Generating package file %s...\n", t.tar_file) tar := archiver.Tar{ MkdirAll: true, } - packageDir := targetPackage.Path + packageDir := t.targetPackage.Path() oldCwd, _ := os.Getwd() _ = os.Chdir(packageDir) - err = tar.Archive(instructions.Package.Artifacts, pkgFile) + err := tar.Archive(t.targetPackage.Manifest.Package.Artifacts, t.tar_file) _ = os.Chdir(oldCwd) if err != nil { diff --git a/cli/remote_build.go b/cli/remote_build.go index 78db0745..d594fc95 100644 --- a/cli/remote_build.go +++ b/cli/remote_build.go @@ -7,6 +7,7 @@ import ( "encoding/json" "flag" "fmt" + "github.com/yourbase/yb/workspace" "io" "io/ioutil" "net/http" @@ -14,7 +15,7 @@ import ( "os" "path" "regexp" - "runtime" + goruntime "runtime" "strconv" "strings" "time" @@ -29,6 +30,7 @@ import ( . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" + "github.com/yourbase/yb/runtime" . "github.com/yourbase/yb/types" ybconfig "github.com/yourbase/yb/config" @@ -94,7 +96,7 @@ func (p *RemoteCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} manifest := targetPackage.Manifest - var target BuildTarget + var target workspace.BuildTarget if len(manifest.BuildTargets) == 0 { target = manifest.Build @@ -110,7 +112,7 @@ func (p *RemoteCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} } } - p.repoDir = targetPackage.Path + p.repoDir = targetPackage.Path() workRepo, err := git.PlainOpen(p.repoDir) if err != nil { @@ -120,11 +122,11 @@ func (p *RemoteCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} if !p.goGitStatus && !p.committed { // Need to check if `git` binary exists and works - if runtime.GOOS == "windows" { + if goruntime.GOOS == "windows" { gitExe = "git.exe" } cmdString := fmt.Sprintf("%s --version", gitExe) - if err := ExecSilently(cmdString, p.repoDir); err != nil { + if err := runtime.ExecSilently(cmdString, p.repoDir); err != nil { if strings.Contains(err.Error(), "executable file not found in $PATH") { log.Warnf("The flag -go-git-status wasn't specified and '%s' wasn't found in PATH", gitExe) } else { @@ -290,7 +292,7 @@ func (p *RemoteCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} } log.Debug("Start backing up the worktree-save") - saver, err := NewWorktreeSave(targetPackage.Path, headCommit.Hash.String(), p.backupWorktree) + saver, err := NewWorktreeSave(targetPackage.Path(), headCommit.Hash.String(), p.backupWorktree) if err != nil { patchErrored() log.Errorf("%s", err) @@ -479,7 +481,7 @@ func (p *RemoteCmd) commandTraverseChanges(worktree *git.Worktree, saver *Worktr cmdString := fmt.Sprintf(gitStatusCmd, gitExe) buf := bytes.NewBuffer(nil) log.Debugf("Executing '%v'...", cmdString) - if err = ExecSilentlyToWriter(cmdString, repoDir, buf); err != nil { + if err = runtime.ExecSilentlyToWriter(cmdString, repoDir, buf); err != nil { return skipped, fmt.Errorf("When running git status: %v", err) } diff --git a/cli/run.go b/cli/run.go index 796f3d51..0bfde0b1 100644 --- a/cli/run.go +++ b/cli/run.go @@ -4,12 +4,11 @@ import ( "context" "flag" "fmt" - "os" - "os/exec" "strings" "github.com/johnewart/subcommands" "github.com/yourbase/yb/plumbing/log" + "github.com/yourbase/yb/runtime" //"path/filepath" ) @@ -35,6 +34,7 @@ Executing the target involves: 3. Start target */ func (b *RunCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { + if len(f.Args()) == 0 { fmt.Println(b.Usage()) return subcommands.ExitFailure @@ -46,43 +46,46 @@ func (b *RunCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s return subcommands.ExitFailure } - instructions := targetPackage.Manifest + runtimeEnv, err := targetPackage.ExecutionRuntime("default") - log.Infof("Setting up dependencies...") - targetPackage.SetupRuntimeDependencies() + if err != nil { + log.Errorf("%v", err) + return subcommands.ExitFailure + } - log.Infof("Setting environment variables...") - for _, property := range instructions.Exec.Environment["default"] { - s := strings.Split(property, "=") - if len(s) == 2 { - log.Infof(" %s", s[0]) - os.Setenv(s[0], s[1]) - } + argList := f.Args() + + runInTarget := false + runtimeTarget := "default" + + if strings.HasPrefix(argList[0], "@") { + runtimeTarget = argList[0][1:] + argList = argList[1:] + runInTarget = true } - if b.environment != "default" { - for _, property := range instructions.Exec.Environment[b.environment] { - s := strings.Split(property, "=") - if len(s) == 2 { - log.Infof(" %s", s[0]) - os.Setenv(s[0], s[1]) - } - } + cmdString := strings.Join(argList, " ") + workDir := "/workspace" + + if runInTarget { + workDir = "/" } - execDir, _ := os.Getwd() - //execDir := filepath.Join(workspace.Path, targetPackage) + log.Infof("Running %s in %s from %s", cmdString, runtimeTarget, workDir) - log.Infof("Running %s from %s", strings.Join(f.Args(), " "), execDir) - cmdName := f.Args()[0] - cmdArgs := f.Args()[1:] - cmd := exec.Command(cmdName, cmdArgs...) - cmd.Dir = execDir - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr + p := runtime.Process{Command: cmdString, Interactive: true, Directory: workDir} - cmd.Run() + if runInTarget { + if err := runtimeEnv.RunInTarget(p, runtimeTarget); err != nil { + log.Errorf("%v", err) + return subcommands.ExitFailure + } + } else { + if err := runtimeEnv.Run(p); err != nil { + log.Errorf("%v", err) + return subcommands.ExitFailure + } + } return subcommands.ExitSuccess } diff --git a/cli/workspace.go b/cli/workspace.go index 3268abc0..b428c95b 100644 --- a/cli/workspace.go +++ b/cli/workspace.go @@ -14,7 +14,6 @@ import ( "path/filepath" "strings" - pkg "github.com/yourbase/yb/packages" util "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" ybtypes "github.com/yourbase/yb/types" @@ -59,7 +58,7 @@ func (w *workspaceLocationCmd) Execute(_ context.Context, f *flag.FlagSet, _ ... if util.PathExists(ybtypes.MANIFEST_FILE) { currentPath, _ := filepath.Abs(".") _, pkgName := filepath.Split(currentPath) - pkg, err := pkg.LoadPackage(pkgName, currentPath) + pkg, err := LoadPackage(pkgName, currentPath) if err != nil { log.Errorf("Error loading package '%s': %v", pkgName, err) return subcommands.ExitFailure diff --git a/config/settings.go b/config/settings.go index 753755fc..a0c38aa8 100644 --- a/config/settings.go +++ b/config/settings.go @@ -6,11 +6,23 @@ import ( "strings" ) +type IsolationType int + +const ( + IsolationContainers IsolationType = 0 + IsolationVMs IsolationType = 1 +) + const ( APP_SETTINGS = "/user/settings" API_TOKEN_VALIDATE = "/apikey/validate/%s" ) +// TODO per-project? +func Isolation() IsolationType { + return IsolationContainers +} + func ShouldUploadBuildLogs() bool { if v, err := GetConfigValue("user", "upload_build_logs"); err == nil { diff --git a/go.mod b/go.mod index cbff5834..63101f39 100644 --- a/go.mod +++ b/go.mod @@ -2,53 +2,58 @@ module github.com/yourbase/yb replace github.com/Sirupsen/logrus => github.com/sirupsen/logrus v1.4.2 +//replace github.com/yourbase/narwhal => /Users/jewart/Work/narwhal + require ( + github.com/Microsoft/hcsshim v0.8.9 // indirect github.com/beholders-eye/diffparser v0.0.0-20190814025112-fcedf0a097ba + github.com/containerd/containerd v1.3.4 // indirect github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6 // indirect github.com/dsnet/compress v0.0.1 // indirect github.com/equinox-io/equinox v1.2.0 github.com/frankban/quicktest v1.5.0 // indirect - github.com/fsouza/go-dockerclient v1.5.0 // indirect github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee // indirect github.com/gobwas/pool v0.2.0 // indirect - github.com/gobwas/ws v1.0.2 - github.com/gogo/protobuf v1.3.1 // indirect + github.com/gobwas/ws v1.0.3 + github.com/golang/protobuf v1.4.2 // indirect github.com/golang/snappy v0.0.1 // indirect - github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f // indirect github.com/johnewart/archiver v3.1.4+incompatible - github.com/johnewart/narwhal v0.0.0-20191030181313-9828f6c923b0 github.com/johnewart/subcommands v0.0.0-20181012225330-46f0354f6315 + github.com/joho/godotenv v1.3.0 github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c // indirect github.com/juju/errors v0.0.0-20190930114154-d42613fe1ab9 // indirect github.com/juju/retry v0.0.0-20180821225755-9058e192b216 // indirect github.com/juju/testing v0.0.0-20191001232224-ce9dec17d28b // indirect github.com/juju/utils v0.0.0-20180820210520-bf9cc5bdd62d // indirect github.com/juju/version v0.0.0-20180108022336-b64dbd566305 // indirect - github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect - github.com/matishsiao/goInfo v0.0.0-20170803142006-617e6440957e + github.com/matishsiao/goInfo v0.0.0-20200404012835-b5f882ee2288 github.com/morikuni/aec v1.0.0 // indirect - github.com/nwaples/rardecode v1.0.0 // indirect - github.com/pierrec/lz4 v2.3.0+incompatible // indirect - github.com/sirupsen/logrus v1.4.2 + github.com/nwaples/rardecode v1.1.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/pierrec/lz4 v2.5.2+incompatible // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/sergi/go-diff v1.1.0 // indirect + github.com/sirupsen/logrus v1.6.0 github.com/smartystreets/assertions v1.0.1 // indirect github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect - github.com/ulikunitz/xz v0.5.6 + github.com/ulikunitz/xz v0.5.7 github.com/waigani/diffparser v0.0.0-20190828052634-7391f219313d // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect - golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 - golang.org/x/mod v0.2.0 - golang.org/x/net v0.0.0-20191112182307-2180aed22343 // indirect + github.com/yourbase/narwhal v0.0.0-20200430180454-3dc9e60c698b + golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 + golang.org/x/mod v0.3.0 + golang.org/x/net v0.0.0-20200513185701-a91f0712d120 // indirect golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect - golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea // indirect - google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a // indirect - google.golang.org/grpc v1.25.1 // indirect - gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect - gopkg.in/ini.v1 v1.51.0 + golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9 // indirect + google.golang.org/genproto v0.0.0-20200514193133-8feb7f20f2a2 // indirect + google.golang.org/grpc v1.29.1 // indirect + gopkg.in/ini.v1 v1.56.0 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect gopkg.in/src-d/go-billy.v4 v4.3.2 gopkg.in/src-d/go-git.v4 v4.13.1 - gopkg.in/yaml.v2 v2.2.5 + gopkg.in/yaml.v2 v2.3.0 ) go 1.13 diff --git a/go.sum b/go.sum index ff9e6b34..b98c84a9 100644 --- a/go.sum +++ b/go.sum @@ -4,10 +4,12 @@ github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Microsoft/go-winio v0.4.12 h1:xAfWHN1IrQ0NJ9TBC0KBZoqLjzDTr1ML+4MywiUOryc= github.com/Microsoft/go-winio v0.4.12/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= -github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= -github.com/Microsoft/hcsshim v0.8.6 h1:ZfF0+zZeYdzMIVMZHKtDKJvLHj76XCuVae/jNkjj0IA= -github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 h1:ygIc8M6trr62pF5DucadTWGdEB4mEyvzi0e2nbcmcyA= +github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= +github.com/Microsoft/hcsshim v0.8.7-0.20191101173118-65519b62243c h1:YMP6olTU903X3gxQJckdmiP8/zkSMq4kN3uipsU9XjU= +github.com/Microsoft/hcsshim v0.8.7-0.20191101173118-65519b62243c/go.mod h1:7xhjOwRV2+0HXGmM0jxaEu+ZiXJFoVZOTfL/dmqbrD8= +github.com/Microsoft/hcsshim v0.8.9 h1:VrfodqvztU8YSOvygU+DN1BGaSGxmrNfqOv5oOuX2Bk= +github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= @@ -16,14 +18,30 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/beholders-eye/diffparser v0.0.0-20190814025112-fcedf0a097ba h1:cmh1vtyJtZtPcoLissi5rpobvQlXcoOlTratoqM1hg4= github.com/beholders-eye/diffparser v0.0.0-20190814025112-fcedf0a097ba/go.mod h1:jzq47XEg545Xz8V+iS6XMc0ajIfbrKVBqW/CdP9YEqs= +github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= +github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.0 h1:xjvXQWABwS2uiv3TWgQt5Uth60Gu86LTGZXMJkjc7rY= github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.4 h1:3o0smo5SKY7H6AJCmJhsnCjR2/V2T8VmiHt7seN2/kI= +github.com/containerd/containerd v1.3.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 h1:4BX8f882bXEDKfWIf0wa8HRvpnBoPszJJXL+TVbBw4M= github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6 h1:NmTXa/uVnDyp0TY5MKi197+3HWcnYWfnHGyaFthlnGw= github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de h1:dlfGmNcE3jDAecLqwKPMNX6nk2qh1c1Vg1/YTzpOOF4= +github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd h1:JNn81o/xG+8NEo3bC/vx9pbi/g2WI8mtP2/nXzu297Y= +github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -32,8 +50,8 @@ github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BU github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.7.3-0.20190309235953-33c3200e0d16 h1:dmUn0SuGx7unKFwxyeQ/oLUHhEfZosEDrpmYM+6MTuc= github.com/docker/docker v0.7.3-0.20190309235953-33c3200e0d16/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v1.4.2-0.20190927142053-ada3c14355ce h1:H3csZuxZESJeeEiOxq4YXPNmLFbjl7u2qVBrAAGX/sA= -github.com/docker/docker v1.4.2-0.20190927142053-ada3c14355ce/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v1.4.2-0.20191101170500-ac7306503d23 h1:oqgGT9O61YAYvI41EBsLePOr+LE6roB0xY4gpkZuFSE= +github.com/docker/docker v1.4.2-0.20191101170500-ac7306503d23/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= @@ -46,6 +64,8 @@ github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdf github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/equinox-io/equinox v1.2.0 h1:bBS7Ou+Y7Jwgmy8TWSYxEh85WctuFn7FPlgbUzX4DBA= github.com/equinox-io/equinox v1.2.0/go.mod h1:6s3HJB0PYUNgs0mxmI8fHdfVl3TQ25ieA/PVfr+eyVo= @@ -53,16 +73,17 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjr github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/frankban/quicktest v1.5.0 h1:Tb4jWdSpdjKzTUicPnY61PZxKbDoGa7ABbrReT3gQVY= github.com/frankban/quicktest v1.5.0/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= -github.com/fsouza/go-dockerclient v1.5.0 h1:7OtayOe5HnoG+KWMHgyyPymwaodnB2IDYuVfseKyxbA= -github.com/fsouza/go-dockerclient v1.5.0/go.mod h1:AqZZK/zFO3phxYxlTsAaeAMSdQ9mgHuhy+bjN034Qds= +github.com/fsouza/go-dockerclient v1.6.0 h1:f7j+AX94143JL1H3TiqSMkM4EcLDI0De1qD4GGn3Hig= +github.com/fsouza/go-dockerclient v1.6.0/go.mod h1:YWwtNPuL4XTX1SKJQk86cWPmmqwx+4np9qfPbb+znGc= github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= -github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/gobwas/ws v1.0.3 h1:ZOigqf7iBxkA4jdQ3am7ATzdlOFp9YzA6NmuvEEZc9g= +github.com/gobwas/ws v1.0.3/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= @@ -73,8 +94,20 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0 h1:kbxbvI4Un1LUWKxufD+BiE6AEExYYgkQLQmLFqA1LFk= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= @@ -83,8 +116,10 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf h1:7+FW5aGwISbqUtkfmIpZJGRgNFg2ioYPvFaUxdqpDsg= -github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f h1:TyqzGm2z1h3AGhjOoRYyeLcW4WlW81MDQkWa+rx/000= @@ -92,6 +127,9 @@ github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f/go.mod h1:wJfORR github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ijc/Gotty v0.0.0-20170406111628-a8b993ba6abd h1:anPrsicrIi2ColgWTVPk+TrN42hJIWlfPHSBP9S0ZkM= github.com/ijc/Gotty v0.0.0-20170406111628-a8b993ba6abd/go.mod h1:3LVOLeyx9XVvwPgrt2be44XgSqndprz1G18rSk8KD84= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= @@ -101,10 +139,10 @@ github.com/johnewart/archiver v3.1.4+incompatible h1:509GRUpPDmAvOLIeqRa1LnbhGvz github.com/johnewart/archiver v3.1.4+incompatible/go.mod h1:RZZSHcya+2fNoD31xh/kmPmwCa7hDaxCm3dFNlPDmaY= github.com/johnewart/go-dockerclient v1.3.7 h1:oyTFh8XsnQ/XdnK8D6RGHQYQSZmZ05sg2BbHbVzwIKQ= github.com/johnewart/go-dockerclient v1.3.7/go.mod h1:FCtHjONdIZH2BKTKbXFJ2Qp8h31iQkiXNboLI3M/TYg= -github.com/johnewart/narwhal v0.0.0-20191030181313-9828f6c923b0 h1:Hrt19BwfJaNsEK29v32epT4+ZPDfL7dZ2vHbkAKvBIQ= -github.com/johnewart/narwhal v0.0.0-20191030181313-9828f6c923b0/go.mod h1:EsUUI8gi1HFwS6NoGEocjpErZEK2ER+b+D0b1qxKM/g= github.com/johnewart/subcommands v0.0.0-20181012225330-46f0354f6315 h1:IHd01G1+fdwW9G/AraiEZzx6GNbGdntaICZjP5PI+oM= github.com/johnewart/subcommands v0.0.0-20181012225330-46f0354f6315/go.mod h1:P9HfiwoEAZ8bQktefnfAuKGlAYDuwfmBqt/cgIBGt+M= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c h1:3UvYABOQRhJAApj9MdCN+Ydv841ETSoy6xLzdmmr/9A= @@ -138,16 +176,16 @@ github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/matishsiao/goInfo v0.0.0-20170803142006-617e6440957e h1:Y+GY+bv5vf1gssphFsGiq6R8qdHxnpDZvYljFnXfhD8= -github.com/matishsiao/goInfo v0.0.0-20170803142006-617e6440957e/go.mod h1:yLZrFIhv+Z20hxHvcZpEyKVQp9HMsOJkXAxx7yDqtvg= +github.com/matishsiao/goInfo v0.0.0-20200404012835-b5f882ee2288 h1:cdM7et8/VlNnSBpq3KbyQWsYLCY0WsB7tvV8Fr0DUNE= +github.com/matishsiao/goInfo v0.0.0-20200404012835-b5f882ee2288/go.mod h1:yLZrFIhv+Z20hxHvcZpEyKVQp9HMsOJkXAxx7yDqtvg= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= @@ -155,30 +193,45 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ= github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U= -github.com/nwaples/rardecode v1.0.0 h1:r7vGuS5akxOnR4JQSkko62RJ1ReCMXxQRPtxsiFMBOs= -github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= +github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= +github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= +github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700 h1:eNUVfm/RFLIi1G7flU5/ZRTHvd4kcVuzfRnL6OFlzCI= +github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/pelletier/go-buffruneio v0.2.0 h1:U4t4R6YkofJ5xHm3dJzuRpPZ0mr5MMCoAWooScCR7aA= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= -github.com/pierrec/lz4 v2.3.0+incompatible h1:CZzRn4Ut9GbUkHlQ7jqBXeZQV41ZSKWFc302ZU6lUTk= -github.com/pierrec/lz4 v2.3.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= +github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= @@ -196,16 +249,26 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.7 h1:YvTNdFzX6+W5m9msiYg/zpkSURPPtOlzbqYjrFn7Yt4= +github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/waigani/diffparser v0.0.0-20190810221810-79c8f4e68fb4 h1:5JA4mMkQVtcQs/ZZcx57GL7sHRXS6TLPs77WkRqsmRk= github.com/waigani/diffparser v0.0.0-20190810221810-79c8f4e68fb4/go.mod h1:CefseIIgCUqtn0B83Lc3+8F2L1V9viokWY2GQlkWGfs= github.com/waigani/diffparser v0.0.0-20190828052634-7391f219313d h1:xQcF7b7cZLWZG/+7A4G7un1qmEDYHIvId9qxRS1mZMs= github.com/waigani/diffparser v0.0.0-20190828052634-7391f219313d/go.mod h1:BzSc3WEF8R+lCaP5iGFRxd5kIXy4JKOZAwNe1w0cdc0= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= +github.com/yourbase/narwhal v0.0.0-20200430180454-3dc9e60c698b h1:0DtxxSaaq5PkME8bPNXBQuKEYiaf+xy4WlsVFrK1YCM= +github.com/yourbase/narwhal v0.0.0-20200430180454-3dc9e60c698b/go.mod h1:iZP/RlitkX0oZU3PAU1hGbxdkJbwNoXirCtsquN8Zfs= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= golang.org/x/crypto v0.0.0-20160824173033-351dc6a5bf92/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -217,8 +280,8 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 h1:pXVtWnwHkrWD9ru3sDxY/qFK/bfc0egRovX91EjWjf4= -golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -227,8 +290,8 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -239,18 +302,21 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191112182307-2180aed22343 h1:00ohfJ4K98s3m6BGUoBd8nyfp4Yl0GoIKvw5abItTjI= -golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120 h1:EZ3cVSzKOlJxAd8e8YAJ7no8nNypTxexh/YE/xW3ZEY= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= @@ -263,11 +329,14 @@ golang.org/x/sys v0.0.0-20190310054646-10058d7d4faa h1:lqti/xP+yD/6zH5TqEwx2MilN golang.org/x/sys v0.0.0-20190310054646-10058d7d4faa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e h1:D5TXcfTk7xF7hvieo4QErS3qqCB4teTffacDWr7CI+0= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea h1:Mz1TMnfJDRJLk8S8OPCoJYgrsp/Se/2TBre2+vwX128= -golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9 h1:YTzHMGlqJu67/uEo1lBv0n3wBXhXNeUbB1XfN2vmTm0= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -285,29 +354,46 @@ golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDq golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200514193133-8feb7f20f2a2 h1:RwW6+LxyOQJ7oeoZ76GIJlwt/O0J5cN2fk+q/jK27kQ= +google.golang.org/genproto v0.0.0-20200514193133-8feb7f20f2a2/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20160105164936-4f90aeace3a2/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.56.0 h1:DPMeDvGTM54DXbPkVIZsp19fp/I2K7zwA/itHYHKo8Y= +gopkg.in/ini.v1 v1.56.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20150529124711-01ee097136da h1:k2jXxOPVI4OLmisXza2wym08Pz6x+ky/f71oJBuA8zg= gopkg.in/mgo.v2 v2.0.0-20150529124711-01ee097136da/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= @@ -323,9 +409,13 @@ gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRN gopkg.in/yaml.v2 v2.0.0-20160715033755-e4d366fc3c79/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= diff --git a/packages/package.go b/packages/package.go deleted file mode 100644 index aea84edd..00000000 --- a/packages/package.go +++ /dev/null @@ -1,91 +0,0 @@ -package packages - -import ( - "crypto/sha256" - "fmt" - "gopkg.in/yaml.v2" - "io/ioutil" - "os" - "os/user" - "path/filepath" - - . "github.com/yourbase/yb/buildpacks" - . "github.com/yourbase/yb/plumbing" - . "github.com/yourbase/yb/types" -) - -type Package struct { - Name string - Path string - Manifest BuildManifest -} - -func LoadPackage(name string, path string) (Package, error) { - manifest := BuildManifest{} - buildYaml := filepath.Join(path, MANIFEST_FILE) - if _, err := os.Stat(buildYaml); os.IsNotExist(err) { - return Package{}, fmt.Errorf("Can't load %s: %v", MANIFEST_FILE, err) - } - - buildyaml, _ := ioutil.ReadFile(buildYaml) - err := yaml.Unmarshal([]byte(buildyaml), &manifest) - if err != nil { - return Package{}, fmt.Errorf("Error loading %s for %s: %v", MANIFEST_FILE, name, err) - } - - p := Package{ - Path: path, - Name: name, - Manifest: manifest, - } - - return p, nil -} - -func (p Package) BuildRoot() string { - // Are we a part of a workspace? - workspaceDir, err := FindWorkspaceRoot() - - if err != nil { - // Nope, just ourselves... - workspacesRoot, exists := os.LookupEnv("YB_WORKSPACES_ROOT") - if !exists { - u, err := user.Current() - if err != nil { - workspacesRoot = "/tmp/yourbase/workspaces" - } else { - workspacesRoot = fmt.Sprintf("%s/.yourbase/workspaces", u.HomeDir) - } - } - - h := sha256.New() - - h.Write([]byte(p.Path)) - workspaceHash := fmt.Sprintf("%x", h.Sum(nil)) - workspaceDir = filepath.Join(workspacesRoot, workspaceHash[0:12]) - } - - MkdirAsNeeded(workspaceDir) - - buildDir := "build" - buildRoot := filepath.Join(workspaceDir, buildDir) - - if _, err := os.Stat(buildRoot); os.IsNotExist(err) { - if err := os.Mkdir(buildRoot, 0700); err != nil { - fmt.Printf("Unable to create build dir in workspace: %v\n", err) - } - } - - return buildRoot - -} - -func (p Package) SetupBuildDependencies() ([]CommandTimer, error) { - return LoadBuildPacks(p.Manifest.Dependencies.Build, p.BuildRoot(), p.Path) -} - -func (p Package) SetupRuntimeDependencies() ([]CommandTimer, error) { - deps := p.Manifest.Dependencies.Runtime - deps = append(deps, p.Manifest.Dependencies.Build...) - return LoadBuildPacks(deps, p.BuildRoot(), p.Path) -} diff --git a/plumbing/util.go b/plumbing/util.go index 98c7a276..cf52024c 100644 --- a/plumbing/util.go +++ b/plumbing/util.go @@ -4,22 +4,15 @@ import ( "bufio" "bytes" "fmt" + "github.com/yourbase/yb/plumbing/log" + . "github.com/yourbase/yb/types" "io" "io/ioutil" - "net/http" "os" - "os/exec" "os/user" "path/filepath" - "regexp" "strings" - "text/template" - "time" - - "github.com/yourbase/yb/plumbing/log" - . "github.com/yourbase/yb/types" - "github.com/google/shlex" "github.com/ulikunitz/xz" "gopkg.in/src-d/go-billy.v4/memfs" @@ -31,125 +24,6 @@ import ( const SNIFFLEN = 8000 -func ExecToStdoutWithExtraEnv(cmdString string, targetDir string, env []string) error { - env = append(os.Environ(), env...) - return ExecToStdoutWithEnv(cmdString, targetDir, env) -} - -func ExecToStdoutWithEnv(cmdString string, targetDir string, env []string) error { - log.Infof("Running: %s in %s", cmdString, targetDir) - cmdArgs, err := shlex.Split(cmdString) - if err != nil { - return fmt.Errorf("Can't parse command string '%s': %v", cmdString, err) - } - - cmd := exec.Command(cmdArgs[0], cmdArgs[1:len(cmdArgs)]...) - cmd.Dir = targetDir - cmd.Stdout = os.Stdout - cmd.Stdin = os.Stdin - cmd.Stderr = os.Stdout - cmd.Env = env - - err = cmd.Run() - - if err != nil { - return fmt.Errorf("Command failed to run with error: %v\n", err) - } - - return nil -} - -func ExecToStdout(cmdString string, targetDir string) error { - return ExecToStdoutWithEnv(cmdString, targetDir, os.Environ()) -} - -func ExecToLog(cmdString string, targetDir string, logPath string) error { - cmdArgs, err := shlex.Split(cmdString) - if err != nil { - return fmt.Errorf("Can't parse command string '%s': %v", cmdString, err) - } - - logfile, err := os.OpenFile(logPath, os.O_RDWR|os.O_CREATE, 0644) - if err != nil { - return fmt.Errorf("Couldn't open log file %s: %v", logPath, err) - } - - defer logfile.Close() - - cmd := exec.Command(cmdArgs[0], cmdArgs[1:len(cmdArgs)]...) - cmd.Dir = targetDir - cmd.Stdout = logfile - cmd.Stdin = os.Stdin - cmd.Stderr = logfile - - err = cmd.Run() - - if err != nil { - return fmt.Errorf("Command '%s' failed to run with error -- see log for information: %s", cmdString, logPath) - } - - return nil - -} - -func ExecSilently(cmdString string, targetDir string) error { - return ExecSilentlyToWriter(cmdString, targetDir, ioutil.Discard) -} -func ExecSilentlyToWriter(cmdString string, targetDir string, writer io.Writer) error { - cmdArgs, err := shlex.Split(cmdString) - if err != nil { - return fmt.Errorf("Can't parse command string '%s': %v", cmdString, err) - } - - cmd := exec.Command(cmdArgs[0], cmdArgs[1:len(cmdArgs)]...) - cmd.Dir = targetDir - cmd.Stdout = writer - cmd.Stdin = os.Stdin - cmd.Stderr = writer - - err = cmd.Run() - - if err != nil { - return fmt.Errorf("Command '%s' failed to run with error %v", cmdString, err) - } - - return nil - -} - -func ExecToLogWithProgressDots(cmdString string, targetDir string, logPath string) error { - stoppedchan := make(chan struct{}) - dotchan := make(chan int) - defer close(stoppedchan) - - go func() { - for { - select { - default: - dotchan <- 1 - time.Sleep(3 * time.Second) - case <-stoppedchan: - return - } - } - }() - - go func() { - for { - select { - default: - case <-dotchan: - fmt.Printf(".") - case <-stoppedchan: - fmt.Printf(" done!\n") - return - } - } - }() - - return ExecToLog(cmdString, targetDir, logPath) -} - func PrependToPath(dir string) { currentPath := os.Getenv("PATH") // Only prepend if it's not already the head; presume that @@ -197,21 +71,6 @@ func MkdirAsNeeded(dir string) error { return nil } -func TemplateToString(templateText string, data interface{}) (string, error) { - t, err := template.New("generic").Parse(templateText) - if err != nil { - return "", err - } - var tpl bytes.Buffer - if err := t.Execute(&tpl, data); err != nil { - log.Errorf("Can't render template:: %v", err) - return "", err - } - - result := tpl.String() - return result, nil -} - func RemoveWritePermission(path string) bool { info, err := os.Stat(path) @@ -227,125 +86,6 @@ func RemoveWritePermission(path string) bool { return true } -func RemoveWritePermissionRecursively(path string) bool { - fileList := []string{} - - err := filepath.Walk(path, func(path string, f os.FileInfo, err error) error { - fileList = append(fileList, path) - return nil - }) - - if err != nil { - return false - } - - for _, file := range fileList { - RemoveWritePermission(file) - } - - return true -} - -func ToolsDir() string { - toolsDir, exists := os.LookupEnv("YB_TOOLS_DIR") - if !exists { - u, err := user.Current() - if err != nil { - toolsDir = "/tmp/yourbase/tools" - } else { - toolsDir = fmt.Sprintf("%s/.yourbase/tools", u.HomeDir) - } - } - - MkdirAsNeeded(toolsDir) - - return toolsDir -} - -func CacheDir() string { - cacheDir, exists := os.LookupEnv("YB_CACHE_DIR") - if !exists { - u, err := user.Current() - if err != nil { - cacheDir = "/tmp/yourbase/cache" - } else { - cacheDir = fmt.Sprintf("%s/.yourbase/cache", u.HomeDir) - } - } - - MkdirAsNeeded(cacheDir) - - return cacheDir -} - -func CacheFilenameForUrl(url string) (string, error) { - cacheDir := CacheDir() - reg, err := regexp.Compile("[^a-zA-Z0-9.]+") - if err != nil { - return "", fmt.Errorf("Can't compile regex: %v", err) - } - - fileName := reg.ReplaceAllString(url, "") - return filepath.Join(cacheDir, fileName), nil -} - -func DownloadFileWithCache(url string) (string, error) { - cacheFilename, err := CacheFilenameForUrl(url) - - if err != nil { - return cacheFilename, err - } - - fileExists := false - fileSizeMismatch := false - - // Exists, don't re-download - if fi, err := os.Stat(cacheFilename); !os.IsNotExist(err) { - fileExists = true - - // try HEAD'ing the URL and comparing to local file - resp, err := http.Head(url) - if err == nil { - if fi.Size() != resp.ContentLength { - log.Infof("Re-downloading %s because remote file and local file differ in size", url) - fileSizeMismatch = true - } - } - - } - - if fileExists && !fileSizeMismatch { - // No mismatch known, but exists, use cached version - log.Infof("Re-using cached version of %s", url) - return cacheFilename, nil - } - - // Otherwise download - err = DownloadFile(cacheFilename, url) - return cacheFilename, err -} - -func DownloadFile(filepath string, url string) error { - - // Get the data - resp, err := http.Get(url) - if err != nil { - return err - } - defer resp.Body.Close() - - // Create the file - out, err := os.Create(filepath) - if err != nil { - return err - } - defer out.Close() - - // Write the body to file - _, err = io.Copy(out, resp.Body) - return err -} - /* func PrintDownloadPercent(done chan int64, path string, total int64) { diff --git a/runtime/common.go b/runtime/common.go new file mode 100644 index 00000000..24c12eb3 --- /dev/null +++ b/runtime/common.go @@ -0,0 +1,167 @@ +package runtime + +import ( + "fmt" + "github.com/google/shlex" + "github.com/yourbase/yb/plumbing/log" + "io" + "io/ioutil" + "os" + "os/exec" + "time" +) + +var ( + runtimeEnvironment = NewRuntime("uuid", "/tmp") +) + + +func SetEnv(key string, value string) error { + runtimeEnvironment.DefaultTarget.SetEnv(key, value) + return nil +} + +func ExecToStdoutWithExtraEnv(cmdString string, targetDir string, env []string) error { + if runtimeEnvironment == nil { + return fmt.Errorf("No runtime environment determined") + } + + return ExecToStdoutWithEnv(cmdString, targetDir, env) +} + +func ExecToStdoutWithEnv(cmdString string, targetDir string, env []string) error { + if runtimeEnvironment == nil { + return fmt.Errorf("No runtime environment determined") + } + + log.Infof("Running: %s in %s", cmdString, targetDir) + + p := Process{ + Command: cmdString, + Directory: targetDir, + Interactive: true, + Environment: env, + } + + if err := runtimeEnvironment.Run(p); err != nil { + return fmt.Errorf("Command failed to run with error: %v", err) + } + + return nil +} + +func ExecToStdout(cmdString string, targetDir string) error { + if runtimeEnvironment == nil { + return fmt.Errorf("No runtime environment determined") + } + + return ExecToStdoutWithEnv(cmdString, targetDir, os.Environ()) +} + +func ExecToLog(cmdString string, targetDir string, logPath string) error { + if runtimeEnvironment == nil { + return fmt.Errorf("No runtime environment determined") + } + + cmdArgs, err := shlex.Split(cmdString) + if err != nil { + return fmt.Errorf("Can't parse command string '%s': %v", cmdString, err) + } + + logfile, err := os.OpenFile(logPath, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return fmt.Errorf("Couldn't open log file %s: %v", logPath, err) + } + + defer logfile.Close() + + cmd := exec.Command(cmdArgs[0], cmdArgs[1:len(cmdArgs)]...) + cmd.Dir = targetDir + cmd.Stdout = logfile + cmd.Stdin = os.Stdin + cmd.Stderr = logfile + + err = cmd.Run() + + if err != nil { + return fmt.Errorf("Command '%s' failed to run with error -- see log for information: %s", cmdString, logPath) + } + + return nil + +} + +func ExecSilently(cmdString string, targetDir string) error { + if runtimeEnvironment == nil { + return fmt.Errorf("No runtime environment determined") + } + + return ExecSilentlyToWriter(cmdString, targetDir, ioutil.Discard) +} + +func ExecSilentlyToWriter(cmdString string, targetDir string, writer io.Writer) error { + if runtimeEnvironment == nil { + return fmt.Errorf("No runtime environment determined") + } + + cmdArgs, err := shlex.Split(cmdString) + if err != nil { + return fmt.Errorf("Can't parse command string '%s': %v", cmdString, err) + } + + cmd := exec.Command(cmdArgs[0], cmdArgs[1:len(cmdArgs)]...) + cmd.Dir = targetDir + cmd.Stdout = writer + cmd.Stdin = os.Stdin + cmd.Stderr = writer + + err = cmd.Run() + + if err != nil { + return fmt.Errorf("Command '%s' failed to run with error %v", cmdString, err) + } + + return nil + +} + +func ExecToLogWithProgressDots(cmdString string, targetDir string, logPath string) error { + if runtimeEnvironment == nil { + return fmt.Errorf("No runtime environment determined!") + } + + stoppedchan := make(chan struct{}) + dotchan := make(chan int) + defer close(stoppedchan) + + go func() { + for { + select { + default: + dotchan <- 1 + time.Sleep(3 * time.Second) + case <-stoppedchan: + return + } + } + }() + + go func() { + for { + select { + default: + case <-dotchan: + fmt.Printf(".") + case <-stoppedchan: + fmt.Printf(" done!\n") + return + } + } + }() + + return ExecToLog(cmdString, targetDir, logPath) +} + +func Init(identifier string, localWorkDir string) { + runtimeEnvironment = NewRuntime(identifier, localWorkDir) +} diff --git a/runtime/containers.go b/runtime/containers.go new file mode 100644 index 00000000..9b78f83d --- /dev/null +++ b/runtime/containers.go @@ -0,0 +1,333 @@ +package runtime + +import ( + "bufio" + "bytes" + "fmt" + "github.com/yourbase/yb/plumbing/log" + "io" + "io/ioutil" + "net" + "os" + "strings" + + "github.com/yourbase/narwhal" +) + +type ContainerTarget struct { + Target + Container *narwhal.Container + Environment []string + workDir string +} + +func (t *ContainerTarget) OS() Os { + return Linux +} + +func (t *ContainerTarget) OSVersion() string { + buf := bytes.Buffer{} + //err := t.Container.DownloadDirectoryToWriter("/etc/os-release", &buf) + err := t.Container.ExecToWriter("cat /etc/os-release", "/", &buf) + + if err != nil { + return "unknown" + } + + rd := bufio.NewReader(&buf) + for { + line, err := rd.ReadString('\n') + if err != nil { + return "" + } + if strings.HasPrefix(line, "VERSION_CODENAME") { + line = strings.TrimSuffix(line, "\n") + parts := strings.Split(line, "=") + if len(parts) == 2 { + return parts[1] + } + } + } + return "" + +} + +func (t *ContainerTarget) Architecture() Architecture { + return Amd64 +} + +func (t *ContainerTarget) ToolsDir() string { + t.Container.MakeDirectoryInContainer("/opt/yb/tools") + return "/opt/yb/tools" +} + +func (t *ContainerTarget) PathExists(path string) bool { + // Assume we can use stat for now + statCmd := fmt.Sprintf("stat %s", path) + + err := t.Container.ExecToWriter(statCmd, "/", ioutil.Discard) + if err != nil { + if execerr, ok := err.(*narwhal.ExecError); ok { + if execerr.ExitCode != 0 { + return false + } else { + // Error but retcode zero... ? + return true + } + } + return false + } + + return true +} + +func (t *ContainerTarget) String() string { + return fmt.Sprintf("Container ID: %s workDir: %s", t.Container.Id, t.workDir) +} + +func (t *ContainerTarget) CacheDir() string { + t.Container.MakeDirectoryInContainer("/opt/yb/cache") + return "/opt/yb/cache" +} + +func (t *ContainerTarget) PrependToPath(dir string) { + pathSet := false + for i, e := range t.Environment { + parts := strings.Split(e, "=") + k := parts[0] + v := parts[1] + if k == "PATH" { + newpath := fmt.Sprintf("PATH=%s:%s", dir, v) + t.Environment[i] = newpath + pathSet = true + } + } + + if !pathSet { + path := fmt.Sprintf("%s:%s", dir, t.GetDefaultPath()) + t.SetEnv("PATH", path) + } +} + +func (t *ContainerTarget) GetDefaultPath() string { + // TODO check other OS defaults, this works for Linux containers + return "/usr/bin:/bin:/sbin:/usr/sbin" +} + +func (t *ContainerTarget) UploadFile(src string, dest string) error { + parts := strings.Split(dest, "/") + filename := parts[len(parts)-1] + destDir := strings.Join(parts[0:len(parts)-1], "/") + if destDir == "" { + destDir = "/" + } + log.Infof("Uploading %s to %s in %s", src, filename, destDir) + err := t.Container.UploadFile(src, filename, destDir) + if err != nil { + return fmt.Errorf("Couldn't upload file to container: %v", err) + } + log.Infof("Done") + return nil + +} + +func (t *ContainerTarget) DownloadFile(url string) (string, error) { + // TODO: upload if locally found + + localFile, err := DownloadFileWithCache(url) + parts := strings.Split(url, "/") + filename := parts[len(parts)-1] + outputFilename := fmt.Sprintf("/tmp/%s", filename) + + if err == nil { + // Downloaded locally, inject + log.Infof("Injecting locally cached file %s as %s", localFile, outputFilename) + err = t.Container.UploadFile(localFile, filename, "/tmp") + } + + // If download or injection failed, fallback + if err != nil { + log.Infof("Failed to download and inject file: %v", err) + log.Infof("Will download via curl in container") + p := Process{ + Command: fmt.Sprintf("curl %s -o %s", url, outputFilename), + Directory: "/tmp", + Interactive: false, + } + + if err := t.Run(p); err != nil { + return "", err + } + } + + return outputFilename, nil +} + +func (t *ContainerTarget) Unarchive(src string, dst string) error { + var command string + t.Container.MakeDirectoryInContainer(dst) + + if strings.HasSuffix(src, "tar.gz") { + command = fmt.Sprintf("tar zxf %s -C %s", src, dst) + } + + if strings.HasSuffix(src, "tar.bz2") { + command = fmt.Sprintf("tar jxf %s -C %s", src, dst) + } + + p := Process{ + Command: command, + Interactive: false, + Directory: "/tmp", + Environment: nil, + } + + return t.Run(p) +} + +func (t *ContainerTarget) WorkDir() string { + return t.workDir +} + +func (t *ContainerTarget) SetEnv(key string, value string) error { + envString := fmt.Sprintf("%s=%s", key, value) + if t.Environment == nil { + t.Environment = make([]string, 0) + } + t.Environment = append(t.Environment, envString) + return nil +} + +func (t *ContainerTarget) Run(p Process) error { + log.Infof("Running container process: %s\n", p.Command) + + p.Environment = append(p.Environment, t.Environment...) + p.Environment = append(p.Environment, t.Container.Definition.Environment...) + + log.Debugf("Process env: %v", p.Environment) + + var output io.Writer + + output = os.Stdout + if p.Output != nil { + output = *(p.Output) + } + + if p.Interactive { + return t.Container.ExecInteractivelyWithEnv(p.Command, p.Directory, p.Environment) + } else { + err := t.Container.ExecToWriterWithEnv(p.Command, p.Directory, output, p.Environment) + if execerr, ok := err.(*narwhal.ExecError); ok { + return &TargetRunError{ + ExitCode: execerr.ExitCode, + Message: execerr.Message, + } + } + + return err + } +} + +func (t *ContainerTarget) WriteFileContents(contents string, remotepath string) error { + if tmpfile, err := ioutil.TempFile("", "injection"); err != nil { + log.Infof("Couldn't make temporary file: %v", err) + return err + } else { + defer os.Remove(tmpfile.Name()) + + if _, err := tmpfile.Write([]byte(contents)); err != nil { + log.Warnf("Couldn't write data to file: %v", err) + return err + } + + tmpfile.Sync() + + log.Infof("Will inject %s as %s", tmpfile.Name(), remotepath) + t.UploadFile(tmpfile.Name(), remotepath) + tmpfile.Close() + } + + return nil +} + +func GetFreePort() (int, error) { + addr, err := net.ResolveTCPAddr("tcp", "localhost:0") + if err != nil { + return 0, err + } + + l, err := net.ListenTCP("tcp", addr) + if err != nil { + return 0, err + } + defer l.Close() + return l.Addr().(*net.TCPAddr).Port, nil +} + +func getDockerInterfaceIp() (string, error) { + ip, err := getOutboundIP() + if err != nil { + return "", err + } + + return ip.String(), nil +} + +func getOutboundIP() (net.IP, error) { + conn, err := net.Dial("udp", "8.8.8.8:80") + if err != nil { + return net.IP{}, err + } + defer conn.Close() + + localAddr := conn.LocalAddr().(*net.UDPAddr) + + return localAddr.IP, nil +} + +func ForwardUnixSocketToTcp(unixSocket string) (string, error) { + port, err := GetFreePort() + if err != nil { + return "", err + } + + dockerInterfaceIp, err := getDockerInterfaceIp() + if err != nil { + return "", err + } + + listenAddr := fmt.Sprintf("%s:%d", dockerInterfaceIp, port) + l, err := net.Listen("tcp", listenAddr) + if err != nil { + log.Fatal(err) + } + + log.Infof("Forwarding %s on %s", unixSocket, listenAddr) + + go func() { + for { + conn, err := l.Accept() + if err != nil { + log.Errorf("accept failed: %s", err) + continue + } + + go func(tconn net.Conn, unixSocket string) { + defer conn.Close() + uconn, err := net.Dial("unix", unixSocket) + if err != nil { + log.Warnf("unix dial failed: %s", err) + return + } + log.Infof("Opened %s", unixSocket) + // copy tcp request -> unix socket + go io.Copy(tconn, uconn) + // copy unix socket -> tcp connection + io.Copy(uconn, tconn) + log.Infof("Done forwarding.") + }(conn, unixSocket) + } + }() + + return listenAddr, nil +} diff --git a/runtime/metal.go b/runtime/metal.go new file mode 100644 index 00000000..8c3ca283 --- /dev/null +++ b/runtime/metal.go @@ -0,0 +1,387 @@ +package runtime + +import ( + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "os/exec" + "os/user" + "path/filepath" + "regexp" + goruntime "runtime" + "strings" + "time" + + "github.com/google/shlex" + "github.com/johnewart/archiver" + "github.com/yourbase/yb/plumbing" + "github.com/yourbase/yb/plumbing/log" +) + +type MetalTarget struct { + Target + workDir string +} + +func (t *MetalTarget) OS() Os { + switch goruntime.GOOS { + case "linux": + return Linux + case "darwin": + return Darwin + case "windows": + return Windows + } + + log.Fatal("Running on an unknown OS - things will likely fail miserably...") + return Unknown +} + +// TODO: Add support for the OS's here +func (t *MetalTarget) OSVersion() string { + return "unknown" +} + +func (t *MetalTarget) Architecture() Architecture { + return Amd64 +} + +func (t *MetalTarget) WriteContentsToFile(contents string, filename string) error { + f, err := os.Open(filename) + if err != nil { + return err + } + + defer f.Close() + + if _, err := f.Write([]byte(contents)); err != nil { + return err + } + + return nil +} + +func (t *MetalTarget) ToolsDir() string { + toolsDir, exists := os.LookupEnv("YB_TOOLS_DIR") + if !exists { + u, err := user.Current() + if err != nil { + toolsDir = "/tmp/yourbase/tools" + } else { + toolsDir = fmt.Sprintf("%s/.yourbase/tools", u.HomeDir) + } + } + + plumbing.MkdirAsNeeded(toolsDir) + + return toolsDir +} + +func (t *MetalTarget) PathExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +func (t *MetalTarget) String() string { + return fmt.Sprintf("Metal target: %s", t.workDir) +} + +func (t *MetalTarget) PrependToPath(dir string) { + currentPath := t.GetDefaultPath() + // Only prepend if it's not already the head; presume that + // whomever asked for this wants to be at the front so it's okay if it's + // duplicated later + if !strings.HasPrefix(currentPath, dir) { + newPath := fmt.Sprintf("%s:%s", dir, currentPath) + t.SetEnv("PATH", newPath) + } +} + +func (t *MetalTarget) GetDefaultPath() string { + // TODO check other OS defaults, this works for Linux containers, maybe for a Mac host we should use "path" + return os.Getenv("PATH") +} + +func (t *MetalTarget) CacheDir() string { + cacheDir, exists := os.LookupEnv("YB_CACHE_DIR") + if !exists { + u, err := user.Current() + if err != nil { + cacheDir = "/tmp/yourbase/cache" + } else { + cacheDir = fmt.Sprintf("%s/.yourbase/cache", u.HomeDir) + } + } + + plumbing.MkdirAsNeeded(cacheDir) + + return cacheDir +} + +func (t *MetalTarget) UploadFile(src string, dst string) error { + source, err := os.Open(src) + if err != nil { + return err + } + + destination, err := os.Open(dst) + if err != nil { + return err + } + + buf := make([]byte, 128*1024) + for { + n, err := source.Read(buf) + if err != nil && err != io.EOF { + return err + } + if n == 0 { + break + } + + if _, err := destination.Write(buf[:n]); err != nil { + return err + } + } + + source.Close() + destination.Close() + + return nil +} + +func (t *MetalTarget) DownloadFile(url string) (string, error) { + + localFile, err := DownloadFileToCache(url, t.CacheDir()) + + if err != nil { + log.Errorf("Unable to download: %v", err) + } + + return localFile, err +} + +func (t *MetalTarget) Unarchive(src string, dst string) error { + err := archiver.Unarchive(src, dst) + + if err != nil { + log.Errorf("Unable to decompress: %v", err) + } + + return err +} + +func (t *MetalTarget) WorkDir() string { + return t.workDir +} + +func (t *MetalTarget) Run(p Process) error { + return t.ExecToStdout(p.Command, p.Directory) +} + +func (t *MetalTarget) SetEnv(key string, value string) error { + return os.Setenv(key, value) +} + +func (t *MetalTarget) ExecToStdoutWithExtraEnv(cmdString string, targetDir string, env []string) error { + env = append(os.Environ(), env...) + return t.ExecToStdoutWithEnv(cmdString, targetDir, env) +} + +func (t *MetalTarget) ExecToStdoutWithEnv(cmdString string, targetDir string, env []string) error { + log.Infof("Running: %s in %s", cmdString, targetDir) + cmdArgs, err := shlex.Split(cmdString) + if err != nil { + return fmt.Errorf("Can't parse command string '%s': %v", cmdString, err) + } + + cmd := exec.Command(cmdArgs[0], cmdArgs[1:len(cmdArgs)]...) + cmd.Dir = targetDir + cmd.Stdout = os.Stdout + cmd.Stdin = os.Stdin + cmd.Stderr = os.Stdout + cmd.Env = env + + err = cmd.Run() + + if err != nil { + return fmt.Errorf("Command failed to run with error: %v", err) + } + + return nil +} + +func (t *MetalTarget) ExecToStdout(cmdString string, targetDir string) error { + return t.ExecToStdoutWithEnv(cmdString, targetDir, os.Environ()) +} + +func (t *MetalTarget) ExecToLog(cmdString string, targetDir string, logPath string) error { + + cmdArgs, err := shlex.Split(cmdString) + if err != nil { + return fmt.Errorf("Can't parse command string '%s': %v", cmdString, err) + } + + logfile, err := os.OpenFile(logPath, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return fmt.Errorf("Couldn't open log file %s: %v", logPath, err) + } + + defer logfile.Close() + + cmd := exec.Command(cmdArgs[0], cmdArgs[1:len(cmdArgs)]...) + cmd.Dir = targetDir + cmd.Stdout = logfile + cmd.Stdin = os.Stdin + cmd.Stderr = logfile + + err = cmd.Run() + + if err != nil { + return fmt.Errorf("Command '%s' failed to run with error -- see log for information: %s", cmdString, logPath) + } + + return nil + +} + +func (t *MetalTarget) ExecSilently(cmdString string, targetDir string) error { + return t.ExecSilentlyToWriter(cmdString, targetDir, ioutil.Discard) +} + +func (t *MetalTarget) ExecSilentlyToWriter(cmdString string, targetDir string, writer io.Writer) error { + cmdArgs, err := shlex.Split(cmdString) + if err != nil { + return fmt.Errorf("Can't parse command string '%s': %v", cmdString, err) + } + + cmd := exec.Command(cmdArgs[0], cmdArgs[1:len(cmdArgs)]...) + cmd.Dir = targetDir + cmd.Stdout = writer + cmd.Stdin = os.Stdin + cmd.Stderr = writer + + err = cmd.Run() + + if err != nil { + return fmt.Errorf("Command '%s' failed to run with error %v", cmdString, err) + } + + return nil + +} + +func (t *MetalTarget) ExecToLogWithProgressDots(cmdString string, targetDir string, logPath string) error { + stoppedchan := make(chan struct{}) + dotchan := make(chan int) + defer close(stoppedchan) + + go func() { + for { + select { + default: + dotchan <- 1 + time.Sleep(3 * time.Second) + case <-stoppedchan: + return + } + } + }() + + go func() { + for { + select { + default: + case <-dotchan: + fmt.Printf(".") + case <-stoppedchan: + fmt.Printf(" done!\n") + return + } + } + }() + + return t.ExecToLog(cmdString, targetDir, logPath) +} + +func CacheFilenameForUrl(url string) (string, error) { + reg, err := regexp.Compile("[^a-zA-Z0-9.]+") + if err != nil { + return "", fmt.Errorf("Can't compile regex: %v", err) + } + fileName := reg.ReplaceAllString(url, "") + return fileName, nil +} + +func DownloadFileWithCache(url string) (string, error) { + cacheDir := "/tmp/yourbase" + if homeDir, exists := os.LookupEnv("HOME"); exists { + cacheDir = filepath.Join(homeDir, ".cache", "yourbase") + } + + return DownloadFileToCache(url, cacheDir) +} + +func DownloadFileToCache(url string, cachedir string) (string, error) { + + filename, err := CacheFilenameForUrl(url) + if err != nil { + return "", err + } + + os.MkdirAll(cachedir, 0700) + + cacheFilename := filepath.Join(cachedir, filename) + log.Infof("Downloading %s to cache as %s", url, cacheFilename) + + fileExists := false + fileSizeMismatch := false + + // Exists, don't re-download + if fi, err := os.Stat(cacheFilename); !os.IsNotExist(err) && fi != nil { + fileExists = true + + // try HEAD'ing the URL and comparing to local file + resp, err := http.Head(url) + if err == nil { + if fi.Size() != resp.ContentLength { + log.Infof("Re-downloading %s because remote file and local file differ in size", url) + fileSizeMismatch = true + } + } + + } + + if fileExists && !fileSizeMismatch { + // No mismatch known, but exists, use cached version + log.Infof("Re-using cached version of %s", url) + return cacheFilename, nil + } + + // Otherwise download + err = doDownload(cacheFilename, url) + return cacheFilename, err +} + +func doDownload(filepath string, url string) error { + + // Get the data + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + // Create the file + out, err := os.Create(filepath) + if err != nil { + return err + } + defer out.Close() + + // Write the body to file + _, err = io.Copy(out, resp.Body) + return err +} diff --git a/runtime/runtime.go b/runtime/runtime.go new file mode 100644 index 00000000..24c84183 --- /dev/null +++ b/runtime/runtime.go @@ -0,0 +1,189 @@ +package runtime + +import ( + "fmt" + "io" + "strings" + + goruntime "runtime" + + "github.com/yourbase/narwhal" +) + +type Os int + +const ( + Linux Os = iota + Darwin + Windows + Unknown +) + +type Architecture int + +const ( + Amd64 Architecture = iota + i386 +) + +type TargetRunError struct { + ExitCode int + Message string +} + +func (e *TargetRunError) Error() string { + return e.Message +} + +type Target interface { + Run(p Process) error + SetEnv(key string, value string) error + UploadFile(localPath string, remotePath string) error + WriteFileContents(contents string, remotepath string) error + WorkDir() string + CacheDir() string + DownloadFile(url string) (string, error) + Unarchive(src string, dst string) error + PrependToPath(dir string) + PathExists(path string) bool + GetDefaultPath() string + ToolsDir() string + OS() Os + OSVersion() string + Architecture() Architecture +} + +type Process struct { + Command string + Interactive bool + Directory string + Environment []string + Output *io.Writer +} + +type Runtime struct { + Identifier string + LocalWorkDir string + Targets map[string]Target + ContainerServiceContext *narwhal.ServiceContext + DefaultTarget Target +} + +func (r *Runtime) AddTarget(targetId string, t Target) error { + if _, exists := r.Targets[targetId]; exists { + return fmt.Errorf("Unable to add target with id %s - already exists", targetId) + } + + r.Targets[targetId] = t + + return nil +} + +func (r *Runtime) Run(p Process) error { + return r.DefaultTarget.Run(p) +} + +func (r *Runtime) RunInTarget(p Process, targetId string) error { + if target, exists := r.Targets[targetId]; exists { + return target.Run(p) + } else { + return fmt.Errorf("Unable to find target %s in runtime", targetId) + } +} + +func (r *Runtime) AddContainer(cd narwhal.ContainerDefinition) (*ContainerTarget, error) { + if r.ContainerServiceContext == nil { + sc, err := narwhal.NewServiceContextWithId(r.Identifier, r.LocalWorkDir) + if err != nil { + return nil, err + } + r.ContainerServiceContext = sc + } + + container, err := r.ContainerServiceContext.StartContainer(cd) + if err != nil { + return nil, fmt.Errorf("Couldn't start container %s in service context: %v", cd.Label, err) + } + + tgt := &ContainerTarget{ + Container: container, + } + + r.AddTarget(cd.Label, tgt) + return tgt, nil +} + +func NewRuntime(identifier string, localWorkDir string) *Runtime { + return &Runtime{ + Identifier: identifier, + LocalWorkDir: localWorkDir, + Targets: make(map[string]Target), + DefaultTarget: &MetalTarget{}, + } +} + +func (r *Runtime) Shutdown() error { + + if r.ContainerServiceContext != nil { + if err := r.ContainerServiceContext.TearDown(); err != nil { + return err + } + } + + return nil +} + +func (r *Runtime) EnvironmentData() RuntimeEnvironmentData { + return RuntimeEnvironmentData{ + Containers: ContainerData{ + serviceCtx: r.ContainerServiceContext, + }, + } +} + +type RuntimeEnvironmentData struct { + Containers ContainerData +} + +type ContainerData struct { + serviceCtx *narwhal.ServiceContext +} + +func (c ContainerData) IP(label string) string { + // Check service context + if c.serviceCtx != nil { + if buildContainer, ok := c.serviceCtx.Containers[label]; ok { + if ipv4, err := buildContainer.IPv4Address(); err == nil { + return ipv4 + } + } + } + + return "" +} + +func (c ContainerData) Environment() map[string]string { + result := make(map[string]string) + if c.serviceCtx != nil { + for label, container := range c.serviceCtx.Containers { + if ipv4, err := container.IPv4Address(); err == nil { + key := fmt.Sprintf("YB_CONTAINER_%s_IP", strings.ToUpper(label)) + result[key] = ipv4 + } + } + } + return result +} + +func HostOS() Os { + switch goruntime.GOOS { + case "darwin": + return Darwin + case "linux": + return Linux + case "windows": + return Windows + default: + return Unknown + } +} diff --git a/plumbing/sandbox_darwin.go b/runtime/sandbox_darwin.go similarity index 95% rename from plumbing/sandbox_darwin.go rename to runtime/sandbox_darwin.go index af70d490..9859f9e8 100644 --- a/plumbing/sandbox_darwin.go +++ b/runtime/sandbox_darwin.go @@ -1,4 +1,4 @@ -package plumbing +package runtime /* import ( @@ -59,7 +59,7 @@ func ExecInSandbox(command string, workingDir string) error { } sandboxFile.Close() sandboxedCommand := fmt.Sprintf("sandbox-exec -f %s %s", sandboxFile.Name(), command) - return ExecToStdout(sandboxedCommand, workingDir) + return runtime.ExecToStdout(sandboxedCommand, workingDir) */ return ExecToStdout(command, workingDir) diff --git a/plumbing/sandbox_linux.go b/runtime/sandbox_linux.go similarity index 89% rename from plumbing/sandbox_linux.go rename to runtime/sandbox_linux.go index 2ec244db..8ebdd0b4 100644 --- a/plumbing/sandbox_linux.go +++ b/runtime/sandbox_linux.go @@ -1,4 +1,4 @@ -package plumbing +package runtime // Currently just a passthrough func ExecInSandbox(command string, workingDir string) error { diff --git a/plumbing/sandbox_windows.go b/runtime/sandbox_windows.go similarity index 89% rename from plumbing/sandbox_windows.go rename to runtime/sandbox_windows.go index 2ec244db..8ebdd0b4 100644 --- a/plumbing/sandbox_windows.go +++ b/runtime/sandbox_windows.go @@ -1,4 +1,4 @@ -package plumbing +package runtime // Currently just a passthrough func ExecInSandbox(command string, workingDir string) error { diff --git a/types/build_manifest.go b/types/build_manifest.go deleted file mode 100644 index 4bf36b37..00000000 --- a/types/build_manifest.go +++ /dev/null @@ -1,75 +0,0 @@ -package types - -import ( - "bytes" - "crypto/sha256" - "fmt" -) - -const DependencyChecksumLength = 12 - -func (b BuildManifest) BuildDependenciesChecksum() string { - buf := bytes.Buffer{} - for _, dep := range b.Dependencies.Build { - buf.Write([]byte(dep)) - } - - sum := sha256.Sum256(buf.Bytes()) - return fmt.Sprintf("%x", sum[:DependencyChecksumLength]) -} - -func (b BuildManifest) IsTargetSandboxed(target BuildTarget) bool { - return b.Sandbox || target.Sandbox -} - -// XXX: Support more than one level? Intuitively that seems like it will breed un-needed complexity -func (b BuildManifest) ResolveBuildTargets(targetName string) ([]BuildTarget, error) { - targetList := make([]BuildTarget, 0) - - target, err := b.BuildTarget(targetName) - if err != nil { - return targetList, err - } - - if len(target.BuildAfter) > 0 { - for _, depName := range target.BuildAfter { - depTarget, err := b.BuildTarget(depName) - if err != nil { - return targetList, err - } - targetList = append(targetList, depTarget) - } - } - - targetList = append(targetList, target) - - return targetList, nil -} - -func (b BuildManifest) CIBuild(buildName string) (CIBuild, error) { - for _, build := range b.CI.CIBuilds { - if build.Name == buildName { - return build, nil - } - } - return CIBuild{}, fmt.Errorf("No such CI build '%s' in build manifest", buildName) -} - -func (b BuildManifest) BuildTarget(targetName string) (BuildTarget, error) { - for _, target := range b.BuildTargets { - if target.Name == targetName { - return target, nil - } - } - return BuildTarget{}, fmt.Errorf("No such target '%s' in build manifest", targetName) -} - -func (b BuildManifest) BuildTargetList() []string { - targets := make([]string, 0, len(b.BuildTargets)) - - for _, t := range b.BuildTargets { - targets = append(targets, t.Name) - } - - return targets -} diff --git a/types/types.go b/types/types.go index 17e1da0c..bb50b57c 100644 --- a/types/types.go +++ b/types/types.go @@ -1,100 +1,16 @@ package types -import ( - "time" - - "github.com/johnewart/narwhal" -) - const ( MANIFEST_FILE = ".yourbase.yml" DOCS_URL = "https://docs.yourbase.io" DEFAULT_YB_CONTAINER = "yourbase/yb_ubuntu:18.04" ) -type BuildManifest struct { - Dependencies DependencySet `yaml:"dependencies"` - Sandbox bool `yaml:"sandbox"` - BuildTargets []BuildTarget `yaml:"build_targets"` - Build BuildTarget `yaml:"build"` - Exec ExecPhase `yaml:"exec"` - Package PackagePhase `yaml:"package"` - CI CIInfo `yaml:"ci"` -} - -type CIInfo struct { - CIBuilds []CIBuild `yaml:"builds"` -} - -type CIBuild struct { - Name string `yaml:"name"` - BuildTarget string `yaml:"build_target"` - When string `yaml:"when"` -} - -type PackagePhase struct { - Artifacts []string `yaml:"artifacts"` -} - -type DependencySet struct { - Build []string `yaml:"build"` - Runtime []string `yaml:"runtime"` -} - -type ExecPhase struct { - Name string `yaml:"name"` - Dependencies ExecDependencies `yaml:"dependencies"` - Container narwhal.ContainerDefinition `yaml:"container"` - Commands []string `yaml:"commands"` - Ports []string `yaml:"ports"` - Environment map[string][]string `yaml:"environment"` - LogFiles []string `yaml:"logfiles"` - Sandbox bool `yaml:"sandbox"` - HostOnly bool `yaml:"host_only"` - BuildFirst []string `yaml:"build_first"` -} - -type BuildDependencies struct { - Containers map[string]narwhal.ContainerDefinition `yaml:"containers"` -} - -func (b BuildDependencies) ContainerList() []narwhal.ContainerDefinition { - containers := make([]narwhal.ContainerDefinition, 0) - for label, c := range b.Containers { - c.Label = label - containers = append(containers, c) - } - return containers -} - -type ExecDependencies struct { - Containers map[string]narwhal.ContainerDefinition `yaml:"containers"` -} - -func (b ExecDependencies) ContainerList() []narwhal.ContainerDefinition { - containers := make([]narwhal.ContainerDefinition, 0) - for label, c := range b.Containers { - c.Label = label - containers = append(containers, c) - } - return containers +type EnvVariable struct { + Key string + Value string } -type BuildTarget struct { - Name string `yaml:"name"` - Container narwhal.ContainerDefinition `yaml:"container"` - Tools []string `yaml:"tools"` - Commands []string `yaml:"commands"` - Artifacts []string `yaml:"artifacts"` - CachePaths []string `yaml:"cache_paths"` - Sandbox bool `yaml:"sandbox"` - HostOnly bool `yaml:"host_only"` - Root string `yaml:"root"` - Environment []string `yaml:"environment"` - Tags map[string]string `yaml:"tags"` - BuildAfter []string `yaml:"build_after"` - Dependencies BuildDependencies `yaml:"dependencies"` -} // API Responses -- TODO use Swagger instead, this is silly type Project struct { @@ -116,16 +32,7 @@ type WorktreeSave struct { Enabled bool } -type CommandTimer struct { - Command string - StartTime time.Time - EndTime time.Time -} -type TargetTimer struct { - Name string - Timers []CommandTimer -} type BuildTool interface { Install() error diff --git a/workspace/build_manifest.go b/workspace/build_manifest.go new file mode 100644 index 00000000..84d8d8d1 --- /dev/null +++ b/workspace/build_manifest.go @@ -0,0 +1,202 @@ +package workspace + +import ( + "bytes" + "crypto/sha256" + "fmt" + "strings" + "text/template" + + "github.com/joho/godotenv" + "github.com/yourbase/narwhal" + "github.com/yourbase/yb/plumbing/log" + "github.com/yourbase/yb/runtime" +) + +const DependencyChecksumLength = 12 + +type CIInfo struct { + CIBuilds []CIBuild `yaml:"builds"` +} + +type CIBuild struct { + Name string `yaml:"name"` + BuildTarget string `yaml:"build_target"` + When string `yaml:"when"` +} + +type PackagePhase struct { + Artifacts []string `yaml:"artifacts"` + Docker DockerArchive `yaml:"docker"` + Tar map[string]string `yaml:"tar"` +} + +type DockerArchive struct { + BaseImage string `yaml:"base_image"` + WorkingDir string `yaml:"working_dir"` + Exec string `yaml:"exec"` +} + +type DependencySet struct { + Build []string `yaml:"build"` + Runtime []string `yaml:"runtime"` +} + +type ExecPhase struct { + Name string `yaml:"name"` + Dependencies ExecDependencies `yaml:"dependencies"` + Container narwhal.ContainerDefinition `yaml:"container"` + Commands []string `yaml:"commands"` + Ports []string `yaml:"ports"` + Environment map[string][]string `yaml:"environment"` + LogFiles []string `yaml:"logfiles"` + Sandbox bool `yaml:"sandbox"` + HostOnly bool `yaml:"host_only"` + BuildFirst []string `yaml:"build_first"` +} + +func (e *ExecPhase) EnvironmentVariables(envName string, data runtime.RuntimeEnvironmentData) []string { + + result := make([]string, 0) + + for _, property := range e.Environment["default"] { + + s := strings.SplitN(property, "=", 2) + if len(s) == 2 { + interpolated, err := TemplateToString(property, data) + if err == nil { + result = append(result, interpolated) + } else { + result = append(result, property) + } + } + } + + for k, v := range data.Containers.Environment() { + result = append(result, k, v) + } + + if envName != "default" { + for _, property := range e.Environment[envName] { + s := strings.SplitN(property, "=", 2) + if len(s) == 2 { + result = append(result, property) + } + } + } + + // Check for local .env file + err := godotenv.Load() + if err == nil { + localEnv, _ := godotenv.Read() + for k, v := range localEnv { + result = append(result, strings.Join([]string{k, v}, "=")) + } + } + + return result +} + +type ExecDependencies struct { + Containers map[string]narwhal.ContainerDefinition `yaml:"containers"` +} + +func (b ExecDependencies) ContainerList() []narwhal.ContainerDefinition { + containers := make([]narwhal.ContainerDefinition, 0) + for label, c := range b.Containers { + c.Label = label + containers = append(containers, c) + } + return containers +} + +type BuildManifest struct { + Dependencies DependencySet `yaml:"dependencies"` + Sandbox bool `yaml:"sandbox"` + BuildTargets []BuildTarget `yaml:"build_targets"` + Build BuildTarget `yaml:"build"` + Exec ExecPhase `yaml:"exec"` + Package PackagePhase `yaml:"package"` + CI CIInfo `yaml:"ci"` +} + +func (b BuildManifest) BuildDependenciesChecksum() string { + buf := bytes.Buffer{} + for _, dep := range b.Dependencies.Build { + buf.Write([]byte(dep)) + } + + sum := sha256.Sum256(buf.Bytes()) + return fmt.Sprintf("%x", sum[:DependencyChecksumLength]) +} + +func (b BuildManifest) IsTargetSandboxed(target BuildTarget) bool { + return b.Sandbox || target.Sandbox +} + +// XXX: Support more than one level? Intuitively that seems like it will breed un-needed complexity +func (b BuildManifest) ResolveBuildTargets(targetName string) ([]BuildTarget, error) { + targetList := make([]BuildTarget, 0) + + target, err := b.BuildTarget(targetName) + if err != nil { + return targetList, err + } + + if len(target.BuildAfter) > 0 { + for _, depName := range target.BuildAfter { + depTarget, err := b.BuildTarget(depName) + if err != nil { + return targetList, err + } + targetList = append(targetList, depTarget) + } + } + + targetList = append(targetList, target) + + return targetList, nil +} + +func (b BuildManifest) CIBuild(buildName string) (CIBuild, error) { + for _, build := range b.CI.CIBuilds { + if build.Name == buildName { + return build, nil + } + } + return CIBuild{}, fmt.Errorf("No such CI build '%s' in build manifest", buildName) +} + +func (b BuildManifest) BuildTarget(targetName string) (BuildTarget, error) { + for _, target := range b.BuildTargets { + if target.Name == targetName { + return target, nil + } + } + return BuildTarget{}, fmt.Errorf("No such target '%s' in build manifest", targetName) +} + +func (b BuildManifest) BuildTargetList() []string { + targets := make([]string, 0, len(b.BuildTargets)) + + for _, t := range b.BuildTargets { + targets = append(targets, t.Name) + } + + return targets +} + +func TemplateToString(templateText string, data interface{}) (string, error) { + t, err := template.New("generic").Parse(templateText) + if err != nil { + return "", err + } + var tpl bytes.Buffer + if err := t.Execute(&tpl, data); err != nil { + log.Errorf("Can't render template:: %v", err) + return "", err + } + + result := tpl.String() + return result, nil +} diff --git a/workspace/build_target.go b/workspace/build_target.go new file mode 100644 index 00000000..16c046f4 --- /dev/null +++ b/workspace/build_target.go @@ -0,0 +1,216 @@ +package workspace + +import ( + "fmt" + "github.com/yourbase/narwhal" + "github.com/yourbase/yb/plumbing/log" + "github.com/yourbase/yb/runtime" + "os" + "strings" + "time" +) + +type CommandTimer struct { + Command string + StartTime time.Time + EndTime time.Time +} + +type TargetTimer struct { + Name string + Timers []CommandTimer +} + +type BuildTarget struct { + Name string `yaml:"name"` + Container narwhal.ContainerDefinition `yaml:"container"` + Tools []string `yaml:"tools"` + Commands []string `yaml:"commands"` + Artifacts []string `yaml:"artifacts"` + CachePaths []string `yaml:"cache_paths"` + Sandbox bool `yaml:"sandbox"` + HostOnly bool `yaml:"host_only"` + Root string `yaml:"root"` + Environment []string `yaml:"environment"` + Tags map[string]string `yaml:"tags"` + BuildAfter []string `yaml:"build_after"` + Dependencies BuildDependencies `yaml:"dependencies"` +} + +type BuildDependencies struct { + Containers map[string]narwhal.ContainerDefinition `yaml:"containers"` +} + +func (b BuildDependencies) ContainerList() []narwhal.ContainerDefinition { + containers := make([]narwhal.ContainerDefinition, 0) + for label, c := range b.Containers { + c.Label = label + containers = append(containers, c) + } + return containers +} + +func (bt BuildTarget) EnvironmentVariables(data runtime.RuntimeEnvironmentData) []string { + result := make([]string, 0) + for _, property := range bt.Environment { + s := strings.SplitN(property, "=", 2) + if len(s) == 2 { + interpolated, err := TemplateToString(property, data) + if err == nil { + result = append(result, interpolated) + } else { + result = append(result, property) + } + } + } + + return result +} + +func (bt BuildTarget) Build(runtimeCtx *runtime.Runtime, flags BuildFlags, packagePath string, buildpacks []string) ([]CommandTimer, error) { + var stepTimes []CommandTimer + + containers := bt.Dependencies.ContainerList() + workDir := packagePath + builder := runtimeCtx.DefaultTarget + + hostOnly := bt.HostOnly || flags.HostOnly + + if !hostOnly { + buildContainer := bt.Container + buildContainer.Command = "/usr/bin/tail -f /dev/null" + buildContainer.Label = "build" + + // Append build environment variables + buildContainer.Environment = []string{} + + // Add package to mounts @ /workspace + sourceMapDir := "/workspace" + if buildContainer.WorkDir != "" { + sourceMapDir = buildContainer.WorkDir + } + + log.Infof("Will mount %s at %s in container", packagePath, sourceMapDir) + mount := fmt.Sprintf("%s:%s", packagePath, sourceMapDir) + buildContainer.Mounts = append(buildContainer.Mounts, mount) + + containers = append(containers, buildContainer) + + var err error + builder, err = runtimeCtx.AddContainer(buildContainer) + + if err != nil { + return []CommandTimer{}, err + } + + runtimeCtx.DefaultTarget = builder + workDir = sourceMapDir + + // Inject a .ssh/config to skip host key checking + sshConfig := "Host github.com\n\tStrictHostKeyChecking no\n" + builder.Run(runtime.Process{Command: "mkdir -p /root/.ssh"}) + builder.WriteFileContents(sshConfig, "/root/.ssh/config") + builder.Run(runtime.Process{Command: "chmod 0600 /root/.ssh/config"}) + builder.Run(runtime.Process{Command: "chown root:root /root/.ssh/config"}) + + // Inject a useful gitconfig + configlines := []string{ + "[url \"ssh://git@github.com/\"]", + "insteadOf = https://github.com/", + "[url \"ssh://git@gitlab.com/\"]", + "insteadOf = https://gitlab.com/", + "[url \"ssh://git@bitbucket.org/\"]", + "insteadOf = https://bitbucket.org/", + } + gitConfig := strings.Join(configlines, "\n") + builder.WriteFileContents(gitConfig, "/root/.gitconfig") + + // TODO: Don't run this multiple times + // Map SSH agent into the container + if agentPath, exists := os.LookupEnv("SSH_AUTH_SOCK"); exists { + log.Infof("Running SSH agent socket forwarder...") + hostAddr, err := runtime.ForwardUnixSocketToTcp(agentPath) + if err != nil { + log.Warnf("Could not forward SSH agent: %v", err) + } else { + log.Infof("Forwarding SSH agent via %s", hostAddr) + } + + //buildContainer.Environment = append(buildContainer.Environment, "SSH_AUTH_SOCK=/ssh_agent") + builder.SetEnv("SSH_AUTH_SOCK", "/ssh_agent") + forwardPath, err := builder.DownloadFile("https://yourbase-artifacts.s3-us-west-2.amazonaws.com/sockforward") + builder.Run(runtime.Process{Command: fmt.Sprintf("chmod a+x %s", forwardPath)}) + forwardCmd := fmt.Sprintf("%s /ssh_agent %s", forwardPath, hostAddr) + go func() { + builder.Run(runtime.Process{Command: forwardCmd}) + }() + } + } + + // Setup dependent containers + for _, cd := range bt.Dependencies.ContainerList() { + if _, err := runtimeCtx.AddContainer(cd); err != nil { + return []CommandTimer{}, fmt.Errorf("can't add container %s: %v", cd.Label, err) + } + } + + // Do this after the containers are up + for _, envString := range bt.EnvironmentVariables(runtimeCtx.EnvironmentData()) { + parts := strings.Split(envString, "=") + key := parts[0] + value := "" + if len(parts) > 1 { + value = parts[1] + } else { + log.Warnf("'%s' doesn't look like an environment variable", envString) + } + builder.SetEnv(key, value) + } + + LoadBuildPacks(builder, buildpacks) + + /*if len(bt.Dependencies.Containers) > 0 { + log.Infof("Available side containers:") + for label, c := range bt.Dependencies.Containers { + ipv4 := buildData.Containers.IP(label) + log.Infof(" * %s (using %s) has IP address %s", label, c.ImageNameWithTag(), ipv4) + } + }*/ + + for _, cmdString := range bt.Commands { + var stepError error + + stepStartTime := time.Now() + p := runtime.Process{ + Directory: workDir, + Command: cmdString, + //Environment: buildData.EnvironmentVariables(), + Interactive: false, + } + + if stepError = builder.Run(p); stepError != nil { + log.Errorf("Failed to run %s: %v", cmdString, stepError) + } + + stepEndTime := time.Now() + stepTotalTime := stepEndTime.Sub(stepStartTime) + + log.Infof("Completed '%s' in %s", cmdString, stepTotalTime) + + cmdTimer := CommandTimer{ + Command: cmdString, + StartTime: stepStartTime, + EndTime: stepEndTime, + } + + stepTimes = append(stepTimes, cmdTimer) + // Make sure our goroutine gets this from stdout + // TODO: There must be a better way... + time.Sleep(10 * time.Millisecond) + if stepError != nil { + return stepTimes, stepError + } + } + + return stepTimes, nil +} diff --git a/buildpacks/buildpack_loader.go b/workspace/buildpack_loader.go similarity index 54% rename from buildpacks/buildpack_loader.go rename to workspace/buildpack_loader.go index 679c4bf0..7035d1ce 100644 --- a/buildpacks/buildpack_loader.go +++ b/workspace/buildpack_loader.go @@ -1,24 +1,17 @@ -package buildpacks +package workspace import ( "fmt" + "github.com/yourbase/yb/buildpacks" + "github.com/yourbase/yb/runtime" "strings" "time" - . "github.com/yourbase/yb/plumbing" "github.com/yourbase/yb/plumbing/log" . "github.com/yourbase/yb/types" ) -type BuildToolSpec struct { - Tool string - Version string - SharedCacheDir string - PackageCacheDir string - PackageDir string -} - -func LoadBuildPacks(dependencies []string, pkgCacheDir string, pkgDir string) ([]CommandTimer, error) { +func LoadBuildPacks(installTarget runtime.Target, dependencies []string) ([]CommandTimer, error) { setupTimers := make([]CommandTimer, 0) for _, toolSpec := range dependencies { @@ -31,62 +24,63 @@ func LoadBuildPacks(dependencies []string, pkgCacheDir string, pkgDir string) ([ versionString = parts[1] } - sharedCacheDir := ToolsDir() + sharedCacheDir := installTarget.ToolsDir() - spec := BuildToolSpec{ + spec := buildpacks.BuildToolSpec{ Tool: buildpackName, Version: versionString, SharedCacheDir: sharedCacheDir, - PackageCacheDir: pkgCacheDir, - PackageDir: pkgDir, + PackageCacheDir: installTarget.CacheDir(), + PackageDir: installTarget.WorkDir(), + InstallTarget: installTarget, } var bt BuildTool - log.Infof("Configuring build tool: %s", toolSpec) + log.Infof("Configuring build tool %s in %s", toolSpec, installTarget) switch buildpackName { case "anaconda2": - bt = NewAnaconda2BuildTool(spec) + bt = buildpacks.NewAnaconda2BuildTool(spec) case "anaconda3": - bt = NewAnaconda3BuildTool(spec) + bt = buildpacks.NewAnaconda3BuildTool(spec) case "ant": - bt = NewAntBuildTool(spec) + bt = buildpacks.NewAntBuildTool(spec) case "r": - bt = NewRLangBuildTool(spec) + bt = buildpacks.NewRLangBuildTool(spec) case "heroku": - bt = NewHerokuBuildTool(spec) + bt = buildpacks.NewHerokuBuildTool(spec) case "node": - bt = NewNodeBuildTool(spec) + bt = buildpacks.NewNodeBuildTool(spec) case "yarn": - bt = NewYarnBuildTool(spec) + bt = buildpacks.NewYarnBuildTool(spec) case "glide": - bt = NewGlideBuildTool(spec) + bt = buildpacks.NewGlideBuildTool(spec) case "androidndk": - bt = NewAndroidNdkBuildTool(spec) + bt = buildpacks.NewAndroidNdkBuildTool(spec) case "android": - bt = NewAndroidBuildTool(spec) + bt = buildpacks.NewAndroidBuildTool(spec) case "gradle": - bt = NewGradleBuildTool(spec) + bt = buildpacks.NewGradleBuildTool(spec) case "flutter": - bt = NewFlutterBuildTool(spec) + bt = buildpacks.NewFlutterBuildTool(spec) case "dart": - bt = NewDartBuildTool(spec) + bt = buildpacks.NewDartBuildTool(spec) case "rust": - bt = NewRustBuildTool(spec) + bt = buildpacks.NewRustBuildTool(spec) case "java": - bt = NewJavaBuildTool(spec) + bt = buildpacks.NewJavaBuildTool(spec) case "maven": - bt = NewMavenBuildTool(spec) + bt = buildpacks.NewMavenBuildTool(spec) case "go": - bt = NewGolangBuildTool(spec) + bt = buildpacks.NewGolangBuildTool(spec) case "python": - bt = NewPythonBuildTool(spec) + bt = buildpacks.NewPythonBuildTool(spec) case "ruby": - bt = NewRubyBuildTool(spec) + bt = buildpacks.NewRubyBuildTool(spec) case "homebrew": - bt = NewHomebrewBuildTool(spec) + bt = buildpacks.NewHomebrewBuildTool(spec) case "protoc": - bt = NewProtocBuildTool(spec) + bt = buildpacks.NewProtocBuildTool(spec) default: return setupTimers, fmt.Errorf("Unknown build tool: %s\n", toolSpec) } diff --git a/workspace/package.go b/workspace/package.go new file mode 100644 index 00000000..1b883389 --- /dev/null +++ b/workspace/package.go @@ -0,0 +1,236 @@ +package workspace + +import ( + "crypto/sha256" + "fmt" + . "github.com/yourbase/yb/plumbing" + "github.com/yourbase/yb/plumbing/log" + "github.com/yourbase/yb/runtime" + . "github.com/yourbase/yb/types" + "gopkg.in/yaml.v2" + "io" + "io/ioutil" + "os" + "os/user" + "path/filepath" + "strings" +) + +// Any flags we want to pass to the build process +type BuildFlags struct { + HostOnly bool + CleanBuild bool +} + +type Package struct { + Name string + path string + Manifest BuildManifest +} + +func (p Package) Path() string { + return p.path +} + +func (p Package) BuildTargetToWriter(buildTargetName string, flags BuildFlags, output io.Writer) ([]CommandTimer, error) { + manifest := p.Manifest + + // Named target, look for that and resolve it + _, err := manifest.ResolveBuildTargets(buildTargetName) + + if err != nil { + log.Errorf("Could not compute build target '%s': %v", buildTargetName, err) + log.Errorf("Valid build targets: %s", strings.Join(manifest.BuildTargetList(), ", ")) + return []CommandTimer{}, err + } + + primaryTarget, err := manifest.BuildTarget(buildTargetName) + if err != nil { + log.Errorf("Couldn't get primary build target '%s' specs: %v", buildTargetName, err) + return []CommandTimer{}, err + } + + contextId := fmt.Sprintf("%s-build-%s", p.Name, primaryTarget.Name) + + runtimeCtx := runtime.NewRuntime(contextId, p.BuildRoot()) + + return primaryTarget.Build(runtimeCtx, flags, p.Path(), p.Manifest.Dependencies.Build) +} + +func (p Package) BuildTarget(name string, flags BuildFlags) ([]CommandTimer, error) { + return p.BuildTargetToWriter(name, flags, os.Stdout) +} + +func LoadPackage(name string, path string) (Package, error) { + manifest := BuildManifest{} + buildYaml := filepath.Join(path, MANIFEST_FILE) + if _, err := os.Stat(buildYaml); os.IsNotExist(err) { + return Package{}, fmt.Errorf("Can't load %s: %v", MANIFEST_FILE, err) + } + + buildyaml, _ := ioutil.ReadFile(buildYaml) + err := yaml.Unmarshal([]byte(buildyaml), &manifest) + if err != nil { + return Package{}, fmt.Errorf("Error loading %s for %s: %v", MANIFEST_FILE, name, err) + } + + p := Package{ + path: path, + Name: name, + Manifest: manifest, + } + + return p, nil +} + +func (p Package) BuildRoot() string { + // Are we a part of a workspace? + workspaceDir, err := FindWorkspaceRoot() + + if err != nil { + // Nope, just ourselves... + workspacesRoot, exists := os.LookupEnv("YB_WORKSPACES_ROOT") + if !exists { + u, err := user.Current() + if err != nil { + workspacesRoot = "/tmp/yourbase/workspaces" + } else { + workspacesRoot = fmt.Sprintf("%s/.yourbase/workspaces", u.HomeDir) + } + } + + h := sha256.New() + + h.Write([]byte(p.path)) + workspaceHash := fmt.Sprintf("%x", h.Sum(nil)) + workspaceDir = filepath.Join(workspacesRoot, workspaceHash[0:12]) + } + + MkdirAsNeeded(workspaceDir) + + buildDir := "build" + buildRoot := filepath.Join(workspaceDir, buildDir) + + if _, err := os.Stat(buildRoot); os.IsNotExist(err) { + if err := os.Mkdir(buildRoot, 0700); err != nil { + log.Warnf("Unable to create build dir in workspace: %v\n", err) + } + } + + return buildRoot + +} + +func (p Package) SetupRuntimeDependencies() ([]CommandTimer, error) { + deps := p.Manifest.Dependencies.Runtime + deps = append(deps, p.Manifest.Dependencies.Build...) + return []CommandTimer{}, nil + //return LoadBuildPacks(deps, p) +} + +func (p Package) Execute(runtimeCtx *runtime.Runtime) error { + return p.ExecuteToWriter(runtimeCtx, os.Stdout) +} + +func (p Package) ExecuteToWriter(runtimeCtx *runtime.Runtime, output io.Writer) error { + + for _, cmdString := range p.Manifest.Exec.Commands { + p := runtime.Process{ + Command: cmdString, + Directory: "/workspace", + Interactive: false, + Output: &output, + } + + if err := runtimeCtx.Run(p); err != nil { + return fmt.Errorf("Unable to run command '%s': %v", cmdString, err) + } + } + + return nil +} + +func (p Package) ExecutionRuntime(environment string) (*runtime.Runtime, error) { + manifest := p.Manifest + containers := manifest.Exec.Dependencies.ContainerList() + contextId := fmt.Sprintf("%s-exec", p.Name) + + runtimeCtx := runtime.NewRuntime(contextId, p.BuildRoot()) + + localContainerWorkDir := filepath.Join(p.BuildRoot(), "containers") + MkdirAsNeeded(localContainerWorkDir) + + log.Infof("Will use %s as the dependency work dir", localContainerWorkDir) + + for _, cd := range containers { + cd.LocalWorkDir = localContainerWorkDir + _, err := runtimeCtx.AddContainer(cd) + if err != nil { + return nil, fmt.Errorf("Couldn't start container dependency: %v", err) + } + } + + // TODO: Support UDP + portMappings := make([]string, 0) + + for _, entry := range p.Manifest.Exec.Ports { + localPort := "" + remotePort := "" + parts := strings.Split(entry, ":") + + if len(parts) == 2 { + localPort = parts[0] + remotePort = parts[1] + } else { + if runtime.HostOS() == runtime.Linux { + log.Infof("No host port specified for port %s - will use %s externaly", entry, entry) + localPort = entry + } else { + log.Infof("Docker is not running natively, will try to pick a random port for %s", entry) + p, err := runtime.GetFreePort() + if err != nil { + log.Warnf("Could not find local port for container port %s: %v", entry, err) + } else { + localPort = fmt.Sprintf("%d", p) + } + } + + remotePort = entry + } + + if localPort != "" && remotePort != "" { + mapString := fmt.Sprintf("%s:%s", localPort, remotePort) + portMappings = append(portMappings, mapString) + } + + log.Infof("Mapping container port %s to %s on the local machine", remotePort, localPort) + } + + execContainer := manifest.Exec.Container + execContainer.Environment = manifest.Exec.EnvironmentVariables(environment, runtimeCtx.EnvironmentData()) + execContainer.Command = "/usr/bin/tail -f /dev/null" + execContainer.Label = "exec" + execContainer.Ports = portMappings + + + // Add package to mounts @ /workspace + sourceMapDir := "/workspace" + if execContainer.WorkDir != "" { + sourceMapDir = execContainer.WorkDir + } + log.Infof("Will mount package %s at %s in container", p.Path(), sourceMapDir) + mount := fmt.Sprintf("%s:%s", p.Path(), sourceMapDir) + execContainer.Mounts = append(execContainer.Mounts, mount) + + execTarget, err := runtimeCtx.AddContainer(execContainer) + if err != nil { + return nil, fmt.Errorf("Couldn't start exec container: %v", err) + } + + LoadBuildPacks(execTarget, p.Manifest.Dependencies.Build) + LoadBuildPacks(execTarget, p.Manifest.Dependencies.Runtime) + + runtimeCtx.DefaultTarget = execTarget + + return runtimeCtx, nil +} diff --git a/workspace/workspace.go b/workspace/workspace.go index 4ceb8461..d3731437 100644 --- a/workspace/workspace.go +++ b/workspace/workspace.go @@ -2,15 +2,15 @@ package workspace import ( "fmt" + "github.com/yourbase/yb/runtime" "gopkg.in/yaml.v2" "io/ioutil" - "log" "os" "path/filepath" "strings" - . "github.com/yourbase/yb/packages" . "github.com/yourbase/yb/plumbing" + "github.com/yourbase/yb/plumbing/log" ) type Workspace struct { @@ -83,7 +83,7 @@ func (w Workspace) BuildRoot() string { } func (w Workspace) SetupEnv() error { - fmt.Println("Clearing environment variables!") + log.Infof("Resetting environment variables!") criticalVariables := []string{"USER", "USERNAME", "UID", "GID", "TTY", "PWD"} oldEnv := make(map[string]string) for _, key := range criticalVariables { @@ -93,12 +93,12 @@ func (w Workspace) SetupEnv() error { os.Clearenv() tmpDir := filepath.Join(w.BuildRoot(), "tmp") MkdirAsNeeded(tmpDir) - os.Setenv("HOME", w.BuildRoot()) - os.Setenv("TMPDIR", tmpDir) + runtime.SetEnv("HOME", w.BuildRoot()) + runtime.SetEnv("TMPDIR", tmpDir) for _, key := range criticalVariables { - fmt.Printf("%s=%s\n", key, oldEnv[key]) - os.Setenv(key, oldEnv[key]) + log.Infof("%s=%s\n", key, oldEnv[key]) + runtime.SetEnv(key, oldEnv[key]) } return nil @@ -135,7 +135,7 @@ func LoadWorkspace() (Workspace, error) { return Workspace{}, fmt.Errorf("Error loading workspace config!") } - fmt.Printf("Workspace path: %s\n", workspacePath) + log.Infof("Workspace path: %s\n", workspacePath) workspace.Path = workspacePath return workspace, nil }