Use Anaconda for Python build pack (#17)

* Use Anaconda for Python build pack - much faster and more reliable than building from source

* Update the env dir so it's different from the old build pack for projects that were using pyenv

buildpacks/python.go

@@ -3,12 +3,7 @@ package buildpacks
 import (
 	"fmt"
 	"os"
-	"os/exec"
 	"path/filepath"
-	"strings"
-
-	"github.com/matishsiao/goInfo"
-	"gopkg.in/src-d/go-git.v4"
 
 	. "github.com/yourbase/yb/plumbing"
 	. "github.com/yourbase/yb/types"
@@ -20,6 +15,10 @@ type PythonBuildTool struct {
 	spec    BuildToolSpec
 }
 
+var ANACONDA_URL_TEMPLATE = "https://repo.continuum.io/miniconda/Miniconda{{.PyNum}}-{{.Version}}-{{.OS}}-{{.Arch}}.{{.Extension}}"
+
+const AnacondaToolVersion = "4.7.10"
+
 func NewPythonBuildTool(toolSpec BuildToolSpec) PythonBuildTool {
 	tool := PythonBuildTool{
 		version: toolSpec.Version,
@@ -33,119 +32,120 @@ func (bt PythonBuildTool) Version() string {
 	return bt.version
 }
 
-func (bt PythonBuildTool) pyenvInstallDir() string {
-	return filepath.Join(bt.spec.SharedCacheDir, "pyenv")
+func (bt PythonBuildTool) AnacondaInstallDir() string {
+	return filepath.Join(bt.spec.SharedCacheDir, "miniconda3", fmt.Sprintf("miniconda-%s", AnacondaToolVersion))
 }
 
-func (bt PythonBuildTool) pythonInstallDir() string {
-	return filepath.Join(bt.pyenvInstallDir(), "versions", bt.Version())
-}
-func (bt PythonBuildTool) pkgVirtualEnvDir() string {
-	return filepath.Join(bt.spec.PackageCacheDir, "python", bt.Version())
+func (bt PythonBuildTool) EnvironmentDir() string {
+	return filepath.Join(bt.spec.PackageCacheDir, "conda-python", bt.Version())
 }
 
-/*
-TODO: Install libssl-dev (or equivalent / warn) and zlib-dev based on platform
-*/
 func (bt PythonBuildTool) Install() error {
-	pyenvGitUrl := "https://github.com/pyenv/pyenv.git"
-	pyenvDir := bt.pyenvInstallDir()
-	pythonVersionDir := bt.pythonInstallDir()
-	virtualenvDir := bt.pkgVirtualEnvDir()
+	anacondaDir := bt.AnacondaInstallDir()
+	setupDir := bt.spec.PackageDir
 
-	bt.InstallPlatformDependencies()
-
-	if _, err := os.Stat(pyenvDir); err == nil {
-		fmt.Printf("pyenv installed in %s\n", pyenvDir)
+	if _, err := os.Stat(anacondaDir); err == nil {
+		fmt.Printf("anaconda installed in %s\n", anacondaDir)
 	} else {
-		fmt.Printf("Installing pyenv\n")
+		fmt.Printf("Installing anaconda\n")
 
-		_, err := git.PlainClone(pyenvDir, false, &git.CloneOptions{
-			URL:      pyenvGitUrl,
-			Progress: os.Stdout,
-		})
+		downloadUrl := bt.DownloadUrl()
 
+		fmt.Printf("Downloading Miniconda from URL %s...\n", downloadUrl)
+		localFile, err := DownloadFileWithCache(downloadUrl)
 		if err != nil {
-			fmt.Printf("Unable to clone pyenv!\n")
-			return fmt.Errorf("Couldn't clone pyenv: %v\n", err)
+			fmt.Printf("Unable to download: %v\n", err)
+			return err
 		}
-	}
 
-	if _, err := os.Stat(pythonVersionDir); err == nil {
-		fmt.Printf("Python %s installed in %s\n", bt.Version(), pythonVersionDir)
-	} else {
-		os.Setenv("PYENV_ROOT", pyenvDir)
-		PrependToPath(filepath.Join(pyenvDir, "bin"))
+		// TODO: Windows
+		for _, cmd := range []string{
+			fmt.Sprintf("chmod +x %s", localFile),
+			fmt.Sprintf("bash %s -b -p %s", localFile, anacondaDir),
+		} {
+			fmt.Printf("Running: '%v' ", cmd)
+			ExecToStdout(cmd, setupDir)
+		}
 
-		installCmd := fmt.Sprintf("pyenv install %s", bt.Version())
-		ExecToStdout(installCmd, pyenvDir)
 	}
 
-	virtualenvBin := filepath.Join(pythonVersionDir, "bin", "virtualenv")
-	if _, err := os.Stat(virtualenvBin); err == nil {
-		fmt.Printf("Virtualenv binary already installed in %s\n", virtualenvBin)
-	} else {
-		fmt.Printf("Installing virtualenv for Python in %s\n", pythonVersionDir)
+	return nil
+}
 
-		shimsDir := filepath.Join(pyenvDir, "shims")
+func (bt PythonBuildTool) DownloadUrl() string {
+	opsys := OS()
+	arch := Arch()
+	extension := "sh"
+	version := bt.Version()
 
-		os.Setenv("PYENV_ROOT", pyenvDir)
-		os.Setenv("PYENV_SHELL", "sh")
-		os.Setenv("PYENV_VERSION", bt.Version())
+	if version == "" {
+		version = "latest"
+	}
 
-		PrependToPath(filepath.Join(pyenvDir, "bin"))
-		PrependToPath(shimsDir)
-		setupCmd := fmt.Sprintf("pyenv rehash")
-		ExecToStdout(setupCmd, pyenvDir)
+	if arch == "amd64" {
+		arch = "x86_64"
+	}
 
-		cmd := "pip install virtualenv"
-		ExecToStdout(cmd, pyenvDir)
+	if opsys == "darwin" {
+		opsys = "MacOSX"
 	}
 
-	if _, err := os.Stat(virtualenvDir); err == nil {
-		fmt.Printf("Virtualenv for %s exists in %s\n", bt.Version(), virtualenvDir)
-	} else {
-		fmt.Println("Creating virtualenv...")
-		MkdirAsNeeded(virtualenvDir)
-		pythonBinPath := filepath.Join(pythonVersionDir, "bin", "python")
-		cmd := fmt.Sprintf("%s -p %s %s", virtualenvBin, pythonBinPath, virtualenvDir)
-		ExecToStdout(cmd, bt.spec.PackageDir)
+	if opsys == "linux" {
+		opsys = "Linux"
 	}
 
-	return nil
-}
+	if opsys == "windows" {
+		opsys = "Windows"
+		extension = "exe"
+	}
 
-func (bt PythonBuildTool) Setup() error {
-	virtualenvDir := bt.pkgVirtualEnvDir()
-	PrependToPath(filepath.Join(virtualenvDir, "bin"))
+	data := struct {
+		PyNum     int
+		OS        string
+		Arch      string
+		Version   string
+		Extension string
+	}{
+		3,
+		opsys,
+		arch,
+		AnacondaToolVersion,
+		extension,
+	}
 
-	return nil
+	url, _ := TemplateToString(ANACONDA_URL_TEMPLATE, data)
 
+	return url
 }
 
-func (bt PythonBuildTool) InstallPlatformDependencies() error {
-	gi := goInfo.GetInfo()
-	if gi.GoOS == "darwin" {
-		if strings.HasPrefix(gi.Core, "18.") {
-			// Need to install the headers on Mojave
-			if !PathExists("/usr/include/zlib.h") {
-				installCmd := "sudo -S installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target /"
-				fmt.Println("Going to run:", installCmd)
-				cmdArgs := strings.Split(installCmd, " ")
-				cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
-				cmd.Stdout = os.Stdout
-				cmd.Stdin = os.Stdin
-				cmd.Stderr = os.Stderr
-				cmd.Run()
+func (bt PythonBuildTool) Setup() error {
+	condaDir := bt.AnacondaInstallDir()
+	envDir := bt.EnvironmentDir()
+
+	if _, err := os.Stat(envDir); err == nil {
+		fmt.Printf("environment installed in %s\n", envDir)
+	} else {
+		currentPath := os.Getenv("PATH")
+		newPath := fmt.Sprintf("PATH=%s:%s", filepath.Join(condaDir, "bin"), currentPath)
+		setupDir := bt.spec.PackageDir
+		condaBin := filepath.Join(condaDir, "bin", "conda")
+
+		for _, cmd := range []string{
+			fmt.Sprintf("%s config --set always_yes yes --set changeps1 no", condaBin),
+			fmt.Sprintf("%s update -q conda", condaBin),
+			fmt.Sprintf("%s create --prefix %s python=%s", condaBin, envDir, bt.Version()),
+		} {
+			fmt.Printf("Running: '%v' ", cmd)
+			if err := ExecToStdoutWithEnv(cmd, setupDir, []string{newPath}); err != nil {
+				fmt.Printf("Unable to run setup command: %s\n", cmd)
+				return fmt.Errorf("Unable to run '%s': %v", cmd, err)
 			}
 		}
-		// Thanks to henriquebastos, here:
-		// https://github.com/pyenv/pyenv/issues/1066#issuecomment-504710495
-		if strings.HasPrefix(gi.Core, "18.6") {
-			os.Setenv("SDKROOT", "/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk")
-			os.Setenv("MACOSX_DEPLOYMENT_TARGET", "10.14")
-		}
 	}
 
+	// Add new env to path
+	PrependToPath(filepath.Join(envDir, "bin"))
+
 	return nil
+
 }

cli/exec.go

@@ -67,7 +67,11 @@ func (b *ExecCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
 		targetPackage = pkg
 	}
 
-	targetPackage.SetupRuntimeDependencies()
+	if _, err := targetPackage.SetupRuntimeDependencies(); err != nil {
+		fmt.Printf("Couldn't configure dependencies: %v\n", err)
+		return subcommands.ExitFailure
+	}
+
 	instructions := targetPackage.Manifest
 	containers := instructions.Exec.Dependencies.Containers
 

plumbing/util.go

@@ -20,15 +20,20 @@ import (
 	"gopkg.in/src-d/go-git.v4/plumbing"
 )
 
-func ExecToStdout(cmdString string, targetDir string) error {
-	fmt.Printf("Running: %s in %s\n", cmdString, targetDir)
+func ExecToStdoutWithExtraEnv(cmdString string, targetDir string, env []string) error {
+	env = append(os.Environ(), env...)
+	return ExecToStdoutWithEnv(cmdString, targetDir, env)
+}
 
+func ExecToStdoutWithEnv(cmdString string, targetDir string, env []string) error {
+	fmt.Printf("Running: %s in %s\n", cmdString, targetDir)
 	cmdArgs := strings.Fields(cmdString)
 	cmd := exec.Command(cmdArgs[0], cmdArgs[1:len(cmdArgs)]...)
 	cmd.Dir = targetDir
 	cmd.Stdout = os.Stdout
 	cmd.Stdin = os.Stdin
 	cmd.Stderr = os.Stdout
+	cmd.Env = env
 
 	err := cmd.Run()
 
@@ -37,7 +42,10 @@ func ExecToStdout(cmdString string, targetDir string) error {
 	}
 
 	return nil
+}
 
+func ExecToStdout(cmdString string, targetDir string) error {
+	return ExecToStdoutWithEnv(cmdString, targetDir, os.Environ())
 }
 
 func PrependToPath(dir string) {