From 7e668473f6fe35c360d0cf93e9d369a403e44829 Mon Sep 17 00:00:00 2001 From: Lou Marvin Caraig Date: Wed, 31 Jul 2019 18:08:33 +0200 Subject: [PATCH 01/10] Replace base64.StdEncoding with base64.URLEncoding Signed-off-by: Lou Marvin Caraig --- cmd/sourced/compose/file/file.go | 4 ++-- cmd/sourced/compose/workdir/common.go | 2 +- cmd/sourced/compose/workdir/org.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/sourced/compose/file/file.go b/cmd/sourced/compose/file/file.go index 82fe0d3..778d2a0 100644 --- a/cmd/sourced/compose/file/file.go +++ b/cmd/sourced/compose/file/file.go @@ -216,7 +216,7 @@ func List() ([]RevOrURL, error) { } func composeName(rev string) string { - if decoded, err := base64.StdEncoding.DecodeString(rev); err == nil { + if decoded, err := base64.URLEncoding.DecodeString(rev); err == nil { return string(decoded) } @@ -248,7 +248,7 @@ func path(revOrURL RevOrURL) (string, error) { subPath := revOrURL if isURL(revOrURL) { - subPath = base64.StdEncoding.EncodeToString([]byte(revOrURL)) + subPath = base64.URLEncoding.EncodeToString([]byte(revOrURL)) } dirPath := filepath.Join(composeDirPath, subPath) diff --git a/cmd/sourced/compose/workdir/common.go b/cmd/sourced/compose/workdir/common.go index dcc3732..93f35c1 100644 --- a/cmd/sourced/compose/workdir/common.go +++ b/cmd/sourced/compose/workdir/common.go @@ -384,7 +384,7 @@ func decodeName(base, target string) (string, error) { } // workdirs for remote orgs encoded into base64 - decoded, err := base64.StdEncoding.DecodeString(p) + decoded, err := base64.URLEncoding.DecodeString(p) if err == nil { return string(decoded), nil } diff --git a/cmd/sourced/compose/workdir/org.go b/cmd/sourced/compose/workdir/org.go index 1286bdd..c7372a6 100644 --- a/cmd/sourced/compose/workdir/org.go +++ b/cmd/sourced/compose/workdir/org.go @@ -11,7 +11,7 @@ func InitWithOrgs(orgs []string, token string) (string, error) { // be indifferent to the order of passed organizations sort.Strings(orgs) - workdir := base64.StdEncoding.EncodeToString([]byte(strings.Join(orgs, ","))) + workdir := base64.URLEncoding.EncodeToString([]byte(strings.Join(orgs, ","))) workdirPath, err := absolutePath(workdir) if err != nil { return "", err From a67ee70f5f10625f74acc9288fbcde1a79acc297 Mon Sep 17 00:00:00 2001 From: Lou Marvin Caraig Date: Wed, 31 Jul 2019 22:02:26 +0200 Subject: [PATCH 02/10] Add workdir.Factory struct Signed-off-by: Lou Marvin Caraig --- cmd/sourced/compose/workdir/common.go | 122 ++++++++++++++++++-------- 1 file changed, 85 insertions(+), 37 deletions(-) diff --git a/cmd/sourced/compose/workdir/common.go b/cmd/sourced/compose/workdir/common.go index 93f35c1..1dca801 100644 --- a/cmd/sourced/compose/workdir/common.go +++ b/cmd/sourced/compose/workdir/common.go @@ -30,6 +30,91 @@ var ( ErrMalformed = goerrors.NewKind("workdir %s is not valid: %s") ) +// Factory is responsible for the initialization of the workdirs +type Factory struct{} + +// InitLocal initializes the workdir for local path and returns the absolute path +func (f *Factory) InitLocal(reposdir string) (string, error) { + dirName := f.encodeDirName(reposdir) + + envf := envFile{ + Workdir: dirName, + ReposDir: reposdir, + } + + return f.init(dirName, "local", envf) +} + +// InitOrgs initializes the workdir for organizationsand returns the absolute path +func (f *Factory) InitOrgs(orgs []string, token string) (string, error) { + // be indifferent to the order of passed organizations + sort.Strings(orgs) + dirName := f.encodeDirName(strings.Join(orgs, ",")) + + envf := envFile{ + Workdir: dirName, + GithubOrganizations: orgs, + GithubToken: token, + } + + return f.init(dirName, "orgs", envf) +} + +func (f *Factory) encodeDirName(dirName string) string { + return base64.URLEncoding.EncodeToString([]byte(dirName)) +} + +func (f *Factory) buildAbsPath(dirName, subPath string) (string, error) { + path, err := workdirsPath() + if err != nil { + return "", err + } + + return filepath.Join(path, subPath, dirName), nil +} + +func (f *Factory) init(dirName string, subPath string, envf envFile) (string, error) { + workdir, err := f.buildAbsPath(dirName, subPath) + if err != nil { + return "", err + } + + err = os.MkdirAll(workdir, 0755) + if err != nil { + return "", errors.Wrap(err, "could not create working directory") + } + + defaultFilePath, err := composefile.InitDefault() + if err != nil { + return "", err + } + + composePath := filepath.Join(workdir, "docker-compose.yml") + if err := link(defaultFilePath, composePath); err != nil { + return "", err + } + + defaultOverridePath, err := composefile.InitDefaultOverride() + if err != nil { + return "", err + } + + workdirOverridePath := filepath.Join(workdir, "docker-compose.override.yml") + if err := link(defaultOverridePath, workdirOverridePath); err != nil { + return "", err + } + + envPath := filepath.Join(workdir, ".env") + contents := envf.String() + err = ioutil.WriteFile(envPath, []byte(contents), 0644) + + if err != nil { + return "", errors.Wrap(err, "could not write .env file") + } + + return workdir, nil +} + type envFile struct { Workdir string ReposDir string @@ -315,43 +400,6 @@ func isEmptyFile(path string) (bool, error) { return strings.TrimSpace(strContents) == "", nil } -// common initialization for both local and remote data -func initWorkdir(workdirPath string, envFile envFile) error { - err := os.MkdirAll(workdirPath, 0755) - if err != nil { - return errors.Wrap(err, "could not create working directory") - } - - defaultFilePath, err := composefile.InitDefault() - if err != nil { - return err - } - - composePath := filepath.Join(workdirPath, "docker-compose.yml") - if err := link(defaultFilePath, composePath); err != nil { - return err - } - - defaultOverridePath, err := composefile.InitDefaultOverride() - if err != nil { - return err - } - - workdirOverridePath := filepath.Join(workdirPath, "docker-compose.override.yml") - if err := link(defaultOverridePath, workdirOverridePath); err != nil { - return err - } - - envPath := filepath.Join(workdirPath, ".env") - contents := envFile.String() - err = ioutil.WriteFile(envPath, []byte(contents), 0644) - if err != nil { - return errors.Wrap(err, "could not write .env file") - } - - return nil -} - func link(linkTargetPath, linkPath string) error { _, err := os.Stat(linkPath) if err == nil { From 3ea7147608f58b09415979d15f689b14f08c155a Mon Sep 17 00:00:00 2001 From: Lou Marvin Caraig Date: Wed, 31 Jul 2019 22:02:34 +0200 Subject: [PATCH 03/10] Simplify cmd init by using workdir.Factory Signed-off-by: Lou Marvin Caraig --- cmd/sourced/cmd/init.go | 51 ++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/cmd/sourced/cmd/init.go b/cmd/sourced/cmd/init.go index 399e36a..fbea9a9 100644 --- a/cmd/sourced/cmd/init.go +++ b/cmd/sourced/cmd/init.go @@ -26,6 +26,8 @@ type initLocalCmd struct { Args struct { Reposdir string `positional-arg-name:"workdir"` } `positional-args:"yes"` + + workdirFactory *workdir.Factory } func (c *initLocalCmd) Execute(args []string) error { @@ -34,22 +36,12 @@ func (c *initLocalCmd) Execute(args []string) error { return err } - dir, err := workdir.InitWithPath(reposdir) + workdir, err := c.workdirFactory.InitLocal(reposdir) if err != nil { return err } - // Before setting a new workdir, stop the current containers - compose.Run(context.Background(), "stop") - - err = workdir.SetActive(reposdir) - if err != nil { - return err - } - - fmt.Printf("docker-compose working directory set to %s\n", dir) - - if err := compose.Run(context.Background(), "up", "--detach"); err != nil { + if err := activate(workdir); err != nil { return err } @@ -86,6 +78,8 @@ type initOrgsCmd struct { Args struct { Orgs []string `required:"yes"` } `positional-args:"yes" required:"1"` + + workdirFactory *workdir.Factory } func (c *initOrgsCmd) Execute(args []string) error { @@ -94,22 +88,12 @@ func (c *initOrgsCmd) Execute(args []string) error { return err } - dir, err := workdir.InitWithOrgs(orgs, c.Token) - if err != nil { - return err - } - - // Before setting a new workdir, stop the current containers - compose.Run(context.Background(), "stop") - - err = workdir.SetActive(dir) + workdir, err := c.workdirFactory.InitOrgs(orgs, c.Token) if err != nil { return err } - fmt.Printf("docker-compose working directory set to %s\n", strings.Join(orgs, ",")) - - if err := compose.Run(context.Background(), "up", "--detach"); err != nil { + if err := activate(workdir); err != nil { return err } @@ -154,6 +138,19 @@ func (c *initOrgsCmd) validate(orgs []string) error { return nil } +func activate(dir string) error { + // Before setting a new workdir, stop the current containers + compose.Run(context.Background(), "stop") + + err := workdir.SetActive(dir) + if err != nil { + return err + } + + fmt.Printf("docker-compose working directory set to %s\n", dir) + return compose.Run(context.Background(), "up", "--detach") +} + type authTransport struct { token string } @@ -165,6 +162,8 @@ func (t *authTransport) RoundTrip(r *http.Request) (*http.Response, error) { func init() { c := rootCmd.AddCommand(&initCmd{}) - c.AddCommand(&initOrgsCmd{}) - c.AddCommand(&initLocalCmd{}) + + workdirFactory := &workdir.Factory{} + c.AddCommand(&initOrgsCmd{workdirFactory: workdirFactory}) + c.AddCommand(&initLocalCmd{workdirFactory: workdirFactory}) } From 5da578a37b740c541e7ad021be323fa725e27f4a Mon Sep 17 00:00:00 2001 From: Lou Marvin Caraig Date: Wed, 31 Jul 2019 23:08:43 +0200 Subject: [PATCH 04/10] Remove unnecessary local.go and org.go Signed-off-by: Lou Marvin Caraig --- cmd/sourced/compose/workdir/local.go | 33 ---------------------------- cmd/sourced/compose/workdir/org.go | 30 ------------------------- 2 files changed, 63 deletions(-) delete mode 100644 cmd/sourced/compose/workdir/local.go delete mode 100644 cmd/sourced/compose/workdir/org.go diff --git a/cmd/sourced/compose/workdir/local.go b/cmd/sourced/compose/workdir/local.go deleted file mode 100644 index 5d8a0e2..0000000 --- a/cmd/sourced/compose/workdir/local.go +++ /dev/null @@ -1,33 +0,0 @@ -// Package workdir provides functions to manage docker compose working -// directories inside the $HOME/.sourced/workdirs directory -package workdir - -import ( - "crypto/sha1" - "encoding/hex" -) - -// InitWithPath creates a working directory in ~/.sourced for the given repositories -// directory. The working directory will contain a docker-compose.yml and a -// .env file. -// If the directory is already initialized the function returns with no error. -// The returned value is the absolute path to $HOME/.sourced/workdirs/reposdir -func InitWithPath(reposdir string) (string, error) { - workdir, err := absolutePath(reposdir) - if err != nil { - return "", err - } - - hash := sha1.Sum([]byte(reposdir)) - hashSt := hex.EncodeToString(hash[:]) - envf := envFile{ - Workdir: hashSt, - ReposDir: reposdir, - } - - if err := initWorkdir(workdir, envf); err != nil { - return "", err - } - - return workdir, nil -} diff --git a/cmd/sourced/compose/workdir/org.go b/cmd/sourced/compose/workdir/org.go deleted file mode 100644 index c7372a6..0000000 --- a/cmd/sourced/compose/workdir/org.go +++ /dev/null @@ -1,30 +0,0 @@ -package workdir - -import ( - "encoding/base64" - "sort" - "strings" -) - -// InitWithOrgs initialize workdir with remote list of organizations -func InitWithOrgs(orgs []string, token string) (string, error) { - // be indifferent to the order of passed organizations - sort.Strings(orgs) - - workdir := base64.URLEncoding.EncodeToString([]byte(strings.Join(orgs, ","))) - workdirPath, err := absolutePath(workdir) - if err != nil { - return "", err - } - - envf := envFile{ - Workdir: workdir, - GithubOrganizations: orgs, - GithubToken: token, - } - if err := initWorkdir(workdirPath, envf); err != nil { - return "", err - } - - return workdir, nil -} From b62eaa5f9ed4ccc4fc6baa581e1739a9c7925354 Mon Sep 17 00:00:00 2001 From: Lou Marvin Caraig Date: Thu, 1 Aug 2019 12:06:56 +0200 Subject: [PATCH 05/10] Fix workdir.SetActive as now workdir is always absolute Signed-off-by: Lou Marvin Caraig --- cmd/sourced/compose/workdir/common.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cmd/sourced/compose/workdir/common.go b/cmd/sourced/compose/workdir/common.go index 1dca801..28c5794 100644 --- a/cmd/sourced/compose/workdir/common.go +++ b/cmd/sourced/compose/workdir/common.go @@ -150,11 +150,6 @@ func SetActive(workdir string) error { return err } - workdirPath, err := absolutePath(workdir) - if err != nil { - return err - } - _, err = os.Stat(activePath) if !os.IsNotExist(err) { err = os.Remove(activePath) @@ -163,7 +158,7 @@ func SetActive(workdir string) error { } } - err = os.Symlink(workdirPath, activePath) + err = os.Symlink(workdir, activePath) if os.IsExist(err) { return nil } From b181a96edccf2e58f25e9f6265c4675d7d538c74 Mon Sep 17 00:00:00 2001 From: Lou Marvin Caraig Date: Thu, 1 Aug 2019 20:03:17 +0200 Subject: [PATCH 06/10] Rename workdir/{common,workdir}.go Signed-off-by: Lou Marvin Caraig --- cmd/sourced/compose/workdir/{common.go => workdir.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cmd/sourced/compose/workdir/{common.go => workdir.go} (100%) diff --git a/cmd/sourced/compose/workdir/common.go b/cmd/sourced/compose/workdir/workdir.go similarity index 100% rename from cmd/sourced/compose/workdir/common.go rename to cmd/sourced/compose/workdir/workdir.go From 31a36db60214f36bc1c6bfd3df039951c0da96c0 Mon Sep 17 00:00:00 2001 From: Lou Marvin Caraig Date: Thu, 1 Aug 2019 20:15:45 +0200 Subject: [PATCH 07/10] Simplify decodeName as now path is always base64 encoded Signed-off-by: Lou Marvin Caraig --- cmd/sourced/compose/workdir/workdir.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/cmd/sourced/compose/workdir/workdir.go b/cmd/sourced/compose/workdir/workdir.go index 28c5794..becf5f7 100644 --- a/cmd/sourced/compose/workdir/workdir.go +++ b/cmd/sourced/compose/workdir/workdir.go @@ -6,7 +6,6 @@ import ( "io/ioutil" "os" "path/filepath" - "runtime" "sort" "strings" @@ -426,17 +425,10 @@ func decodeName(base, target string) (string, error) { return "", err } - // workdirs for remote orgs encoded into base64 decoded, err := base64.URLEncoding.DecodeString(p) if err == nil { return string(decoded), nil } - // for windows local path convert C\path to C:\path - if runtime.GOOS == "windows" { - return string(p[0]) + ":" + p[1:len(p)], nil - } - - // for *nix prepend root, User/path to /Users/path - return filepath.Join("/", p), nil + return "", err } From e362a3c215bd81b128bd03af80e26822f652834d Mon Sep 17 00:00:00 2001 From: Lou Marvin Caraig Date: Thu, 1 Aug 2019 20:39:51 +0200 Subject: [PATCH 08/10] Specialize absolutePath function into activeAbsolutePath function Signed-off-by: Lou Marvin Caraig --- cmd/sourced/compose/workdir/workdir.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/cmd/sourced/compose/workdir/workdir.go b/cmd/sourced/compose/workdir/workdir.go index becf5f7..5ece8ff 100644 --- a/cmd/sourced/compose/workdir/workdir.go +++ b/cmd/sourced/compose/workdir/workdir.go @@ -144,7 +144,7 @@ func (f *envFile) String() string { // SetActive creates a symlink from the fixed active workdir path // to the workdir for the given repos dir. func SetActive(workdir string) error { - activePath, err := absolutePath(activeDir) + activePath, err := activeAbsolutePath() if err != nil { return err } @@ -167,7 +167,7 @@ func SetActive(workdir string) error { // UnsetActive removes symlink for active workdir func UnsetActive() error { - dir, err := absolutePath(activeDir) + dir, err := activeAbsolutePath() if err != nil { return err } @@ -200,7 +200,7 @@ func Active() (string, error) { // ActivePath returns absolute path to active working directory func ActivePath() (string, error) { - path, err := absolutePath(activeDir) + path, err := activeAbsolutePath() if err != nil { return "", err } @@ -354,18 +354,14 @@ func ValidatePath(dir string) error { return nil } -// path returns the absolute path to -// $HOME/.sourced/workdirs/workdir -func absolutePath(workdir string) (string, error) { +// activeAbsolutePath returns the absolute path to the current active workdir +func activeAbsolutePath() (string, error) { path, err := workdirsPath() if err != nil { return "", err } - // On windows replace C:\path with C\path - workdir = strings.Replace(workdir, ":", "", 1) - - return filepath.Join(path, workdir), nil + return filepath.Join(path, activeDir), nil } func hasContent(path, file string) bool { From 1dc2fea985a6a4b87ac451c8378ccc9cf177cfd5 Mon Sep 17 00:00:00 2001 From: Lou Marvin Caraig Date: Thu, 1 Aug 2019 23:16:40 +0200 Subject: [PATCH 09/10] Fix broken workdir.decodeName function due to orgs and local path split Signed-off-by: Lou Marvin Caraig --- cmd/sourced/compose/workdir/workdir.go | 32 +++++++++++++++----------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/cmd/sourced/compose/workdir/workdir.go b/cmd/sourced/compose/workdir/workdir.go index 5ece8ff..d461e9a 100644 --- a/cmd/sourced/compose/workdir/workdir.go +++ b/cmd/sourced/compose/workdir/workdir.go @@ -190,12 +190,12 @@ func Active() (string, error) { return "", err } - wpath, err := workdirsPath() + decoded, err := decodeName(path) if err != nil { return "nil", err } - return decodeName(wpath, path) + return decoded, nil } // ActivePath returns absolute path to active working directory @@ -215,11 +215,6 @@ func ActivePath() (string, error) { // List returns array of working directories names func List() ([]string, error) { - wpath, err := workdirsPath() - if err != nil { - return nil, err - } - workdirs, err := ListPaths() if err != nil { return nil, err @@ -227,10 +222,11 @@ func List() ([]string, error) { res := make([]string, len(workdirs)) for i, d := range workdirs { - res[i], err = decodeName(wpath, d) + res[i], err = decodeName(d) if err != nil { return nil, err } + } sort.Strings(res) @@ -413,17 +409,25 @@ func workdirsPath() (string, error) { return filepath.Join(path, "workdirs"), nil } -// decodeName takes workdirs root and absolute path to workdir +// decodeName takes absolute path to workdir // return human-readable name. It returns an error if the path could not be built -func decodeName(base, target string) (string, error) { - p, err := filepath.Rel(base, target) +func decodeName(target string) (string, error) { + wpath, err := workdirsPath() if err != nil { return "", err } - decoded, err := base64.URLEncoding.DecodeString(p) - if err == nil { - return string(decoded), nil + subPaths := [2]string{"orgs", "local"} + for _, sp := range subPaths { + p, err := filepath.Rel(filepath.Join(wpath, sp), target) + if err != nil { + continue + } + + decoded, err := base64.URLEncoding.DecodeString(p) + if err == nil { + return string(decoded), nil + } } return "", err From 5601dca3eb02fba843ff321011f1317c89698fb6 Mon Sep 17 00:00:00 2001 From: Lou Marvin Caraig Date: Fri, 2 Aug 2019 00:28:00 +0200 Subject: [PATCH 10/10] Refactor workdir module by splitting into multiple modules and structs Signed-off-by: Lou Marvin Caraig --- cmd/sourced/cmd/init.go | 33 ++- cmd/sourced/cmd/prune.go | 23 +- cmd/sourced/cmd/workdirs.go | 17 +- cmd/sourced/compose/compose.go | 25 +- cmd/sourced/compose/workdir/factory.go | 129 +++++++++ cmd/sourced/compose/workdir/handler.go | 197 +++++++++++++ cmd/sourced/compose/workdir/workdir.go | 380 ++++--------------------- 7 files changed, 443 insertions(+), 361 deletions(-) create mode 100644 cmd/sourced/compose/workdir/factory.go create mode 100644 cmd/sourced/compose/workdir/handler.go diff --git a/cmd/sourced/cmd/init.go b/cmd/sourced/cmd/init.go index fbea9a9..7ae81a2 100644 --- a/cmd/sourced/cmd/init.go +++ b/cmd/sourced/cmd/init.go @@ -26,22 +26,25 @@ type initLocalCmd struct { Args struct { Reposdir string `positional-arg-name:"workdir"` } `positional-args:"yes"` - - workdirFactory *workdir.Factory } func (c *initLocalCmd) Execute(args []string) error { + wdHandler, err := workdir.NewHandler() + if err != nil { + return err + } + reposdir, err := c.reposdirArg() if err != nil { return err } - workdir, err := c.workdirFactory.InitLocal(reposdir) + wd, err := workdir.InitLocal(reposdir) if err != nil { return err } - if err := activate(workdir); err != nil { + if err := activate(wdHandler, wd); err != nil { return err } @@ -78,22 +81,25 @@ type initOrgsCmd struct { Args struct { Orgs []string `required:"yes"` } `positional-args:"yes" required:"1"` - - workdirFactory *workdir.Factory } func (c *initOrgsCmd) Execute(args []string) error { + wdHandler, err := workdir.NewHandler() + if err != nil { + return err + } + orgs := c.orgsList() if err := c.validate(orgs); err != nil { return err } - workdir, err := c.workdirFactory.InitOrgs(orgs, c.Token) + wd, err := workdir.InitOrgs(orgs, c.Token) if err != nil { return err } - if err := activate(workdir); err != nil { + if err := activate(wdHandler, wd); err != nil { return err } @@ -138,16 +144,16 @@ func (c *initOrgsCmd) validate(orgs []string) error { return nil } -func activate(dir string) error { +func activate(wdHandler *workdir.Handler, workdir *workdir.Workdir) error { // Before setting a new workdir, stop the current containers compose.Run(context.Background(), "stop") - err := workdir.SetActive(dir) + err := wdHandler.SetActive(workdir) if err != nil { return err } - fmt.Printf("docker-compose working directory set to %s\n", dir) + fmt.Printf("docker-compose working directory set to %s\n", workdir.Path) return compose.Run(context.Background(), "up", "--detach") } @@ -163,7 +169,6 @@ func (t *authTransport) RoundTrip(r *http.Request) (*http.Response, error) { func init() { c := rootCmd.AddCommand(&initCmd{}) - workdirFactory := &workdir.Factory{} - c.AddCommand(&initOrgsCmd{workdirFactory: workdirFactory}) - c.AddCommand(&initLocalCmd{workdirFactory: workdirFactory}) + c.AddCommand(&initOrgsCmd{}) + c.AddCommand(&initLocalCmd{}) } diff --git a/cmd/sourced/cmd/prune.go b/cmd/sourced/cmd/prune.go index 2360b3a..d90a487 100644 --- a/cmd/sourced/cmd/prune.go +++ b/cmd/sourced/cmd/prune.go @@ -15,21 +15,26 @@ type pruneCmd struct { } func (c *pruneCmd) Execute(args []string) error { + workdirHandler, err := workdir.NewHandler() + if err != nil { + return err + } + if !c.All { - return c.pruneActive() + return c.pruneActive(workdirHandler) } - dirs, err := workdir.ListPaths() + wds, err := workdirHandler.List() if err != nil { return err } - for _, dir := range dirs { - if err := workdir.SetActivePath(dir); err != nil { + for _, wd := range wds { + if err := workdirHandler.SetActive(wd); err != nil { return err } - if err = c.pruneActive(); err != nil { + if err = c.pruneActive(workdirHandler); err != nil { return err } } @@ -37,7 +42,7 @@ func (c *pruneCmd) Execute(args []string) error { return nil } -func (c *pruneCmd) pruneActive() error { +func (c *pruneCmd) pruneActive(workdirHandler *workdir.Handler) error { a := []string{"down", "--volumes"} if c.Images { a = append(a, "--rmi", "all") @@ -47,16 +52,16 @@ func (c *pruneCmd) pruneActive() error { return err } - dir, err := workdir.ActivePath() + wd, err := workdirHandler.Active() if err != nil { return err } - if err := workdir.RemovePath(dir); err != nil { + if err := workdirHandler.Remove(wd); err != nil { return err } - return workdir.UnsetActive() + return workdirHandler.UnsetActive() } func init() { diff --git a/cmd/sourced/cmd/workdirs.go b/cmd/sourced/cmd/workdirs.go index 706033c..a38c643 100644 --- a/cmd/sourced/cmd/workdirs.go +++ b/cmd/sourced/cmd/workdirs.go @@ -11,21 +11,26 @@ type workdirsCmd struct { } func (c *workdirsCmd) Execute(args []string) error { - dirs, err := workdir.List() + workdirHandler, err := workdir.NewHandler() if err != nil { return err } - active, err := workdir.Active() + wds, err := workdirHandler.List() if err != nil { return err } - for _, dir := range dirs { - if dir == active { - fmt.Printf("* %s\n", dir) + active, err := workdirHandler.Active() + if err != nil { + return err + } + + for _, wd := range wds { + if wd.Path == active.Path { + fmt.Printf("* %s\n", wd.Name) } else { - fmt.Printf(" %s\n", dir) + fmt.Printf(" %s\n", wd.Name) } } diff --git a/cmd/sourced/compose/compose.go b/cmd/sourced/compose/compose.go index 4967c9d..319ac0f 100644 --- a/cmd/sourced/compose/compose.go +++ b/cmd/sourced/compose/compose.go @@ -23,7 +23,8 @@ const dockerComposeVersion = "1.24.0" var composeContainerURL = fmt.Sprintf("https://github.com/docker/compose/releases/download/%s/run.sh", dockerComposeVersion) type Compose struct { - bin string + bin string + workdirHandler *workdir.Handler } func (c *Compose) Run(ctx context.Context, arg ...string) error { @@ -34,16 +35,16 @@ func (c *Compose) RunWithIO(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer, arg ...string) error { cmd := exec.CommandContext(ctx, c.bin, arg...) - dir, err := workdir.ActivePath() + wd, err := c.workdirHandler.Active() if err != nil { return err } - if err := workdir.ValidatePath(dir); err != nil { + if err := c.workdirHandler.Validate(wd); err != nil { return err } - cmd.Dir = dir + cmd.Dir = wd.Path cmd.Stdin = stdin cmd.Stdout = stdout cmd.Stderr = stderr @@ -51,13 +52,21 @@ func (c *Compose) RunWithIO(ctx context.Context, stdin io.Reader, return cmd.Run() } -func NewCompose() (*Compose, error) { +func newCompose() (*Compose, error) { + workdirHandler, err := workdir.NewHandler() + if err != nil { + return nil, err + } + bin, err := getOrInstallComposeBinary() if err != nil { return nil, err } - return &Compose{bin: bin}, nil + return &Compose{ + bin: bin, + workdirHandler: workdirHandler, + }, nil } func getOrInstallComposeBinary() (string, error) { @@ -118,7 +127,7 @@ func downloadCompose(path string) error { } func Run(ctx context.Context, arg ...string) error { - comp, err := NewCompose() + comp, err := newCompose() if err != nil { return err } @@ -127,7 +136,7 @@ func Run(ctx context.Context, arg ...string) error { } func RunWithIO(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer, arg ...string) error { - comp, err := NewCompose() + comp, err := newCompose() if err != nil { return err } diff --git a/cmd/sourced/compose/workdir/factory.go b/cmd/sourced/compose/workdir/factory.go new file mode 100644 index 0000000..d61747a --- /dev/null +++ b/cmd/sourced/compose/workdir/factory.go @@ -0,0 +1,129 @@ +package workdir + +import ( + "encoding/base64" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/pkg/errors" + composefile "github.com/src-d/sourced-ce/cmd/sourced/compose/file" +) + +// InitLocal initializes the workdir for local path and returns the Workdir instance +func InitLocal(reposdir string) (*Workdir, error) { + dirName := encodeDirName(reposdir) + + envf := envFile{ + Workdir: dirName, + ReposDir: reposdir, + } + + return initialize(dirName, "local", envf) +} + +// InitOrgs initializes the workdir for organizations and returns the Workdir instance +func InitOrgs(orgs []string, token string) (*Workdir, error) { + // be indifferent to the order of passed organizations + sort.Strings(orgs) + dirName := encodeDirName(strings.Join(orgs, ",")) + + envf := envFile{ + Workdir: dirName, + GithubOrganizations: orgs, + GithubToken: token, + } + + return initialize(dirName, "orgs", envf) +} + +func encodeDirName(dirName string) string { + return base64.URLEncoding.EncodeToString([]byte(dirName)) +} + +func buildAbsPath(dirName, subPath string) (string, error) { + path, err := workdirsPath() + if err != nil { + return "", err + } + + return filepath.Join(path, subPath, dirName), nil +} + +func initialize(dirName string, subPath string, envf envFile) (*Workdir, error) { + path, err := workdirsPath() + if err != nil { + return nil, err + } + + workdir := filepath.Join(path, subPath, dirName) + if err != nil { + return nil, err + } + + err = os.MkdirAll(workdir, 0755) + if err != nil { + return nil, errors.Wrap(err, "could not create working directory") + } + + defaultFilePath, err := composefile.InitDefault() + if err != nil { + return nil, err + } + + composePath := filepath.Join(workdir, "docker-compose.yml") + if err := link(defaultFilePath, composePath); err != nil { + return nil, err + } + + defaultOverridePath, err := composefile.InitDefaultOverride() + if err != nil { + return nil, err + } + + workdirOverridePath := filepath.Join(workdir, "docker-compose.override.yml") + if err := link(defaultOverridePath, workdirOverridePath); err != nil { + return nil, err + } + + envPath := filepath.Join(workdir, ".env") + contents := envf.String() + err = ioutil.WriteFile(envPath, []byte(contents), 0644) + + if err != nil { + return nil, errors.Wrap(err, "could not write .env file") + } + + b := &builder{workdirsPath: path} + return b.build(workdir) +} + +type envFile struct { + Workdir string + ReposDir string + GithubOrganizations []string + GithubToken string +} + +func (f *envFile) String() string { + volumeType := "bind" + volumeSource := f.ReposDir + gitbaseSiva := "" + if f.ReposDir == "" { + volumeType = "volume" + volumeSource = "gitbase_repositories" + gitbaseSiva = "true" + } + + return fmt.Sprintf(`COMPOSE_PROJECT_NAME=srcd-%s + GITBASE_VOLUME_TYPE=%s + GITBASE_VOLUME_SOURCE=%s + GITBASE_SIVA=%s + GITHUB_ORGANIZATIONS=%s + GITHUB_TOKEN=%s + `, f.Workdir, volumeType, volumeSource, gitbaseSiva, + strings.Join(f.GithubOrganizations, ","), f.GithubToken) +} diff --git a/cmd/sourced/compose/workdir/handler.go b/cmd/sourced/compose/workdir/handler.go new file mode 100644 index 0000000..9f5f868 --- /dev/null +++ b/cmd/sourced/compose/workdir/handler.go @@ -0,0 +1,197 @@ +package workdir + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +// Handler provides a way to interact with all the workdirs by exposing the following operations: +// - read/set/unset active workdir, +// - remove/validate a workdir, +// - list workdirs. +type Handler struct { + workdirsPath string + builder *builder +} + +// NewHandler creates a handler that manages workdirs in the path returned by +// the `workdirsPath` function +func NewHandler() (*Handler, error) { + path, err := workdirsPath() + if err != nil { + return nil, err + } + + return &Handler{ + workdirsPath: path, + builder: &builder{workdirsPath: path}, + }, nil +} + +// SetActive creates a symlink from the fixed active workdir path to the prodived workdir +func (h *Handler) SetActive(w *Workdir) error { + path, err := h.activeAbsolutePath() + if err != nil { + return err + } + + if err := h.UnsetActive(); err != nil { + return err + } + + err = os.Symlink(w.Path, path) + if os.IsExist(err) { + return nil + } + + return err +} + +// UnsetActive removes symlink for active workdir +func (h *Handler) UnsetActive() error { + path, err := h.activeAbsolutePath() + if err != nil { + return err + } + + _, err = os.Lstat(path) + if !os.IsNotExist(err) { + err = os.Remove(path) + if err != nil { + return errors.Wrap(err, "could not delete the previous active workdir directory symlink") + } + } + + return nil +} + +// Active returns active working directory +func (h *Handler) Active() (*Workdir, error) { + path, err := h.activeAbsolutePath() + if err != nil { + return nil, err + } + + resolvedPath, err := filepath.EvalSymlinks(path) + if os.IsNotExist(err) { + return nil, ErrMalformed.New("active", err) + } + + return h.builder.build(resolvedPath) +} + +// List returns array of working directories +func (h *Handler) List() ([]*Workdir, error) { + dirs := make([]string, 0) + err := filepath.Walk(h.workdirsPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + return nil + } + for _, f := range RequiredFiles { + if !hasContent(path, f) { + return nil + } + } + + dirs = append(dirs, path) + return nil + }) + + if os.IsNotExist(err) { + return nil, ErrMalformed.New(h.workdirsPath, err) + } + + if err != nil { + return nil, err + } + + wds := make([]*Workdir, 0, len(dirs)) + for _, p := range dirs { + wd, err := h.builder.build(p) + if err != nil { + return nil, err + } + + wds = append(wds, wd) + } + + return wds, nil + +} + +// Validate validates that the passed working directoy is valid +// It's path must be a directory (or a symlink) containing docker-compose.yml and .env files +func (h *Handler) Validate(w *Workdir) error { + pointedDir, err := filepath.EvalSymlinks(w.Path) + if err != nil { + return ErrMalformed.New(w.Path, "is not a directory") + } + + if info, err := os.Lstat(pointedDir); err != nil || !info.IsDir() { + return ErrMalformed.New(pointedDir, "is not a directory") + } + + for _, f := range RequiredFiles { + if !hasContent(pointedDir, f) { + return ErrMalformed.New(pointedDir, fmt.Sprintf("%s not found", f)) + } + } + + return nil +} + +// Remove removes working directory by removing required and optional files, +// and recursively removes directories up to the workdirs root as long as they are empty +func (h *Handler) Remove(w *Workdir) error { + path := w.Path + var subPath string + switch w.Type { + case Local: + subPath = "local" + case Orgs: + subPath = "orgs" + } + + basePath := filepath.Join(h.workdirsPath, subPath) + + for _, f := range append(RequiredFiles, OptionalFiles...) { + file := filepath.Join(path, f) + if _, err := os.Stat(file); os.IsNotExist(err) { + continue + } + + if err := os.Remove(file); err != nil { + return errors.Wrap(err, "could not remove from workdir directory") + } + } + + for { + files, err := ioutil.ReadDir(path) + if err != nil { + return errors.Wrap(err, "could not read workdir directory") + } + if len(files) > 0 { + return nil + } + + if err := os.Remove(path); err != nil { + return errors.Wrap(err, "could not delete workdir directory") + } + + path = filepath.Dir(path) + if path == basePath { + return nil + } + } +} + +func (h *Handler) activeAbsolutePath() (string, error) { + return filepath.Join(h.workdirsPath, activeDir), nil +} diff --git a/cmd/sourced/compose/workdir/workdir.go b/cmd/sourced/compose/workdir/workdir.go index d461e9a..6062c33 100644 --- a/cmd/sourced/compose/workdir/workdir.go +++ b/cmd/sourced/compose/workdir/workdir.go @@ -6,13 +6,11 @@ import ( "io/ioutil" "os" "path/filepath" - "sort" "strings" "github.com/pkg/errors" goerrors "gopkg.in/src-d/go-errors.v1" - composefile "github.com/src-d/sourced-ce/cmd/sourced/compose/file" datadir "github.com/src-d/sourced-ce/cmd/sourced/dir" ) @@ -29,335 +27,93 @@ var ( ErrMalformed = goerrors.NewKind("workdir %s is not valid: %s") ) -// Factory is responsible for the initialization of the workdirs -type Factory struct{} +// WorkdirType defines the type of the workdir +type WorkdirType int -// InitLocal initializes the workdir for local path and returns the absolute path -func (f *Factory) InitLocal(reposdir string) (string, error) { - dirName := f.encodeDirName(reposdir) - - envf := envFile{ - Workdir: dirName, - ReposDir: reposdir, - } - - return f.init(dirName, "local", envf) -} - -// InitOrgs initializes the workdir for organizationsand returns the absolute path -func (f *Factory) InitOrgs(orgs []string, token string) (string, error) { - // be indifferent to the order of passed organizations - sort.Strings(orgs) - dirName := f.encodeDirName(strings.Join(orgs, ",")) - - envf := envFile{ - Workdir: dirName, - GithubOrganizations: orgs, - GithubToken: token, - } - - return f.init(dirName, "orgs", envf) -} - -func (f *Factory) encodeDirName(dirName string) string { - return base64.URLEncoding.EncodeToString([]byte(dirName)) -} - -func (f *Factory) buildAbsPath(dirName, subPath string) (string, error) { - path, err := workdirsPath() - if err != nil { - return "", err - } - - return filepath.Join(path, subPath, dirName), nil -} - -func (f *Factory) init(dirName string, subPath string, envf envFile) (string, error) { - workdir, err := f.buildAbsPath(dirName, subPath) - if err != nil { - return "", err - } - - err = os.MkdirAll(workdir, 0755) - if err != nil { - return "", errors.Wrap(err, "could not create working directory") - } - - defaultFilePath, err := composefile.InitDefault() - if err != nil { - return "", err - } - - composePath := filepath.Join(workdir, "docker-compose.yml") - if err := link(defaultFilePath, composePath); err != nil { - return "", err - } - - defaultOverridePath, err := composefile.InitDefaultOverride() - if err != nil { - return "", err - } - - workdirOverridePath := filepath.Join(workdir, "docker-compose.override.yml") - if err := link(defaultOverridePath, workdirOverridePath); err != nil { - return "", err - } - - envPath := filepath.Join(workdir, ".env") - contents := envf.String() - err = ioutil.WriteFile(envPath, []byte(contents), 0644) - - if err != nil { - return "", errors.Wrap(err, "could not write .env file") - } - - return workdir, nil -} - -type envFile struct { - Workdir string - ReposDir string - GithubOrganizations []string - GithubToken string -} - -func (f *envFile) String() string { - volumeType := "bind" - volumeSource := f.ReposDir - gitbaseSiva := "" - if f.ReposDir == "" { - volumeType = "volume" - volumeSource = "gitbase_repositories" - gitbaseSiva = "true" - } - - return fmt.Sprintf(`COMPOSE_PROJECT_NAME=srcd-%s - GITBASE_VOLUME_TYPE=%s - GITBASE_VOLUME_SOURCE=%s - GITBASE_SIVA=%s - GITHUB_ORGANIZATIONS=%s - GITHUB_TOKEN=%s - `, f.Workdir, volumeType, volumeSource, gitbaseSiva, - strings.Join(f.GithubOrganizations, ","), f.GithubToken) -} - -// SetActive creates a symlink from the fixed active workdir path -// to the workdir for the given repos dir. -func SetActive(workdir string) error { - activePath, err := activeAbsolutePath() - if err != nil { - return err - } - - _, err = os.Stat(activePath) - if !os.IsNotExist(err) { - err = os.Remove(activePath) - if err != nil { - return errors.Wrap(err, "could not delete the previous active workdir directory symlink") - } - } - - err = os.Symlink(workdir, activePath) - if os.IsExist(err) { - return nil - } - - return err -} - -// UnsetActive removes symlink for active workdir -func UnsetActive() error { - dir, err := activeAbsolutePath() - if err != nil { - return err - } - - _, err = os.Lstat(dir) - if !os.IsNotExist(err) { - err = os.Remove(dir) - if err != nil { - return errors.Wrap(err, "could not delete active workdir directory symlink") - } - } - - return nil -} - -// Active returns active working directory name -func Active() (string, error) { - path, err := ActivePath() - if err != nil { - return "", err - } - - decoded, err := decodeName(path) - if err != nil { - return "nil", err - } - - return decoded, nil -} - -// ActivePath returns absolute path to active working directory -func ActivePath() (string, error) { - path, err := activeAbsolutePath() - if err != nil { - return "", err - } - - resolvedPath, err := filepath.EvalSymlinks(path) - if os.IsNotExist(err) { - return "", ErrMalformed.New("active", err) - } +const ( + // None refers to a failure in identifying the type of the workdir + None WorkdirType = iota + // Local refers to a workdir that has been initialized for local repos + Local + // Orgs refers to a workdir that has been initialized for organizations + Orgs +) - return resolvedPath, err +// Workdir represents a workdir associated with a local or an orgs initialization +type Workdir struct { + // Type is the WorkdirType + Type WorkdirType + // Name is a human-friendly string to identify the workdir + Name string + // Path is the absolute path corresponding to the workdir + Path string } -// List returns array of working directories names -func List() ([]string, error) { - workdirs, err := ListPaths() - if err != nil { - return nil, err - } - - res := make([]string, len(workdirs)) - for i, d := range workdirs { - res[i], err = decodeName(d) - if err != nil { - return nil, err - } - - } - - sort.Strings(res) - return res, nil +type builder struct { + workdirsPath string } -// ListPaths returns array of absolute paths to working directories -func ListPaths() ([]string, error) { - wpath, err := workdirsPath() +// build returns the Workdir instance corresponding to the provided absolute path +func (b *builder) build(path string) (*Workdir, error) { + wdType, err := b.typeFromPath(path) if err != nil { return nil, err } - dirs := make(map[string]bool) - err = filepath.Walk(wpath, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() { - return nil - } - for _, f := range RequiredFiles { - if !hasContent(path, f) { - return nil - } - } - - dirs[path] = true - return nil - }) - - if os.IsNotExist(err) { - return nil, ErrMalformed.New(wpath, err) + if wdType == None { + return nil, fmt.Errorf("invalid workdir type for path %s", path) } + wdName, err := b.workdirName(wdType, path) if err != nil { return nil, err } - res := make([]string, 0) - for dir := range dirs { - res = append(res, dir) - } - - return res, nil + return &Workdir{ + Type: wdType, + Name: wdName, + Path: path, + }, nil } -// RemovePath removes working directory by removing required and optional files, -// and recursively removes directories up to the workdirs root as long as they are empty -func RemovePath(path string) error { - workdirsRoot, err := workdirsPath() - if err != nil { - return err - } - - for _, f := range append(RequiredFiles, OptionalFiles...) { - file := filepath.Join(path, f) - if _, err := os.Stat(file); os.IsNotExist(err) { - continue - } - - if err := os.Remove(file); err != nil { - return errors.Wrap(err, "could not remove from workdir directory") - } - } - - for { - files, err := ioutil.ReadDir(path) - if err != nil { - return errors.Wrap(err, "could not read workdir directory") - } - if len(files) > 0 { - return nil - } - - if err := os.Remove(path); err != nil { - return errors.Wrap(err, "could not delete workdir directory") - } - - path = filepath.Dir(path) - if path == workdirsRoot { - return nil - } +// workdirName returns the workdir name given its type and absolute path +func (b *builder) workdirName(wdType WorkdirType, path string) (string, error) { + var subPath string + switch wdType { + case Local: + subPath = "local" + case Orgs: + subPath = "orgs" } -} -// SetActivePath similar to SetActive -// but accepts absolute path to a directory instead of a relative one -func SetActivePath(path string) error { - wpath, err := workdirsPath() + encoded, err := filepath.Rel(filepath.Join(b.workdirsPath, subPath), path) if err != nil { - return err + return "", err } - wd, err := filepath.Rel(wpath, path) - if err != nil { - return err + decoded, err := base64.URLEncoding.DecodeString(encoded) + if err == nil { + return string(decoded), nil } - return SetActive(wd) + return "", err } -// ValidatePath validates that the passed dir is valid -// Must be a directory (or a symlink) containing docker-compose.yml and .env files -func ValidatePath(dir string) error { - pointedDir, err := filepath.EvalSymlinks(dir) +// typeFromPath returns the WorkdirType corresponding to the provided absolute path +func (b *builder) typeFromPath(path string) (WorkdirType, error) { + suffix, err := filepath.Rel(b.workdirsPath, path) if err != nil { - return ErrMalformed.New(dir, "is not a directory") - } - - if info, err := os.Lstat(pointedDir); err != nil || !info.IsDir() { - return ErrMalformed.New(pointedDir, "is not a directory") - } - - for _, f := range RequiredFiles { - if !hasContent(pointedDir, f) { - return ErrMalformed.New(pointedDir, fmt.Sprintf("%s not found", f)) - } + return None, err } - return nil -} - -// activeAbsolutePath returns the absolute path to the current active workdir -func activeAbsolutePath() (string, error) { - path, err := workdirsPath() - if err != nil { - return "", err + switch filepath.Dir(suffix) { + case "local": + return Local, nil + case "orgs": + return Orgs, nil + default: + return None, nil } - - return filepath.Join(path, activeDir), nil } func hasContent(path, file string) bool { @@ -408,27 +164,3 @@ func workdirsPath() (string, error) { return filepath.Join(path, "workdirs"), nil } - -// decodeName takes absolute path to workdir -// return human-readable name. It returns an error if the path could not be built -func decodeName(target string) (string, error) { - wpath, err := workdirsPath() - if err != nil { - return "", err - } - - subPaths := [2]string{"orgs", "local"} - for _, sp := range subPaths { - p, err := filepath.Rel(filepath.Join(wpath, sp), target) - if err != nil { - continue - } - - decoded, err := base64.URLEncoding.DecodeString(p) - if err == nil { - return string(decoded), nil - } - } - - return "", err -}