diff --git a/README.md b/README.md index 321b7d5..65e0103 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ [![build](https://github.com/insightsengineering/git-synchronizer/actions/workflows/test.yml/badge.svg)](https://github.com/insightsengineering/git-synchronizer/actions/workflows/test.yml) -`git-synchronizer` allows you to mirror a collection of `git` repositories from one location to another. For each source repository, you can set a destination repository to which the source should be mirrored (see example [configuration file](#configuration-file)). +`git-synchronizer` allows you to mirror a collection of `git` repositories from one location to another. +For each source repository, you can set a destination repository to which the source should be mirrored (see example [configuration file](#configuration-file)). +Synchronization between all source-destination repository pairs in performed concurrently. `git-synchronizer` will: * push all branches and tags from source to destination repository, @@ -10,7 +12,8 @@ ## Installing -Simply download the project for your distribution from the [releases](https://github.com/insightsengineering/git-synchronizer/releases) page. `git-synchronizer` is distributed as a single binary file and does not require any additional system requirements. +Simply download the project for your distribution from the [releases](https://github.com/insightsengineering/git-synchronizer/releases) page. +`git-synchronizer` is distributed as a single binary file and does not require any additional system requirements. ## Usage @@ -35,12 +38,14 @@ defaults: source: auth: method: token - # Name of environment variable storing the Personal Access Token with permissions to read source repositories. + # Name of environment variable storing the Personal Access Token + # with permissions to read source repositories. token_name: GITHUB_TOKEN destination: auth: method: token - # Name of environment variable storing the Personal Access Token with permissions to push to destination repositories. + # Name of environment variable storing the Personal Access Token + # with permissions to push to destination repositories. token_name: GITLAB_TOKEN # List of repository pairs to be synchronized. @@ -91,7 +96,8 @@ This project is built with the [Go programming language](https://go.dev/). ### Development Environment -It is recommended to use Go 1.21+ for developing this project. This project uses a pre-commit configuration and it is recommended to [install and use pre-commit](https://pre-commit.com/#install) when you are developing this project. +It is recommended to use Go 1.21+ for developing this project. +This project uses a pre-commit configuration and it is recommended to [install and use pre-commit](https://pre-commit.com/#install) when you are developing this project. ### Common Commands diff --git a/cmd/mirror.go b/cmd/mirror.go index b8acc1a..45f52ef 100644 --- a/cmd/mirror.go +++ b/cmd/mirror.go @@ -22,8 +22,11 @@ import ( "strings" "time" + backoff "github.com/cenkalti/backoff/v4" git "github.com/go-git/go-git/v5" gitconfig "github.com/go-git/go-git/v5/config" + gitplumbing "github.com/go-git/go-git/v5/plumbing" + gittransport "github.com/go-git/go-git/v5/plumbing/transport" githttp "github.com/go-git/go-git/v5/plumbing/transport/http" ) @@ -85,15 +88,33 @@ func ValidateRepositories(repositories []RepositoryPair) { } } +func ListRemote(remote *git.Remote, listOptions *git.ListOptions) ([]*gitplumbing.Reference, error) { + refList, err := remote.List(listOptions) + if err == gittransport.ErrAuthenticationRequired { + return nil, backoff.Permanent(err) + } else if err != nil { + log.Warn("Retrying listing remote...") + } + return refList, err +} + // GetBranchesAndTagsFromRemote returns list of branches and tags present in remoteName of repository. func GetBranchesAndTagsFromRemote(repository *git.Repository, remoteName string, listOptions *git.ListOptions) ([]string, []string, error) { var branchList []string var tagList []string + var err error + remote, err := repository.Remote(remoteName) if err != nil { return branchList, tagList, err } - refList, err := remote.List(listOptions) + + listRemoteBackoff := backoff.NewExponentialBackOff() + listRemoteBackoff.MaxElapsedTime = time.Minute + refList, err := backoff.RetryWithData( + func() ([]*gitplumbing.Reference, error) { return ListRemote(remote, listOptions) }, + listRemoteBackoff, + ) if err != nil { return branchList, tagList, err } @@ -207,6 +228,47 @@ func GetDestinationAuth(destAuth Authentication) *githttp.BasicAuth { return destinationAuth } +// GitPlainClone clones git repository and is retried in case of error. +func GitPlainClone(gitDirectory string, cloneOptions *git.CloneOptions) (*git.Repository, error) { + repository, err := git.PlainClone(gitDirectory, false, cloneOptions) + if err == gittransport.ErrAuthenticationRequired { + // Terminate backoff. + return nil, backoff.Permanent(err) + } else if err != nil { + log.Warn("Retrying cloning repository...") + } + return repository, err +} + +// GitFetchBranches fetches all branches and is retried in case of error. +func GitFetchBranches(sourceRemote *git.Remote, sourceAuthentication Authentication) error { + gitFetchOptions := GetFetchOptions("refs/heads/*:refs/heads/*", sourceAuthentication) + err := sourceRemote.Fetch(gitFetchOptions) + if err == gittransport.ErrAuthenticationRequired { + // Terminate backoff. + return backoff.Permanent(err) + } else if err != nil { + log.Warn("Retrying fetching branches...") + } + return err +} + +// PushRefs pushes refs defined in refSpecString to destination remote and is retried in case of error. +func PushRefs(repository *git.Repository, auth *githttp.BasicAuth, refSpecString string) error { + err := repository.Push(&git.PushOptions{ + RemoteName: "destination", + RefSpecs: []gitconfig.RefSpec{gitconfig.RefSpec(refSpecString)}, + Auth: auth, Force: true, Atomic: true}, + ) + if err == gittransport.ErrAuthenticationRequired || err == git.NoErrAlreadyUpToDate { + // Terminate backoff. + return backoff.Permanent(err) + } else if err != nil { + log.Warn("Retrying pushing refs...") + } + return err +} + // MirrorRepository mirrors branches and tags from source to destination. Tags and branches // no longer present in source are removed from destination. func MirrorRepository(messages chan MirrorStatus, source, destination string, sourceAuthentication, destinationAuthentication Authentication) { @@ -217,7 +279,13 @@ func MirrorRepository(messages chan MirrorStatus, source, destination string, so defer os.RemoveAll(gitDirectory) var allErrors []string gitCloneOptions := GetCloneOptions(source, sourceAuthentication) - repository, err := git.PlainClone(gitDirectory, false, gitCloneOptions) + + cloneBackoff := backoff.NewExponentialBackOff() + cloneBackoff.MaxElapsedTime = 2 * time.Minute + repository, err := backoff.RetryWithData( + func() (*git.Repository, error) { return GitPlainClone(gitDirectory, gitCloneOptions) }, + cloneBackoff, + ) if err != nil { ProcessError(err, "cloning repository from ", source, &allErrors) messages <- MirrorStatus{allErrors, time.Now(), 0, 0} @@ -242,8 +310,12 @@ func MirrorRepository(messages chan MirrorStatus, source, destination string, so return } - gitFetchOptions := GetFetchOptions("refs/heads/*:refs/heads/*", sourceAuthentication) - err = sourceRemote.Fetch(gitFetchOptions) + fetchBranchesBackoff := backoff.NewExponentialBackOff() + fetchBranchesBackoff.MaxElapsedTime = time.Minute + err = backoff.Retry( + func() error { return GitFetchBranches(sourceRemote, sourceAuthentication) }, + fetchBranchesBackoff, + ) if err != nil { ProcessError(err, "fetching branches from ", source, &allErrors) messages <- MirrorStatus{allErrors, time.Now(), 0, 0} @@ -275,10 +347,14 @@ func MirrorRepository(messages chan MirrorStatus, source, destination string, so log.Info("Pushing all branches from ", source, " to ", destination) for _, branch := range sourceBranchList { log.Debug("Pushing branch ", branch, " to ", destination) - err = repository.Push(&git.PushOptions{ - RemoteName: "destination", - RefSpecs: []gitconfig.RefSpec{gitconfig.RefSpec("+" + refBranchPrefix + branch + ":" + refBranchPrefix + branch)}, - Auth: destinationAuth, Force: true, Atomic: true}) + pushBranchesBackoff := backoff.NewExponentialBackOff() + pushBranchesBackoff.MaxElapsedTime = 2 * time.Minute + err = backoff.Retry( + func() error { + return PushRefs(repository, destinationAuth, "+"+refBranchPrefix+branch+":"+refBranchPrefix+branch) + }, + pushBranchesBackoff, + ) ProcessError(err, "pushing branch "+branch+" to ", destination, &allErrors) } @@ -286,29 +362,35 @@ func MirrorRepository(messages chan MirrorStatus, source, destination string, so for _, branch := range destinationBranchList { if !stringInSlice(branch, sourceBranchList) { log.Info("Removing branch ", branch, " from ", destination) - err = repository.Push(&git.PushOptions{ - RemoteName: "destination", - RefSpecs: []gitconfig.RefSpec{gitconfig.RefSpec(":" + refBranchPrefix + branch)}, - Auth: destinationAuth, Force: true, Atomic: true}) + removeBranchesBackoff := backoff.NewExponentialBackOff() + removeBranchesBackoff.MaxElapsedTime = time.Minute + err = backoff.Retry( + func() error { return PushRefs(repository, destinationAuth, ":"+refBranchPrefix+branch) }, + removeBranchesBackoff, + ) ProcessError(err, "removing branch "+branch+" from ", destination, &allErrors) } } log.Info("Pushing all tags from ", source, " to ", destination) - err = repository.Push(&git.PushOptions{ - RemoteName: "destination", - RefSpecs: []gitconfig.RefSpec{gitconfig.RefSpec("+" + refTagPrefix + "*:" + refTagPrefix + "*")}, - Auth: destinationAuth, Force: true, Atomic: true}) + pushTagsBackoff := backoff.NewExponentialBackOff() + pushTagsBackoff.MaxElapsedTime = time.Minute + err = backoff.Retry( + func() error { return PushRefs(repository, destinationAuth, "+"+refTagPrefix+"*:"+refTagPrefix+"*") }, + pushTagsBackoff, + ) ProcessError(err, "pushing all tags to ", destination, &allErrors) // Remove any tags not present in the source repository anymore. for _, tag := range destinationTagList { if !stringInSlice(tag, sourceTagList) { log.Info("Removing tag ", tag, " from ", destination) - err := repository.Push(&git.PushOptions{ - RemoteName: "destination", - RefSpecs: []gitconfig.RefSpec{gitconfig.RefSpec(":" + refTagPrefix + tag)}, - Auth: destinationAuth, Force: true, Atomic: true}) + removeTagsBackoff := backoff.NewExponentialBackOff() + removeTagsBackoff.MaxElapsedTime = time.Minute + err = backoff.Retry( + func() error { return PushRefs(repository, destinationAuth, ":"+refTagPrefix+tag) }, + removeTagsBackoff, + ) ProcessError(err, "removing tag "+tag+" from ", destination, &allErrors) } } diff --git a/cmd/root.go b/cmd/root.go index f22491e..c4eea3d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -49,7 +49,6 @@ type Authentication struct { // Repository list provided in YAML configuration file. var inputRepositories []RepositoryPair - var defaultSettings RepositoryPair var localTempDirectory string diff --git a/go.mod b/go.mod index 44878f6..25732af 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21 toolchain go1.21.6 require ( + github.com/cenkalti/backoff/v4 v4.2.1 github.com/go-git/go-git/v5 v5.11.0 github.com/jamiealquiza/envy v1.1.0 github.com/sirupsen/logrus v1.9.3 diff --git a/go.sum b/go.sum index 8ca4b07..9950f82 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=