From 4532c9c7db1cd97c5478a56bbf930a3b4e16ba1d Mon Sep 17 00:00:00 2001 From: Zac Bergquist Date: Mon, 10 Feb 2025 15:59:26 -0700 Subject: [PATCH] Troubleshoot backport failures Lately we see a lot of backport workflows fail. The pattern always seems to be that the first backport (to branch/v17) succeeds, but subsequent backports (to v15 and v16) almost always fail. The failure looks to be a GitHub-specific failure, not a generic git failure: remote: fatal error in commit_refs To https://github.com/gravitational/teleport ! [remote rejected] bot/backport-51989-branch/v15 > bot/backport-51989-branch/v15 (failure) error: failed to push some refs to https://github.com/gravitational/teleport To aid in troubleshooting we: 1. Temporarily reverse the order of branches. This will tell us if it's always the first branch that succeeds, or if it's always branch/v17 that succeeds. 2. Add the --verbose flag to the git push command in case we get any more debug info. --- bot/internal/bot/backport.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bot/internal/bot/backport.go b/bot/internal/bot/backport.go index d015542c..6076d948 100644 --- a/bot/internal/bot/backport.go +++ b/bot/internal/bot/backport.go @@ -84,6 +84,10 @@ func (b *Bot) Backport(ctx context.Context) error { g = b.c.Git } + // Temporary experiment to change the order of backport attempts. + // TODO(zmb3): remove this + slices.Reverse(branches) + // Loop over all requested backport branches and create backport branch and // GitHub Pull Request. for _, base := range branches { @@ -249,7 +253,7 @@ func (b *Bot) createBackportBranch(ctx context.Context, organization string, rep } // Push the backport branch to Github. - if err := git("push", "origin", newHead); err != nil { + if err := git("push", "--verbose", "origin", newHead); err != nil { return trace.Wrap(err) }