Merge pull request #2599 from buildkite/gh-app-git-credentials

BK github app git credentials helper

api/github_code_access_token.go

@@ -0,0 +1,65 @@
+package api
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/buildkite/roko"
+)
+
+type GithubCodeAccessTokenRequest struct {
+	RepoURL string `json:"repo_url,omitempty"`
+}
+
+type GithubCodeAccessTokenResponse struct {
+	Token string `json:"token,omitempty"`
+}
+
+func (c *Client) GenerateGithubCodeAccessToken(ctx context.Context, repoURL, jobID string) (string, *Response, error) {
+	u := fmt.Sprintf("jobs/%s/github_code_access_token", railsPathEscape(jobID))
+
+	req, err := c.newRequest(ctx, http.MethodPost, u, GithubCodeAccessTokenRequest{RepoURL: repoURL})
+	if err != nil {
+		return "", nil, err
+	}
+
+	r := roko.NewRetrier(
+		roko.WithMaxAttempts(3),
+		roko.WithStrategy(roko.Constant(5*time.Second)),
+	)
+
+	var g *GithubCodeAccessTokenResponse
+	var resp *Response
+
+	err = r.Do(func(r *roko.Retrier) error {
+		var err error
+		resp, err = c.doRequest(req, g)
+		if err == nil {
+			return nil
+		}
+
+		if resp != nil {
+			if !IsRetryableStatus(resp) {
+				r.Break()
+				return err
+			}
+
+			if resp.Header.Get("Retry-After") != "" {
+				retryAfter, errParseDuration := time.ParseDuration(resp.Header.Get("Retry-After") + "s")
+				if errParseDuration == nil {
+					r.SetNextInterval(retryAfter)
+				}
+			}
+		}
+
+		return err
+	})
+
+	if err != nil {
+		return "", resp, err
+	}
+
+	return g.Token, resp, nil
+}

clicommand/commands.go

@@ -23,6 +23,7 @@ var BuildkiteAgentCommands = []cli.Command{
 			ArtifactShasumCommand,
 		},
 	},
+	GitCredentialsHelperCommand,
 	{
 		Name:  "env",
 		Usage: "Process environment subcommands",

clicommand/config_completeness_test.go

@@ -23,10 +23,11 @@ var commandConfigPairs = []configCommandPair{
 	{Config: ArtifactShasumConfig{}, Command: ArtifactShasumCommand},
 	{Config: ArtifactUploadConfig{}, Command: ArtifactUploadCommand},
 	{Config: BootstrapConfig{}, Command: BootstrapCommand},
-	{Config: EnvGetConfig{}, Command: EnvGetCommand},
 	{Config: EnvDumpConfig{}, Command: EnvDumpCommand},
+	{Config: EnvGetConfig{}, Command: EnvGetCommand},
 	{Config: EnvSetConfig{}, Command: EnvSetCommand},
 	{Config: EnvUnsetConfig{}, Command: EnvUnsetCommand},
+	{Config: GitCredentialsHelperConfig{}, Command: GitCredentialsHelperCommand},
 	{Config: LockAcquireConfig{}, Command: LockAcquireCommand},
 	{Config: LockDoConfig{}, Command: LockDoCommand},
 	{Config: LockDoneConfig{}, Command: LockDoneCommand},

clicommand/git_credentials_helper.go

@@ -0,0 +1,172 @@
+package clicommand
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"strings"
+
+	"github.com/buildkite/agent/v3/api"
+	"github.com/buildkite/agent/v3/logger"
+	"github.com/urfave/cli"
+)
+
+const gitCredentialsHelperHelpDescription = `Usage:
+
+    buildkite-agent git-credential-helper [options...]
+
+Description:
+
+[EXPERIMENTAL]
+
+Ask buildkite for credentials to use to authenticate with Github when cloning via HTTPS.
+The credentials are returned in the git-credential format.
+
+This command will only work if the organization running the job has connected a Github app with Code Access enabled, and
+if the pipeline has this feature enabled. All hosted compute jobs automatically qualify for this feature.
+
+This command is intended to be used as a git credential helper, and not called directly.`
+
+type GitCredentialsHelperConfig struct {
+	JobID string `cli:"job-id" validate:"required"`
+
+	// Global flags
+	Debug       bool     `cli:"debug"`
+	LogLevel    string   `cli:"log-level"`
+	NoColor     bool     `cli:"no-color"`
+	Experiments []string `cli:"experiment" normalize:"list"`
+	Profile     string   `cli:"profile"`
+
+	// API config
+	DebugHTTP        bool   `cli:"debug-http"`
+	AgentAccessToken string `cli:"agent-access-token" validate:"required"`
+	Endpoint         string `cli:"endpoint" validate:"required"`
+	NoHTTP2          bool   `cli:"no-http2"`
+}
+
+var GitCredentialsHelperCommand = cli.Command{
+	Name:        "git-credentials-helper",
+	Usage:       "Ask buildkite for credentials to use to authenticate with Github when cloning",
+	Description: gitCredentialsHelperHelpDescription,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:   "job-id",
+			Usage:  "The job id to get credentials for",
+			EnvVar: "BUILDKITE_JOB_ID",
+		},
+
+		// API Flags
+		AgentAccessTokenFlag,
+		EndpointFlag,
+		NoHTTP2Flag,
+		DebugHTTPFlag,
+
+		// Global flags
+		NoColorFlag,
+		DebugFlag,
+		LogLevelFlag,
+		ExperimentsFlag,
+		ProfileFlag,
+	},
+	Action: func(c *cli.Context) error {
+		ctx := context.Background()
+		ctx, cfg, l, _, done := setupLoggerAndConfig[GitCredentialsHelperConfig](ctx, c)
+		defer done()
+
+		l.Info("Authenticating checkout using Buildkite Github App Credentials...")
+
+		// ie, if the flags are from the command line rather than from the environment, which is how they should be passed
+		// to this process when it's called through the job executor
+		if os.Getenv("BUILDKITE_JOB_ID") == "" {
+			l.Warn("📎💬 It looks like you're calling this command directly in a step, rather than having the agent automatically call it")
+			l.Warn("This command is intended to be used as a git credential helper, and not called directly.")
+		}
+
+		// git passes the details of the current clone process to the credential helper via stdin
+		// we need to parse this to get the repo URL
+		// see: https://git-scm.com/docs/git-credential
+		stdin, err := io.ReadAll(os.Stdin)
+		if err != nil {
+			return handleAuthError(c, l, fmt.Errorf("failed to read stdin: %v", err))
+		}
+
+		repo, err := parseGitURLFromCredentialInput(string(stdin))
+		if err != nil {
+			return handleAuthError(c, l, fmt.Errorf("failed to parse git URL from stdin: %v", err))
+		}
+
+		client := api.NewClient(l, loadAPIClientConfig(cfg, "AgentAccessToken"))
+		tok, _, err := client.GenerateGithubCodeAccessToken(ctx, repo, cfg.JobID)
+		if err != nil {
+			return handleAuthError(c, l, fmt.Errorf("failed to get github app credentials: %v", err))
+		}
+
+		fmt.Fprintln(c.App.Writer, "username=token")
+		fmt.Fprintln(c.App.Writer, "password="+tok)
+		fmt.Fprintln(c.App.Writer, "")
+
+		l.Info("Authentication successful! 🎉")
+
+		return nil
+	},
+}
+
+// handleAuthError is a helper function that logs an error and outputs a dummy password
+// git continues with clones etc even when the credential helper fails, so we should output something that will 100% cause
+// the clone to fail
+// this function always returns a cli.ExitError
+func handleAuthError(c *cli.Context, l logger.Logger, err error) error {
+	l.Error("Error: %v. Authentication will proceed, but will fail.", err)
+	fmt.Fprintln(c.App.Writer, "username=fail")
+	fmt.Fprintln(c.App.Writer, "password=fail")
+	fmt.Fprintln(c.App.Writer, "")
+
+	return cli.NewExitError("", 1)
+}
+
+var (
+	errMissingComponent = errors.New("missing component in git credential input")
+	errNotHTTPS         = errors.New("git remote must be using the https protocol to use Github App authentication")
+)
+
+func parseGitURLFromCredentialInput(input string) (string, error) {
+	lines := strings.Split(input, "\n")
+
+	components := map[string]string{
+		"protocol": "",
+		"host":     "",
+		"path":     "",
+	}
+	for _, line := range lines {
+		if p, ok := strings.CutPrefix(line, "protocol="); ok {
+			components["protocol"] = strings.TrimSpace(p)
+		}
+		if p, ok := strings.CutPrefix(line, "host="); ok {
+			components["host"] = strings.TrimSpace(p)
+		}
+		if p, ok := strings.CutPrefix(line, "path="); ok {
+			components["path"] = strings.TrimSpace(p)
+		}
+	}
+
+	for k, v := range components {
+		if v == "" {
+			return "", fmt.Errorf("%w: %s", errMissingComponent, k)
+		}
+	}
+
+	if components["protocol"] != "https" {
+		return "", errNotHTTPS
+	}
+
+	u := url.URL{
+		Scheme: components["protocol"],
+		Host:   components["host"],
+		Path:   components["path"],
+	}
+
+	return u.String(), nil
+}

clicommand/git_credentials_helper_test.go

@@ -0,0 +1,77 @@
+package clicommand
+
+import (
+	"errors"
+	"strings"
+	"testing"
+)
+
+func TestParseGitCredentialInput(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		name        string
+		lines       []string
+		expected    string
+		expectedErr error
+	}{
+		{
+			name: "happy path",
+			lines: []string{
+				"protocol=https",
+				"host=github.com",
+				"path=buildkite/agent",
+			},
+			expected: "https://github.com/buildkite/agent",
+		},
+		{
+			name: "missing protocol",
+			lines: []string{
+				"host=github.com",
+				"path=buildkite/agent",
+			},
+			expectedErr: errMissingComponent,
+		},
+		{
+			name: "missing host",
+			lines: []string{
+				"protocol=https",
+				"path=buildkite/agent",
+			},
+			expectedErr: errMissingComponent,
+		},
+		{
+			name: "missing path",
+			lines: []string{
+				"protocol=https",
+				"host=github.com",
+			},
+			expectedErr: errMissingComponent,
+		},
+		{
+			name: "non-https protocol",
+			lines: []string{
+				"protocol=ssh",
+				"host=github.com",
+				"path=buildkite/agent",
+			},
+			expectedErr: errNotHTTPS,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			input := strings.Join(tc.lines, "\n")
+			actual, actualErr := parseGitURLFromCredentialInput(input)
+			if !errors.Is(actualErr, tc.expectedErr) {
+				t.Fatalf("parseGitURLFromCredentialInput(%q) = error(%q), want error(%q)", input, actualErr, tc.expectedErr)
+			}
+
+			if actual != tc.expected {
+				t.Fatalf("parseGitURLFromCredentialInput(%q) = %q, want %q", input, actual, tc.expected)
+			}
+		})
+	}
+}

internal/job/checkout.go

@@ -16,6 +16,35 @@ import (
 	"github.com/buildkite/roko"
 )
 
+// configureGitCredentialHelper sets up the agent to use a git credential helper that calls the Buildkite Agent API
+// asking for a Github App token to use when cloning. This feature is turned on serverside
+func (e *Executor) configureGitCredentialHelper(ctx context.Context) error {
+	// credential.useHttpPath is a git config setting that tells git to tell the credential helper the full URL of the repo
+	// this means that we can pass the repo being cloned up to the BK API, which can then choose (or not, if it's not permitted)
+	// to return a token for that repo.
+	//
+	// This is important for the case where a user clones multiple repos in a step - ie, if we always crammed
+	// os.Getenv("BUILDKITE_REPO") into credential helper, we'd only ever get a token for the repo that the step is running
+	// in, and not for any other repos that the step might clone.
+	err := e.shell.RunWithoutPrompt(ctx, "git", "config", "--global", "credential.useHttpPath", "true")
+	if err != nil {
+		return fmt.Errorf("enabling git credential.useHttpPath: %v", err)
+	}
+
+	buildkiteAgent, err := os.Executable()
+	if err != nil {
+		return fmt.Errorf("getting executable path: %v", err)
+	}
+
+	helper := fmt.Sprintf(`%s git-credentials-helper`, buildkiteAgent)
+	err = e.shell.RunWithoutPrompt(ctx, "git", "config", "--global", "credential.helper", helper)
+	if err != nil {
+		return fmt.Errorf("configuring git credential.helper: %v", err)
+	}
+
+	return nil
+}
+
 func (e *Executor) removeCheckoutDir() error {
 	checkoutPath, _ := e.shell.Env.Get("BUILDKITE_BUILD_CHECKOUT_PATH")
 

internal/job/executor.go

@@ -139,6 +139,14 @@ func (e *Executor) Run(ctx context.Context) (exitCode int) {
 		}
 	}()
 
+	if env, ok := e.shell.Env.Get("BUILDKITE_USE_GITHUB_APP_GIT_CREDENTIALS"); ok && env == "true" {
+		err := e.configureGitCredentialHelper(ctx)
+		if err != nil {
+			e.shell.Errorf("Error configuring git credential helper: %v", err)
+			return shell.GetExitCode(err)
+		}
+	}
+
 	// Initialize the environment, a failure here will still call the tearDown
 	if err = e.setUp(ctx); err != nil {
 		e.shell.Errorf("Error setting up job executor: %v", err)