backend: Create GetFileContents method in GitHub service (#3224)

backend/mock/service/githubmock/githubmock.go

@@ -128,6 +128,15 @@ func (s *svc) SearchCode(ctx context.Context, query string, opts *githubv3.Searc
 	}, nil
 }
 
+func (s *svc) GetFileContents(ctx context.Context, ref *github.RemoteRef, path string) (*githubv3.RepositoryContent, error) {
+	return &githubv3.RepositoryContent{
+		Name:     githubv3.String("README.md"),
+		Path:     githubv3.String("README.md"),
+		Content:  githubv3.String("# Hello World"),
+		Encoding: githubv3.String(""),
+	}, nil
+}
+
 func NewAsService(*any.Any, *zap.Logger, tally.Scope) (service.Service, error) {
 	return New(), nil
 }

backend/service/github/github.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"path"
 	"strconv"
 	"strings"
 	"time"
@@ -45,14 +46,25 @@ const (
 type FileMap map[string]io.ReadCloser
 
 type StatsRoundTripper struct {
-	Wrapped http.RoundTripper
-	scope   tally.Scope
+	Wrapped   http.RoundTripper
+	scope     tally.Scope
+	AcceptRaw bool
 }
 
 func (st *StatsRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
-	resp, err := st.Wrapped.RoundTrip(req)
+	if st.AcceptRaw && GetRepositoryContentRegex.Match([]byte(req.URL.Path)) {
+		req.Header.Set("accept", AcceptGithubRawMediaType)
+	}
 
+	resp, err := st.Wrapped.RoundTrip(req)
 	if resp != nil {
+		if st.AcceptRaw && GetRepositoryContentRegex.Match([]byte(req.URL.Path)) {
+			err = InterceptGetRepositoryContentResponse(resp)
+			if err != nil {
+				return nil, err
+			}
+		}
+
 		if hdr := resp.Header.Get("X-RateLimit-Remaining"); hdr != "" {
 			if v, err := strconv.Atoi(hdr); err == nil {
 				st.scope.Gauge("rate_limit_remaining").Update(float64(v))
@@ -129,6 +141,7 @@ type Client interface {
 	DeleteFile(ctx context.Context, ref *RemoteRef, path, sha, message string) (*githubv3.RepositoryContentResponse, error)
 	CreateCommit(ctx context.Context, ref *RemoteRef, message string, files FileMap) (*Commit, error)
 	SearchCode(ctx context.Context, query string, opts *githubv3.SearchOptions) (*githubv3.CodeSearchResult, error)
+	GetFileContents(ctx context.Context, ref *RemoteRef, path string) (*githubv3.RepositoryContent, error)
 }
 
 // This func can be used to create comments for PRs or Issues
@@ -154,6 +167,7 @@ type svc struct {
 	rest    v3client
 
 	appTransport        *ghinstallation.Transport
+	httpTransport       *StatsRoundTripper
 	personalAccessToken string
 }
 
@@ -398,7 +412,7 @@ func newService(config *githubv1.Config, scope tally.Scope, logger *zap.Logger)
 	default:
 		return nil, fmt.Errorf("did not recognize auth config type '%T'", auth)
 	}
-	httpClient.Transport = &StatsRoundTripper{Wrapped: httpClient.Transport, scope: scope}
+	transport := &StatsRoundTripper{Wrapped: httpClient.Transport, scope: scope, AcceptRaw: false}
 
 	restClient := githubv3.NewClient(httpClient)
 	ret.rest = v3client{
@@ -410,6 +424,9 @@ func newService(config *githubv1.Config, scope tally.Scope, logger *zap.Logger)
 		Users:         restClient.Users,
 	}
 
+	httpClient.Transport = transport
+	ret.httpTransport = transport
+
 	ret.graphQL = githubv4.NewClient(httpClient)
 
 	return ret, nil
@@ -673,3 +690,32 @@ func (s *svc) SearchCode(ctx context.Context, query string, opts *githubv3.Searc
 
 	return results, nil
 }
+
+func (s *svc) GetFileContents(ctx context.Context, ref *RemoteRef, filePath string) (*githubv3.RepositoryContent, error) {
+	options := &githubv3.RepositoryContentGetOptions{Ref: ref.Ref}
+	file, _, _, err := s.rest.Repositories.GetContents(ctx, ref.RepoOwner, ref.RepoName, filePath, options)
+	if err != nil {
+		return nil, err
+	}
+
+	// From https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#get-repository-content
+	// If the requested file's size is between 1-100 MB:
+	// Only the raw or object custom media types are supported.  Both will work as normal, except that
+	// when using the object media type, the content field will be an empty string and the encoding field
+	// will be "none". To get the contents of these larger files, use the raw media type.
+	if file.Content != nil && *file.Content == "" && file.GetEncoding() == "none" {
+		s.httpTransport.AcceptRaw = true
+		file, _, _, err = s.rest.Repositories.GetContents(ctx, ref.RepoOwner, ref.RepoName, filePath, options)
+		s.httpTransport.AcceptRaw = false
+		if err != nil {
+			return nil, err
+		}
+
+		file.Path = &filePath
+		file.Name = githubv3.String(path.Base(filePath))
+
+		return file, err
+	}
+
+	return file, nil
+}

backend/service/github/helpers.go

@@ -0,0 +1,66 @@
+package github
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"net/http"
+	"regexp"
+	"strings"
+
+	githubv3 "github.com/google/go-github/v54/github"
+)
+
+const (
+	// From https://docs.github.com/en/rest/using-the-rest-api/getting-started-with-the-rest-api?apiVersion=2022-11-28#media-types
+	// and https://docs2.lfe.io/v3/media/
+	AcceptGithubRawMediaType = "application/vnd.github.v3.raw+json"
+	// From https://cs.opensource.google/go/go/+/master:src/encoding/json/scanner.go;l=593?q=scanner.go&ss=go%2Fgo
+	JSONSyntaxErrorPrefix = "invalid character"
+)
+
+// RegEx only for `Repositories.GetContents` must match something like `/repos/lyft/clutch/contents/README.md`
+var GetRepositoryContentRegex = regexp.MustCompile(`^\/repos\/[\w-]+\/[\w-]+\/contents\/[\w-.\/]+$`)
+
+// `Repositories.GetContents“ method cannot process a raw response, so we intercept it
+func InterceptGetRepositoryContentResponse(res *http.Response) error {
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+
+	var fileContentBytes []byte
+	var fileContent *githubv3.RepositoryContent
+
+	// Sometimes we get a githubv3.RepositoryContent body and we have to check
+	err = json.Unmarshal(body, &fileContent)
+	switch {
+	case fileContent != nil &&
+		fileContent.Content != nil &&
+		fileContent.Encoding != nil &&
+		fileContent.Path != nil &&
+		fileContent.GitURL != nil:
+		fileContentBytes = body
+	case err != nil:
+		if !strings.HasPrefix(err.Error(), JSONSyntaxErrorPrefix) {
+			return err
+		}
+		fallthrough
+	default:
+		// Set the body as a `githubv3.RepositoryContent`
+		fileContentBytes, err = json.Marshal(&githubv3.RepositoryContent{
+			Content:  githubv3.String(string(body)),
+			Encoding: githubv3.String(""), // The content is not encoded
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	// Recreate the response body
+	res.Body = io.NopCloser(bytes.NewReader(fileContentBytes))
+	res.ContentLength = int64(len(fileContentBytes))
+
+	return nil
+}