Refactoring for other packages (#190)

* Export config package

* datastore can use without MySQL

* Implement fallback target scope for datastore don't store target

* Add PLUGIN_OUTPUT for environment that read-only filesystem in root

* Add comments

* Use action

* Update action

* Use token

* Generate by GitHub Apps

.github/workflows/test.yaml

@@ -15,11 +15,11 @@ jobs:
           - ubuntu-latest
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
       - name: setup go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version-file: 'go.mod'
       - name: lint
@@ -36,8 +36,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
      - name: checkout
-       uses: actions/checkout@v3
+       uses: actions/checkout@v4
        with:
          fetch-depth: 1
-     - name: docker build
-       run: docker build .
+     - name: Build container image
+       uses: docker/build-push-action@v5
+       with:
+         push: false
+         tags: ${{ steps.meta.outputs.tags }}

cmd/server/cmd.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/whywaita/myshoes/internal/config"
+	"github.com/whywaita/myshoes/pkg/config"
 	"github.com/whywaita/myshoes/pkg/datastore"
 	"github.com/whywaita/myshoes/pkg/datastore/mysql"
 	"github.com/whywaita/myshoes/pkg/gh"
@@ -22,6 +22,9 @@ import (
 
 func init() {
 	config.Load()
+	mysqlURL := config.LoadMySQLURL()
+	config.Config.MySQLDSN = mysqlURL
+
 	if err := gh.InitializeCache(config.Config.GitHub.AppID, config.Config.GitHub.PEMByte); err != nil {
 		log.Panicf("failed to create a cache: %+v", err)
 	}

docs/01_01_for_admin_setup.md

@@ -89,6 +89,9 @@ A config variables can set from environment values.
   - required
   - set path of myshoes-provider binary.
   - example) `./shoes-mock` `https://example.com/shoes-mock` `https://github.com/whywaita/myshoes-providers/releases/download/v0.1.0/shoes-lxd-linux-amd64`
+- `PLUGIN_OUTPUT`
+  - default: `.`
+  - set path of directory that contains myshoes-provider binary.
 - `GITHUB_URL`
   - default: `https://github.com`
   - The URL of GitHub Enterprise Server.

internal/config/config.go → pkg/config/config.go

@@ -10,17 +10,13 @@ var Config Conf
 
 // Conf is type of Config
 type Conf struct {
-	GitHub struct {
-		AppID     int64
-		AppSecret []byte
-		PEMByte   []byte
-		PEM       *rsa.PrivateKey
-	}
+	GitHub GitHubApp
 
-	MySQLDSN        string
-	Port            int
-	ShoesPluginPath string
-	RunnerUser      string
+	MySQLDSN              string
+	Port                  int
+	ShoesPluginPath       string
+	ShoesPluginOutputPath string
+	RunnerUser            string
 
 	Debug           bool
 	Strict          bool // check to registered runner before delete job
@@ -33,6 +29,14 @@ type Conf struct {
 	RunnerVersion string
 }
 
+// GitHubApp is type of config value
+type GitHubApp struct {
+	AppID     int64
+	AppSecret []byte
+	PEMByte   []byte
+	PEM       *rsa.PrivateKey
+}
+
 // Config Environment keys
 const (
 	EnvGitHubAppID               = "GITHUB_APP_ID"
@@ -41,6 +45,7 @@ const (
 	EnvMySQLURL                  = "MYSQL_URL"
 	EnvPort                      = "PORT"
 	EnvShoesPluginPath           = "PLUGIN"
+	EnvShoesPluginOutputPath     = "PLUGIN_OUTPUT"
 	EnvRunnerUser                = "RUNNER_USER"
 	EnvDebug                     = "DEBUG"
 	EnvStrict                    = "STRICT"

internal/config/init.go → pkg/config/init.go

@@ -21,57 +21,11 @@ import (
 func Load() {
 	c := LoadWithDefault()
 
-	appID, err := strconv.ParseInt(os.Getenv(EnvGitHubAppID), 10, 64)
-	if err != nil {
-		log.Panicf("failed to parse %s: %+v", EnvGitHubAppID, err)
-	}
-	c.GitHub.AppID = appID
+	ga := LoadGitHubApps()
+	c.GitHub = *ga
 
-	pemBase64ed := os.Getenv(EnvGitHubAppPrivateKeyBase64)
-	if pemBase64ed == "" {
-		log.Panicf("%s must be set", EnvGitHubAppPrivateKeyBase64)
-	}
-	pemByte, err := base64.StdEncoding.DecodeString(pemBase64ed)
-	if err != nil {
-		log.Panicf("failed to decode base64 %s: %+v", EnvGitHubAppPrivateKeyBase64, err)
-	}
-	c.GitHub.PEMByte = pemByte
-	block, _ := pem.Decode(pemByte)
-	if block == nil {
-		log.Panicf("%s is invalid format, please input private key ", EnvGitHubAppPrivateKeyBase64)
-	}
-	key, err := x509.ParsePKCS1PrivateKey(block.Bytes)
-	if err != nil {
-		log.Panicf("%s is invalid format, failed to parse private key: %+v", EnvGitHubAppPrivateKeyBase64, err)
-	}
-	c.GitHub.PEM = key
-
-	appSecret := os.Getenv(EnvGitHubAppSecret)
-	if appSecret == "" {
-		log.Panicf("%s must be set", EnvGitHubAppSecret)
-	}
-	c.GitHub.AppSecret = []byte(appSecret)
-
-	mysqlURL := os.Getenv(EnvMySQLURL)
-	if mysqlURL == "" {
-		log.Panicf("%s must be set", EnvMySQLURL)
-	}
-	c.MySQLDSN = mysqlURL
-
-	pluginPath := os.Getenv(EnvShoesPluginPath)
-	if pluginPath == "" {
-		log.Panicf("%s must be set", EnvShoesPluginPath)
-	}
-	fp, err := fetch(pluginPath)
-	if err != nil {
-		log.Panicf("failed to fetch plugin binary: %+v", err)
-	}
-	absPath, err := checkBinary(fp)
-	if err != nil {
-		log.Panicf("failed to check plugin binary: %+v", err)
-	}
-	c.ShoesPluginPath = absPath
-	log.Printf("use plugin path is %s\n", c.ShoesPluginPath)
+	pluginPath := LoadPluginPath()
+	c.ShoesPluginPath = pluginPath
 
 	Config = c
 }
@@ -167,10 +121,80 @@ func LoadWithDefault() Conf {
 		}
 	}
 
+	c.ShoesPluginOutputPath = "."
+	if os.Getenv(EnvShoesPluginOutputPath) != "" {
+		c.ShoesPluginOutputPath = os.Getenv(EnvShoesPluginOutputPath)
+	}
+
 	Config = c
 	return c
 }
 
+// LoadGitHubApps load config for GitHub Apps
+func LoadGitHubApps() *GitHubApp {
+	var ga GitHubApp
+	appID, err := strconv.ParseInt(os.Getenv(EnvGitHubAppID), 10, 64)
+	if err != nil {
+		log.Panicf("failed to parse %s: %+v", EnvGitHubAppID, err)
+	}
+	ga.AppID = appID
+
+	pemBase64ed := os.Getenv(EnvGitHubAppPrivateKeyBase64)
+	if pemBase64ed == "" {
+		log.Panicf("%s must be set", EnvGitHubAppPrivateKeyBase64)
+	}
+	pemByte, err := base64.StdEncoding.DecodeString(pemBase64ed)
+	if err != nil {
+		log.Panicf("failed to decode base64 %s: %+v", EnvGitHubAppPrivateKeyBase64, err)
+	}
+	ga.PEMByte = pemByte
+
+	block, _ := pem.Decode(pemByte)
+	if block == nil {
+		log.Panicf("%s is invalid format, please input private key ", EnvGitHubAppPrivateKeyBase64)
+	}
+	key, err := x509.ParsePKCS1PrivateKey(block.Bytes)
+	if err != nil {
+		log.Panicf("%s is invalid format, failed to parse private key: %+v", EnvGitHubAppPrivateKeyBase64, err)
+	}
+	ga.PEM = key
+
+	appSecret := os.Getenv(EnvGitHubAppSecret)
+	if appSecret == "" {
+		log.Panicf("%s must be set", EnvGitHubAppSecret)
+	}
+	ga.AppSecret = []byte(appSecret)
+
+	return &ga
+}
+
+// LoadMySQLURL load MySQL URL from environment
+func LoadMySQLURL() string {
+	mysqlURL := os.Getenv(EnvMySQLURL)
+	if mysqlURL == "" {
+		log.Panicf("%s must be set", EnvMySQLURL)
+	}
+	return mysqlURL
+}
+
+// LoadPluginPath load plugin path from environment
+func LoadPluginPath() string {
+	pluginPath := os.Getenv(EnvShoesPluginPath)
+	if pluginPath == "" {
+		log.Panicf("%s must be set", EnvShoesPluginPath)
+	}
+	fp, err := fetch(pluginPath)
+	if err != nil {
+		log.Panicf("failed to fetch plugin binary: %+v", err)
+	}
+	absPath, err := checkBinary(fp)
+	if err != nil {
+		log.Panicf("failed to check plugin binary: %+v", err)
+	}
+	log.Printf("use plugin path is %s\n", absPath)
+	return absPath
+}
+
 func checkBinary(p string) (string, error) {
 	if _, err := os.Stat(p); err != nil {
 		return "", fmt.Errorf("failed to stat file: %w", err)
@@ -210,23 +234,27 @@ func fetch(p string) (string, error) {
 	case "http", "https":
 		return fetchHTTP(u)
 	default:
-		return "", fmt.Errorf("unsupported fetch schema")
+		return "", fmt.Errorf("unsupported fetch schema (scheme: %s)", u.Scheme)
 	}
 }
 
 // fetchHTTP fetch plugin binary over HTTP(s).
 // save to current directory.
 func fetchHTTP(u *url.URL) (string, error) {
 	log.Printf("fetch plugin binary from %s\n", u.String())
-	pwd, err := os.Getwd()
-	if err != nil {
-		return "", fmt.Errorf("failed to working directory: %w", err)
+	dir := Config.ShoesPluginOutputPath
+	if strings.EqualFold(dir, ".") {
+		pwd, err := os.Getwd()
+		if err != nil {
+			return "", fmt.Errorf("failed to working directory: %w", err)
+		}
+		dir = pwd
 	}
 
 	p := strings.Split(u.Path, "/")
 	fileName := p[len(p)-1]
 
-	fp := filepath.Join(pwd, fileName)
+	fp := filepath.Join(dir, fileName)
 	f, err := os.Create(fp)
 	if err != nil {
 		return "", fmt.Errorf("failed to create os file: %w", err)

pkg/datastore/mysql/lock.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 
 	"github.com/go-sql-driver/mysql"
-	"github.com/whywaita/myshoes/internal/config"
+	"github.com/whywaita/myshoes/pkg/config"
 	"github.com/whywaita/myshoes/pkg/datastore"
 )
 

pkg/gh/github.go

@@ -12,7 +12,7 @@ import (
 	"github.com/google/go-github/v47/github"
 	"github.com/m4ns0ur/httpcache"
 	"github.com/patrickmn/go-cache"
-	"github.com/whywaita/myshoes/internal/config"
+	"github.com/whywaita/myshoes/pkg/config"
 	"golang.org/x/oauth2"
 )
 

pkg/gh/github_test.go

@@ -4,7 +4,7 @@ import (
 	"os"
 	"testing"
 
-	"github.com/whywaita/myshoes/internal/config"
+	"github.com/whywaita/myshoes/pkg/config"
 )
 
 func TestDetectScope(t *testing.T) {

pkg/gh/jwt.go

@@ -6,8 +6,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/whywaita/myshoes/internal/config"
-
+	"github.com/whywaita/myshoes/pkg/config"
 	"github.com/whywaita/myshoes/pkg/logger"
 
 	"github.com/google/go-github/v47/github"

pkg/gh/runner.go

@@ -85,7 +85,19 @@ func listRunners(ctx context.Context, client *github.Client, owner, repo string,
 }
 
 // GetLatestRunnerVersion get a latest version of actions/runner
-func GetLatestRunnerVersion(ctx context.Context, scope, token string) (string, error) {
+func GetLatestRunnerVersion(ctx context.Context, scope string) (string, error) {
+	clientApps, err := NewClientGitHubApps()
+	if err != nil {
+		return "", fmt.Errorf("failed to create a client from Apps: %+v", err)
+	}
+	installationID, err := IsInstalledGitHubApp(ctx, scope)
+	if err != nil {
+		return "", fmt.Errorf("failed to get installlation id: %w", err)
+	}
+	token, _, err := GenerateGitHubAppsToken(ctx, clientApps, installationID, scope)
+	if err != nil {
+		return "", fmt.Errorf("failed to get registration token: %w", err)
+	}
 	client, err := NewClient(token)
 	if err != nil {
 		return "", fmt.Errorf("failed to create GitHub client: %w", err)

pkg/logger/logger.go

@@ -5,7 +5,7 @@ import (
 	"os"
 	"sync"
 
-	"github.com/whywaita/myshoes/internal/config"
+	"github.com/whywaita/myshoes/pkg/config"
 )
 
 var (

pkg/metric/scrape_memory.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/whywaita/myshoes/internal/config"
+	"github.com/whywaita/myshoes/pkg/config"
 	"github.com/whywaita/myshoes/pkg/datastore"
 	"github.com/whywaita/myshoes/pkg/gh"
 	"github.com/whywaita/myshoes/pkg/runner"

pkg/runner/runner_delete.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"github.com/google/go-github/v47/github"
-	"github.com/whywaita/myshoes/internal/config"
+	"github.com/whywaita/myshoes/pkg/config"
 	"github.com/whywaita/myshoes/pkg/datastore"
 	"github.com/whywaita/myshoes/pkg/gh"
 	"github.com/whywaita/myshoes/pkg/logger"

pkg/shoes/shoes.go

@@ -9,7 +9,7 @@ import (
 	"github.com/hashicorp/go-plugin"
 
 	pb "github.com/whywaita/myshoes/api/proto.go"
-	"github.com/whywaita/myshoes/internal/config"
+	"github.com/whywaita/myshoes/pkg/config"
 	"github.com/whywaita/myshoes/pkg/datastore"
 
 	"google.golang.org/grpc"