Skip to content

Commit

Permalink
Build-and-push in submit when arg is dir
Browse files Browse the repository at this point in the history
  • Loading branch information
discordianfish committed Jan 20, 2025
1 parent 4e75dac commit a8eecfa
Show file tree
Hide file tree
Showing 10 changed files with 153 additions and 48 deletions.
8 changes: 1 addition & 7 deletions pkg/cmd/agent/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (

"github.com/diambra/cli/pkg/container"
"github.com/diambra/cli/pkg/log"
dclient "github.com/docker/docker/client"
"github.com/go-kit/log/level"
"github.com/spf13/cobra"
)
Expand All @@ -35,13 +34,8 @@ func NewBuildCmd(logger *log.Logger) *cobra.Command {
if len(args) == 0 {
args = []string{"."}
}
client, err := dclient.NewClientWithOpts(dclient.FromEnv, dclient.WithAPIVersionNegotiation())
if err != nil {
level.Error(logger).Log("msg", "failed to create docker client", "err", err)
os.Exit(1)
}

runner, err := container.NewDockerRunner(logger, client, false)
runner, err := container.NewDockerRunner(logger, false)
if err != nil {
level.Error(logger).Log("msg", "failed to create docker runner", "err", err)
os.Exit(1)
Expand Down
8 changes: 1 addition & 7 deletions pkg/cmd/agent/build_and_push.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
"github.com/diambra/cli/pkg/container"
"github.com/diambra/cli/pkg/diambra/client"
"github.com/diambra/cli/pkg/log"
dclient "github.com/docker/docker/client"
"github.com/go-kit/log/level"
"github.com/spf13/cobra"
)
Expand All @@ -45,13 +44,8 @@ func NewBuildAndPushCmd(logger *log.Logger) *cobra.Command {
if len(args) == 0 {
args = []string{"."}
}
dc, err := dclient.NewClientWithOpts(dclient.FromEnv, dclient.WithAPIVersionNegotiation())
if err != nil {
level.Error(logger).Log("msg", "failed to create docker client", "err", err)
os.Exit(1)
}

runner, err := container.NewDockerRunner(logger, dc, false)
runner, err := container.NewDockerRunner(logger, false)
if err != nil {
level.Error(logger).Log("msg", "failed to create docker runner", "err", err)
os.Exit(1)
Expand Down
75 changes: 71 additions & 4 deletions pkg/cmd/agent/submit.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,31 +17,40 @@ package agent

import (
"fmt"
"net/url"
"os"
"path/filepath"
"time"

"github.com/diambra/cli/pkg/container"
"github.com/diambra/cli/pkg/diambra"
"github.com/diambra/cli/pkg/diambra/client"
"github.com/diambra/cli/pkg/git"
"github.com/diambra/cli/pkg/log"
"github.com/go-kit/log/level"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
)

func NewSubmitCmd(logger *log.Logger) *cobra.Command {
dump := false
submissionConfig := diambra.SubmissionConfig{}
var (
dump = false
submissionConfig = diambra.SubmissionConfig{}
name = ""
version = ""
)
submissionConfig.RegisterCredentialsProviders()

c, err := diambra.NewConfig(logger)
if err != nil {
level.Error(logger).Log("msg", err.Error())
os.Exit(1)
}

cmd := &cobra.Command{
Use: "submit [flags] {--submission.manifest submission-manifest.yaml | docker-image} [args/command(s) ...]",
Use: "submit [flags] (directory | --submission.manifest=submission-manifest.yaml | docker-image) [args/command(s) ...]",
Short: "Submits an agent for evaluation",
Long: `This takes a docker image or submission manifest and submits it for evaluation.`,
Long: `This takes a directory, existing docker image or submission manifest and submits it for evaluation.`,
Run: func(cmd *cobra.Command, args []string) {
if err := diambra.EnsureCredentials(logger, c.CredPath); err != nil {
level.Error(logger).Log("msg", err.Error())
Expand All @@ -61,11 +70,67 @@ func NewSubmitCmd(logger *log.Logger) *cobra.Command {
fmt.Println(string(b))
return
}

cl, err := client.NewClient(logger, c.CredPath)
if err != nil {
level.Error(logger).Log("msg", "failed to create client", "err", err.Error())
os.Exit(1)
}
// If submission.Image is a directory, we build and push it, then update the name to the resulting image
if stat, err := os.Stat(submission.Manifest.Image); err == nil && stat.IsDir() {
context := submission.Manifest.Image
level.Info(logger).Log("msg", "Building and pushing image", "context", context)

if name == "" {
name, err = container.TagFromDir(context)
if err != nil {
level.Error(logger).Log("msg", "failed to get tag from dir", "err", err)
os.Exit(1)
}
}

if version == "" {
version, err = git.GitHeadSHAShort(context, 0)
if err != nil {
level.Warn(logger).Log("msg", "failed to get git head sha", "err", err)
version = time.Now().Format("20060102-150405")
}
}
level.Info(logger).Log("msg", "Building agent", "name", name, "version", version)
runner, err := container.NewDockerRunner(logger, false)
if err != nil {
level.Error(logger).Log("msg", "failed to create docker runner", "err", err)
os.Exit(1)
}
credentials, err := cl.Credentials()
if err != nil {
level.Error(logger).Log("msg", "failed to get credentials", "err", err.Error())
os.Exit(1)
}

repositoryURL, err := url.Parse(credentials.Repository)
if err != nil {
level.Error(logger).Log("msg", "failed to parse repository URL", "err", err)
os.Exit(1)
}

tag := fmt.Sprintf("%s%s:%s-%s", repositoryURL.Host, repositoryURL.Path, name, version)

if err := runner.Build(context, tag); err != nil {
level.Error(logger).Log("msg", "failed to build and push image", "err", err.Error())
os.Exit(1)
}
if err := runner.Push(tag, credentials.Username, credentials.Password, repositoryURL.Host); err != nil {
level.Error(logger).Log("msg", "failed to push agent", "err", err)
os.Exit(1)
}

submission.Manifest.Image = tag
} else {
level.Warn(logger).Log("msg", "Using existing images or submission manifest is not recommended and might get deprecated in the future")
}
os.Exit(1)

id, err := cl.Submit(submission)
if err != nil {
level.Error(logger).Log("msg", "failed to submit agent", "err", err.Error())
Expand All @@ -79,5 +144,7 @@ func NewSubmitCmd(logger *log.Logger) *cobra.Command {
cmd.Flags().StringVar(&c.CredPath, "path.credentials", filepath.Join(c.Home, ".diambra/credentials"), "Path to credentials file")
cmd.Flags().BoolVar(&dump, "dump", false, "Dump the manifest to stdout instead of submitting")
cmd.Flags().SetInterspersed(false)
cmd.Flags().StringVar(&name, "name", name, "Name of the agent image (only used when giving a directory)")
cmd.Flags().StringVar(&version, "version", version, "Version of the agent image (only used when giving a directory)")
return cmd
}
7 changes: 1 addition & 6 deletions pkg/cmd/agent/test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
"github.com/diambra/cli/pkg/diambra"
"github.com/diambra/cli/pkg/diambra/client"
"github.com/diambra/cli/pkg/log"
dclient "github.com/docker/docker/client"
"github.com/go-kit/log/level"
"github.com/spf13/cobra"
)
Expand Down Expand Up @@ -58,11 +57,7 @@ func NewTestCmd(logger *log.Logger) *cobra.Command {
func TestFn(logger *log.Logger, c *diambra.EnvConfig, submission *client.Submission) error {
level.Debug(logger).Log("manifest", fmt.Sprintf("%#v", submission.Manifest), "config", fmt.Sprintf("%#v", c))

client, err := dclient.NewClientWithOpts(dclient.FromEnv, dclient.WithAPIVersionNegotiation())
if err != nil {
return err
}
runner, err := container.NewDockerRunner(logger, client, c.AutoRemove)
runner, err := container.NewDockerRunner(logger, c.AutoRemove)
if err != nil {
return err
}
Expand Down
8 changes: 1 addition & 7 deletions pkg/cmd/arena/down.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (

"github.com/diambra/cli/pkg/container"
"github.com/diambra/cli/pkg/log"
"github.com/docker/docker/client"
"github.com/go-kit/log/level"
"github.com/spf13/cobra"
)
Expand All @@ -31,12 +30,7 @@ func NewDownCmd(logger *log.Logger) *cobra.Command {
Short: "Stop DIAMBRA Arena",
Long: `This stops a DIAMBRA Arena running in the background.`,
Run: func(_ *cobra.Command, _ []string) {
client, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
level.Error(logger).Log("msg", "failed to create docker client", "err", err.Error())
os.Exit(1)
}
runner, err := container.NewDockerRunner(logger, client, true)
runner, err := container.NewDockerRunner(logger, true)
if err != nil {
level.Error(logger).Log("msg", "msg", "failed to create runner", "err", err.Error())
os.Exit(1)
Expand Down
7 changes: 1 addition & 6 deletions pkg/cmd/arena/up.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
"github.com/diambra/cli/pkg/container"
"github.com/diambra/cli/pkg/diambra"
"github.com/diambra/cli/pkg/log"
"github.com/docker/docker/client"
"github.com/go-kit/log/level"
"github.com/spf13/cobra"
)
Expand Down Expand Up @@ -68,11 +67,7 @@ func NewUpCmd(logger *log.Logger) *cobra.Command {
func RunFn(logger *log.Logger, c *diambra.EnvConfig, args []string) error {
level.Debug(logger).Log("config", fmt.Sprintf("%#v", c))

client, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
return err
}
runner, err := container.NewDockerRunner(logger, client, c.AutoRemove)
runner, err := container.NewDockerRunner(logger, c.AutoRemove)
if err != nil {
return err
}
Expand Down
7 changes: 1 addition & 6 deletions pkg/cmd/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ import (
"github.com/diambra/cli/pkg/diambra"
"github.com/diambra/cli/pkg/log"

"github.com/docker/docker/client"
"github.com/go-kit/log/level"
"github.com/spf13/cobra"
)
Expand Down Expand Up @@ -76,11 +75,7 @@ The flag --agent-image can be used to run the commands in the given image.`,
func RunFn(logger *log.Logger, c *diambra.EnvConfig, args []string) error {
level.Debug(logger).Log("config", fmt.Sprintf("%#v", c))

client, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
return err
}
runner, err := container.NewDockerRunner(logger, client, c.AutoRemove)
runner, err := container.NewDockerRunner(logger, c.AutoRemove)
if err != nil {
return err
}
Expand Down
11 changes: 8 additions & 3 deletions pkg/container/docker.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,14 +48,19 @@ type DockerRunner struct {
AutoRemove bool
}

func NewDockerRunner(logger log.Logger, client *client.Client, autoRemove bool) (*DockerRunner, error) {
_, err := client.Ping(context.TODO())
func NewDockerRunner(logger log.Logger, autoRemove bool) (*DockerRunner, error) {
cl, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
level.Error(logger).Log("msg", "failed to create docker client", "err", err.Error())
os.Exit(1)
}
_, err = cl.Ping(context.TODO())
if err != nil {
return nil, fmt.Errorf("couldn't connect to docker. Make sure your user has docker access: %w", err)
}
return &DockerRunner{
Logger: logger,
Client: client,
Client: cl,
TimeoutStop: 10 * time.Second,
AutoRemove: autoRemove,
}, nil
Expand Down
4 changes: 2 additions & 2 deletions pkg/diambra/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ const (
DifficultyHard Difficulty = "hard"
)

var ErrInvalidArgs = errors.New("either image, manifest path or submission id must be provided")
var ErrInvalidArgs = errors.New("either directory, image, manifest path or submission id must be provided")

type SubmissionConfig struct {
Mode string
Expand Down Expand Up @@ -296,7 +296,7 @@ func (c *SubmissionConfig) Submission(config *EnvConfig, args []string) (*client
}
default:
if nargs == 0 {
return nil, fmt.Errorf("either image, manifest path or submission id must be provided")
return nil, fmt.Errorf("either directory, image, manifest path or submission id must be provided")
}
// If we don't have a manifest, args are image and args
manifest = &client.Manifest{}
Expand Down
66 changes: 66 additions & 0 deletions pkg/git/git.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package git

import (
"fmt"
"os"
"path/filepath"
"strings"
)

const RefPrefix = "ref: "

func findGitDir(startDir string) (string, error) {
dir, err := filepath.Abs(startDir)
if err != nil {
return "", fmt.Errorf("unable to resolve absolute path: %w", err)
}

for {
gitPath := filepath.Join(dir, ".git")
if info, err := os.Stat(gitPath); err == nil && info.IsDir() {
return gitPath, nil
}

parent := filepath.Dir(dir)
// root
if parent == dir {
break
}
dir = parent
}
return "", fmt.Errorf("no .git directory found in parents of %s", dir)
}

func GitHeadSHA(dir string) (string, error) {
dir, err := findGitDir(dir)
if err != nil {
return "", err
}
file, err := os.ReadFile(filepath.Join(dir, "HEAD"))
if err != nil {
return "", err
}
if !strings.HasPrefix(string(file), RefPrefix) {
return string(file), nil
}

refFile, err := os.ReadFile(filepath.Join(dir, strings.TrimSpace(string(file)[len(RefPrefix):])))
if err != nil {
return "", err
}
return strings.TrimSpace(string(refFile)), nil
}

func GitHeadSHAShort(dir string, n int) (string, error) {
if n <= 0 {
n = 7
}
sha, err := GitHeadSHA(dir)
if err != nil {
return "", err
}
if len(sha) < n {
return sha, nil
}
return sha[:n], nil
}

0 comments on commit a8eecfa

Please sign in to comment.