Skip to content

Commit

Permalink
Fix: Dont sent stop build on success (#867)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dleviminzi
dleviminzi authored Jan 17, 2025
1 parent cb94880 commit 1e459fa
Showing 1 changed file with 25 additions and 18 deletions.
43 changes: 25 additions & 18 deletions pkg/abstractions/image/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"regexp"
"runtime"
"strings"
"sync/atomic"
"text/template"
"time"

Expand Down Expand Up @@ -222,6 +223,7 @@ func (b *Builder) Build(ctx context.Context, opts *BuildOpts, outputChan chan co
dockerfile *string
authInfo, _ = auth.AuthInfoFromContext(ctx)
containerSpinupTimeout = defaultContainerSpinupTimeout
success = atomic.Bool{}
)

switch {
Expand All @@ -239,13 +241,16 @@ func (b *Builder) Build(ctx context.Context, opts *BuildOpts, outputChan chan co

containerRequest, err := b.generateContainerRequest(opts, dockerfile, containerId, authInfo.Workspace)
if err != nil {
outputChan <- common.OutputMsg{Done: true, Success: false, Msg: "Error occured while generating container request: " + err.Error()}
outputChan <- common.OutputMsg{Done: true, Success: success.Load(), Msg: "Error occured while generating container request: " + err.Error()}
return err
}

// If user cancels the build, send a stop-build event to the worker
go func() {
<-ctx.Done()
if success.Load() {
return
}
err := b.stopBuild(containerId)
if err != nil {
log.Error().Str("container_id", containerId).Err(err).Msg("failed to stop build")
Expand All @@ -254,33 +259,33 @@ func (b *Builder) Build(ctx context.Context, opts *BuildOpts, outputChan chan co

err = b.scheduler.Run(containerRequest)
if err != nil {
outputChan <- common.OutputMsg{Done: true, Success: false, Msg: err.Error() + "\n"}
outputChan <- common.OutputMsg{Done: true, Success: success.Load(), Msg: err.Error() + "\n"}
return err
}

hostname, err := b.containerRepo.GetWorkerAddress(ctx, containerId)
if err != nil {
outputChan <- common.OutputMsg{Done: true, Success: false, Msg: "Failed to connect to build container.\n"}
outputChan <- common.OutputMsg{Done: true, Success: success.Load(), Msg: "Failed to connect to build container.\n"}
return err
}

err = b.rdb.Set(ctx, Keys.imageBuildContainerTTL(containerId), "1", time.Duration(imageContainerTtlS)*time.Second).Err()
if err != nil {
outputChan <- common.OutputMsg{Done: true, Success: false, Msg: "Failed to connect to build container.\n"}
outputChan <- common.OutputMsg{Done: true, Success: success.Load(), Msg: "Failed to connect to build container.\n"}
return err
}

go b.keepAlive(ctx, containerId, ctx.Done())

conn, err := network.ConnectToHost(ctx, hostname, time.Second*30, b.tailscale, b.config.Tailscale)
if err != nil {
outputChan <- common.OutputMsg{Done: true, Success: false, Msg: "Failed to connect to build container.\n"}
outputChan <- common.OutputMsg{Done: true, Success: success.Load(), Msg: "Failed to connect to build container.\n"}
return err
}

client, err := common.NewRunCClient(hostname, authInfo.Token.Key, conn)
if err != nil {
outputChan <- common.OutputMsg{Done: true, Success: false, Msg: "Failed to connect to build container.\n"}
outputChan <- common.OutputMsg{Done: true, Success: success.Load(), Msg: "Failed to connect to build container.\n"}
return err
}
go client.StreamLogs(ctx, containerId, outputChan)
Expand All @@ -291,21 +296,21 @@ func (b *Builder) Build(ctx context.Context, opts *BuildOpts, outputChan chan co
}()
defer client.Kill(containerId) // Kill and remove container after the build completes

outputChan <- common.OutputMsg{Done: false, Success: false, Msg: "Waiting for build container to start...\n"}
outputChan <- common.OutputMsg{Done: false, Success: success.Load(), Msg: "Waiting for build container to start...\n"}
start := time.Now()
buildContainerRunning := false

for !buildContainerRunning {
select {
case <-ctx.Done():
log.Info().Str("container_id", containerId).Msg("build was aborted")
outputChan <- common.OutputMsg{Done: true, Success: false, Msg: "Build was aborted.\n"}
outputChan <- common.OutputMsg{Done: true, Success: success.Load(), Msg: "Build was aborted.\n"}
return ctx.Err()

case <-time.After(100 * time.Millisecond):
r, err := client.Status(containerId)
if err != nil {
outputChan <- common.OutputMsg{Done: true, Success: false, Msg: "Error occurred while checking container status: " + err.Error()}
outputChan <- common.OutputMsg{Done: true, Success: success.Load(), Msg: "Error occurred while checking container status: " + err.Error()}
return err
}

Expand All @@ -322,7 +327,7 @@ func (b *Builder) Build(ctx context.Context, opts *BuildOpts, outputChan chan co
}
// Wait for any final logs to get sent before returning
time.Sleep(200 * time.Millisecond)
outputChan <- common.OutputMsg{Done: true, Success: false, Msg: fmt.Sprintf("Container exited with error: %s\n", msg)}
outputChan <- common.OutputMsg{Done: true, Success: success.Load(), Msg: fmt.Sprintf("Container exited with error: %s\n", msg)}
return errors.New(fmt.Sprintf("container exited with error: %s\n", msg))
}

Expand All @@ -331,20 +336,20 @@ func (b *Builder) Build(ctx context.Context, opts *BuildOpts, outputChan chan co
if err != nil {
log.Error().Str("container_id", containerId).Err(err).Msg("failed to stop build")
}
outputChan <- common.OutputMsg{Done: true, Success: false, Msg: fmt.Sprintf("Timeout: container not running after %s seconds.\n", containerSpinupTimeout)}
outputChan <- common.OutputMsg{Done: true, Success: success.Load(), Msg: fmt.Sprintf("Timeout: container not running after %s seconds.\n", containerSpinupTimeout)}
return errors.New(fmt.Sprintf("timeout: container not running after %s seconds", containerSpinupTimeout))
}
}
}

imageId, err := b.GetImageId(opts)
if err != nil {
outputChan <- common.OutputMsg{Done: true, Success: false, Msg: "Error occured while generating image id: " + err.Error()}
outputChan <- common.OutputMsg{Done: true, Success: success.Load(), Msg: "Error occured while generating image id: " + err.Error()}
return err
}

if !buildContainerRunning {
outputChan <- common.OutputMsg{Done: true, Success: false, Msg: "Unable to connect to build container.\n"}
outputChan <- common.OutputMsg{Done: true, Success: success.Load(), Msg: "Unable to connect to build container.\n"}
return errors.New("container not running")
}

Expand All @@ -365,10 +370,10 @@ func (b *Builder) Build(ctx context.Context, opts *BuildOpts, outputChan chan co
// Detect if python3.x is installed in the container, if not install it
checkPythonVersionCmd := fmt.Sprintf("%s --version", opts.PythonVersion)
if resp, err := client.Exec(containerId, checkPythonVersionCmd); (err != nil || !resp.Ok) && !micromambaEnv {
outputChan <- common.OutputMsg{Done: false, Success: false, Msg: fmt.Sprintf("%s not detected, installing it for you...\n", opts.PythonVersion)}
outputChan <- common.OutputMsg{Done: false, Success: success.Load(), Msg: fmt.Sprintf("%s not detected, installing it for you...\n", opts.PythonVersion)}
installCmd, err := getPythonStandaloneInstallCommand(b.config.ImageService.Runner.PythonStandalone, opts.PythonVersion)
if err != nil {
outputChan <- common.OutputMsg{Done: true, Success: false, Msg: err.Error() + "\n"}
outputChan <- common.OutputMsg{Done: true, Success: success.Load(), Msg: err.Error() + "\n"}
return err
}
opts.Commands = append([]string{installCmd}, opts.Commands...)
Expand All @@ -391,19 +396,20 @@ func (b *Builder) Build(ctx context.Context, opts *BuildOpts, outputChan chan co
}

time.Sleep(defaultImageBuildGracefulShutdownS) // Wait for logs to be passed through
outputChan <- common.OutputMsg{Done: true, Success: false, Msg: errMsg}
outputChan <- common.OutputMsg{Done: true, Success: success.Load(), Msg: errMsg}
return err
}
}
log.Info().Str("container_id", containerId).Dur("duration", time.Since(startTime)).Msg("container build took")

err = client.Archive(ctx, containerId, imageId, outputChan)
if err != nil {
outputChan <- common.OutputMsg{Done: true, Archiving: true, Success: false, Msg: err.Error() + "\n"}
outputChan <- common.OutputMsg{Done: true, Archiving: true, Success: success.Load(), Msg: err.Error() + "\n"}
return err
}

outputChan <- common.OutputMsg{Done: true, Archiving: true, Success: true, ImageId: imageId}
success.Store(true)
outputChan <- common.OutputMsg{Done: true, Archiving: true, Success: success.Load(), ImageId: imageId}
return nil
}

Expand Down Expand Up @@ -803,6 +809,7 @@ func extractPackageName(pkg string) string {
}

func (b *Builder) stopBuild(containerId string) error {

_, err := b.eventBus.Send(&common.Event{
Type: common.StopBuildEventType(containerId),
Args: map[string]any{"container_id": containerId},
Expand Down

0 comments on commit 1e459fa

Please sign in to comment.