Skip to content

Commit

Permalink
implements build image extension by daemon
Browse files Browse the repository at this point in the history 
Signed-off-by: Darshan Kumar <[email protected]>
  • Loading branch information
@itsdarshankumar
itsdarshankumar committed Jun 25, 2023
1 parent 5c460c5 commit 8710e46
Show file tree
Hide file tree
Showing 3 changed files with 102 additions and 16 deletions.
26 changes: 18 additions & 8 deletions internal/build/helper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,18 @@ import (
"os"
"path/filepath"
"strings"
"time"

"github.com/BurntSushi/toml"
"github.com/buildpacks/imgutil/layout/sparse"
"github.com/buildpacks/lifecycle/buildpack"
"github.com/buildpacks/lifecycle/cmd"
"github.com/buildpacks/pack/pkg/logging"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/daemon"
"golang.org/x/sync/errgroup"

"github.com/buildpacks/pack/pkg/logging"
)

const (
Expand Down Expand Up @@ -94,22 +96,24 @@ func escapeID(id string) string {
return strings.ReplaceAll(id, "/", "_")
}

func SaveLayers(group *errgroup.Group, image v1.Image, origTopLayerHash string, dest string) error {
func SaveLayers(group *errgroup.Group, image v1.Image, origTopLayerHash string, dest string) (*time.Duration, error) {
var totalSaveExecutionTime time.Duration
savetime := time.Now()
layoutPath, err := sparse.NewImage(dest, image)
if err != nil {
fmt.Println("sparse.NewImage err", err)
return err
return nil, err
}
if err = layoutPath.Save(); err != nil {
return err
return nil, err
}
if err != nil {
fmt.Println("sparse.NewImage err", err)
return err
return nil, err
}
layers, err := image.Layers()
if err != nil {
return fmt.Errorf("getting image layers: %w", err)
return nil, fmt.Errorf("getting image layers: %w", err)
}
var (
currentHash v1.Hash
Expand All @@ -121,12 +125,17 @@ func SaveLayers(group *errgroup.Group, image v1.Image, origTopLayerHash string,
for _, currentLayer := range layers {
currentHash, err = currentLayer.Digest()
if err != nil {
return fmt.Errorf("getting layer hash: %w", err)
return nil, fmt.Errorf("getting layer hash: %w", err)
}
switch {
case needsCopying:
currentLayer := currentLayer
start := time.Now()
group.Go(func() error {
defer func() {
duration := time.Since(start)
totalSaveExecutionTime += duration
}()
return copyLayer(currentLayer, dest)
})
case currentHash.String() == origTopLayerHash:
Expand All @@ -136,7 +145,8 @@ func SaveLayers(group *errgroup.Group, image v1.Image, origTopLayerHash string,
continue
}
}
return nil
totalSaveExecutionTime += time.Since(savetime)
return &totalSaveExecutionTime, nil
}

func copyLayer(layer v1.Layer, toSparseImage string) error {
Expand Down
86 changes: 78 additions & 8 deletions internal/build/lifecycle_execution.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,10 +234,20 @@ func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseF

group, _ := errgroup.WithContext(context.TODO())
if l.platformAPI.AtLeast("0.10") && l.hasExtensionsForBuild() {
group.Go(func() error {
l.logger.Info(style.Step("EXTENDING (BUILD)"))
return l.ExtendBuild(ctx, buildCache, phaseFactory)
})
if l.opts.Publish {
group.Go(func() error {
l.logger.Info(style.Step("EXTENDING (BUILD)"))
return l.ExtendBuild(ctx, buildCache, phaseFactory)
})
} else {
group.Go(func() error {
l.logger.Info(style.Step("EXTENDING (BUILD) BY DAEMON"))
if err := l.ExtendBuildByDaemon(ctx, group); err != nil {
return err
}
return l.Build(ctx, phaseFactory)
})
}
} else {
group.Go(func() error {
l.logger.Info(style.Step("BUILDING"))
Expand All @@ -262,15 +272,17 @@ func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseF
start = time.Now()
group.Go(func() error {
l.logger.Info(style.Step("EXTENDING (RUN) BY DAEMON"))
defer func() {
duration := time.Since(start)
fmt.Println("Execution time:", duration)
}()
return l.ExtendRunByDaemon(ctx, group, &currentRunImage)
})
}
}
if err := group.Wait(); err != nil {
return err
}
duration := time.Since(start)
fmt.Println("Execution time:", duration)

l.logger.Info(style.Step("EXPORTING"))
return l.Export(ctx, buildCache, launchCache, phaseFactory)
Expand Down Expand Up @@ -657,6 +669,8 @@ func (l *LifecycleExecution) Build(ctx context.Context, phaseFactory PhaseFactor
WithNetwork(l.opts.Network),
WithBinds(l.opts.Volumes...),
WithFlags(flags...),
If((!l.opts.Publish && l.hasExtensionsForBuild()), WithImage("newbuilder-image:latest")),
If((!l.opts.Publish && l.hasExtensionsForBuild()), WithoutPrivilege()),
)

build := phaseFactory.New(configProvider)
Expand Down Expand Up @@ -725,27 +739,78 @@ func (l *LifecycleExecution) ExtendRun(ctx context.Context, buildCache Cache, ph
defer extend.Cleanup()
return extend.Run(ctx)
}
func (l *LifecycleExecution) ExtendBuildByDaemon(ctx context.Context, group *errgroup.Group) error {
builderImageName := l.opts.BuilderImage
defaultFilterFunc := func(file string) bool { return true }
var extensions Extensions
time1 := time.Now()
extensions.SetExtensions(l.tmpDir, l.logger)
fmt.Println("extensions.SetExtensions took", time.Since(time1))
time2 := time.Now()
dockerfiles, err := extensions.DockerFiles(DockerfileKindBuild, l.tmpDir, l.logger)
if err != nil {
return fmt.Errorf("getting %s.Dockerfiles: %w", DockerfileKindBuild, err)
}
fmt.Println("extensions.DockerFiles took", time.Since(time2))
dockerapplytime := time.Now()
for _, dockerfile := range dockerfiles {
buildContext := archive.ReadDirAsTar(filepath.Dir(dockerfile.Path), "/", 0, 0, -1, true, false, defaultFilterFunc)
buildArguments := map[string]*string{}
if dockerfile.WithBase == "" {
buildArguments["base_image"] = &builderImageName
}
buildOptions := types.ImageBuildOptions{
Context: buildContext,
Dockerfile: "Dockerfile",
Tags: []string{"newbuilder-image"},
Remove: true,
BuildArgs: buildArguments,
}
response, err := l.docker.ImageBuild(ctx, buildContext, buildOptions)
if err != nil {
return err
}
defer response.Body.Close()
_, err = io.Copy(os.Stdout, response.Body)
if err != nil {
return err
}
l.logger.Debugf("build response for the extend: %v", response)
}
l.logger.Debugf("docker apply time: %v", time.Since(dockerapplytime))

return nil
}

func (l *LifecycleExecution) ExtendRunByDaemon(ctx context.Context, group *errgroup.Group, currentRunImage *string) error {
defaultFilterFunc := func(file string) bool { return true }
var extensions Extensions
l.logger.Debugf("extending run image %s", *currentRunImage)
time1 := time.Now()
extensions.SetExtensions(l.tmpDir, l.logger)
fmt.Println("extensions.SetExtensions took", time.Since(time1))
time2 := time.Now()
dockerfiles, err := extensions.DockerFiles(DockerfileKindRun, l.tmpDir, l.logger)
if err != nil {
return fmt.Errorf("getting %s.Dockerfiles: %w", DockerfileKindRun, err)
}
fmt.Println("extensions.DockerFiles took", time.Since(time2))
nestedCtx, cancel := context.WithCancel(ctx)
defer cancel()
nestedGroup, _ := errgroup.WithContext(nestedCtx)
var origTopLayerHash string = ""
var origTopLayerHash = ""
time3 := time.Now()
nestedGroup.Go(func() error {
defer func() {
fmt.Println("topLayerHash took", time.Since(time3))
}()
origTopLayerHash, err = topLayerHash(currentRunImage)
if err != nil {
return fmt.Errorf("getting top layer hash of run image: %w", err)
}
return nil
})
dockerapplytime := time.Now()
for _, dockerfile := range dockerfiles {
if dockerfile.Extend {
buildContext := archive.ReadDirAsTar(filepath.Dir(dockerfile.Path), "/", 0, 0, -1, true, false, defaultFilterFunc)
Expand All @@ -772,6 +837,8 @@ func (l *LifecycleExecution) ExtendRunByDaemon(ctx context.Context, group *errgr
l.logger.Debugf("build response for the extend: %v", response)
}
}
l.logger.Debugf("docker apply time: %v", time.Since(dockerapplytime))
time4 := time.Now()
ref, err := name.ParseReference("run-image:latest")
if err != nil {
return fmt.Errorf("failed to parse reference: %v", err)
Expand All @@ -785,13 +852,16 @@ func (l *LifecycleExecution) ExtendRunByDaemon(ctx context.Context, group *errgr
return fmt.Errorf("getting image hash: %w", err)
}
dest := filepath.Join(l.tmpDir, "extended-new", "run", imageHash.String())
fmt.Println("exporting to OCI took", time.Since(time4))
waiterr := nestedGroup.Wait()
if waiterr != nil {
return err
}
if err = SaveLayers(group, image, origTopLayerHash, dest); err != nil {
var savetime *time.Duration
if savetime, err = SaveLayers(group, image, origTopLayerHash, dest); err != nil {
return fmt.Errorf("copying selective image to output directory: %w", err)
}
fmt.Println("Total Save execution time:", savetime)
return nil
}

Expand Down
6 changes: 6 additions & 0 deletions internal/build/phase_config_provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,12 @@ func WithRoot() PhaseConfigProviderOperation {
}
}

func WithoutPrivilege() PhaseConfigProviderOperation {
return func(provider *PhaseConfigProvider) {
provider.ctrConf.User = "1000:1001"
}
}

func WithContainerOperations(operations ...ContainerOperation) PhaseConfigProviderOperation {
return func(provider *PhaseConfigProvider) {
provider.containerOps = append(provider.containerOps, operations...)
Expand Down

0 comments on commit 8710e46

Please sign in to comment.