diff --git a/internal/build/helper.go b/internal/build/helper.go index 0c603d07f..9f8822eef 100644 --- a/internal/build/helper.go +++ b/internal/build/helper.go @@ -6,16 +6,18 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/BurntSushi/toml" "github.com/buildpacks/imgutil/layout/sparse" "github.com/buildpacks/lifecycle/buildpack" "github.com/buildpacks/lifecycle/cmd" - "github.com/buildpacks/pack/pkg/logging" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/daemon" "golang.org/x/sync/errgroup" + + "github.com/buildpacks/pack/pkg/logging" ) const ( @@ -94,22 +96,24 @@ func escapeID(id string) string { return strings.ReplaceAll(id, "/", "_") } -func SaveLayers(group *errgroup.Group, image v1.Image, origTopLayerHash string, dest string) error { +func SaveLayers(group *errgroup.Group, image v1.Image, origTopLayerHash string, dest string) (*time.Duration, error) { + var totalSaveExecutionTime time.Duration + savetime := time.Now() layoutPath, err := sparse.NewImage(dest, image) if err != nil { fmt.Println("sparse.NewImage err", err) - return err + return nil, err } if err = layoutPath.Save(); err != nil { - return err + return nil, err } if err != nil { fmt.Println("sparse.NewImage err", err) - return err + return nil, err } layers, err := image.Layers() if err != nil { - return fmt.Errorf("getting image layers: %w", err) + return nil, fmt.Errorf("getting image layers: %w", err) } var ( currentHash v1.Hash @@ -121,12 +125,17 @@ func SaveLayers(group *errgroup.Group, image v1.Image, origTopLayerHash string, for _, currentLayer := range layers { currentHash, err = currentLayer.Digest() if err != nil { - return fmt.Errorf("getting layer hash: %w", err) + return nil, fmt.Errorf("getting layer hash: %w", err) } switch { case needsCopying: currentLayer := currentLayer + start := time.Now() group.Go(func() error { + defer func() { + duration := time.Since(start) + totalSaveExecutionTime += duration + }() return copyLayer(currentLayer, dest) }) case currentHash.String() == origTopLayerHash: @@ -136,7 +145,8 @@ func SaveLayers(group *errgroup.Group, image v1.Image, origTopLayerHash string, continue } } - return nil + totalSaveExecutionTime += time.Since(savetime) + return &totalSaveExecutionTime, nil } func copyLayer(layer v1.Layer, toSparseImage string) error { diff --git a/internal/build/lifecycle_execution.go b/internal/build/lifecycle_execution.go index f87422332..d09f9b67c 100644 --- a/internal/build/lifecycle_execution.go +++ b/internal/build/lifecycle_execution.go @@ -234,10 +234,20 @@ func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseF group, _ := errgroup.WithContext(context.TODO()) if l.platformAPI.AtLeast("0.10") && l.hasExtensionsForBuild() { - group.Go(func() error { - l.logger.Info(style.Step("EXTENDING (BUILD)")) - return l.ExtendBuild(ctx, buildCache, phaseFactory) - }) + if l.opts.Publish { + group.Go(func() error { + l.logger.Info(style.Step("EXTENDING (BUILD)")) + return l.ExtendBuild(ctx, buildCache, phaseFactory) + }) + } else { + group.Go(func() error { + l.logger.Info(style.Step("EXTENDING (BUILD) BY DAEMON")) + if err := l.ExtendBuildByDaemon(ctx, group); err != nil { + return err + } + return l.Build(ctx, phaseFactory) + }) + } } else { group.Go(func() error { l.logger.Info(style.Step("BUILDING")) @@ -262,6 +272,10 @@ func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseF start = time.Now() group.Go(func() error { l.logger.Info(style.Step("EXTENDING (RUN) BY DAEMON")) + defer func() { + duration := time.Since(start) + fmt.Println("Execution time:", duration) + }() return l.ExtendRunByDaemon(ctx, group, ¤tRunImage) }) } @@ -269,8 +283,6 @@ func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseF if err := group.Wait(); err != nil { return err } - duration := time.Since(start) - fmt.Println("Execution time:", duration) l.logger.Info(style.Step("EXPORTING")) return l.Export(ctx, buildCache, launchCache, phaseFactory) @@ -657,6 +669,8 @@ func (l *LifecycleExecution) Build(ctx context.Context, phaseFactory PhaseFactor WithNetwork(l.opts.Network), WithBinds(l.opts.Volumes...), WithFlags(flags...), + If((!l.opts.Publish && l.hasExtensionsForBuild()), WithImage("newbuilder-image:latest")), + If((!l.opts.Publish && l.hasExtensionsForBuild()), WithoutPrivilege()), ) build := phaseFactory.New(configProvider) @@ -725,27 +739,78 @@ func (l *LifecycleExecution) ExtendRun(ctx context.Context, buildCache Cache, ph defer extend.Cleanup() return extend.Run(ctx) } +func (l *LifecycleExecution) ExtendBuildByDaemon(ctx context.Context, group *errgroup.Group) error { + builderImageName := l.opts.BuilderImage + defaultFilterFunc := func(file string) bool { return true } + var extensions Extensions + time1 := time.Now() + extensions.SetExtensions(l.tmpDir, l.logger) + fmt.Println("extensions.SetExtensions took", time.Since(time1)) + time2 := time.Now() + dockerfiles, err := extensions.DockerFiles(DockerfileKindBuild, l.tmpDir, l.logger) + if err != nil { + return fmt.Errorf("getting %s.Dockerfiles: %w", DockerfileKindBuild, err) + } + fmt.Println("extensions.DockerFiles took", time.Since(time2)) + dockerapplytime := time.Now() + for _, dockerfile := range dockerfiles { + buildContext := archive.ReadDirAsTar(filepath.Dir(dockerfile.Path), "/", 0, 0, -1, true, false, defaultFilterFunc) + buildArguments := map[string]*string{} + if dockerfile.WithBase == "" { + buildArguments["base_image"] = &builderImageName + } + buildOptions := types.ImageBuildOptions{ + Context: buildContext, + Dockerfile: "Dockerfile", + Tags: []string{"newbuilder-image"}, + Remove: true, + BuildArgs: buildArguments, + } + response, err := l.docker.ImageBuild(ctx, buildContext, buildOptions) + if err != nil { + return err + } + defer response.Body.Close() + _, err = io.Copy(os.Stdout, response.Body) + if err != nil { + return err + } + l.logger.Debugf("build response for the extend: %v", response) + } + l.logger.Debugf("docker apply time: %v", time.Since(dockerapplytime)) + + return nil +} func (l *LifecycleExecution) ExtendRunByDaemon(ctx context.Context, group *errgroup.Group, currentRunImage *string) error { defaultFilterFunc := func(file string) bool { return true } var extensions Extensions l.logger.Debugf("extending run image %s", *currentRunImage) + time1 := time.Now() extensions.SetExtensions(l.tmpDir, l.logger) + fmt.Println("extensions.SetExtensions took", time.Since(time1)) + time2 := time.Now() dockerfiles, err := extensions.DockerFiles(DockerfileKindRun, l.tmpDir, l.logger) if err != nil { return fmt.Errorf("getting %s.Dockerfiles: %w", DockerfileKindRun, err) } + fmt.Println("extensions.DockerFiles took", time.Since(time2)) nestedCtx, cancel := context.WithCancel(ctx) defer cancel() nestedGroup, _ := errgroup.WithContext(nestedCtx) - var origTopLayerHash string = "" + var origTopLayerHash = "" + time3 := time.Now() nestedGroup.Go(func() error { + defer func() { + fmt.Println("topLayerHash took", time.Since(time3)) + }() origTopLayerHash, err = topLayerHash(currentRunImage) if err != nil { return fmt.Errorf("getting top layer hash of run image: %w", err) } return nil }) + dockerapplytime := time.Now() for _, dockerfile := range dockerfiles { if dockerfile.Extend { buildContext := archive.ReadDirAsTar(filepath.Dir(dockerfile.Path), "/", 0, 0, -1, true, false, defaultFilterFunc) @@ -772,6 +837,8 @@ func (l *LifecycleExecution) ExtendRunByDaemon(ctx context.Context, group *errgr l.logger.Debugf("build response for the extend: %v", response) } } + l.logger.Debugf("docker apply time: %v", time.Since(dockerapplytime)) + time4 := time.Now() ref, err := name.ParseReference("run-image:latest") if err != nil { return fmt.Errorf("failed to parse reference: %v", err) @@ -785,13 +852,16 @@ func (l *LifecycleExecution) ExtendRunByDaemon(ctx context.Context, group *errgr return fmt.Errorf("getting image hash: %w", err) } dest := filepath.Join(l.tmpDir, "extended-new", "run", imageHash.String()) + fmt.Println("exporting to OCI took", time.Since(time4)) waiterr := nestedGroup.Wait() if waiterr != nil { return err } - if err = SaveLayers(group, image, origTopLayerHash, dest); err != nil { + var savetime *time.Duration + if savetime, err = SaveLayers(group, image, origTopLayerHash, dest); err != nil { return fmt.Errorf("copying selective image to output directory: %w", err) } + fmt.Println("Total Save execution time:", savetime) return nil } diff --git a/internal/build/phase_config_provider.go b/internal/build/phase_config_provider.go index 93861633a..a6566858a 100644 --- a/internal/build/phase_config_provider.go +++ b/internal/build/phase_config_provider.go @@ -244,6 +244,12 @@ func WithRoot() PhaseConfigProviderOperation { } } +func WithoutPrivilege() PhaseConfigProviderOperation { + return func(provider *PhaseConfigProvider) { + provider.ctrConf.User = "1000:1001" + } +} + func WithContainerOperations(operations ...ContainerOperation) PhaseConfigProviderOperation { return func(provider *PhaseConfigProvider) { provider.containerOps = append(provider.containerOps, operations...)