Skip to content

Commit

Permalink
Move restorer logic out of main and into phase package
Browse files Browse the repository at this point in the history 
Signed-off-by: Natalie Arellano <[email protected]>
  • Loading branch information
@natalieparellano
natalieparellano committed Apr 16, 2024
1 parent 82fdf23 commit 67ec91e
Show file tree
Hide file tree
Showing 11 changed files with 399 additions and 239 deletions.
18 changes: 9 additions & 9 deletions cmd/lifecycle/creator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,15 +124,15 @@ func (c *createCmd) Exec() error {
plan files.Plan
)
cmd.DefaultLogger.Phase("ANALYZING")
analyzerFactory := phase.NewConnectedFactory(
connectedFactory := phase.NewConnectedFactory(
c.PlatformAPI,
&cmd.BuildpackAPIVerifier{},
NewCacheHandler(c.keychain),
files.NewHandler(),
image.NewHandler(c.docker, c.keychain, c.LayoutDir, c.UseLayout, c.InsecureRegistries),
image.NewRegistryHandler(c.keychain, c.InsecureRegistries),
)
analyzer, err := analyzerFactory.NewAnalyzer(c.Inputs(), cmd.DefaultLogger)
analyzer, err := connectedFactory.NewAnalyzer(c.Inputs(), cmd.DefaultLogger)
if err != nil {
return unwrapErrorFailWithMessage(err, "initialize analyzer")
}
Expand All @@ -146,13 +146,13 @@ func (c *createCmd) Exec() error {

// Detect
cmd.DefaultLogger.Phase("DETECTING")
detectorFactory := phase.NewHermeticFactory(
hermeticFactory := phase.NewHermeticFactory(
c.PlatformAPI,
&cmd.BuildpackAPIVerifier{},
files.NewHandler(),
dirStore,
)
detector, err := detectorFactory.NewDetector(c.Inputs(), cmd.DefaultLogger)
detector, err := hermeticFactory.NewDetector(c.Inputs(), cmd.DefaultLogger)
if err != nil {
return unwrapErrorFailWithMessage(err, "initialize detector")
}
Expand All @@ -171,12 +171,12 @@ func (c *createCmd) Exec() error {
// Restore
if !c.SkipLayers || c.PlatformAPI.AtLeast("0.10") {
cmd.DefaultLogger.Phase("RESTORING")
restoreCmd := &restoreCmd{
Platform: c.Platform,
keychain: c.keychain,
restorer, err := connectedFactory.NewRestorer(c.Inputs(), cmd.DefaultLogger, buildpack.Group{})
if err != nil {
return unwrapErrorFailWithMessage(err, "initialize restorer")
}
if err := restoreCmd.restore(analyzedMD.LayersMetadata, group, cacheStore); err != nil {
return err
if err = restorer.RestoreCache(); err != nil {
return cmd.FailErrCode(err, c.CodeFor(platform.RestoreError), "restore")
}
}

Expand Down
209 changes: 14 additions & 195 deletions cmd/lifecycle/restorer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,7 @@ package main
import (
"errors"
"fmt"
"os"
"path/filepath"

"github.com/buildpacks/imgutil"
"github.com/buildpacks/imgutil/layout"
"github.com/buildpacks/imgutil/layout/sparse"
"github.com/buildpacks/imgutil/remote"
"github.com/docker/docker/client"
"github.com/google/go-containerregistry/pkg/authn"

Expand All @@ -18,16 +12,12 @@ import (
"github.com/buildpacks/lifecycle/cmd"
"github.com/buildpacks/lifecycle/cmd/lifecycle/cli"
"github.com/buildpacks/lifecycle/image"
"github.com/buildpacks/lifecycle/internal/encoding"
"github.com/buildpacks/lifecycle/internal/layer"
"github.com/buildpacks/lifecycle/phase"
"github.com/buildpacks/lifecycle/platform"
"github.com/buildpacks/lifecycle/platform/files"
"github.com/buildpacks/lifecycle/priv"
)

const kanikoDir = "/kaniko"

type restoreCmd struct {
*platform.Platform

Expand Down Expand Up @@ -95,196 +85,25 @@ func (r *restoreCmd) Privileges() error {
}

func (r *restoreCmd) Exec() error {
group, err := files.Handler.ReadGroup(r.GroupPath)
if err != nil {
return err
}
if err = verifyBuildpackApis(group); err != nil {
return err
}

var analyzedMD files.Analyzed
if analyzedMD, err = files.Handler.ReadAnalyzed(r.AnalyzedPath, cmd.DefaultLogger); err == nil {
if r.supportsBuildImageExtension() && r.BuildImageRef != "" {
cmd.DefaultLogger.Debugf("Pulling builder image metadata for %s...", r.BuildImageRef)
remoteBuildImage, err := r.pullSparse(r.BuildImageRef)
if err != nil {
return cmd.FailErr(err, fmt.Sprintf("pull builder image %s", r.BuildImageRef))
}
digestRef, err := remoteBuildImage.Identifier()
if err != nil {
return cmd.FailErr(err, "get digest reference for builder image")
}
analyzedMD.BuildImage = &files.ImageIdentifier{Reference: digestRef.String()}
cmd.DefaultLogger.Debugf("Adding build image info to analyzed metadata: ")
cmd.DefaultLogger.Debugf(encoding.ToJSONMaybe(analyzedMD.BuildImage))
}
var (
runImage imgutil.Image
)
runImageName := analyzedMD.RunImageImage() // FIXME: if we have a digest reference available in `Reference` (e.g., in the non-daemon case) we should use it
if r.supportsRunImageExtension() && needsPulling(analyzedMD.RunImage) {
cmd.DefaultLogger.Debugf("Pulling run image metadata for %s...", runImageName)
runImage, err = r.pullSparse(runImageName)
if err != nil {
return cmd.FailErr(err, fmt.Sprintf("pull run image %s", runImageName))
}
// update analyzed metadata, even if we only needed to pull the image metadata, because
// the extender needs a digest reference in analyzed.toml,
// and daemon images will only have a daemon image ID
if err = r.updateAnalyzedMD(&analyzedMD, runImage); err != nil {
return cmd.FailErr(err, "update analyzed metadata")
}
} else if r.needsUpdating(analyzedMD.RunImage, group) {
cmd.DefaultLogger.Debugf("Updating run image info in analyzed metadata...")
h := image.NewHandler(r.docker, r.keychain, r.LayoutDir, r.UseLayout, r.InsecureRegistries)
runImage, err = h.InitImage(runImageName)
if err != nil || !runImage.Found() {
return cmd.FailErr(err, fmt.Sprintf("get run image %s", runImageName))
}
if err = r.updateAnalyzedMD(&analyzedMD, runImage); err != nil {
return cmd.FailErr(err, "update analyzed metadata")
}
}
if err = files.Handler.WriteAnalyzed(r.AnalyzedPath, &analyzedMD, cmd.DefaultLogger); err != nil {
return cmd.FailErr(err, "write analyzed metadata")
}
} else {
cmd.DefaultLogger.Warnf("Not using analyzed data, usable file not found: %s", err)
}

cacheStore, err := initCache(r.CacheImageRef, r.CacheDir, r.keychain, r.PlatformAPI.LessThan("0.13"))
if err != nil {
return err
}
return r.restore(analyzedMD.LayersMetadata, group, cacheStore)
}

func (r *restoreCmd) updateAnalyzedMD(analyzedMD *files.Analyzed, runImage imgutil.Image) error {
if r.PlatformAPI.LessThan("0.10") {
return nil
}
digestRef, err := runImage.Identifier()
if err != nil {
return cmd.FailErr(err, "get digest reference for run image")
}
var targetData *files.TargetMetadata
if r.PlatformAPI.AtLeast("0.12") {
targetData, err = platform.GetTargetMetadata(runImage)
if err != nil {
return cmd.FailErr(err, "read target data from run image")
}
}
cmd.DefaultLogger.Debugf("Run image info in analyzed metadata was: ")
cmd.DefaultLogger.Debugf(encoding.ToJSONMaybe(analyzedMD.RunImage))
analyzedMD.RunImage.Reference = digestRef.String()
analyzedMD.RunImage.TargetMetadata = targetData
cmd.DefaultLogger.Debugf("Run image info in analyzed metadata is: ")
cmd.DefaultLogger.Debugf(encoding.ToJSONMaybe(analyzedMD.RunImage))
return nil
}

func needsPulling(runImage *files.RunImage) bool {
if runImage == nil {
// sanity check to prevent panic, should be unreachable
return false
}
return runImage.Extend
}

func (r *restoreCmd) needsUpdating(runImage *files.RunImage, group buildpack.Group) bool {
if r.PlatformAPI.LessThan("0.10") {
return false
}
if !group.HasExtensions() {
return false
}
if runImage == nil {
// sanity check to prevent panic, should be unreachable
return false
}
if isPopulated(runImage.TargetMetadata) {
return false
}
return true
}

func isPopulated(metadata *files.TargetMetadata) bool {
return metadata != nil && metadata.OS != ""
}

func (r *restoreCmd) supportsBuildImageExtension() bool {
return r.PlatformAPI.AtLeast("0.10")
}

func (r *restoreCmd) supportsRunImageExtension() bool {
return r.PlatformAPI.AtLeast("0.12") && !r.UseLayout // FIXME: add layout support as part of https://github.com/buildpacks/lifecycle/issues/1102
}

func (r *restoreCmd) supportsTargetData() bool {
return r.PlatformAPI.AtLeast("0.12")
}

func (r *restoreCmd) pullSparse(imageRef string) (imgutil.Image, error) {
baseCacheDir := filepath.Join(kanikoDir, "cache", "base")
if err := os.MkdirAll(baseCacheDir, 0755); err != nil {
return nil, fmt.Errorf("failed to create cache directory: %w", err)
}

var opts []imgutil.ImageOption
opts = append(opts, append(image.GetInsecureOptions(r.InsecureRegistries), remote.FromBaseImage(imageRef))...)

// get remote image
remoteImage, err := remote.NewImage(imageRef, r.keychain, opts...)
if err != nil {
return nil, fmt.Errorf("failed to initialize remote image: %w", err)
}
if !remoteImage.Found() {
return nil, fmt.Errorf("failed to get remote image")
}
// check for usable kaniko dir
if _, err := os.Stat(kanikoDir); err != nil {
if !os.IsNotExist(err) {
return nil, fmt.Errorf("failed to read kaniko directory: %w", err)
}
return nil, nil
}
// save to disk
h, err := remoteImage.UnderlyingImage().Digest()
if err != nil {
return nil, fmt.Errorf("failed to get remote image digest: %w", err)
}
path := filepath.Join(baseCacheDir, h.String())
cmd.DefaultLogger.Debugf("Saving image metadata to %s...", path)
sparseImage, err := sparse.NewImage(
path,
remoteImage.UnderlyingImage(),
layout.WithMediaTypes(imgutil.DefaultTypes),
factory := phase.NewConnectedFactory(
r.PlatformAPI,
&cmd.BuildpackAPIVerifier{},
NewCacheHandler(r.keychain),
files.Handler,
image.NewHandler(r.docker, r.keychain, r.LayoutDir, r.UseLayout, r.InsecureRegistries),
image.NewRegistryHandler(r.keychain, r.InsecureRegistries),
)
restorer, err := factory.NewRestorer(r.Inputs(), cmd.DefaultLogger, buildpack.Group{})
if err != nil {
return nil, fmt.Errorf("failed to initialize sparse image: %w", err)
return unwrapErrorFailWithMessage(err, "initialize restorer")
}
if err = sparseImage.Save(); err != nil {
return nil, fmt.Errorf("failed to save sparse image: %w", err)
if err = restorer.RestoreAnalyzed(); err != nil {
return cmd.FailErrCode(err, r.CodeFor(platform.RestoreError), "restore")
}
return remoteImage, nil
}

func (r *restoreCmd) restore(layerMetadata files.LayersMetadata, group buildpack.Group, cacheStore phase.Cache) error {
restorer := &phase.Restorer{
LayersDir: r.LayersDir,
Buildpacks: group.Group,
Logger: cmd.DefaultLogger,
PlatformAPI: r.PlatformAPI,
LayerMetadataRestorer: layer.NewDefaultMetadataRestorer(r.LayersDir, r.SkipLayers, cmd.DefaultLogger),
LayersMetadata: layerMetadata,
SBOMRestorer: layer.NewSBOMRestorer(layer.SBOMRestorerOpts{
LayersDir: r.LayersDir,
Logger: cmd.DefaultLogger,
Nop: r.SkipLayers,
}, r.PlatformAPI),
if err = files.Handler.WriteAnalyzed(r.AnalyzedPath, &restorer.AnalyzedMD, cmd.DefaultLogger); err != nil {
return cmd.FailErr(err, "write analyzed metadata")
}
if err := restorer.Restore(cacheStore); err != nil {
if err = restorer.RestoreCache(); err != nil {
return cmd.FailErrCode(err, r.CodeFor(platform.RestoreError), "restore")
}
return nil
Expand Down
17 changes: 14 additions & 3 deletions image/handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
//go:generate mockgen -package testmock -destination ../phase/testmock/image_handler.go github.com/buildpacks/lifecycle/image Handler
type Handler interface {
InitImage(imageRef string) (imgutil.Image, error)
InitRemoteImage(imageRef string) (imgutil.Image, error)
Kind() string
}

Expand All @@ -22,15 +23,25 @@ type Handler interface {
// - WHEN a docker client is provided then it returns a LocalHandler
// - WHEN an auth.Keychain is provided then it returns a RemoteHandler
// - Otherwise nil is returned
func NewHandler(docker client.CommonAPIClient, keychain authn.Keychain, layoutDir string, useLayout bool, insecureRegistries []string) Handler {
func NewHandler(
docker client.CommonAPIClient,
keychain authn.Keychain,
layoutDir string,
useLayout bool,
insecureRegistries []string,
) Handler {
if layoutDir != "" && useLayout {
return &LayoutHandler{
layoutDir: layoutDir,
layoutDir: layoutDir,
keychain: keychain,
insecureRegistries: insecureRegistries,
}
}
if docker != nil {
return &LocalHandler{
docker: docker,
docker: docker,
keychain: keychain,
insecureRegistries: insecureRegistries,
}
}
if keychain != nil {
Expand Down
25 changes: 24 additions & 1 deletion image/layout_handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,17 @@ import (

"github.com/buildpacks/imgutil"
"github.com/buildpacks/imgutil/layout"
"github.com/buildpacks/imgutil/remote"
"github.com/google/go-containerregistry/pkg/authn"
v1 "github.com/google/go-containerregistry/pkg/v1"
)

const LayoutKind = "layout"

type LayoutHandler struct {
layoutDir string
layoutDir string
keychain authn.Keychain
insecureRegistries []string
}

func (h *LayoutHandler) InitImage(imageRef string) (imgutil.Image, error) {
Expand All @@ -29,6 +33,25 @@ func (h *LayoutHandler) InitImage(imageRef string) (imgutil.Image, error) {
return layout.NewImage(path, layout.FromBaseImagePath(path))
}

// InitRemoteImage TODO
func (h *LayoutHandler) InitRemoteImage(imageRef string) (imgutil.Image, error) {
if imageRef == "" {
return nil, nil
}

options := []imgutil.ImageOption{
remote.FromBaseImage(imageRef),
}

options = append(options, GetInsecureOptions(h.insecureRegistries)...)

return remote.NewImage(
imageRef,
h.keychain,
options...,
)
}

func (h *LayoutHandler) Kind() string {
return LayoutKind
}
Expand Down
Loading

0 comments on commit 67ec91e

Please sign in to comment.