diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 63badc8f..56a3fcd9 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -13,9 +13,17 @@ import ( "github.com/canonical/chisel/internal/deb" ) +type PackageInfo interface { + Name() string + Version() string + Arch() string + SHA256() string +} + type Archive interface { Options() *Options Fetch(pkg string) (io.ReadCloser, error) + Info(pkg string) PackageInfo Exists(pkg string) bool } @@ -86,6 +94,22 @@ func (a *ubuntuArchive) Exists(pkg string) bool { return err == nil } +type pkgInfo struct{ control.Section } + +var _ PackageInfo = pkgInfo{} + +func (info pkgInfo) Name() string { return info.Get("Package") } +func (info pkgInfo) Version() string { return info.Get("Version") } +func (info pkgInfo) Arch() string { return info.Get("Architecture") } +func (info pkgInfo) SHA256() string { return info.Get("SHA256") } + +func (a *ubuntuArchive) Info(pkg string) PackageInfo { + if section, _, _ := a.selectPackage(pkg); section != nil { + return &pkgInfo{section} + } + return nil +} + func (a *ubuntuArchive) selectPackage(pkg string) (control.Section, *ubuntuIndex, error) { var selectedVersion string var selectedSection control.Section diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index cb9f3278..b18fc60c 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -332,6 +332,39 @@ func (s *httpSuite) TestArchiveLabels(c *C) { c.Assert(err, ErrorMatches, `.*\bno Ubuntu section`) } +func (s *httpSuite) TestPackageInfo(c *C) { + s.prepareArchive("jammy", "22.04", "amd64", []string{"main", "universe"}) + + options := archive.Options{ + Label: "ubuntu", + Version: "22.04", + Arch: "amd64", + Suites: []string{"jammy"}, + Components: []string{"main", "universe"}, + CacheDir: c.MkDir(), + } + + archive, err := archive.Open(&options) + c.Assert(err, IsNil) + + info1 := archive.Info("mypkg1") + c.Assert(info1, NotNil) + c.Assert(info1.Name(), Equals, "mypkg1") + c.Assert(info1.Version(), Equals, "1.1") + c.Assert(info1.Arch(), Equals, "amd64") + c.Assert(info1.SHA256(), Equals, "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05") + + info3 := archive.Info("mypkg3") + c.Assert(info3, NotNil) + c.Assert(info3.Name(), Equals, "mypkg3") + c.Assert(info3.Version(), Equals, "1.3") + c.Assert(info3.Arch(), Equals, "amd64") + c.Assert(info3.SHA256(), Equals, "fe377bf13ba1a5cb287cb4e037e6e7321281c929405ae39a72358ef0f5d179aa") + + info99 := archive.Info("mypkg99") + c.Assert(info99, IsNil) +} + func read(r io.Reader) string { data, err := io.ReadAll(r) if err != nil { diff --git a/internal/deb/extract.go b/internal/deb/extract.go index 5de2f403..27f644f5 100644 --- a/internal/deb/extract.go +++ b/internal/deb/extract.go @@ -6,6 +6,7 @@ import ( "compress/gzip" "fmt" "io" + "io/fs" "os" "path/filepath" "sort" @@ -20,11 +21,56 @@ import ( "github.com/canonical/chisel/internal/strdist" ) +type ConsumeData func(reader io.Reader) error + +// DataCallback function is called when a tarball entry of a regular file is +// going to be extracted. +// +// The source and size parameters are set to the file's path in the tarball and +// the size of its content, respectively. When the returned ConsumeData function +// is not nil, it is called with the file's content. +// +// When the source path occurs in the tarball more than once, this function will +// be called for each occurrence. +// +// If either DataCallback or ConsumeData function returns a non-nil error, the +// Extract() function will fail with that error. +type DataCallback func(source string, size int64) (ConsumeData, error) + +// The CreateCallback function is called when a tarball entry is going to be +// extracted to a target path. +// +// The target, link, and mode parameters are set to the target's path, symbolic +// link target, and mode, respectively. The target's filesystem type can be +// determined from these parameters. If the link is not empty, the target is a +// symbolic link. Otherwise, if the target's path ends with /, it is a +// directory. Otherwise, it is a regular file. +// +// When the source parameter is not empty, the target is going to be extracted +// from a tarball entry with the source path. The function may be called more +// than once with the same source when the tarball entry is extracted to +// multiple different targets. +// +// Otherwise, the mode is 0755 and the target is going to be an implicitly +// created parent directory of another target, and when a directory entry with +// that path is later encountered in the tarball with a different mode, the +// function will be called again with the same target, source equal to the +// target, and the different mode. +// +// When the source path occurs in the tarball more than once, this function will +// be called for each target path for each occurrence. +// +// If CreateCallback function returns a non-nil error, the Extract() function +// will fail with that error. +type CreateCallback func(source, target, link string, mode fs.FileMode) error + type ExtractOptions struct { Package string TargetDir string Extract map[string][]ExtractInfo Globbed map[string][]string + OnData DataCallback + OnCreate CreateCallback } type ExtractInfo struct { @@ -109,34 +155,8 @@ func extractData(dataReader io.Reader, options *ExtractOptions) error { syscall.Umask(oldUmask) }() - shouldExtract := func(pkgPath string) (globPath string, ok bool) { - if pkgPath == "" { - return "", false - } - pkgPathIsDir := pkgPath[len(pkgPath)-1] == '/' - for extractPath, extractInfos := range options.Extract { - if extractPath == "" { - continue - } - switch { - case strings.ContainsAny(extractPath, "*?"): - if strdist.GlobPath(extractPath, pkgPath) { - return extractPath, true - } - case extractPath == pkgPath: - return "", true - case pkgPathIsDir: - for _, extractInfo := range extractInfos { - if strings.HasPrefix(extractInfo.Path, pkgPath) { - return "", true - } - } - } - } - return "", false - } - pendingPaths := make(map[string]bool) + var globs []string for extractPath, extractInfos := range options.Extract { for _, extractInfo := range extractInfos { if !extractInfo.Optional { @@ -144,7 +164,25 @@ func extractData(dataReader io.Reader, options *ExtractOptions) error { break } } + if strings.ContainsAny(extractPath, "*?") { + globs = append(globs, extractPath) + } + } + + // dirInfo represents a directory that we + // a) encountered in the tarball, + // b) created, or + // c) both a) and c). + type dirInfo struct { + // mode of the directory with which we + // a) encountered it in the tarball, or + // b) created it + mode fs.FileMode + // whether we created this directory + created bool } + // directories we encountered and/or created + dirInfos := make(map[string]dirInfo) tarReader := tar.NewReader(dataReader) for { @@ -161,81 +199,151 @@ func extractData(dataReader io.Reader, options *ExtractOptions) error { continue } sourcePath = sourcePath[1:] - globPath, ok := shouldExtract(sourcePath) - if !ok { - continue + sourceMode := tarHeader.FileInfo().Mode() + + globPath := "" + extractInfos := options.Extract[sourcePath] + + if extractInfos == nil { + for _, glob := range globs { + if strdist.GlobPath(glob, sourcePath) { + globPath = glob + extractInfos = []ExtractInfo{{Path: glob}} + break + } + } } - sourceIsDir := sourcePath[len(sourcePath)-1] == '/' + // Is this a directory path that was not requested? + if extractInfos == nil && sourceMode.IsDir() { + if info := dirInfos[sourcePath]; info.mode != sourceMode { + // We have not seen this directory yet, or we + // have seen or created it with a different mode + // before. Record the source path mode. + info.mode = sourceMode + dirInfos[sourcePath] = info + if info.created { + // We have created this directory before + // with a different mode. Create it + // again with the proper mode. + extractInfos = []ExtractInfo{{Path: sourcePath}} + } + } + } - //debugf("Extracting header: %#v", tarHeader) + if extractInfos == nil { + continue + } - var extractInfos []ExtractInfo if globPath != "" { - extractInfos = options.Extract[globPath] - delete(pendingPaths, globPath) if options.Globbed != nil { options.Globbed[globPath] = append(options.Globbed[globPath], sourcePath) } + delete(pendingPaths, globPath) } else { - extractInfos, ok = options.Extract[sourcePath] - if ok { - delete(pendingPaths, sourcePath) - } else { - // Base directory for extracted content. Relevant mainly to preserve - // the metadata, since the extracted content itself will also create - // any missing directories unaccounted for in the options. - err := fsutil.Create(&fsutil.CreateOptions{ - Path: filepath.Join(options.TargetDir, sourcePath), - Mode: tarHeader.FileInfo().Mode(), - MakeParents: true, - }) - if err != nil { + delete(pendingPaths, sourcePath) + } + + // createParents creates missing parent directories of the path + // with modes with which they were encountered in the tarball or + // 0755 if they were not encountered yet. + var createParents func(path string) error + createParents = func(path string) error { + dir := fsutil.SlashedPathDir(path) + if dir == "/" { + return nil + } + info := dirInfos[dir] + source := dir + if info.created { + return nil + } else if info.mode == 0 { + info.mode = fs.ModeDir | 0755 + source = "" + } + if err := createParents(dir); err != nil { + return err + } + if options.OnCreate != nil { + if err := options.OnCreate(source, dir, "", info.mode); err != nil { return err } - continue } + create := fsutil.CreateOptions{ + Path: filepath.Join(options.TargetDir, dir), + Mode: info.mode, + } + if err := fsutil.Create(&create); err != nil { + return err + } + info.created = true + dirInfos[dir] = info + return nil } - var contentCache []byte - var contentIsCached = len(extractInfos) > 1 && !sourceIsDir && globPath == "" - if contentIsCached { - // Read and cache the content so it may be reused. - // As an alternative, to avoid having an entire file in - // memory at once this logic might open the first file - // written and copy it every time. For now, the choice - // is speed over memory efficiency. - data, err := io.ReadAll(tarReader) - if err != nil { - return err + getReader := func() io.Reader { return tarReader } + + if sourceMode.IsRegular() { + var consumeData ConsumeData + if options.OnData != nil { + var err error + if consumeData, err = options.OnData(sourcePath, tarHeader.Size); err != nil { + return err + } + } + if consumeData != nil || (len(extractInfos) > 1 && globPath == "") { + // Read and cache the content so it may be reused. + // As an alternative, to avoid having an entire file in + // memory at once this logic might open the first file + // written and copy it every time. For now, the choice + // is speed over memory efficiency. + data, err := io.ReadAll(tarReader) + if err != nil { + return err + } + getReader = func() io.Reader { return bytes.NewReader(data) } + } + if consumeData != nil { + if err := consumeData(getReader()); err != nil { + return err + } } - contentCache = data } - var pathReader io.Reader = tarReader + origMode := tarHeader.Mode for _, extractInfo := range extractInfos { - if contentIsCached { - pathReader = bytes.NewReader(contentCache) - } var targetPath string if globPath == "" { - targetPath = filepath.Join(options.TargetDir, extractInfo.Path) + targetPath = extractInfo.Path } else { - targetPath = filepath.Join(options.TargetDir, sourcePath) + targetPath = sourcePath + } + if err := createParents(targetPath); err != nil { + return err } + tarHeader.Mode = origMode if extractInfo.Mode != 0 { tarHeader.Mode = int64(extractInfo.Mode) } + fsMode := tarHeader.FileInfo().Mode() + if options.OnCreate != nil { + if err := options.OnCreate(sourcePath, targetPath, tarHeader.Linkname, fsMode); err != nil { + return err + } + } err := fsutil.Create(&fsutil.CreateOptions{ - Path: targetPath, - Mode: tarHeader.FileInfo().Mode(), - Data: pathReader, - Link: tarHeader.Linkname, - MakeParents: true, + Path: filepath.Join(options.TargetDir, targetPath), + Mode: fsMode, + Data: getReader(), + Link: tarHeader.Linkname, }) if err != nil { return err } + if fsMode.IsDir() { + // Record the target directory mode. + dirInfos[targetPath] = dirInfo{fsMode, true} + } if globPath != "" { break } diff --git a/internal/deb/extract_test.go b/internal/deb/extract_test.go index 3147dc36..f18ad24f 100644 --- a/internal/deb/extract_test.go +++ b/internal/deb/extract_test.go @@ -2,6 +2,8 @@ package deb_test import ( "bytes" + "io" + "io/fs" . "gopkg.in/check.v1" @@ -9,6 +11,12 @@ import ( "github.com/canonical/chisel/internal/testutil" ) +var ( + Reg = testutil.Reg + Dir = testutil.Dir + Lnk = testutil.Lnk +) + type extractTest struct { summary string pkgdata []byte @@ -260,10 +268,7 @@ var extractTests = []extractTest{{ }, }, result: map[string]string{ - "/etc/": "dir 0755", - "/usr/": "dir 0755", - "/usr/bin/": "dir 0755", - "/tmp/": "dir 01777", + "/etc/": "dir 0755", }, }, { summary: "Optional entries mixed in cannot be missing", @@ -280,6 +285,162 @@ var extractTests = []extractTest{{ }, }, error: `cannot extract from package "base-files": no content at /usr/bin/hallo`, +}, { + summary: "Implicit parent directories", + pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + Dir(0701, "./a/"), + Dir(0702, "./a/b/"), + Reg(0601, "./a/b/c", ""), + }), + options: deb.ExtractOptions{ + Extract: map[string][]deb.ExtractInfo{ + "/a/b/c": []deb.ExtractInfo{{Path: "/a/b/c"}}, + }, + }, + result: map[string]string{ + "/a/": "dir 0701", + "/a/b/": "dir 0702", + "/a/b/c": "file 0601 empty", + }, +}, { + summary: "Implicit parent directories with different target path", + pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + Dir(0701, "./a/"), + Dir(0702, "./b/"), + Reg(0601, "./b/x", "shark"), + Dir(0703, "./c/"), + Reg(0602, "./c/y", "octopus"), + Dir(0704, "./d/"), + }), + options: deb.ExtractOptions{ + Extract: map[string][]deb.ExtractInfo{ + "/b/x": []deb.ExtractInfo{{Path: "/a/x"}}, + "/c/y": []deb.ExtractInfo{{Path: "/d/y"}}, + }, + }, + result: map[string]string{ + "/a/": "dir 0701", + "/a/x": "file 0601 31fc1594", + "/d/": "dir 0704", + "/d/y": "file 0602 5633c9b8", + }, +}, { + summary: "Implicit parent directories with a glob", + pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + Dir(0701, "./a/"), + Dir(0702, "./a/aa/"), + Dir(0703, "./a/aa/aaa/"), + Reg(0601, "./a/aa/aaa/ffff", ""), + }), + options: deb.ExtractOptions{ + Extract: map[string][]deb.ExtractInfo{ + "/a/aa/a**": []deb.ExtractInfo{{ + Path: "/a/aa/a**", + }}, + }, + }, + result: map[string]string{ + "/a/": "dir 0701", + "/a/aa/": "dir 0702", + "/a/aa/aaa/": "dir 0703", + "/a/aa/aaa/ffff": "file 0601 empty", + }, +}, { + summary: "Implicit parent directories with a glob and non-sorted tarball", + pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + Reg(0601, "./a/b/c/d", ""), + Dir(0702, "./a/b/"), + Dir(0703, "./a/b/c/"), + Dir(0701, "./a/"), + }), + options: deb.ExtractOptions{ + Extract: map[string][]deb.ExtractInfo{ + "/a/b/c/*": []deb.ExtractInfo{{ + Path: "/a/b/c/*", + }}, + }, + }, + result: map[string]string{ + "/a/": "dir 0701", + "/a/b/": "dir 0702", + "/a/b/c/": "dir 0703", + "/a/b/c/d": "file 0601 empty", + }, +}, { + summary: "Implicit parent directories with a glob and some parents missing in the tarball", + pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + Reg(0601, "./a/b/c/d", ""), + Dir(0702, "./a/b/"), + }), + options: deb.ExtractOptions{ + Extract: map[string][]deb.ExtractInfo{ + "/a/b/c/*": []deb.ExtractInfo{{ + Path: "/a/b/c/*", + }}, + }, + }, + result: map[string]string{ + "/a/": "dir 0755", + "/a/b/": "dir 0702", + "/a/b/c/": "dir 0755", + "/a/b/c/d": "file 0601 empty", + }, +}, { + summary: "Implicit parent directories with copied dirs and different modes", + pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + Dir(0701, "./a/"), + Dir(0702, "./a/b/"), + Dir(0703, "./a/b/c/"), + Reg(0601, "./a/b/c/d", ""), + Dir(0704, "./e/"), + Dir(0705, "./e/f/"), + }), + options: deb.ExtractOptions{ + Extract: map[string][]deb.ExtractInfo{ + "/a/b/**": []deb.ExtractInfo{{ + Path: "/a/b/**", + }}, + "/e/f/": []deb.ExtractInfo{{ + Path: "/a/", + }}, + "/e/": []deb.ExtractInfo{{ + Path: "/a/b/c/", + Mode: 0706, + }}, + }, + }, + result: map[string]string{ + "/a/": "dir 0705", + "/a/b/": "dir 0702", + "/a/b/c/": "dir 0706", + "/a/b/c/d": "file 0601 empty", + }, +}, { + summary: "Copies with different permissions", + pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + Dir(0701, "./a/"), + Reg(0601, "./b", ""), + }), + options: deb.ExtractOptions{ + Extract: map[string][]deb.ExtractInfo{ + "/a/": []deb.ExtractInfo{ + {Path: "/b/"}, + {Path: "/c/", Mode: 0702}, + {Path: "/d/", Mode: 01777}, + {Path: "/e/"}, + {Path: "/f/", Mode: 0723}, + {Path: "/g/"}, + }, + }, + }, + result: map[string]string{ + "/b/": "dir 0701", + "/c/": "dir 0702", + "/d/": "dir 01777", + "/e/": "dir 0701", + "/f/": "dir 0723", + "/g/": "dir 0701", + }, }} func (s *S) TestExtract(c *C) { @@ -311,3 +472,229 @@ func (s *S) TestExtract(c *C) { c.Assert(result, DeepEquals, test.result) } } + +type callbackTest struct { + summary string + pkgdata []byte + extract map[string][]deb.ExtractInfo + callbacks [][]any + noConsume bool + noData bool +} + +var callbackTests = []callbackTest{{ + summary: "Trivial", + pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + Dir(0701, "./a/"), + Dir(0702, "./a/b/"), + Reg(0601, "./a/b/c", ""), + }), + extract: map[string][]deb.ExtractInfo{ + "/**": []deb.ExtractInfo{{Path: "/**"}}, + }, + callbacks: [][]any{ + {"create", "/a/", "/a/", "", 0701}, + {"create", "/a/b/", "/a/b/", "", 0702}, + {"data", "/a/b/c", 0, []byte{}}, + {"create", "/a/b/c", "/a/b/c", "", 0601}, + }, +}, { + summary: "Data", + pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + Dir(0701, "./a/"), + Reg(0601, "./a/b", "foo"), + Reg(0602, "./a/c", "bar"), + }), + extract: map[string][]deb.ExtractInfo{ + "/**": []deb.ExtractInfo{{Path: "/**"}}, + }, + callbacks: [][]any{ + {"create", "/a/", "/a/", "", 0701}, + {"data", "/a/b", 3, []byte("foo")}, + {"create", "/a/b", "/a/b", "", 0601}, + {"data", "/a/c", 3, []byte("bar")}, + {"create", "/a/c", "/a/c", "", 0602}, + }, +}, { + summary: "Symlinks", + pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + Reg(0601, "./a", ""), + Lnk(0777, "./b", "/a"), + Lnk(0777, "./c", "/d"), + }), + extract: map[string][]deb.ExtractInfo{ + "/**": []deb.ExtractInfo{{Path: "/**"}}, + }, + noData: true, + callbacks: [][]any{ + {"create", "/a", "/a", "", 0601}, + {"create", "/b", "/b", "/a", 0777}, + {"create", "/c", "/c", "/d", 0777}, + }, +}, { + summary: "Simple copied paths", + pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + Reg(0601, "./a", ""), + Reg(0602, "./b", ""), + Dir(0701, "./c/"), + Reg(0603, "./c/d", ""), + }), + extract: map[string][]deb.ExtractInfo{ + "/a": []deb.ExtractInfo{{Path: "/a"}}, + "/c/d": []deb.ExtractInfo{{Path: "/b"}}, + }, + noData: true, + callbacks: [][]any{ + {"create", "/a", "/a", "", 0601}, + {"create", "/c/d", "/b", "", 0603}, + }, +}, { + summary: "Parent directories", + pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + Dir(0701, "./a/"), + Dir(0702, "./a/b/"), + Dir(0703, "./a/b/c/"), + Reg(0601, "./a/b/c/d", ""), + }), + extract: map[string][]deb.ExtractInfo{ + "/a/b/c/": []deb.ExtractInfo{{Path: "/a/b/c/"}}, + }, + callbacks: [][]any{ + {"create", "/a/", "/a/", "", 0701}, + {"create", "/a/b/", "/a/b/", "", 0702}, + {"create", "/a/b/c/", "/a/b/c/", "", 0703}, + }, +}, { + summary: "Parent directories with globs", + pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + Dir(0701, "./a/"), + Dir(0702, "./a/b/"), + Dir(0703, "./a/b/c/"), + Reg(0601, "./a/b/c/d", ""), + }), + extract: map[string][]deb.ExtractInfo{ + "/a/b/*/": []deb.ExtractInfo{{Path: "/a/b/*/"}}, + }, + callbacks: [][]any{ + {"create", "/a/", "/a/", "", 0701}, + {"create", "/a/b/", "/a/b/", "", 0702}, + {"create", "/a/b/c/", "/a/b/c/", "", 0703}, + }, +}, { + summary: "Parent directories out of order", + pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + Reg(0601, "./a/b/c/d", ""), + Dir(0703, "./a/b/c/"), + Dir(0702, "./a/b/"), + Dir(0701, "./a/"), + }), + extract: map[string][]deb.ExtractInfo{ + "/a/b/*/": []deb.ExtractInfo{{Path: "/a/b/*/"}}, + }, + callbacks: [][]any{ + {"create", "", "/a/", "", 0755}, + {"create", "", "/a/b/", "", 0755}, + {"create", "/a/b/c/", "/a/b/c/", "", 0703}, + {"create", "/a/b/", "/a/b/", "", 0702}, + {"create", "/a/", "/a/", "", 0701}, + }, +}, { + summary: "Parent directories with early copy path", + pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + Dir(0701, "./a/"), + Reg(0601, "./a/b", ""), + Dir(0702, "./c/"), + Reg(0602, "./c/d", ""), + }), + extract: map[string][]deb.ExtractInfo{ + "/a/b": []deb.ExtractInfo{{Path: "/c/d"}}, + }, + noData: true, + callbacks: [][]any{ + {"create", "", "/c/", "", 0755}, + {"create", "/a/b", "/c/d", "", 0601}, + {"create", "/c/", "/c/", "", 0702}, + }, +}, { + summary: "Same file twice with different content", + pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + Reg(0601, "./a", "foo"), + Reg(0602, "./b", "bar"), + Reg(0603, "./a", "baz"), + }), + extract: map[string][]deb.ExtractInfo{ + "/*": []deb.ExtractInfo{{Path: "/*"}}, + }, + callbacks: [][]any{ + {"data", "/a", 3, []byte("foo")}, + {"create", "/a", "/a", "", 0601}, + {"data", "/b", 3, []byte("bar")}, + {"create", "/b", "/b", "", 0602}, + {"data", "/a", 3, []byte("baz")}, + {"create", "/a", "/a", "", 0603}, + }, +}, { + summary: "Source with multiple targets", + pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + Reg(0601, "./a", "aaa"), + Reg(0602, "./b", "bu bu bu"), + }), + extract: map[string][]deb.ExtractInfo{ + "/a": []deb.ExtractInfo{{Path: "/b"}}, + "/b": []deb.ExtractInfo{ + {Path: "/c", Mode: 0603}, + {Path: "/d"}, + }, + }, + callbacks: [][]any{ + {"data", "/a", 3, []byte("aaa")}, + {"create", "/a", "/b", "", 0601}, + {"data", "/b", 8, []byte("bu bu bu")}, + {"create", "/b", "/c", "", 0603}, + {"create", "/b", "/d", "", 0602}, + }, +}} + +func (s *S) TestExtractCallbacks(c *C) { + for _, test := range callbackTests { + c.Logf("Test: %s", test.summary) + dir := c.MkDir() + var callbacks [][]any + onData := func(source string, size int64) (deb.ConsumeData, error) { + if test.noConsume { + args := []any{"data", source, int(size), nil} + callbacks = append(callbacks, args) + return nil, nil + } + consume := func(reader io.Reader) error { + data, err := io.ReadAll(reader) + if err != nil { + return err + } + args := []any{"data", source, int(size), data} + callbacks = append(callbacks, args) + return nil + } + return consume, nil + } + if test.noData { + onData = nil + } + onCreate := func(source, target, link string, mode fs.FileMode) error { + modeInt := int(07777 & mode) + args := []any{"create", source, target, link, modeInt} + callbacks = append(callbacks, args) + return nil + } + options := deb.ExtractOptions{ + Package: "test", + TargetDir: dir, + Extract: test.extract, + OnData: onData, + OnCreate: onCreate, + } + err := deb.Extract(bytes.NewBuffer(test.pkgdata), &options) + c.Assert(err, IsNil) + c.Assert(callbacks, DeepEquals, test.callbacks) + } +} diff --git a/internal/fsutil/path.go b/internal/fsutil/path.go new file mode 100644 index 00000000..34469b46 --- /dev/null +++ b/internal/fsutil/path.go @@ -0,0 +1,66 @@ +package fsutil + +import ( + "path/filepath" +) + +// isDirPath returns whether the path refers to a directory. +// The path refers to a directory when it ends with "/", "/." or "/..", or when +// it equals "." or "..". +func isDirPath(path string) bool { + i := len(path) - 1 + if i < 0 { + return true + } + if path[i] == '.' { + i-- + if i < 0 { + return true + } + if path[i] == '.' { + i-- + if i < 0 { + return true + } + } + } + if path[i] == '/' { + return true + } + return false +} + +// Debian package tarballs present paths slightly differently to what we would +// normally classify as clean paths. While a traditional clean file path is identical +// to a clean deb package file path, the deb package directory path always ends +// with a slash. Although the change only affects directory paths, the implication +// is that a directory path without a slash is interpreted as a file path. For this +// reason, we need to be very careful and handle both file and directory paths using +// a new set of functions. We call this new path type a Slashed Path. A slashed path +// allows us to identify a file or directory simply using lexical analysis. + +// SlashedPathClean takes a file or slashed directory path as input, and produces +// the shortest equivalent as output. An input path ending without a slash will be +// interpreted as a file path. Directory paths should always end with a slash. +// These functions exists because we work with slash terminated directory paths +// that come from deb package tarballs but standard library path functions +// treat slash terminated paths as unclean. +func SlashedPathClean(path string) string { + clean := filepath.Clean(path) + if clean != "/" && isDirPath(path) { + clean += "/" + } + return clean +} + +// SlashedPathDir takes a file or slashed directory path as input, cleans the +// path and returns the parent directory. An input path ending without a slash +// will be interpreted as a file path. Directory paths should always end with a slash. +// Clean is like filepath.Clean() but trailing slash is kept. +func SlashedPathDir(path string) string { + parent := filepath.Dir(filepath.Clean(path)) + if parent != "/" { + parent += "/" + } + return parent +} diff --git a/internal/fsutil/path_test.go b/internal/fsutil/path_test.go new file mode 100644 index 00000000..864ec594 --- /dev/null +++ b/internal/fsutil/path_test.go @@ -0,0 +1,54 @@ +package fsutil_test + +import ( + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/fsutil" +) + +var cleanAndDirTestCases = []struct { + inputPath string + resultClean string + resultDir string +}{ + {"/a/b/c", "/a/b/c", "/a/b/"}, + {"/a/b/c/", "/a/b/c/", "/a/b/"}, + {"/a/b/c//", "/a/b/c/", "/a/b/"}, + {"/a/b//c", "/a/b/c", "/a/b/"}, + {"/a/b/c/.", "/a/b/c/", "/a/b/"}, + {"/a/b/c/.///.", "/a/b/c/", "/a/b/"}, + {"/a/b/./c/", "/a/b/c/", "/a/b/"}, + {"/a/b/.///./c", "/a/b/c", "/a/b/"}, + {"/a/b/c/..", "/a/b/", "/a/"}, + {"/a/b/c/..///./", "/a/b/", "/a/"}, + {"/a/b/c/../.", "/a/b/", "/a/"}, + {"/a/b/../c/", "/a/c/", "/a/"}, + {"/a/b/..///./c", "/a/c", "/a/"}, + {"a/b/./c", "a/b/c", "a/b/"}, + {"./a/b/./c", "a/b/c", "a/b/"}, + {"/", "/", "/"}, + {"///", "/", "/"}, + {"///.///", "/", "/"}, + {"/././.", "/", "/"}, + {".", "./", "./"}, + {".///", "./", "./"}, + {"..", "../", "./"}, + {"..///.", "../", "./"}, + {"../../..", "../../../", "../../"}, + {"..///.///../..", "../../../", "../../"}, + {"", "./", "./"}, +} + +func (s *S) TestSlashedPathClean(c *C) { + for _, t := range cleanAndDirTestCases { + c.Logf("%s => %s", t.inputPath, t.resultClean) + c.Assert(fsutil.SlashedPathClean(t.inputPath), Equals, t.resultClean) + } +} + +func (s *S) TestSlashedPathDir(c *C) { + for _, t := range cleanAndDirTestCases { + c.Logf("%s => %s", t.inputPath, t.resultDir) + c.Assert(fsutil.SlashedPathDir(t.inputPath), Equals, t.resultDir) + } +} diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 2b684b5d..3152ca67 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -32,25 +32,21 @@ func Run(options *RunOptions) error { knownPaths["/"] = true + // addKnownPath path adds path and all its directory parent paths into + // knownPaths set. addKnownPath := func(path string) { if path[0] != '/' { panic("bug: tried to add relative path to known paths") } - cleanPath := filepath.Clean(path) - slashPath := cleanPath - if path[len(path)-1] == '/' && cleanPath != "/" { - slashPath += "/" - } + path = fsutil.SlashedPathClean(path) for { - if _, ok := knownPaths[slashPath]; ok { + if _, ok := knownPaths[path]; ok { break } - knownPaths[slashPath] = true - cleanPath = filepath.Dir(cleanPath) - if cleanPath == "/" { + knownPaths[path] = true + if path = fsutil.SlashedPathDir(path); path == "/" { break } - slashPath = cleanPath + "/" } } @@ -113,14 +109,13 @@ func Run(options *RunOptions) error { hasCopyright = true } } else { - targetDir := filepath.Dir(strings.TrimRight(targetPath, "/")) + "/" - if targetDir == "" || targetDir == "/" { - continue + parent := fsutil.SlashedPathDir(targetPath) + for ; parent != "/"; parent = fsutil.SlashedPathDir(parent) { + extractPackage[parent] = append(extractPackage[parent], deb.ExtractInfo{ + Path: parent, + Optional: true, + }) } - extractPackage[targetDir] = append(extractPackage[targetDir], deb.ExtractInfo{ - Path: targetDir, - Optional: true, - }) } } if !hasCopyright { diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index aa9d0f51..cdb68f12 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -16,9 +16,21 @@ import ( "github.com/canonical/chisel/internal/testutil" ) +var ( + Reg = testutil.Reg + Dir = testutil.Dir + Lnk = testutil.Lnk +) + +type testPackage struct { + info map[string]string + content []byte +} + type slicerTest struct { summary string arch string + pkgs map[string]map[string]testPackage release map[string]string slices []setup.SliceKey hackopt func(c *C, opts *slicer.RunOptions) @@ -515,6 +527,77 @@ var slicerTests = []slicerTest{{ "/usr/bin/": "dir 0755", "/usr/bin/hello": "file 0775 eaf29575", }, +}, { + summary: "Custom archives with custom packages", + pkgs: map[string]map[string]testPackage{ + "leptons": { + "electron": testPackage{ + content: testutil.MustMakeDeb([]testutil.TarEntry{ + Dir(0755, "./"), + Dir(0755, "./mass/"), + Reg(0644, "./mass/electron", "9.1093837015E−31 kg\n"), + Dir(0755, "./usr/"), + Dir(0755, "./usr/share/"), + Dir(0755, "./usr/share/doc/"), + Dir(0755, "./usr/share/doc/electron/"), + Reg(0644, "./usr/share/doc/electron/copyright", ""), + }), + }, + }, + "hadrons": { + "proton": testPackage{ + content: testutil.MustMakeDeb([]testutil.TarEntry{ + Dir(0755, "./"), + Dir(0755, "./mass/"), + Reg(0644, "./mass/proton", "1.67262192369E−27 kg\n"), + }), + }, + }, + }, + release: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + leptons: + version: 1 + suites: [main] + components: [main, universe] + default: true + hadrons: + version: 1 + suites: [main] + components: [main] + `, + "slices/mydir/electron.yaml": ` + package: electron + slices: + mass: + contents: + /mass/electron: + `, + "slices/mydir/proton.yaml": ` + package: proton + archive: hadrons + slices: + mass: + contents: + /mass/proton: + `, + }, + slices: []setup.SliceKey{ + {"electron", "mass"}, + {"proton", "mass"}, + }, + result: map[string]string{ + "/mass/": "dir 0755", + "/mass/electron": "file 0644 a1258e30", + "/mass/proton": "file 0644 a2390d10", + "/usr/": "dir 0755", + "/usr/share/": "dir 0755", + "/usr/share/doc/": "dir 0755", + "/usr/share/doc/electron/": "dir 0755", + "/usr/share/doc/electron/copyright": "file 0644 empty", + }, }} const defaultChiselYaml = ` @@ -525,9 +608,25 @@ const defaultChiselYaml = ` components: [main, universe] ` +type testPackageInfo map[string]string + +var _ archive.PackageInfo = (testPackageInfo)(nil) + +func (info testPackageInfo) Name() string { return info["Package"] } +func (info testPackageInfo) Version() string { return info["Version"] } +func (info testPackageInfo) Arch() string { return info["Architecture"] } +func (info testPackageInfo) SHA256() string { return info["SHA256"] } + +func (s testPackageInfo) Get(key string) (value string) { + if s != nil { + value = s[key] + } + return +} + type testArchive struct { options archive.Options - pkgs map[string][]byte + pkgs map[string]testPackage } func (a *testArchive) Options() *archive.Options { @@ -536,7 +635,7 @@ func (a *testArchive) Options() *archive.Options { func (a *testArchive) Fetch(pkg string) (io.ReadCloser, error) { if data, ok := a.pkgs[pkg]; ok { - return io.NopCloser(bytes.NewBuffer(data)), nil + return io.NopCloser(bytes.NewBuffer(data.content)), nil } return nil, fmt.Errorf("attempted to open %q package", pkg) } @@ -546,6 +645,18 @@ func (a *testArchive) Exists(pkg string) bool { return ok } +func (a *testArchive) Info(pkg string) archive.PackageInfo { + var info map[string]string + if pkgData, ok := a.pkgs[pkg]; ok { + if info = pkgData.info; info == nil { + info = map[string]string{ + "Version": "1.0", + } + } + } + return testPackageInfo(info) +} + func (s *S) TestRun(c *C) { for _, test := range slicerTests { c.Logf("Summary: %s", test.summary) @@ -569,16 +680,23 @@ func (s *S) TestRun(c *C) { selection, err := setup.Select(release, test.slices) c.Assert(err, IsNil) - pkgs := map[string][]byte{ - "base-files": testutil.PackageData["base-files"], + pkgs := map[string]testPackage{ + "base-files": testPackage{content: testutil.PackageData["base-files"]}, } for name, entries := range packageEntries { deb, err := testutil.MakeDeb(entries) c.Assert(err, IsNil) - pkgs[name] = deb + pkgs[name] = testPackage{content: deb} } archives := map[string]archive.Archive{} for name, setupArchive := range release.Archives { + var archivePkgs map[string]testPackage + if test.pkgs != nil { + archivePkgs = test.pkgs[name] + } + if archivePkgs == nil { + archivePkgs = pkgs + } archive := &testArchive{ options: archive.Options{ Label: setupArchive.Name, @@ -587,7 +705,7 @@ func (s *S) TestRun(c *C) { Components: setupArchive.Components, Arch: test.arch, }, - pkgs: pkgs, + pkgs: archivePkgs, } archives[name] = archive } @@ -611,8 +729,15 @@ func (s *S) TestRun(c *C) { if test.result != nil { result := make(map[string]string, len(copyrightEntries)+len(test.result)) - for k, v := range copyrightEntries { - result[k] = v + if test.pkgs == nil { + // This was added in order to not specify copyright entries for each + // existing test. These tests use only the base-files embedded + // package. Custom packages may not include copyright entries + // though. So if a test defines any custom packages, it must include + // copyright entries explicitly in the results. + for k, v := range copyrightEntries { + result[k] = v + } } for k, v := range test.result { result[k] = v