diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 06c43920..e5ed42bc 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -79,7 +79,7 @@ func (cmd *cmdCut) Execute(args []string) error { archives[archiveName] = openArchive } - _, err = slicer.Run(&slicer.RunOptions{ + err = slicer.Run(&slicer.RunOptions{ Selection: selection, Archives: archives, TargetDir: cmd.RootDir, diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 2a552084..c09fbdfb 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -18,8 +18,16 @@ import ( type Archive interface { Options() *Options - Fetch(pkg string) (io.ReadCloser, error) + Fetch(pkg string) (io.ReadCloser, *PackageInfo, error) Exists(pkg string) bool + Info(pkg string) (*PackageInfo, error) +} + +type PackageInfo struct { + Name string + Version string + Arch string + SHA256 string } type Options struct { @@ -112,18 +120,28 @@ func (a *ubuntuArchive) selectPackage(pkg string) (control.Section, *ubuntuIndex return selectedSection, selectedIndex, nil } -func (a *ubuntuArchive) Fetch(pkg string) (io.ReadCloser, error) { +func (a *ubuntuArchive) Fetch(pkg string) (io.ReadCloser, *PackageInfo, error) { section, index, err := a.selectPackage(pkg) if err != nil { - return nil, err + return nil, nil, err } suffix := section.Get("Filename") logf("Fetching %s...", suffix) reader, err := index.fetch("../../"+suffix, section.Get("SHA256"), fetchBulk) + if err != nil { + return nil, nil, err + } + info := sectionPackageInfo(section) + return reader, info, nil +} + +func (a *ubuntuArchive) Info(pkg string) (*PackageInfo, error) { + section, _, err := a.selectPackage(pkg) if err != nil { return nil, err } - return reader, nil + info := sectionPackageInfo(section) + return info, nil } const ubuntuURL = "http://archive.ubuntu.com/ubuntu/" @@ -336,3 +354,12 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea return index.archive.cache.Open(writer.Digest()) } + +func sectionPackageInfo(section control.Section) *PackageInfo { + return &PackageInfo{ + Name: section.Get("Package"), + Version: section.Get("Version"), + Arch: section.Get("Architecture"), + SHA256: section.Get("SHA256"), + } +} diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index af741dbe..703148fb 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -205,17 +205,29 @@ func (s *httpSuite) TestFetchPackage(c *C) { PubKeys: []*packet.PublicKey{s.pubKey}, } - archive, err := archive.Open(&options) + testArchive, err := archive.Open(&options) c.Assert(err, IsNil) // First on component main. - pkg, err := archive.Fetch("mypkg1") + pkg, info, err := testArchive.Fetch("mypkg1") c.Assert(err, IsNil) + c.Assert(info, DeepEquals, &archive.PackageInfo{ + Name: "mypkg1", + Version: "1.1", + Arch: "amd64", + SHA256: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05", + }) c.Assert(read(pkg), Equals, "mypkg1 1.1 data") // Last on component universe. - pkg, err = archive.Fetch("mypkg4") + pkg, info, err = testArchive.Fetch("mypkg4") c.Assert(err, IsNil) + c.Assert(info, DeepEquals, &archive.PackageInfo{ + Name: "mypkg4", + Version: "1.4", + Arch: "amd64", + SHA256: "54af70097b30b33cfcbb6911ad3d0df86c2d458928169e348fa7873e4fc678e4", + }) c.Assert(read(pkg), Equals, "mypkg4 1.4 data") } @@ -235,17 +247,29 @@ func (s *httpSuite) TestFetchPortsPackage(c *C) { PubKeys: []*packet.PublicKey{s.pubKey}, } - archive, err := archive.Open(&options) + testArchive, err := archive.Open(&options) c.Assert(err, IsNil) // First on component main. - pkg, err := archive.Fetch("mypkg1") + pkg, info, err := testArchive.Fetch("mypkg1") c.Assert(err, IsNil) + c.Assert(info, DeepEquals, &archive.PackageInfo{ + Name: "mypkg1", + Version: "1.1", + Arch: "arm64", + SHA256: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05", + }) c.Assert(read(pkg), Equals, "mypkg1 1.1 data") // Last on component universe. - pkg, err = archive.Fetch("mypkg4") + pkg, info, err = testArchive.Fetch("mypkg4") c.Assert(err, IsNil) + c.Assert(info, DeepEquals, &archive.PackageInfo{ + Name: "mypkg4", + Version: "1.4", + Arch: "arm64", + SHA256: "54af70097b30b33cfcbb6911ad3d0df86c2d458928169e348fa7873e4fc678e4", + }) c.Assert(read(pkg), Equals, "mypkg4 1.4 data") } @@ -273,15 +297,27 @@ func (s *httpSuite) TestFetchSecurityPackage(c *C) { PubKeys: []*packet.PublicKey{s.pubKey}, } - archive, err := archive.Open(&options) + testArchive, err := archive.Open(&options) c.Assert(err, IsNil) - pkg, err := archive.Fetch("mypkg1") + pkg, info, err := testArchive.Fetch("mypkg1") c.Assert(err, IsNil) + c.Assert(info, DeepEquals, &archive.PackageInfo{ + Name: "mypkg1", + Version: "1.1.2.2", + Arch: "amd64", + SHA256: "5448585bdd916e5023eff2bc1bc3b30bcc6ee9db9c03e531375a6a11ddf0913c", + }) c.Assert(read(pkg), Equals, "package from jammy-security") - pkg, err = archive.Fetch("mypkg2") + pkg, info, err = testArchive.Fetch("mypkg2") c.Assert(err, IsNil) + c.Assert(info, DeepEquals, &archive.PackageInfo{ + Name: "mypkg2", + Version: "1.2", + Arch: "amd64", + SHA256: "a4b4f3f3a8fa09b69e3ba23c60a41a1f8144691fd371a2455812572fd02e6f79", + }) c.Assert(read(pkg), Equals, "mypkg2 1.2 data") } @@ -399,6 +435,53 @@ func (s *httpSuite) TestVerifyArchiveRelease(c *C) { } } +var packageInfoTests = []struct { + summary string + pkg string + info *archive.PackageInfo + error string +}{{ + summary: "Basic", + pkg: "mypkg1", + info: &archive.PackageInfo{ + Name: "mypkg1", + Version: "1.1", + Arch: "amd64", + SHA256: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05", + }, +}, { + summary: "Package not found in archive", + pkg: "mypkg99", + error: `cannot find package "mypkg99" in archive`, +}} + +func (s *httpSuite) TestPackageInfo(c *C) { + s.prepareArchive("jammy", "22.04", "amd64", []string{"main", "universe"}) + + options := archive.Options{ + Label: "ubuntu", + Version: "22.04", + Arch: "amd64", + Suites: []string{"jammy"}, + Components: []string{"main", "universe"}, + CacheDir: c.MkDir(), + PubKeys: []*packet.PublicKey{s.pubKey}, + } + + testArchive, err := archive.Open(&options) + c.Assert(err, IsNil) + + for _, test := range packageInfoTests { + info, err := testArchive.Info(test.pkg) + if test.error != "" { + c.Assert(err, ErrorMatches, test.error) + continue + } + c.Assert(err, IsNil) + c.Assert(info, DeepEquals, test.info) + } +} + func read(r io.Reader) string { data, err := io.ReadAll(r) if err != nil { @@ -481,13 +564,15 @@ func (s *S) testOpenArchiveArch(c *C, release ubuntuRelease, arch string) { PubKeys: release.archivePubKeys, } - archive, err := archive.Open(&options) + testArchive, err := archive.Open(&options) c.Assert(err, IsNil) extractDir := c.MkDir() - pkg, err := archive.Fetch("hostname") + pkg, info, err := testArchive.Fetch("hostname") c.Assert(err, IsNil) + c.Assert(info.Name, DeepEquals, "hostname") + c.Assert(info.Arch, DeepEquals, arch) err = deb.Extract(pkg, &deb.ExtractOptions{ Package: "hostname", diff --git a/internal/fsutil/create.go b/internal/fsutil/create.go index 48b12cb7..f76271f1 100644 --- a/internal/fsutil/create.go +++ b/internal/fsutil/create.go @@ -22,11 +22,11 @@ type CreateOptions struct { } type Entry struct { - Path string - Mode fs.FileMode - Hash string - Size int - Link string + Path string + Mode fs.FileMode + SHA256 string + Size int + Link string } // Create creates a filesystem entry according to the provided options and returns @@ -66,15 +66,44 @@ func Create(options *CreateOptions) (*Entry, error) { return nil, err } entry := &Entry{ - Path: o.Path, - Mode: s.Mode(), - Hash: hash, - Size: rp.size, - Link: o.Link, + Path: o.Path, + Mode: s.Mode(), + SHA256: hash, + Size: rp.size, + Link: o.Link, } return entry, nil } +// CreateWriter handles the creation of a regular file and collects the +// information recorded in Entry. The Hash and Size attributes are set on +// calling Close() on the Writer. +func CreateWriter(options *CreateOptions) (io.WriteCloser, *Entry, error) { + if !options.Mode.IsRegular() { + return nil, nil, fmt.Errorf("unsupported file type: %s", options.Path) + } + if options.MakeParents { + if err := os.MkdirAll(filepath.Dir(options.Path), 0755); err != nil { + return nil, nil, err + } + } + file, err := os.OpenFile(options.Path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, options.Mode) + if err != nil { + return nil, nil, err + } + entry := &Entry{ + Path: options.Path, + Mode: options.Mode, + } + wp := &writerProxy{ + entry: entry, + inner: file, + h: sha256.New(), + size: 0, + } + return wp, entry, nil +} + func createDir(o *CreateOptions) error { debugf("Creating directory: %s (mode %#o)", o.Path, o.Mode) err := os.Mkdir(o.Path, o.Mode) @@ -137,3 +166,29 @@ func (rp *readerProxy) Read(p []byte) (n int, err error) { rp.size += n return n, err } + +// writerProxy implements the io.WriteCloser interface proxying the calls to its +// inner io.WriteCloser. On each write, the proxy keeps track of the file size +// and hash. The associated entry hash and size are updated when Close() is +// called. +type writerProxy struct { + inner io.WriteCloser + h hash.Hash + size int + entry *Entry +} + +var _ io.WriteCloser = (*writerProxy)(nil) + +func (rp *writerProxy) Write(p []byte) (n int, err error) { + n, err = rp.inner.Write(p) + rp.h.Write(p[:n]) + rp.size += n + return n, err +} + +func (rp *writerProxy) Close() error { + rp.entry.SHA256 = hex.EncodeToString(rp.h.Sum(nil)) + rp.entry.Size = rp.size + return rp.inner.Close() +} diff --git a/internal/fsutil/create_test.go b/internal/fsutil/create_test.go index 7288d878..d41bbf5a 100644 --- a/internal/fsutil/create_test.go +++ b/internal/fsutil/create_test.go @@ -133,3 +133,100 @@ func (s *S) TestCreate(c *C) { c.Assert(testutil.TreeDumpEntry(entry), DeepEquals, test.result[slashPath]) } } + +type createWriterTest struct { + options fsutil.CreateOptions + data []byte + hackdir func(c *C, dir string) + result map[string]string + error string +} + +var createWriterTests = []createWriterTest{{ + options: fsutil.CreateOptions{ + Path: "foo", + Mode: 0644, + }, + data: []byte("foo"), + result: map[string]string{ + "/foo": "file 0644 2c26b46b", + }, +}, { + options: fsutil.CreateOptions{ + Path: "foo", + Mode: 0644 | fs.ModeDir, + }, + error: `unsupported file type: \/[a-z0-9\-\/]*foo`, +}, { + options: fsutil.CreateOptions{ + Path: "foo", + Mode: 0644 | fs.ModeSymlink, + }, + error: `unsupported file type: /[a-z0-9\-\/]*/foo`, +}, { + options: fsutil.CreateOptions{ + Path: "foo/bar", + Mode: 0644, + MakeParents: true, + }, + data: []byte("foo"), + result: map[string]string{ + "/foo/": "dir 0755", + "/foo/bar": "file 0644 2c26b46b", + }, +}, { + options: fsutil.CreateOptions{ + Path: "foo/bar", + Mode: 0644, + MakeParents: false, + }, + error: `open /[a-z0-9\-\/]*/foo/bar: no such file or directory`, +}} + +func (s *S) TestCreateWriter(c *C) { + oldUmask := syscall.Umask(0) + defer func() { + syscall.Umask(oldUmask) + }() + + for _, test := range createWriterTests { + if test.result == nil { + // Empty map for no files created. + test.result = make(map[string]string) + } + c.Logf("Options: %v", test.options) + dir := c.MkDir() + if test.hackdir != nil { + test.hackdir(c, dir) + } + options := test.options + options.Path = filepath.Join(dir, options.Path) + writer, entry, err := fsutil.CreateWriter(&options) + if test.error != "" { + c.Assert(err, ErrorMatches, test.error) + continue + } + c.Assert(err, IsNil) + + // Hash and Size are only set when the writer is closed. + _, err = writer.Write(test.data) + c.Assert(err, IsNil) + c.Assert(entry.Path, Equals, options.Path) + c.Assert(entry.Mode, Equals, options.Mode) + c.Assert(entry.SHA256, Equals, "") + c.Assert(entry.Size, Equals, 0) + err = writer.Close() + c.Assert(err, IsNil) + + c.Assert(testutil.TreeDump(dir), DeepEquals, test.result) + // [fsutil.CreateWriter] does not return information about parent + // directories created implicitly. We only check for the requested path. + entry.Path = strings.TrimPrefix(entry.Path, dir) + // Add the slashes that TreeDump adds to the path. + slashPath := "/" + test.options.Path + if test.options.Mode.IsDir() { + slashPath = slashPath + "/" + } + c.Assert(testutil.TreeDumpEntry(entry), DeepEquals, test.result[slashPath]) + } +} diff --git a/internal/manifest/manifest.go b/internal/manifest/manifest.go index fc8a1366..8c0b430b 100644 --- a/internal/manifest/manifest.go +++ b/internal/manifest/manifest.go @@ -3,13 +3,19 @@ package manifest import ( "fmt" "io" + "io/fs" + "path/filepath" "slices" + "sort" + "strings" + "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/jsonwall" "github.com/canonical/chisel/internal/setup" ) -const schema = "1.0" +const Schema = "1.0" +const DefaultFilename = "manifest.wall" type Package struct { Kind string `json:"kind"` @@ -25,14 +31,14 @@ type Slice struct { } type Path struct { - Kind string `json:"kind"` - Path string `json:"path,omitempty"` - Mode string `json:"mode,omitempty"` - Slices []string `json:"slices,omitempty"` - Hash string `json:"sha256,omitempty"` - FinalHash string `json:"final_sha256,omitempty"` - Size uint64 `json:"size,omitempty"` - Link string `json:"link,omitempty"` + Kind string `json:"kind"` + Path string `json:"path,omitempty"` + Mode string `json:"mode,omitempty"` + Slices []string `json:"slices,omitempty"` + SHA256 string `json:"sha256,omitempty"` + FinalSHA256 string `json:"final_sha256,omitempty"` + Size uint64 `json:"size,omitempty"` + Link string `json:"link,omitempty"` } type Content struct { @@ -59,7 +65,7 @@ func Read(reader io.Reader) (manifest *Manifest, err error) { return nil, err } mfestSchema := db.Schema() - if mfestSchema != schema { + if mfestSchema != Schema { return nil, fmt.Errorf("unknown schema version %q", mfestSchema) } @@ -158,6 +164,52 @@ func Validate(manifest *Manifest) (err error) { return nil } +// FindPaths finds the paths marked with "generate:manifest" and +// returns a map from the manifest path to all the slices that declare it. +func FindPaths(slices []*setup.Slice) map[string][]*setup.Slice { + manifestSlices := make(map[string][]*setup.Slice) + for _, slice := range slices { + for path, info := range slice.Contents { + if info.Generate == setup.GenerateManifest { + dir := strings.TrimSuffix(path, "**") + path = filepath.Join(dir, DefaultFilename) + manifestSlices[path] = append(manifestSlices[path], slice) + } + } + } + return manifestSlices +} + +type WriteOptions struct { + PackageInfo []*archive.PackageInfo + Selection []*setup.Slice + Report *Report +} + +func Write(options *WriteOptions, writer io.Writer) error { + dbw := jsonwall.NewDBWriter(&jsonwall.DBWriterOptions{ + Schema: Schema, + }) + + err := manifestAddPackages(dbw, options.PackageInfo) + if err != nil { + return err + } + + err = manifestAddSlices(dbw, options.Selection) + if err != nil { + return err + } + + err = manifestAddReport(dbw, options.Report) + if err != nil { + return err + } + + _, err = dbw.WriteTo(writer) + return err +} + type prefixable interface { Path | Content | Package | Slice } @@ -180,3 +232,72 @@ func iteratePrefix[T prefixable](manifest *Manifest, prefix *T, onMatch func(*T) } return nil } + +func manifestAddPackages(dbw *jsonwall.DBWriter, infos []*archive.PackageInfo) error { + for _, info := range infos { + err := dbw.Add(&Package{ + Kind: "package", + Name: info.Name, + Version: info.Version, + Digest: info.SHA256, + Arch: info.Arch, + }) + if err != nil { + return err + } + } + return nil +} + +func manifestAddSlices(dbw *jsonwall.DBWriter, slices []*setup.Slice) error { + for _, slice := range slices { + err := dbw.Add(&Slice{ + Kind: "slice", + Name: slice.String(), + }) + if err != nil { + return err + } + } + return nil +} + +func manifestAddReport(dbw *jsonwall.DBWriter, report *Report) error { + for _, entry := range report.Entries { + sliceNames := []string{} + for slice := range entry.Slices { + err := dbw.Add(&Content{ + Kind: "content", + Slice: slice.String(), + Path: entry.Path, + }) + if err != nil { + return err + } + sliceNames = append(sliceNames, slice.String()) + } + sort.Strings(sliceNames) + err := dbw.Add(&Path{ + Kind: "path", + Path: entry.Path, + Mode: fmt.Sprintf("0%o", unixPerm(entry.Mode)), + Slices: sliceNames, + SHA256: entry.SHA256, + FinalSHA256: entry.FinalSHA256, + Size: uint64(entry.Size), + Link: entry.Link, + }) + if err != nil { + return err + } + } + return nil +} + +func unixPerm(mode fs.FileMode) (perm uint32) { + perm = uint32(mode.Perm()) + if mode&fs.ModeSticky != 0 { + perm |= 01000 + } + return perm +} diff --git a/internal/manifest/manifest_test.go b/internal/manifest/manifest_test.go index 1da510da..9e757bdd 100644 --- a/internal/manifest/manifest_test.go +++ b/internal/manifest/manifest_test.go @@ -1,6 +1,8 @@ package manifest_test import ( + "bytes" + "io" "os" "path" "slices" @@ -8,7 +10,9 @@ import ( . "gopkg.in/check.v1" + "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/manifest" + "github.com/canonical/chisel/internal/setup" ) type manifestContents struct { @@ -18,7 +22,7 @@ type manifestContents struct { Contents []*manifest.Content } -var manifestTests = []struct { +var readManifestTests = []struct { summary string input string mfest *manifestContents @@ -45,10 +49,10 @@ var manifestTests = []struct { `, mfest: &manifestContents{ Paths: []*manifest.Path{ - {Kind: "path", Path: "/dir/file", Mode: "0644", Slices: []string{"pkg1_myslice"}, Hash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", FinalHash: "8067926c032c090867013d14fb0eb21ae858344f62ad07086fd32375845c91a6", Size: 0x15, Link: ""}, - {Kind: "path", Path: "/dir/foo/bar/", Mode: "01777", Slices: []string{"pkg2_myotherslice", "pkg1_myslice"}, Hash: "", FinalHash: "", Size: 0x0, Link: ""}, - {Kind: "path", Path: "/dir/link/file", Mode: "0644", Slices: []string{"pkg1_myslice"}, Hash: "", FinalHash: "", Size: 0x0, Link: "/dir/file"}, - {Kind: "path", Path: "/manifest/manifest.wall", Mode: "0644", Slices: []string{"pkg1_manifest"}, Hash: "", FinalHash: "", Size: 0x0, Link: ""}, + {Kind: "path", Path: "/dir/file", Mode: "0644", Slices: []string{"pkg1_myslice"}, SHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", FinalSHA256: "8067926c032c090867013d14fb0eb21ae858344f62ad07086fd32375845c91a6", Size: 0x15, Link: ""}, + {Kind: "path", Path: "/dir/foo/bar/", Mode: "01777", Slices: []string{"pkg2_myotherslice", "pkg1_myslice"}, SHA256: "", FinalSHA256: "", Size: 0x0, Link: ""}, + {Kind: "path", Path: "/dir/link/file", Mode: "0644", Slices: []string{"pkg1_myslice"}, SHA256: "", FinalSHA256: "", Size: 0x0, Link: "/dir/file"}, + {Kind: "path", Path: "/manifest/manifest.wall", Mode: "0644", Slices: []string{"pkg1_manifest"}, SHA256: "", FinalSHA256: "", Size: 0x0, Link: ""}, }, Packages: []*manifest.Package{ {Kind: "package", Name: "pkg1", Version: "v1", Digest: "hash1", Arch: "arch1"}, @@ -123,8 +127,8 @@ var manifestTests = []struct { readError: `cannot read manifest: unknown schema version "2.0"`, }} -func (s *S) TestRun(c *C) { - for _, test := range manifestTests { +func (s *S) TestManifestReadValidate(c *C) { + for _, test := range readManifestTests { c.Logf("Summary: %s", test.summary) // Reindent the jsonwall to remove leading tabs in each line. @@ -169,6 +173,222 @@ func (s *S) TestRun(c *C) { } } +var findPathsTests = []struct { + summary string + slices []*setup.Slice + expected map[string][]string +}{{ + summary: "Single slice", + slices: []*setup.Slice{{ + Name: "slice1", + Contents: map[string]setup.PathInfo{ + "/folder/**": { + Kind: "generate", + Generate: "manifest", + }, + }, + }}, + expected: map[string][]string{ + "/folder/manifest.wall": []string{"slice1"}, + }, +}, { + summary: "No slice matched", + slices: []*setup.Slice{{ + Name: "slice1", + Contents: map[string]setup.PathInfo{}, + }}, + expected: map[string][]string{}, +}, { + summary: "Several matches with several groups", + slices: []*setup.Slice{{ + Name: "slice1", + Contents: map[string]setup.PathInfo{ + "/folder/**": { + Kind: "generate", + Generate: "manifest", + }, + }, + }, { + Name: "slice2", + Contents: map[string]setup.PathInfo{ + "/folder/**": { + Kind: "generate", + Generate: "manifest", + }, + }, + }, { + Name: "slice3", + Contents: map[string]setup.PathInfo{}, + }, { + Name: "slice4", + Contents: map[string]setup.PathInfo{ + "/other-folder/**": { + Kind: "generate", + Generate: "manifest", + }, + }, + }, { + Name: "slice5", + Contents: map[string]setup.PathInfo{ + "/other-folder/**": { + Kind: "generate", + Generate: "manifest", + }, + }, + }}, + expected: map[string][]string{ + "/folder/manifest.wall": {"slice1", "slice2"}, + "/other-folder/manifest.wall": {"slice4", "slice5"}, + }, +}} + +func (s *S) TestFindPaths(c *C) { + for _, test := range findPathsTests { + c.Logf("Summary: %s", test.summary) + + manifestSlices := manifest.FindPaths(test.slices) + + slicesByName := map[string]*setup.Slice{} + for _, slice := range test.slices { + _, ok := slicesByName[slice.Name] + c.Assert(ok, Equals, false, Commentf("duplicated slice name")) + slicesByName[slice.Name] = slice + } + + c.Assert(manifestSlices, HasLen, len(test.expected)) + for path, slices := range manifestSlices { + c.Assert(slices, HasLen, len(test.expected[path])) + for i, sliceName := range test.expected[path] { + c.Assert(slicesByName[sliceName], DeepEquals, slices[i]) + } + } + } +} + +func (s *S) TestGenerateManifests(c *C) { + slice1 := &setup.Slice{ + Package: "package1", + Name: "slice1", + } + slice2 := &setup.Slice{ + Package: "package2", + Name: "slice2", + } + report := &manifest.Report{ + Root: "/", + Entries: map[string]manifest.ReportEntry{ + "/file": { + Path: "/file", + Mode: 0456, + SHA256: "hash", + Size: 1234, + Slices: map[*setup.Slice]bool{slice1: true}, + FinalSHA256: "final-hash", + }, + "/link": { + Path: "/link", + Mode: 0567, + Link: "/target", + Slices: map[*setup.Slice]bool{slice1: true, slice2: true}, + }, + }, + } + packageInfo := []*archive.PackageInfo{{ + Name: "package1", + Version: "v1", + Arch: "a1", + SHA256: "s1", + }, { + Name: "package2", + Version: "v2", + Arch: "a2", + SHA256: "s2", + }} + + expected := &manifestContents{ + Paths: []*manifest.Path{{ + Kind: "path", + Path: "/file", + Mode: "0456", + Slices: []string{"package1_slice1"}, + Size: 1234, + SHA256: "hash", + FinalSHA256: "final-hash", + }, { + Kind: "path", + Path: "/link", + Link: "/target", + Mode: "0567", + Slices: []string{"package1_slice1", "package2_slice2"}, + }}, + Packages: []*manifest.Package{{ + Kind: "package", + Name: "package1", + Version: "v1", + Digest: "s1", + Arch: "a1", + }, { + Kind: "package", + Name: "package2", + Version: "v2", + Digest: "s2", + Arch: "a2", + }}, + Slices: []*manifest.Slice{{ + Kind: "slice", + Name: "package1_slice1", + }, { + Kind: "slice", + Name: "package2_slice2", + }}, + Contents: []*manifest.Content{{ + Kind: "content", + Slice: "package1_slice1", + Path: "/file", + }, { + Kind: "content", + Slice: "package1_slice1", + Path: "/link", + }, { + Kind: "content", + Slice: "package2_slice2", + Path: "/link", + }}, + } + + options := &manifest.WriteOptions{ + PackageInfo: packageInfo, + Selection: []*setup.Slice{slice1, slice2}, + Report: report, + } + var buffer bytes.Buffer + err := manifest.Write(options, &buffer) + c.Assert(err, IsNil) + mfest, err := manifest.Read(&buffer) + c.Assert(err, IsNil) + err = manifest.Validate(mfest) + c.Assert(err, IsNil) + contents := dumpManifestContents(c, mfest) + c.Assert(contents, DeepEquals, expected) +} + +func (s *S) TestGenerateNoManifests(c *C) { + report, err := manifest.NewReport("/") + c.Assert(err, IsNil) + options := &manifest.WriteOptions{ + Report: report, + } + var buffer bytes.Buffer + err = manifest.Write(options, &buffer) + c.Assert(err, IsNil) + + var reader io.Reader = &buffer + var bs []byte + n, err := reader.Read(bs) + c.Assert(err, IsNil) + c.Assert(n, Equals, 0) +} + func dumpManifestContents(c *C, mfest *manifest.Manifest) *manifestContents { var slices []*manifest.Slice err := mfest.IterateSlices("", func(slice *manifest.Slice) error { diff --git a/internal/slicer/report.go b/internal/manifest/report.go similarity index 83% rename from internal/slicer/report.go rename to internal/manifest/report.go index 8897f518..092ed823 100644 --- a/internal/slicer/report.go +++ b/internal/manifest/report.go @@ -1,4 +1,4 @@ -package slicer +package manifest import ( "fmt" @@ -11,13 +11,13 @@ import ( ) type ReportEntry struct { - Path string - Mode fs.FileMode - Hash string - Size int - Slices map[*setup.Slice]bool - Link string - FinalHash string + Path string + Mode fs.FileMode + SHA256 string + Size int + Slices map[*setup.Slice]bool + Link string + FinalSHA256 string } // Report holds the information about files and directories created when slicing @@ -35,8 +35,12 @@ func NewReport(root string) (*Report, error) { if !filepath.IsAbs(root) { return nil, fmt.Errorf("cannot use relative path for report root: %q", root) } + root = filepath.Clean(root) + if root != "/" { + root = filepath.Clean(root) + "/" + } report := &Report{ - Root: filepath.Clean(root) + "/", + Root: root, Entries: make(map[string]ReportEntry), } return report, nil @@ -55,8 +59,8 @@ func (r *Report) Add(slice *setup.Slice, fsEntry *fsutil.Entry) error { return fmt.Errorf("path %s reported twice with diverging link: %q != %q", relPath, fsEntry.Link, entry.Link) } else if fsEntry.Size != entry.Size { return fmt.Errorf("path %s reported twice with diverging size: %d != %d", relPath, fsEntry.Size, entry.Size) - } else if fsEntry.Hash != entry.Hash { - return fmt.Errorf("path %s reported twice with diverging hash: %q != %q", relPath, fsEntry.Hash, entry.Hash) + } else if fsEntry.SHA256 != entry.SHA256 { + return fmt.Errorf("path %s reported twice with diverging hash: %q != %q", relPath, fsEntry.SHA256, entry.SHA256) } entry.Slices[slice] = true r.Entries[relPath] = entry @@ -64,7 +68,7 @@ func (r *Report) Add(slice *setup.Slice, fsEntry *fsutil.Entry) error { r.Entries[relPath] = ReportEntry{ Path: relPath, Mode: fsEntry.Mode, - Hash: fsEntry.Hash, + SHA256: fsEntry.SHA256, Size: fsEntry.Size, Slices: map[*setup.Slice]bool{slice: true}, Link: fsEntry.Link, @@ -73,7 +77,7 @@ func (r *Report) Add(slice *setup.Slice, fsEntry *fsutil.Entry) error { return nil } -// Mutate updates the FinalHash and Size of an existing path entry. +// Mutate updates the FinalSHA256 and Size of an existing path entry. func (r *Report) Mutate(fsEntry *fsutil.Entry) error { relPath, err := r.sanitizeAbsPath(fsEntry.Path, fsEntry.Mode.IsDir()) if err != nil { @@ -87,11 +91,11 @@ func (r *Report) Mutate(fsEntry *fsutil.Entry) error { if entry.Mode.IsDir() { return fmt.Errorf("cannot mutate path in report: %s is a directory", relPath) } - if entry.Hash == fsEntry.Hash { + if entry.SHA256 == fsEntry.SHA256 { // Content has not changed, nothing to do. return nil } - entry.FinalHash = fsEntry.Hash + entry.FinalSHA256 = fsEntry.SHA256 entry.Size = fsEntry.Size r.Entries[relPath] = entry return nil diff --git a/internal/slicer/report_test.go b/internal/manifest/report_test.go similarity index 75% rename from internal/slicer/report_test.go rename to internal/manifest/report_test.go index 762b35ad..9c001509 100644 --- a/internal/slicer/report_test.go +++ b/internal/manifest/report_test.go @@ -1,4 +1,4 @@ -package slicer_test +package manifest_test import ( "io/fs" @@ -6,8 +6,8 @@ import ( . "gopkg.in/check.v1" "github.com/canonical/chisel/internal/fsutil" + "github.com/canonical/chisel/internal/manifest" "github.com/canonical/chisel/internal/setup" - "github.com/canonical/chisel/internal/slicer" ) var oneSlice = &setup.Slice{ @@ -33,25 +33,25 @@ var sampleDir = fsutil.Entry{ } var sampleFile = fsutil.Entry{ - Path: "/base/example-file", - Mode: 0777, - Hash: "example-file_hash", - Size: 5678, - Link: "", + Path: "/base/example-file", + Mode: 0777, + SHA256: "example-file_hash", + Size: 5678, + Link: "", } var sampleLink = fsutil.Entry{ - Path: "/base/example-link", - Mode: 0777, - Hash: "example-file_hash", - Size: 5678, - Link: "/base/example-file", + Path: "/base/example-link", + Mode: 0777, + SHA256: "example-file_hash", + Size: 5678, + Link: "/base/example-file", } var sampleFileMutated = fsutil.Entry{ - Path: sampleFile.Path, - Hash: sampleFile.Hash + "_changed", - Size: sampleFile.Size + 10, + Path: sampleFile.Path, + SHA256: sampleFile.SHA256 + "_changed", + Size: sampleFile.Size + 10, } type sliceAndEntry struct { @@ -64,13 +64,13 @@ var reportTests = []struct { add []sliceAndEntry mutate []*fsutil.Entry // indexed by path. - expected map[string]slicer.ReportEntry + expected map[string]manifest.ReportEntry // error after adding the last [sliceAndEntry]. err string }{{ summary: "Regular directory", add: []sliceAndEntry{{entry: sampleDir, slice: oneSlice}}, - expected: map[string]slicer.ReportEntry{ + expected: map[string]manifest.ReportEntry{ "/example-dir/": { Path: "/example-dir/", Mode: fs.ModeDir | 0654, @@ -83,7 +83,7 @@ var reportTests = []struct { {entry: sampleDir, slice: oneSlice}, {entry: sampleDir, slice: otherSlice}, }, - expected: map[string]slicer.ReportEntry{ + expected: map[string]manifest.ReportEntry{ "/example-dir/": { Path: "/example-dir/", Mode: fs.ModeDir | 0654, @@ -93,11 +93,11 @@ var reportTests = []struct { }, { summary: "Regular file", add: []sliceAndEntry{{entry: sampleFile, slice: oneSlice}}, - expected: map[string]slicer.ReportEntry{ + expected: map[string]manifest.ReportEntry{ "/example-file": { Path: "/example-file", Mode: 0777, - Hash: "example-file_hash", + SHA256: "example-file_hash", Size: 5678, Slices: map[*setup.Slice]bool{oneSlice: true}, Link: "", @@ -105,11 +105,11 @@ var reportTests = []struct { }, { summary: "Regular file link", add: []sliceAndEntry{{entry: sampleLink, slice: oneSlice}}, - expected: map[string]slicer.ReportEntry{ + expected: map[string]manifest.ReportEntry{ "/example-link": { Path: "/example-link", Mode: 0777, - Hash: "example-file_hash", + SHA256: "example-file_hash", Size: 5678, Slices: map[*setup.Slice]bool{oneSlice: true}, Link: "/base/example-file", @@ -120,7 +120,7 @@ var reportTests = []struct { {entry: sampleDir, slice: oneSlice}, {entry: sampleFile, slice: otherSlice}, }, - expected: map[string]slicer.ReportEntry{ + expected: map[string]manifest.ReportEntry{ "/example-dir/": { Path: "/example-dir/", Mode: fs.ModeDir | 0654, @@ -130,7 +130,7 @@ var reportTests = []struct { "/example-file": { Path: "/example-file", Mode: 0777, - Hash: "example-file_hash", + SHA256: "example-file_hash", Size: 5678, Slices: map[*setup.Slice]bool{otherSlice: true}, Link: "", @@ -141,11 +141,11 @@ var reportTests = []struct { {entry: sampleFile, slice: oneSlice}, {entry: sampleFile, slice: oneSlice}, }, - expected: map[string]slicer.ReportEntry{ + expected: map[string]manifest.ReportEntry{ "/example-file": { Path: "/example-file", Mode: 0777, - Hash: "example-file_hash", + SHA256: "example-file_hash", Size: 5678, Slices: map[*setup.Slice]bool{oneSlice: true}, Link: "", @@ -155,11 +155,11 @@ var reportTests = []struct { add: []sliceAndEntry{ {entry: sampleFile, slice: oneSlice}, {entry: fsutil.Entry{ - Path: sampleFile.Path, - Mode: 0, - Hash: sampleFile.Hash, - Size: sampleFile.Size, - Link: sampleFile.Link, + Path: sampleFile.Path, + Mode: 0, + SHA256: sampleFile.SHA256, + Size: sampleFile.Size, + Link: sampleFile.Link, }, slice: oneSlice}, }, err: `path /example-file reported twice with diverging mode: 0000 != 0777`, @@ -168,11 +168,11 @@ var reportTests = []struct { add: []sliceAndEntry{ {entry: sampleFile, slice: oneSlice}, {entry: fsutil.Entry{ - Path: sampleFile.Path, - Mode: sampleFile.Mode, - Hash: "distinct hash", - Size: sampleFile.Size, - Link: sampleFile.Link, + Path: sampleFile.Path, + Mode: sampleFile.Mode, + SHA256: "distinct hash", + Size: sampleFile.Size, + Link: sampleFile.Link, }, slice: oneSlice}, }, err: `path /example-file reported twice with diverging hash: "distinct hash" != "example-file_hash"`, @@ -181,11 +181,11 @@ var reportTests = []struct { add: []sliceAndEntry{ {entry: sampleFile, slice: oneSlice}, {entry: fsutil.Entry{ - Path: sampleFile.Path, - Mode: sampleFile.Mode, - Hash: sampleFile.Hash, - Size: 0, - Link: sampleFile.Link, + Path: sampleFile.Path, + Mode: sampleFile.Mode, + SHA256: sampleFile.SHA256, + Size: 0, + Link: sampleFile.Link, }, slice: oneSlice}, }, err: `path /example-file reported twice with diverging size: 0 != 5678`, @@ -194,11 +194,11 @@ var reportTests = []struct { add: []sliceAndEntry{ {entry: sampleFile, slice: oneSlice}, {entry: fsutil.Entry{ - Path: sampleFile.Path, - Mode: sampleFile.Mode, - Hash: sampleFile.Hash, - Size: sampleFile.Size, - Link: "distinct link", + Path: sampleFile.Path, + Mode: sampleFile.Mode, + SHA256: sampleFile.SHA256, + Size: sampleFile.Size, + Link: "distinct link", }, slice: oneSlice}, }, err: `path /example-file reported twice with diverging link: "distinct link" != ""`, @@ -225,7 +225,7 @@ var reportTests = []struct { {entry: sampleDir, slice: oneSlice}, }, mutate: []*fsutil.Entry{&sampleFileMutated}, - expected: map[string]slicer.ReportEntry{ + expected: map[string]manifest.ReportEntry{ "/example-dir/": { Path: "/example-dir/", Mode: fs.ModeDir | 0654, @@ -233,13 +233,13 @@ var reportTests = []struct { Link: "", }, "/example-file": { - Path: "/example-file", - Mode: 0777, - Hash: "example-file_hash", - Size: 5688, - Slices: map[*setup.Slice]bool{oneSlice: true}, - Link: "", - FinalHash: "example-file_hash_changed", + Path: "/example-file", + Mode: 0777, + SHA256: "example-file_hash", + Size: 5688, + Slices: map[*setup.Slice]bool{oneSlice: true}, + Link: "", + FinalSHA256: "example-file_hash_changed", }}, }, { summary: "Calling mutated with identical content to initial file", @@ -247,16 +247,16 @@ var reportTests = []struct { {entry: sampleFile, slice: oneSlice}, }, mutate: []*fsutil.Entry{&sampleFile}, - expected: map[string]slicer.ReportEntry{ + expected: map[string]manifest.ReportEntry{ "/example-file": { Path: "/example-file", Mode: 0777, - Hash: "example-file_hash", + SHA256: "example-file_hash", Size: 5678, Slices: map[*setup.Slice]bool{oneSlice: true}, Link: "", - // FinalHash is not updated. - FinalHash: "", + // FinalSHA256 is not updated. + FinalSHA256: "", }}, }, { summary: "Mutated paths must refer to previously added entries", @@ -272,7 +272,7 @@ var reportTests = []struct { func (s *S) TestReport(c *C) { for _, test := range reportTests { var err error - report, err := slicer.NewReport("/base/") + report, err := manifest.NewReport("/base/") c.Assert(err, IsNil) for _, si := range test.add { err = report.Add(si.slice, &si.entry) @@ -290,6 +290,12 @@ func (s *S) TestReport(c *C) { } func (s *S) TestRootRelativePath(c *C) { - _, err := slicer.NewReport("../base/") + _, err := manifest.NewReport("../base/") c.Assert(err, ErrorMatches, `cannot use relative path for report root: "../base/"`) } + +func (s *S) TestRootOnlySlash(c *C) { + report, err := manifest.NewReport("/") + c.Assert(err, IsNil) + c.Assert(report.Root, Equals, "/") +} diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index f164efd8..e76b07af 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -5,6 +5,7 @@ import ( "bytes" "fmt" "io" + "io/fs" "os" "path/filepath" "slices" @@ -12,13 +13,18 @@ import ( "strings" "syscall" + "github.com/klauspost/compress/zstd" + "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" + "github.com/canonical/chisel/internal/manifest" "github.com/canonical/chisel/internal/scripts" "github.com/canonical/chisel/internal/setup" ) +const manifestMode fs.FileMode = 0644 + type RunOptions struct { Selection *setup.Selection Archives map[string]archive.Archive @@ -65,7 +71,7 @@ func (cc *contentChecker) checkKnown(path string) error { return err } -func Run(options *RunOptions) (*Report, error) { +func Run(options *RunOptions) error { oldUmask := syscall.Umask(0) defer func() { syscall.Umask(oldUmask) @@ -75,7 +81,7 @@ func Run(options *RunOptions) (*Report, error) { if !filepath.IsAbs(targetDir) { dir, err := os.Getwd() if err != nil { - return nil, fmt.Errorf("cannot obtain current directory: %w", err) + return fmt.Errorf("cannot obtain current directory: %w", err) } targetDir = filepath.Join(dir, targetDir) } @@ -89,10 +95,10 @@ func Run(options *RunOptions) (*Report, error) { archiveName := options.Selection.Release.Packages[slice.Package].Archive archive := options.Archives[archiveName] if archive == nil { - return nil, fmt.Errorf("archive %q not defined", archiveName) + return fmt.Errorf("archive %q not defined", archiveName) } if !archive.Exists(slice.Package) { - return nil, fmt.Errorf("slice package %q missing from archive", slice.Package) + return fmt.Errorf("slice package %q missing from archive", slice.Package) } archives[slice.Package] = archive extractPackage = make(map[string][]deb.ExtractInfo) @@ -145,26 +151,27 @@ func Run(options *RunOptions) (*Report, error) { // Fetch all packages, using the selection order. packages := make(map[string]io.ReadCloser) + var pkgInfos []*archive.PackageInfo for _, slice := range options.Selection.Slices { if packages[slice.Package] != nil { continue } - reader, err := archives[slice.Package].Fetch(slice.Package) + reader, info, err := archives[slice.Package].Fetch(slice.Package) if err != nil { - return nil, err + return err } defer reader.Close() packages[slice.Package] = reader + pkgInfos = append(pkgInfos, info) } // When creating content, record if a path is known and whether they are // listed as until: mutate in all the slices that reference them. knownPaths := map[string]pathData{} addKnownPath(knownPaths, "/", pathData{}) - - report, err := NewReport(targetDir) + report, err := manifest.NewReport(targetDir) if err != nil { - return nil, fmt.Errorf("internal error: cannot create report: %w", err) + return fmt.Errorf("internal error: cannot create report: %w", err) } // Creates the filesystem entry and adds it to the report. It also updates @@ -235,38 +242,60 @@ func Run(options *RunOptions) (*Report, error) { reader.Close() packages[slice.Package] = nil if err != nil { - return nil, err + return err } } - // Create new content not coming from packages. - done := make(map[string]bool) + // Create new content not extracted from packages, e.g. TextPath or DirPath + // with {make: true}. The only exception is the manifest which will be created + // later. + // First group them by their relative path. Then create them and attribute + // them to the appropriate slices. + relPaths := map[string][]*setup.Slice{} for _, slice := range options.Selection.Slices { arch := archives[slice.Package].Options().Arch for relPath, pathInfo := range slice.Contents { if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, arch) { continue } - if done[relPath] || pathInfo.Kind == setup.CopyPath || pathInfo.Kind == setup.GlobPath { + if pathInfo.Kind == setup.CopyPath || pathInfo.Kind == setup.GlobPath || + pathInfo.Kind == setup.GeneratePath { continue } - done[relPath] = true - data := pathData{ - until: pathInfo.Until, - mutable: pathInfo.Mutable, - } - addKnownPath(knownPaths, relPath, data) - targetPath := filepath.Join(targetDir, relPath) - entry, err := createFile(targetPath, pathInfo) - if err != nil { - return nil, err + relPaths[relPath] = append(relPaths[relPath], slice) + } + } + for relPath, slices := range relPaths { + until := setup.UntilMutate + for _, slice := range slices { + if slice.Contents[relPath].Until == setup.UntilNone { + until = setup.UntilNone + break } + } + // It is okay to take the first pathInfo because the release has been + // validated when read and there are no conflicts. The only field that + // was not checked was until because it is not used for conflict + // validation. + pathInfo := slices[0].Contents[relPath] + pathInfo.Until = until + data := pathData{ + until: pathInfo.Until, + mutable: pathInfo.Mutable, + } + addKnownPath(knownPaths, relPath, data) + targetPath := filepath.Join(targetDir, relPath) + entry, err := createFile(targetPath, pathInfo) + if err != nil { + return err + } - // Do not add paths with "until: mutate". - if pathInfo.Until != setup.UntilMutate { + // Do not add paths with "until: mutate". + if pathInfo.Until != setup.UntilMutate { + for _, slice := range slices { err = report.Add(slice, entry) if err != nil { - return nil, err + return err } } } @@ -291,16 +320,59 @@ func Run(options *RunOptions) (*Report, error) { } err := scripts.Run(&opts) if err != nil { - return nil, fmt.Errorf("slice %s: %w", slice, err) + return fmt.Errorf("slice %s: %w", slice, err) } } err = removeAfterMutate(targetDir, knownPaths) if err != nil { - return nil, err + return err } - return report, nil + return generateManifests(targetDir, options.Selection, report, pkgInfos) +} + +func generateManifests(targetDir string, selection *setup.Selection, + report *manifest.Report, pkgInfos []*archive.PackageInfo) error { + manifestSlices := manifest.FindPaths(selection.Slices) + if len(manifestSlices) == 0 { + // Nothing to do. + return nil + } + var writers []io.Writer + for relPath, slices := range manifestSlices { + logf("Generating manifest at %s...", relPath) + absPath := filepath.Join(targetDir, relPath) + createOptions := &fsutil.CreateOptions{ + Path: absPath, + Mode: manifestMode, + MakeParents: true, + } + writer, info, err := fsutil.CreateWriter(createOptions) + if err != nil { + return err + } + defer writer.Close() + writers = append(writers, writer) + for _, slice := range slices { + err := report.Add(slice, info) + if err != nil { + return err + } + } + } + w, err := zstd.NewWriter(io.MultiWriter(writers...)) + if err != nil { + return err + } + defer w.Close() + writeOptions := &manifest.WriteOptions{ + PackageInfo: pkgInfos, + Selection: selection.Slices, + Report: report, + } + err = manifest.Write(writeOptions, w) + return err } // removeAfterMutate removes entries marked with until: mutate. A path is marked diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 4f9dee55..8035ef0d 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2,18 +2,19 @@ package slicer_test import ( "archive/tar" - "bytes" "fmt" - "io" "io/fs" "os" + "path" "path/filepath" "sort" "strings" + "github.com/klauspost/compress/zstd" . "gopkg.in/check.v1" "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/manifest" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" "github.com/canonical/chisel/internal/testutil" @@ -24,15 +25,16 @@ var ( ) type slicerTest struct { - summary string - arch string - release map[string]string - pkgs map[string][]byte - slices []setup.SliceKey - hackopt func(c *C, opts *slicer.RunOptions) - filesystem map[string]string - report map[string]string - error string + summary string + arch string + release map[string]string + pkgs map[string]testutil.TestPackage + slices []setup.SliceKey + hackopt func(c *C, opts *slicer.RunOptions) + filesystem map[string]string + manifestPaths map[string]string + manifestPkgs map[string]string + error string } var packageEntries = map[string][]testutil.TarEntry{ @@ -97,7 +99,7 @@ var slicerTests = []slicerTest{{ "/other-dir/": "dir 0755", "/other-dir/file": "symlink ../dir/file", }, - report: map[string]string{ + manifestPaths: map[string]string{ "/dir/file": "file 0644 cc55e2ec {test-package_myslice}", "/dir/file-copy": "file 0644 cc55e2ec {test-package_myslice}", "/dir/foo/bar/": "dir 01777 {test-package_myslice}", @@ -122,7 +124,7 @@ var slicerTests = []slicerTest{{ "/dir/nested/other-file": "file 0644 6b86b273", "/dir/other-file": "file 0644 63d5dd49", }, - report: map[string]string{ + manifestPaths: map[string]string{ "/dir/nested/other-file": "file 0644 6b86b273 {test-package_myslice}", "/dir/other-file": "file 0644 63d5dd49 {test-package_myslice}", }, @@ -143,7 +145,7 @@ var slicerTests = []slicerTest{{ "/parent/": "dir 01777", // This is the magic. "/parent/new": "file 0644 5b41362b", }, - report: map[string]string{ + manifestPaths: map[string]string{ "/parent/new": "file 0644 5b41362b {test-package_myslice}", }, }, { @@ -164,7 +166,7 @@ var slicerTests = []slicerTest{{ "/parent/permissions/": "dir 0764", // This is the magic. "/parent/permissions/new": "file 0644 5b41362b", }, - report: map[string]string{ + manifestPaths: map[string]string{ "/parent/permissions/new": "file 0644 5b41362b {test-package_myslice}", }, }, { @@ -184,15 +186,12 @@ var slicerTests = []slicerTest{{ "/parent/": "dir 01777", // This is the magic. "/parent/new/": "dir 0755", }, - report: map[string]string{ + manifestPaths: map[string]string{ "/parent/new/": "dir 0755 {test-package_myslice}", }, }, { summary: "Create new file using glob and preserve parent directory permissions", slices: []setup.SliceKey{{"test-package", "myslice"}}, - pkgs: map[string][]byte{ - "test-package": testutil.PackageData["test-package"], - }, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -208,7 +207,7 @@ var slicerTests = []slicerTest{{ "/parent/permissions/": "dir 0764", // This is the magic. "/parent/permissions/file": "file 0755 722c14b3", }, - report: map[string]string{ + manifestPaths: map[string]string{ "/parent/": "dir 01777 {test-package_myslice}", "/parent/permissions/": "dir 0764 {test-package_myslice}", "/parent/permissions/file": "file 0755 722c14b3 {test-package_myslice}", @@ -239,7 +238,7 @@ var slicerTests = []slicerTest{{ "/dir/nested/copy-1": "file 0644 84237a05", "/dir/nested/copy-3": "file 0644 84237a05", }, - report: map[string]string{ + manifestPaths: map[string]string{ "/dir/nested/copy-1": "file 0644 84237a05 {test-package_myslice}", "/dir/nested/copy-3": "file 0644 84237a05 {test-package_myslice}", "/dir/text-file-1": "file 0644 5b41362b {test-package_myslice}", @@ -248,9 +247,11 @@ var slicerTests = []slicerTest{{ }, { summary: "Copyright is installed", slices: []setup.SliceKey{{"test-package", "myslice"}}, - pkgs: map[string][]byte{ - // Add the copyright entries to the package. - "test-package": testutil.MustMakeDeb(append(testutil.TestPackageEntries, testPackageCopyrightEntries...)), + pkgs: map[string]testutil.TestPackage{ + "test-package": { + // Add the copyright entries to the package. + Data: testutil.MustMakeDeb(append(testutil.TestPackageEntries, testPackageCopyrightEntries...)), + }, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -271,7 +272,7 @@ var slicerTests = []slicerTest{{ "/usr/share/doc/test-package/": "dir 0755", "/usr/share/doc/test-package/copyright": "file 0644 c2fca2aa", }, - report: map[string]string{ + manifestPaths: map[string]string{ "/dir/file": "file 0644 cc55e2ec {test-package_myslice}", }, }, { @@ -279,9 +280,13 @@ var slicerTests = []slicerTest{{ slices: []setup.SliceKey{ {"test-package", "myslice"}, {"other-package", "myslice"}}, - pkgs: map[string][]byte{ - "test-package": testutil.PackageData["test-package"], - "other-package": testutil.PackageData["other-package"], + pkgs: map[string]testutil.TestPackage{ + "test-package": { + Data: testutil.PackageData["test-package"], + }, + "other-package": { + Data: testutil.PackageData["other-package"], + }, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -308,7 +313,7 @@ var slicerTests = []slicerTest{{ "/file": "file 0644 fc02ca0e", "/foo/": "dir 0755", }, - report: map[string]string{ + manifestPaths: map[string]string{ "/foo/": "dir 0755 {test-package_myslice}", "/dir/file": "file 0644 cc55e2ec {test-package_myslice}", "/bar/": "dir 0755 {other-package_myslice}", @@ -319,14 +324,18 @@ var slicerTests = []slicerTest{{ slices: []setup.SliceKey{ {"implicit-parent", "myslice"}, {"explicit-dir", "myslice"}}, - pkgs: map[string][]byte{ - "implicit-parent": testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0755, "./dir/"), - testutil.Reg(0644, "./dir/file", "random"), - }), - "explicit-dir": testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(01777, "./dir/"), - }), + pkgs: map[string]testutil.TestPackage{ + "implicit-parent": { + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Dir(0755, "./dir/"), + testutil.Reg(0644, "./dir/file", "random"), + }), + }, + "explicit-dir": { + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Dir(01777, "./dir/"), + }), + }, }, release: map[string]string{ "slices/mydir/implicit-parent.yaml": ` @@ -348,7 +357,7 @@ var slicerTests = []slicerTest{{ "/dir/": "dir 01777", "/dir/file": "file 0644 a441b15f", }, - report: map[string]string{ + manifestPaths: map[string]string{ "/dir/": "dir 01777 {explicit-dir_myslice}", "/dir/file": "file 0644 a441b15f {implicit-parent_myslice}", }, @@ -357,9 +366,13 @@ var slicerTests = []slicerTest{{ slices: []setup.SliceKey{ {"test-package", "myslice"}, {"other-package", "myslice"}}, - pkgs: map[string][]byte{ - "test-package": testutil.PackageData["test-package"], - "other-package": testutil.PackageData["other-package"], + pkgs: map[string]testutil.TestPackage{ + "test-package": { + Data: testutil.PackageData["test-package"], + }, + "other-package": { + Data: testutil.PackageData["other-package"], + }, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -380,11 +393,8 @@ var slicerTests = []slicerTest{{ filesystem: map[string]string{ "/textFile": "file 0644 c6c83d10", }, - report: map[string]string{ - // Note: This is the only case where two slices can declare the same - // file without conflicts. - // TODO which slice(s) should own the file. - "/textFile": "file 0644 c6c83d10 {other-package_myslice}", + manifestPaths: map[string]string{ + "/textFile": "file 0644 c6c83d10 {other-package_myslice,test-package_myslice}", }, }, { summary: "Script: write a file", @@ -404,7 +414,7 @@ var slicerTests = []slicerTest{{ "/dir/": "dir 0755", "/dir/text-file": "file 0644 d98cf53e", }, - report: map[string]string{ + manifestPaths: map[string]string{ "/dir/text-file": "file 0644 5b41362b d98cf53e {test-package_myslice}", }, }, { @@ -429,7 +439,7 @@ var slicerTests = []slicerTest{{ "/foo/": "dir 0755", "/foo/text-file-2": "file 0644 5b41362b", }, - report: map[string]string{ + manifestPaths: map[string]string{ "/dir/text-file-1": "file 0644 5b41362b {test-package_myslice}", "/foo/text-file-2": "file 0644 d98cf53e 5b41362b {test-package_myslice}", }, @@ -454,7 +464,7 @@ var slicerTests = []slicerTest{{ "/foo/": "dir 0755", "/foo/text-file-2": "file 0644 5b41362b", }, - report: map[string]string{ + manifestPaths: map[string]string{ "/foo/text-file-2": "file 0644 d98cf53e 5b41362b {test-package_myslice}", }, }, { @@ -474,7 +484,7 @@ var slicerTests = []slicerTest{{ "/dir/": "dir 0755", "/other-dir/": "dir 0755", }, - report: map[string]string{}, + manifestPaths: map[string]string{}, }, { summary: "Script: 'until' does not remove non-empty directories", slices: []setup.SliceKey{{"test-package", "myslice"}}, @@ -493,7 +503,7 @@ var slicerTests = []slicerTest{{ "/dir/nested/": "dir 0755", "/dir/nested/file-copy": "file 0644 cc55e2ec", }, - report: map[string]string{ + manifestPaths: map[string]string{ "/dir/nested/file-copy": "file 0644 cc55e2ec {test-package_myslice}", }, }, { @@ -514,7 +524,7 @@ var slicerTests = []slicerTest{{ "/dir/": "dir 0755", "/dir/text-file": "file 0644 5b41362b", }, - report: map[string]string{ + manifestPaths: map[string]string{ "/dir/text-file": "file 0644 5b41362b {test-package_myslice}", }, }, { @@ -694,9 +704,13 @@ var slicerTests = []slicerTest{{ }, { summary: "Duplicate copyright symlink is ignored", slices: []setup.SliceKey{{"copyright-symlink-openssl", "bins"}}, - pkgs: map[string][]byte{ - "copyright-symlink-openssl": testutil.MustMakeDeb(packageEntries["copyright-symlink-openssl"]), - "copyright-symlink-libssl3": testutil.MustMakeDeb(packageEntries["copyright-symlink-libssl3"]), + pkgs: map[string]testutil.TestPackage{ + "copyright-symlink-openssl": { + Data: testutil.MustMakeDeb(packageEntries["copyright-symlink-openssl"]), + }, + "copyright-symlink-libssl3": { + Data: testutil.MustMakeDeb(packageEntries["copyright-symlink-libssl3"]), + }, }, release: map[string]string{ "slices/mydir/copyright-symlink-libssl3.yaml": ` @@ -784,12 +798,15 @@ var slicerTests = []slicerTest{{ /dir/nested/file: `, }, + hackopt: func(c *C, opts *slicer.RunOptions) { + delete(opts.Archives, "foo") + }, filesystem: map[string]string{ "/dir/": "dir 0755", "/dir/nested/": "dir 0755", "/dir/nested/file": "file 0644 84237a05", }, - report: map[string]string{ + manifestPaths: map[string]string{ "/dir/nested/file": "file 0644 84237a05 {test-package_myslice}", }, }, { @@ -823,7 +840,7 @@ var slicerTests = []slicerTest{{ "/other-dir/": "dir 0755", "/other-dir/file": "symlink ../dir/file", }, - report: map[string]string{ + manifestPaths: map[string]string{ "/dir/file": "file 0644 cc55e2ec {test-package_myslice1}", "/dir/file-copy": "file 0644 cc55e2ec {test-package_myslice1}", "/dir/foo/bar/": "dir 01777 {test-package_myslice1}", @@ -864,7 +881,7 @@ var slicerTests = []slicerTest{{ "/dir/other-file": "file 0644 63d5dd49", "/dir/several/levels/deep/": "dir 0755", }, - report: map[string]string{ + manifestPaths: map[string]string{ "/dir/": "dir 0755 {test-package_myslice2}", "/dir/file": "file 0644 cc55e2ec {test-package_myslice2}", "/dir/nested/": "dir 0755 {test-package_myslice2}", @@ -910,7 +927,7 @@ var slicerTests = []slicerTest{{ "/dir/several/levels/deep/": "dir 0755", "/dir/several/levels/deep/file": "file 0644 6bc26dff", }, - report: map[string]string{ + manifestPaths: map[string]string{ "/dir/": "dir 0755 {test-package_myslice1}", "/dir/file": "file 0644 cc55e2ec {test-package_myslice1}", "/dir/nested/": "dir 0755 {test-package_myslice1}", @@ -956,7 +973,7 @@ var slicerTests = []slicerTest{{ "/dir/several/levels/deep/": "dir 0755", "/dir/several/levels/deep/file": "file 0644 6bc26dff", }, - report: map[string]string{ + manifestPaths: map[string]string{ "/dir/": "dir 0755 {test-package_myslice1}", "/dir/file": "file 0644 cc55e2ec {test-package_myslice1}", "/dir/nested/": "dir 0755 {test-package_myslice1}", @@ -994,7 +1011,7 @@ var slicerTests = []slicerTest{{ "/dir/": "dir 0755", "/dir/file": "file 0644 cc55e2ec", }, - report: map[string]string{ + manifestPaths: map[string]string{ "/dir/file": "file 0644 cc55e2ec {test-package_myslice2}", }, }, { @@ -1019,21 +1036,125 @@ var slicerTests = []slicerTest{{ content.read("/dir/file") `, }, - filesystem: map[string]string{}, - report: map[string]string{}, + filesystem: map[string]string{}, + manifestPaths: map[string]string{}, +}, { + summary: "Content not created in packages with until:mutate on one and reading from script", + slices: []setup.SliceKey{ + {"test-package", "myslice1"}, + {"test-package", "myslice2"}, + }, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice1: + contents: + /file: {text: foo, until: mutate} + mutate: | + content.read("/file") + myslice2: + contents: + /file: {text: foo} + mutate: | + content.read("/file") + `, + }, + filesystem: map[string]string{"/file": "file 0644 2c26b46b"}, + manifestPaths: map[string]string{"/file": "file 0644 2c26b46b {test-package_myslice1,test-package_myslice2}"}, +}, { + summary: "Install two packages, both are recorded", + slices: []setup.SliceKey{ + {"test-package", "myslice"}, + {"other-package", "myslice"}, + }, + pkgs: map[string]testutil.TestPackage{ + "test-package": { + Name: "test-package", + Hash: "h1", + Version: "v1", + Arch: "a1", + Data: testutil.PackageData["test-package"], + }, + "other-package": { + Name: "other-package", + Hash: "h2", + Version: "v2", + Arch: "a2", + Data: testutil.PackageData["other-package"], + }, + }, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + `, + "slices/mydir/other-package.yaml": ` + package: other-package + slices: + myslice: + contents: + `, + }, + manifestPkgs: map[string]string{ + "test-package": "test-package v1 a1 h1", + "other-package": "other-package v2 a2 h2", + }, +}, { + summary: "Two packages, only one is selected and recorded", + slices: []setup.SliceKey{ + {"test-package", "myslice"}, + }, + pkgs: map[string]testutil.TestPackage{ + "test-package": { + Name: "test-package", + Hash: "h1", + Version: "v1", + Arch: "a1", + Data: testutil.PackageData["test-package"], + }, + "other-package": { + Name: "other-package", + Hash: "h2", + Version: "v2", + Arch: "a2", + Data: testutil.PackageData["other-package"], + }, + }, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + `, + "slices/mydir/other-package.yaml": ` + package: other-package + slices: + myslice: + contents: + `, + }, + manifestPkgs: map[string]string{ + "test-package": "test-package v1 a1 h1", + }, }, { summary: "Relative paths are properly trimmed during extraction", slices: []setup.SliceKey{{"test-package", "myslice"}}, - pkgs: map[string][]byte{ - "test-package": testutil.MustMakeDeb([]testutil.TarEntry{ - // This particular path starting with "/foo" is chosen to test for - // a particular bug; which appeared due to the usage of - // strings.TrimLeft() instead strings.TrimPrefix() to determine a - // relative path. Since TrimLeft takes in a cutset instead of a - // prefix, the desired relative path was not produced. - // See https://github.com/canonical/chisel/pull/145. - testutil.Dir(0755, "./foo-bar/"), - }), + pkgs: map[string]testutil.TestPackage{ + "test-package": { + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + // This particular path starting with "/foo" is chosen to test for + // a particular bug; which appeared due to the usage of + // strings.TrimLeft() instead strings.TrimPrefix() to determine a + // relative path. Since TrimLeft takes in a cutset instead of a + // prefix, the desired relative path was not produced. + // See https://github.com/canonical/chisel/pull/145. + testutil.Dir(0755, "./foo-bar/"), + }), + }, }, hackopt: func(c *C, opts *slicer.RunOptions) { opts.TargetDir = filepath.Join(filepath.Clean(opts.TargetDir), "foo") @@ -1051,6 +1172,28 @@ var slicerTests = []slicerTest{{ content.list("/foo-bar/") `, }, +}, { + summary: "Producing a manifest is not mandatory", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + hackopt: func(c *C, opts *slicer.RunOptions) { + // Remove the manifest slice that the tests add automatically. + var index int + for i, slice := range opts.Selection.Slices { + if slice.Name == "manifest" { + index = i + break + } + } + opts.Selection.Slices = append(opts.Selection.Slices[:index], opts.Selection.Slices[index+1:]...) + }, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + `, + }, }} var defaultChiselYaml = ` @@ -1066,27 +1209,6 @@ var defaultChiselYaml = ` armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` ` -type testArchive struct { - options archive.Options - pkgs map[string][]byte -} - -func (a *testArchive) Options() *archive.Options { - return &a.options -} - -func (a *testArchive) Fetch(pkg string) (io.ReadCloser, error) { - if data, ok := a.pkgs[pkg]; ok { - return io.NopCloser(bytes.NewBuffer(data)), nil - } - return nil, fmt.Errorf("attempted to open %q package", pkg) -} - -func (a *testArchive) Exists(pkg string) bool { - _, ok := a.pkgs[pkg] - return ok -} - func (s *S) TestRun(c *C) { // Run tests for format chisel-v1. runSlicerTests(c, slicerTests) @@ -1114,12 +1236,20 @@ func runSlicerTests(c *C, tests []slicerTest) { c.Logf("Summary: %s", test.summary) if _, ok := test.release["chisel.yaml"]; !ok { - test.release["chisel.yaml"] = string(defaultChiselYaml) + test.release["chisel.yaml"] = defaultChiselYaml } - if test.pkgs == nil { - test.pkgs = map[string][]byte{ - "test-package": testutil.PackageData["test-package"], + test.pkgs = map[string]testutil.TestPackage{ + "test-package": { + Data: testutil.PackageData["test-package"], + }, + } + } + for pkgName, pkg := range test.pkgs { + if pkg.Name == "" { + // We need to add the name for the manifest validation. + pkg.Name = pkgName + test.pkgs[pkgName] = pkg } } @@ -1135,86 +1265,161 @@ func runSlicerTests(c *C, tests []slicerTest) { release, err := setup.ReadRelease(releaseDir) c.Assert(err, IsNil) + // Create a manifest slice and add it to the selection. + manifestPackage := test.slices[0].Package + manifestPath := "/chisel-data/manifest.wall" + release.Packages[manifestPackage].Slices["manifest"] = &setup.Slice{ + Package: manifestPackage, + Name: "manifest", + Essential: nil, + Contents: map[string]setup.PathInfo{ + "/chisel-data/**": { + Kind: "generate", + Generate: "manifest", + }, + }, + Scripts: setup.SliceScripts{}, + } + slices = append(slices, setup.SliceKey{ + Package: manifestPackage, + Slice: "manifest", + }) + selection, err := setup.Select(release, slices) c.Assert(err, IsNil) archives := map[string]archive.Archive{} for name, setupArchive := range release.Archives { - archive := &testArchive{ - options: archive.Options{ + archive := &testutil.TestArchive{ + Opts: archive.Options{ Label: setupArchive.Name, Version: setupArchive.Version, Suites: setupArchive.Suites, Components: setupArchive.Components, Arch: test.arch, }, - pkgs: test.pkgs, + Packages: test.pkgs, } archives[name] = archive } - targetDir := c.MkDir() options := slicer.RunOptions{ Selection: selection, Archives: archives, - TargetDir: targetDir, + TargetDir: c.MkDir(), } if test.hackopt != nil { test.hackopt(c, &options) } - report, err := slicer.Run(&options) - if test.error == "" { - c.Assert(err, IsNil) - } else { + err = slicer.Run(&options) + if test.error != "" { c.Assert(err, ErrorMatches, test.error) continue } + c.Assert(err, IsNil) + + if test.filesystem == nil && test.manifestPaths == nil && test.manifestPkgs == nil { + continue + } + mfest := readManifest(c, options.TargetDir, manifestPath) + // Assert state of final filesystem. if test.filesystem != nil { - c.Assert(testutil.TreeDump(targetDir), DeepEquals, test.filesystem) + filesystem := testutil.TreeDump(options.TargetDir) + c.Assert(filesystem["/chisel-data/"], Not(HasLen), 0) + c.Assert(filesystem[manifestPath], Not(HasLen), 0) + delete(filesystem, "/chisel-data/") + delete(filesystem, manifestPath) + c.Assert(filesystem, DeepEquals, test.filesystem) } - if test.report != nil { - c.Assert(treeDumpReport(report), DeepEquals, test.report) + // Assert state of the files recorded in the manifest. + if test.manifestPaths != nil { + pathsDump, err := treeDumpManifestPaths(mfest) + c.Assert(err, IsNil) + c.Assert(pathsDump[manifestPath], Not(HasLen), 0) + delete(pathsDump, manifestPath) + c.Assert(pathsDump, DeepEquals, test.manifestPaths) + } + + // Assert state of the packages recorded in the manifest. + if test.manifestPkgs != nil { + pkgsDump, err := dumpManifestPkgs(mfest) + c.Assert(err, IsNil) + c.Assert(pkgsDump, DeepEquals, test.manifestPkgs) } } } } -// treeDumpReport returns the file information in the same format as -// [testutil.TreeDump] with the added slices that have installed each path. -func treeDumpReport(report *slicer.Report) map[string]string { +func treeDumpManifestPaths(mfest *manifest.Manifest) (map[string]string, error) { result := make(map[string]string) - for _, entry := range report.Entries { - fperm := entry.Mode.Perm() - if entry.Mode&fs.ModeSticky != 0 { - fperm |= 01000 - } + err := mfest.IteratePaths("", func(path *manifest.Path) error { var fsDump string - switch entry.Mode.Type() { - case fs.ModeDir: - fsDump = fmt.Sprintf("dir %#o", fperm) - case fs.ModeSymlink: - fsDump = fmt.Sprintf("symlink %s", entry.Link) - case 0: // Regular - if entry.Size == 0 { - fsDump = fmt.Sprintf("file %#o empty", entry.Mode.Perm()) - } else if entry.FinalHash != "" { - fsDump = fmt.Sprintf("file %#o %s %s", fperm, entry.Hash[:8], entry.FinalHash[:8]) + switch { + case strings.HasSuffix(path.Path, "/"): + fsDump = fmt.Sprintf("dir %s", path.Mode) + case path.Link != "": + fsDump = fmt.Sprintf("symlink %s", path.Link) + default: // Regular + if path.Size == 0 { + fsDump = fmt.Sprintf("file %s empty", path.Mode) + } else if path.FinalSHA256 != "" { + fsDump = fmt.Sprintf("file %s %s %s", path.Mode, path.SHA256[:8], path.FinalSHA256[:8]) } else { - fsDump = fmt.Sprintf("file %#o %s", fperm, entry.Hash[:8]) + fsDump = fmt.Sprintf("file %s %s", path.Mode, path.SHA256[:8]) } - default: - panic(fmt.Errorf("unknown file type %d: %s", entry.Mode.Type(), entry.Path)) } - // append {slice1, ..., sliceN} to the end of the entry dump. - slicesStr := make([]string, 0, len(entry.Slices)) - for slice := range entry.Slices { - slicesStr = append(slicesStr, slice.String()) + // append {slice1, ..., sliceN} to the end of the path dump. + slicesStr := make([]string, 0, len(path.Slices)) + for _, slice := range path.Slices { + slicesStr = append(slicesStr, slice) } sort.Strings(slicesStr) - result[entry.Path] = fmt.Sprintf("%s {%s}", fsDump, strings.Join(slicesStr, ",")) + result[path.Path] = fmt.Sprintf("%s {%s}", fsDump, strings.Join(slicesStr, ",")) + return nil + }) + if err != nil { + return nil, err + } + return result, nil +} + +func dumpManifestPkgs(mfest *manifest.Manifest) (map[string]string, error) { + result := map[string]string{} + err := mfest.IteratePackages(func(pkg *manifest.Package) error { + result[pkg.Name] = fmt.Sprintf("%s %s %s %s", pkg.Name, pkg.Version, pkg.Arch, pkg.Digest) + return nil + }) + if err != nil { + return nil, err } - return result + return result, nil +} + +func readManifest(c *C, targetDir, manifestPath string) *manifest.Manifest { + f, err := os.Open(path.Join(targetDir, manifestPath)) + c.Assert(err, IsNil) + defer f.Close() + r, err := zstd.NewReader(f) + c.Assert(err, IsNil) + defer r.Close() + mfest, err := manifest.Read(r) + c.Assert(err, IsNil) + err = manifest.Validate(mfest) + c.Assert(err, IsNil) + + // Assert that the mode of the manifest.wall file matches the one recorded + // in the manifest itself. + s, err := os.Stat(path.Join(targetDir, manifestPath)) + c.Assert(err, IsNil) + c.Assert(s.Mode(), Equals, fs.FileMode(0644)) + err = mfest.IteratePaths(manifestPath, func(p *manifest.Path) error { + c.Assert(p.Mode, Equals, fmt.Sprintf("%#o", fs.FileMode(0644))) + return nil + }) + c.Assert(err, IsNil) + + return mfest } diff --git a/internal/testutil/archive.go b/internal/testutil/archive.go new file mode 100644 index 00000000..77b945cc --- /dev/null +++ b/internal/testutil/archive.go @@ -0,0 +1,58 @@ +package testutil + +import ( + "bytes" + "fmt" + "io" + + "github.com/canonical/chisel/internal/archive" +) + +type TestArchive struct { + Opts archive.Options + Packages map[string]TestPackage +} + +type TestPackage struct { + Name string + Version string + Hash string + Arch string + Data []byte +} + +func (a *TestArchive) Options() *archive.Options { + return &a.Opts +} + +func (a *TestArchive) Fetch(pkgName string) (io.ReadCloser, *archive.PackageInfo, error) { + pkg, ok := a.Packages[pkgName] + if !ok { + return nil, nil, fmt.Errorf("cannot find package %q in archive", pkgName) + } + info := &archive.PackageInfo{ + Name: pkg.Name, + Version: pkg.Version, + SHA256: pkg.Hash, + Arch: pkg.Arch, + } + return io.NopCloser(bytes.NewBuffer(pkg.Data)), info, nil +} + +func (a *TestArchive) Exists(pkg string) bool { + _, ok := a.Packages[pkg] + return ok +} + +func (a *TestArchive) Info(pkgName string) (*archive.PackageInfo, error) { + pkg, ok := a.Packages[pkgName] + if !ok { + return nil, fmt.Errorf("cannot find package %q in archive", pkgName) + } + return &archive.PackageInfo{ + Name: pkg.Name, + Version: pkg.Version, + SHA256: pkg.Hash, + Arch: pkg.Arch, + }, nil +} diff --git a/internal/testutil/treedump.go b/internal/testutil/treedump.go index 26aa164d..83b275e3 100644 --- a/internal/testutil/treedump.go +++ b/internal/testutil/treedump.go @@ -78,7 +78,7 @@ func TreeDumpEntry(entry *fsutil.Entry) string { if entry.Size == 0 { return fmt.Sprintf("file %#o empty", entry.Mode.Perm()) } else { - return fmt.Sprintf("file %#o %s", fperm, entry.Hash[:8]) + return fmt.Sprintf("file %#o %s", fperm, entry.SHA256[:8]) } default: panic(fmt.Errorf("unknown file type %d: %s", entry.Mode.Type(), entry.Path)) diff --git a/spread.yaml b/spread.yaml index 510025a1..a8de3d36 100644 --- a/spread.yaml +++ b/spread.yaml @@ -29,7 +29,7 @@ backends: docker run -e usr=$SPREAD_SYSTEM_USERNAME -e pass=$SPREAD_SYSTEM_PASSWORD --name $SPREAD_SYSTEM -d $image sh -c ' set -x apt update - apt install -y openssh-server sudo + apt install -y openssh-server sudo zstd jq mkdir /run/sshd useradd -rm -d /home/ubuntu -s /bin/bash -g root -G sudo -u 1000 ubuntu echo "$usr:$pass" | chpasswd diff --git a/tests/use-a-custom-chisel-release/task.yaml b/tests/use-a-custom-chisel-release/task.yaml index e124be13..2345bf0b 100644 --- a/tests/use-a-custom-chisel-release/task.yaml +++ b/tests/use-a-custom-chisel-release/task.yaml @@ -20,12 +20,23 @@ execute: | myslice: contents: /etc/: + manifest: + contents: + /chisel/**: {generate: manifest} EOF - chisel cut --release $chisel_release --root $rootfs_folder base-files_myslice + chisel cut --release $chisel_release --root $rootfs_folder base-files_myslice base-files_manifest # make sure $rootfs_folder is not empty ls ${rootfs_folder}/* # make sure the custom slice has been installed test -d ${rootfs_folder}/etc + test -d ${rootfs_folder}/chisel + test -f ${rootfs_folder}/chisel/manifest.wall + + # make sure the manifest can be decompressed and each line is valid json + zstd -d ${rootfs_folder}/chisel/manifest.wall -o ${rootfs_folder}/chisel/manifest.jsonwall + while read line; do + echo $line | jq + done < ${rootfs_folder}/chisel/manifest.jsonwall