feat: parse generate property in sdf

README.md

@@ -269,11 +269,16 @@ have additional information for identifying the kind of content to expect:
  which are only available for certain architectures. Example:
  `/usr/bin/hello: {arch: amd64}` will instruct Chisel to extract and install
  the "/usr/bin/hello" file only when chiselling an amd64 filesystem.
+ - **generate**: accepts a `manifest` value to instruct Chisel to generate the
+ manifest files in the directory. Example: `/var/lib/chisel/**:{generate:
+ manifest}`. NOTE: the provided path has to be of the form
+ `/slashed/path/to/dir/**` and no wildcards can appear apart from the trailing
+ `**`.
 
 ## TODO
 
 - [ ] Preserve ownerships when possible
-- [ ] GPG signature checking for archives
+- [x] GPG signature checking for archives
 - [ ] Use a fake server for the archive tests
 - [ ] Functional tests
 

internal/setup/setup.go

@@ -60,11 +60,12 @@ type SliceScripts struct {
 type PathKind string
 
 const (
-	DirPath     PathKind = "dir"
-	CopyPath    PathKind = "copy"
-	GlobPath    PathKind = "glob"
-	TextPath    PathKind = "text"
-	SymlinkPath PathKind = "symlink"
+	DirPath      PathKind = "dir"
+	CopyPath     PathKind = "copy"
+	GlobPath     PathKind = "glob"
+	TextPath     PathKind = "text"
+	SymlinkPath  PathKind = "symlink"
+	GeneratePath PathKind = "generate"
 
 	// TODO Maybe in the future, for binary support.
 	//Base64Path PathKind = "base64"
@@ -77,14 +78,25 @@ const (
 	UntilMutate PathUntil = "mutate"
 )
 
+type GenerateKind string
+
+const (
+	GenerateNone     GenerateKind = ""
+	GenerateManifest GenerateKind = "manifest"
+)
+
 type PathInfo struct {
+	// Disable ==.
+	_ [0]func()
+
 	Kind PathKind
 	Info string
 	Mode uint
 
-	Mutable bool
-	Until   PathUntil
-	Arch    []string
+	Mutable  bool
+	Until    PathUntil
+	Arch     []string
+	Generate GenerateKind
 }
 
 // SameContent returns whether the path has the same content properties as some
@@ -95,7 +107,8 @@ func (pi *PathInfo) SameContent(other *PathInfo) bool {
 	return (pi.Kind == other.Kind &&
 		pi.Info == other.Info &&
 		pi.Mode == other.Mode &&
-		pi.Mutable == other.Mutable)
+		pi.Mutable == other.Mutable &&
+		pi.Generate == other.Generate)
 }
 
 type SliceKey struct {
@@ -141,15 +154,37 @@ func ReadRelease(dir string) (*Release, error) {
 
 func (r *Release) validate() error {
 	keys := []SliceKey(nil)
-	paths := make(map[string]*Slice)
-	globs := make(map[string]*Slice)
 
-	// Check for info conflicts and prepare for following checks.
+	type pathSlice struct {
+		path  string
+		slice *Slice
+	}
+	allPaths := make(map[string]*Slice)
+	// globs contains all the paths of kind GlobPath.
+	var globs []pathSlice
+	// copies contains all the paths of kind CopyPath.
+	var copies []pathSlice
+	// generates contains all the paths of kind GeneratePath.
+	var generates []pathSlice
+	// rest contains the paths which are not in globs, copies or generates.
+	var rest []pathSlice
+
+	// Check for info conflicts and prepare for following checks. A conflict
+	// means that two slices attempt to extract different files or directories
+	// to the same location.
+	// Conflict validation is done without downloading packages which means that
+	// if we are extracting content from different packages to the same location
+	// we cannot be sure that it will be the same. On the contrary, content
+	// extracted from the same package will never conflict because it is
+	// guaranteed to be the same.
+	// The above also means that generated content (e.g. text files, directories
+	// with make:true) will always conflict with extracted content, because we
+	// cannot validate that they are the same without downloading the package.
 	for _, pkg := range r.Packages {
 		for _, new := range pkg.Slices {
 			keys = append(keys, SliceKey{pkg.Name, new.Name})
 			for newPath, newInfo := range new.Contents {
-				if old, ok := paths[newPath]; ok {
+				if old, ok := allPaths[newPath]; ok {
 					oldInfo := old.Contents[newPath]
 					if !newInfo.SameContent(&oldInfo) || (newInfo.Kind == CopyPath || newInfo.Kind == GlobPath) && new.Package != old.Package {
 						if old.Package > new.Package || old.Package == new.Package && old.Name > new.Name {
@@ -158,35 +193,63 @@ func (r *Release) validate() error {
 						return fmt.Errorf("slices %s and %s conflict on %s", old, new, newPath)
 					}
 				} else {
-					if newInfo.Kind == GlobPath {
-						globs[newPath] = new
+					switch newInfo.Kind {
+					case GlobPath:
+						globs = append(globs, pathSlice{path: newPath, slice: new})
+					case CopyPath:
+						copies = append(copies, pathSlice{path: newPath, slice: new})
+					case GeneratePath:
+						generates = append(generates, pathSlice{path: newPath, slice: new})
+					default:
+						rest = append(rest, pathSlice{path: newPath, slice: new})
 					}
-					paths[newPath] = new
+					allPaths[newPath] = new
 				}
 			}
 		}
 	}
 
-	// Check for cycles.
-	_, err := order(r.Packages, keys)
-	if err != nil {
-		return err
+	checkConflict := func(old, new pathSlice) error {
+		if strdist.GlobPath(new.path, old.path) {
+			if (old.slice.Package > new.slice.Package) || (old.slice.Package == new.slice.Package && old.slice.Name > new.slice.Name) {
+				old, new = new, old
+			}
+			return fmt.Errorf("slices %s and %s conflict on %s and %s", old.slice, new.slice, old.path, new.path)
+		}
+		return nil
 	}
-
-	// Check for glob conflicts.
-	for newPath, new := range globs {
-		for oldPath, old := range paths {
-			if new.Package == old.Package {
+	for _, new := range globs {
+		// TODO replace with slices.Concat once we upgrade to go 1.22.
+		for _, old := range append(globs, copies...) {
+			if new.slice.Package == old.slice.Package {
 				continue
 			}
-			if strdist.GlobPath(newPath, oldPath) {
-				if old.Package > new.Package || old.Package == new.Package && old.Name > new.Name {
-					old, oldPath, new, newPath = new, newPath, old, oldPath
-				}
-				return fmt.Errorf("slices %s and %s conflict on %s and %s", old, new, oldPath, newPath)
+			err := checkConflict(old, new)
+			if err != nil {
+				return err
+			}
+		}
+		for _, old := range rest {
+			err := checkConflict(old, new)
+			if err != nil {
+				return err
 			}
 		}
-		paths[newPath] = new
+	}
+	for _, new := range generates {
+		// TODO replace with slices.Concat once we upgrade to go 1.22.
+		for _, old := range append(copies, append(globs, rest...)...) {
+			err := checkConflict(old, new)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	// Check for cycles.
+	_, err := order(r.Packages, keys)
+	if err != nil {
+		return err
 	}
 
 	return nil
@@ -357,8 +420,9 @@ type yamlPath struct {
 	Symlink string  `yaml:"symlink"`
 	Mutable bool    `yaml:"mutable"`
 
-	Until PathUntil `yaml:"until"`
-	Arch  yamlArch  `yaml:"arch"`
+	Until    PathUntil    `yaml:"until"`
+	Arch     yamlArch     `yaml:"arch"`
+	Generate GenerateKind `yaml:"generate"`
 }
 
 // SameContent returns whether the path has the same content properties as some
@@ -583,7 +647,19 @@ func parsePackage(baseDir, pkgName, pkgPath string, data []byte) (*Package, erro
 			var mutable bool
 			var until PathUntil
 			var arch []string
-			if strings.ContainsAny(contPath, "*?") {
+			var generate GenerateKind
+			if yamlPath != nil && yamlPath.Generate != "" {
+				zeroPathGenerate := zeroPath
+				zeroPathGenerate.Generate = yamlPath.Generate
+				if !yamlPath.SameContent(&zeroPathGenerate) || yamlPath.Until != UntilNone {
+					return nil, fmt.Errorf("slice %s_%s path %s has invalid generate options",
+						pkgName, sliceName, contPath)
+				}
+				if _, err := validateGeneratePath(contPath); err != nil {
+					return nil, fmt.Errorf("slice %s_%s has invalid generate path: %s", pkgName, sliceName, err)
+				}
+				kinds = append(kinds, GeneratePath)
+			} else if strings.ContainsAny(contPath, "*?") {
 				if yamlPath != nil {
 					if !yamlPath.SameContent(&zeroPath) {
 						return nil, fmt.Errorf("slice %s_%s path %s has invalid wildcard options",
@@ -595,6 +671,7 @@ func parsePackage(baseDir, pkgName, pkgPath string, data []byte) (*Package, erro
 			if yamlPath != nil {
 				mode = yamlPath.Mode
 				mutable = yamlPath.Mutable
+				generate = yamlPath.Generate
 				if yamlPath.Dir {
 					if !strings.HasSuffix(contPath, "/") {
 						return nil, fmt.Errorf("slice %s_%s path %s must end in / for 'make' to be valid",
@@ -644,12 +721,13 @@ func parsePackage(baseDir, pkgName, pkgPath string, data []byte) (*Package, erro
 				return nil, fmt.Errorf("slice %s_%s mutable is not a regular file: %s", pkgName, sliceName, contPath)
 			}
 			slice.Contents[contPath] = PathInfo{
-				Kind:    kinds[0],
-				Info:    info,
-				Mode:    mode,
-				Mutable: mutable,
-				Until:   until,
-				Arch:    arch,
+				Kind:     kinds[0],
+				Info:     info,
+				Mode:     mode,
+				Mutable:  mutable,
+				Until:    until,
+				Arch:     arch,
+				Generate: generate,
 			}
 		}
 
@@ -659,6 +737,22 @@ func parsePackage(baseDir, pkgName, pkgPath string, data []byte) (*Package, erro
 	return &pkg, err
 }
 
+// validateGeneratePath validates that the path follows the following format:
+//   - /slashed/path/to/dir/**
+//
+// Wildcard characters can only appear at the end as **, and the path before
+// those wildcards must be a directory.
+func validateGeneratePath(path string) (string, error) {
+	if !strings.HasSuffix(path, "/**") {
+		return "", fmt.Errorf("%s does not end with /**", path)
+	}
+	dirPath := strings.TrimSuffix(path, "**")
+	if strings.ContainsAny(dirPath, "*?") {
+		return "", fmt.Errorf("%s contains wildcard characters in addition to trailing **", path)
+	}
+	return dirPath, nil
+}
+
 func stripBase(baseDir, path string) string {
 	// Paths must be clean for this to work correctly.
 	return strings.TrimPrefix(path, baseDir+string(filepath.Separator))
@@ -691,9 +785,17 @@ func Select(release *Release, slices []SliceKey) (*Selection, error) {
 					}
 					return nil, fmt.Errorf("slices %s and %s conflict on %s", old, new, newPath)
 				}
-				continue
+			} else {
+				paths[newPath] = new
+			}
+			// An invalid "generate" value should only throw an error if that
+			// particular slice is selected. Hence, the check is here.
+			switch newInfo.Generate {
+			case GenerateNone, GenerateManifest:
+			default:
+				return nil, fmt.Errorf("slice %s has invalid 'generate' for path %s: %q, consider an update if available",
+					new, newPath, newInfo.Generate)
 			}
-			paths[newPath] = new
 		}
 	}