diff --git a/shared/api/event_lifecycle.go b/shared/api/event_lifecycle.go index 6e8cfc9..4e3a5fe 100644 --- a/shared/api/event_lifecycle.go +++ b/shared/api/event_lifecycle.go @@ -49,6 +49,7 @@ const ( EventLifecycleInstanceMetadataTemplateRetrieved = "instance-metadata-template-retrieved" EventLifecycleInstanceMetadataUpdated = "instance-metadata-updated" EventLifecycleInstancePaused = "instance-paused" + EventLifecycleInstanceReady = "instance-ready" EventLifecycleInstanceRenamed = "instance-renamed" EventLifecycleInstanceRestarted = "instance-restarted" EventLifecycleInstanceRestored = "instance-restored" diff --git a/shared/idmap/idmapset_linux.go b/shared/idmap/idmapset_linux.go index 27ffe86..56038d6 100644 --- a/shared/idmap/idmapset_linux.go +++ b/shared/idmap/idmapset_linux.go @@ -591,7 +591,7 @@ func (set *IdmapSet) doUidshiftIntoContainer(dir string, testmode bool, how stri tmp := filepath.Dir(dir) tmp, err := filepath.EvalSymlinks(tmp) if err != nil { - return fmt.Errorf("Expand symlinks: %w", err) + return fmt.Errorf("Failed expanding symlinks of %q: %w", tmp, err) } dir = filepath.Join(tmp, filepath.Base(dir)) @@ -607,7 +607,7 @@ func (set *IdmapSet) doUidshiftIntoContainer(dir string, testmode bool, how stri return filepath.SkipDir } - intUid, intGid, _, _, inode, nlink, err := shared.GetFileStat(path) + intUID, intGID, _, _, inode, nlink, err := shared.GetFileStat(path) if err != nil { return err } @@ -623,8 +623,8 @@ func (set *IdmapSet) doUidshiftIntoContainer(dir string, testmode bool, how stri hardLinks = append(hardLinks, inode) } - uid := int64(intUid) - gid := int64(intGid) + uid := int64(intUID) + gid := int64(intGID) caps := []byte{} var newuid, newgid int64 @@ -661,15 +661,15 @@ func (set *IdmapSet) doUidshiftIntoContainer(dir string, testmode bool, how stri // Shift capabilities if len(caps) != 0 { - rootUid := int64(0) + rootUID := int64(0) if how == "in" { - rootUid, _ = set.ShiftIntoNs(0, 0) + rootUID, _ = set.ShiftIntoNs(0, 0) } if how != "in" || atomic.LoadInt32(&VFS3Fscaps) == VFS3FscapsSupported { - err = SetCaps(path, caps, rootUid) + err = SetCaps(path, caps, rootUID) if err != nil { - logger.Warnf("Unable to set file capabilities on %s", path) + logger.Warnf("Unable to set file capabilities on %q: %v", path, err) } } } diff --git a/shared/logger/log.go b/shared/logger/log.go index f9c841d..d27c234 100644 --- a/shared/logger/log.go +++ b/shared/logger/log.go @@ -13,7 +13,7 @@ import ( // Setup a basic empty logger on init. func init() { - logger := logrus.StandardLogger() + logger := logrus.New() logger.SetOutput(ioutil.Discard) Log = newWrapper(logger) @@ -21,7 +21,7 @@ func init() { // InitLogger intializes a full logging instance. func InitLogger(filepath string, syslogName string, verbose bool, debug bool, hook logrus.Hook) error { - logger := logrus.StandardLogger() + logger := logrus.New() logger.Level = logrus.DebugLevel logger.SetOutput(io.Discard) diff --git a/shared/osarch/architectures.go b/shared/osarch/architectures.go index 8c3850e..14b9c5f 100644 --- a/shared/osarch/architectures.go +++ b/shared/osarch/architectures.go @@ -18,12 +18,16 @@ const ( ARCH_64BIT_MIPS = 10 ARCH_32BIT_RISCV_LITTLE_ENDIAN = 11 ARCH_64BIT_RISCV_LITTLE_ENDIAN = 12 + ARCH_32BIT_ARMV6_LITTLE_ENDIAN = 13 + ARCH_32BIT_ARMV8_LITTLE_ENDIAN = 14 ) var architectureNames = map[int]string{ ARCH_32BIT_INTEL_X86: "i686", ARCH_64BIT_INTEL_X86: "x86_64", + ARCH_32BIT_ARMV6_LITTLE_ENDIAN: "armv6l", ARCH_32BIT_ARMV7_LITTLE_ENDIAN: "armv7l", + ARCH_32BIT_ARMV8_LITTLE_ENDIAN: "armv8l", ARCH_64BIT_ARMV8_LITTLE_ENDIAN: "aarch64", ARCH_32BIT_POWERPC_BIG_ENDIAN: "ppc", ARCH_64BIT_POWERPC_BIG_ENDIAN: "ppc64", @@ -38,7 +42,9 @@ var architectureNames = map[int]string{ var architectureAliases = map[int][]string{ ARCH_32BIT_INTEL_X86: {"i386", "i586", "386", "x86", "generic_32"}, ARCH_64BIT_INTEL_X86: {"amd64", "generic_64"}, - ARCH_32BIT_ARMV7_LITTLE_ENDIAN: {"armel", "armhf", "arm", "armhfp", "armv7a_hardfp", "armv7", "armv7a_vfpv3_hardfp"}, + ARCH_32BIT_ARMV6_LITTLE_ENDIAN: {"armel", "arm"}, + ARCH_32BIT_ARMV7_LITTLE_ENDIAN: {"armhf", "armhfp", "armv7a_hardfp", "armv7", "armv7a_vfpv3_hardfp"}, + ARCH_32BIT_ARMV8_LITTLE_ENDIAN: {}, ARCH_64BIT_ARMV8_LITTLE_ENDIAN: {"arm64", "arm64_generic"}, ARCH_32BIT_POWERPC_BIG_ENDIAN: {"powerpc"}, ARCH_64BIT_POWERPC_BIG_ENDIAN: {"powerpc64", "ppc64"}, @@ -52,7 +58,9 @@ var architectureAliases = map[int][]string{ var architecturePersonalities = map[int]string{ ARCH_32BIT_INTEL_X86: "linux32", ARCH_64BIT_INTEL_X86: "linux64", + ARCH_32BIT_ARMV6_LITTLE_ENDIAN: "linux32", ARCH_32BIT_ARMV7_LITTLE_ENDIAN: "linux32", + ARCH_32BIT_ARMV8_LITTLE_ENDIAN: "linux32", ARCH_64BIT_ARMV8_LITTLE_ENDIAN: "linux64", ARCH_32BIT_POWERPC_BIG_ENDIAN: "linux32", ARCH_64BIT_POWERPC_BIG_ENDIAN: "linux64", @@ -67,8 +75,10 @@ var architecturePersonalities = map[int]string{ var architectureSupportedPersonalities = map[int][]int{ ARCH_32BIT_INTEL_X86: {}, ARCH_64BIT_INTEL_X86: {ARCH_32BIT_INTEL_X86}, - ARCH_32BIT_ARMV7_LITTLE_ENDIAN: {}, - ARCH_64BIT_ARMV8_LITTLE_ENDIAN: {ARCH_32BIT_ARMV7_LITTLE_ENDIAN}, + ARCH_32BIT_ARMV6_LITTLE_ENDIAN: {}, + ARCH_32BIT_ARMV7_LITTLE_ENDIAN: {ARCH_32BIT_ARMV6_LITTLE_ENDIAN}, + ARCH_32BIT_ARMV8_LITTLE_ENDIAN: {ARCH_32BIT_ARMV6_LITTLE_ENDIAN, ARCH_32BIT_ARMV7_LITTLE_ENDIAN}, + ARCH_64BIT_ARMV8_LITTLE_ENDIAN: {ARCH_32BIT_ARMV6_LITTLE_ENDIAN, ARCH_32BIT_ARMV7_LITTLE_ENDIAN, ARCH_32BIT_ARMV8_LITTLE_ENDIAN}, ARCH_32BIT_POWERPC_BIG_ENDIAN: {}, ARCH_64BIT_POWERPC_BIG_ENDIAN: {ARCH_32BIT_POWERPC_BIG_ENDIAN}, ARCH_64BIT_POWERPC_LITTLE_ENDIAN: {}, diff --git a/shared/subprocess/bgpm_test.go b/shared/subprocess/bgpm_test.go index 0680b5d..282aa68 100644 --- a/shared/subprocess/bgpm_test.go +++ b/shared/subprocess/bgpm_test.go @@ -21,7 +21,7 @@ func TestSignalHandling(t *testing.T) { t.Error("Failed process creation: ", err) } - err = p.Start() + err = p.Start(context.Background()) if err != nil { t.Error("Failed to start process ", err) } @@ -90,7 +90,7 @@ func TestStopRestart(t *testing.T) { t.Error("Failed process creation: ", err) } - err = p.Start() + err = p.Start(context.Background()) if err != nil { t.Error("Failed to start process: ", err) } @@ -110,12 +110,12 @@ func TestStopRestart(t *testing.T) { t.Error("Failed to import process: ", err) } - err = p.Start() + err = p.Start(context.Background()) if err != nil { t.Error("Failed to start process: ", err) } - err = p.Restart() + err = p.Restart(context.Background()) if err != nil { t.Error("Failed to restart process: ", err) } @@ -144,7 +144,7 @@ func TestProcessStartWaitExit(t *testing.T) { t.Error("Failed process creation: ", err) } - err = p.Start() + err = p.Start(context.Background()) if err != nil { t.Error("Failed to start process: ", err) } diff --git a/shared/subprocess/manager.go b/shared/subprocess/manager.go index b675883..8cb7922 100644 --- a/shared/subprocess/manager.go +++ b/shared/subprocess/manager.go @@ -32,18 +32,14 @@ func NewProcess(name string, args []string, stdoutPath string, stderrPath string } } - p, err := NewProcessWithFds(name, args, nil, stdout, stderr) - if err != nil { - return nil, fmt.Errorf("Error when creating process object: %w", err) - } - + p := NewProcessWithFds(name, args, nil, stdout, stderr) p.closeFds = true return p, nil } // NewProcessWithFds is a constructor for a process object. Represents a process with argument config. Returns an address to process. -func NewProcessWithFds(name string, args []string, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) (*Process, error) { +func NewProcessWithFds(name string, args []string, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) *Process { proc := Process{ Name: name, Args: args, @@ -52,7 +48,7 @@ func NewProcessWithFds(name string, args []string, stdin io.ReadCloser, stdout i Stderr: stderr, } - return &proc, nil + return &proc } // ImportProcess imports a saved process into a subprocess object. diff --git a/shared/subprocess/proc.go b/shared/subprocess/proc.go index efa9736..8f50f21 100644 --- a/shared/subprocess/proc.go +++ b/shared/subprocess/proc.go @@ -106,22 +106,22 @@ func (p *Process) Stop() error { } // Start will start the given process object. -func (p *Process) Start() error { - return p.start(nil) +func (p *Process) Start(ctx context.Context) error { + return p.start(ctx, nil) } // StartWithFiles will start the given process object with extra file descriptors. -func (p *Process) StartWithFiles(fds []*os.File) error { - return p.start(fds) +func (p *Process) StartWithFiles(ctx context.Context, fds []*os.File) error { + return p.start(ctx, fds) } -func (p *Process) start(fds []*os.File) error { +func (p *Process) start(ctx context.Context, fds []*os.File) error { var cmd *exec.Cmd if p.Apparmor != "" && p.hasApparmor() { - cmd = exec.Command("aa-exec", append([]string{"-p", p.Apparmor, p.Name}, p.Args...)...) + cmd = exec.CommandContext(ctx, "aa-exec", append([]string{"-p", p.Apparmor, p.Name}, p.Args...)...) } else { - cmd = exec.Command(p.Name, p.Args...) + cmd = exec.CommandContext(ctx, p.Name, p.Args...) } cmd.Stdout = p.Stdout @@ -189,13 +189,13 @@ func (p *Process) start(fds []*os.File) error { } // Restart stop and starts the given process object. -func (p *Process) Restart() error { +func (p *Process) Restart(ctx context.Context) error { err := p.Stop() if err != nil { return fmt.Errorf("Unable to stop process: %w", err) } - err = p.Start() + err = p.Start(ctx) if err != nil { return fmt.Errorf("Unable to start process: %w", err) } diff --git a/shared/util.go b/shared/util.go index d1b4893..4955e2c 100644 --- a/shared/util.go +++ b/shared/util.go @@ -37,6 +37,9 @@ const HTTPSDefaultPort = 8443 const HTTPDefaultPort = 8080 const HTTPSMetricsDefaultPort = 9100 +// HTTPSStorageBucketsDefaultPort the default port for the storage buckets listener. +const HTTPSStorageBucketsDefaultPort = 9000 + // URLEncode encodes a path and query parameters to a URL. func URLEncode(path string, query map[string]string) (string, error) { u, err := url.Parse(path) diff --git a/shared/util_linux.go b/shared/util_linux.go index 4c16941..071f060 100644 --- a/shared/util_linux.go +++ b/shared/util_linux.go @@ -16,6 +16,7 @@ import ( "sync/atomic" "unsafe" + "github.com/pkg/xattr" "golang.org/x/sys/unix" "github.com/lxc/lxd/lxd/revert" @@ -68,106 +69,27 @@ func SetSize(fd int, width int, height int) (err error) { return nil } -// This uses ssize_t llistxattr(const char *path, char *list, size_t size); to -// handle symbolic links (should it in the future be possible to set extended -// attributed on symlinks): If path is a symbolic link the extended attributes -// associated with the link itself are retrieved. -func llistxattr(path string, list []byte) (sz int, err error) { - var _p0 *byte - _p0, err = unix.BytePtrFromString(path) +// GetAllXattr retrieves all extended attributes associated with a file, directory or symbolic link. +func GetAllXattr(path string) (map[string]string, error) { + xattrNames, err := xattr.LList(path) if err != nil { - return - } - - var _p1 unsafe.Pointer - if len(list) > 0 { - _p1 = unsafe.Pointer(&list[0]) - } else { - _p1 = unsafe.Pointer(nil) - } - - r0, _, e1 := unix.Syscall(unix.SYS_LLISTXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(list))) - sz = int(r0) - if e1 != 0 { - err = e1 - } - - return -} - -// GetAllXattr retrieves all extended attributes associated with a file, -// directory or symbolic link. -func GetAllXattr(path string) (xattrs map[string]string, err error) { - // Call llistxattr() twice: First, to determine the size of the buffer - // we need to allocate to store the extended attributes, second, to - // actually store the extended attributes in the buffer. Also, check if - // the size/number of extended attributes hasn't increased between the - // two calls. - pre, err := llistxattr(path, nil) - if err != nil || pre < 0 { - if err == unix.EOPNOTSUPP { + // Some filesystems don't support llistxattr() for various reasons. + // Interpret this as a set of no xattrs, instead of an error. + if errors.Is(err, unix.EOPNOTSUPP) { return nil, nil } - return nil, err + return nil, fmt.Errorf("Failed getting extended attributes from %q: %w", path, err) } - if pre == 0 { - return nil, nil - } - - dest := make([]byte, pre) - - post, err := llistxattr(path, dest) - if err != nil || post < 0 { - return nil, err - } - - if post > pre { - return nil, fmt.Errorf("Extended attribute list size increased from %d to %d during retrieval", pre, post) - } - - split := strings.Split(string(dest), "\x00") - if split == nil { - return nil, fmt.Errorf("No valid extended attribute key found") - } - // *listxattr functions return a list of names as an unordered array - // of null-terminated character strings (attribute names are separated - // by null bytes ('\0')), like this: user.name1\0system.name1\0user.name2\0 - // Since we split at the '\0'-byte the last element of the slice will be - // the empty string. We remove it: - if split[len(split)-1] == "" { - split = split[:len(split)-1] - } - - xattrs = make(map[string]string, len(split)) - - for _, x := range split { - xattr := string(x) - // Call Getxattr() twice: First, to determine the size of the - // buffer we need to allocate to store the extended attributes, - // second, to actually store the extended attributes in the - // buffer. Also, check if the size of the extended attribute - // hasn't increased between the two calls. - pre, err = unix.Getxattr(path, xattr, nil) - if err != nil || pre < 0 { - return nil, err - } - - dest = make([]byte, pre) - post := 0 - if pre > 0 { - post, err = unix.Getxattr(path, xattr, dest) - if err != nil || post < 0 { - return nil, err - } - } - - if post > pre { - return nil, fmt.Errorf("Extended attribute '%s' size increased from %d to %d during retrieval", xattr, pre, post) + var xattrs = make(map[string]string, len(xattrNames)) + for _, xattrName := range xattrNames { + value, err := xattr.LGet(path, xattrName) + if err != nil { + return nil, fmt.Errorf("Failed getting %q extended attribute from %q: %w", xattrName, path, err) } - xattrs[xattr] = string(dest) + xattrs[xattrName] = string(value) } return xattrs, nil diff --git a/shared/validate/validate.go b/shared/validate/validate.go index 90c2220..95e6d9e 100644 --- a/shared/validate/validate.go +++ b/shared/validate/validate.go @@ -2,6 +2,8 @@ package validate import ( "bytes" + "crypto/x509" + "encoding/pem" "fmt" "net" "net/url" @@ -268,19 +270,6 @@ func IsNetworkAddress(value string) error { return nil } -// IsNetworkAddressList validates a comma delimited list of IPv4 or IPv6 addresses. -func IsNetworkAddressList(value string) error { - for _, v := range strings.Split(value, ",") { - v = strings.TrimSpace(v) - err := IsNetworkAddress(v) - if err != nil { - return err - } - } - - return nil -} - // IsNetwork validates an IP network CIDR string. func IsNetwork(value string) error { ip, subnet, err := net.ParseCIDR(value) @@ -295,18 +284,6 @@ func IsNetwork(value string) error { return nil } -// IsNetworkList validates a comma delimited list of IP network CIDR strings. -func IsNetworkList(value string) error { - for _, network := range strings.Split(value, ",") { - err := IsNetwork(strings.TrimSpace(network)) - if err != nil { - return err - } - } - - return nil -} - // IsNetworkAddressCIDR validates an IP addresss string in CIDR format. func IsNetworkAddressCIDR(value string) error { _, _, err := net.ParseCIDR(value) @@ -363,19 +340,6 @@ func IsNetworkV4(value string) error { return nil } -// IsNetworkV4List validates a comma delimited list of IPv4 CIDR strings. -func IsNetworkV4List(value string) error { - for _, network := range strings.Split(value, ",") { - network = strings.TrimSpace(network) - err := IsNetworkV4(network) - if err != nil { - return err - } - } - - return nil -} - // IsNetworkAddressV4 validates an IPv4 addresss string. func IsNetworkAddressV4(value string) error { ip := net.ParseIP(value) @@ -386,19 +350,6 @@ func IsNetworkAddressV4(value string) error { return nil } -// IsNetworkAddressV4List validates a comma delimited list of IPv4 addresses. -func IsNetworkAddressV4List(value string) error { - for _, v := range strings.Split(value, ",") { - v = strings.TrimSpace(v) - err := IsNetworkAddressV4(v) - if err != nil { - return err - } - } - - return nil -} - // IsNetworkAddressCIDRV4 validates an IPv4 addresss string in CIDR format. func IsNetworkAddressCIDRV4(value string) error { ip, subnet, err := net.ParseCIDR(value) @@ -434,18 +385,6 @@ func IsNetworkRangeV4(value string) error { return nil } -// IsNetworkRangeV4List validates a comma delimited list of IPv4 ranges. -func IsNetworkRangeV4List(value string) error { - for _, ipRange := range strings.Split(value, ",") { - err := IsNetworkRangeV4(strings.TrimSpace(ipRange)) - if err != nil { - return err - } - } - - return nil -} - // IsNetworkV6 validates an IPv6 CIDR string. func IsNetworkV6(value string) error { ip, subnet, err := net.ParseCIDR(value) @@ -464,19 +403,6 @@ func IsNetworkV6(value string) error { return nil } -// IsNetworkV6List validates a comma delimited list of IPv6 CIDR strings. -func IsNetworkV6List(value string) error { - for _, network := range strings.Split(value, ",") { - network = strings.TrimSpace(network) - err := IsNetworkV6(network) - if err != nil { - return err - } - } - - return nil -} - // IsNetworkAddressV6 validates an IPv6 addresss string. func IsNetworkAddressV6(value string) error { ip := net.ParseIP(value) @@ -487,18 +413,6 @@ func IsNetworkAddressV6(value string) error { return nil } -// IsNetworkAddressV6List validates a comma delimited list of IPv6 addresses. -func IsNetworkAddressV6List(value string) error { - for _, v := range strings.Split(value, ",") { - v = strings.TrimSpace(v) - err := IsNetworkAddressV6(v) - if err != nil { - return err - } - } - return nil -} - // IsNetworkAddressCIDRV6 validates an IPv6 addresss string in CIDR format. func IsNetworkAddressCIDRV6(value string) error { ip, subnet, err := net.ParseCIDR(value) @@ -534,18 +448,6 @@ func IsNetworkRangeV6(value string) error { return nil } -// IsNetworkRangeV6List validates a comma delimited list of IPv6 ranges. -func IsNetworkRangeV6List(value string) error { - for _, ipRange := range strings.Split(value, ",") { - err := IsNetworkRangeV6(strings.TrimSpace(ipRange)) - if err != nil { - return err - } - } - - return nil -} - // IsNetworkVLAN validates a VLAN ID. func IsNetworkVLAN(value string) error { vlanID, err := strconv.Atoi(value) @@ -755,6 +657,18 @@ func IsListenAddress(allowDNS bool, allowWildcard bool, requirePort bool) func(v } } +// IsX509Certificate checks if the value is a valid x509 PEM Certificate. +func IsX509Certificate(value string) error { + certBlock, _ := pem.Decode([]byte(value)) + if certBlock == nil { + return fmt.Errorf("Invalid certificate") + } + + _, err := x509.ParseCertificate(certBlock.Bytes) + + return err +} + // IsAbsFilePath checks if value is an absolute file path. func IsAbsFilePath(value string) error { if !filepath.IsAbs(value) { diff --git a/shared/version/api.go b/shared/version/api.go index e17c145..53fc21d 100644 --- a/shared/version/api.go +++ b/shared/version/api.go @@ -343,6 +343,11 @@ var APIExtensions = []string{ "storage_volumes_all_projects", "metrics_memory_oom_total", "storage_buckets", + "storage_buckets_create_credentials", + "metrics_cpu_effective_total", + "projects_networks_restricted_access", + "storage_buckets_local", + "loki", } // APIExtensionsCount returns the number of available API extensions. diff --git a/shared/version/flex.go b/shared/version/flex.go index 9f84f98..db6f784 100644 --- a/shared/version/flex.go +++ b/shared/version/flex.go @@ -1,4 +1,4 @@ package version // Version contains the LXD version number. -var Version = "5.5" +var Version = "5.6"