Skip to content

Commit

Permalink
Enable multipath + iSCSI as installation disk
Browse files Browse the repository at this point in the history 
This commit removes the disablement of multipath+iSCSI, updates the required kernel arguments, and extends the `nicReapplyManifest` workaround previously added for iSCSI to also support multipath+iSCSI.
  • Loading branch information
@linoyaslan
linoyaslan committed Jan 16, 2025
1 parent 2e03b51 commit c6e41ad
Show file tree
Hide file tree
Showing 3 changed files with 59 additions and 49 deletions.
45 changes: 10 additions & 35 deletions internal/hardware/validator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,12 @@ import (
)

const (
tooSmallDiskTemplate = "Disk is too small (disk only has %s, but %s are required)"
wrongDriveTypeTemplate = "Drive type is %s, it must be one of %s."
wrongMultipathTypeTemplate = "Multipath device has path of type %s, it must be %s"
iSCSIWithMultipathHolder = "iSCSI disk with a multipath holder is not eligible"
wrongISCSINetworkTemplate = "iSCSI host IP %s is the same as host IP, they must be different"
tooSmallDiskTemplate = "Disk is too small (disk only has %s, but %s are required)"
wrongDriveTypeTemplate = "Drive type is %s, it must be one of %s."
wrongMultipathTypeTemplate = "Multipath device has path of type %s, it must be %s"
iSCSIWithMultipathHolder = "iSCSI disk with a multipath holder is not eligible"
wrongISCSINetworkTemplate = "iSCSI host IP %s is the same as host IP, they must be different"
errsInIscsiDisableMultipathInstallation = "Installation in multipath is not possible due to errors in at least one iSCSI disk."
)

//go:generate mockgen -source=validator.go -package=hardware -destination=mock_validator.go
Expand Down Expand Up @@ -147,25 +148,20 @@ func (v *validator) DiskIsEligible(ctx context.Context, disk *models.Disk, infra
fmt.Sprintf(wrongDriveTypeTemplate, disk.DriveType, strings.Join(v.getValidDeviceStorageTypes(hostArchitecture, clusterVersion), ", ")))
}

// We only allow multipath if all paths are FC
if disk.DriveType == models.DriveTypeMultipath {
for _, inventoryDisk := range inventory.Disks {
if lo.Contains(strings.Split(inventoryDisk.Holders, ","), disk.Name) {
if inventoryDisk.DriveType != models.DriveTypeFC {
if strings.Contains(inventoryDisk.Holders, disk.Name) {
// We only allow multipath if all paths are FC/ iSCSI
if inventoryDisk.DriveType != models.DriveTypeFC && inventoryDisk.DriveType != models.DriveTypeISCSI {
notEligibleReasons = append(notEligibleReasons,
fmt.Sprintf(wrongMultipathTypeTemplate, inventoryDisk.DriveType, string(models.DriveTypeFC)))
fmt.Sprintf(wrongMultipathTypeTemplate, inventoryDisk.DriveType, fmt.Sprintf("%s or %s", string(models.DriveTypeFC), string(models.DriveTypeISCSI))))
break
}
}
}
}

if disk.DriveType == models.DriveTypeISCSI {
err := areISCSIHoldersValid(disk, inventory)
if err != nil {
notEligibleReasons = append(notEligibleReasons, err.Error())
}

// Check if network is configured properly to install on iSCSI boot drive
err = isISCSINetworkingValid(disk, inventory)
if err != nil {
Expand All @@ -176,27 +172,6 @@ func (v *validator) DiskIsEligible(ctx context.Context, disk *models.Disk, infra
return notEligibleReasons, nil
}

// Validate holders of iSCSI disk. We do not allow iSCSI disk with multipath holder.
func areISCSIHoldersValid(disk *models.Disk, inventory *models.Inventory) error {
multipathDiskNamesMap := make(map[string]struct{})
for _, inventoryDisk := range inventory.Disks {
if inventoryDisk.DriveType == models.DriveTypeMultipath {
multipathDiskNamesMap[inventoryDisk.Name] = struct{}{}
}
}

// Check if the iSCSI disk has any holders that are multipath disks
holders := strings.Split(disk.Holders, ",")
for _, holder := range holders {
if _, exists := multipathDiskNamesMap[holder]; exists {
return fmt.Errorf(iSCSIWithMultipathHolder)
}
}

return nil

}

// isISCSINetworkingValid checks if the iSCSI disk is not connected through the
// default network interface. The default network interface is the interface
// which is used by the default gateway.
Expand Down
34 changes: 28 additions & 6 deletions internal/host/hostcommands/install_cmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"k8s.io/utils/strings/slices"
"net"
"net/netip"
"strings"
Expand Down Expand Up @@ -300,7 +301,10 @@ func constructHostInstallerArgs(cluster *common.Cluster, host *models.Host, inve
// append kargs depending on installation drive type
installationDisk := hostutil.GetDiskByInstallationPath(inventory.Disks, hostutil.GetHostInstallationPath(host))
if installationDisk != nil {
installerArgs = appendMultipathArgs(installerArgs, installationDisk)
installerArgs, err = appendMultipathArgs(installerArgs, installationDisk, inventory, hasUserConfiguredIP)
if err != nil {
return "", err
}
installerArgs, err = appendISCSIArgs(installerArgs, installationDisk, inventory, hasUserConfiguredIP)
if err != nil {
return "", err
Expand Down Expand Up @@ -370,7 +374,9 @@ func appendISCSIArgs(installerArgs []string, installationDisk *models.Disk, inve
}

// enable iSCSI on boot
installerArgs = append(installerArgs, "--append-karg", "rd.iscsi.firmware=1")
if !slices.Contains(installerArgs, "rd.iscsi.firmware=1") {
installerArgs = append(installerArgs, "--append-karg", "rd.iscsi.firmware=1")
}

if hasUserConfiguredIP {
return installerArgs, nil
Expand All @@ -391,16 +397,32 @@ func appendISCSIArgs(installerArgs []string, installationDisk *models.Disk, inve
if iSCSIHostIP.Is6() {
dhcp = "dhcp6"
}
installerArgs = append(installerArgs, "--append-karg", fmt.Sprintf("ip=%s:%s", nic.Name, dhcp))

if !slices.Contains(installerArgs, fmt.Sprintf("ip=%s:%s", nic.Name, dhcp)) {
installerArgs = append(installerArgs, "--append-karg", fmt.Sprintf("ip=%s:%s", nic.Name, dhcp))
}

return installerArgs, nil
}

func appendMultipathArgs(installerArgs []string, installationDisk *models.Disk) []string {
func appendMultipathArgs(installerArgs []string, installationDisk *models.Disk, inventory *models.Inventory, hasUserConfiguredIP bool) ([]string, error) {
if installationDisk.DriveType != models.DriveTypeMultipath {
return installerArgs
return installerArgs, nil
}
return append(installerArgs, "--append-karg", "root=/dev/disk/by-label/dm-mpath-root", "--append-karg", "rw", "--append-karg", "rd.multipath=default")

installerArgs = append(installerArgs, "--append-karg", "root=/dev/disk/by-label/dm-mpath-root", "--append-karg", "rw", "--append-karg", "rd.multipath=default")

for _, disk := range inventory.Disks {
if disk.DriveType == models.DriveTypeISCSI && strings.Contains(disk.Holders, installationDisk.Name) {
iSCSIInstallerArgs, err := appendISCSIArgs(installerArgs, disk, inventory, hasUserConfiguredIP)
if err != nil {
return nil, err
}
installerArgs = append(installerArgs, iSCSIInstallerArgs...)
break
}
}
return installerArgs, nil
}

func appends390xArgs(inventory *models.Inventory, installerArgs []string, log logrus.FieldLogger) ([]string, bool) {
Expand Down
29 changes: 21 additions & 8 deletions internal/network/manifests_generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -555,12 +555,14 @@ spec:
[Service]
Type=oneshot
ExecStart=-/bin/sh -c ' \
lsblk -o NAME,TRAN,MOUNTPOINTS --json | jq -e \'.blockdevices[] | select(.tran == "iscsi") | select(.children) | .children[].mountpoints | select(.) | index("/sysroot") | select(.)\' > /dev/null; \
if [ $? = 0 ]; \
then \
echo "iSCSI boot volume detected, force network reconfiguration..."; \
nmcli -t -f DEVICE device status | xargs -l nmcli device reapply; \
ExecStart=-/bin/sh -c ' set -x; \
MULTIPATH=".blockdevices[] | select(.tran == \\\"iscsi\\\") | select(.children) | .children[] | select(.children) | .children[].mountpoints | select(.) | index(\\\"/sysroot\\\")"; \
ISCSI_ALONE=".blockdevices[] | select(.tran == \\\"iscsi\\\") | select(.children) | .children[].mountpoints | select(.) | index(\\\"/sysroot\\\")"; \
(lsblk -o NAME,TRAN,MOUNTPOINTS --json | jq -e "$MULTIPATH" > /dev/null) || (lsblk -o NAME,TRAN,MOUNTPOINTS --json | jq -e "$ISCSI_ALONE" > /dev/null); \
if [ $? = 0 ]; \
then \
echo "iSCSI OR multipath iSCSI boot volume detected, force network reconfiguration..."; \
nmcli -t -f DEVICE device status | xargs -l nmcli device reapply; \
fi'
ExecStartPost=-systemctl disable iscsi-nic-reapply.service
Expand All @@ -569,12 +571,23 @@ spec:
`

func (m *ManifestsGenerator) AddNicReapply(ctx context.Context, log logrus.FieldLogger, c *common.Cluster) error {
// Add this manifest only is one of the host is installting on an iSCSI boot drive
// Add this manifest only if one of the host is installing on an iSCSI / multiapth + iSCSI boot drive
_, isUsingISCSIBootDrive := lo.Find(c.Cluster.Hosts, func(h *models.Host) bool {
installationDisk, err := hostutil.GetHostInstallationDisk(h)
inventory, err := common.UnmarshalInventory(h.Inventory)
if err != nil {
return false
}
installationDisk := hostutil.GetDiskByInstallationPath(inventory.Disks, hostutil.GetHostInstallationPath(h))
if installationDisk.DriveType == models.DriveTypeMultipath {
if err != nil {
return false
}
for _, disk := range inventory.Disks {
if disk.DriveType == models.DriveTypeISCSI && strings.Contains(disk.Holders, installationDisk.Name) {
return true
}
}
}
return installationDisk.DriveType == models.DriveTypeISCSI
})

Expand Down

0 comments on commit c6e41ad

Please sign in to comment.