Configure interfaces to mimic default AL2023

nodeadm/internal/api/status.go

@@ -91,7 +91,7 @@ func getNetworkCardsDetails(ctx context.Context, imdsFunc func(ctx context.Conte
 			return nil, fmt.Errorf("failed to get network card details for MAC %s: %w", mac, err)
 		}
 		// ip address can be empty for efa-only cards
-		if cardDetails.IpAddress == "" {
+		if cardDetails.IpV4Address == "" {
 			continue
 		}
 
@@ -113,7 +113,11 @@ func getNetworkCardDetail(ctx context.Context, imdsFunc func(ctx context.Context
 	// imds will return 404 if we query network-card object for instance that doesn't support multiple cards
 	cardIndexPath := imds.IMDSProperty(fmt.Sprintf("network/interfaces/macs/%s/network-card", mac))
 	// imds will return 404 if we query local-ipv4s object if ip-address is not confirured on the interface from EC2 (efa-only)
-	ipAddressPath := imds.IMDSProperty(fmt.Sprintf("network/interfaces/macs/%s/local-ipv4s", mac))
+	ipV4AddressPath := imds.IMDSProperty(fmt.Sprintf("network/interfaces/macs/%s/local-ipv4s", mac))
+	ipV4SubnetPath := imds.IMDSProperty(fmt.Sprintf("network/interfaces/macs/%s/subnet-ipv4-cidr-block", mac))
+	ipV6SubnetPath := imds.IMDSProperty(fmt.Sprintf("network/interfaces/macs/%s/subnet-ipv6-cidr-blocks", mac))
+	ipV6AddressPath := imds.IMDSProperty(fmt.Sprintf("network/interfaces/macs/%s/ipv6s", mac))
+	interfaceIdPath := imds.IMDSProperty(fmt.Sprintf("network/interfaces/macs/%s/interface-id", mac))
 
 	cardIndex, err := imdsFunc(ctx, cardIndexPath)
 	if err != nil {
@@ -124,15 +128,39 @@ func getNetworkCardDetail(ctx context.Context, imdsFunc func(ctx context.Context
 		return NetworkCardDetails{}, fmt.Errorf("invalid card index: %w", err)
 	}
 
-	ipAddress, err := imdsFunc(ctx, ipAddressPath)
+	ipV4Address, err := imdsFunc(ctx, ipV4AddressPath)
+	if err != nil {
+		return NetworkCardDetails{}, err
+	}
+
+	ipV4Subnet, err := imdsFunc(ctx, ipV4SubnetPath)
+	if err != nil {
+		return NetworkCardDetails{}, err
+	}
+
+	ipV6Address, err := imdsFunc(ctx, ipV6AddressPath)
+	if err != nil {
+		return NetworkCardDetails{}, err
+	}
+
+	ipV6Subnet, err := imdsFunc(ctx, ipV6SubnetPath)
+	if err != nil {
+		return NetworkCardDetails{}, err
+	}
+
+	interfaceId, err := imdsFunc(ctx, interfaceIdPath)
 	if err != nil {
 		return NetworkCardDetails{}, err
 	}
 
 	return NetworkCardDetails{
-		MAC:       mac,
-		CardIndex: cardIndexInt,
-		IpAddress: ipAddress,
+		MAC:         mac,
+		CardIndex:   cardIndexInt,
+		IpV4Address: ipV4Address,
+		IpV4Subnet:  ipV4Subnet,
+		IpV6Address: ipV6Address,
+		IpV6Subnet:  ipV6Subnet,
+		InterfaceId: interfaceId,
 	}, nil
 }
 

nodeadm/internal/api/status_test.go

@@ -20,12 +20,12 @@ var (
 06:83:e7:fc:fc:fd/
 `
 	validTwoNetworkCardDetails = []NetworkCardDetails{
-		{MAC: "06:83:e7:fe:fe:fe", IpAddress: "1.2.3.4", CardIndex: 1},
-		{MAC: "06:83:e7:ff:ff:ff", IpAddress: "5.6.7.8", CardIndex: 0},
+		{MAC: "06:83:e7:fe:fe:fe", IpV4Address: "1.2.3.4", CardIndex: 1},
+		{MAC: "06:83:e7:ff:ff:ff", IpV4Address: "5.6.7.8", CardIndex: 0},
 	}
 
 	validOneNetworkCardDetails = []NetworkCardDetails{
-		{MAC: "06:83:e7:fb:fb:fb", IpAddress: "1.2.3.4", CardIndex: 0},
+		{MAC: "06:83:e7:fb:fb:fb", IpV4Address: "1.2.3.4", CardIndex: 0},
 	}
 	imds404     = errors.New("http response error StatusCode: 404, request to EC2 IMDS failed")
 	imdsGeneric = errors.New("IMDS error")

nodeadm/internal/api/types.go

@@ -41,9 +41,13 @@ type NodeConfigStatus struct {
 }
 
 type NetworkCardDetails struct {
-	MAC       string `json:"mac,omitempty"`
-	IpAddress string `json:"ipAddress,omitempty"`
-	CardIndex int    `json:"cardIndex,omitempty"`
+	MAC         string `json:"mac,omitempty"`
+	IpV4Address string `json:"ipV4Address,omitempty"`
+	IpV4Subnet  string `json:"ipV4Subnet,omitempty"`
+	IpV6Address string `json:"ipV6Address,omitempty"`
+	IpV6Subnet  string `json:"ipV6Subnet,omitempty"`
+	CardIndex   int    `json:"cardIndex,omitempty"`
+	InterfaceId string `json:"interfaceId,omitempty"`
 }
 
 type InstanceDetails struct {

nodeadm/internal/system/_assets/10-eks_primary_eni_only.conf.template

@@ -1,2 +1,9 @@
 [Match]
-PermanentMACAddress={{.PermanentMACAddress}}
+PermanentMACAddress={{.PermanentMACAddress}}
+
+[DHCPv4]
+RouteMetric=512
+
+[IPv6AcceptRA]
+RouteMetric=512
+UseGateway=true

nodeadm/internal/system/_assets/interface.network.template

@@ -1,23 +1,59 @@
 [Match]
-Driver=ena ixgbevf vif
 PermanentMACAddress={{.PermanentMACAddress}}
 
 [Link]
 MTUBytes=9001
 
+[Link]
+MTUBytes=9001
+
 [Network]
-DHCP=ipv4
+DHCP=yes
+IPv6DuplicateAddressDetection=0
+LLMNR=no
 DNSDefaultRoute=yes
-IPv6AcceptRA=no
 
 [DHCPv4]
 UseHostname=no
 UseDNS=yes
 UseNTP=yes
 UseDomains=yes
-RouteTable={{.RouteTableId}}
+RouteMetric={{.RouteTableMetric}}
+UseRoutes=true
+UseGateway=true
+
+[DHCPv6]
+UseHostname=no
+UseDNS=yes
+UseNTP=yes
+WithoutRA=solicit
+
+[RoutingPolicyRule]
+From={{.IpV4Address}}
+Table={{.RouteTableId}}
+Priority={{.RouteTableId}}
 
 [RoutingPolicyRule]
-From={{.IPAddress}}
+From={{.IpV6Address}}
+Table={{.RouteTableId}}
+Priority={{.RouteTableId}}
+
+[IPv6AcceptRA]
+RouteMetric={{.RouteTableMetric}}
+UseGateway=true
+
+[Route]
+Table={{.RouteTableId}}
+Gateway=_ipv6ra
+
+[Route]
+Table={{.RouteTableId}}
+Destination={{.IpV6Subnet}}
+
+[Route]
+Gateway=_dhcp4
+Table={{.RouteTableId}}
+
+[Route]
 Table={{.RouteTableId}}
-Priority={{.RoutePriority}}
+Destination={{.IpV4Subnet}}

nodeadm/internal/system/_assets/test_10-eks_primary_eni_only.conf

@@ -1,2 +1,9 @@
 [Match]
-PermanentMACAddress=0e:f7:72:74:2d:43
+PermanentMACAddress=0e:f7:72:74:2d:43
+
+[DHCPv4]
+RouteMetric=512
+
+[IPv6AcceptRA]
+RouteMetric=512
+UseGateway=true

nodeadm/internal/system/networking.go

@@ -57,6 +57,9 @@ func (a *networkingAspect) Setup(cfg *api.NodeConfig) error {
 	if err := a.ensureMulticardNetworkConfiguration(cfg); err != nil {
 		return fmt.Errorf("failed to ensure multicard network configuration: %w", err)
 	}
+	if err := a.reloadNetworkConfigurations(); err != nil {
+		return fmt.Errorf("failed to reload network configurations: %w", err)
+	}
 	return nil
 }
 
@@ -90,23 +93,22 @@ func (a *networkingAspect) ensureEKSNetworkConfiguration(cfg *api.NodeConfig) er
 	if err := os.WriteFile(eksPrimaryENIOnlyConfPathName, eksPrimaryENIOnlyConfContent, networkConfFilePerms); err != nil {
 		return fmt.Errorf("failed to write eks_primary_eni_only network configuration: %w", err)
 	}
-	if err := a.reloadNetworkConfigurations(); err != nil {
-		return fmt.Errorf("failed to reload network configurations: %w", err)
-	}
 	return nil
 }
 
+// ensureMulticardNetworkConfiguration configures the non-zero card interfaces in a way that mimics the
+// default AL2023 configuration. Non-zero card interfaces are not managed by vpc-cni and we're creating
+// systemd-networkd .network files for each interface.
 func (a *networkingAspect) ensureMulticardNetworkConfiguration(cfg *api.NodeConfig) error {
-	var networkRestartRequired bool
-	routeTableId := 1001
-	routePriority := 32765
+	routeTableId := 10101
+	routeTableMetric := 613
 
 	for _, card := range cfg.Status.Instance.NetworkCards {
 		if card.CardIndex == 0 {
 			continue
 		}
 
-		networkInterfaceConfName := fmt.Sprintf("80-card%d.network", card.CardIndex)
+		networkInterfaceConfName := fmt.Sprintf("70-%s.network", card.InterfaceId)
 		networkInterfaceConfPathName := fmt.Sprintf("%s/%s", administrationNetworkDir, networkInterfaceConfName)
 
 		if exists, err := util.IsFilePathExists(networkInterfaceConfPathName); err != nil {
@@ -118,12 +120,15 @@ func (a *networkingAspect) ensureMulticardNetworkConfiguration(cfg *api.NodeConf
 
 		templateVars := networkInterfaceTemplateVars{
 			PermanentMACAddress: card.MAC,
-			IPAddress:           card.IpAddress,
+			IpV4Address:         card.IpV4Address,
+			IpV4Subnet:          card.IpV4Subnet,
+			IpV6Address:         card.IpV6Address,
+			IpV6Subnet:          card.IpV6Subnet,
 			RouteTableId:        int16(routeTableId),
-			RoutePriority:       int16(routePriority),
+			RouteTableMetric:    int16(routeTableMetric),
 		}
-		routeTableId++
-		routePriority--
+		routeTableId += 100
+		routeTableMetric += 100
 
 		interfaceConfigContent, err := a.generateNetworkConfigFile(networkInterfaceConfName, templateVars)
 		if err != nil {
@@ -134,14 +139,6 @@ func (a *networkingAspect) ensureMulticardNetworkConfiguration(cfg *api.NodeConf
 			return fmt.Errorf("failed to write %s configuration: %w", networkInterfaceConfName, err)
 		}
 		zap.L().Sugar().Infof("Multicard instance found, configuring card with index: %d, network file: %s", card.CardIndex, networkInterfaceConfPathName)
-		networkRestartRequired = true
-	}
-
-	if networkRestartRequired {
-		if err := a.reloadNetworkConfigurations(); err != nil {
-			return fmt.Errorf("failed to reload network configurations: %w", err)
-		}
-		zap.L().Sugar().Infof("Reloading network network..")
 	}
 	return nil
 }
@@ -164,9 +161,12 @@ type eksPrimaryENIOnlyTemplateVars struct {
 // networkInterfaceTemplateVars holds the variables for networkInterfaceConfTemplate
 type networkInterfaceTemplateVars struct {
 	PermanentMACAddress string
-	IPAddress           string
+	IpV4Address         string
+	IpV4Subnet          string
+	IpV6Address         string
+	IpV6Subnet          string
 	RouteTableId        int16
-	RoutePriority       int16
+	RouteTableMetric    int16
 }
 
 // generateEKSPrimaryENIOnlyConfiguration generates the eks primary eni only network configuration.