node annotations: add k8s.ovn.org/node-mgmt-port-info

Instead of making an assumption about the IP address on ovn-k8s-mp0, where it is expected to be the second IP of node-subnet for the default/primary network, it is better to provide this information through an annotation. k8s.ovn.org/node-mgmt-port-info: { "default": { "ip-addresses": [ "10.192.13.2/26" ], } } Signed-off-by: Flavio Fernandes <[email protected]>
ovn-org · Oct 14, 2024 · 48a5d60 · 48a5d60
1 parent 689e95a
commit 48a5d60
Show file tree

Hide file tree

Showing 5 changed files with 196 additions and 2 deletions.
diff --git a/go-controller/pkg/ovnwebhook/nodeadmission.go b/go-controller/pkg/ovnwebhook/nodeadmission.go
@@ -33,6 +33,7 @@ var commonNodeAnnotationChecks = map[string]checkNodeAnnot{
 	util.OvnNodeMasqCIDR:                   nil,
 	util.OvnNodeGatewayMtuSupport:          nil,
 	util.OvnNodeManagementPort:             nil,
+	util.OvnNodeManagementPortInfo:         nil,
 	util.OvnNodeChassisID: func(v annotationChange, nodeName string) error {
 		if v.action == removed {
 			return fmt.Errorf("%s cannot be removed", util.OvnNodeChassisID)

diff --git a/go-controller/pkg/ovnwebhook/nodeadmission_test.go b/go-controller/pkg/ovnwebhook/nodeadmission_test.go
@@ -269,6 +269,25 @@ func TestNodeAdmission_ValidateUpdate(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "ovnkube-node can set util.OvnNodeManagementPortInfo",
+			ctx: admission.NewContextWithRequest(context.TODO(), admission.Request{
+				AdmissionRequest: v1.AdmissionRequest{UserInfo: authenticationv1.UserInfo{
+					Username: userName,
+				}},
+			}),
+			oldObj: &corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: nodeName,
+				},
+			},
+			newObj: &corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        nodeName,
+					Annotations: map[string]string{util.OvnNodeManagementPortInfo: `{"default":[1.2.3.4/24]}`},
+				},
+			},
+		},
 		{
 			name: "ovnkube-node can set util.OvnNodeGatewayMtuSupport",
 			ctx: admission.NewContextWithRequest(context.TODO(), admission.Request{

diff --git a/go-controller/pkg/util/node_annotations.go b/go-controller/pkg/util/node_annotations.go
@@ -2,6 +2,7 @@ package util
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"math"
 	"net"
@@ -62,6 +63,21 @@ const (
 	// OvnNodeManagementPort is the constant string representing the annotation key
 	OvnNodeManagementPort = "k8s.ovn.org/node-mgmt-port"
 
+	// OvnNodeManagementPortInfo contains all ips, mac addresses and mgmt ports
+	// on all networks keyed by the network-name
+	// k8s.ovn.org/node-mgmt-port-info: {
+	// 		"default": {
+	// 			"ip-addresses": [ "10.192.13.2/26" ],
+	// 			"mac-address": "ca:53:88:23:bc:98",
+	// 			"function-info": { "PfId": 0, "FuncId": 0 }
+	//      },
+	// 		"l2-network": {
+	// 			"ip-addresses": [ "10.200.10.2/26" ],
+	// 			"mac-address": "5e:52:2a:c0:98:f4"
+	//      }
+	// }
+	OvnNodeManagementPortInfo = "k8s.ovn.org/node-mgmt-port-info"
+
 	// OvnNodeManagementPortMacAddresses contains all mac addresses of the management ports
 	// on all networks keyed by the network-name
 	// k8s.ovn.org/node-mgmt-port-mac-addresses: {
@@ -1454,3 +1470,80 @@ func GetNetworkID(nodes []*corev1.Node, nInfo BasicNetInfo) (int, error) {
 	}
 	return InvalidNetworkID, fmt.Errorf("missing network id for network '%s'", nInfo.GetNetworkName())
 }
+
+type ManagementPortInfo struct {
+	IpAddresses []string `json:"ip-addresses,omitempty"`
+}
+
+func updateNodeMgmtPortInfoIpAddresses(netName string, hostSubnets []*net.IPNet, annotations map[string]string) error {
+	if annotations == nil {
+		return errors.New("annotaions must not be nil")
+	}
+
+	mgmtPortInfoMap, err := parseNodeMgmtPortInfoAnnotation(annotations)
+	if err != nil {
+		return fmt.Errorf("failed to decode ManagementPortInfo for IpAddresses: %w", err)
+	}
+
+	// Convert IPNet slice into a string slice of CIDRs
+	ipAddressesStr := make([]string, len(hostSubnets))
+	for i, hostSubnet := range hostSubnets {
+		ipNet := GetNodeManagementIfAddr(hostSubnet) // Get the IPNet representation
+		ipAddressesStr[i] = ipNet.String()           // Convert IPNet to CIDR string
+	}
+
+	// Check if the netName already exists in the map
+	if mgmtPortInfo, exists := mgmtPortInfoMap[netName]; exists {
+		// Update the IpAddresses for the specified netName
+		mgmtPortInfo.IpAddresses = ipAddressesStr
+
+		// Check if the netName should be deleted based on conditions
+		if len(ipAddressesStr) == 0 {
+			delete(mgmtPortInfoMap, netName)
+		} else {
+			// Otherwise, update the map with new IpAddresses
+			mgmtPortInfoMap[netName] = mgmtPortInfo
+		}
+	} else if len(ipAddressesStr) > 0 {
+		// If netName doesn't exist and there are IP addresses to set, create a new entry
+		mgmtPortInfoMap[netName] = ManagementPortInfo{
+			IpAddresses: ipAddressesStr,
+		}
+	}
+
+	// Encode the updated map back to JSON
+	bytes, err := encodeNodeMgmtPortInfoAnnotation(mgmtPortInfoMap)
+	if err != nil {
+		return fmt.Errorf("failed to encode ManagementPortInfo: %w", err)
+	}
+
+	// Set the JSON-encoded string back into the annotations
+	annotations[OvnNodeManagementPortInfo] = bytes
+	return nil
+}
+
+// ParseAnnotations parses the annotations map and returns a map of ManagementPortInfo
+func parseNodeMgmtPortInfoAnnotation(annotations map[string]string) (map[string]ManagementPortInfo, error) {
+	result := make(map[string]ManagementPortInfo)
+
+	// Check if the annotation exists
+	if value, ok := annotations[OvnNodeManagementPortInfo]; ok {
+		// Parse the JSON data into the result map
+		err := json.Unmarshal([]byte(value), &result)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse annotation %s: %w", OvnNodeManagementPortInfo, err)
+		}
+	}
+
+	return result, nil
+}
+
+// SetManagementPortInfo sets the JSON encoded ManagementPortInfo to the annotations map
+func encodeNodeMgmtPortInfoAnnotation(info map[string]ManagementPortInfo) (string, error) {
+	// JSON encode the map
+	jsonData, err := json.Marshal(info)
+	if err != nil {
+		return "", fmt.Errorf("failed to encode ManagementPortInfo: %w", err)
+	}
+	return string(jsonData), nil
+}
diff --git a/go-controller/pkg/util/node_annotations_unit_test.go b/go-controller/pkg/util/node_annotations_unit_test.go
@@ -826,3 +826,84 @@ func TestGetNetworkID(t *testing.T) {
 		})
 	}
 }
+
+func TestParseNodeManagementPortIPAddresses(t *testing.T) {
+	net1 := ovntest.MustParseIPNet("1.1.1.0/24")
+	net2 := ovntest.MustParseIPNet("2.2.2.0/24")
+	tests := []struct {
+		desc           string
+		inpNetName     string
+		inpHostSubnets []*net.IPNet
+		inpAnnotations map[string]string
+		errExpected    bool
+		expOutput      map[string]string
+	}{
+		{
+			desc:        "ip address annotation not provided",
+			errExpected: true,
+		},
+		{
+			desc:           "ip address annotation not found for node, however, does not return error",
+			inpNetName:     "default",
+			inpAnnotations: map[string]string{},
+			expOutput: map[string]string{
+				"k8s.ovn.org/node-mgmt-port-info": `{}`,
+			},
+		},
+		{
+			desc:       "ip address annotation removal",
+			inpNetName: "default",
+			inpAnnotations: map[string]string{
+				"k8s.ovn.org/node-mgmt-port-info": `{"default":{"ip-addresses":["1.2.3.4/24"]}}`,
+			},
+			expOutput: map[string]string{
+				"k8s.ovn.org/node-mgmt-port-info": `{}`,
+			},
+		},
+		{
+			desc:       "ip address annotation removal from one of the networks",
+			inpNetName: "one",
+			inpAnnotations: map[string]string{
+				"k8s.ovn.org/node-mgmt-port-info": `{"one":{"ip-addresses":["10.0.0.2/24"]},"default":{"ip-addresses":["11.0.0.2/24"]},"two":{"ip-addresses":["20.0.0.2/24"]}}`,
+			},
+			expOutput: map[string]string{
+				"k8s.ovn.org/node-mgmt-port-info": `{"default":{"ip-addresses":["11.0.0.2/24"]},"two":{"ip-addresses":["20.0.0.2/24"]}}`,
+			},
+		},
+		{
+			desc:           "ip address annotation add new network",
+			inpNetName:     "one",
+			inpHostSubnets: []*net.IPNet{net1},
+			inpAnnotations: map[string]string{
+				"k8s.ovn.org/node-mgmt-port-info": `{"default":{"ip-addresses":["11.0.0.2/24"]},"two":{"ip-addresses":["20.0.0.2/24"]}}`,
+			},
+			expOutput: map[string]string{
+				"k8s.ovn.org/node-mgmt-port-info": `{"default":{"ip-addresses":["11.0.0.2/24"]},"one":{"ip-addresses":["1.1.1.2/24"]},"two":{"ip-addresses":["20.0.0.2/24"]}}`,
+			},
+		},
+		{
+			desc:           "ip address annotation update address of existing network",
+			inpNetName:     "one",
+			inpHostSubnets: []*net.IPNet{net1, net2},
+			inpAnnotations: map[string]string{
+				"k8s.ovn.org/node-mgmt-port-info": `{"default":{"ip-addresses":["11.0.0.2/24"]},"one":{"ip-addresses":["9.9.9.9/24"]},"two":{"ip-addresses":["20.0.0.2/24"]}}`,
+			},
+			expOutput: map[string]string{
+				"k8s.ovn.org/node-mgmt-port-info": `{"default":{"ip-addresses":["11.0.0.2/24"]},"one":{"ip-addresses":["1.1.1.2/24","2.2.2.2/24"]},"two":{"ip-addresses":["20.0.0.2/24"]}}`,
+			},
+		},
+	}
+
+	for i, tc := range tests {
+		t.Run(fmt.Sprintf("%d:%s", i, tc.desc), func(t *testing.T) {
+			e := updateNodeMgmtPortInfoIpAddresses(tc.inpNetName, tc.inpHostSubnets, tc.inpAnnotations)
+			if tc.errExpected {
+				t.Log(e)
+				assert.Error(t, e)
+			} else {
+				assert.NoError(t, e)
+			}
+			assert.Equal(t, tc.inpAnnotations, tc.expOutput)
+		})
+	}
+}
diff --git a/go-controller/pkg/util/subnet_annotations.go b/go-controller/pkg/util/subnet_annotations.go
@@ -65,7 +65,7 @@ func updateSubnetAnnotation(annotations map[string]string, annotationName, netNa
 	// if no host subnet left, just delete the host subnet annotation from node annotations.
 	if len(subnetsMap) == 0 {
 		delete(annotations, annotationName)
-		return nil
+		return updateNodeMgmtPortInfoIpAddresses(netName, hostSubnets, annotations)
 	}
 
 	// Marshal all host subnets of all networks back to annotations.
@@ -82,7 +82,7 @@ func updateSubnetAnnotation(annotations map[string]string, annotationName, netNa
 		return err
 	}
 	annotations[annotationName] = string(bytes)
-	return nil
+	return updateNodeMgmtPortInfoIpAddresses(netName, hostSubnets, annotations)
 }
 
 func setSubnetAnnotation(nodeAnnotator kube.Annotator, annotationName string, defaultSubnets []*net.IPNet) error {