diff --git a/README.md b/README.md index 1c72dc14..cec8e97d 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,40 @@ If the two above are in place, users can orhestrate k8s-netperf to launch VMs by `k8s-netperf --vm` +### Using a linux bridge interface +When using `--bridge`, a NetworkAttachmentDefinition defining a bridge interface is attached to the VMs and is used for the test. It requires the name of the bridge as it is defined in the NetworkNodeConfigurationPolicy, NMstate operator is required. For example: +```yaml +apiVersion: nmstate.io/v1alpha1 +kind: NodeNetworkConfigurationPolicy +metadata: + name: br0-eth1 +spec: + desiredState: + interfaces: + - name: br0 + description: Linux bridge with eno2 as a port + type: linux-bridge + state: up + ipv4: + dhcp: true + enabled: true + bridge: + options: + stp: + enabled: false + port: + - name: eno2 +``` +Then you can launch a test using the bridge interface: +``` +./bin/amd64/k8s-netperf --vm --bridge br0 +``` +By default, it will read the `bridgeNetwork.json` file from the git repository. If the default IP addresses (10.10.10.12/24 and 10.10.10.14/24) are not available for your setup, it is possible to change it by passing a JSON file as a parameter with `--bridgeNetwork`, like follow: +``` +k8s-netperf --vm --bridge br0 --bridgeNetwork /path/to/my/bridgeConfig.json + +``` + ### Config file #### Config File v2 The v2 config file will be executed in the order the tests are presented in the config file. diff --git a/bridgeNetwork.json b/bridgeNetwork.json new file mode 100644 index 00000000..6f27bc73 --- /dev/null +++ b/bridgeNetwork.json @@ -0,0 +1,4 @@ +{ + "bridgeServerNetwork": "10.10.10.12/24", + "bridgeClientNetwork": "10.10.10.14/24" +} diff --git a/cmd/k8s-netperf/k8s-netperf.go b/cmd/k8s-netperf/k8s-netperf.go index 790dea18..bd43fa1a 100644 --- a/cmd/k8s-netperf/k8s-netperf.go +++ b/cmd/k8s-netperf/k8s-netperf.go @@ -2,7 +2,9 @@ package main import ( "context" + encodeJson "encoding/json" "fmt" + "io" "os" "regexp" "strings" @@ -33,27 +35,29 @@ const index = "k8s-netperf" const retry = 3 var ( - cfgfile string - nl bool - clean bool - netperf bool - iperf3 bool - uperf bool - udn bool - acrossAZ bool - full bool - vm bool - vmimage string - debug bool - promURL string - id string - searchURL string - showMetrics bool - tcpt float64 - json bool - version bool - csvArchive bool - searchIndex string + cfgfile string + nl bool + clean bool + netperf bool + iperf3 bool + uperf bool + udn bool + acrossAZ bool + full bool + vm bool + vmimage string + debug bool + bridge string + bridgeNetwork string + promURL string + id string + searchURL string + showMetrics bool + tcpt float64 + json bool + version bool + csvArchive bool + searchIndex string ) var rootCmd = &cobra.Command{ @@ -184,6 +188,16 @@ var rootCmd = &cobra.Command{ log.Error(err) } s.KClient = kclient + if len(bridge) > 0 { + err := k8s.DeployNADBridge(s.DClient, bridge) + if err != nil { + log.Error(err) + } + s.BridgeServerNetwork, s.BridgeClientNetwork, err = parseNetworkConfig(bridgeNetwork) + if err != nil { + log.Error(err) + } + } } // Build the SUT (Deployments) @@ -395,6 +409,40 @@ func cleanup(client *kubernetes.Clientset) { } } +// Function to parse the JSON from a file and return the IP parts (before '/') +func parseNetworkConfig(jsonFile string) (string, string, error) { + + // Open the JSON file + file, err := os.Open(jsonFile) + if err != nil { + return "", "", fmt.Errorf("error opening file: %v", err) + } + defer file.Close() + + // Read the file contents + log.Debugf("Reading BridgeNetwork configuration from JSON file: %s ", jsonFile) + content, err := io.ReadAll(file) + if err != nil { + return "", "", fmt.Errorf("error reading file: %v", err) + } + + // Create an instance of the struct + var netConfig config.BridgeNetworkConfig + + // Unmarshal the JSON string into the struct + err = encodeJson.Unmarshal(content, &netConfig) + if err != nil { + return "", "", fmt.Errorf("error parsing JSON: %v", err) + } + + // Extract the IP parts (before '/') + serverIP := netConfig.BridgeServerNetwork + clientIP := netConfig.BridgeClientNetwork + + // Return the extracted IPs + return serverIP, clientIP, nil +} + // executeWorkload executes the workload and returns the result data. func executeWorkload(nc config.Config, s config.PerfScenarios, @@ -417,6 +465,9 @@ func executeWorkload(nc config.Config, if err != nil { log.Fatal(err) } + //when using a bridge + } else if s.BridgeServerNetwork != "" { + serverIP = strings.Split(s.BridgeServerNetwork, "/")[0] } else { if hostNet { serverIP = s.ServerHost.Items[0].Status.PodIP @@ -516,6 +567,8 @@ func main() { rootCmd.Flags().BoolVar(&full, "all", false, "Run all tests scenarios - hostNet and podNetwork (if possible)") rootCmd.Flags().BoolVar(&debug, "debug", false, "Enable debug log") rootCmd.Flags().BoolVar(&udn, "udn", false, "Create and use a UDN called 'udn-l2-primary' as primary network.") + rootCmd.Flags().StringVar(&bridge, "bridge", "", "Name of the NNCP to be used for creating bridge interface - VM only.") + rootCmd.Flags().StringVar(&bridgeNetwork, "bridgeNetwork", "bridgeNetwork.json", "Json file for the network defined by the bridge interface - bridge should be enabled") rootCmd.Flags().StringVar(&promURL, "prom", "", "Prometheus URL") rootCmd.Flags().StringVar(&id, "uuid", "", "User provided UUID") rootCmd.Flags().StringVar(&searchURL, "search", "", "OpenSearch URL, if you have auth, pass in the format of https://user:pass@url:port") diff --git a/pkg/config/config.go b/pkg/config/config.go index 4f1cb0ff..3408e55a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -32,29 +32,37 @@ type Config struct { // PerfScenarios describes the different scenarios type PerfScenarios struct { - NodeLocal bool - AcrossAZ bool - HostNetwork bool - Configs []Config - VM bool - VMImage string - VMHost string - Udn bool - ServerNodeInfo metrics.NodeInfo - ClientNodeInfo metrics.NodeInfo - Client apiv1.PodList - Server apiv1.PodList - ClientAcross apiv1.PodList - ClientHost apiv1.PodList - ServerHost apiv1.PodList - NetperfService *apiv1.Service - IperfService *apiv1.Service - UperfService *apiv1.Service - RestConfig rest.Config - ClientSet *kubernetes.Clientset - KClient *kubevirtv1.KubevirtV1Client - DClient *dynamic.DynamicClient - SSHClient *goph.Client + NodeLocal bool + AcrossAZ bool + HostNetwork bool + Configs []Config + VM bool + VMImage string + VMHost string + Udn bool + BridgeServerNetwork string + BridgeClientNetwork string + ServerNodeInfo metrics.NodeInfo + ClientNodeInfo metrics.NodeInfo + Client apiv1.PodList + Server apiv1.PodList + ClientAcross apiv1.PodList + ClientHost apiv1.PodList + ServerHost apiv1.PodList + NetperfService *apiv1.Service + IperfService *apiv1.Service + UperfService *apiv1.Service + RestConfig rest.Config + ClientSet *kubernetes.Clientset + KClient *kubevirtv1.KubevirtV1Client + DClient *dynamic.DynamicClient + SSHClient *goph.Client +} + +// struct for bridge options +type BridgeNetworkConfig struct { + BridgeServerNetwork string `json:"bridgeServerNetwork"` + BridgeClientNetwork string `json:"bridgeClientNetwork"` } // Tests we will support in k8s-netperf diff --git a/pkg/k8s/kubernetes.go b/pkg/k8s/kubernetes.go index e36d68b0..c31c72bb 100644 --- a/pkg/k8s/kubernetes.go +++ b/pkg/k8s/kubernetes.go @@ -171,6 +171,36 @@ func DeployL2Udn(dynamicClient *dynamic.DynamicClient) error { return nil } +// Create a NetworkAttachcmentDefinition object for a bridge connection +func DeployNADBridge(dyn *dynamic.DynamicClient, bridgeName string) error { + nadBridge := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": map[string]interface{}{ + "name": "br-netperf", + "namespace": "netperf", + "annotations": map[string]interface{}{ + "k8s.v1.cni.cncf.io/resourceName": "bridge.network.kubevirt.io/" + bridgeName, + }, + }, + "spec": map[string]interface{}{ + "config": `{"cniVersion": "0.3.1", "type": "bridge", "name": "br-netperf", "bridge": "` + bridgeName + `"}`, + }, + }, + } + gvr := schema.GroupVersionResource{ + Group: "k8s.cni.cncf.io", + Version: "v1", + Resource: "network-attachment-definitions", + } + _, err := dyn.Resource(gvr).Namespace(namespace).Create(context.TODO(), nadBridge, metav1.CreateOptions{}) + if err != nil { + return err + } + return nil +} + // BuildSUT Build the k8s env to run network performance tests func BuildSUT(client *kubernetes.Clientset, s *config.PerfScenarios) error { var netperfDataPorts []int32 @@ -557,7 +587,7 @@ func ExtractUdnIp(s config.PerfScenarios) (string, error) { // launchServerVM will create the ServerVM with the specific node and pod affinity. func launchServerVM(perf *config.PerfScenarios, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) error { - _, err := CreateVMServer(perf.KClient, serverRole, serverRole, *podAff, *nodeAff, perf.VMImage) + _, err := CreateVMServer(perf.KClient, serverRole, serverRole, *podAff, *nodeAff, perf.VMImage, perf.BridgeServerNetwork) if err != nil { return err } @@ -582,7 +612,7 @@ func launchServerVM(perf *config.PerfScenarios, name string, podAff *corev1.PodA // launchClientVM will create the ClientVM with the specific node and pod affinity. func launchClientVM(perf *config.PerfScenarios, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) error { - host, err := CreateVMClient(perf.KClient, perf.ClientSet, perf.DClient, name, podAff, nodeAff, perf.VMImage) + host, err := CreateVMClient(perf.KClient, perf.ClientSet, perf.DClient, name, podAff, nodeAff, perf.VMImage, perf.BridgeClientNetwork) if err != nil { return err } diff --git a/pkg/k8s/kubevirt.go b/pkg/k8s/kubevirt.go index 59b99e68..16c3956d 100644 --- a/pkg/k8s/kubevirt.go +++ b/pkg/k8s/kubevirt.go @@ -158,7 +158,7 @@ func exposeService(client *kubernetes.Clientset, dynamicClient *dynamic.DynamicC // CreateVMClient takes in the affinity rules and deploys the VMI func CreateVMClient(kclient *kubevirtv1.KubevirtV1Client, client *kubernetes.Clientset, - dyn *dynamic.DynamicClient, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity, vmimage string) (string, error) { + dyn *dynamic.DynamicClient, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity, vmimage string, bridgeNetwork string) (string, error) { label := map[string]string{ "app": name, "role": name, @@ -171,6 +171,7 @@ func CreateVMClient(kclient *kubevirtv1.KubevirtV1Client, client *kubernetes.Cli if err != nil { return "", err } + netData := "{}" data := fmt.Sprintf(`#cloud-config users: - name: fedora @@ -184,7 +185,7 @@ chpasswd: { expire: False } runcmd: - export HOME=/home/fedora - dnf install -y --nodocs uperf iperf3 git ethtool automake gcc bc lksctp-tools-devel texinfo --enablerepo=* - - git clone https://github.com/HewlettPackard/netperf + - git clone https://github.com/HewlettPackard/netperf.git - cd netperf - git reset --hard 3bc455b23f901dae377ca0a558e1e32aa56b31c4 - curl -o netperf.diff https://raw.githubusercontent.com/cloud-bulldozer/k8s-netperf/main/containers/netperf.diff @@ -196,7 +197,43 @@ runcmd: - curl -o /usr/bin/super-netperf https://raw.githubusercontent.com/cloud-bulldozer/k8s-netperf/main/containers/super-netperf - chmod 0777 /usr/bin/super-netperf `, ssh) - _, err = CreateVMI(kclient, name, label, b64.StdEncoding.EncodeToString([]byte(data)), *podAff, *nodeAff, vmimage) + interfaces := []v1.Interface{ + { + Name: "default", + InterfaceBindingMethod: v1.InterfaceBindingMethod{ + Bridge: &v1.InterfaceBridge{}, + }, + }, + } + networks := []v1.Network{ + { + Name: "default", + NetworkSource: v1.NetworkSource{ + Pod: &v1.PodNetwork{}, + }, + }, + } + if bridgeNetwork != "" { + interfaces = append(interfaces, v1.Interface{ + Name: "br-netperf", + InterfaceBindingMethod: v1.InterfaceBindingMethod{ + Bridge: &v1.InterfaceBridge{}, + }, + }) + networks = append(networks, v1.Network{ + Name: "br-netperf", + NetworkSource: v1.NetworkSource{ + Multus: &v1.MultusNetwork{ + NetworkName: "netperf/br-netperf", + }, + }, + }) + netData = fmt.Sprintf(`version: 2 +ethernets: + eth1: + addresses: [ %s ]`, bridgeNetwork) + } + _, err = CreateVMI(kclient, name, label, b64.StdEncoding.EncodeToString([]byte(data)), *podAff, *nodeAff, vmimage, interfaces, networks, b64.StdEncoding.EncodeToString([]byte(netData))) if err != nil { return "", err } @@ -213,11 +250,12 @@ runcmd: // CreateVMServer will take the pod and node affinity and deploy the VMI func CreateVMServer(client *kubevirtv1.KubevirtV1Client, name string, role string, podAff corev1.PodAntiAffinity, - nodeAff corev1.NodeAffinity, vmimage string) (*v1.VirtualMachineInstance, error) { + nodeAff corev1.NodeAffinity, vmimage string, bridgeNetwork string) (*v1.VirtualMachineInstance, error) { label := map[string]string{ "app": name, "role": role, } + netData := "{}" dirname, err := os.UserHomeDir() if err != nil { return nil, err @@ -239,7 +277,7 @@ chpasswd: { expire: False } runcmd: - dnf install -y --nodocs uperf iperf3 git ethtool - dnf install -y --nodocs automake gcc bc lksctp-tools-devel texinfo --enablerepo=* - - git clone https://github.com/HewlettPackard/netperf + - git clone https://github.com/HewlettPackard/netperf.git - cd netperf - git reset --hard 3bc455b23f901dae377ca0a558e1e32aa56b31c4 - curl -o netperf.diff https://raw.githubusercontent.com/cloud-bulldozer/k8s-netperf/main/containers/netperf.diff @@ -252,12 +290,48 @@ runcmd: - iperf3 -s -p %d & - netserver & `, string(ssh), UperfServerCtlPort, IperfServerCtlPort) - return CreateVMI(client, name, label, b64.StdEncoding.EncodeToString([]byte(data)), podAff, nodeAff, vmimage) + interfaces := []v1.Interface{ + { + Name: "default", + InterfaceBindingMethod: v1.InterfaceBindingMethod{ + Bridge: &v1.InterfaceBridge{}, + }, + }, + } + networks := []v1.Network{ + { + Name: "default", + NetworkSource: v1.NetworkSource{ + Pod: &v1.PodNetwork{}, + }, + }, + } + if bridgeNetwork != "" { + interfaces = append(interfaces, v1.Interface{ + Name: "br-netperf", + InterfaceBindingMethod: v1.InterfaceBindingMethod{ + Bridge: &v1.InterfaceBridge{}, + }, + }) + networks = append(networks, v1.Network{ + Name: "br-netperf", + NetworkSource: v1.NetworkSource{ + Multus: &v1.MultusNetwork{ + NetworkName: "netperf/br-netperf", + }, + }, + }) + netData = fmt.Sprintf(`version: 2 +ethernets: + eth1: + addresses: [ %s ]`, bridgeNetwork) + } + return CreateVMI(client, name, label, b64.StdEncoding.EncodeToString([]byte(data)), podAff, nodeAff, vmimage, interfaces, networks, b64.StdEncoding.EncodeToString([]byte(netData))) } // CreateVMI creates the desired Virtual Machine instance with the cloud-init config with affinity. func CreateVMI(client *kubevirtv1.KubevirtV1Client, name string, label map[string]string, b64data string, podAff corev1.PodAntiAffinity, - nodeAff corev1.NodeAffinity, vmimage string) (*v1.VirtualMachineInstance, error) { + nodeAff corev1.NodeAffinity, vmimage string, interfaces []v1.Interface, networks []v1.Network, netDatab64 string) (*v1.VirtualMachineInstance, error) { delSeconds := int64(0) mutliQ := true vmi, err := client.VirtualMachineInstances(namespace).Create(context.TODO(), &v1.VirtualMachineInstance{ @@ -300,8 +374,10 @@ func CreateVMI(client *kubevirtv1.KubevirtV1Client, name string, label map[strin }, }, }, + Interfaces: interfaces, }, }, + Networks: networks, Volumes: []v1.Volume{ v1.Volume{ Name: "disk0", @@ -315,7 +391,8 @@ func CreateVMI(client *kubevirtv1.KubevirtV1Client, name string, label map[strin Name: "cloudinit", VolumeSource: v1.VolumeSource{ CloudInitNoCloud: &v1.CloudInitNoCloudSource{ - UserDataBase64: b64data, + UserDataBase64: b64data, + NetworkDataBase64: netDatab64, }, }, },