Skip to content

Commit

Permalink
Adding the support of Linux bridge interface for VM setup (#158)
Browse files Browse the repository at this point in the history 
* Adding the support of Linux bridge interface for VM setup
Use the --bridge option to create a NAD that will be attached to the VMs

* Updating the README for the lxn-bridge option

* adding the --bridgeNetwork option to change the default configuration for bridge setup by passing a JSON file
  • Loading branch information
@capolrik
capolrik authored Jan 20, 2025
1 parent dc1a1b1 commit 747be21
Show file tree
Hide file tree
Showing 6 changed files with 260 additions and 54 deletions.
34 changes: 34 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,40 @@ If the two above are in place, users can orhestrate k8s-netperf to launch VMs by
`k8s-netperf --vm`
### Using a linux bridge interface
When using `--bridge`, a NetworkAttachmentDefinition defining a bridge interface is attached to the VMs and is used for the test. It requires the name of the bridge as it is defined in the NetworkNodeConfigurationPolicy, NMstate operator is required. For example:
```yaml
apiVersion: nmstate.io/v1alpha1
kind: NodeNetworkConfigurationPolicy
metadata:
name: br0-eth1
spec:
desiredState:
interfaces:
- name: br0
description: Linux bridge with eno2 as a port
type: linux-bridge
state: up
ipv4:
dhcp: true
enabled: true
bridge:
options:
stp:
enabled: false
port:
- name: eno2
```
Then you can launch a test using the bridge interface:
```
./bin/amd64/k8s-netperf --vm --bridge br0
```
By default, it will read the `bridgeNetwork.json` file from the git repository. If the default IP addresses (10.10.10.12/24 and 10.10.10.14/24) are not available for your setup, it is possible to change it by passing a JSON file as a parameter with `--bridgeNetwork`, like follow:
```
k8s-netperf --vm --bridge br0 --bridgeNetwork /path/to/my/bridgeConfig.json
```
### Config file
#### Config File v2
The v2 config file will be executed in the order the tests are presented in the config file.
Expand Down
4 changes: 4 additions & 0 deletions bridgeNetwork.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"bridgeServerNetwork": "10.10.10.12/24",
"bridgeClientNetwork": "10.10.10.14/24"
}
95 changes: 74 additions & 21 deletions cmd/k8s-netperf/k8s-netperf.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@ package main

import (
"context"
encodeJson "encoding/json"
"fmt"
"io"
"os"
"regexp"
"strings"
Expand Down Expand Up @@ -33,27 +35,29 @@ const index = "k8s-netperf"
const retry = 3

var (
cfgfile string
nl bool
clean bool
netperf bool
iperf3 bool
uperf bool
udn bool
acrossAZ bool
full bool
vm bool
vmimage string
debug bool
promURL string
id string
searchURL string
showMetrics bool
tcpt float64
json bool
version bool
csvArchive bool
searchIndex string
cfgfile string
nl bool
clean bool
netperf bool
iperf3 bool
uperf bool
udn bool
acrossAZ bool
full bool
vm bool
vmimage string
debug bool
bridge string
bridgeNetwork string
promURL string
id string
searchURL string
showMetrics bool
tcpt float64
json bool
version bool
csvArchive bool
searchIndex string
)

var rootCmd = &cobra.Command{
Expand Down Expand Up @@ -184,6 +188,16 @@ var rootCmd = &cobra.Command{
log.Error(err)
}
s.KClient = kclient
if len(bridge) > 0 {
err := k8s.DeployNADBridge(s.DClient, bridge)
if err != nil {
log.Error(err)
}
s.BridgeServerNetwork, s.BridgeClientNetwork, err = parseNetworkConfig(bridgeNetwork)
if err != nil {
log.Error(err)
}
}
}

// Build the SUT (Deployments)
Expand Down Expand Up @@ -395,6 +409,40 @@ func cleanup(client *kubernetes.Clientset) {
}
}

// Function to parse the JSON from a file and return the IP parts (before '/')
func parseNetworkConfig(jsonFile string) (string, string, error) {

// Open the JSON file
file, err := os.Open(jsonFile)
if err != nil {
return "", "", fmt.Errorf("error opening file: %v", err)
}
defer file.Close()

// Read the file contents
log.Debugf("Reading BridgeNetwork configuration from JSON file: %s ", jsonFile)
content, err := io.ReadAll(file)
if err != nil {
return "", "", fmt.Errorf("error reading file: %v", err)
}

// Create an instance of the struct
var netConfig config.BridgeNetworkConfig

// Unmarshal the JSON string into the struct
err = encodeJson.Unmarshal(content, &netConfig)
if err != nil {
return "", "", fmt.Errorf("error parsing JSON: %v", err)
}

// Extract the IP parts (before '/')
serverIP := netConfig.BridgeServerNetwork
clientIP := netConfig.BridgeClientNetwork

// Return the extracted IPs
return serverIP, clientIP, nil
}

// executeWorkload executes the workload and returns the result data.
func executeWorkload(nc config.Config,
s config.PerfScenarios,
Expand All @@ -417,6 +465,9 @@ func executeWorkload(nc config.Config,
if err != nil {
log.Fatal(err)
}
//when using a bridge
} else if s.BridgeServerNetwork != "" {
serverIP = strings.Split(s.BridgeServerNetwork, "/")[0]
} else {
if hostNet {
serverIP = s.ServerHost.Items[0].Status.PodIP
Expand Down Expand Up @@ -516,6 +567,8 @@ func main() {
rootCmd.Flags().BoolVar(&full, "all", false, "Run all tests scenarios - hostNet and podNetwork (if possible)")
rootCmd.Flags().BoolVar(&debug, "debug", false, "Enable debug log")
rootCmd.Flags().BoolVar(&udn, "udn", false, "Create and use a UDN called 'udn-l2-primary' as primary network.")
rootCmd.Flags().StringVar(&bridge, "bridge", "", "Name of the NNCP to be used for creating bridge interface - VM only.")
rootCmd.Flags().StringVar(&bridgeNetwork, "bridgeNetwork", "bridgeNetwork.json", "Json file for the network defined by the bridge interface - bridge should be enabled")
rootCmd.Flags().StringVar(&promURL, "prom", "", "Prometheus URL")
rootCmd.Flags().StringVar(&id, "uuid", "", "User provided UUID")
rootCmd.Flags().StringVar(&searchURL, "search", "", "OpenSearch URL, if you have auth, pass in the format of https://user:pass@url:port")
Expand Down
54 changes: 31 additions & 23 deletions pkg/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,29 +32,37 @@ type Config struct {

// PerfScenarios describes the different scenarios
type PerfScenarios struct {
NodeLocal bool
AcrossAZ bool
HostNetwork bool
Configs []Config
VM bool
VMImage string
VMHost string
Udn bool
ServerNodeInfo metrics.NodeInfo
ClientNodeInfo metrics.NodeInfo
Client apiv1.PodList
Server apiv1.PodList
ClientAcross apiv1.PodList
ClientHost apiv1.PodList
ServerHost apiv1.PodList
NetperfService *apiv1.Service
IperfService *apiv1.Service
UperfService *apiv1.Service
RestConfig rest.Config
ClientSet *kubernetes.Clientset
KClient *kubevirtv1.KubevirtV1Client
DClient *dynamic.DynamicClient
SSHClient *goph.Client
NodeLocal bool
AcrossAZ bool
HostNetwork bool
Configs []Config
VM bool
VMImage string
VMHost string
Udn bool
BridgeServerNetwork string
BridgeClientNetwork string
ServerNodeInfo metrics.NodeInfo
ClientNodeInfo metrics.NodeInfo
Client apiv1.PodList
Server apiv1.PodList
ClientAcross apiv1.PodList
ClientHost apiv1.PodList
ServerHost apiv1.PodList
NetperfService *apiv1.Service
IperfService *apiv1.Service
UperfService *apiv1.Service
RestConfig rest.Config
ClientSet *kubernetes.Clientset
KClient *kubevirtv1.KubevirtV1Client
DClient *dynamic.DynamicClient
SSHClient *goph.Client
}

// struct for bridge options
type BridgeNetworkConfig struct {
BridgeServerNetwork string `json:"bridgeServerNetwork"`
BridgeClientNetwork string `json:"bridgeClientNetwork"`
}

// Tests we will support in k8s-netperf
Expand Down
34 changes: 32 additions & 2 deletions pkg/k8s/kubernetes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,36 @@ func DeployL2Udn(dynamicClient *dynamic.DynamicClient) error {
return nil
}

// Create a NetworkAttachcmentDefinition object for a bridge connection
func DeployNADBridge(dyn *dynamic.DynamicClient, bridgeName string) error {
nadBridge := &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "k8s.cni.cncf.io/v1",
"kind": "NetworkAttachmentDefinition",
"metadata": map[string]interface{}{
"name": "br-netperf",
"namespace": "netperf",
"annotations": map[string]interface{}{
"k8s.v1.cni.cncf.io/resourceName": "bridge.network.kubevirt.io/" + bridgeName,
},
},
"spec": map[string]interface{}{
"config": `{"cniVersion": "0.3.1", "type": "bridge", "name": "br-netperf", "bridge": "` + bridgeName + `"}`,
},
},
}
gvr := schema.GroupVersionResource{
Group: "k8s.cni.cncf.io",
Version: "v1",
Resource: "network-attachment-definitions",
}
_, err := dyn.Resource(gvr).Namespace(namespace).Create(context.TODO(), nadBridge, metav1.CreateOptions{})
if err != nil {
return err
}
return nil
}

// BuildSUT Build the k8s env to run network performance tests
func BuildSUT(client *kubernetes.Clientset, s *config.PerfScenarios) error {
var netperfDataPorts []int32
Expand Down Expand Up @@ -557,7 +587,7 @@ func ExtractUdnIp(s config.PerfScenarios) (string, error) {

// launchServerVM will create the ServerVM with the specific node and pod affinity.
func launchServerVM(perf *config.PerfScenarios, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) error {
_, err := CreateVMServer(perf.KClient, serverRole, serverRole, *podAff, *nodeAff, perf.VMImage)
_, err := CreateVMServer(perf.KClient, serverRole, serverRole, *podAff, *nodeAff, perf.VMImage, perf.BridgeServerNetwork)
if err != nil {
return err
}
Expand All @@ -582,7 +612,7 @@ func launchServerVM(perf *config.PerfScenarios, name string, podAff *corev1.PodA

// launchClientVM will create the ClientVM with the specific node and pod affinity.
func launchClientVM(perf *config.PerfScenarios, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) error {
host, err := CreateVMClient(perf.KClient, perf.ClientSet, perf.DClient, name, podAff, nodeAff, perf.VMImage)
host, err := CreateVMClient(perf.KClient, perf.ClientSet, perf.DClient, name, podAff, nodeAff, perf.VMImage, perf.BridgeClientNetwork)
if err != nil {
return err
}
Expand Down
Loading

0 comments on commit 747be21

Please sign in to comment.