Support DSR mode for Service's external addresses (#5202)

This commit adds support for DSR mode for Service's external addresses, including LoadBalancerIPs and ExternalIPs. A configuration option, `antreaProxy.defaultLoadBalancerMode` is added to determine how external traffic is processed when it's load balanced across Nodes by default. It has two options: `nat` (default) and `dsr`. In NAT mode, external traffic is SNAT'd when it's load balanced across Nodes to ensure symmetric path. In DSR mode, external traffic is never SNAT'd and backend Pods running on Nodes that are not the ingress Node can reply to clients directly, bypassing the ingress Node. Additionally, a Service's load balancer mode can be overridden by annotating it with `service.antrea.io/load-balancer-mode`. A feature gate, `LoadBalancerModeDSR` is added to control whether it's allowed to use DSR mode. When a Service's LoadBalancerMode is DSR, the following changes will be applied to the OpenFlow flows and groups: 1. ClusterGroup will be used by traffic working in DSR mode on ingress Node. * If a local Endpoint is selected, it will just be handled normally as DSR is not applicable in this case. * If a remote Endpoint is selected, it will be sent to the backend Node that hosts the Endpoint without being NAT'd, the eventual Endpoint will be determined on the backend Node and may be different from the one selected here. 2. LocalGroup will be used by traffic working in DSR mode on backend Node. In this way, each Endpoint has the same chance to be selected eventually. 3. Traffic working in DSR mode on ingress Node will be marked and treated specially, e.g. bypassing SNAT. 4. Learned flow will be created for each connection to ensure consistent load balance decision for a connection of DSR mode. Learned flow is necessary because connections of DSR mode will remain invalid on ingress Node as it can only see requests and not responses. And OVS doesn't provide ct_state and ct_label for invalid connections. Thus, we can't store the load balance decision of the connection to ct_state or ct_label. To ensure consistent load balancing decision for packets of a connection, we use "learn" action to generate a learned flow when processing the first packet of a connection, and rely on the learned flow to process subsequent packets of the same connection. DSR mode usually means lower latency, higher output bandwidth, and preserved client IP. However, due to the use of learned flow, creating new connections may be slightly slower than NAT mode, this may be improved in the future. The benchmark of the current implementation is as below: ``` Test NAT DSR delta TCP_CRR 1105.69 1007.82 -8.86% TCP_RR 6802.55 9054.44 +33.1% ``` This feature is currently only supported for Linux Nodes, `encap` mode, and IPv4 cluster. The support for Windows and IPv6 can be added in the future. Signed-off-by: Quan Tian <[email protected]>
antrea-io · Jul 20, 2023 · 8261b12 · 8261b12
1 parent f7adfa7
commit 8261b12
Show file tree

Hide file tree

Showing 48 changed files with 2,597 additions and 855 deletions.
diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml
@@ -125,8 +125,7 @@ jobs:
         retention-days: 30
 
   test-e2e-encap-non-default:
-    # NodeIPAM=true cannot run with proxyAll enabled due to https://github.com/antrea-io/antrea/issues/5022.
-    name: E2e tests on a Kind cluster on Linux with non default values (AntreaProxy=false, NodeIPAM=true)
+    name: E2e tests on a Kind cluster on Linux with non default values (proxyAll=true, LoadBalancerMode=DSR, NodeIPAM=true)
     needs: [build-antrea-coverage-image]
     runs-on: [ubuntu-latest]
     steps:
@@ -155,7 +154,13 @@ jobs:
       run: |
         mkdir log
         mkdir test-e2e-encap-non-default-coverage
-        ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-non-default-coverage ./ci/kind/test-e2e-kind.sh --encap-mode encap --feature-gates AntreaProxy=false --node-ipam --coverage
+        ANTREA_LOG_DIR=$PWD/log ANTREA_COV_DIR=$PWD/test-e2e-encap-non-default-coverage ./ci/kind/test-e2e-kind.sh \
+          --coverage \
+          --encap-mode encap \
+          --proxy-all \
+          --feature-gates LoadBalancerModeDSR=true \
+          --load-balancer-mode dsr \
+          --node-ipam
     - name: Tar coverage files
       run: tar -czf test-e2e-encap-non-default-coverage.tar.gz test-e2e-encap-non-default-coverage
     - name: Upload coverage for test-e2e-encap-non-default-coverage

diff --git a/build/charts/antrea/README.md b/build/charts/antrea/README.md
@@ -41,6 +41,7 @@ Kubernetes: `>= 1.16.0-0`
 | agent.priorityClassName | string | `"system-node-critical"` | Prority class to use for the antrea-agent Pods. |
 | agent.tolerations | list | `[{"key":"CriticalAddonsOnly","operator":"Exists"},{"effect":"NoSchedule","operator":"Exists"},{"effect":"NoExecute","operator":"Exists"}]` | Tolerations for the antrea-agent Pods. |
 | agent.updateStrategy | object | `{"type":"RollingUpdate"}` | Update strategy for the antrea-agent DaemonSet. |
+| antreaProxy.defaultLoadBalancerMode | string | `"nat"` | Determines how external traffic is processed when it's load balanced across Nodes by default. It must be one of "nat" or "dsr". |
 | antreaProxy.nodePortAddresses | list | `[]` | String array of values which specifies the host IPv4/IPv6 addresses for NodePort. By default, all host addresses are used. |
 | antreaProxy.proxyAll | bool | `false` | Proxy all Service traffic, for all Service types, regardless of where it comes from. |
 | antreaProxy.proxyLoadBalancerIPs | bool | `true` | When set to false, AntreaProxy no longer load-balances traffic destined to the External IPs of LoadBalancer Services. |

diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf
@@ -73,6 +73,9 @@ featureGates:
 # into account application context.
 {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "L7NetworkPolicy" "default" false) }}
 
+# Allow users to specify the load balancer mode as DSR (Direct Server Return).
+{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "LoadBalancerModeDSR" "default" false) }}
+
 # Name of the OpenVSwitch bridge antrea-agent will create and use.
 # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
 ovsBridge: {{ .Values.ovs.bridgeName | quote }}
@@ -347,6 +350,13 @@ antreaProxy:
   # then AntreaProxy will only handle Services without the "service.kubernetes.io/service-proxy-name" label,
   # but ignore Services with the label no matter what is the value.
   serviceProxyName: {{ .serviceProxyName | quote }}
+  # Determines how external traffic is processed when it's load balanced across Nodes by default.
+  # It has the following options:
+  # - nat (default): External traffic is SNAT'd when it's load balanced across Nodes to ensure symmetric path.
+  # - dsr:           External traffic is never SNAT'd. Backend Pods running on Nodes that are not the ingress Node
+  #                  can reply to clients directly, bypassing the ingress Node.
+  # A Service's load balancer mode can be overridden by annotating it with `service.antrea.io/load-balancer-mode`.
+  defaultLoadBalancerMode: {{ .defaultLoadBalancerMode | quote }}
 {{- end }}
 
 # IPsec tunnel related configurations.

diff --git a/build/charts/antrea/values.yaml b/build/charts/antrea/values.yaml
@@ -141,6 +141,9 @@ antreaProxy:
   # will only handle Services without the "service.kubernetes.io/service-proxy-name"
   # label, but ignore Services with the label no matter what is the value.
   serviceProxyName: ""
+  # -- Determines how external traffic is processed when it's load balanced across Nodes by default. It must be one of "nat" or
+  # "dsr".
+  defaultLoadBalancerMode: "nat"
 
 nodeIPAM:
   # -- Enable Node IPAM in Antrea

diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -3163,6 +3163,9 @@ data:
     # into account application context.
     #  L7NetworkPolicy: false
 
+    # Allow users to specify the load balancer mode as DSR (Direct Server Return).
+    #  LoadBalancerModeDSR: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -3411,6 +3414,13 @@ data:
       # then AntreaProxy will only handle Services without the "service.kubernetes.io/service-proxy-name" label,
       # but ignore Services with the label no matter what is the value.
       serviceProxyName: ""
+      # Determines how external traffic is processed when it's load balanced across Nodes by default.
+      # It has the following options:
+      # - nat (default): External traffic is SNAT'd when it's load balanced across Nodes to ensure symmetric path.
+      # - dsr:           External traffic is never SNAT'd. Backend Pods running on Nodes that are not the ingress Node
+      #                  can reply to clients directly, bypassing the ingress Node.
+      # A Service's load balancer mode can be overridden by annotating it with `service.antrea.io/load-balancer-mode`.
+      defaultLoadBalancerMode: "nat"
 
     # IPsec tunnel related configurations.
     ipsec:
@@ -4542,7 +4552,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 720e2b412e83992caf5874a01e67507617e079b896796e588c92fd75c9e06ad6
+        checksum/config: 4ebea7300356a753d716270575de36dd3584f67dd62607cd6c9c2a115ac92e62
       labels:
         app: antrea
         component: antrea-agent
@@ -4783,7 +4793,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 720e2b412e83992caf5874a01e67507617e079b896796e588c92fd75c9e06ad6
+        checksum/config: 4ebea7300356a753d716270575de36dd3584f67dd62607cd6c9c2a115ac92e62
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -3163,6 +3163,9 @@ data:
     # into account application context.
     #  L7NetworkPolicy: false
 
+    # Allow users to specify the load balancer mode as DSR (Direct Server Return).
+    #  LoadBalancerModeDSR: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -3411,6 +3414,13 @@ data:
       # then AntreaProxy will only handle Services without the "service.kubernetes.io/service-proxy-name" label,
       # but ignore Services with the label no matter what is the value.
       serviceProxyName: ""
+      # Determines how external traffic is processed when it's load balanced across Nodes by default.
+      # It has the following options:
+      # - nat (default): External traffic is SNAT'd when it's load balanced across Nodes to ensure symmetric path.
+      # - dsr:           External traffic is never SNAT'd. Backend Pods running on Nodes that are not the ingress Node
+      #                  can reply to clients directly, bypassing the ingress Node.
+      # A Service's load balancer mode can be overridden by annotating it with `service.antrea.io/load-balancer-mode`.
+      defaultLoadBalancerMode: "nat"
 
     # IPsec tunnel related configurations.
     ipsec:
@@ -4542,7 +4552,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 720e2b412e83992caf5874a01e67507617e079b896796e588c92fd75c9e06ad6
+        checksum/config: 4ebea7300356a753d716270575de36dd3584f67dd62607cd6c9c2a115ac92e62
       labels:
         app: antrea
         component: antrea-agent
@@ -4784,7 +4794,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 720e2b412e83992caf5874a01e67507617e079b896796e588c92fd75c9e06ad6
+        checksum/config: 4ebea7300356a753d716270575de36dd3584f67dd62607cd6c9c2a115ac92e62
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -3163,6 +3163,9 @@ data:
     # into account application context.
     #  L7NetworkPolicy: false
 
+    # Allow users to specify the load balancer mode as DSR (Direct Server Return).
+    #  LoadBalancerModeDSR: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -3411,6 +3414,13 @@ data:
       # then AntreaProxy will only handle Services without the "service.kubernetes.io/service-proxy-name" label,
       # but ignore Services with the label no matter what is the value.
       serviceProxyName: ""
+      # Determines how external traffic is processed when it's load balanced across Nodes by default.
+      # It has the following options:
+      # - nat (default): External traffic is SNAT'd when it's load balanced across Nodes to ensure symmetric path.
+      # - dsr:           External traffic is never SNAT'd. Backend Pods running on Nodes that are not the ingress Node
+      #                  can reply to clients directly, bypassing the ingress Node.
+      # A Service's load balancer mode can be overridden by annotating it with `service.antrea.io/load-balancer-mode`.
+      defaultLoadBalancerMode: "nat"
 
     # IPsec tunnel related configurations.
     ipsec:
@@ -4542,7 +4552,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 46b91206a96d91e7e4861f20fa0e255ed660cf96e796116e562061efc09fdfcc
+        checksum/config: 48b346133ac76c11a6c456d99aa93c2421e5598a13b9d18d5dd58d6cce5408ff
       labels:
         app: antrea
         component: antrea-agent
@@ -4781,7 +4791,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 46b91206a96d91e7e4861f20fa0e255ed660cf96e796116e562061efc09fdfcc
+        checksum/config: 48b346133ac76c11a6c456d99aa93c2421e5598a13b9d18d5dd58d6cce5408ff
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -3176,6 +3176,9 @@ data:
     # into account application context.
     #  L7NetworkPolicy: false
 
+    # Allow users to specify the load balancer mode as DSR (Direct Server Return).
+    #  LoadBalancerModeDSR: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -3424,6 +3427,13 @@ data:
       # then AntreaProxy will only handle Services without the "service.kubernetes.io/service-proxy-name" label,
       # but ignore Services with the label no matter what is the value.
       serviceProxyName: ""
+      # Determines how external traffic is processed when it's load balanced across Nodes by default.
+      # It has the following options:
+      # - nat (default): External traffic is SNAT'd when it's load balanced across Nodes to ensure symmetric path.
+      # - dsr:           External traffic is never SNAT'd. Backend Pods running on Nodes that are not the ingress Node
+      #                  can reply to clients directly, bypassing the ingress Node.
+      # A Service's load balancer mode can be overridden by annotating it with `service.antrea.io/load-balancer-mode`.
+      defaultLoadBalancerMode: "nat"
 
     # IPsec tunnel related configurations.
     ipsec:
@@ -4555,7 +4565,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: f7e797321f4228539c43945a503637bcabf4d4eee4f3d5393ba9e69a778a0916
+        checksum/config: c464a20c63a45190125a9bacb8d0b25cf04ad0e1f45e9bc2be76ebdb74d758bf
         checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4
       labels:
         app: antrea
@@ -4840,7 +4850,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: f7e797321f4228539c43945a503637bcabf4d4eee4f3d5393ba9e69a778a0916
+        checksum/config: c464a20c63a45190125a9bacb8d0b25cf04ad0e1f45e9bc2be76ebdb74d758bf
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -3163,6 +3163,9 @@ data:
     # into account application context.
     #  L7NetworkPolicy: false
 
+    # Allow users to specify the load balancer mode as DSR (Direct Server Return).
+    #  LoadBalancerModeDSR: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -3411,6 +3414,13 @@ data:
       # then AntreaProxy will only handle Services without the "service.kubernetes.io/service-proxy-name" label,
       # but ignore Services with the label no matter what is the value.
       serviceProxyName: ""
+      # Determines how external traffic is processed when it's load balanced across Nodes by default.
+      # It has the following options:
+      # - nat (default): External traffic is SNAT'd when it's load balanced across Nodes to ensure symmetric path.
+      # - dsr:           External traffic is never SNAT'd. Backend Pods running on Nodes that are not the ingress Node
+      #                  can reply to clients directly, bypassing the ingress Node.
+      # A Service's load balancer mode can be overridden by annotating it with `service.antrea.io/load-balancer-mode`.
+      defaultLoadBalancerMode: "nat"
 
     # IPsec tunnel related configurations.
     ipsec:
@@ -4542,7 +4552,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 520e5bfad080176d9e77896e35756f8f947a1777817847cad27c764ecf6e3bec
+        checksum/config: 38c3b07d25dc21a29a2e7c91aaa95475191b53ca77639ceada4a2604b6425666
       labels:
         app: antrea
         component: antrea-agent
@@ -4781,7 +4791,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 520e5bfad080176d9e77896e35756f8f947a1777817847cad27c764ecf6e3bec
+        checksum/config: 38c3b07d25dc21a29a2e7c91aaa95475191b53ca77639ceada4a2604b6425666
       labels:
         app: antrea
         component: antrea-controller

diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh
@@ -28,6 +28,7 @@ _usage="Usage: $0 [--encap-mode <mode>] [--ip-family <v4|v6>] [--coverage] [--he
         --feature-gates               A comma-separated list of key=value pairs that describe feature gates, e.g. AntreaProxy=true,Egress=false.
         --run                         Run only tests matching the regexp.
         --proxy-all                   Enables Antrea proxy with all Service support.
+        --load-balancer-mode          LoadBalancer mode.
         --node-ipam                   Enables Antrea NodeIPAN.
         --multicast                   Enables Multicast.
         --flow-visibility             Only run flow visibility related e2e tests.
@@ -64,6 +65,7 @@ mode=""
 ipfamily="v4"
 feature_gates=""
 proxy_all=false
+load_balancer_mode=""
 node_ipam=false
 multicast=false
 flow_visibility=false
@@ -90,6 +92,10 @@ case $key in
     proxy_all=true
     shift
     ;;
+    --load-balancer-mode)
+    load_balancer_mode="$2"
+    shift 2
+    ;;
     --node-ipam)
     node_ipam=true
     shift
@@ -155,6 +161,9 @@ fi
 if $proxy_all; then
     manifest_args="$manifest_args --proxy-all"
 fi
+if [ -n "$load_balancer_mode" ]; then
+    manifest_args="$manifest_args --extra-helm-values antreaProxy.defaultLoadBalancerMode=$load_balancer_mode"
+fi
 if $node_ipam; then
     manifest_args="$manifest_args --extra-helm-values nodeIPAM.enable=true,nodeIPAM.clusterCIDRs={10.244.0.0/16}"
 fi

diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -51,6 +51,7 @@ import (
 	"antrea.io/antrea/pkg/agent/metrics"
 	"antrea.io/antrea/pkg/agent/multicast"
 	mcroute "antrea.io/antrea/pkg/agent/multicluster"
+	"antrea.io/antrea/pkg/agent/nodeip"
 	npl "antrea.io/antrea/pkg/agent/nodeportlocal"
 	"antrea.io/antrea/pkg/agent/openflow"
 	"antrea.io/antrea/pkg/agent/proxy"
@@ -139,20 +140,24 @@ func run(o *Options) error {
 	enableMulticlusterGW := features.DefaultFeatureGate.Enabled(features.Multicluster) && o.config.Multicluster.EnableGateway
 	enableMulticlusterNP := features.DefaultFeatureGate.Enabled(features.Multicluster) && o.config.Multicluster.EnableStretchedNetworkPolicy
 
+	nodeIPTracker := nodeip.NewTracker(nodeInformer)
 	// Bridging mode will connect the uplink interface to the OVS bridge.
 	connectUplinkToBridge := enableBridgingMode
 	ovsDatapathType := ovsconfig.OVSDatapathType(o.config.OVSDatapathType)
 	ovsBridgeClient := ovsconfig.NewOVSBridge(o.config.OVSBridge, ovsDatapathType, ovsdbConnection)
 	ovsCtlClient := ovsctl.NewClient(o.config.OVSBridge)
 	ovsBridgeMgmtAddr := ofconfig.GetMgmtAddress(o.config.OVSRunDir, o.config.OVSBridge)
 	multicastEnabled := features.DefaultFeatureGate.Enabled(features.Multicast) && o.config.Multicast.Enable
-	ofClient := openflow.NewClient(o.config.OVSBridge, ovsBridgeMgmtAddr,
+	ofClient := openflow.NewClient(o.config.OVSBridge,
+		ovsBridgeMgmtAddr,
+		nodeIPTracker,
 		features.DefaultFeatureGate.Enabled(features.AntreaProxy),
 		features.DefaultFeatureGate.Enabled(features.AntreaPolicy),
 		l7NetworkPolicyEnabled,
 		o.enableEgress,
 		features.DefaultFeatureGate.Enabled(features.FlowExporter) && o.config.FlowExporter.Enable,
 		o.config.AntreaProxy.ProxyAll,
+		features.DefaultFeatureGate.Enabled(features.LoadBalancerModeDSR),
 		connectUplinkToBridge,
 		multicastEnabled,
 		features.DefaultFeatureGate.Enabled(features.TrafficControl),
@@ -396,11 +401,13 @@ func run(o *Options) error {
 			k8sClient,
 			ofClient,
 			routeClient,
+			nodeIPTracker,
 			v4Enabled,
 			v6Enabled,
 			nodePortAddressesIPv4,
 			nodePortAddressesIPv6,
 			o.config.AntreaProxy,
+			o.defaultLoadBalancerMode,
 			v4GroupCounter,
 			v6GroupCounter,
 			enableMulticlusterGW,