From 9ae8636bb0d5d466b22e237558a9bbbe0acf757a Mon Sep 17 00:00:00 2001 From: Hang Yan Date: Wed, 10 Apr 2024 09:13:29 +0000 Subject: [PATCH] Add packetsampling feature Signed-off-by: Hang Yan --- build/charts/antrea/conf/antrea-agent.conf | 3 + build/charts/antrea/crds/packetsampling.yaml | 173 ++++ .../antrea/templates/agent/clusterrole.yaml | 20 + .../webhooks/validating/crdvalidator.yaml | 15 + build/yamls/antrea-aks.yml | 218 +++- build/yamls/antrea-crds.yml | 174 ++++ build/yamls/antrea-eks.yml | 218 +++- build/yamls/antrea-gke.yml | 218 +++- build/yamls/antrea-ipsec.yml | 218 +++- build/yamls/antrea.yml | 218 +++- ci/kind/test-e2e-kind.sh | 6 + cmd/antrea-agent/agent.go | 21 + docs/packetsampling-guide.md | 76 ++ go.mod | 1 + go.sum | 5 + hack/.notableofcontents | 1 + .../controller/networkpolicy/audit_logging.go | 8 +- .../controller/networkpolicy/packetin.go | 19 +- pkg/agent/controller/networkpolicy/reject.go | 4 +- .../controller/packetsampling/packetin.go | 109 ++ .../packetsampling/packetin_test.go | 259 +++++ .../packetsampling_controller.go | 718 +++++++++++++ .../packetsampling_controller_test.go | 977 ++++++++++++++++++ pkg/agent/openflow/client.go | 34 + pkg/agent/openflow/client_test.go | 154 ++- pkg/agent/openflow/cookie/allocator.go | 3 + pkg/agent/openflow/fields.go | 3 + pkg/agent/openflow/framework.go | 15 + pkg/agent/openflow/packetin.go | 2 + pkg/agent/openflow/packetsampling.go | 55 + pkg/agent/openflow/pipeline.go | 253 ++++- pkg/agent/openflow/pipeline_test.go | 92 ++ pkg/agent/openflow/testing/mock_openflow.go | 28 + .../support_bundle_controller.go | 108 +- .../support_bundle_controller_test.go | 11 +- pkg/apis/crd/v1alpha1/register.go | 2 + pkg/apis/crd/v1alpha1/types.go | 69 ++ .../crd/v1alpha1/zz_generated.deepcopy.go | 123 +++ pkg/apiserver/apiserver.go | 6 + .../handlers/featuregates/handler_test.go | 2 + .../typed/crd/v1alpha1/crd_client.go | 7 +- .../crd/v1alpha1/fake/fake_crd_client.go | 6 +- .../typed/crd/v1alpha1/generated_expansion.go | 4 +- .../crd/v1alpha1/interface.go | 9 +- .../informers/externalversions/generic.go | 2 + .../crd/v1alpha1/expansion_generated.go | 6 +- pkg/controller/packetsampling/validate.go | 108 ++ .../packetsampling/validate_test.go | 227 ++++ .../supportbundlecollection/controller.go | 58 +- .../controller_test.go | 105 -- pkg/features/antrea_features.go | 7 + pkg/util/ftp/auth.go | 102 ++ pkg/util/ftp/auth_test.go | 183 ++++ pkg/util/ftp/ftp.go | 109 ++ pkg/util/ftp/ftp_test.go | 66 ++ test/e2e/framework.go | 15 + test/e2e/packetsampling_test.go | 636 ++++++++++++ test/integration/agent/openflow_test.go | 20 +- 58 files changed, 5989 insertions(+), 320 deletions(-) create mode 100644 build/charts/antrea/crds/packetsampling.yaml create mode 100644 docs/packetsampling-guide.md create mode 100644 pkg/agent/controller/packetsampling/packetin.go create mode 100644 pkg/agent/controller/packetsampling/packetin_test.go create mode 100644 pkg/agent/controller/packetsampling/packetsampling_controller.go create mode 100644 pkg/agent/controller/packetsampling/packetsampling_controller_test.go create mode 100644 pkg/agent/openflow/packetsampling.go create mode 100644 pkg/controller/packetsampling/validate.go create mode 100644 pkg/controller/packetsampling/validate_test.go create mode 100644 pkg/util/ftp/auth.go create mode 100644 pkg/util/ftp/auth_test.go create mode 100644 pkg/util/ftp/ftp.go create mode 100644 pkg/util/ftp/ftp_test.go create mode 100644 test/e2e/packetsampling_test.go diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf index 11ee1f54b0e..5cfe934ff6f 100644 --- a/build/charts/antrea/conf/antrea-agent.conf +++ b/build/charts/antrea/conf/antrea-agent.conf @@ -23,6 +23,9 @@ featureGates: # Enable traceflow which provides packet tracing feature to diagnose network issue. {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "Traceflow" "default" true) }} +# Enable PacketSampling feature which provides packets sampling (capture) feature to diagnose network issue. +{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "PacketSampling" "default" false) }} + # Enable NodePortLocal feature to make the Pods reachable externally through NodePort {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "NodePortLocal" "default" true) }} diff --git a/build/charts/antrea/crds/packetsampling.yaml b/build/charts/antrea/crds/packetsampling.yaml new file mode 100644 index 00000000000..3b1bf30e4a9 --- /dev/null +++ b/build/charts/antrea/crds/packetsampling.yaml @@ -0,0 +1,173 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: packetsamplings.crd.antrea.io + labels: + app: antrea +spec: + group: crd.antrea.io + versions: + - name: v1alpha1 + served: true + storage: true + additionalPrinterColumns: + - jsonPath: .status.phase + description: The phase of the PacketSampling. + name: Phase + type: string + - jsonPath: .spec.source.pod + description: The name of the source Pod. + name: Source-Pod + type: string + priority: 10 + - jsonPath: .spec.destination.pod + description: The name of the destination Pod. + name: Destination-Pod + type: string + priority: 10 + - jsonPath: .spec.destination.ip + description: The IP address of the destination. + name: Destination-IP + type: string + priority: 10 + - jsonPath: .spec.timeout + description: Timeout in seconds. + name: Timeout + type: integer + priority: 10 + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + schema: + openAPIV3Schema: + type: object + required: + - spec + properties: + spec: + type: object + properties: + source: + type: object + properties: + pod: + type: string + namespace: + type: string + ip: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + destination: + type: object + properties: + pod: + type: string + service: + type: string + namespace: + type: string + ip: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + packet: + type: object + properties: + ipHeader: + type: object + properties: + protocol: + type: integer + minimum: 0 + maximum: 255 + ipv6Header: + type: object + properties: + nextHeader: + type: integer + minimum: 0 + maximum: 65535 + transportHeader: + type: object + properties: + udp: + type: object + properties: + srcPort: + type: integer + minimum: 1 + maximum: 65535 + dstPort: + type: integer + minimum: 1 + maximum: 65535 + tcp: + type: object + properties: + srcPort: + type: integer + minimum: 1 + maximum: 65535 + dstPort: + type: integer + minimum: 1 + maximum: 65535 + flags: + type: integer + minimum: 0 + maximum: 255 + timeout: + type: integer + minimum: 1 + maximum: 300 + type: + type: string + firstNSamplingConfig: + type: object + properties: + number: + type: integer + fileServer: + type: object + properties: + url: + type: string + authentication: + type: object + properties: + authType: + type: string + enum: [ "BearerToken", "APIKey", "BasicAuthentication" ] + authSecret: + type: object + properties: + name: + type: string + namespace: + type: string + status: + type: object + properties: + reason: + type: string + phase: + type: string + startTime: + type: string + numCapturedPackets: + type: integer + packetsPath: + type: string + + subresources: + status: {} + scope: Cluster + names: + plural: packetsamplings + singular: packetsampling + kind: PacketSampling + shortNames: + - ps diff --git a/build/charts/antrea/templates/agent/clusterrole.yaml b/build/charts/antrea/templates/agent/clusterrole.yaml index 05ca58c1ab6..1cabd7c6605 100644 --- a/build/charts/antrea/templates/agent/clusterrole.yaml +++ b/build/charts/antrea/templates/agent/clusterrole.yaml @@ -33,6 +33,12 @@ rules: - pods/status verbs: - patch + - apiGroups: + - "" + resources: + - secrets + verbs: + - get - apiGroups: - "" resources: @@ -154,6 +160,20 @@ rules: - patch - create - delete + - apiGroups: + - crd.antrea.io + resources: + - packetsamplings + verbs: + - get + - watch + - list + - apiGroups: + - crd.antrea.io + resources: + - packetsamplings/status + verbs: + - patch - apiGroups: - crd.antrea.io resources: diff --git a/build/charts/antrea/templates/webhooks/validating/crdvalidator.yaml b/build/charts/antrea/templates/webhooks/validating/crdvalidator.yaml index 0dfe1f8acd3..c5ba497552c 100644 --- a/build/charts/antrea/templates/webhooks/validating/crdvalidator.yaml +++ b/build/charts/antrea/templates/webhooks/validating/crdvalidator.yaml @@ -184,3 +184,18 @@ webhooks: admissionReviewVersions: ["v1", "v1beta1"] sideEffects: None timeoutSeconds: 5 + - name: "packetsamplingvalidator.antrea.io" + clientConfig: + service: + name: "antrea" + namespace: {{ .Release.Namespace }} + path: "/validate/packetsampling" + rules: + - operations: ["CREATE", "UPDATE"] + apiGroups: ["crd.antrea.io"] + apiVersions: ["v1alpha1"] + resources: ["packetsampling"] + scope: "Cluster" + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + timeoutSeconds: 5 diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index f4bc9bb2162..95ecef577e4 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -4313,6 +4313,182 @@ spec: # Deprecated shortName and shall be removed in Antrea v1.14.0 - anp +--- +# Source: crds/packetsampling.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: packetsamplings.crd.antrea.io + labels: + app: antrea +spec: + group: crd.antrea.io + versions: + - name: v1alpha1 + served: true + storage: true + additionalPrinterColumns: + - jsonPath: .status.phase + description: The phase of the PacketSampling. + name: Phase + type: string + - jsonPath: .spec.source.pod + description: The name of the source Pod. + name: Source-Pod + type: string + priority: 10 + - jsonPath: .spec.destination.pod + description: The name of the destination Pod. + name: Destination-Pod + type: string + priority: 10 + - jsonPath: .spec.destination.ip + description: The IP address of the destination. + name: Destination-IP + type: string + priority: 10 + - jsonPath: .spec.timeout + description: Timeout in seconds. + name: Timeout + type: integer + priority: 10 + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + schema: + openAPIV3Schema: + type: object + required: + - spec + properties: + spec: + type: object + properties: + source: + type: object + properties: + pod: + type: string + namespace: + type: string + ip: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + destination: + type: object + properties: + pod: + type: string + service: + type: string + namespace: + type: string + ip: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + packet: + type: object + properties: + ipHeader: + type: object + properties: + protocol: + type: integer + minimum: 0 + maximum: 255 + ipv6Header: + type: object + properties: + nextHeader: + type: integer + minimum: 0 + maximum: 65535 + transportHeader: + type: object + properties: + udp: + type: object + properties: + srcPort: + type: integer + minimum: 1 + maximum: 65535 + dstPort: + type: integer + minimum: 1 + maximum: 65535 + tcp: + type: object + properties: + srcPort: + type: integer + minimum: 1 + maximum: 65535 + dstPort: + type: integer + minimum: 1 + maximum: 65535 + flags: + type: integer + minimum: 0 + maximum: 255 + timeout: + type: integer + minimum: 1 + maximum: 300 + type: + type: string + firstNSamplingConfig: + type: object + properties: + number: + type: integer + fileServer: + type: object + properties: + url: + type: string + authentication: + type: object + properties: + authType: + type: string + enum: [ "BearerToken", "APIKey", "BasicAuthentication" ] + authSecret: + type: object + properties: + name: + type: string + namespace: + type: string + status: + type: object + properties: + reason: + type: string + phase: + type: string + startTime: + type: string + numCapturedPackets: + type: integer + packetsPath: + type: string + + subresources: + status: {} + scope: Cluster + names: + plural: packetsamplings + singular: packetsampling + kind: PacketSampling + shortNames: + - ps + --- # Source: crds/supportbundlecollection.yaml apiVersion: apiextensions.k8s.io/v1 @@ -5449,6 +5625,9 @@ data: # Enable traceflow which provides packet tracing feature to diagnose network issue. # Traceflow: true + # Enable PacketSampling feature which provides packets sampling (capture) feature to diagnose network issue. + # PacketSampling: false + # Enable NodePortLocal feature to make the Pods reachable externally through NodePort # NodePortLocal: true @@ -6008,6 +6187,12 @@ rules: - pods/status verbs: - patch + - apiGroups: + - "" + resources: + - secrets + verbs: + - get - apiGroups: - "" resources: @@ -6129,6 +6314,20 @@ rules: - patch - create - delete + - apiGroups: + - crd.antrea.io + resources: + - packetsamplings + verbs: + - get + - watch + - list + - apiGroups: + - crd.antrea.io + resources: + - packetsamplings/status + verbs: + - patch - apiGroups: - crd.antrea.io resources: @@ -6810,7 +7009,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 30843b57762c91dfcffb560917191e3bc7e662c06552759bac2a173bc060b82c + checksum/config: 8c4aced766a99cc91b0f614593a3f326865f0b882ad92a105827253ffb98d0db labels: app: antrea component: antrea-agent @@ -7048,7 +7247,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 30843b57762c91dfcffb560917191e3bc7e662c06552759bac2a173bc060b82c + checksum/config: 8c4aced766a99cc91b0f614593a3f326865f0b882ad92a105827253ffb98d0db labels: app: antrea component: antrea-controller @@ -7431,3 +7630,18 @@ webhooks: admissionReviewVersions: ["v1", "v1beta1"] sideEffects: None timeoutSeconds: 5 + - name: "packetsamplingvalidator.antrea.io" + clientConfig: + service: + name: "antrea" + namespace: kube-system + path: "/validate/packetsampling" + rules: + - operations: ["CREATE", "UPDATE"] + apiGroups: ["crd.antrea.io"] + apiVersions: ["v1alpha1"] + resources: ["packetsampling"] + scope: "Cluster" + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + timeoutSeconds: 5 diff --git a/build/yamls/antrea-crds.yml b/build/yamls/antrea-crds.yml index e0497dcf8b2..e71b0148ef2 100644 --- a/build/yamls/antrea-crds.yml +++ b/build/yamls/antrea-crds.yml @@ -4294,6 +4294,180 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + name: packetsamplings.crd.antrea.io + labels: + app: antrea +spec: + group: crd.antrea.io + versions: + - name: v1alpha1 + served: true + storage: true + additionalPrinterColumns: + - jsonPath: .status.phase + description: The phase of the PacketSampling. + name: Phase + type: string + - jsonPath: .spec.source.pod + description: The name of the source Pod. + name: Source-Pod + type: string + priority: 10 + - jsonPath: .spec.destination.pod + description: The name of the destination Pod. + name: Destination-Pod + type: string + priority: 10 + - jsonPath: .spec.destination.ip + description: The IP address of the destination. + name: Destination-IP + type: string + priority: 10 + - jsonPath: .spec.timeout + description: Timeout in seconds. + name: Timeout + type: integer + priority: 10 + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + schema: + openAPIV3Schema: + type: object + required: + - spec + properties: + spec: + type: object + properties: + source: + type: object + properties: + pod: + type: string + namespace: + type: string + ip: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + destination: + type: object + properties: + pod: + type: string + service: + type: string + namespace: + type: string + ip: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + packet: + type: object + properties: + ipHeader: + type: object + properties: + protocol: + type: integer + minimum: 0 + maximum: 255 + ipv6Header: + type: object + properties: + nextHeader: + type: integer + minimum: 0 + maximum: 65535 + transportHeader: + type: object + properties: + udp: + type: object + properties: + srcPort: + type: integer + minimum: 1 + maximum: 65535 + dstPort: + type: integer + minimum: 1 + maximum: 65535 + tcp: + type: object + properties: + srcPort: + type: integer + minimum: 1 + maximum: 65535 + dstPort: + type: integer + minimum: 1 + maximum: 65535 + flags: + type: integer + minimum: 0 + maximum: 255 + timeout: + type: integer + minimum: 1 + maximum: 300 + type: + type: string + firstNSamplingConfig: + type: object + properties: + number: + type: integer + fileServer: + type: object + properties: + url: + type: string + authentication: + type: object + properties: + authType: + type: string + enum: [ "BearerToken", "APIKey", "BasicAuthentication" ] + authSecret: + type: object + properties: + name: + type: string + namespace: + type: string + status: + type: object + properties: + reason: + type: string + phase: + type: string + startTime: + type: string + numCapturedPackets: + type: integer + packetsPath: + type: string + + subresources: + status: {} + scope: Cluster + names: + plural: packetsamplings + singular: packetsampling + kind: PacketSampling + shortNames: + - ps +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: name: supportbundlecollections.crd.antrea.io spec: diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index bec701d3056..24d473777af 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -4313,6 +4313,182 @@ spec: # Deprecated shortName and shall be removed in Antrea v1.14.0 - anp +--- +# Source: crds/packetsampling.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: packetsamplings.crd.antrea.io + labels: + app: antrea +spec: + group: crd.antrea.io + versions: + - name: v1alpha1 + served: true + storage: true + additionalPrinterColumns: + - jsonPath: .status.phase + description: The phase of the PacketSampling. + name: Phase + type: string + - jsonPath: .spec.source.pod + description: The name of the source Pod. + name: Source-Pod + type: string + priority: 10 + - jsonPath: .spec.destination.pod + description: The name of the destination Pod. + name: Destination-Pod + type: string + priority: 10 + - jsonPath: .spec.destination.ip + description: The IP address of the destination. + name: Destination-IP + type: string + priority: 10 + - jsonPath: .spec.timeout + description: Timeout in seconds. + name: Timeout + type: integer + priority: 10 + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + schema: + openAPIV3Schema: + type: object + required: + - spec + properties: + spec: + type: object + properties: + source: + type: object + properties: + pod: + type: string + namespace: + type: string + ip: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + destination: + type: object + properties: + pod: + type: string + service: + type: string + namespace: + type: string + ip: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + packet: + type: object + properties: + ipHeader: + type: object + properties: + protocol: + type: integer + minimum: 0 + maximum: 255 + ipv6Header: + type: object + properties: + nextHeader: + type: integer + minimum: 0 + maximum: 65535 + transportHeader: + type: object + properties: + udp: + type: object + properties: + srcPort: + type: integer + minimum: 1 + maximum: 65535 + dstPort: + type: integer + minimum: 1 + maximum: 65535 + tcp: + type: object + properties: + srcPort: + type: integer + minimum: 1 + maximum: 65535 + dstPort: + type: integer + minimum: 1 + maximum: 65535 + flags: + type: integer + minimum: 0 + maximum: 255 + timeout: + type: integer + minimum: 1 + maximum: 300 + type: + type: string + firstNSamplingConfig: + type: object + properties: + number: + type: integer + fileServer: + type: object + properties: + url: + type: string + authentication: + type: object + properties: + authType: + type: string + enum: [ "BearerToken", "APIKey", "BasicAuthentication" ] + authSecret: + type: object + properties: + name: + type: string + namespace: + type: string + status: + type: object + properties: + reason: + type: string + phase: + type: string + startTime: + type: string + numCapturedPackets: + type: integer + packetsPath: + type: string + + subresources: + status: {} + scope: Cluster + names: + plural: packetsamplings + singular: packetsampling + kind: PacketSampling + shortNames: + - ps + --- # Source: crds/supportbundlecollection.yaml apiVersion: apiextensions.k8s.io/v1 @@ -5449,6 +5625,9 @@ data: # Enable traceflow which provides packet tracing feature to diagnose network issue. # Traceflow: true + # Enable PacketSampling feature which provides packets sampling (capture) feature to diagnose network issue. + # PacketSampling: false + # Enable NodePortLocal feature to make the Pods reachable externally through NodePort # NodePortLocal: true @@ -6008,6 +6187,12 @@ rules: - pods/status verbs: - patch + - apiGroups: + - "" + resources: + - secrets + verbs: + - get - apiGroups: - "" resources: @@ -6129,6 +6314,20 @@ rules: - patch - create - delete + - apiGroups: + - crd.antrea.io + resources: + - packetsamplings + verbs: + - get + - watch + - list + - apiGroups: + - crd.antrea.io + resources: + - packetsamplings/status + verbs: + - patch - apiGroups: - crd.antrea.io resources: @@ -6810,7 +7009,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 30843b57762c91dfcffb560917191e3bc7e662c06552759bac2a173bc060b82c + checksum/config: 8c4aced766a99cc91b0f614593a3f326865f0b882ad92a105827253ffb98d0db labels: app: antrea component: antrea-agent @@ -7049,7 +7248,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 30843b57762c91dfcffb560917191e3bc7e662c06552759bac2a173bc060b82c + checksum/config: 8c4aced766a99cc91b0f614593a3f326865f0b882ad92a105827253ffb98d0db labels: app: antrea component: antrea-controller @@ -7432,3 +7631,18 @@ webhooks: admissionReviewVersions: ["v1", "v1beta1"] sideEffects: None timeoutSeconds: 5 + - name: "packetsamplingvalidator.antrea.io" + clientConfig: + service: + name: "antrea" + namespace: kube-system + path: "/validate/packetsampling" + rules: + - operations: ["CREATE", "UPDATE"] + apiGroups: ["crd.antrea.io"] + apiVersions: ["v1alpha1"] + resources: ["packetsampling"] + scope: "Cluster" + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + timeoutSeconds: 5 diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index a4ae810f7b0..1d2d050b3ed 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -4313,6 +4313,182 @@ spec: # Deprecated shortName and shall be removed in Antrea v1.14.0 - anp +--- +# Source: crds/packetsampling.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: packetsamplings.crd.antrea.io + labels: + app: antrea +spec: + group: crd.antrea.io + versions: + - name: v1alpha1 + served: true + storage: true + additionalPrinterColumns: + - jsonPath: .status.phase + description: The phase of the PacketSampling. + name: Phase + type: string + - jsonPath: .spec.source.pod + description: The name of the source Pod. + name: Source-Pod + type: string + priority: 10 + - jsonPath: .spec.destination.pod + description: The name of the destination Pod. + name: Destination-Pod + type: string + priority: 10 + - jsonPath: .spec.destination.ip + description: The IP address of the destination. + name: Destination-IP + type: string + priority: 10 + - jsonPath: .spec.timeout + description: Timeout in seconds. + name: Timeout + type: integer + priority: 10 + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + schema: + openAPIV3Schema: + type: object + required: + - spec + properties: + spec: + type: object + properties: + source: + type: object + properties: + pod: + type: string + namespace: + type: string + ip: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + destination: + type: object + properties: + pod: + type: string + service: + type: string + namespace: + type: string + ip: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + packet: + type: object + properties: + ipHeader: + type: object + properties: + protocol: + type: integer + minimum: 0 + maximum: 255 + ipv6Header: + type: object + properties: + nextHeader: + type: integer + minimum: 0 + maximum: 65535 + transportHeader: + type: object + properties: + udp: + type: object + properties: + srcPort: + type: integer + minimum: 1 + maximum: 65535 + dstPort: + type: integer + minimum: 1 + maximum: 65535 + tcp: + type: object + properties: + srcPort: + type: integer + minimum: 1 + maximum: 65535 + dstPort: + type: integer + minimum: 1 + maximum: 65535 + flags: + type: integer + minimum: 0 + maximum: 255 + timeout: + type: integer + minimum: 1 + maximum: 300 + type: + type: string + firstNSamplingConfig: + type: object + properties: + number: + type: integer + fileServer: + type: object + properties: + url: + type: string + authentication: + type: object + properties: + authType: + type: string + enum: [ "BearerToken", "APIKey", "BasicAuthentication" ] + authSecret: + type: object + properties: + name: + type: string + namespace: + type: string + status: + type: object + properties: + reason: + type: string + phase: + type: string + startTime: + type: string + numCapturedPackets: + type: integer + packetsPath: + type: string + + subresources: + status: {} + scope: Cluster + names: + plural: packetsamplings + singular: packetsampling + kind: PacketSampling + shortNames: + - ps + --- # Source: crds/supportbundlecollection.yaml apiVersion: apiextensions.k8s.io/v1 @@ -5449,6 +5625,9 @@ data: # Enable traceflow which provides packet tracing feature to diagnose network issue. # Traceflow: true + # Enable PacketSampling feature which provides packets sampling (capture) feature to diagnose network issue. + # PacketSampling: false + # Enable NodePortLocal feature to make the Pods reachable externally through NodePort # NodePortLocal: true @@ -6008,6 +6187,12 @@ rules: - pods/status verbs: - patch + - apiGroups: + - "" + resources: + - secrets + verbs: + - get - apiGroups: - "" resources: @@ -6129,6 +6314,20 @@ rules: - patch - create - delete + - apiGroups: + - crd.antrea.io + resources: + - packetsamplings + verbs: + - get + - watch + - list + - apiGroups: + - crd.antrea.io + resources: + - packetsamplings/status + verbs: + - patch - apiGroups: - crd.antrea.io resources: @@ -6810,7 +7009,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: d5cdb5356795c44a69c66fad1b4d67f7c00cdcbe837f3b3b50260e4d9dfd1e7e + checksum/config: e5155e0f5b5f1d56a497787fbb04ceed66a301b8debf12b489508278f9310463 labels: app: antrea component: antrea-agent @@ -7046,7 +7245,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: d5cdb5356795c44a69c66fad1b4d67f7c00cdcbe837f3b3b50260e4d9dfd1e7e + checksum/config: e5155e0f5b5f1d56a497787fbb04ceed66a301b8debf12b489508278f9310463 labels: app: antrea component: antrea-controller @@ -7429,3 +7628,18 @@ webhooks: admissionReviewVersions: ["v1", "v1beta1"] sideEffects: None timeoutSeconds: 5 + - name: "packetsamplingvalidator.antrea.io" + clientConfig: + service: + name: "antrea" + namespace: kube-system + path: "/validate/packetsampling" + rules: + - operations: ["CREATE", "UPDATE"] + apiGroups: ["crd.antrea.io"] + apiVersions: ["v1alpha1"] + resources: ["packetsampling"] + scope: "Cluster" + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + timeoutSeconds: 5 diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 393cb59da0a..72a42061c83 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -4313,6 +4313,182 @@ spec: # Deprecated shortName and shall be removed in Antrea v1.14.0 - anp +--- +# Source: crds/packetsampling.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: packetsamplings.crd.antrea.io + labels: + app: antrea +spec: + group: crd.antrea.io + versions: + - name: v1alpha1 + served: true + storage: true + additionalPrinterColumns: + - jsonPath: .status.phase + description: The phase of the PacketSampling. + name: Phase + type: string + - jsonPath: .spec.source.pod + description: The name of the source Pod. + name: Source-Pod + type: string + priority: 10 + - jsonPath: .spec.destination.pod + description: The name of the destination Pod. + name: Destination-Pod + type: string + priority: 10 + - jsonPath: .spec.destination.ip + description: The IP address of the destination. + name: Destination-IP + type: string + priority: 10 + - jsonPath: .spec.timeout + description: Timeout in seconds. + name: Timeout + type: integer + priority: 10 + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + schema: + openAPIV3Schema: + type: object + required: + - spec + properties: + spec: + type: object + properties: + source: + type: object + properties: + pod: + type: string + namespace: + type: string + ip: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + destination: + type: object + properties: + pod: + type: string + service: + type: string + namespace: + type: string + ip: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + packet: + type: object + properties: + ipHeader: + type: object + properties: + protocol: + type: integer + minimum: 0 + maximum: 255 + ipv6Header: + type: object + properties: + nextHeader: + type: integer + minimum: 0 + maximum: 65535 + transportHeader: + type: object + properties: + udp: + type: object + properties: + srcPort: + type: integer + minimum: 1 + maximum: 65535 + dstPort: + type: integer + minimum: 1 + maximum: 65535 + tcp: + type: object + properties: + srcPort: + type: integer + minimum: 1 + maximum: 65535 + dstPort: + type: integer + minimum: 1 + maximum: 65535 + flags: + type: integer + minimum: 0 + maximum: 255 + timeout: + type: integer + minimum: 1 + maximum: 300 + type: + type: string + firstNSamplingConfig: + type: object + properties: + number: + type: integer + fileServer: + type: object + properties: + url: + type: string + authentication: + type: object + properties: + authType: + type: string + enum: [ "BearerToken", "APIKey", "BasicAuthentication" ] + authSecret: + type: object + properties: + name: + type: string + namespace: + type: string + status: + type: object + properties: + reason: + type: string + phase: + type: string + startTime: + type: string + numCapturedPackets: + type: integer + packetsPath: + type: string + + subresources: + status: {} + scope: Cluster + names: + plural: packetsamplings + singular: packetsampling + kind: PacketSampling + shortNames: + - ps + --- # Source: crds/supportbundlecollection.yaml apiVersion: apiextensions.k8s.io/v1 @@ -5462,6 +5638,9 @@ data: # Enable traceflow which provides packet tracing feature to diagnose network issue. # Traceflow: true + # Enable PacketSampling feature which provides packets sampling (capture) feature to diagnose network issue. + # PacketSampling: false + # Enable NodePortLocal feature to make the Pods reachable externally through NodePort # NodePortLocal: true @@ -6021,6 +6200,12 @@ rules: - pods/status verbs: - patch + - apiGroups: + - "" + resources: + - secrets + verbs: + - get - apiGroups: - "" resources: @@ -6142,6 +6327,20 @@ rules: - patch - create - delete + - apiGroups: + - crd.antrea.io + resources: + - packetsamplings + verbs: + - get + - watch + - list + - apiGroups: + - crd.antrea.io + resources: + - packetsamplings/status + verbs: + - patch - apiGroups: - crd.antrea.io resources: @@ -6823,7 +7022,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 50f2864cf09e4732327b963130bd59a9fc06c560784b161c94e813c000367615 + checksum/config: 051af47db7b16f375bac60985fb4984c63f443fe27758702613eef508fc9fced checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4 labels: app: antrea @@ -7105,7 +7304,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 50f2864cf09e4732327b963130bd59a9fc06c560784b161c94e813c000367615 + checksum/config: 051af47db7b16f375bac60985fb4984c63f443fe27758702613eef508fc9fced labels: app: antrea component: antrea-controller @@ -7488,3 +7687,18 @@ webhooks: admissionReviewVersions: ["v1", "v1beta1"] sideEffects: None timeoutSeconds: 5 + - name: "packetsamplingvalidator.antrea.io" + clientConfig: + service: + name: "antrea" + namespace: kube-system + path: "/validate/packetsampling" + rules: + - operations: ["CREATE", "UPDATE"] + apiGroups: ["crd.antrea.io"] + apiVersions: ["v1alpha1"] + resources: ["packetsampling"] + scope: "Cluster" + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + timeoutSeconds: 5 diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 2451670ca39..ac804a13f75 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -4313,6 +4313,182 @@ spec: # Deprecated shortName and shall be removed in Antrea v1.14.0 - anp +--- +# Source: crds/packetsampling.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: packetsamplings.crd.antrea.io + labels: + app: antrea +spec: + group: crd.antrea.io + versions: + - name: v1alpha1 + served: true + storage: true + additionalPrinterColumns: + - jsonPath: .status.phase + description: The phase of the PacketSampling. + name: Phase + type: string + - jsonPath: .spec.source.pod + description: The name of the source Pod. + name: Source-Pod + type: string + priority: 10 + - jsonPath: .spec.destination.pod + description: The name of the destination Pod. + name: Destination-Pod + type: string + priority: 10 + - jsonPath: .spec.destination.ip + description: The IP address of the destination. + name: Destination-IP + type: string + priority: 10 + - jsonPath: .spec.timeout + description: Timeout in seconds. + name: Timeout + type: integer + priority: 10 + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + schema: + openAPIV3Schema: + type: object + required: + - spec + properties: + spec: + type: object + properties: + source: + type: object + properties: + pod: + type: string + namespace: + type: string + ip: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + destination: + type: object + properties: + pod: + type: string + service: + type: string + namespace: + type: string + ip: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + packet: + type: object + properties: + ipHeader: + type: object + properties: + protocol: + type: integer + minimum: 0 + maximum: 255 + ipv6Header: + type: object + properties: + nextHeader: + type: integer + minimum: 0 + maximum: 65535 + transportHeader: + type: object + properties: + udp: + type: object + properties: + srcPort: + type: integer + minimum: 1 + maximum: 65535 + dstPort: + type: integer + minimum: 1 + maximum: 65535 + tcp: + type: object + properties: + srcPort: + type: integer + minimum: 1 + maximum: 65535 + dstPort: + type: integer + minimum: 1 + maximum: 65535 + flags: + type: integer + minimum: 0 + maximum: 255 + timeout: + type: integer + minimum: 1 + maximum: 300 + type: + type: string + firstNSamplingConfig: + type: object + properties: + number: + type: integer + fileServer: + type: object + properties: + url: + type: string + authentication: + type: object + properties: + authType: + type: string + enum: [ "BearerToken", "APIKey", "BasicAuthentication" ] + authSecret: + type: object + properties: + name: + type: string + namespace: + type: string + status: + type: object + properties: + reason: + type: string + phase: + type: string + startTime: + type: string + numCapturedPackets: + type: integer + packetsPath: + type: string + + subresources: + status: {} + scope: Cluster + names: + plural: packetsamplings + singular: packetsampling + kind: PacketSampling + shortNames: + - ps + --- # Source: crds/supportbundlecollection.yaml apiVersion: apiextensions.k8s.io/v1 @@ -5449,6 +5625,9 @@ data: # Enable traceflow which provides packet tracing feature to diagnose network issue. # Traceflow: true + # Enable PacketSampling feature which provides packets sampling (capture) feature to diagnose network issue. + # PacketSampling: false + # Enable NodePortLocal feature to make the Pods reachable externally through NodePort # NodePortLocal: true @@ -6008,6 +6187,12 @@ rules: - pods/status verbs: - patch + - apiGroups: + - "" + resources: + - secrets + verbs: + - get - apiGroups: - "" resources: @@ -6129,6 +6314,20 @@ rules: - patch - create - delete + - apiGroups: + - crd.antrea.io + resources: + - packetsamplings + verbs: + - get + - watch + - list + - apiGroups: + - crd.antrea.io + resources: + - packetsamplings/status + verbs: + - patch - apiGroups: - crd.antrea.io resources: @@ -6810,7 +7009,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: ac3c14eed7ca0dc28bf2d659cd2c4e4a39d55278fb9a8759c30ea12eff89e518 + checksum/config: 9f89f6497340a41f1c461aefd52c1c4d255ecc7494a180d88b645e06153dafa9 labels: app: antrea component: antrea-agent @@ -7046,7 +7245,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: ac3c14eed7ca0dc28bf2d659cd2c4e4a39d55278fb9a8759c30ea12eff89e518 + checksum/config: 9f89f6497340a41f1c461aefd52c1c4d255ecc7494a180d88b645e06153dafa9 labels: app: antrea component: antrea-controller @@ -7429,3 +7628,18 @@ webhooks: admissionReviewVersions: ["v1", "v1beta1"] sideEffects: None timeoutSeconds: 5 + - name: "packetsamplingvalidator.antrea.io" + clientConfig: + service: + name: "antrea" + namespace: kube-system + path: "/validate/packetsampling" + rules: + - operations: ["CREATE", "UPDATE"] + apiGroups: ["crd.antrea.io"] + apiVersions: ["v1alpha1"] + resources: ["packetsampling"] + scope: "Cluster" + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + timeoutSeconds: 5 diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh index 0d4b5992c3e..2c6f0f6f389 100755 --- a/ci/kind/test-e2e-kind.sh +++ b/ci/kind/test-e2e-kind.sh @@ -55,6 +55,7 @@ THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" TESTBED_CMD="$THIS_DIR/kind-setup.sh" YML_CMD="$THIS_DIR/../../hack/generate-manifest.sh" FLOWAGGREGATOR_YML_CMD="$THIS_DIR/../../hack/generate-manifest-flow-aggregator.sh" +SFTP_DEPLOYMENT_YML="$THIS_DIR/../../hack/externalnode/sftp-deployment.yml" FLOW_VISIBILITY_HELM_VALUES="$THIS_DIR/values-flow-exporter.yml" CH_OPERATOR_YML="$THIS_DIR/../../build/yamls/clickhouse-operator-install-bundle.yml" FLOW_VISIBILITY_CHART="$THIS_DIR/../../test/e2e/charts/flow-visibility" @@ -320,10 +321,15 @@ function run_test { coverage_args="" flow_visibility_args="" + + # used for PacketSampling tests. + cat "$SFTP_DEPLOYMENT_YML" | docker exec -i kind-control-plane dd of=/root/sftp-deployment.yml + if $use_non_default_images; then export AGENT_IMG_NAME=${antrea_agent_image} export CONTROLLER_IMG_NAME=${antrea_controller_image} fi + if $coverage; then $YML_CMD --encap-mode $current_mode $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea-coverage.yml $YML_CMD --ipsec $manifest_args | docker exec -i kind-control-plane dd of=/root/antrea-ipsec-coverage.yml diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 3678b665fc7..5f2bb859504 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -43,6 +43,7 @@ import ( "antrea.io/antrea/pkg/agent/controller/networkpolicy" "antrea.io/antrea/pkg/agent/controller/networkpolicy/l7engine" "antrea.io/antrea/pkg/agent/controller/noderoute" + "antrea.io/antrea/pkg/agent/controller/packetsampling" "antrea.io/antrea/pkg/agent/controller/serviceexternalip" "antrea.io/antrea/pkg/agent/controller/traceflow" "antrea.io/antrea/pkg/agent/controller/trafficcontrol" @@ -114,6 +115,7 @@ func run(o *Options) error { informerFactory := informers.NewSharedInformerFactory(k8sClient, informerDefaultResync) crdInformerFactory := crdinformers.NewSharedInformerFactory(crdClient, informerDefaultResync) traceflowInformer := crdInformerFactory.Crd().V1beta1().Traceflows() + packetSamplingInformer := crdInformerFactory.Crd().V1alpha1().PacketSamplings() egressInformer := crdInformerFactory.Crd().V1beta1().Egresses() externalIPPoolInformer := crdInformerFactory.Crd().V1beta1().ExternalIPPools() trafficControlInformer := crdInformerFactory.Crd().V1alpha2().TrafficControls() @@ -179,6 +181,7 @@ func run(o *Options) error { enableMulticlusterGW, groupIDAllocator, *o.config.EnablePrometheusMetrics, + features.DefaultFeatureGate.Enabled(features.PacketSampling), o.config.PacketInRate, ) @@ -630,6 +633,20 @@ func run(o *Options) error { o.enableAntreaProxy) } + var packetSamplingController *packetsampling.Controller + if features.DefaultFeatureGate.Enabled(features.PacketSampling) { + packetSamplingController = packetsampling.NewPacketSamplingController( + k8sClient, + crdClient, + serviceInformer, + endpointsInformer, + packetSamplingInformer, + ofClient, + ifaceStore, + nodeConfig, + ) + } + // TODO: we should call this after installing flows for initial node routes // and initial NetworkPolicies so that no packets will be mishandled. if err := agentInitializer.FlowRestoreComplete(); err != nil { @@ -771,6 +788,10 @@ func run(o *Options) error { go traceflowController.Run(stopCh) } + if features.DefaultFeatureGate.Enabled(features.PacketSampling) { + go packetSamplingController.Run(stopCh) + } + if o.enableAntreaProxy { go proxier.GetProxyProvider().Run(stopCh) diff --git a/docs/packetsampling-guide.md b/docs/packetsampling-guide.md new file mode 100644 index 00000000000..cde673a23f7 --- /dev/null +++ b/docs/packetsampling-guide.md @@ -0,0 +1,76 @@ +# PacketSampling User Guide + +Starting with Antrea v2.0, Antrea supports using PacketSampling for network diagnosis. +It can capture specified number of packets from real traffic and upload them to a +supported storage location. Users can create a PacketSampling CRD to trigger +such actions on the target traffic flow. + + +- [Prerequisites](#prerequisites) +- [Start a new PacketSampling](#start-a-new-packetsampling) + + +## Prerequisites + +The PacketSampling feature is disabled by default. If you +want to enable this feature, you need to set PacketSampling feature gate to true in `antrea-config` +ConfigMap for antrea-agent. + +```yaml + antrea-agent.conf: | + featureGates: + # Enable packetsampling feature to capture real traffic packets. + PacketSampling: true +``` + +## Start a new PacketSampling + +When start a new packet sampling, you can provide the following information to identify +the target flow: + +* Source Pod +* Destination Pod, Service or IP address +* Transport protocol (TCP/UDP/ICMP) +* Transport ports + +You can start a new packet sampling by creating a PacketSampling CR via +`kubectl` and a yaml file which contains the essential configuration of +PacketSampling CRD. Following is an example of PacketSampling CR: + +```yaml +apiVersion: crd.antrea.io/v1alpha1 +kind: PacketSampling +metadata: + name: ps-test +spec: + fileServer: + url: sftp://127.0.0.1:22/upload # define your own ftp url here. + authentication: + authType: "BasicAuthentication" + authSecret: + name: test-secret + namespace: default + timeout: 600 + type: FirstNSampling + firstNSamplingConfig: + number: 5 + source: + namespace: default + pod: frontend + destination: + namespace: default + pod: backend + # Destination can also be an IP address ('ip' field) or a Service name ('service' field); the 3 choices are mutually exclusive. + packet: + ipHeader: # If ipHeader/ipv6Header is not set, the default value is IPv4 + ICMP. + protocol: 6 # Protocol here can be 6 (TCP), 17 (UDP) or 1 (ICMP), default value is 1 (ICMP) + transportHeader: + tcp: + dstPort: 8080 # Destination port needs to be set when the protocol is TCP/UDP. +``` + +The CRD above starts a new packet sampling from a Pod named `frontend` +to the port 8080 of a Pod named `backend` using TCP protocol. It will capture the first 5 packets +that meet this criterion and upload them to the file server specified in the PacketSampling's +specifications. Users can download the packet file from the ftp server and analysis its content +with common network diagnose tools like Wireshark or `tcpdump`. diff --git a/go.mod b/go.mod index 80241bb3dc0..21282664ba1 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/go-logr/logr v1.4.1 github.com/gogo/protobuf v1.3.2 github.com/google/btree v1.1.2 + github.com/google/gopacket v1.1.19 github.com/google/uuid v1.6.0 github.com/hashicorp/memberlist v0.5.1 github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.3.0 diff --git a/go.sum b/go.sum index fcbea231037..f6f210facbd 100644 --- a/go.sum +++ b/go.sum @@ -363,6 +363,8 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= +github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -825,6 +827,8 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -984,6 +988,7 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= diff --git a/hack/.notableofcontents b/hack/.notableofcontents index 476abf7da99..99ca1129f53 100644 --- a/hack/.notableofcontents +++ b/hack/.notableofcontents @@ -38,6 +38,7 @@ docs/noencap-hybrid-modes.md docs/octant-plugin-installation.md docs/os-issues.md docs/ovs-offload.md +docs/packetsampling-guide.md docs/prometheus-integration.md docs/secondary-network.md docs/security.md diff --git a/pkg/agent/controller/networkpolicy/audit_logging.go b/pkg/agent/controller/networkpolicy/audit_logging.go index 3dd3a82695b..717b133ccb2 100644 --- a/pkg/agent/controller/networkpolicy/audit_logging.go +++ b/pkg/agent/controller/networkpolicy/audit_logging.go @@ -233,7 +233,7 @@ func getNetworkPolicyInfo(pktIn *ofctrl.PacketIn, packet *binding.Packet, c *Con // Get disposition Allow or Drop. match = getMatchRegField(matchers, openflow.APDispositionField) - disposition, err := getInfoInReg(match, openflow.APDispositionField.GetRange().ToNXRange()) + disposition, err := openflow.GetInfoInReg(match, openflow.APDispositionField.GetRange().ToNXRange()) if err != nil { return fmt.Errorf("received error while unloading disposition from reg: %v", err) } @@ -241,7 +241,7 @@ func getNetworkPolicyInfo(pktIn *ofctrl.PacketIn, packet *binding.Packet, c *Con // Get layer 7 NetworkPolicy redirect action, if traffic is redirected, disposition log should be overwritten. if match = getMatchRegField(matchers, openflow.L7NPRegField); match != nil { - l7NPRegVal, err := getInfoInReg(match, openflow.L7NPRegField.GetRange().ToNXRange()) + l7NPRegVal, err := openflow.GetInfoInReg(match, openflow.L7NPRegField.GetRange().ToNXRange()) if err != nil { return fmt.Errorf("received error while unloading l7 NP redirect value from reg: %v", err) } @@ -252,7 +252,7 @@ func getNetworkPolicyInfo(pktIn *ofctrl.PacketIn, packet *binding.Packet, c *Con // Get K8s default deny action, if traffic is default deny, no conjunction could be matched. if match = getMatchRegField(matchers, openflow.APDenyRegMark.GetField()); match != nil { - apDenyRegVal, err := getInfoInReg(match, openflow.APDenyRegMark.GetField().GetRange().ToNXRange()) + apDenyRegVal, err := openflow.GetInfoInReg(match, openflow.APDenyRegMark.GetField().GetRange().ToNXRange()) if err != nil { return fmt.Errorf("received error while unloading deny mark from reg: %v", err) } @@ -269,7 +269,7 @@ func getNetworkPolicyInfo(pktIn *ofctrl.PacketIn, packet *binding.Packet, c *Con match = getMatch(matchers, tableID, disposition) // Get NetworkPolicy full name and OF priority of the conjunction. - conjID, err := getInfoInReg(match, nil) + conjID, err := openflow.GetInfoInReg(match, nil) if err != nil { return fmt.Errorf("received error while unloading conjunction id from reg: %v", err) } diff --git a/pkg/agent/controller/networkpolicy/packetin.go b/pkg/agent/controller/networkpolicy/packetin.go index ac7de7f95ea..1c1fd2e7403 100644 --- a/pkg/agent/controller/networkpolicy/packetin.go +++ b/pkg/agent/controller/networkpolicy/packetin.go @@ -21,7 +21,6 @@ import ( "net/netip" "time" - "antrea.io/libOpenflow/openflow15" "antrea.io/ofnet/ofctrl" "github.com/vmware/go-ipfix/pkg/registry" "k8s.io/klog/v2" @@ -91,18 +90,6 @@ func getMatch(matchers *ofctrl.Matchers, tableID uint8, disposition uint32) *ofc return nil } -// getInfoInReg unloads and returns data stored in the match field. -func getInfoInReg(regMatch *ofctrl.MatchField, rng *openflow15.NXRange) (uint32, error) { - regValue, ok := regMatch.GetValue().(*ofctrl.NXRegister) - if !ok { - return 0, errors.New("register value cannot be retrieved") - } - if rng != nil { - return ofctrl.GetUint32ValueWithRange(regValue.Data, rng), nil - } - return regValue.Data, nil -} - func (c *Controller) storeDenyConnection(pktIn *ofctrl.PacketIn) error { packet, err := binding.ParsePacketIn(pktIn) if err != nil { @@ -147,7 +134,7 @@ func (c *Controller) storeDenyConnection(pktIn *ofctrl.PacketIn) error { tableID := getPacketInTableID(pktIn) // Get disposition Allow, Drop or Reject match = getMatchRegField(matchers, openflow.APDispositionField) - id, err := getInfoInReg(match, openflow.APDispositionField.GetRange().ToNXRange()) + id, err := openflow.GetInfoInReg(match, openflow.APDispositionField.GetRange().ToNXRange()) if err != nil { return fmt.Errorf("error when getting disposition from reg: %v", err) } @@ -156,7 +143,7 @@ func (c *Controller) storeDenyConnection(pktIn *ofctrl.PacketIn) error { // Set match to corresponding ingress/egress reg according to disposition match = getMatch(matchers, tableID, id) if match != nil { - ruleID, err := getInfoInReg(match, nil) + ruleID, err := openflow.GetInfoInReg(match, nil) if err != nil { return fmt.Errorf("error when obtaining rule id from reg: %v", err) } @@ -223,7 +210,7 @@ func getPacketInTableID(pktIn *ofctrl.PacketIn) uint8 { tableID := pktIn.TableId matchers := pktIn.GetMatches() if match := getMatchRegField(matchers, openflow.PacketInTableField); match != nil { - tableVal, err := getInfoInReg(match, openflow.PacketInTableField.GetRange().ToNXRange()) + tableVal, err := openflow.GetInfoInReg(match, openflow.PacketInTableField.GetRange().ToNXRange()) if err == nil { return uint8(tableVal) } else { diff --git a/pkg/agent/controller/networkpolicy/reject.go b/pkg/agent/controller/networkpolicy/reject.go index 75de5abd6b5..4c2fcc95635 100644 --- a/pkg/agent/controller/networkpolicy/reject.go +++ b/pkg/agent/controller/networkpolicy/reject.go @@ -142,7 +142,7 @@ func (c *Controller) rejectRequest(pktIn *ofctrl.PacketIn) error { if c.antreaProxyEnabled { matches := pktIn.GetMatches() if match := getMatchRegField(matches, openflow.ServiceEPStateField); match != nil { - svcEpstate, err := getInfoInReg(match, openflow.ServiceEPStateField.GetRange().ToNXRange()) + svcEpstate, err := openflow.GetInfoInReg(match, openflow.ServiceEPStateField.GetRange().ToNXRange()) if err != nil { return false } @@ -343,7 +343,7 @@ func parseFlexibleIPAMStatus(pktIn *ofctrl.PacketIn, nodeConfig *config.NodeConf // The generated reject packet should have same ctZone with the incoming packet, otherwise the conntrack cannot work properly. matches := pktIn.GetMatches() if match := getMatchRegField(matches, openflow.CtZoneField); match != nil { - ctZone, err = getInfoInReg(match, openflow.CtZoneField.GetRange().ToNXRange()) + ctZone, err = openflow.GetInfoInReg(match, openflow.CtZoneField.GetRange().ToNXRange()) if err != nil { return false, false, 0, err } diff --git a/pkg/agent/controller/packetsampling/packetin.go b/pkg/agent/controller/packetsampling/packetin.go new file mode 100644 index 00000000000..073a80b5814 --- /dev/null +++ b/pkg/agent/controller/packetsampling/packetin.go @@ -0,0 +1,109 @@ +// Copyright 2024 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package packetsampling + +import ( + "fmt" + "time" + + "antrea.io/libOpenflow/util" + "antrea.io/ofnet/ofctrl" + "github.com/google/gopacket" + "k8s.io/klog/v2" + + "antrea.io/antrea/pkg/agent/openflow" + crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" +) + +// HandlePacketIn processes PacketIn messages from the OFSwitch. If the register value match, it will be counted and captured. +// Once the total number reaches the target, the PacketSampling will be marked as Succeed. +func (c *Controller) HandlePacketIn(pktIn *ofctrl.PacketIn) error { + klog.V(4).InfoS("PacketIn for PacketSampling", "PacketIn", pktIn.PacketIn) + samplingState, samplingFinished, err := c.parsePacketIn(pktIn) + if err != nil { + return fmt.Errorf("parsePacketIn error: %w", err) + } + if samplingFinished { + return nil + } + rawData := pktIn.Data.(*util.Buffer).Bytes() + ci := gopacket.CaptureInfo{ + Timestamp: time.Now(), + CaptureLength: len(rawData), + Length: len(rawData), + } + err = samplingState.pcapngWriter.WritePacket(ci, rawData) + if err != nil { + return fmt.Errorf("couldn't write packet: %w", err) + } + reachTarget := samplingState.numCapturedPackets == samplingState.maxNumCapturedPackets + // use rate limiter to reduce the times we need to update status. + if reachTarget || samplingState.updateRateLimiter.Allow() { + ps, err := c.packetSamplingLister.Get(samplingState.name) + if err != nil { + return fmt.Errorf("get PacketSampling failed: %w", err) + } + // if reach the target. flush the file and upload it. + if reachTarget { + if err := samplingState.pcapngWriter.Flush(); err != nil { + return err + } + if err := c.uploadPackets(ps, samplingState.pcapngFile); err != nil { + return err + } + } + err = c.updatePacketSamplingStatus(ps, crdv1alpha1.PacketSamplingRunning, "", samplingState.numCapturedPackets) + if err != nil { + return fmt.Errorf("failed to update the PacketSampling: %w", err) + } + klog.InfoS("Updated PacketSampling", "PacketSampling", klog.KObj(ps), "numCapturedPackets", samplingState.numCapturedPackets) + } + return nil +} + +// parsePacketIn parses the packet-in message. If the value in register match with existing PacketSampling's state(tag), +// it will be counted. If the total count reach the target, the ovs flow will be uninstalled. +func (c *Controller) parsePacketIn(pktIn *ofctrl.PacketIn) (_ *packetSamplingState, samplingFinished bool, _ error) { + var tag uint8 + matchers := pktIn.GetMatches() + match := openflow.GetMatchFieldByRegID(matchers, openflow.PacketSamplingMark.GetRegID()) + if match != nil { + value, err := openflow.GetInfoInReg(match, openflow.PacketSamplingMark.GetRange().ToNXRange()) + if err != nil { + return nil, false, fmt.Errorf("failed to get PacketSampling tag from packet-in message: %w", err) + } + tag = uint8(value) + } + c.runningPacketSamplingsMutex.Lock() + psState, exists := c.runningPacketSamplings[tag] + if exists { + if psState.numCapturedPackets == psState.maxNumCapturedPackets { + c.runningPacketSamplingsMutex.Unlock() + return nil, true, nil + } + psState.numCapturedPackets++ + if psState.numCapturedPackets == psState.maxNumCapturedPackets { + err := c.ofClient.UninstallPacketSamplingFlows(tag) + if err != nil { + return nil, false, fmt.Errorf("uninstall PacketSampling ovs flow failed: %v", err) + } + } + } + c.runningPacketSamplingsMutex.Unlock() + if !exists { + return nil, false, fmt.Errorf("PacketSampling for dataplane tag %d not found in cache", tag) + } + return psState, false, nil +} diff --git a/pkg/agent/controller/packetsampling/packetin_test.go b/pkg/agent/controller/packetsampling/packetin_test.go new file mode 100644 index 00000000000..d86df2fba7c --- /dev/null +++ b/pkg/agent/controller/packetsampling/packetin_test.go @@ -0,0 +1,259 @@ +// Copyright 2024 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package packetsampling + +import ( + "context" + "net" + "testing" + + "antrea.io/libOpenflow/openflow15" + "antrea.io/libOpenflow/protocol" + "antrea.io/libOpenflow/util" + "antrea.io/ofnet/ofctrl" + "github.com/google/gopacket/layers" + "github.com/google/gopacket/pcapgo" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/crypto/ssh" + "golang.org/x/time/rate" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2" + + "antrea.io/antrea/pkg/agent/config" + "antrea.io/antrea/pkg/agent/openflow" + openflowtest "antrea.io/antrea/pkg/agent/openflow/testing" + crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" +) + +const ( + maxNum = 5 +) + +const ( + testTag = uint8(1) + testUID = "1-2-3-4" + testSFTPUrl = "sftp://10.220.175.92:22/root/packetsamplings" +) + +var ( + // parse to tag(1) + testTagData = []byte{0x11, 0x00, 0x00, 0x11} +) + +func genMatchers() []openflow15.MatchField { + m := generateMatch(openflow.PacketSamplingMark.GetRegID(), testTagData) + matchers := []openflow15.MatchField{m} + return matchers +} + +func generateMatch(regID int, data []byte) openflow15.MatchField { + baseData := make([]byte, 8, 8) + if regID%2 == 0 { + copy(baseData[0:4], data) + } else { + copy(baseData[4:8], data) + } + return openflow15.MatchField{ + Class: openflow15.OXM_CLASS_PACKET_REGS, + // convert reg (4-byte) ID to xreg (8-byte) ID + Field: uint8(regID / 2), + HasMask: false, + Value: &openflow15.ByteArrayField{Data: baseData}, + } +} + +func getTestPacketBytes(dstIP string) []byte { + ipPacket := &protocol.IPv4{ + Version: 0x4, + IHL: 5, + Protocol: uint8(8), + Length: 20, + NWSrc: net.IP(pod1IPv4), + NWDst: net.IP(dstIP), + } + ethernetPkt := protocol.NewEthernet() + ethernetPkt.HWSrc = pod1MAC + ethernetPkt.Ethertype = protocol.IPv4_MSG + ethernetPkt.Data = ipPacket + pktBytes, _ := ethernetPkt.MarshalBinary() + return pktBytes +} + +func generateTestPsState(name string, pcapngFile afero.File, writer *pcapgo.NgWriter, num int32) *packetSamplingState { + return &packetSamplingState{ + name: name, + maxNumCapturedPackets: maxNum, + numCapturedPackets: num, + tag: testTag, + pcapngWriter: writer, + pcapngFile: pcapngFile, + shouldSyncPackets: true, + updateRateLimiter: rate.NewLimiter(rate.Every(samplingStatusUpdatePeriod), 1), + } +} + +func generatePacketSampling(name string) *crdv1alpha1.PacketSampling { + return &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: testUID, + }, + Status: crdv1alpha1.PacketSamplingStatus{}, + Spec: crdv1alpha1.PacketSamplingSpec{ + FirstNSamplingConfig: &crdv1alpha1.FirstNSamplingConfig{ + Number: 5, + }, + FileServer: crdv1alpha1.BundleFileServer{ + URL: testSFTPUrl, + }, + Authentication: crdv1alpha1.BundleServerAuthConfiguration{ + AuthType: crdv1alpha1.BasicAuthentication, + AuthSecret: &v1.SecretReference{ + Name: "AAA", + Namespace: "default", + }, + }, + }, + } +} + +func generateTestSecret() *v1.Secret { + return &v1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "AAA", + Namespace: "default", + }, + Data: map[string][]byte{ + "username": []byte("AAA"), + "password": []byte("BBBCCC"), + }, + } +} + +type testUploader struct { +} + +func (uploader *testUploader) Upload(url string, fileName string, config *ssh.ClientConfig, outputFile afero.File) error { + klog.Info("Called test uploader") + return nil +} + +func TestHandlePacketSamplingPacketIn(t *testing.T) { + + invalidPktBytes := getTestPacketBytes("89.207.132.170") + pktBytesPodToPod := getTestPacketBytes(pod2IPv4) + + // create test os + defaultFS = afero.NewMemMapFs() + defaultFS.MkdirAll("/tmp/packetsampling/packets", 0755) + file, err := defaultFS.Create(uidToPath(testUID)) + if err != nil { + t.Fatal("create pcapng file error: ", err) + } + + testWriter, err := pcapgo.NewNgWriter(file, layers.LinkTypeEthernet) + if err != nil { + t.Fatal("create test pcapng writer failed: ", err) + } + + tests := []struct { + name string + networkConfig *config.NetworkConfig + nodeConfig *config.NodeConfig + psState *packetSamplingState + pktIn *ofctrl.PacketIn + expectedPS *crdv1alpha1.PacketSampling + expectedErrStr string + expectedCalls func(mockOFClient *openflowtest.MockClient) + expectedNum int32 + }{ + { + name: "invalid packets", + psState: generateTestPsState("ps-with-invalid-packet", nil, testWriter, 0), + expectedPS: generatePacketSampling("ps-with-invalid-packet"), + pktIn: &ofctrl.PacketIn{ + PacketIn: &openflow15.PacketIn{ + Data: util.NewBuffer(invalidPktBytes), + }, + }, + expectedErrStr: "parsePacketIn error: PacketSampling for dataplane tag 0 not found in cache", + }, + { + name: "not hitting target number", + psState: generateTestPsState("ps-with-less-num", nil, testWriter, 1), + expectedPS: generatePacketSampling("ps-with-less-num"), + expectedNum: 2, + pktIn: &ofctrl.PacketIn{ + PacketIn: &openflow15.PacketIn{ + Data: util.NewBuffer(pktBytesPodToPod), + Match: openflow15.Match{ + Fields: genMatchers(), + }, + }, + }, + }, + { + name: "hit target number", + psState: generateTestPsState("ps-with-max-num", file, testWriter, maxNum-1), + expectedPS: generatePacketSampling("ps-with-max-num"), + expectedNum: maxNum, + pktIn: &ofctrl.PacketIn{ + PacketIn: &openflow15.PacketIn{ + Data: util.NewBuffer(pktBytesPodToPod), + Match: openflow15.Match{ + Fields: genMatchers(), + }, + }, + }, + expectedCalls: func(mockOFClient *openflowtest.MockClient) { + mockOFClient.EXPECT().UninstallPacketSamplingFlows(testTag) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + psc := newFakePacketSamplingController(t, nil, []runtime.Object{tt.expectedPS}, nil, &config.NodeConfig{Name: "node1"}) + if tt.expectedCalls != nil { + tt.expectedCalls(psc.mockOFClient) + } + stopCh := make(chan struct{}) + defer close(stopCh) + psc.crdInformerFactory.Start(stopCh) + psc.crdInformerFactory.WaitForCacheSync(stopCh) + psc.runningPacketSamplings[tt.psState.tag] = tt.psState + + err := psc.HandlePacketIn(tt.pktIn) + if err == nil { + assert.Equal(t, tt.expectedErrStr, "") + // check target num in status + ps, err := psc.crdClient.CrdV1alpha1().PacketSamplings().Get(context.TODO(), tt.expectedPS.Name, metav1.GetOptions{}) + require.Nil(t, err) + assert.Equal(t, tt.expectedNum, ps.Status.NumCapturedPackets) + } else { + assert.Equal(t, tt.expectedErrStr, err.Error()) + } + + }) + } +} diff --git a/pkg/agent/controller/packetsampling/packetsampling_controller.go b/pkg/agent/controller/packetsampling/packetsampling_controller.go new file mode 100644 index 00000000000..dbe29a5c3d7 --- /dev/null +++ b/pkg/agent/controller/packetsampling/packetsampling_controller.go @@ -0,0 +1,718 @@ +// Copyright 2024 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package packetsampling + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "sync" + "time" + + "antrea.io/libOpenflow/protocol" + "github.com/google/gopacket/layers" + "github.com/google/gopacket/pcapgo" + "github.com/spf13/afero" + "golang.org/x/time/rate" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + coreinformers "k8s.io/client-go/informers/core/v1" + clientset "k8s.io/client-go/kubernetes" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + "antrea.io/antrea/pkg/agent/config" + "antrea.io/antrea/pkg/agent/interfacestore" + "antrea.io/antrea/pkg/agent/openflow" + "antrea.io/antrea/pkg/agent/util" + crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" + clientsetversioned "antrea.io/antrea/pkg/client/clientset/versioned" + crdinformers "antrea.io/antrea/pkg/client/informers/externalversions/crd/v1alpha1" + crdlisters "antrea.io/antrea/pkg/client/listers/crd/v1alpha1" + binding "antrea.io/antrea/pkg/ovs/openflow" + "antrea.io/antrea/pkg/util/ftp" +) + +type StorageProtocolType string + +const ( + sftpProtocol StorageProtocolType = "sftp" +) + +const ( + controllerName = "AntreaAgentPacketSamplingController" + resyncPeriod time.Duration = 0 + + minRetryDelay = 5 * time.Second + maxRetryDelay = 300 * time.Second + + defaultWorkers = 4 + + // 4bits in ovs reg4 + minTagNum uint8 = 1 + maxTagNum uint8 = 15 + + // reason for timeout + samplingTimeoutReason = "PacketSampling timeout" + defaultTimeoutDuration = time.Second * time.Duration(crdv1alpha1.DefaultPacketSamplingTimeout) + + samplingStatusUpdatePeriod = 10 * time.Second +) + +var ( + timeoutCheckInterval = 10 * time.Second + + packetDirectory = getPacketDirectory() + defaultFS = afero.NewOsFs() +) + +func getPacketDirectory() string { + return filepath.Join(os.TempDir(), "antrea", "packetsampling", "packets") +} + +type packetSamplingState struct { + // name is the PacketSampling name + name string + // tag is a node scope unique id for the PacketSampling. It will be write into ovs reg and parsed in packetIn handler + // to match with existing PacketSampling. + tag uint8 + // shouldSyncPackets means this node will be responsible for doing the actual packet capture job. + shouldSyncPackets bool + // numCapturedPackets record how many packets has been captured. Due to the RateLimiter, + // this maybe not be realtime data. + numCapturedPackets int32 + // maxNumCapturedPackets is target number limit for our capture. If numCapturedPackets=maxNumCapturedPackets, means + // the PacketSampling is succeeded. + maxNumCapturedPackets int32 + // updateRateLimiter controls the frequency of the updates to PacketSampling status. + updateRateLimiter *rate.Limiter + // pcapngFile is the file object for the packet file. + pcapngFile afero.File + // pcapngWriter is the writer for the packet file. + pcapngWriter *pcapgo.NgWriter +} + +type Controller struct { + kubeClient clientset.Interface + crdClient clientsetversioned.Interface + serviceLister corelisters.ServiceLister + serviceListerSynced cache.InformerSynced + endpointLister corelisters.EndpointsLister + endpointSynced cache.InformerSynced + packetSamplingInformer crdinformers.PacketSamplingInformer + packetSamplingLister crdlisters.PacketSamplingLister + packetSamplingSynced cache.InformerSynced + ofClient openflow.Client + interfaceStore interfacestore.InterfaceStore + nodeConfig *config.NodeConfig + queue workqueue.RateLimitingInterface + runningPacketSamplingsMutex sync.RWMutex + runningPacketSamplings map[uint8]*packetSamplingState + sftpUploader ftp.UpLoader +} + +func NewPacketSamplingController( + kubeClient clientset.Interface, + crdClient clientsetversioned.Interface, + serviceInformer coreinformers.ServiceInformer, + endpointInformer coreinformers.EndpointsInformer, + packetSamplingInformer crdinformers.PacketSamplingInformer, + client openflow.Client, + interfaceStore interfacestore.InterfaceStore, + nodeConfig *config.NodeConfig, +) *Controller { + c := &Controller{ + kubeClient: kubeClient, + crdClient: crdClient, + packetSamplingInformer: packetSamplingInformer, + packetSamplingLister: packetSamplingInformer.Lister(), + packetSamplingSynced: packetSamplingInformer.Informer().HasSynced, + ofClient: client, + interfaceStore: interfaceStore, + nodeConfig: nodeConfig, + queue: workqueue.NewRateLimitingQueueWithConfig(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), + workqueue.RateLimitingQueueConfig{Name: "packetsampling"}), + runningPacketSamplings: make(map[uint8]*packetSamplingState), + sftpUploader: &ftp.SftpUploader{}, + } + + packetSamplingInformer.Informer().AddEventHandlerWithResyncPeriod(cache.ResourceEventHandlerFuncs{ + AddFunc: c.addPacketSampling, + UpdateFunc: c.updatePacketSampling, + DeleteFunc: c.deletePacketSampling, + }, resyncPeriod) + + c.ofClient.RegisterPacketInHandler(uint8(openflow.PacketInCategoryPS), c) + + c.serviceLister = serviceInformer.Lister() + c.serviceListerSynced = serviceInformer.Informer().HasSynced + c.endpointLister = endpointInformer.Lister() + c.endpointSynced = endpointInformer.Informer().HasSynced + return c +} + +func (c *Controller) enqueuePacketSampling(ps *crdv1alpha1.PacketSampling) { + c.queue.Add(ps.Name) +} + +// Run will create defaultWorkers workers (go routines) which will process the PacketSampling events from the +// workqueue. +func (c *Controller) Run(stopCh <-chan struct{}) { + defer c.queue.ShutDown() + + klog.InfoS("Starting packetsampling controller", "name", controllerName) + defer klog.InfoS("Shutting down packetsampling controller", "name", controllerName) + + cacheSynced := []cache.InformerSynced{c.packetSamplingSynced, c.serviceListerSynced, c.endpointSynced} + if !cache.WaitForNamedCacheSync(controllerName, stopCh, cacheSynced...) { + return + } + + err := defaultFS.MkdirAll(packetDirectory, 0755) + if err != nil { + klog.ErrorS(err, "Couldn't create directory for storing sampling packets", "directory", packetDirectory) + return + } + + go func() { + wait.Until(c.checkPacketSamplingTimeout, timeoutCheckInterval, stopCh) + }() + + for i := 0; i < defaultWorkers; i++ { + go wait.Until(c.worker, time.Second, stopCh) + } + <-stopCh +} + +func (c *Controller) checkPacketSamplingTimeout() { + c.runningPacketSamplingsMutex.RLock() + ss := make([]string, 0, len(c.runningPacketSamplings)) + for _, psState := range c.runningPacketSamplings { + ss = append(ss, psState.name) + } + c.runningPacketSamplingsMutex.RUnlock() + for _, psName := range ss { + // Re-post all running PacketSampling requests to the work queue to + // be processed and checked for timeout. + c.queue.Add(psName) + } +} + +func (c *Controller) addPacketSampling(obj interface{}) { + ps := obj.(*crdv1alpha1.PacketSampling) + klog.InfoS("Processing PacketSampling ADD event", "name", ps.Name) + c.enqueuePacketSampling(ps) +} + +func (c *Controller) updatePacketSampling(_, obj interface{}) { + ps := obj.(*crdv1alpha1.PacketSampling) + klog.InfoS("Processing PacketSampling UPDATE EVENT", "name", ps.Name) + c.enqueuePacketSampling(ps) +} + +func (c *Controller) deletePacketSampling(obj interface{}) { + ps := obj.(*crdv1alpha1.PacketSampling) + klog.InfoS("Processing PacketSampling DELETE event", "name", ps.Name) + err := deletePcapngFile(string(ps.UID)) + if err != nil { + klog.ErrorS(err, "Couldn't delete pcapng file") + } + c.enqueuePacketSampling(ps) +} + +func deletePcapngFile(uid string) error { + return defaultFS.Remove(uidToPath(uid)) +} + +func uidToPath(uid string) string { + return filepath.Join(packetDirectory, uid+".pcapng") +} + +func (c *Controller) worker() { + for c.processPacketSamplingItem() { + } +} + +func (c *Controller) processPacketSamplingItem() bool { + obj, quit := c.queue.Get() + if quit { + return false + } + + defer c.queue.Done(obj) + if key, ok := obj.(string); !ok { + c.queue.Forget(obj) + klog.ErrorS(nil, "Expected string in work queue but got", "obj", obj) + return true + } else if err := c.syncPacketSampling(key); err == nil { + c.queue.Forget(key) + } else { + klog.ErrorS(err, "Error syncing PacketSampling, exiting", "key", key) + } + return true +} + +func (c *Controller) cleanupPacketSampling(psName string) { + psState := c.deletePacketSamplingState(psName) + if psState != nil { + err := c.ofClient.UninstallPacketSamplingFlows(psState.tag) + if err != nil { + klog.ErrorS(err, "Error cleaning up flows for PacketSampling", "name", psName) + } + if err := psState.pcapngFile.Close(); err != nil { + klog.ErrorS(err, "Error closing pcap file", "name", psName) + } + } +} + +func (c *Controller) deletePacketSamplingState(psName string) *packetSamplingState { + c.runningPacketSamplingsMutex.Lock() + defer c.runningPacketSamplingsMutex.Unlock() + + for tag, state := range c.runningPacketSamplings { + if state.name == psName { + delete(c.runningPacketSamplings, tag) + return state + } + } + return nil +} + +func (c *Controller) startPacketSampling(ps *crdv1alpha1.PacketSampling, psState *packetSamplingState) error { + var err error + defer func() { + if err != nil { + c.cleanupPacketSampling(ps.Name) + c.updatePacketSamplingStatus(ps, crdv1alpha1.PacketSamplingFailed, fmt.Sprintf("Node: %s, error:%+v", c.nodeConfig.Name, err), 0) + + } + }() + receiverOnly := false + senderOnly := false + var pod, ns string + + if ps.Spec.Source.Pod != "" { + pod = ps.Spec.Source.Pod + ns = ps.Spec.Source.Namespace + if ps.Spec.Destination.Pod == "" { + senderOnly = true + } + } else { + pod = ps.Spec.Destination.Pod + ns = ps.Spec.Destination.Namespace + receiverOnly = true + } + + podInterfaces := c.interfaceStore.GetContainerInterfacesByPod(pod, ns) + psState.shouldSyncPackets = len(podInterfaces) > 0 + if !psState.shouldSyncPackets { + return nil + } + var packet, senderPacket *binding.Packet + var endpointPackets []binding.Packet + var ofPort uint32 + packet, err = c.preparePacket(ps, podInterfaces[0], receiverOnly) + if err != nil { + return err + } + ofPort = uint32(podInterfaces[0].OFPort) + senderPacket = packet + klog.V(2).InfoS("PacketSampling sender packet", "packet", *packet) + if senderOnly && ps.Spec.Destination.Service != "" { + endpointPackets, err = c.genEndpointMatchPackets(ps) + if err != nil { + return fmt.Errorf("couldn't generate endpoint match packets: %w", err) + } + } + + c.runningPacketSamplingsMutex.Lock() + psState.maxNumCapturedPackets = ps.Spec.FirstNSamplingConfig.Number + var file afero.File + filePath := uidToPath(string(ps.UID)) + if _, err := os.Stat(filePath); err == nil { + return fmt.Errorf("packet file already exists. this may be due to an unexpected termination") + } else if os.IsNotExist(err) { + file, err = defaultFS.Create(filePath) + if err != nil { + return fmt.Errorf("failed to create pcapng file: %w", err) + } + } else { + return fmt.Errorf("couldn't check if the file exists: %w", err) + } + writer, err := pcapgo.NewNgWriter(file, layers.LinkTypeEthernet) + if err != nil { + return fmt.Errorf("couldn't init pcap writer: %w", err) + } + psState.shouldSyncPackets = len(podInterfaces) > 0 + psState.pcapngFile = file + psState.pcapngWriter = writer + psState.updateRateLimiter = rate.NewLimiter(rate.Every(samplingStatusUpdatePeriod), 1) + c.runningPacketSamplings[psState.tag] = psState + c.runningPacketSamplingsMutex.Unlock() + + timeout := ps.Spec.Timeout + if timeout == 0 { + timeout = crdv1alpha1.DefaultPacketSamplingTimeout + } + klog.V(2).InfoS("Installing flow entries for PacketSampling", "name", ps.Name) + err = c.ofClient.InstallPacketSamplingFlows(psState.tag, senderOnly, receiverOnly, senderPacket, endpointPackets, ofPort, timeout) + if err != nil { + klog.ErrorS(err, "Install flow entries failed", "name", ps.Name) + } + return err +} + +// genEndpointMatchPackets generates match packets (with destination Endpoint's IP/port info) besides the normal match packet. +// these match packets will help the pipeline to capture the pod -> svc traffic. +// TODO: 1. support name based port name 2. dual-stack support +func (c *Controller) genEndpointMatchPackets(ps *crdv1alpha1.PacketSampling) ([]binding.Packet, error) { + var port int32 + if ps.Spec.Packet.TransportHeader.TCP != nil { + port = ps.Spec.Packet.TransportHeader.TCP.DstPort + } else if ps.Spec.Packet.TransportHeader.UDP != nil { + port = ps.Spec.Packet.TransportHeader.UDP.DstPort + } + var packets []binding.Packet + dstSvc, err := c.serviceLister.Services(ps.Spec.Destination.Namespace).Get(ps.Spec.Destination.Service) + if err != nil { + return nil, err + } + for _, item := range dstSvc.Spec.Ports { + if item.Port == port { + if item.TargetPort.Type == intstr.Int { + port = item.TargetPort.IntVal + } + } + } + dstEndpoint, err := c.endpointLister.Endpoints(ps.Spec.Destination.Namespace).Get(ps.Spec.Destination.Service) + if err != nil { + return nil, err + } + for _, item := range dstEndpoint.Subsets[0].Addresses { + packet := binding.Packet{} + packet.DestinationIP = net.ParseIP(item.IP) + if port != 0 { + packet.DestinationPort = uint16(port) + } + packet.IPProto, _ = parseTargetProto(&ps.Spec.Packet) + packets = append(packets, packet) + } + return packets, nil +} + +func (c *Controller) preparePacket(ps *crdv1alpha1.PacketSampling, intf *interfacestore.InterfaceConfig, receiverOnly bool) (*binding.Packet, error) { + packet := new(binding.Packet) + packet.IsIPv6 = ps.Spec.Packet.IPv6Header != nil + + if receiverOnly { + if ps.Spec.Source.IP != "" { + packet.SourceIP = net.ParseIP(ps.Spec.Source.IP) + } + packet.DestinationMAC = intf.MAC + } else if ps.Spec.Destination.IP != "" { + packet.DestinationIP = net.ParseIP(ps.Spec.Destination.IP) + } else if ps.Spec.Destination.Pod != "" { + dstPodInterfaces := c.interfaceStore.GetContainerInterfacesByPod(ps.Spec.Destination.Pod, ps.Spec.Destination.Namespace) + if len(dstPodInterfaces) > 0 { + if packet.IsIPv6 { + packet.DestinationIP = dstPodInterfaces[0].GetIPv6Addr() + } else { + packet.DestinationIP = dstPodInterfaces[0].GetIPv4Addr() + } + } else { + dstPod, err := c.kubeClient.CoreV1().Pods(ps.Spec.Destination.Namespace).Get(context.TODO(), ps.Spec.Destination.Pod, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get the destination pod %s/%s: %v", ps.Spec.Destination.Namespace, ps.Spec.Destination.Pod, err) + } + podIPs := make([]net.IP, len(dstPod.Status.PodIPs)) + for i, ip := range dstPod.Status.PodIPs { + podIPs[i] = net.ParseIP(ip.IP) + } + if packet.IsIPv6 { + packet.DestinationIP, _ = util.GetIPWithFamily(podIPs, util.FamilyIPv6) + } else { + packet.DestinationIP = util.GetIPv4Addr(podIPs) + } + } + if packet.DestinationIP == nil { + if packet.IsIPv6 { + return nil, errors.New("destination Pod does not have an IPv6 address") + } + return nil, errors.New("destination Pod does not have an IPv4 address") + } + } else if ps.Spec.Destination.Service != "" { + dstSvc, err := c.serviceLister.Services(ps.Spec.Destination.Namespace).Get(ps.Spec.Destination.Service) + if err != nil { + return nil, fmt.Errorf("failed to get the destination service %s/%s: %v", ps.Spec.Destination.Namespace, ps.Spec.Destination.Service, err) + } + if dstSvc.Spec.ClusterIP == "" { + return nil, errors.New("destination Service does not have a ClusterIP") + } + + packet.DestinationIP = net.ParseIP(dstSvc.Spec.ClusterIP) + if !packet.IsIPv6 { + packet.DestinationIP = packet.DestinationIP.To4() + if packet.DestinationIP == nil { + return nil, errors.New("destination Service does not have an IPv4 address") + } + } else if packet.DestinationIP.To4() != nil { + return nil, errors.New("destination Service does not have an IPv6 address") + } + } else { + return nil, errors.New("destination is not specified") + } + + if ps.Spec.Packet.TransportHeader.TCP != nil { + packet.SourcePort = uint16(ps.Spec.Packet.TransportHeader.TCP.SrcPort) + packet.DestinationPort = uint16(ps.Spec.Packet.TransportHeader.TCP.DstPort) + if ps.Spec.Packet.TransportHeader.TCP.Flags != 0 { + packet.TCPFlags = uint8(ps.Spec.Packet.TransportHeader.TCP.Flags) + } + } else if ps.Spec.Packet.TransportHeader.UDP != nil { + packet.SourcePort = uint16(ps.Spec.Packet.TransportHeader.UDP.SrcPort) + packet.DestinationPort = uint16(ps.Spec.Packet.TransportHeader.UDP.DstPort) + } + + proto, err := parseTargetProto(&ps.Spec.Packet) + if err != nil { + return nil, err + } + packet.IPProto = proto + return packet, nil +} + +func parseTargetProto(packet *crdv1alpha1.Packet) (uint8, error) { + var ipProto uint8 + var isIPv6 bool + if packet.IPv6Header != nil { + isIPv6 = true + if packet.IPv6Header.NextHeader != nil { + ipProto = uint8(*packet.IPv6Header.NextHeader) + } + } else if packet.IPHeader.Protocol != 0 { + ipProto = uint8(packet.IPHeader.Protocol) + } + + proto2 := ipProto + if packet.TransportHeader.TCP != nil { + proto2 = protocol.Type_TCP + } else if packet.TransportHeader.UDP != nil { + proto2 = protocol.Type_UDP + } else if packet.TransportHeader.ICMP != nil || ipProto == 0 { + proto2 = protocol.Type_ICMP + if isIPv6 { + proto2 = protocol.Type_IPv6ICMP + } + } + + if ipProto != 0 && proto2 != ipProto { + return 0, errors.New("conflicting protocol settings in ipHeader and transportHeader") + } + return proto2, nil +} + +func (c *Controller) syncPacketSampling(psName string) error { + startTime := time.Now() + defer func() { + klog.V(4).InfoS("Finished syncing PacketSampling", "name", psName, "startTime", time.Since(startTime)) + }() + + ps, err := c.packetSamplingLister.Get(psName) + if err != nil { + if apierrors.IsNotFound(err) { + c.cleanupPacketSampling(psName) + return nil + } + return err + } + + switch ps.Status.Phase { + case "": + err = c.initPacketSampling(ps) + case crdv1alpha1.PacketSamplingRunning: + err = c.checkPacketSamplingStatus(ps) + default: + c.cleanupPacketSampling(psName) + } + return err + +} + +// Allocates a tag. If the PacketSampling request has been allocated with a tag +// already, 0 is returned. If number of existing PacketSampling requests reaches +// the upper limit, an error is returned. +func (c *Controller) allocateTag(name string) (uint8, error) { + c.runningPacketSamplingsMutex.Lock() + defer c.runningPacketSamplingsMutex.Unlock() + + for _, state := range c.runningPacketSamplings { + if state != nil && state.name == name { + // The packetsampling request has been processed already. + return 0, nil + } + } + for i := minTagNum; i <= maxTagNum; i += 1 { + if _, ok := c.runningPacketSamplings[i]; !ok { + c.runningPacketSamplings[i] = &packetSamplingState{ + name: name, + tag: i, + } + return i, nil + } + } + return 0, fmt.Errorf("number of on-going PacketSampling operations already reached the upper limit: %d", maxTagNum) +} + +func (c *Controller) getUploaderByProtocol(protocol StorageProtocolType) (ftp.UpLoader, error) { + if protocol == sftpProtocol { + return c.sftpUploader, nil + } + return nil, fmt.Errorf("unsupported protocol %s", protocol) +} + +func (c *Controller) generatePacketsPathForServer(name string) string { + return name + ".pcapng" +} + +func (c *Controller) uploadPackets(ps *crdv1alpha1.PacketSampling, outputFile afero.File) error { + klog.V(2).InfoS("Uploading captured packets for PacketSampling", "name", ps.Name) + uploader, err := c.getUploaderByProtocol(sftpProtocol) + if err != nil { + return fmt.Errorf("failed to upload support bundle while getting uploader: %v", err) + } + serverAuth, err := ftp.ParseBundleAuth(ps.Spec.Authentication, c.kubeClient) + if err != nil { + klog.ErrorS(err, "Failed to get authentication defined in the PacketSampling CR", "name", ps.Name, "authentication", ps.Spec.Authentication) + return err + } + cfg := ftp.GenSSHClientConfig(serverAuth.BasicAuthentication.Username, serverAuth.BasicAuthentication.Password) + return uploader.Upload(ps.Spec.FileServer.URL, c.generatePacketsPathForServer(string(ps.UID)), cfg, outputFile) + +} + +// initPacketSampling mark the packetsampling as running and allocate tag for it, then start the sampling. the tag will +// serve as a unique id for concurrent processing. +func (c *Controller) initPacketSampling(ps *crdv1alpha1.PacketSampling) error { + tag, err := c.allocateTag(ps.Name) + if err != nil { + return err + } + if tag == 0 { + return nil + } + err = c.updatePacketSamplingStatus(ps, crdv1alpha1.PacketSamplingRunning, "", 0) + if err != nil { + c.deallocateTag(ps.Name, tag) + return err + } + return c.startPacketSampling(ps, c.runningPacketSamplings[tag]) +} + +func (c *Controller) updatePacketSamplingStatus(ps *crdv1alpha1.PacketSampling, phase crdv1alpha1.PacketSamplingPhase, reason string, numCapturedPackets int32) error { + type PacketSampling struct { + Status crdv1alpha1.PacketSamplingStatus `json:"status,omitempty"` + } + patchData := PacketSampling{Status: crdv1alpha1.PacketSamplingStatus{Phase: phase}} + if phase == crdv1alpha1.PacketSamplingRunning && ps.Status.StartTime == nil { + t := metav1.Now() + patchData.Status.StartTime = &t + } + if reason != "" { + patchData.Status.Reason = reason + } + if numCapturedPackets != 0 { + patchData.Status.NumCapturedPackets = numCapturedPackets + } + if phase == crdv1alpha1.PacketSamplingSucceeded { + patchData.Status.PacketsPath = c.generatePacketsPathForServer(string(ps.UID)) + } + payloads, _ := json.Marshal(patchData) + _, err := c.crdClient.CrdV1alpha1().PacketSamplings().Patch(context.TODO(), ps.Name, types.MergePatchType, payloads, metav1.PatchOptions{}, "status") + return err +} + +func (c *Controller) deallocateTag(name string, tag uint8) { + c.runningPacketSamplingsMutex.Lock() + defer c.runningPacketSamplingsMutex.Unlock() + if state, ok := c.runningPacketSamplings[tag]; ok { + if state != nil && name == state.name { + delete(c.runningPacketSamplings, tag) + } + } +} + +func (c *Controller) getTagForPacketSampling(name string) uint8 { + c.runningPacketSamplingsMutex.Lock() + defer c.runningPacketSamplingsMutex.Unlock() + for tag, state := range c.runningPacketSamplings { + if state != nil && state.name == name { + // The packetsampling request has been processed already. + return tag + } + } + return 0 +} + +// checkPacketSamplingStatus is only called for PacketSamplings in the Running phase +func (c *Controller) checkPacketSamplingStatus(ps *crdv1alpha1.PacketSampling) error { + tag := c.getTagForPacketSampling(ps.Name) + if tag == 0 { + return nil + } + if checkPacketSamplingSucceeded(ps) { + c.deallocateTag(ps.Name, tag) + return c.updatePacketSamplingStatus(ps, crdv1alpha1.PacketSamplingSucceeded, "", 0) + } + + if isPacketSamplingTimeout(ps) { + c.deallocateTag(ps.Name, tag) + return c.updatePacketSamplingStatus(ps, crdv1alpha1.PacketSamplingFailed, samplingTimeoutReason, 0) + } + return nil +} + +func checkPacketSamplingSucceeded(ps *crdv1alpha1.PacketSampling) bool { + succeeded := false + if ps.Spec.Type == crdv1alpha1.FirstNSampling && ps.Status.NumCapturedPackets == ps.Spec.FirstNSamplingConfig.Number { + succeeded = true + } + return succeeded +} + +func isPacketSamplingTimeout(ps *crdv1alpha1.PacketSampling) bool { + var timeout time.Duration + if ps.Spec.Timeout != 0 { + timeout = time.Duration(ps.Spec.Timeout) * time.Second + } else { + timeout = defaultTimeoutDuration + } + var startTime time.Time + if ps.Status.StartTime != nil { + startTime = ps.Status.StartTime.Time + } else { + klog.V(2).InfoS("StartTime field in PacketSampling Status should not be empty", "PacketSampling", klog.KObj(ps)) + startTime = ps.CreationTimestamp.Time + } + return startTime.Add(timeout).Before(time.Now()) +} diff --git a/pkg/agent/controller/packetsampling/packetsampling_controller_test.go b/pkg/agent/controller/packetsampling/packetsampling_controller_test.go new file mode 100644 index 00000000000..27d486873a9 --- /dev/null +++ b/pkg/agent/controller/packetsampling/packetsampling_controller_test.go @@ -0,0 +1,977 @@ +// Copyright 2024 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package packetsampling + +import ( + "bytes" + "net" + "os" + "reflect" + "testing" + "time" + + "github.com/google/gopacket/layers" + "github.com/google/gopacket/pcapgo" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/klog/v2" + "antrea.io/libOpenflow/protocol" + + "antrea.io/antrea/pkg/agent/config" + "antrea.io/antrea/pkg/agent/interfacestore" + openflowtest "antrea.io/antrea/pkg/agent/openflow/testing" + "antrea.io/antrea/pkg/agent/util" + crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" + fakeversioned "antrea.io/antrea/pkg/client/clientset/versioned/fake" + crdinformers "antrea.io/antrea/pkg/client/informers/externalversions" + binding "antrea.io/antrea/pkg/ovs/openflow" + "antrea.io/antrea/pkg/util/k8s" +) + +var ( + pod1IPv4 = "192.168.10.10" + pod2IPv4 = "192.168.11.10" + service1IPv4 = "10.96.0.10" + dstIPv4 = "192.168.99.99" + pod1MAC, _ = net.ParseMAC("aa:bb:cc:dd:ee:0f") + pod2MAC, _ = net.ParseMAC("aa:bb:cc:dd:ee:00") + ofPortPod1 = uint32(1) + ofPortPod2 = uint32(2) + protocolICMPv6 = int32(58) + + pod1 = v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-1", + Namespace: "default", + }, + Status: v1.PodStatus{ + PodIP: pod1IPv4, + }, + } + pod2 = v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-2", + Namespace: "default", + }, + Status: v1.PodStatus{ + PodIP: pod2IPv4, + }, + } + pod3 = v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-3", + Namespace: "default", + }, + } + + service1 = v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "service-1", + Namespace: "default", + }, + Spec: v1.ServiceSpec{ + ClusterIP: service1IPv4, + }, + } +) + +type fakePacketSamplingController struct { + *Controller + kubeClient kubernetes.Interface + mockController *gomock.Controller + mockOFClient *openflowtest.MockClient + crdClient *fakeversioned.Clientset + crdInformerFactory crdinformers.SharedInformerFactory + informerFactory informers.SharedInformerFactory +} + +func newFakePacketSamplingController(t *testing.T, runtimeObjects []runtime.Object, initObjects []runtime.Object, networkConfig *config.NetworkConfig, nodeConfig *config.NodeConfig) *fakePacketSamplingController { + controller := gomock.NewController(t) + objs := []runtime.Object{ + &pod1, + &pod2, + &pod3, + &service1, + } + objs = append(objs, generateTestSecret()) + if runtimeObjects != nil { + objs = append(objs, runtimeObjects...) + } + kubeClient := fake.NewSimpleClientset(objs...) + mockOFClient := openflowtest.NewMockClient(controller) + crdClient := fakeversioned.NewSimpleClientset(initObjects...) + crdInformerFactory := crdinformers.NewSharedInformerFactory(crdClient, 0) + packetSamplingInformer := crdInformerFactory.Crd().V1alpha1().PacketSamplings() + informerFactory := informers.NewSharedInformerFactory(kubeClient, 0) + serviceInformer := informerFactory.Core().V1().Services() + endpointInformer := informerFactory.Core().V1().Endpoints() + + ifaceStore := interfacestore.NewInterfaceStore() + addPodInterface(ifaceStore, pod1.Namespace, pod1.Name, pod1IPv4, pod1MAC.String(), int32(ofPortPod1)) + addPodInterface(ifaceStore, pod2.Namespace, pod2.Name, pod2IPv4, pod2MAC.String(), int32(ofPortPod2)) + + mockOFClient.EXPECT().RegisterPacketInHandler(gomock.Any(), gomock.Any()).Times(1) + psController := NewPacketSamplingController( + kubeClient, + crdClient, + serviceInformer, + endpointInformer, + packetSamplingInformer, + mockOFClient, + ifaceStore, + nodeConfig, + ) + psController.sftpUploader = &testUploader{} + + return &fakePacketSamplingController{ + Controller: psController, + kubeClient: kubeClient, + mockController: controller, + mockOFClient: mockOFClient, + crdClient: crdClient, + crdInformerFactory: crdInformerFactory, + informerFactory: informerFactory, + } +} + +func addPodInterface(ifaceStore interfacestore.InterfaceStore, podNamespace, podName, podIP, podMac string, ofPort int32) { + containerName := k8s.NamespacedName(podNamespace, podName) + ifIPs := []net.IP{net.ParseIP(podIP)} + mac, _ := net.ParseMAC(podMac) + ifaceStore.AddInterface(&interfacestore.InterfaceConfig{ + IPs: ifIPs, + MAC: mac, + InterfaceName: util.GenerateContainerInterfaceName(podName, podNamespace, containerName), + ContainerInterfaceConfig: &interfacestore.ContainerInterfaceConfig{PodName: podName, PodNamespace: podNamespace, ContainerID: containerName}, + OVSPortConfig: &interfacestore.OVSPortConfig{OFPort: ofPort}, + }) +} + +func TestErrPacketSamplingCRD(t *testing.T) { + ps := &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ps", + UID: "uid", + }, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: pod1.Namespace, + Pod: pod1.Name, + }, + Destination: crdv1alpha1.Destination{ + Namespace: pod2.Namespace, + Pod: pod2.Name, + }, + }, + Status: crdv1alpha1.PacketSamplingStatus{ + Phase: crdv1alpha1.PacketSamplingRunning, + }, + } + expectedPS := ps + reason := "failed" + expectedPS.Status.Phase = crdv1alpha1.PacketSamplingFailed + expectedPS.Status.Reason = reason + + psc := newFakePacketSamplingController(t, nil, []runtime.Object{ps}, nil, nil) + + err := psc.updatePacketSamplingStatus(ps, crdv1alpha1.PacketSamplingFailed, reason, 0) + require.NoError(t, err) +} + +func TestPreparePacket(t *testing.T) { + pss := []struct { + name string + ps *crdv1alpha1.PacketSampling + intf *interfacestore.InterfaceConfig + receiverOnly bool + expectedPacket *binding.Packet + expectedErr string + }{ + { + name: "empty destination", + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{Name: "ps2", UID: "uid2"}, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: pod1.Namespace, + Pod: pod1.Name, + }, + }, + }, + expectedErr: "destination is not specified", + }, + { + name: "ipv4 tcp packet", + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{Name: "ps3", UID: "uid3"}, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: pod1.Namespace, + Pod: pod1.Name, + }, + Destination: crdv1alpha1.Destination{ + Namespace: pod2.Namespace, + Pod: pod2.Name, + }, + Packet: crdv1alpha1.Packet{ + TransportHeader: crdv1alpha1.TransportHeader{ + TCP: &crdv1alpha1.TCPHeader{ + SrcPort: 80, + DstPort: 81, + Flags: 11, + }, + }, + }, + }, + }, + expectedPacket: &binding.Packet{ + DestinationIP: net.ParseIP(pod2IPv4), + IPProto: protocol.Type_TCP, + SourcePort: 80, + DestinationPort: 81, + TCPFlags: 11, + }, + }, + { + name: "receiver only with source ip", + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{Name: "ps4", UID: "uid4"}, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + IP: "192.168.12.4", + }, + Destination: crdv1alpha1.Destination{ + Namespace: pod1.Namespace, + Pod: pod1.Name, + }, + Packet: crdv1alpha1.Packet{ + IPHeader: crdv1alpha1.IPHeader{}, + }, + }, + }, + receiverOnly: true, + expectedPacket: &binding.Packet{ + SourceIP: net.ParseIP("192.168.12.4"), + DestinationMAC: pod1MAC, + IPProto: 1, + }, + }, + { + name: "destination Pod without IPv6 address", + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{Name: "ps4", UID: "uid4"}, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: pod1.Namespace, + Pod: pod1.Name, + }, + Destination: crdv1alpha1.Destination{ + Namespace: pod2.Namespace, + Pod: pod2.Name, + }, + Packet: crdv1alpha1.Packet{ + IPv6Header: &crdv1alpha1.IPv6Header{}, + }, + }, + }, + expectedErr: "destination Pod does not have an IPv6 address", + }, + { + name: "pod to ipv6 packet sampling", + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{Name: "ps5", UID: "uid5"}, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: pod1.Namespace, + Pod: pod1.Name, + }, + Destination: crdv1alpha1.Destination{ + IP: "2001:db8::68", + }, + Packet: crdv1alpha1.Packet{ + IPv6Header: &crdv1alpha1.IPv6Header{NextHeader: &protocolICMPv6}, + }, + }, + }, + expectedPacket: &binding.Packet{ + IsIPv6: true, + DestinationIP: net.ParseIP("2001:db8::68"), + IPProto: protocol.Type_IPv6ICMP, + }, + }, + { + name: "tcp packet without flags", + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{Name: "ps6", UID: "uid6"}, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: pod1.Namespace, + Pod: pod1.Name, + }, + Destination: crdv1alpha1.Destination{ + Namespace: pod2.Namespace, + Pod: pod2.Name, + }, + Packet: crdv1alpha1.Packet{ + TransportHeader: crdv1alpha1.TransportHeader{ + TCP: &crdv1alpha1.TCPHeader{ + SrcPort: 80, + DstPort: 81, + }, + }, + }, + }, + }, + expectedPacket: &binding.Packet{ + DestinationIP: net.ParseIP(pod2IPv4), + IPProto: protocol.Type_TCP, + SourcePort: 80, + DestinationPort: 81, + }, + }, + { + name: "udp packet", + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{Name: "ps7", UID: "uid7"}, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: pod1.Namespace, + Pod: pod1.Name, + }, + Destination: crdv1alpha1.Destination{ + Namespace: pod2.Namespace, + Pod: pod2.Name, + }, + Packet: crdv1alpha1.Packet{ + TransportHeader: crdv1alpha1.TransportHeader{ + UDP: &crdv1alpha1.UDPHeader{ + SrcPort: 80, + DstPort: 100, + }, + }, + }, + }, + }, + expectedPacket: &binding.Packet{ + DestinationIP: net.ParseIP(pod2IPv4), + IPProto: protocol.Type_UDP, + SourcePort: 80, + DestinationPort: 100, + }, + }, + { + name: "icmp packet", + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{Name: "ps8", UID: "uid8"}, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: pod1.Namespace, + Pod: pod1.Name, + }, + Destination: crdv1alpha1.Destination{ + Namespace: pod2.Namespace, + Pod: pod2.Name, + }, + Packet: crdv1alpha1.Packet{ + TransportHeader: crdv1alpha1.TransportHeader{ + ICMP: &crdv1alpha1.ICMPEchoRequestHeader{}, + }, + }, + }, + }, + expectedPacket: &binding.Packet{ + DestinationIP: net.ParseIP(pod2IPv4), + IPProto: protocol.Type_ICMP, + }, + }, + { + name: "destination Pod unavailable", + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{Name: "ps11", UID: "uid11"}, + Spec: crdv1alpha1.PacketSamplingSpec{ + Destination: crdv1alpha1.Destination{ + Pod: "unknown pod", + Namespace: "default", + }, + }, + }, + expectedErr: "failed to get the destination pod default/unknown pod: pods \"unknown pod\"", + }, + { + name: "to service packet", + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{Name: "ps12", UID: "uid12"}, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: pod1.Namespace, + Pod: pod1.Name, + }, + Destination: crdv1alpha1.Destination{ + Service: service1.Name, + Namespace: service1.Namespace, + }, + Packet: crdv1alpha1.Packet{ + TransportHeader: crdv1alpha1.TransportHeader{ + TCP: &crdv1alpha1.TCPHeader{ + SrcPort: 80, + DstPort: 81, + Flags: 11, + }, + }, + }, + }, + }, + expectedPacket: &binding.Packet{ + DestinationIP: net.ParseIP(service1IPv4).To4(), + IPProto: protocol.Type_TCP, + SourcePort: 80, + DestinationPort: 81, + TCPFlags: 11, + }, + }, + } + for _, ps := range pss { + t.Run(ps.name, func(t *testing.T) { + psc := newFakePacketSamplingController(t, nil, []runtime.Object{ps.ps}, nil, nil) + podInterfaces := psc.interfaceStore.GetContainerInterfacesByPod(pod1.Name, pod1.Namespace) + if ps.intf != nil { + podInterfaces[0] = ps.intf + } + stopCh := make(chan struct{}) + defer close(stopCh) + psc.crdInformerFactory.Start(stopCh) + psc.crdInformerFactory.WaitForCacheSync(stopCh) + psc.informerFactory.Start(stopCh) + psc.informerFactory.WaitForCacheSync(stopCh) + + pkt, err := psc.preparePacket(ps.ps, podInterfaces[0], ps.receiverOnly) + if ps.expectedErr == "" { + require.NoError(t, err) + assert.Equal(t, ps.expectedPacket, pkt) + } else { + assert.ErrorContains(t, err, ps.expectedErr) + assert.Nil(t, pkt) + } + }) + } +} + +func TestSyncPacketSampling(t *testing.T) { + // create test os + defaultFS = afero.NewMemMapFs() + defaultFS.MkdirAll("/tmp/antrea/packetsampling/packets", 0755) + file, err := defaultFS.Create(uidToPath(testUID)) + if err != nil { + t.Fatal("create pcapng file error: ", err) + } + + testWriter, err := pcapgo.NewNgWriter(file, layers.LinkTypeEthernet) + if err != nil { + t.Fatal("create test pcapng writer failed: ", err) + } + + pcs := []struct { + name string + ps *crdv1alpha1.PacketSampling + existingState *packetSamplingState + newState *packetSamplingState + expectedCalls func(mockOFClient *openflowtest.MockClient) + }{ + { + name: "start packetsampling", + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{Name: "ps1", UID: "uid1"}, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: pod1.Namespace, + Pod: pod1.Name, + }, + Destination: crdv1alpha1.Destination{ + Namespace: pod2.Namespace, + Pod: pod2.Name, + }, + }, + }, + existingState: &packetSamplingState{ + name: "ps1", + tag: 1, + }, + newState: &packetSamplingState{ + name: "ps1", + tag: 1, + }, + }, + + { + name: "packetsampling in failed phase", + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{Name: "ps1", UID: types.UID(testUID)}, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: pod1.Namespace, + Pod: pod1.Name, + }, + Destination: crdv1alpha1.Destination{ + Namespace: pod2.Namespace, + Pod: pod2.Name, + }, + Type: crdv1alpha1.FirstNSampling, + FirstNSamplingConfig: &crdv1alpha1.FirstNSamplingConfig{ + Number: 5, + }, + }, + Status: crdv1alpha1.PacketSamplingStatus{ + Phase: crdv1alpha1.PacketSamplingFailed, + }, + }, + existingState: &packetSamplingState{ + name: "ps1", + pcapngFile: file, + pcapngWriter: testWriter, + tag: 1, + }, + expectedCalls: func(mockOFClient *openflowtest.MockClient) { + mockOFClient.EXPECT().UninstallPacketSamplingFlows(uint8(1)) + }, + }, + } + + for _, ps := range pcs { + t.Run(ps.name, func(t *testing.T) { + psc := newFakePacketSamplingController(t, nil, []runtime.Object{ps.ps}, nil, nil) + stopCh := make(chan struct{}) + defer close(stopCh) + psc.crdInformerFactory.Start(stopCh) + psc.crdInformerFactory.WaitForCacheSync(stopCh) + + if ps.existingState != nil { + psc.runningPacketSamplings[ps.existingState.tag] = ps.existingState + } + + if ps.expectedCalls != nil { + ps.expectedCalls(psc.mockOFClient) + } + + err := psc.syncPacketSampling(ps.ps.Name) + require.NoError(t, err) + assert.Equal(t, ps.newState, psc.runningPacketSamplings[ps.existingState.tag]) + }) + } +} + +// TestPacketSamplingControllerRun was used to validate the whole run process is working. It doesn't wait for +// the testing ps to finish. +func TestPacketSamplingControllerRun(t *testing.T) { + // create test os + defaultFS = afero.NewMemMapFs() + defaultFS.MkdirAll("/tmp/antrea/packetsampling/packets", 0755) + ps := struct { + name string + ps *crdv1alpha1.PacketSampling + newState *packetSamplingState + }{ + name: "start packetsampling", + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{Name: "ps1", UID: "uid1"}, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: pod1.Namespace, + Pod: pod1.Name, + }, + Destination: crdv1alpha1.Destination{ + Namespace: pod2.Namespace, + Pod: pod2.Name, + }, + Type: crdv1alpha1.FirstNSampling, + FirstNSamplingConfig: &crdv1alpha1.FirstNSamplingConfig{ + Number: 5, + }, + }, + }, + newState: &packetSamplingState{tag: 1}, + } + + psc := newFakePacketSamplingController(t, nil, []runtime.Object{ps.ps}, nil, nil) + stopCh := make(chan struct{}) + defer close(stopCh) + psc.crdInformerFactory.Start(stopCh) + psc.crdInformerFactory.WaitForCacheSync(stopCh) + psc.informerFactory.Start(stopCh) + psc.informerFactory.WaitForCacheSync(stopCh) + psc.mockOFClient.EXPECT().InstallPacketSamplingFlows(ps.newState.tag, false, false, + &binding.Packet{DestinationIP: net.ParseIP(pod2.Status.PodIP), IPProto: protocol.Type_ICMP}, + nil, ofPortPod1, crdv1alpha1.DefaultPacketSamplingTimeout) + go psc.Run(stopCh) + time.Sleep(300 * time.Millisecond) +} + +func TestProcessPacketSamplingItem(t *testing.T) { + // create test os + defaultFS = afero.NewMemMapFs() + defaultFS.MkdirAll("/tmp/antrea/packetsampling/packets", 0755) + pc := struct { + ps *crdv1alpha1.PacketSampling + ofPort uint32 + receiverOnly bool + packet *binding.Packet + expected bool + }{ + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{Name: "ps1", UID: "uid1"}, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: pod1.Namespace, + Pod: pod1.Name, + }, + Destination: crdv1alpha1.Destination{ + Namespace: pod2.Namespace, + Pod: pod2.Name, + }, + FirstNSamplingConfig: &crdv1alpha1.FirstNSamplingConfig{ + Number: 5, + }, + Type: crdv1alpha1.FirstNSampling, + }, + }, + ofPort: ofPortPod1, + packet: &binding.Packet{ + DestinationIP: net.ParseIP(pod2IPv4), + IPProto: 1, + }, + expected: true, + } + + psc := newFakePacketSamplingController(t, nil, []runtime.Object{pc.ps}, nil, nil) + stopCh := make(chan struct{}) + defer close(stopCh) + psc.crdInformerFactory.Start(stopCh) + psc.crdInformerFactory.WaitForCacheSync(stopCh) + + psc.mockOFClient.EXPECT().InstallPacketSamplingFlows(uint8(1), false, pc.receiverOnly, pc.packet, nil, pc.ofPort, uint16(crdv1alpha1.DefaultPacketSamplingTimeout)) + psc.enqueuePacketSampling(pc.ps) + got := psc.processPacketSamplingItem() + assert.Equal(t, pc.expected, got) +} + +func TestStartPacketSampling(t *testing.T) { + defaultFS = afero.NewMemMapFs() + defaultFS.MkdirAll(packetDirectory, 0755) + tcs := []struct { + name string + ps *crdv1alpha1.PacketSampling + state *packetSamplingState + ofPort uint32 + receiverOnly bool + packet *binding.Packet + expectedCalls func(mockOFClient *openflowtest.MockClient) + nodeConfig *config.NodeConfig + expectedErr string + expectedErrLog string + }{ + { + name: "Pod-to-Pod PacketSampling", + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{Name: "ps1", UID: "uid1"}, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: pod1.Namespace, + Pod: pod1.Name, + }, + Destination: crdv1alpha1.Destination{ + Namespace: pod2.Namespace, + Pod: pod2.Name, + }, + FirstNSamplingConfig: &crdv1alpha1.FirstNSamplingConfig{ + Number: 5, + }, + }, + + Status: crdv1alpha1.PacketSamplingStatus{ + Phase: crdv1alpha1.PacketSamplingRunning, + }, + }, + state: &packetSamplingState{tag: 1}, + ofPort: ofPortPod1, + packet: &binding.Packet{ + SourceIP: net.ParseIP(pod1IPv4), + SourceMAC: pod1MAC, + DestinationIP: net.ParseIP(pod2IPv4), + DestinationMAC: pod2MAC, + IPProto: 1, + TTL: 64, + ICMPType: 8, + }, + expectedCalls: func(mockOFClient *openflowtest.MockClient) { + mockOFClient.EXPECT().InstallPacketSamplingFlows(uint8(1), false, false, + &binding.Packet{ + DestinationIP: net.ParseIP(pod2IPv4), + IPProto: 1, + }, + nil, ofPortPod1, crdv1alpha1.DefaultPacketSamplingTimeout) + }, + }, + { + name: "Pod-to-IPv4 packetsampling", + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{Name: "ps2", UID: "uid2"}, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: pod1.Namespace, + Pod: pod1.Name, + }, + Destination: crdv1alpha1.Destination{ + IP: dstIPv4, + }, + FirstNSamplingConfig: &crdv1alpha1.FirstNSamplingConfig{ + Number: 5, + }, + }, + Status: crdv1alpha1.PacketSamplingStatus{ + Phase: crdv1alpha1.PacketSamplingRunning, + }, + }, + state: &packetSamplingState{tag: 2}, + ofPort: ofPortPod1, + packet: &binding.Packet{ + SourceIP: net.ParseIP(pod1IPv4), + SourceMAC: pod1MAC, + DestinationIP: net.ParseIP(dstIPv4), + IPProto: 1, + TTL: 64, + ICMPType: 8, + }, + expectedCalls: func(mockOFClient *openflowtest.MockClient) { + mockOFClient.EXPECT().InstallPacketSamplingFlows(uint8(2), true, false, &binding.Packet{ + DestinationIP: net.ParseIP(dstIPv4), + IPProto: 1, + }, nil, ofPortPod1, crdv1alpha1.DefaultPacketSamplingTimeout) + }, + }, + } + + for _, tt := range tcs { + t.Run(tt.name, func(t *testing.T) { + tfc := newFakePacketSamplingController(t, nil, []runtime.Object{tt.ps}, nil, tt.nodeConfig) + if tt.expectedCalls != nil { + tt.expectedCalls(tfc.mockOFClient) + } + + bufWriter := bytes.NewBuffer(nil) + klog.SetOutput(bufWriter) + klog.LogToStderr(false) + defer func() { + klog.SetOutput(os.Stderr) + klog.LogToStderr(true) + }() + + err := tfc.startPacketSampling(tt.ps, tt.state) + if tt.expectedErr != "" { + assert.ErrorContains(t, err, tt.expectedErr) + } else { + require.NoError(t, err) + } + if tt.expectedErrLog != "" { + assert.Contains(t, bufWriter.String(), tt.expectedErrLog) + } + }) + } +} + +func TestPrepareEndpointsPackets(t *testing.T) { + pss := []struct { + name string + ps *crdv1alpha1.PacketSampling + expectedPackets []binding.Packet + objs []runtime.Object + expectedErr string + }{ + { + name: "svc-not-exist", + expectedErr: "service \"svc1\" not found", + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{Name: "ps2", UID: "uid2"}, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: pod1.Namespace, + Pod: pod1.Name, + }, + Destination: crdv1alpha1.Destination{ + Namespace: pod1.Namespace, + Service: "svc1", + }, + Packet: crdv1alpha1.Packet{ + TransportHeader: crdv1alpha1.TransportHeader{ + TCP: &crdv1alpha1.TCPHeader{ + DstPort: 80, + }, + }, + }, + }, + }, + }, + { + name: "ep-not-exist", + expectedErr: "endpoints \"svc1\" not found", + objs: []runtime.Object{&v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: pod1.Namespace, + Name: "svc1", + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Name: "http", + Port: 80, + TargetPort: intstr.IntOrString{ + Type: intstr.Type(intstr.Int), + IntVal: 8080, + }, + }, + }, + }, + }}, + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{Name: "ps2", UID: "uid2"}, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: pod1.Namespace, + Pod: pod1.Name, + }, + Destination: crdv1alpha1.Destination{ + Namespace: pod1.Namespace, + Service: "svc1", + }, + Packet: crdv1alpha1.Packet{ + TransportHeader: crdv1alpha1.TransportHeader{ + TCP: &crdv1alpha1.TCPHeader{ + DstPort: 80, + }, + }, + }, + }, + }, + }, + { + name: "tcp-2-backends-svc", + expectedPackets: []binding.Packet{ + { + DestinationIP: net.ParseIP(pod1.Status.PodIP), + DestinationPort: 8080, + IPProto: protocol.Type_TCP, + }, + { + DestinationIP: net.ParseIP(pod2.Status.PodIP), + DestinationPort: 8080, + IPProto: protocol.Type_TCP, + }, + }, + objs: []runtime.Object{&v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: pod1.Namespace, + Name: "svc1", + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Name: "http", + Port: 80, + TargetPort: intstr.IntOrString{ + Type: intstr.Type(intstr.Int), + IntVal: 8080, + }, + }, + }, + }, + }, &v1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: pod1.Namespace, + Name: "svc1", + }, + Subsets: []v1.EndpointSubset{ + { + Addresses: []v1.EndpointAddress{ + { + IP: pod1.Status.PodIP, + }, + { + IP: pod2.Status.PodIP, + }, + }, + Ports: []v1.EndpointPort{ + { + Name: "http", + Port: 8080, + }, + }, + }, + }, + }}, + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{Name: "ps1", UID: "uid1"}, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: pod1.Namespace, + Pod: pod1.Name, + }, + Destination: crdv1alpha1.Destination{ + Namespace: pod1.Namespace, + Service: "svc1", + }, + Packet: crdv1alpha1.Packet{ + TransportHeader: crdv1alpha1.TransportHeader{ + TCP: &crdv1alpha1.TCPHeader{ + DstPort: 80, + }, + }, + }, + }, + }, + }, + } + + for _, ps := range pss { + t.Run(ps.name, func(t *testing.T) { + psc := newFakePacketSamplingController(t, ps.objs, []runtime.Object{ps.ps}, nil, nil) + stopCh := make(chan struct{}) + defer close(stopCh) + psc.crdInformerFactory.Start(stopCh) + psc.crdInformerFactory.WaitForCacheSync(stopCh) + psc.informerFactory.Start(stopCh) + psc.informerFactory.WaitForCacheSync(stopCh) + + pkts, err := psc.genEndpointMatchPackets(ps.ps) + if ps.expectedErr == "" { + require.NoError(t, err) + if !reflect.DeepEqual(ps.expectedPackets, pkts) { + t.Errorf("expected packets: %+v, got: %+v", ps.expectedPackets, pkts) + } + + } else { + assert.ErrorContains(t, err, ps.expectedErr) + assert.Nil(t, pkts) + } + }) + } +} diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index 8354e938850..71fc97a6fe4 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -234,9 +234,15 @@ type Client interface { // InstallTraceflowFlows installs flows for a Traceflow request. InstallTraceflowFlows(dataplaneTag uint8, liveTraffic, droppedOnly, receiverOnly bool, packet *binding.Packet, ofPort uint32, timeoutSeconds uint16) error + // InstallPacketSamplingFlows installs flows for a Packet Sampling request. + InstallPacketSamplingFlows(dataplaneTag uint8, senderOnly bool, receiverOnly bool, packet *binding.Packet, endpointPackets []binding.Packet, ofPort uint32, timeoutSeconds uint16) error + // UninstallTraceflowFlows uninstalls flows for a Traceflow request. UninstallTraceflowFlows(dataplaneTag uint8) error + // UninstallPacketSamplingFlows uninstalls flows for a Packet Sampling request. + UninstallPacketSamplingFlows(dataplaneTag uint8) error + // GetPolicyInfoFromConjunction returns the following policy information for the provided conjunction ID: // NetworkPolicy reference, OF priority, rule name, label // The boolean return value indicates whether the policy information was found. @@ -928,6 +934,7 @@ func (c *client) generatePipelines() { c.enableL7FlowExporter) c.activatedFeatures = append(c.activatedFeatures, c.featurePodConnectivity) c.traceableFeatures = append(c.traceableFeatures, c.featurePodConnectivity) + c.samplingFeatures = append(c.samplingFeatures, c.featurePodConnectivity) c.featureService = newFeatureService(c.cookieAllocator, c.nodeIPChecker, @@ -943,6 +950,7 @@ func (c *client) generatePipelines() { c.connectUplinkToBridge) c.activatedFeatures = append(c.activatedFeatures, c.featureService) c.traceableFeatures = append(c.traceableFeatures, c.featureService) + c.samplingFeatures = append(c.samplingFeatures, c.featureService) } if c.nodeType == config.ExternalNode { @@ -990,6 +998,11 @@ func (c *client) generatePipelines() { c.featureTraceflow = newFeatureTraceflow() c.activatedFeatures = append(c.activatedFeatures, c.featureTraceflow) + if c.enablePacketSampling { + c.featurePacketSampling = newFeaturePacketSampling() + c.activatedFeatures = append(c.activatedFeatures, c.featurePacketSampling) + } + // Pipelines to generate. pipelineIDs := []binding.PipelineID{pipelineRoot, pipelineIP} if c.networkConfig.IPv4Enabled { @@ -1234,6 +1247,22 @@ func (c *client) SendTraceflowPacket(dataplaneTag uint8, packet *binding.Packet, return c.bridge.SendPacketOut(packetOutObj) } +func (c *client) InstallPacketSamplingFlows(dataplaneTag uint8, senderOnly, receiverOnly bool, packet *binding.Packet, endpointPackets []binding.Packet, ofPort uint32, timeoutSeconds uint16) error { + cacheKey := fmt.Sprintf("%x", dataplaneTag) + var flows []binding.Flow + for _, f := range c.samplingFeatures { + flows = append(flows, f.flowsToSampling(dataplaneTag, + c.ovsMetersAreSupported, + senderOnly, + receiverOnly, + packet, + endpointPackets, + ofPort, + timeoutSeconds)...) + } + return c.addFlows(c.featurePacketSampling.cachedFlows, cacheKey, flows) +} + func (c *client) InstallTraceflowFlows(dataplaneTag uint8, liveTraffic, droppedOnly, receiverOnly bool, packet *binding.Packet, ofPort uint32, timeoutSeconds uint16) error { cacheKey := fmt.Sprintf("%x", dataplaneTag) var flows []binding.Flow @@ -1255,6 +1284,11 @@ func (c *client) UninstallTraceflowFlows(dataplaneTag uint8) error { return c.deleteFlows(c.featureTraceflow.cachedFlows, cacheKey) } +func (c *client) UninstallPacketSamplingFlows(dataplaneTag uint8) error { + cacheKey := fmt.Sprintf("%x", dataplaneTag) + return c.deleteFlows(c.featurePacketSampling.cachedFlows, cacheKey) +} + // setBasePacketOutBuilder sets base IP properties of a packetOutBuilder which can have more packet data added. func setBasePacketOutBuilder(packetOutBuilder binding.PacketOutBuilder, srcMAC string, dstMAC string, srcIP string, dstIP string, inPort uint32, outPort uint32) (binding.PacketOutBuilder, error) { // Set ethernet header. diff --git a/pkg/agent/openflow/client_test.go b/pkg/agent/openflow/client_test.go index 3ff5d7fd808..5ff01c3da5f 100644 --- a/pkg/agent/openflow/client_test.go +++ b/pkg/agent/openflow/client_test.go @@ -94,6 +94,7 @@ type clientOptions struct { enableMulticluster bool enableL7NetworkPolicy bool enableL7FlowExporter bool + enablePacketSampling bool trafficEncryptionMode config.TrafficEncryptionModeType } @@ -168,6 +169,10 @@ func enableMulticluster(o *clientOptions) { o.enableMulticluster = true } +func enablePacketSampling(o *clientOptions) { + o.enablePacketSampling = true +} + func setTrafficEncryptionMode(trafficEncryptionMode config.TrafficEncryptionModeType) clientOptionsFn { return func(o *clientOptions) { o.trafficEncryptionMode = trafficEncryptionMode @@ -419,6 +424,7 @@ func newFakeClientWithBridge( o.enableMulticluster, NewGroupAllocator(), false, + o.enablePacketSampling, defaultPacketInRate) // Meters must be supported to enable Egress traffic shaping. @@ -1778,6 +1784,126 @@ func Test_client_InstallEgressQoS(t *testing.T) { require.False(t, ok) } +func Test_client_InstallPacketSamplingFlows(t *testing.T) { + type fields struct { + } + type args struct { + dataplaneTag uint8 + senderOnly bool + receiverOnly bool + packet *binding.Packet + endpointsPacket []binding.Packet + } + srcMAC, _ := net.ParseMAC("11:22:33:44:55:66") + dstMAC, _ := net.ParseMAC("11:22:33:44:55:77") + tests := []struct { + name string + fields fields + args args + wantErr bool + prepareFunc func(*gomock.Controller) *client + }{ + { + name: "packetsampling flow", + fields: fields{}, + args: args{ + dataplaneTag: 1, + packet: &binding.Packet{ + SourceMAC: srcMAC, + DestinationMAC: dstMAC, + SourceIP: net.ParseIP("1.2.3.4"), + DestinationIP: net.ParseIP("1.2.3.5"), + IPProto: 1, + TTL: 64, + }, + }, + wantErr: false, + prepareFunc: preparePacketSamplingFlow, + }, + { + name: "packetsampling flow with receiver only", + fields: fields{}, + args: args{ + dataplaneTag: 1, + receiverOnly: true, + packet: &binding.Packet{ + SourceMAC: srcMAC, + DestinationMAC: dstMAC, + SourceIP: net.ParseIP("1.2.3.4"), + DestinationIP: net.ParseIP("1.2.3.5"), + IPProto: 1, + TTL: 64, + }, + }, + wantErr: false, + prepareFunc: preparePacketSamplingFlow, + }, + { + name: "packetsampling flow with sender only", + fields: fields{}, + args: args{ + dataplaneTag: 1, + senderOnly: true, + packet: &binding.Packet{ + SourceMAC: srcMAC, + DestinationMAC: dstMAC, + SourceIP: net.ParseIP("1.2.3.4"), + DestinationIP: net.ParseIP("1.2.3.5"), + IPProto: 1, + TTL: 64, + }, + }, + wantErr: false, + prepareFunc: preparePacketSamplingFlow, + }, + { + name: "packetsampling flow with endpoints packets", + fields: fields{}, + args: args{ + dataplaneTag: 1, + senderOnly: true, + packet: &binding.Packet{ + SourceMAC: srcMAC, + DestinationMAC: dstMAC, + SourceIP: net.ParseIP("1.2.3.4"), + DestinationIP: net.ParseIP("1.2.3.5"), + IPProto: 1, + TTL: 64, + }, + endpointsPacket: []binding.Packet{ + { + SourceMAC: srcMAC, + DestinationMAC: dstMAC, + SourceIP: net.ParseIP("1.2.3.4"), + DestinationIP: net.ParseIP("1.2.3.6"), + IPProto: 1, + TTL: 64, + }, + { + SourceMAC: srcMAC, + DestinationMAC: dstMAC, + SourceIP: net.ParseIP("1.2.3.4"), + DestinationIP: net.ParseIP("1.2.3.7"), + IPProto: 1, + TTL: 64, + }, + }, + }, + wantErr: false, + prepareFunc: preparePacketSamplingFlow, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + c := tt.prepareFunc(ctrl) + if err := c.InstallPacketSamplingFlows(tt.args.dataplaneTag, tt.args.senderOnly, tt.args.receiverOnly, tt.args.packet, nil, 0, 300); (err != nil) != tt.wantErr { + t.Errorf("InstallPacketSamplingFlows() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + func Test_client_InstallTraceflowFlows(t *testing.T) { type fields struct { } @@ -1918,6 +2044,32 @@ func Test_client_SendTraceflowPacket(t *testing.T) { } } +func preparePacketSamplingFlow(ctrl *gomock.Controller) *client { + m := opstest.NewMockOFEntryOperations(ctrl) + fc := newFakeClientWithBridge(m, true, false, config.K8sNode, config.TrafficEncapModeEncap, ovsoftest.NewMockBridge(ctrl), enablePacketSampling) + defer resetPipelines() + + m.EXPECT().AddAll(gomock.Any()).Return(nil).Times(1) + _, ipCIDR, _ := net.ParseCIDR("192.168.2.30/32") + flows, _ := EgressDefaultTable.ofTable.BuildFlow(priority100).Action().Drop().Done().GetBundleMessages(binding.AddMessage) + flowMsg := flows[0].GetMessage().(*openflow15.FlowMod) + ctx := &conjMatchFlowContext{ + dropFlow: flowMsg, + dropFlowEnableLogging: false, + conjunctiveMatch: &conjunctiveMatch{ + tableID: 1, + matchPairs: []matchPair{ + { + matchKey: MatchCTSrcIPNet, + matchValue: *ipCIDR, + }, + }, + }} + fc.featureNetworkPolicy.globalConjMatchFlowCache["mockContext"] = ctx + fc.featureNetworkPolicy.policyCache.Add(&policyRuleConjunction{metricFlows: []*openflow15.FlowMod{flowMsg}}) + return fc +} + func prepareTraceflowFlow(ctrl *gomock.Controller) *client { m := opstest.NewMockOFEntryOperations(ctrl) fc := newFakeClientWithBridge(m, true, false, config.K8sNode, config.TrafficEncapModeEncap, ovsoftest.NewMockBridge(ctrl)) @@ -2031,7 +2183,7 @@ func Test_client_setBasePacketOutBuilder(t *testing.T) { } func prepareSetBasePacketOutBuilder(ctrl *gomock.Controller, success bool) *client { - ofClient := NewClient(bridgeName, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), true, true, false, false, false, false, false, false, false, false, false, false, false, nil, false, defaultPacketInRate) + ofClient := NewClient(bridgeName, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), true, true, false, false, false, false, false, false, false, false, false, false, false, nil, false, false, defaultPacketInRate) m := ovsoftest.NewMockBridge(ctrl) ofClient.bridge = m bridge := binding.OFBridge{} diff --git a/pkg/agent/openflow/cookie/allocator.go b/pkg/agent/openflow/cookie/allocator.go index 3aef3db4c84..4ec3dc01f4e 100644 --- a/pkg/agent/openflow/cookie/allocator.go +++ b/pkg/agent/openflow/cookie/allocator.go @@ -39,6 +39,7 @@ const ( Multicluster Traceflow ExternalNodeConnectivity + PacketSampling ) func (c Category) String() string { @@ -61,6 +62,8 @@ func (c Category) String() string { return "Traceflow" case ExternalNodeConnectivity: return "ExternalNodeConnectivity" + case PacketSampling: + return "PacketSampling" default: return "Invalid" } diff --git a/pkg/agent/openflow/fields.go b/pkg/agent/openflow/fields.go index 87d0521af2e..7708d1dea7e 100644 --- a/pkg/agent/openflow/fields.go +++ b/pkg/agent/openflow/fields.go @@ -148,6 +148,9 @@ var ( // consider the packet external sourced as the other IPs are routable externally anyway. FromExternalRegMark = binding.NewOneBitRegMark(4, 27) + // reg4[28..31]: Field mark the flow for packet sampling case. + PacketSamplingMark = binding.NewRegField(4, 28, 31) + // reg5(NXM_NX_REG5) // Field to cache the Egress conjunction ID hit by TraceFlow packet. TFEgressConjIDField = binding.NewRegField(5, 0, 31) diff --git a/pkg/agent/openflow/framework.go b/pkg/agent/openflow/framework.go index d8c353ee2a2..d64073e8c0e 100644 --- a/pkg/agent/openflow/framework.go +++ b/pkg/agent/openflow/framework.go @@ -309,6 +309,10 @@ func (f *featureTraceflow) getRequiredTables() []*Table { return nil } +func (f *featurePacketSampling) getRequiredTables() []*Table { + return nil +} + func (f *featureExternalNodeConnectivity) getRequiredTables() []*Table { return []*Table{ ConntrackTable, @@ -336,3 +340,14 @@ type traceableFeature interface { ofPort uint32, timeoutSeconds uint16) []binding.Flow } + +type samplingFeature interface { + flowsToSampling(dataplaneTag uint8, + ovsMetersAreSupported, + senderOnly bool, + receiverOnly bool, + packet *binding.Packet, + endpointPackets []binding.Packet, + ofPort uint32, + timeoutSeconds uint16) []binding.Flow +} diff --git a/pkg/agent/openflow/packetin.go b/pkg/agent/openflow/packetin.go index 608f1759e34..cc4cdf5d0ea 100644 --- a/pkg/agent/openflow/packetin.go +++ b/pkg/agent/openflow/packetin.go @@ -53,6 +53,8 @@ const ( // PacketInCategorySvcReject is used to process the Service packets not matching any // Endpoints within packetIn message. PacketInCategorySvcReject + // PacketInCategoryPS is used for packetIn messages related to sampling. + PacketInCategoryPS // PacketIn operations below are used to decide which operation(s) should be // executed by a handler. It(they) should be loaded in the second byte of the diff --git a/pkg/agent/openflow/packetsampling.go b/pkg/agent/openflow/packetsampling.go new file mode 100644 index 00000000000..9605ebfe7f6 --- /dev/null +++ b/pkg/agent/openflow/packetsampling.go @@ -0,0 +1,55 @@ +// Copyright 2024 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openflow + +import ( + "antrea.io/libOpenflow/openflow15" + + binding "antrea.io/antrea/pkg/ovs/openflow" +) + +type featurePacketSampling struct { + cachedFlows *flowCategoryCache +} + +func (f *featurePacketSampling) getFeatureName() string { + return "PacketSampling" +} + +func newFeaturePacketSampling() *featurePacketSampling { + return &featurePacketSampling{ + cachedFlows: newFlowCategoryCache(), + } +} + +func (f *featurePacketSampling) initFlows() []*openflow15.FlowMod { + return []*openflow15.FlowMod{} +} + +func (f *featurePacketSampling) replayFlows() []*openflow15.FlowMod { + return []*openflow15.FlowMod{} +} + +func (f *featurePacketSampling) initGroups() []binding.OFEntry { + return nil +} + +func (f *featurePacketSampling) replayGroups() []binding.OFEntry { + return nil +} + +func (f *featurePacketSampling) replayMeters() []binding.OFEntry { + return nil +} diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go index 9daed3adc33..b25578ce6bc 100644 --- a/pkg/agent/openflow/pipeline.go +++ b/pkg/agent/openflow/pipeline.go @@ -402,6 +402,7 @@ type client struct { enableL7FlowExporter bool enableMulticluster bool enablePrometheusMetrics bool + enablePacketSampling bool connectUplinkToBridge bool nodeType config.NodeType roundInfo types.RoundInfo @@ -421,6 +422,9 @@ type client struct { featureTraceflow *featureTraceflow traceableFeatures []traceableFeature + featurePacketSampling *featurePacketSampling + samplingFeatures []samplingFeature + pipelines map[binding.PipelineID]binding.Pipeline // ofEntryOperations is a wrapper interface for operating multiple OpenFlow entries with action AddAll / ModifyAll / DeleteAll. @@ -847,6 +851,206 @@ func (f *featureService) snatConntrackFlows() []binding.Flow { return flows } +func matchTransportHeader(packet *binding.Packet, flowBuilder binding.FlowBuilder, endpointPackets []binding.Packet) binding.FlowBuilder { + // Match transport header + switch packet.IPProto { + case protocol.Type_ICMP: + flowBuilder = flowBuilder.MatchProtocol(binding.ProtocolICMP) + case protocol.Type_IPv6ICMP: + flowBuilder = flowBuilder.MatchProtocol(binding.ProtocolICMPv6) + case protocol.Type_TCP: + if packet.IsIPv6 { + flowBuilder = flowBuilder.MatchProtocol(binding.ProtocolTCPv6) + } else { + flowBuilder = flowBuilder.MatchProtocol(binding.ProtocolTCP) + } + case protocol.Type_UDP: + if packet.IsIPv6 { + flowBuilder = flowBuilder.MatchProtocol(binding.ProtocolUDPv6) + } else { + flowBuilder = flowBuilder.MatchProtocol(binding.ProtocolUDP) + } + default: + flowBuilder = flowBuilder.MatchIPProtocolValue(packet.IsIPv6, packet.IPProto) + } + if packet.IPProto == protocol.Type_TCP || packet.IPProto == protocol.Type_UDP { + if endpointPackets != nil && endpointPackets[0].DestinationPort != 0 { + flowBuilder = flowBuilder.MatchDstPort(endpointPackets[0].DestinationPort, nil) + } else if packet.DestinationPort != 0 { + flowBuilder = flowBuilder.MatchDstPort(packet.DestinationPort, nil) + } + if packet.SourcePort != 0 { + flowBuilder = flowBuilder.MatchSrcPort(packet.SourcePort, nil) + } + } + + return flowBuilder +} + +// flowsToSampling generates flows for packet sampling. dataplaneTag is used as a mark for the target flow. +func (f *featurePodConnectivity) flowsToSampling(dataplaneTag uint8, + ovsMetersAreSupported, + senderOnly bool, + receiverOnly bool, + packet *binding.Packet, + endpointPackets []binding.Packet, + ofPort uint32, + timeout uint16) []binding.Flow { + cookieID := f.cookieAllocator.Request(cookie.PacketSampling).Raw() + var flows []binding.Flow + tag := uint32(dataplaneTag) + var flowBuilder binding.FlowBuilder + if !receiverOnly { + // if not receiverOnly, ofPort is inPort + if endpointPackets == nil { + flowBuilder = ConntrackStateTable.ofTable.BuildFlow(priorityHigh). + Cookie(cookieID). + MatchInPort(ofPort). + MatchCTStateTrk(true). + Action().LoadToRegField(PacketSamplingMark, tag). + SetHardTimeout(timeout). + Action().GotoStage(stagePreRouting) + if packet.DestinationIP != nil { + flowBuilder = flowBuilder.MatchDstIP(packet.DestinationIP) + } + } else { + // generate flows to endpoints. + for _, epPacket := range endpointPackets { + tmpFlowBuilder := ConntrackStateTable.ofTable.BuildFlow(priorityHigh). + Cookie(cookieID). + MatchInPort(ofPort). + MatchCTStateTrk(true). + Action().LoadToRegField(PacketSamplingMark, tag). + SetHardTimeout(timeout). + Action().GotoStage(stagePreRouting) + tmpFlowBuilder.MatchDstIP(epPacket.DestinationIP) + flow := matchTransportHeader(packet, tmpFlowBuilder, endpointPackets).Done() + flows = append(flows, flow) + } + } + } else { + flowBuilder = L2ForwardingCalcTable.ofTable.BuildFlow(priorityHigh). + Cookie(cookieID). + MatchCTStateTrk(true). + MatchDstMAC(packet.DestinationMAC). + Action().LoadToRegField(TargetOFPortField, ofPort). + Action().LoadRegMark(OutputToOFPortRegMark). + Action().LoadToRegField(PacketSamplingMark, tag). + SetHardTimeout(timeout). + Action().GotoStage(stageIngressSecurity) + if packet.SourceIP != nil { + flowBuilder = flowBuilder.MatchSrcIP(packet.SourceIP) + } + } + + // for sender only case, capture the first tracked packet for svc. + if senderOnly { + for _, ipProtocol := range f.ipProtocols { + tmpFlowBuilder := ConntrackStateTable.ofTable.BuildFlow(priorityHigh). + Cookie(cookieID). + MatchInPort(ofPort). + MatchProtocol(ipProtocol). + MatchCTStateNew(true). + MatchCTStateTrk(true). + Action().LoadRegMark(RewriteMACRegMark). + Action().LoadToRegField(PacketSamplingMark, tag). + SetHardTimeout(timeout). + Action().GotoStage(stagePreRouting) + tmpFlowBuilder.MatchDstIP(packet.DestinationIP) + tmpFlowBuilder = matchTransportHeader(packet, tmpFlowBuilder, nil) + flows = append(flows, tmpFlowBuilder.Done()) + } + } + + if flowBuilder != nil { + flow := matchTransportHeader(packet, flowBuilder, nil).Done() + flows = append(flows, flow) + } + + output := func(fb binding.FlowBuilder) binding.FlowBuilder { + return fb.Action().OutputToRegField(TargetOFPortField) + } + + sendToController := func(fb binding.FlowBuilder) binding.FlowBuilder { + if ovsMetersAreSupported { + fb = fb.Action().Meter(PacketInMeterIDTF) + } + fb = fb.Action().SendToController([]byte{uint8(PacketInCategoryPS)}, false) + return fb + } + + // This generates PacketSampling specific flows that outputs sampling + // non-hairpin packets to OVS port and Antrea Agent after + // L2 forwarding calculation. + for _, ipProtocol := range f.ipProtocols { + if f.networkConfig.TrafficEncapMode.SupportsEncap() { + // SendToController and Output if output port is tunnel port. + fb := OutputTable.ofTable.BuildFlow(priorityNormal+3). + Cookie(cookieID). + MatchRegFieldWithValue(TargetOFPortField, f.tunnelPort). + MatchProtocol(ipProtocol). + MatchRegMark(OutputToOFPortRegMark). + MatchRegFieldWithValue(PacketSamplingMark, tag). + SetHardTimeout(timeout). + Action().OutputToRegField(TargetOFPortField) + fb = sendToController(fb) + flows = append(flows, fb.Done()) + // For injected packets, only SendToController if output port is local gateway. In encapMode, a PacketSampling + // packet going out of the gateway port (i.e. exiting the overlay) essentially means that the PacketSampling + // request is complete. + fb = OutputTable.ofTable.BuildFlow(priorityNormal+2). + Cookie(cookieID). + MatchRegFieldWithValue(TargetOFPortField, f.gatewayPort). + MatchProtocol(ipProtocol). + MatchRegMark(OutputToOFPortRegMark). + MatchRegFieldWithValue(PacketSamplingMark, tag). + SetHardTimeout(timeout) + fb = sendToController(fb) + fb = output(fb) + flows = append(flows, fb.Done()) + } else { + // SendToController and Output if output port is local gateway. Unlike in encapMode, inter-Node Pod-to-Pod + // traffic is expected to go out of the gateway port on the way to its destination. + fb := OutputTable.ofTable.BuildFlow(priorityNormal+2). + Cookie(cookieID). + MatchRegFieldWithValue(TargetOFPortField, f.gatewayPort). + MatchProtocol(ipProtocol). + MatchRegMark(OutputToOFPortRegMark). + MatchRegFieldWithValue(PacketSamplingMark, tag). + SetHardTimeout(timeout). + Action().OutputToRegField(TargetOFPortField) + fb = sendToController(fb) + flows = append(flows, fb.Done()) + } + + gatewayIP := f.gatewayIPs[ipProtocol] + if gatewayIP != nil { + fb := OutputTable.ofTable.BuildFlow(priorityNormal+3). + Cookie(cookieID). + MatchRegFieldWithValue(TargetOFPortField, f.gatewayPort). + MatchProtocol(ipProtocol). + MatchDstIP(gatewayIP). + MatchRegMark(OutputToOFPortRegMark). + MatchRegFieldWithValue(PacketSamplingMark, tag). + SetHardTimeout(timeout) + fb = sendToController(fb) + fb = output(fb) + flows = append(flows, fb.Done()) + } + + fb := OutputTable.ofTable.BuildFlow(priorityNormal+2). + Cookie(cookieID). + MatchProtocol(ipProtocol). + MatchRegMark(OutputToOFPortRegMark). + MatchRegFieldWithValue(PacketSamplingMark, tag). + SetHardTimeout(timeout) + fb = sendToController(fb) + fb = output(fb) + flows = append(flows, fb.Done()) + } + return flows +} + // TODO: Use DuplicateToBuilder or integrate this function into original one to avoid unexpected difference. // flowsToTrace generates Traceflow specific flows in the connectionTrackStateTable or L2ForwardingCalcTable for featurePodConnectivity. // When packet is not provided, the flows bypass the drop flow in conntrackStateFlow to avoid unexpected drop of the @@ -936,14 +1140,7 @@ func (f *featurePodConnectivity) flowsToTrace(dataplaneTag uint8, default: flowBuilder = flowBuilder.MatchIPProtocolValue(packet.IsIPv6, packet.IPProto) } - if packet.IPProto == protocol.Type_TCP || packet.IPProto == protocol.Type_UDP { - if packet.DestinationPort != 0 { - flowBuilder = flowBuilder.MatchDstPort(packet.DestinationPort, nil) - } - if packet.SourcePort != 0 { - flowBuilder = flowBuilder.MatchSrcPort(packet.SourcePort, nil) - } - } + flows = append(flows, flowBuilder.Done()) } @@ -1040,6 +1237,44 @@ func (f *featurePodConnectivity) flowsToTrace(dataplaneTag uint8, return flows } +// flowsToSampling is used to generate flows for PacketSampling in featureService. +func (f *featureService) flowsToSampling(dataplaneTag uint8, + ovsMetersAreSupported, + senderOnly bool, + receiverOnly bool, + packet *binding.Packet, + endpointPackets []binding.Packet, + ofPort uint32, + timeout uint16) []binding.Flow { + cookieID := f.cookieAllocator.Request(cookie.PacketSampling).Raw() + var flows []binding.Flow + + sendToController := func(fb binding.FlowBuilder) binding.FlowBuilder { + if ovsMetersAreSupported { + fb = fb.Action().Meter(PacketInMeterIDTF) + } + fb = fb.Action().SendToController([]byte{uint8(PacketInCategoryPS)}, false) + return fb + } + + // This generates PacketSampling specific flows that outputs hairpin PacketSampling packets to OVS port and Antrea Agent after + // L2forwarding calculation. + for _, ipProtocol := range f.ipProtocols { + if f.enableProxy { + fb := OutputTable.ofTable.BuildFlow(priorityHigh+2). + Cookie(cookieID). + MatchProtocol(ipProtocol). + MatchCTMark(HairpinCTMark). + MatchRegFieldWithValue(PacketSamplingMark, uint32(dataplaneTag)). + SetHardTimeout(timeout) + fb = sendToController(fb) + fb = fb.Action().OutputToRegField(TargetOFPortField) + flows = append(flows, fb.Done()) + } + } + return flows +} + // flowsToTrace is used to generate flows for Traceflow in featureService. func (f *featureService) flowsToTrace(dataplaneTag uint8, ovsMetersAreSupported, @@ -2834,6 +3069,7 @@ func NewClient(bridgeName string, enableMulticluster bool, groupIDAllocator GroupAllocator, enablePrometheusMetrics bool, + enablePacketSampling bool, packetInRate int, ) *client { bridge := binding.NewOFBridge(bridgeName, mgmtAddr) @@ -2853,6 +3089,7 @@ func NewClient(bridgeName string, enableL7FlowExporter: enableL7FlowExporter, enableMulticluster: enableMulticluster, enablePrometheusMetrics: enablePrometheusMetrics, + enablePacketSampling: enablePacketSampling, connectUplinkToBridge: connectUplinkToBridge, pipelines: make(map[binding.PipelineID]binding.Pipeline), packetInHandlers: map[uint8]PacketInHandler{}, diff --git a/pkg/agent/openflow/pipeline_test.go b/pkg/agent/openflow/pipeline_test.go index 451abe7ccc1..3d094d15ca8 100644 --- a/pkg/agent/openflow/pipeline_test.go +++ b/pkg/agent/openflow/pipeline_test.go @@ -19,6 +19,7 @@ import ( "testing" "antrea.io/libOpenflow/openflow15" + "antrea.io/libOpenflow/protocol" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -301,3 +302,94 @@ func getGroupModLen(g *openflow15.GroupMod) uint32 { } return n } + +func TestMatchTransportHeader(t *testing.T) { + + testCases := []struct { + name string + packet *binding.Packet + endpointPackets []binding.Packet + expectCalls func(ctrl *gomock.Controller, builder *openflowtest.MockFlowBuilder) *openflowtest.MockFlowBuilder + }{ + { + name: "tcp proto", + packet: &binding.Packet{ + IPProto: protocol.Type_TCP, + }, + expectCalls: func(ctrl *gomock.Controller, builder *openflowtest.MockFlowBuilder) *openflowtest.MockFlowBuilder { + builder.EXPECT().MatchProtocol(binding.ProtocolTCP) + return builder + + }, + }, + { + name: "udp proto", + packet: &binding.Packet{ + IPProto: protocol.Type_UDP, + }, + expectCalls: func(ctrl *gomock.Controller, builder *openflowtest.MockFlowBuilder) *openflowtest.MockFlowBuilder { + builder.EXPECT().MatchProtocol(binding.ProtocolUDP) + return builder + }, + }, + { + name: "ipv6-tcp", + packet: &binding.Packet{ + IPProto: protocol.Type_TCP, + IsIPv6: true, + }, + expectCalls: func(ctrl *gomock.Controller, builder *openflowtest.MockFlowBuilder) *openflowtest.MockFlowBuilder { + builder.EXPECT().MatchProtocol(binding.ProtocolTCPv6) + return builder + }, + }, + { + name: "udp-with-src-and-dst-port", + packet: &binding.Packet{ + IPProto: protocol.Type_UDP, + SourcePort: 1000, + DestinationPort: 53, + }, + expectCalls: func(ctrl *gomock.Controller, builder *openflowtest.MockFlowBuilder) *openflowtest.MockFlowBuilder { + builder.EXPECT().MatchProtocol(binding.ProtocolUDP).Return(builder).AnyTimes() + builder.EXPECT().MatchDstPort(uint16(53), nil).Return(builder).AnyTimes() + builder.EXPECT().MatchSrcPort(uint16(1000), nil).Return(builder).AnyTimes() + return builder + }, + }, + { + name: "with endpoints packets", + packet: &binding.Packet{ + IPProto: protocol.Type_TCP, + SourcePort: 1000, + DestinationPort: 53, + }, + endpointPackets: []binding.Packet{ + { + IPProto: protocol.Type_TCP, + SourcePort: 1000, + DestinationPort: 54, + }, + }, + expectCalls: func(ctrl *gomock.Controller, builder *openflowtest.MockFlowBuilder) *openflowtest.MockFlowBuilder { + builder.EXPECT().MatchProtocol(binding.ProtocolTCP).Return(builder).AnyTimes() + builder.EXPECT().MatchDstPort(uint16(54), nil).Return(builder).AnyTimes() + builder.EXPECT().MatchSrcPort(uint16(1000), nil).Return(builder).AnyTimes() + return builder + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + fakeOfTable := openflowtest.NewMockTable(ctrl) + ConntrackStateTable.ofTable = fakeOfTable + defer func() { + ConntrackStateTable.ofTable = nil + }() + testBuilder := openflowtest.NewMockFlowBuilder(ctrl) + tc.expectCalls(ctrl, testBuilder) + matchTransportHeader(tc.packet, testBuilder, tc.endpointPackets) + }) + } +} diff --git a/pkg/agent/openflow/testing/mock_openflow.go b/pkg/agent/openflow/testing/mock_openflow.go index cb1f4cb8cbc..22b83dfc317 100644 --- a/pkg/agent/openflow/testing/mock_openflow.go +++ b/pkg/agent/openflow/testing/mock_openflow.go @@ -419,6 +419,20 @@ func (mr *MockClientMockRecorder) InstallNodeFlows(arg0, arg1, arg2, arg3, arg4 return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallNodeFlows", reflect.TypeOf((*MockClient)(nil).InstallNodeFlows), arg0, arg1, arg2, arg3, arg4) } +// InstallPacketSamplingFlows mocks base method. +func (m *MockClient) InstallPacketSamplingFlows(arg0 byte, arg1, arg2 bool, arg3 *openflow0.Packet, arg4 []openflow0.Packet, arg5 uint32, arg6 uint16) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InstallPacketSamplingFlows", arg0, arg1, arg2, arg3, arg4, arg5, arg6) + ret0, _ := ret[0].(error) + return ret0 +} + +// InstallPacketSamplingFlows indicates an expected call of InstallPacketSamplingFlows. +func (mr *MockClientMockRecorder) InstallPacketSamplingFlows(arg0, arg1, arg2, arg3, arg4, arg5, arg6 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallPacketSamplingFlows", reflect.TypeOf((*MockClient)(nil).InstallPacketSamplingFlows), arg0, arg1, arg2, arg3, arg4, arg5, arg6) +} + // InstallPodFlows mocks base method. func (m *MockClient) InstallPodFlows(arg0 string, arg1 []net.IP, arg2 net.HardwareAddr, arg3 uint32, arg4 uint16, arg5 *uint32) error { m.ctrl.T.Helper() @@ -959,6 +973,20 @@ func (mr *MockClientMockRecorder) UninstallNodeFlows(arg0 any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UninstallNodeFlows", reflect.TypeOf((*MockClient)(nil).UninstallNodeFlows), arg0) } +// UninstallPacketSamplingFlows mocks base method. +func (m *MockClient) UninstallPacketSamplingFlows(arg0 byte) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UninstallPacketSamplingFlows", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// UninstallPacketSamplingFlows indicates an expected call of UninstallPacketSamplingFlows. +func (mr *MockClientMockRecorder) UninstallPacketSamplingFlows(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UninstallPacketSamplingFlows", reflect.TypeOf((*MockClient)(nil).UninstallPacketSamplingFlows), arg0) +} + // UninstallPodFlows mocks base method. func (m *MockClient) UninstallPodFlows(arg0 string) error { m.ctrl.T.Helper() diff --git a/pkg/agent/supportbundlecollection/support_bundle_controller.go b/pkg/agent/supportbundlecollection/support_bundle_controller.go index 72b410f97e7..538f7fe6ea8 100644 --- a/pkg/agent/supportbundlecollection/support_bundle_controller.go +++ b/pkg/agent/supportbundlecollection/support_bundle_controller.go @@ -17,16 +17,11 @@ package supportbundlecollection import ( "context" "fmt" - "io" - "net/url" - "path" "reflect" "sync" "time" - "github.com/pkg/sftp" "github.com/spf13/afero" - "golang.org/x/crypto/ssh" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/wait" @@ -43,18 +38,15 @@ import ( "antrea.io/antrea/pkg/querier" "antrea.io/antrea/pkg/support" "antrea.io/antrea/pkg/util/compress" + "antrea.io/antrea/pkg/util/ftp" "antrea.io/antrea/pkg/util/k8s" ) type ProtocolType string const ( - sftpProtocol ProtocolType = "sftp" - - controllerName = "SupportBundleCollectionController" - - uploadToFileServerTries = 5 - uploadToFileServerRetryDelay = 5 * time.Second + sftpProtocol ProtocolType = "sftp" + controllerName string = "SupportBundleCollectionController" ) var ( @@ -78,7 +70,7 @@ type SupportBundleController struct { npq querier.AgentNetworkPolicyInfoQuerier v4Enabled bool v6Enabled bool - sftpUploader uploader + sftpUploader ftp.UpLoader } func NewSupportBundleController(nodeName string, @@ -101,7 +93,7 @@ func NewSupportBundleController(nodeName string, npq: npq, v4Enabled: v4Enabled, v6Enabled: v6Enabled, - sftpUploader: &sftpUploader{}, + sftpUploader: &ftp.SftpUploader{}, } return c } @@ -299,100 +291,20 @@ func (c *SupportBundleController) uploadSupportBundle(supportBundle *cpv1b2.Supp if err != nil { return fmt.Errorf("failed to upload support bundle while getting uploader: %v", err) } - if _, err := outputFile.Seek(0, 0); err != nil { - return fmt.Errorf("failed to upload support bundle to file server while setting offset: %v", err) - } - // fileServer.URL should be like: 10.92.23.154:22/path or sftp://10.92.23.154:22/path - parsedURL, err := parseUploadUrl(supportBundle.FileServer.URL) - if err != nil { - return fmt.Errorf("failed to upload support bundle while parsing upload URL: %v", err) - } - triesLeft := uploadToFileServerTries - var uploadErr error - for triesLeft > 0 { - if uploadErr = c.uploadToFileServer(uploader, supportBundle.Name, parsedURL, &supportBundle.Authentication, outputFile); uploadErr == nil { - return nil - } - triesLeft-- - if triesLeft == 0 { - return fmt.Errorf("failed to upload support bundle after %d attempts", uploadToFileServerTries) - } - klog.InfoS("Failed to upload support bundle", "UploadError", uploadErr, "TriesLeft", triesLeft) - time.Sleep(uploadToFileServerRetryDelay) - } - return nil -} -func parseUploadUrl(uploadUrl string) (*url.URL, error) { - parsedURL, err := url.Parse(uploadUrl) - if err != nil { - parsedURL, err = url.Parse("sftp://" + uploadUrl) - if err != nil { - return nil, err - } - } - if parsedURL.Scheme != "sftp" { - return nil, fmt.Errorf("not sftp protocol") - } - return parsedURL, nil + fileName := c.nodeName + "_" + supportBundle.Name + ".tar.gz" + serverAuth := supportBundle.Authentication.BasicAuthentication + cfg := ftp.GenSSHClientConfig(serverAuth.Username, serverAuth.Password) + return uploader.Upload(supportBundle.FileServer.URL, fileName, cfg, outputFile) } -func (c *SupportBundleController) uploadToFileServer(up uploader, bundleName string, parsedURL *url.URL, serverAuth *cpv1b2.BundleServerAuthConfiguration, tarGzFile io.Reader) error { - joinedPath := path.Join(parsedURL.Path, c.nodeName+"_"+bundleName+".tar.gz") - cfg := &ssh.ClientConfig{ - User: serverAuth.BasicAuthentication.Username, - Auth: []ssh.AuthMethod{ssh.Password(serverAuth.BasicAuthentication.Password)}, - // #nosec G106: skip host key check here and users can specify their own checks if needed - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - Timeout: time.Second, - } - return up.upload(parsedURL.Host, joinedPath, cfg, tarGzFile) -} - -func (c *SupportBundleController) getUploaderByProtocol(protocol ProtocolType) (uploader, error) { +func (c *SupportBundleController) getUploaderByProtocol(protocol ProtocolType) (ftp.UpLoader, error) { if protocol == sftpProtocol { return c.sftpUploader, nil } return nil, fmt.Errorf("unsupported protocol %s", protocol) } -type uploader interface { - upload(addr string, path string, config *ssh.ClientConfig, tarGzFile io.Reader) error -} - -type sftpUploader struct { -} - -func (uploader *sftpUploader) upload(address string, path string, config *ssh.ClientConfig, tarGzFile io.Reader) error { - conn, err := ssh.Dial("tcp", address, config) - if err != nil { - return fmt.Errorf("error when connecting to fs server: %w", err) - } - sftpClient, err := sftp.NewClient(conn) - if err != nil { - return fmt.Errorf("error when setting up sftp client: %w", err) - } - defer func() { - if err := sftpClient.Close(); err != nil { - klog.ErrorS(err, "Error when closing sftp client") - } - }() - targetFile, err := sftpClient.Create(path) - if err != nil { - return fmt.Errorf("error when creating target file on remote: %v", err) - } - defer func() { - if err := targetFile.Close(); err != nil { - klog.ErrorS(err, "Error when closing target file on remote") - } - }() - if written, err := io.Copy(targetFile, tarGzFile); err != nil { - return fmt.Errorf("error when copying target file: %v, written: %d", err, written) - } - klog.InfoS("Successfully upload file to path", "filePath", path) - return nil -} - func (c *SupportBundleController) updateSupportBundleCollectionStatus(key string, complete bool, genErr error) error { antreaClient, err := c.antreaClientGetter.GetAntreaClient() if err != nil { diff --git a/pkg/agent/supportbundlecollection/support_bundle_controller_test.go b/pkg/agent/supportbundlecollection/support_bundle_controller_test.go index 51a92767050..109e8b8af6e 100644 --- a/pkg/agent/supportbundlecollection/support_bundle_controller_test.go +++ b/pkg/agent/supportbundlecollection/support_bundle_controller_test.go @@ -16,7 +16,6 @@ package supportbundlecollection import ( "fmt" - "io" "testing" "github.com/spf13/afero" @@ -29,6 +28,8 @@ import ( "k8s.io/klog/v2" "k8s.io/utils/exec" + "antrea.io/antrea/pkg/util/ftp" + agentquerier "antrea.io/antrea/pkg/agent/querier" "antrea.io/antrea/pkg/apis/controlplane" cpv1b2 "antrea.io/antrea/pkg/apis/controlplane/v1beta2" @@ -69,7 +70,7 @@ func TestSupportBundleCollectionAdd(t *testing.T) { supportBundleCollection *cpv1b2.SupportBundleCollection expectedCompleted bool agentDumper *mockAgentDumper - uploader uploader + uploader ftp.UpLoader }{ { name: "Add SupportBundleCollection", @@ -90,7 +91,7 @@ func TestSupportBundleCollectionAdd(t *testing.T) { supportBundleCollection: generateSupportbundleCollection("supportBundle3", "https://10.220.175.92:22/root/supportbundle"), expectedCompleted: false, agentDumper: &mockAgentDumper{}, - uploader: &testUploader{}, + uploader: &testFailedUploader{}, }, { name: "Add SupportBundleCollection with retry logics", @@ -191,7 +192,7 @@ func TestSupportBundleCollectionDelete(t *testing.T) { type testUploader struct { } -func (uploader *testUploader) upload(address string, path string, config *ssh.ClientConfig, tarGzFile io.Reader) error { +func (uploader *testUploader) Upload(url string, fileName string, config *ssh.ClientConfig, outputFile afero.File) error { klog.Info("Called test uploader") return nil } @@ -199,7 +200,7 @@ func (uploader *testUploader) upload(address string, path string, config *ssh.Cl type testFailedUploader struct { } -func (uploader *testFailedUploader) upload(address string, path string, config *ssh.ClientConfig, tarGzFile io.Reader) error { +func (uploader *testFailedUploader) Upload(url string, fileName string, config *ssh.ClientConfig, outputFile afero.File) error { klog.Info("Called test uploader for failed case") return fmt.Errorf("uploader failed") } diff --git a/pkg/apis/crd/v1alpha1/register.go b/pkg/apis/crd/v1alpha1/register.go index a9492ab5ac0..fc9924fa7c9 100644 --- a/pkg/apis/crd/v1alpha1/register.go +++ b/pkg/apis/crd/v1alpha1/register.go @@ -61,6 +61,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &ExternalNodeList{}, &SupportBundleCollection{}, &SupportBundleCollectionList{}, + &PacketSampling{}, + &PacketSamplingList{}, ) metav1.AddToGroupVersion( diff --git a/pkg/apis/crd/v1alpha1/types.go b/pkg/apis/crd/v1alpha1/types.go index 8682ddeb09b..b8c820c2b09 100644 --- a/pkg/apis/crd/v1alpha1/types.go +++ b/pkg/apis/crd/v1alpha1/types.go @@ -935,3 +935,72 @@ type TLSProtocol struct { // SNI (Server Name Indication) indicates the server domain name in the TLS/SSL hello message. SNI string `json:"sni,omitempty"` } + +type PacketSamplingType string + +const ( + FirstNSampling PacketSamplingType = "FirstNSampling" +) + +type FirstNSamplingConfig struct { + Number int32 `json:"number,omitempty"` +} + +const DefaultPacketSamplingTimeout uint16 = 60 + +type PacketSamplingPhase string + +const ( + PacketSamplingRunning PacketSamplingPhase = "Running" + PacketSamplingSucceeded PacketSamplingPhase = "Succeeded" + PacketSamplingFailed PacketSamplingPhase = "Failed" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +type PacketSamplingList struct { + metav1.TypeMeta `json:",inline"` + // +optional + metav1.ListMeta `json:"metadata,omitempty"` + + Items []PacketSampling `json:"items"` +} + +// +genclient +// +genclient:nonNamespaced +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +type PacketSampling struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec PacketSamplingSpec `json:"spec,omitempty"` + Status PacketSamplingStatus `json:"status,omitempty"` +} + +type PacketSamplingSpec struct { + Timeout uint16 `json:"timeout,omitempty"` + // Type is the sampling type. Currently only FirstN is supported. + Type PacketSamplingType `json:"type,omitempty"` + // FirstNSamplingConfig contains the config for the FirstN type sampling. The only supported parameter is + // `Number` at the moment, means capture the first specified number of packet in a flow. + FirstNSamplingConfig *FirstNSamplingConfig `json:"firstNSamplingConfig,omitempty"` + Source Source `json:"source,omitempty"` + Destination Destination `json:"destination,omitempty"` + Packet Packet `json:"packet,omitempty"` + // FileServer the sftp url config for the fileServer. Captured packets will be uploaded to this server. + FileServer BundleFileServer `json:"fileServer,omitempty"` + Authentication BundleServerAuthConfiguration `json:"authentication,omitempty"` +} + +type PacketSamplingStatus struct { + Phase PacketSamplingPhase `json:"phase,omitempty"` + // Reason recorded the failed reason when the sampling failed. + Reason string `json:"reason,omitempty"` + // NumCapturedPackets record how many packets has been captured. If it reach the target number, the sampling + // can be considered as finished. + NumCapturedPackets int32 `json:"numCapturedPackets,omitempty"` + // PacketsPath is the path where the captured packets are temporarily stored in the container. It will be + // removed after the PacketSampling is deleted. + PacketsPath string `json:"packetsPath,omitempty"` + StartTime *metav1.Time `json:"startTime,omitempty"` +} diff --git a/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go index f80a99a2c64..493708adcb4 100644 --- a/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go @@ -353,6 +353,22 @@ func (in *ExternalNodeSpec) DeepCopy() *ExternalNodeSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FirstNSamplingConfig) DeepCopyInto(out *FirstNSamplingConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FirstNSamplingConfig. +func (in *FirstNSamplingConfig) DeepCopy() *FirstNSamplingConfig { + if in == nil { + return nil + } + out := new(FirstNSamplingConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPProtocol) DeepCopyInto(out *HTTPProtocol) { *out = *in @@ -864,6 +880,113 @@ func (in *Packet) DeepCopy() *Packet { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PacketSampling) DeepCopyInto(out *PacketSampling) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PacketSampling. +func (in *PacketSampling) DeepCopy() *PacketSampling { + if in == nil { + return nil + } + out := new(PacketSampling) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PacketSampling) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PacketSamplingList) DeepCopyInto(out *PacketSamplingList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]PacketSampling, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PacketSamplingList. +func (in *PacketSamplingList) DeepCopy() *PacketSamplingList { + if in == nil { + return nil + } + out := new(PacketSamplingList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PacketSamplingList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PacketSamplingSpec) DeepCopyInto(out *PacketSamplingSpec) { + *out = *in + if in.FirstNSamplingConfig != nil { + in, out := &in.FirstNSamplingConfig, &out.FirstNSamplingConfig + *out = new(FirstNSamplingConfig) + **out = **in + } + out.Source = in.Source + out.Destination = in.Destination + in.Packet.DeepCopyInto(&out.Packet) + out.FileServer = in.FileServer + in.Authentication.DeepCopyInto(&out.Authentication) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PacketSamplingSpec. +func (in *PacketSamplingSpec) DeepCopy() *PacketSamplingSpec { + if in == nil { + return nil + } + out := new(PacketSamplingSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PacketSamplingStatus) DeepCopyInto(out *PacketSamplingStatus) { + *out = *in + if in.StartTime != nil { + in, out := &in.StartTime, &out.StartTime + *out = (*in).DeepCopy() + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PacketSamplingStatus. +func (in *PacketSamplingStatus) DeepCopy() *PacketSamplingStatus { + if in == nil { + return nil + } + out := new(PacketSamplingStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PeerNamespaces) DeepCopyInto(out *PeerNamespaces) { *out = *in diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 1ce26c162d5..bc3068d23a7 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -66,6 +66,7 @@ import ( "antrea.io/antrea/pkg/controller/externalippool" "antrea.io/antrea/pkg/controller/ipam" controllernetworkpolicy "antrea.io/antrea/pkg/controller/networkpolicy" + "antrea.io/antrea/pkg/controller/packetsampling" "antrea.io/antrea/pkg/controller/querier" "antrea.io/antrea/pkg/controller/stats" controllerbundlecollection "antrea.io/antrea/pkg/controller/supportbundlecollection" @@ -342,6 +343,11 @@ func installHandlers(c *ExtraConfig, s *genericapiserver.GenericAPIServer) { if features.DefaultFeatureGate.Enabled(features.Traceflow) { s.Handler.NonGoRestfulMux.HandleFunc("/validate/traceflow", webhook.HandlerForValidateFunc(c.traceflowController.Validate)) } + + if features.DefaultFeatureGate.Enabled(features.PacketSampling) { + s.Handler.NonGoRestfulMux.HandleFunc("/validate/packetsampling", webhook.HandlerForValidateFunc(packetsampling.Validate)) + } + } func DefaultCAConfig() *certificate.CAConfig { diff --git a/pkg/apiserver/handlers/featuregates/handler_test.go b/pkg/apiserver/handlers/featuregates/handler_test.go index 51f49a68ff5..b06d3ecaeb1 100644 --- a/pkg/apiserver/handlers/featuregates/handler_test.go +++ b/pkg/apiserver/handlers/featuregates/handler_test.go @@ -70,6 +70,7 @@ func Test_getGatesResponse(t *testing.T) { {Component: "agent", Name: "NetworkPolicyStats", Status: "Enabled", Version: "BETA"}, {Component: "agent", Name: "NodeNetworkPolicy", Status: "Disabled", Version: "ALPHA"}, {Component: "agent", Name: "NodePortLocal", Status: "Enabled", Version: "GA"}, + {Component: "agent", Name: "PacketSampling", Status: "Disabled", Version: "ALPHA"}, {Component: "agent", Name: "SecondaryNetwork", Status: "Disabled", Version: "ALPHA"}, {Component: "agent", Name: "ServiceExternalIP", Status: "Disabled", Version: "ALPHA"}, {Component: "agent", Name: "SupportBundleCollection", Status: "Disabled", Version: "ALPHA"}, @@ -201,6 +202,7 @@ func Test_getControllerGatesResponse(t *testing.T) { {Component: "controller", Name: "Multicluster", Status: "Disabled", Version: "ALPHA"}, {Component: "controller", Name: "NetworkPolicyStats", Status: "Enabled", Version: "BETA"}, {Component: "controller", Name: "NodeIPAM", Status: "Enabled", Version: "BETA"}, + {Component: "controller", Name: "PacketSampling", Status: "Disabled", Version: "ALPHA"}, {Component: "controller", Name: "ServiceExternalIP", Status: "Disabled", Version: "ALPHA"}, {Component: "controller", Name: "SupportBundleCollection", Status: "Disabled", Version: "ALPHA"}, {Component: "controller", Name: "Traceflow", Status: "Enabled", Version: "BETA"}, diff --git a/pkg/client/clientset/versioned/typed/crd/v1alpha1/crd_client.go b/pkg/client/clientset/versioned/typed/crd/v1alpha1/crd_client.go index 140c2de5f47..d0d743b938b 100644 --- a/pkg/client/clientset/versioned/typed/crd/v1alpha1/crd_client.go +++ b/pkg/client/clientset/versioned/typed/crd/v1alpha1/crd_client.go @@ -1,4 +1,4 @@ -// Copyright 2022 Antrea Authors +// Copyright 2023 Antrea Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -29,6 +29,7 @@ type CrdV1alpha1Interface interface { ClusterNetworkPoliciesGetter ExternalNodesGetter NetworkPoliciesGetter + PacketSamplingsGetter SupportBundleCollectionsGetter TiersGetter TraceflowsGetter @@ -51,6 +52,10 @@ func (c *CrdV1alpha1Client) NetworkPolicies(namespace string) NetworkPolicyInter return newNetworkPolicies(c, namespace) } +func (c *CrdV1alpha1Client) PacketSamplings() PacketSamplingInterface { + return newPacketSamplings(c) +} + func (c *CrdV1alpha1Client) SupportBundleCollections() SupportBundleCollectionInterface { return newSupportBundleCollections(c) } diff --git a/pkg/client/clientset/versioned/typed/crd/v1alpha1/fake/fake_crd_client.go b/pkg/client/clientset/versioned/typed/crd/v1alpha1/fake/fake_crd_client.go index a90f91178bc..c9bbe091330 100644 --- a/pkg/client/clientset/versioned/typed/crd/v1alpha1/fake/fake_crd_client.go +++ b/pkg/client/clientset/versioned/typed/crd/v1alpha1/fake/fake_crd_client.go @@ -1,4 +1,4 @@ -// Copyright 2022 Antrea Authors +// Copyright 2023 Antrea Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -38,6 +38,10 @@ func (c *FakeCrdV1alpha1) NetworkPolicies(namespace string) v1alpha1.NetworkPoli return &FakeNetworkPolicies{c, namespace} } +func (c *FakeCrdV1alpha1) PacketSamplings() v1alpha1.PacketSamplingInterface { + return &FakePacketSamplings{c} +} + func (c *FakeCrdV1alpha1) SupportBundleCollections() v1alpha1.SupportBundleCollectionInterface { return &FakeSupportBundleCollections{c} } diff --git a/pkg/client/clientset/versioned/typed/crd/v1alpha1/generated_expansion.go b/pkg/client/clientset/versioned/typed/crd/v1alpha1/generated_expansion.go index d4d390824f7..4569fbeaaf6 100644 --- a/pkg/client/clientset/versioned/typed/crd/v1alpha1/generated_expansion.go +++ b/pkg/client/clientset/versioned/typed/crd/v1alpha1/generated_expansion.go @@ -1,4 +1,4 @@ -// Copyright 2022 Antrea Authors +// Copyright 2023 Antrea Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,6 +22,8 @@ type ExternalNodeExpansion interface{} type NetworkPolicyExpansion interface{} +type PacketSamplingExpansion interface{} + type SupportBundleCollectionExpansion interface{} type TierExpansion interface{} diff --git a/pkg/client/informers/externalversions/crd/v1alpha1/interface.go b/pkg/client/informers/externalversions/crd/v1alpha1/interface.go index 5f34c67e3e2..78e3db6f7e5 100644 --- a/pkg/client/informers/externalversions/crd/v1alpha1/interface.go +++ b/pkg/client/informers/externalversions/crd/v1alpha1/interface.go @@ -1,4 +1,4 @@ -// Copyright 2022 Antrea Authors +// Copyright 2023 Antrea Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -28,6 +28,8 @@ type Interface interface { ExternalNodes() ExternalNodeInformer // NetworkPolicies returns a NetworkPolicyInformer. NetworkPolicies() NetworkPolicyInformer + // PacketSamplings returns a PacketSamplingInformer. + PacketSamplings() PacketSamplingInformer // SupportBundleCollections returns a SupportBundleCollectionInformer. SupportBundleCollections() SupportBundleCollectionInformer // Tiers returns a TierInformer. @@ -62,6 +64,11 @@ func (v *version) NetworkPolicies() NetworkPolicyInformer { return &networkPolicyInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} } +// PacketSamplings returns a PacketSamplingInformer. +func (v *version) PacketSamplings() PacketSamplingInformer { + return &packetSamplingInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} +} + // SupportBundleCollections returns a SupportBundleCollectionInformer. func (v *version) SupportBundleCollections() SupportBundleCollectionInformer { return &supportBundleCollectionInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go index e3f7e903ce7..1eb0c2df1e9 100644 --- a/pkg/client/informers/externalversions/generic.go +++ b/pkg/client/informers/externalversions/generic.go @@ -60,6 +60,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource return &genericInformer{resource: resource.GroupResource(), informer: f.Crd().V1alpha1().ExternalNodes().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("networkpolicies"): return &genericInformer{resource: resource.GroupResource(), informer: f.Crd().V1alpha1().NetworkPolicies().Informer()}, nil + case v1alpha1.SchemeGroupVersion.WithResource("packetsamplings"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Crd().V1alpha1().PacketSamplings().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("supportbundlecollections"): return &genericInformer{resource: resource.GroupResource(), informer: f.Crd().V1alpha1().SupportBundleCollections().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("tiers"): diff --git a/pkg/client/listers/crd/v1alpha1/expansion_generated.go b/pkg/client/listers/crd/v1alpha1/expansion_generated.go index 99f199fd8a1..004e8cf7768 100644 --- a/pkg/client/listers/crd/v1alpha1/expansion_generated.go +++ b/pkg/client/listers/crd/v1alpha1/expansion_generated.go @@ -1,4 +1,4 @@ -// Copyright 2022 Antrea Authors +// Copyright 2023 Antrea Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -36,6 +36,10 @@ type NetworkPolicyListerExpansion interface{} // NetworkPolicyNamespaceLister. type NetworkPolicyNamespaceListerExpansion interface{} +// PacketSamplingListerExpansion allows custom methods to be added to +// PacketSamplingLister. +type PacketSamplingListerExpansion interface{} + // SupportBundleCollectionListerExpansion allows custom methods to be added to // SupportBundleCollectionLister. type SupportBundleCollectionListerExpansion interface{} diff --git a/pkg/controller/packetsampling/validate.go b/pkg/controller/packetsampling/validate.go new file mode 100644 index 00000000000..8481a83309e --- /dev/null +++ b/pkg/controller/packetsampling/validate.go @@ -0,0 +1,108 @@ +// Copyright 2024 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package packetsampling + +import ( + "encoding/json" + "fmt" + "net" + + admv1 "k8s.io/api/admission/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + + crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" + "antrea.io/antrea/pkg/util/ftp" +) + +func Validate(review *admv1.AdmissionReview) *admv1.AdmissionResponse { + newResponse := func(allowed bool, deniedReason string) *admv1.AdmissionResponse { + resp := &admv1.AdmissionResponse{ + UID: review.Request.UID, + Allowed: allowed, + } + if !allowed { + resp.Result = &metav1.Status{ + Message: deniedReason, + } + } + return resp + } + + klog.V(2).InfoS("Validating PacketSampling", "request", review.Request) + + var newObj crdv1alpha1.PacketSampling + if review.Request.Object.Raw != nil { + if err := json.Unmarshal(review.Request.Object.Raw, &newObj); err != nil { + klog.ErrorS(err, "Error de-serializing current Traceflow") + return newResponse(false, err.Error()) + } + } + + switch review.Request.Operation { + case admv1.Create: + klog.V(2).InfoS("Validating CREATE request for PacketSampling", "name", newObj.Name) + allowed, deniedReason := validate(&newObj) + return newResponse(allowed, deniedReason) + case admv1.Update: + klog.V(2).InfoS("Validating UPDATE request for PacketSampling", "name", newObj.Name) + allowed, deniedReason := validate(&newObj) + return newResponse(allowed, deniedReason) + default: + err := fmt.Errorf("invalid request operation %s for Traceflow", review.Request.Operation) + klog.ErrorS(err, "Failed to validate PacketSampling", "name", newObj.Name) + return newResponse(false, err.Error()) + } +} + +func validate(ps *crdv1alpha1.PacketSampling) (allowed bool, deniedReason string) { + if ps.Spec.Source.Pod == "" && ps.Spec.Destination.Pod == "" { + return false, fmt.Sprintf("PacketSampling %s has neither source nor destination Pod specified", ps.Name) + } + + if ps.Spec.Type != crdv1alpha1.FirstNSampling { + return false, fmt.Sprintf("PacketSampling %s has invalid type %s, supported type is [%s]", ps.Name, ps.Spec.Type, crdv1alpha1.FirstNSampling) + } + + if ps.Spec.FirstNSamplingConfig == nil { + return false, fmt.Sprintf("PacketSampling %s has no FirstNSamplingConfig", ps.Name) + } + + isIPv6 := ps.Spec.Packet.IPv6Header != nil + if ps.Spec.Source.IP != "" { + sourceIP := net.ParseIP(ps.Spec.Source.IP) + if sourceIP == nil { + return false, "source IP is not valid" + } + if isIPv6 != (sourceIP.To4() == nil) { + return false, "source IP does not match the IP header family" + } + } + + if ps.Spec.Destination.IP != "" { + destIP := net.ParseIP(ps.Spec.Destination.IP) + if destIP == nil { + return false, "destination IP is not valid" + } + if isIPv6 != (destIP.To4() == nil) { + return false, "destination IP does not match the IP header family" + } + } + + if _, err := ftp.ParseFTPUploadUrl(ps.Spec.FileServer.URL); err != nil { + return false, fmt.Sprintf("invalid file server address: %s", err.Error()) + } + return true, "" +} diff --git a/pkg/controller/packetsampling/validate_test.go b/pkg/controller/packetsampling/validate_test.go new file mode 100644 index 00000000000..516cc163497 --- /dev/null +++ b/pkg/controller/packetsampling/validate_test.go @@ -0,0 +1,227 @@ +// Copyright 2024 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package packetsampling + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + admv1 "k8s.io/api/admission/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" +) + +func TestControllerValidate(t *testing.T) { + tests := []struct { + name string + + // input + oldSpec *crdv1alpha1.PacketSamplingSpec + newSpec *crdv1alpha1.PacketSamplingSpec + + // expected output + allowed bool + deniedReason string + }{ + { + name: "Traceflow should have either source or destination Pod assigned", + newSpec: &crdv1alpha1.PacketSamplingSpec{}, + deniedReason: "PacketSampling ps has neither source nor destination Pod specified", + }, + { + name: "Must assign sampling type", + newSpec: &crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: "test-ns", + Pod: "test-pod", + }, + }, + deniedReason: "PacketSampling ps has invalid type (supported are [FirstNSampling])", + }, + { + name: "FistNSampling config not set", + newSpec: &crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: "test-ns", + Pod: "test-pod", + }, + Type: crdv1alpha1.FirstNSampling, + }, + deniedReason: "PacketSampling ps has no FirstNSamplingConfig", + }, + { + name: "Source IP family does not match", + newSpec: &crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + IP: "127.0.0.1", + }, + Type: crdv1alpha1.FirstNSampling, + FirstNSamplingConfig: &crdv1alpha1.FirstNSamplingConfig{ + Number: 4, + }, + Destination: crdv1alpha1.Destination{ + Namespace: "test-ns", + Pod: "test-pod", + }, + Packet: crdv1alpha1.Packet{ + IPv6Header: &crdv1alpha1.IPv6Header{ + HopLimit: 1, + }, + }, + }, + allowed: false, + deniedReason: "source IP does not match the IP header family", + }, + { + name: "Destination IP family does not match", + newSpec: &crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: "test-ns", + Pod: "test-pod", + }, + Type: crdv1alpha1.FirstNSampling, + FirstNSamplingConfig: &crdv1alpha1.FirstNSamplingConfig{ + Number: 4, + }, + Destination: crdv1alpha1.Destination{ + IP: "fe80::aede:48ff:fe00:1122", + }, + Packet: crdv1alpha1.Packet{}, + }, + allowed: false, + deniedReason: "destination IP does not match the IP header family", + }, + { + name: "Destination IP not valid", + newSpec: &crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: "test-ns", + Pod: "test-pod", + }, + Type: crdv1alpha1.FirstNSampling, + FirstNSamplingConfig: &crdv1alpha1.FirstNSamplingConfig{ + Number: 4, + }, + Destination: crdv1alpha1.Destination{ + IP: "aaa:111", + }, + Packet: crdv1alpha1.Packet{}, + }, + allowed: false, + deniedReason: "destination IP is not valid", + }, + { + name: "source IP not valid", + newSpec: &crdv1alpha1.PacketSamplingSpec{ + Destination: crdv1alpha1.Destination{ + Namespace: "test-ns", + Pod: "test-pod", + }, + Type: crdv1alpha1.FirstNSampling, + FirstNSamplingConfig: &crdv1alpha1.FirstNSamplingConfig{ + Number: 4, + }, + Source: crdv1alpha1.Source{ + IP: "aaa:111", + }, + Packet: crdv1alpha1.Packet{}, + }, + allowed: false, + deniedReason: "source IP is not valid", + }, + { + name: "invalid ftp server address", + newSpec: &crdv1alpha1.PacketSamplingSpec{ + Destination: crdv1alpha1.Destination{ + Namespace: "test-ns", + Pod: "test-pod", + }, + Type: crdv1alpha1.FirstNSampling, + FirstNSamplingConfig: &crdv1alpha1.FirstNSamplingConfig{ + Number: 4, + }, + FileServer: crdv1alpha1.BundleFileServer{ + URL: "https://127.0.0.1:22/root/supportbundle", + }, + Source: crdv1alpha1.Source{ + Namespace: "test-ns", + Pod: "test-pod", + }, + Packet: crdv1alpha1.Packet{}, + }, + allowed: false, + deniedReason: "invalid file server address: not sftp protocol", + }, + { + name: "Valid request", + newSpec: &crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: "test-ns", + Pod: "test-pod", + }, + Type: crdv1alpha1.FirstNSampling, + FirstNSamplingConfig: &crdv1alpha1.FirstNSamplingConfig{ + Number: 4, + }, + FileServer: crdv1alpha1.BundleFileServer{ + URL: "sftp://127.0.0.1:22/root/supportbundle", + }, + }, + allowed: true, + }, + } + for _, ps := range tests { + t.Run(ps.name, func(t *testing.T) { + var request *admv1.AdmissionRequest + if ps.oldSpec != nil && ps.newSpec != nil { + request = &admv1.AdmissionRequest{ + Operation: admv1.Update, + OldObject: toRawExtension(ps.oldSpec), + Object: toRawExtension(ps.newSpec), + } + } else if ps.newSpec != nil { + request = &admv1.AdmissionRequest{ + Operation: admv1.Create, + Object: toRawExtension(ps.newSpec), + } + } + review := &admv1.AdmissionReview{ + Request: request, + } + + expectedResponse := &admv1.AdmissionResponse{ + Allowed: ps.allowed, + } + if !ps.allowed { + expectedResponse.Result = &metav1.Status{ + Message: ps.deniedReason, + } + } + + response := Validate(review) + assert.Equal(t, expectedResponse, response) + }) + } +} + +func toRawExtension(spec *crdv1alpha1.PacketSamplingSpec) runtime.RawExtension { + ps := &crdv1alpha1.PacketSampling{Spec: *spec} + ps.Name = "ps" + raw, _ := json.Marshal(ps) + return runtime.RawExtension{Raw: raw} +} diff --git a/pkg/controller/supportbundlecollection/controller.go b/pkg/controller/supportbundlecollection/controller.go index aabf9101f0b..98d52a66aab 100644 --- a/pkg/controller/supportbundlecollection/controller.go +++ b/pkg/controller/supportbundlecollection/controller.go @@ -15,7 +15,6 @@ package supportbundlecollection import ( - "bytes" "context" "fmt" "reflect" @@ -45,6 +44,7 @@ import ( crdinformers "antrea.io/antrea/pkg/client/informers/externalversions/crd/v1alpha1" crdlisters "antrea.io/antrea/pkg/client/listers/crd/v1alpha1" "antrea.io/antrea/pkg/controller/types" + "antrea.io/antrea/pkg/util/ftp" "antrea.io/antrea/pkg/util/k8s" ) @@ -390,7 +390,7 @@ func (c *Controller) createInternalSupportBundleCollection(bundle *v1alpha1.Supp } nodeSpan := nodeNames.Union(externalNodeNames) // Get authentication from the Secret provided in authentication field in the CRD - authentication, err := c.parseBundleAuth(bundle.Spec.Authentication) + authentication, err := ftp.ParseBundleAuth(bundle.Spec.Authentication, c.kubeClient) if err != nil { klog.ErrorS(err, "Failed to get authentication defined in the SupportBundleCollection CR", "name", bundle.Name, "authentication", bundle.Spec.Authentication) return nil, err @@ -511,60 +511,6 @@ func (c *Controller) deleteInternalSupportBundleCollection(key string) error { return nil } -// parseBundleAuth returns the authentication from the Secret provided in BundleServerAuthConfiguration. -// The authentication is stored in the Secret Data with a key decided by the AuthType, and encoded using base64. -func (c *Controller) parseBundleAuth(authentication v1alpha1.BundleServerAuthConfiguration) (*controlplane.BundleServerAuthConfiguration, error) { - secretReference := authentication.AuthSecret - if secretReference == nil { - return nil, fmt.Errorf("authentication is not specified") - } - secret, err := c.kubeClient.CoreV1().Secrets(secretReference.Namespace).Get(context.TODO(), secretReference.Name, metav1.GetOptions{}) - if err != nil { - return nil, fmt.Errorf("unable to get Secret with name %s in Namespace %s: %v", secretReference.Name, secretReference.Namespace, err) - } - parseAuthValue := func(secretData map[string][]byte, key string) (string, error) { - authValue, found := secret.Data[key] - if !found { - return "", fmt.Errorf("not found authentication in Secret %s/%s with key %s", secretReference.Namespace, secretReference.Name, key) - } - return bytes.NewBuffer(authValue).String(), nil - } - switch authentication.AuthType { - case v1alpha1.APIKey: - value, err := parseAuthValue(secret.Data, secretKeyWithAPIKey) - if err != nil { - return nil, err - } - return &controlplane.BundleServerAuthConfiguration{ - APIKey: value, - }, nil - case v1alpha1.BearerToken: - value, err := parseAuthValue(secret.Data, secretKeyWithBearerToken) - if err != nil { - return nil, err - } - return &controlplane.BundleServerAuthConfiguration{ - BearerToken: value, - }, nil - case v1alpha1.BasicAuthentication: - username, err := parseAuthValue(secret.Data, secretKeyWithUsername) - if err != nil { - return nil, err - } - password, err := parseAuthValue(secret.Data, secretKeyWithPassword) - if err != nil { - return nil, err - } - return &controlplane.BundleServerAuthConfiguration{ - BasicAuthentication: &controlplane.BasicAuthentication{ - Username: username, - Password: password, - }, - }, nil - } - return nil, fmt.Errorf("unsupported authentication type %s", authentication.AuthType) -} - // addInternalSupportBundleCollection adds internalBundle into supportBundleCollectionStore, and creates a // supportBundleCollectionAppliedTo resource to maintain the SupportBundleCollection's required Nodes or ExternalNodes. func (c *Controller) addInternalSupportBundleCollection( diff --git a/pkg/controller/supportbundlecollection/controller_test.go b/pkg/controller/supportbundlecollection/controller_test.go index 678215dadd5..1b3d2eb490c 100644 --- a/pkg/controller/supportbundlecollection/controller_test.go +++ b/pkg/controller/supportbundlecollection/controller_test.go @@ -660,111 +660,6 @@ type secretConfig struct { data map[string][]byte } -func TestParseBundleAuth(t *testing.T) { - ns := "ns-auth" - apiKey := testKeyString - token := testTokenString - usr := "user" - pwd := "pwd123456" - var secretObjects []runtime.Object - for _, s := range prepareSecrets(ns, []secretConfig{ - {name: "s1", data: map[string][]byte{secretKeyWithAPIKey: []byte(apiKey)}}, - {name: "s2", data: map[string][]byte{secretKeyWithBearerToken: []byte(token)}}, - {name: "s3", data: map[string][]byte{secretKeyWithUsername: []byte(usr), secretKeyWithPassword: []byte(pwd)}}, - {name: "invalid-base64", data: map[string][]byte{secretKeyWithAPIKey: []byte("invalid string to decode with base64")}}, - {name: "invalid-secret", data: map[string][]byte{"unknown": []byte(apiKey)}}, - }) { - secretObjects = append(secretObjects, s) - } - - testClient := newTestClient(secretObjects, nil) - controller := newController(testClient) - stopCh := make(chan struct{}) - testClient.start(stopCh) - - testClient.waitForSync(stopCh) - - for _, tc := range []struct { - authentication v1alpha1.BundleServerAuthConfiguration - expectedError string - expectedAuth *controlplane.BundleServerAuthConfiguration - }{ - { - authentication: v1alpha1.BundleServerAuthConfiguration{ - AuthType: v1alpha1.APIKey, - AuthSecret: &corev1.SecretReference{ - Namespace: ns, - Name: "s1", - }, - }, - expectedAuth: &controlplane.BundleServerAuthConfiguration{ - APIKey: testKeyString, - }, - }, - { - authentication: v1alpha1.BundleServerAuthConfiguration{ - AuthType: v1alpha1.BearerToken, - AuthSecret: &corev1.SecretReference{ - Namespace: ns, - Name: "s2", - }, - }, - expectedAuth: &controlplane.BundleServerAuthConfiguration{ - BearerToken: testTokenString, - }, - }, - { - authentication: v1alpha1.BundleServerAuthConfiguration{ - AuthType: v1alpha1.BasicAuthentication, - AuthSecret: &corev1.SecretReference{ - Namespace: ns, - Name: "s3", - }, - }, - expectedAuth: &controlplane.BundleServerAuthConfiguration{ - BasicAuthentication: &controlplane.BasicAuthentication{ - Username: usr, - Password: pwd, - }, - }, - }, - { - authentication: v1alpha1.BundleServerAuthConfiguration{ - AuthType: v1alpha1.BearerToken, - AuthSecret: &corev1.SecretReference{ - Namespace: ns, - Name: "invalid-secret", - }, - }, - expectedError: fmt.Sprintf("not found authentication in Secret %s/invalid-secret with key %s", ns, secretKeyWithBearerToken), - }, - { - authentication: v1alpha1.BundleServerAuthConfiguration{ - AuthType: v1alpha1.BearerToken, - AuthSecret: &corev1.SecretReference{ - Namespace: ns, - Name: "not-exist", - }, - }, - expectedError: fmt.Sprintf("unable to get Secret with name not-exist in Namespace %s", ns), - }, - { - authentication: v1alpha1.BundleServerAuthConfiguration{ - AuthType: v1alpha1.APIKey, - AuthSecret: nil, - }, - expectedError: "authentication is not specified", - }, - } { - auth, err := controller.parseBundleAuth(tc.authentication) - if tc.expectedError != "" { - assert.Contains(t, err.Error(), tc.expectedError) - } else { - assert.Equal(t, tc.expectedAuth, auth) - } - } -} - func TestCreateAndDeleteInternalSupportBundleCollection(t *testing.T) { coreObjects, crdObjects := prepareTopology() testClient := newTestClient(coreObjects, crdObjects) diff --git a/pkg/features/antrea_features.go b/pkg/features/antrea_features.go index 36fb997ffdc..5f8d0ab5844 100644 --- a/pkg/features/antrea_features.go +++ b/pkg/features/antrea_features.go @@ -67,6 +67,10 @@ const ( // Allows to trace path from a generated packet. Traceflow featuregate.Feature = "Traceflow" + // alpha: v2.0 + // Allows to capture sampling packets for a flow. + PacketSampling featuregate.Feature = "PacketSampling" + // alpha: v0.9 // Flow exporter exports IPFIX flow records of Antrea flows seen in conntrack module. FlowExporter featuregate.Feature = "FlowExporter" @@ -179,6 +183,7 @@ var ( TopologyAwareHints: {Default: true, PreRelease: featuregate.Beta}, CleanupStaleUDPSvcConntrack: {Default: false, PreRelease: featuregate.Alpha}, Traceflow: {Default: true, PreRelease: featuregate.Beta}, + PacketSampling: {Default: false, PreRelease: featuregate.Alpha}, AntreaIPAM: {Default: false, PreRelease: featuregate.Alpha}, FlowExporter: {Default: false, PreRelease: featuregate.Alpha}, NetworkPolicyStats: {Default: true, PreRelease: featuregate.Beta}, @@ -224,6 +229,7 @@ var ( SupportBundleCollection, TopologyAwareHints, Traceflow, + PacketSampling, TrafficControl, EgressTrafficShaping, EgressSeparateSubnet, @@ -276,6 +282,7 @@ var ( EgressSeparateSubnet: {}, NodeNetworkPolicy: {}, L7FlowExporter: {}, + PacketSampling: {}, } // supportedFeaturesOnExternalNode records the features supported on an external // Node. Antrea Agent checks the enabled features if it is running on an diff --git a/pkg/util/ftp/auth.go b/pkg/util/ftp/auth.go new file mode 100644 index 00000000000..2900c0e4812 --- /dev/null +++ b/pkg/util/ftp/auth.go @@ -0,0 +1,102 @@ +// Copyright 2024 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ftp + +import ( + "bytes" + "context" + "fmt" + "time" + + "golang.org/x/crypto/ssh" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + + "antrea.io/antrea/pkg/apis/controlplane" + crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" +) + +const ( + secretKeyWithAPIKey = "apikey" + secretKeyWithBearerToken = "token" + secretKeyWithUsername = "username" + secretKeyWithPassword = "password" +) + +// GenSSHClientConfig generates ssh.ClientConfig from username and password +func GenSSHClientConfig(username, password string) *ssh.ClientConfig { + cfg := &ssh.ClientConfig{ + User: username, + Auth: []ssh.AuthMethod{ssh.Password(password)}, + // #nosec G106: skip host key check here and users can specify their own checks if needed + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Timeout: time.Second, + } + return cfg +} + +// ParseBundleAuth returns the authentication from the Secret provided in BundleServerAuthConfiguration. +// The authentication is stored in the Secret Data with a key decided by the AuthType, and encoded using base64. +func ParseBundleAuth(authentication crdv1alpha1.BundleServerAuthConfiguration, kubeClient clientset.Interface) (*controlplane.BundleServerAuthConfiguration, error) { + secretReference := authentication.AuthSecret + if secretReference == nil { + return nil, fmt.Errorf("authentication is not specified") + } + secret, err := kubeClient.CoreV1().Secrets(secretReference.Namespace).Get(context.TODO(), secretReference.Name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("unable to get Secret with name %s in Namespace %s: %v", secretReference.Name, secretReference.Namespace, err) + } + parseAuthValue := func(secretData map[string][]byte, key string) (string, error) { + authValue, found := secret.Data[key] + if !found { + return "", fmt.Errorf("not found authentication in Secret %s/%s with key %s", secretReference.Namespace, secretReference.Name, key) + } + return bytes.NewBuffer(authValue).String(), nil + } + switch authentication.AuthType { + case crdv1alpha1.APIKey: + value, err := parseAuthValue(secret.Data, secretKeyWithAPIKey) + if err != nil { + return nil, err + } + return &controlplane.BundleServerAuthConfiguration{ + APIKey: value, + }, nil + case crdv1alpha1.BearerToken: + value, err := parseAuthValue(secret.Data, secretKeyWithBearerToken) + if err != nil { + return nil, err + } + return &controlplane.BundleServerAuthConfiguration{ + BearerToken: value, + }, nil + case crdv1alpha1.BasicAuthentication: + username, err := parseAuthValue(secret.Data, secretKeyWithUsername) + if err != nil { + return nil, err + } + password, err := parseAuthValue(secret.Data, secretKeyWithPassword) + if err != nil { + return nil, err + } + return &controlplane.BundleServerAuthConfiguration{ + BasicAuthentication: &controlplane.BasicAuthentication{ + Username: username, + Password: password, + }, + }, nil + } + return nil, fmt.Errorf("unsupported authentication type %s", authentication.AuthType) +} diff --git a/pkg/util/ftp/auth_test.go b/pkg/util/ftp/auth_test.go new file mode 100644 index 00000000000..f2d4a3dc0bf --- /dev/null +++ b/pkg/util/ftp/auth_test.go @@ -0,0 +1,183 @@ +// Copyright 2024 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ftp + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + + "antrea.io/antrea/pkg/apis/controlplane" + "antrea.io/antrea/pkg/apis/crd/v1alpha1" +) + +const ( + informerDefaultResync = 30 * time.Second + + testKeyString = "it is a valid API key" + testTokenString = "it is a valid token" +) + +type secretConfig struct { + name string + data map[string][]byte +} + +func prepareSecrets(ns string, secretConfigs []secretConfig) []*corev1.Secret { + secrets := make([]*corev1.Secret, 0) + for _, s := range secretConfigs { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: s.name, + Namespace: ns, + }, + Data: s.data, + } + secrets = append(secrets, secret) + } + return secrets +} + +type testClient struct { + client kubernetes.Interface + informerFactory informers.SharedInformerFactory +} + +func (c *testClient) start(stopCh <-chan struct{}) { + c.informerFactory.Start(stopCh) +} + +func (c *testClient) waitForSync(stopCh <-chan struct{}) { + c.informerFactory.WaitForCacheSync(stopCh) +} + +func newTestClient(coreObjects []runtime.Object, crdObjects []runtime.Object) *testClient { + client := fake.NewSimpleClientset(coreObjects...) + return &testClient{ + client: client, + informerFactory: informers.NewSharedInformerFactory(client, informerDefaultResync), + } +} + +func TestParseBundleAuth(t *testing.T) { + ns := "ns-auth" + apiKey := testKeyString + token := testTokenString + usr := "user" + pwd := "pwd123456" + var secretObjects []runtime.Object + for _, s := range prepareSecrets(ns, []secretConfig{ + {name: "s1", data: map[string][]byte{secretKeyWithAPIKey: []byte(apiKey)}}, + {name: "s2", data: map[string][]byte{secretKeyWithBearerToken: []byte(token)}}, + {name: "s3", data: map[string][]byte{secretKeyWithUsername: []byte(usr), secretKeyWithPassword: []byte(pwd)}}, + {name: "invalid-base64", data: map[string][]byte{secretKeyWithAPIKey: []byte("invalid string to decode with base64")}}, + {name: "invalid-secret", data: map[string][]byte{"unknown": []byte(apiKey)}}, + }) { + secretObjects = append(secretObjects, s) + } + + testClient := newTestClient(secretObjects, nil) + stopCh := make(chan struct{}) + testClient.start(stopCh) + testClient.waitForSync(stopCh) + + for _, tc := range []struct { + authentication v1alpha1.BundleServerAuthConfiguration + expectedError string + expectedAuth *controlplane.BundleServerAuthConfiguration + }{ + { + authentication: v1alpha1.BundleServerAuthConfiguration{ + AuthType: v1alpha1.APIKey, + AuthSecret: &corev1.SecretReference{ + Namespace: ns, + Name: "s1", + }, + }, + expectedAuth: &controlplane.BundleServerAuthConfiguration{ + APIKey: testKeyString, + }, + }, + { + authentication: v1alpha1.BundleServerAuthConfiguration{ + AuthType: v1alpha1.BearerToken, + AuthSecret: &corev1.SecretReference{ + Namespace: ns, + Name: "s2", + }, + }, + expectedAuth: &controlplane.BundleServerAuthConfiguration{ + BearerToken: testTokenString, + }, + }, + { + authentication: v1alpha1.BundleServerAuthConfiguration{ + AuthType: v1alpha1.BasicAuthentication, + AuthSecret: &corev1.SecretReference{ + Namespace: ns, + Name: "s3", + }, + }, + expectedAuth: &controlplane.BundleServerAuthConfiguration{ + BasicAuthentication: &controlplane.BasicAuthentication{ + Username: usr, + Password: pwd, + }, + }, + }, + { + authentication: v1alpha1.BundleServerAuthConfiguration{ + AuthType: v1alpha1.BearerToken, + AuthSecret: &corev1.SecretReference{ + Namespace: ns, + Name: "invalid-secret", + }, + }, + expectedError: fmt.Sprintf("not found authentication in Secret %s/invalid-secret with key %s", ns, secretKeyWithBearerToken), + }, + { + authentication: v1alpha1.BundleServerAuthConfiguration{ + AuthType: v1alpha1.BearerToken, + AuthSecret: &corev1.SecretReference{ + Namespace: ns, + Name: "not-exist", + }, + }, + expectedError: fmt.Sprintf("unable to get Secret with name not-exist in Namespace %s", ns), + }, + { + authentication: v1alpha1.BundleServerAuthConfiguration{ + AuthType: v1alpha1.APIKey, + AuthSecret: nil, + }, + expectedError: "authentication is not specified", + }, + } { + auth, err := ParseBundleAuth(tc.authentication, testClient.client) + if tc.expectedError != "" { + assert.Contains(t, err.Error(), tc.expectedError) + } else { + assert.Equal(t, tc.expectedAuth, auth) + } + } +} diff --git a/pkg/util/ftp/ftp.go b/pkg/util/ftp/ftp.go new file mode 100644 index 00000000000..0283fff474d --- /dev/null +++ b/pkg/util/ftp/ftp.go @@ -0,0 +1,109 @@ +// Copyright 2024 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ftp + +import ( + "fmt" + "io" + "net/url" + "path" + "time" + + "github.com/pkg/sftp" + "github.com/spf13/afero" + "golang.org/x/crypto/ssh" + "k8s.io/klog/v2" +) + +const ( + uploadToFileServerTries = 5 + uploadToFileServerRetryDelay = 5 * time.Second +) + +func ParseFTPUploadUrl(uploadUrl string) (*url.URL, error) { + parsedURL, err := url.Parse(uploadUrl) + if err != nil { + parsedURL, err = url.Parse("sftp://" + uploadUrl) + if err != nil { + return nil, err + } + } + if parsedURL.Scheme != "sftp" { + return nil, fmt.Errorf("not sftp protocol") + } + return parsedURL, nil +} + +type UpLoader interface { + // Upload uploads a file to the target sftp address using ssh config. + Upload(url string, fileName string, config *ssh.ClientConfig, outputFile afero.File) error +} + +type SftpUploader struct { +} + +func (uploader *SftpUploader) Upload(url string, fileName string, config *ssh.ClientConfig, outputFile afero.File) error { + if _, err := outputFile.Seek(0, 0); err != nil { + return fmt.Errorf("failed to upload to file server while setting offset: %v", err) + } + // url should be like: 10.92.23.154:22/path or sftp://10.92.23.154:22/path + parsedURL, _ := ParseFTPUploadUrl(url) + joinedPath := path.Join(parsedURL.Path, fileName) + + triesLeft := uploadToFileServerTries + var uploadErr error + for triesLeft > 0 { + if uploadErr = upload(parsedURL.Host, joinedPath, config, outputFile); uploadErr == nil { + return nil + } + triesLeft-- + if triesLeft == 0 { + return fmt.Errorf("failed to upload file after %d attempts", uploadToFileServerTries) + } + klog.InfoS("Failed to upload file", "UploadError", uploadErr, "TriesLeft", triesLeft) + time.Sleep(uploadToFileServerRetryDelay) + } + return nil +} + +func upload(address string, path string, config *ssh.ClientConfig, file io.Reader) error { + conn, err := ssh.Dial("tcp", address, config) + if err != nil { + return fmt.Errorf("error when connecting to fs server: %w", err) + } + sftpClient, err := sftp.NewClient(conn) + if err != nil { + return fmt.Errorf("error when setting up sftp client: %w", err) + } + defer func() { + if err := sftpClient.Close(); err != nil { + klog.ErrorS(err, "Error when closing sftp client") + } + }() + targetFile, err := sftpClient.Create(path) + if err != nil { + return fmt.Errorf("error when creating target file on remote: %v", err) + } + defer func() { + if err := targetFile.Close(); err != nil { + klog.ErrorS(err, "Error when closing target file on remote") + } + }() + if written, err := io.Copy(targetFile, file); err != nil { + return fmt.Errorf("error when copying target file: %v, written: %d", err, written) + } + klog.InfoS("Successfully upload file to path", "filePath", path) + return nil +} diff --git a/pkg/util/ftp/ftp_test.go b/pkg/util/ftp/ftp_test.go new file mode 100644 index 00000000000..37b4dfb14ae --- /dev/null +++ b/pkg/util/ftp/ftp_test.go @@ -0,0 +1,66 @@ +// Copyright 2024 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ftp + +import ( + "net/url" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseFTPUploadUrl(t *testing.T) { + cases := []struct { + url string + expectedError string + expectedURL url.URL + }{ + { + url: "127.0.0.1:22/path", + expectedURL: url.URL{ + Scheme: "sftp", + Host: "127.0.0.1:22", + Path: "/path", + }, + }, + { + url: "sftp://127.0.0.1:22/path", + expectedURL: url.URL{ + Scheme: "sftp", + Host: "127.0.0.1:22", + Path: "/path", + }, + }, + { + url: "https://127.0.0.1:22/root/supportbundle", + expectedError: "not sftp protocol", + }, + } + + for _, tc := range cases { + uploadUrl, err := ParseFTPUploadUrl(tc.url) + if tc.expectedError == "" { + assert.NoError(t, err) + if !reflect.DeepEqual(tc.expectedURL, *uploadUrl) { + t.Errorf("expected %v, got %v", tc.expectedURL, *uploadUrl) + + } + } else { + assert.Equal(t, tc.expectedError, err.Error()) + } + } + +} diff --git a/test/e2e/framework.go b/test/e2e/framework.go index f81bc17891d..561b9e7df07 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -1527,6 +1527,21 @@ func (data *TestData) createNginxPodOnNode(name string, ns string, nodeName stri }).WithHostNetwork(hostNetwork).Create(data) } +func (data *TestData) createUDPServerPod(name string, ns string, portNum int32, serverNode string) error { + cmd := []string{"/bin/bash", "-c"} + args := []string{ + fmt.Sprintf("/agnhost serve-hostname --udp --http=false --port %v", portNum), + } + port := corev1.ContainerPort{Name: fmt.Sprintf("port-%d", portNum), ContainerPort: portNum} + return NewPodBuilder(name, ns, agnhostImage). + OnNode(serverNode). + WithContainerName("agnhost"). + WithCommand(cmd). + WithArgs(args). + WithPorts([]corev1.ContainerPort{port}). + Create(testData) +} + // createServerPod creates a Pod that can listen to specified port and have named port set. func (data *TestData) createServerPod(name string, ns string, portName string, portNum int32, setHostPort bool, hostNetwork bool) error { // See https://github.com/kubernetes/kubernetes/blob/master/test/images/agnhost/porter/porter.go#L17 for the image's detail. diff --git a/test/e2e/packetsampling_test.go b/test/e2e/packetsampling_test.go new file mode 100644 index 00000000000..c1349955ca7 --- /dev/null +++ b/test/e2e/packetsampling_test.go @@ -0,0 +1,636 @@ +// Copyright 2024 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "fmt" + "net" + "testing" + "time" + + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + + crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" + agentconfig "antrea.io/antrea/pkg/config/agent" + controllerconfig "antrea.io/antrea/pkg/config/controller" + "antrea.io/antrea/pkg/features" +) + +var ( + psNamespace = "default" + psSecretName = "ps-secret" + tcpServerPodName = "tcp-server" + psToolboxPodName = "toolbox" + udpServerPodName = "udp-server" + nonExistPodName = "non-existing-pod" + dstServiceName = "svc" + dstServiceIP = "" +) + +type psTestCase struct { + name string + ps *crdv1alpha1.PacketSampling + expectedPhase crdv1alpha1.PacketSamplingPhase + expectedReason string + expectedNum int32 + // required IP version, skip if not match, default is 0 (no restrict) + ipVersion int + // Source Pod to run ping for live-traffic PacketSampling. + srcPod string + skipIfNeeded func(t *testing.T) +} + +// TestPacketSampling is the top-level test which contains all subtests for +// PacketSampling related test cases so they can share setup, teardown. +func TestPacketSampling(t *testing.T) { + + data, err := setupTest(t) + if err != nil { + t.Fatalf("Error when setting up test: %v", err) + } + defer teardownTest(t, data) + + var previousAgentPacketSamplingEnableState bool + var previousControllerPacketSamplingEnableState bool + + ac := func(config *agentconfig.AgentConfig) { + previousAgentPacketSamplingEnableState = config.FeatureGates[string(features.PacketSampling)] + config.FeatureGates[string(features.PacketSampling)] = true + } + cc := func(config *controllerconfig.ControllerConfig) { + previousControllerPacketSamplingEnableState = config.FeatureGates[string(features.PacketSampling)] + config.FeatureGates[string(features.PacketSampling)] = true + } + if err := data.mutateAntreaConfigMap(cc, ac, true, true); err != nil { + t.Fatalf("Failed to enable PacketSampling flag: %v", err) + } + defer func() { + ac := func(config *agentconfig.AgentConfig) { + config.FeatureGates[string(features.PacketSampling)] = previousAgentPacketSamplingEnableState + } + cc := func(config *controllerconfig.ControllerConfig) { + config.FeatureGates[string(features.PacketSampling)] = previousControllerPacketSamplingEnableState + } + if err := data.mutateAntreaConfigMap(cc, ac, true, true); err != nil { + t.Errorf("Failed to disable PacketSampling flag: %v", err) + } + }() + + // setup sftp server for test. + sftpServiceYAML := "sftp-deployment.yml" + secretUserName := "foo" + secretPassword := "pass" + + applySFTPYamlCommand := fmt.Sprintf("kubectl apply -f %s -n %s", sftpServiceYAML, data.testNamespace) + code, stdout, stderr, err := data.RunCommandOnNode(controlPlaneNodeName(), applySFTPYamlCommand) + require.NoError(t, err) + defer func() { + deleteSFTPYamlCommand := fmt.Sprintf("kubectl delete -f %s -n %s", sftpServiceYAML, data.testNamespace) + data.RunCommandOnNode(controlPlaneNodeName(), deleteSFTPYamlCommand) + }() + t.Logf("Stdout of the command '%s': %s", applySFTPYamlCommand, stdout) + if code != 0 { + t.Errorf("Error when applying %s: %v", sftpServiceYAML, stderr) + } + failOnError(data.waitForDeploymentReady(t, data.testNamespace, "sftp", defaultTimeout), t) + + sec := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: psSecretName, + }, + Data: map[string][]byte{ + "username": []byte(secretUserName), + "password": []byte(secretPassword), + }, + } + _, err = data.clientset.CoreV1().Secrets(psNamespace).Create(context.TODO(), sec, metav1.CreateOptions{}) + require.NoError(t, err) + defer data.clientset.CoreV1().Secrets(psNamespace).Delete(context.TODO(), psSecretName, metav1.DeleteOptions{}) + + t.Run("testPacketSamplingBasic", func(t *testing.T) { + testPacketSamplingBasic(t, data) + }) + t.Run("testPacketSampling", func(t *testing.T) { + testPacketSampling(t, data) + }) +} + +func testPacketSampling(t *testing.T, data *TestData) { + nodeIdx := 0 + if len(clusterInfo.windowsNodes) != 0 { + nodeIdx = clusterInfo.windowsNodes[0] + } + node1 := nodeName(nodeIdx) + + err := data.createServerPodWithLabels(tcpServerPodName, data.testNamespace, serverPodPort, nil) + require.NoError(t, err) + err = data.createToolboxPodOnNode(psToolboxPodName, data.testNamespace, node1, false) + require.NoError(t, err) + + svc, cleanup := data.createAgnhostServiceAndBackendPods(t, dstServiceName, data.testNamespace, node1, v1.ServiceTypeClusterIP) + defer cleanup() + t.Logf("%s Service is ready", dstServiceName) + dstServiceIP = svc.Spec.ClusterIP + + podIPs := waitForPodIPs(t, data, []PodInfo{ + {tcpServerPodName, getOSString(), "", data.testNamespace}, + {psToolboxPodName, getOSString(), "", data.testNamespace}, + }) + + // Give a little time for Windows containerd Nodes to setup OVS. + // Containerd configures port asynchronously, which could cause execution time of installing flow longer than docker. + time.Sleep(time.Second * 1) + + testcases := []psTestCase{ + { + name: "to-ipv4-ip", + ipVersion: 4, + srcPod: psToolboxPodName, + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{ + Name: randName(fmt.Sprintf("%s-%s-to-%s-%s-", data.testNamespace, psToolboxPodName, data.testNamespace, tcpServerPodName)), + }, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: data.testNamespace, + Pod: psToolboxPodName, + }, + Destination: crdv1alpha1.Destination{ + IP: podIPs[tcpServerPodName].IPv4.String(), + }, + Type: crdv1alpha1.FirstNSampling, + FirstNSamplingConfig: &crdv1alpha1.FirstNSamplingConfig{ + Number: 5, + }, + FileServer: crdv1alpha1.BundleFileServer{ + URL: fmt.Sprintf("%s:30010/upload", controlPlaneNodeIPv4()), + }, + Authentication: crdv1alpha1.BundleServerAuthConfiguration{ + AuthType: "BasicAuthentication", + AuthSecret: &v1.SecretReference{ + Name: psSecretName, + Namespace: psNamespace, + }, + }, + Packet: crdv1alpha1.Packet{ + IPHeader: crdv1alpha1.IPHeader{ + Protocol: protocolTCP, + }, + TransportHeader: crdv1alpha1.TransportHeader{ + TCP: &crdv1alpha1.TCPHeader{ + DstPort: serverPodPort, + }, + }, + }, + }, + }, + + expectedPhase: crdv1alpha1.PacketSamplingSucceeded, + expectedNum: 5, + }, + { + name: "to-svc", + ipVersion: 4, + srcPod: psToolboxPodName, + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{ + Name: randName(fmt.Sprintf("%s-%s-to-%s-%s-", data.testNamespace, psToolboxPodName, data.testNamespace, tcpServerPodName)), + }, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: data.testNamespace, + Pod: psToolboxPodName, + }, + Destination: crdv1alpha1.Destination{ + Service: dstServiceName, + Namespace: data.testNamespace, + }, + Type: crdv1alpha1.FirstNSampling, + FirstNSamplingConfig: &crdv1alpha1.FirstNSamplingConfig{ + Number: 5, + }, + FileServer: crdv1alpha1.BundleFileServer{ + URL: fmt.Sprintf("%s:30010/upload", controlPlaneNodeIPv4()), + }, + Authentication: crdv1alpha1.BundleServerAuthConfiguration{ + AuthType: "BasicAuthentication", + AuthSecret: &v1.SecretReference{ + Name: psSecretName, + Namespace: psNamespace, + }, + }, + Packet: crdv1alpha1.Packet{ + IPHeader: crdv1alpha1.IPHeader{ + Protocol: protocolTCP, + }, + TransportHeader: crdv1alpha1.TransportHeader{ + TCP: &crdv1alpha1.TCPHeader{ + DstPort: serverPodPort, + }, + }, + }, + }, + }, + + expectedPhase: crdv1alpha1.PacketSamplingSucceeded, + expectedNum: 5, + }, + } + t.Run("testPacketSampling", func(t *testing.T) { + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + runPacketSamplingTest(t, data, tc) + }) + } + }) + +} + +// testPacketSamplingTCP verifies if PacketSampling can capture tcp packets. this function only contains basic +// cases with pod-to-pod. +func testPacketSamplingBasic(t *testing.T, data *TestData) { + nodeIdx := 0 + if len(clusterInfo.windowsNodes) != 0 { + nodeIdx = clusterInfo.windowsNodes[0] + } + node1 := nodeName(nodeIdx) + + node1Pods, _, _ := createTestAgnhostPods(t, data, 3, data.testNamespace, node1) + err := data.createUDPServerPod(udpServerPodName, data.testNamespace, serverPodPort, node1) + defer data.DeletePodAndWait(defaultTimeout, udpServerPodName, data.testNamespace) + require.NoError(t, err) + // test tcp server pod + err = data.createServerPodWithLabels(tcpServerPodName, data.testNamespace, serverPodPort, nil) + defer data.DeletePodAndWait(defaultTimeout, tcpServerPodName, data.testNamespace) + require.NoError(t, err) + err = data.createToolboxPodOnNode(psToolboxPodName, data.testNamespace, node1, false) + defer data.DeletePodAndWait(defaultTimeout, psToolboxPodName, data.testNamespace) + require.NoError(t, err) + + // Give a little time for Windows containerd Nodes to setup OVS. + // Containerd configures port asynchronously, which could cause execution time of installing flow longer than docker. + time.Sleep(time.Second * 1) + + testcases := []psTestCase{ + { + name: "ipv4-tcp", + ipVersion: 4, + srcPod: psToolboxPodName, + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{ + Name: randName(fmt.Sprintf("%s-%s-to-%s-%s-", data.testNamespace, psToolboxPodName, data.testNamespace, tcpServerPodName)), + }, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: data.testNamespace, + Pod: psToolboxPodName, + }, + Destination: crdv1alpha1.Destination{ + Namespace: data.testNamespace, + Pod: tcpServerPodName, + }, + Type: crdv1alpha1.FirstNSampling, + FirstNSamplingConfig: &crdv1alpha1.FirstNSamplingConfig{ + Number: 5, + }, + FileServer: crdv1alpha1.BundleFileServer{ + URL: fmt.Sprintf("%s:30010/upload", controlPlaneNodeIPv4()), + }, + Authentication: crdv1alpha1.BundleServerAuthConfiguration{ + AuthType: "BasicAuthentication", + AuthSecret: &v1.SecretReference{ + Name: psSecretName, + Namespace: psNamespace, + }, + }, + Packet: crdv1alpha1.Packet{ + IPHeader: crdv1alpha1.IPHeader{ + Protocol: protocolTCP, + }, + TransportHeader: crdv1alpha1.TransportHeader{ + TCP: &crdv1alpha1.TCPHeader{ + DstPort: serverPodPort, + }, + }, + }, + }, + }, + expectedPhase: crdv1alpha1.PacketSamplingSucceeded, + expectedNum: 5, + }, + { + name: "ipv4-udp", + ipVersion: 4, + srcPod: psToolboxPodName, + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{ + Name: randName(fmt.Sprintf("%s-%s-to-%s-%s-", data.testNamespace, psToolboxPodName, data.testNamespace, udpServerPodName)), + }, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: data.testNamespace, + Pod: psToolboxPodName, + }, + Destination: crdv1alpha1.Destination{ + Namespace: data.testNamespace, + Pod: udpServerPodName, + }, + + Type: crdv1alpha1.FirstNSampling, + Timeout: 300, + FirstNSamplingConfig: &crdv1alpha1.FirstNSamplingConfig{ + Number: 5, + }, + FileServer: crdv1alpha1.BundleFileServer{ + URL: fmt.Sprintf("%s:30010/upload", controlPlaneNodeIPv4()), + }, + Authentication: crdv1alpha1.BundleServerAuthConfiguration{ + AuthType: "BasicAuthentication", + AuthSecret: &v1.SecretReference{ + Name: psSecretName, + Namespace: psNamespace, + }, + }, + Packet: crdv1alpha1.Packet{ + IPHeader: crdv1alpha1.IPHeader{ + Protocol: protocolUDP, + }, + TransportHeader: crdv1alpha1.TransportHeader{ + UDP: &crdv1alpha1.UDPHeader{ + DstPort: serverPodPort, + }, + }, + }, + }, + }, + expectedPhase: crdv1alpha1.PacketSamplingSucceeded, + expectedNum: 5, + }, + { + name: "ipv4-icmp", + ipVersion: 4, + srcPod: node1Pods[0], + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{ + Name: randName(fmt.Sprintf("%s-%s-to-%s-%s-", data.testNamespace, node1Pods[0], data.testNamespace, node1Pods[1])), + }, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: data.testNamespace, + Pod: node1Pods[0], + }, + Destination: crdv1alpha1.Destination{ + Namespace: data.testNamespace, + Pod: node1Pods[1], + }, + + Type: crdv1alpha1.FirstNSampling, + FirstNSamplingConfig: &crdv1alpha1.FirstNSamplingConfig{ + Number: 5, + }, + FileServer: crdv1alpha1.BundleFileServer{ + URL: fmt.Sprintf("%s:30010/upload", controlPlaneNodeIPv4()), + }, + Authentication: crdv1alpha1.BundleServerAuthConfiguration{ + AuthType: "BasicAuthentication", + AuthSecret: &v1.SecretReference{ + Name: psSecretName, + Namespace: psNamespace, + }, + }, + Packet: crdv1alpha1.Packet{ + IPHeader: crdv1alpha1.IPHeader{ + Protocol: protocolICMP, + }, + }, + }, + }, + expectedPhase: crdv1alpha1.PacketSamplingSucceeded, + expectedNum: 5, + }, + { + name: "ipv6-icmp", + ipVersion: 6, + srcPod: node1Pods[0], + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{ + Name: randName(fmt.Sprintf("%s-%s-to-%s-%s-ipv6", data.testNamespace, node1Pods[0], data.testNamespace, node1Pods[1])), + }, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: data.testNamespace, + Pod: node1Pods[0], + }, + Destination: crdv1alpha1.Destination{ + Namespace: data.testNamespace, + Pod: node1Pods[1], + }, + + Type: crdv1alpha1.FirstNSampling, + FirstNSamplingConfig: &crdv1alpha1.FirstNSamplingConfig{ + Number: 5, + }, + FileServer: crdv1alpha1.BundleFileServer{ + URL: fmt.Sprintf("%s:30010/upload", controlPlaneNodeIPv4()), + }, + Authentication: crdv1alpha1.BundleServerAuthConfiguration{ + AuthType: "BasicAuthentication", + AuthSecret: &v1.SecretReference{ + Name: psSecretName, + Namespace: psNamespace, + }, + }, + Packet: crdv1alpha1.Packet{ + IPv6Header: &crdv1alpha1.IPv6Header{ + NextHeader: &protocolICMPv6, + }, + }, + }, + }, + expectedPhase: crdv1alpha1.PacketSamplingSucceeded, + expectedNum: 5, + }, + { + + name: "non-exist-pod", + ipVersion: 4, + srcPod: node1Pods[0], + ps: &crdv1alpha1.PacketSampling{ + ObjectMeta: metav1.ObjectMeta{ + Name: randName(fmt.Sprintf("%s-%s-to-%s-%s-", data.testNamespace, node1Pods[0], data.testNamespace, nonExistPodName)), + }, + Spec: crdv1alpha1.PacketSamplingSpec{ + Source: crdv1alpha1.Source{ + Namespace: data.testNamespace, + Pod: node1Pods[0], + }, + Destination: crdv1alpha1.Destination{ + Namespace: data.testNamespace, + Pod: nonExistPodName, + }, + Type: crdv1alpha1.FirstNSampling, + FirstNSamplingConfig: &crdv1alpha1.FirstNSamplingConfig{ + Number: 5, + }, + FileServer: crdv1alpha1.BundleFileServer{ + URL: fmt.Sprintf("%s:30010/upload", controlPlaneNodeIPv4()), + }, + Authentication: crdv1alpha1.BundleServerAuthConfiguration{ + AuthType: "BasicAuthentication", + AuthSecret: &v1.SecretReference{ + Name: psSecretName, + Namespace: psNamespace, + }, + }, + }, + }, + expectedPhase: crdv1alpha1.PacketSamplingFailed, + expectedReason: fmt.Sprintf("Node: %s, error:failed to get the destination pod %s/%s: pods \"%s\" not found", node1, data.testNamespace, nonExistPodName, nonExistPodName), + }, + } + t.Run("testPacketSamplingBasic", func(t *testing.T) { + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + runPacketSamplingTest(t, data, tc) + }) + } + }) +} + +func getOSString() string { + if len(clusterInfo.windowsNodes) != 0 { + return "windows" + } else { + return "linux" + } +} + +func runPacketSamplingTest(t *testing.T, data *TestData, tc psTestCase) { + switch tc.ipVersion { + case 4: + skipIfNotIPv4Cluster(t) + case 6: + skipIfNotIPv6Cluster(t) + } + if tc.skipIfNeeded != nil { + tc.skipIfNeeded(t) + } + + dstPodName := tc.ps.Spec.Destination.Pod + var dstPodIPs *PodIPs + if dstPodName != nonExistPodName && dstPodName != "" { + // wait for pods to be ready first , or the ps will skip install flow + podIPs := waitForPodIPs(t, data, []PodInfo{{dstPodName, getOSString(), "", data.testNamespace}}) + dstPodIPs = podIPs[dstPodName] + } + + if _, err := data.crdClient.CrdV1alpha1().PacketSamplings().Create(context.TODO(), tc.ps, metav1.CreateOptions{}); err != nil { + t.Fatalf("Error when creating PacketSampling: %v", err) + } + defer func() { + if err := data.crdClient.CrdV1alpha1().PacketSamplings().Delete(context.TODO(), tc.ps.Name, metav1.DeleteOptions{}); err != nil { + t.Errorf("Error when deleting PacketSampling: %v", err) + } + }() + + if tc.ps.Spec.Destination.Pod != nonExistPodName { + srcPod := tc.srcPod + if dstIP := tc.ps.Spec.Destination.IP; dstIP != "" { + ip := net.ParseIP(dstIP) + if ip.To4() != nil { + dstPodIPs = &PodIPs{IPv4: &ip} + } else { + dstPodIPs = &PodIPs{IPv6: &ip} + } + } else if tc.ps.Spec.Destination.Service != "" { + ip := net.ParseIP(dstServiceIP) + if ip.To4() != nil { + dstPodIPs = &PodIPs{IPv4: &ip} + } else { + dstPodIPs = &PodIPs{IPv6: &ip} + } + } + // Give a little time for Nodes to install OVS flows. + time.Sleep(time.Second * 2) + protocol := tc.ps.Spec.Packet.IPHeader.Protocol + if tc.ps.Spec.Packet.IPv6Header != nil { + protocol = *tc.ps.Spec.Packet.IPv6Header.NextHeader + } + server := dstPodIPs.IPv4.String() + if tc.ipVersion == 6 { + server = dstPodIPs.IPv6.String() + } + // Send an ICMP echo packet from the source Pod to the destination. + if protocol == protocolICMP || protocol == protocolICMPv6 { + if err := data.RunPingCommandFromTestPod(PodInfo{srcPod, getOSString(), "", data.testNamespace}, + data.testNamespace, dstPodIPs, agnhostContainerName, 10, 0, false); err != nil { + t.Logf("Ping(%d) '%s' -> '%v' failed: ERROR (%v)", protocol, srcPod, *dstPodIPs, err) + } + } else if protocol == protocolTCP { + for i := 1; i <= 5; i++ { + if err := data.runNetcatCommandFromTestPodWithProtocol(tc.srcPod, data.testNamespace, toolboxContainerName, server, serverPodPort, "tcp"); err != nil { + t.Logf("Netcat(TCP) '%s' -> '%v' failed: ERROR (%v)", srcPod, server, err) + } + } + } else if protocol == protocolUDP { + for i := 1; i <= 5; i++ { + if err := data.runNetcatCommandFromTestPodWithProtocol(tc.srcPod, data.testNamespace, toolboxContainerName, server, serverPodPort, "udp"); err != nil { + t.Logf("Netcat(UDP) '%s' -> '%v' failed: ERROR (%v)", srcPod, server, err) + } + } + } + } + + ps, err := data.waitForPacketSampling(t, tc.ps.Name, tc.expectedPhase) + if err != nil { + t.Fatalf("Error: Get PacketSampling failed: %v", err) + } + if tc.expectedPhase == crdv1alpha1.PacketSamplingFailed { + if ps.Status.Reason != tc.expectedReason { + t.Fatalf("Error: PacketSampling Error Reason should be %v, but got %s", tc.expectedReason, ps.Status.Reason) + } + } + if ps.Status.NumCapturedPackets != tc.expectedNum { + t.Fatalf("Error: PacketSampling captured packets count should be %v, but got %v", tc.expectedNum, ps.Status.NumCapturedPackets) + } + +} + +func (data *TestData) waitForPacketSampling(t *testing.T, name string, phase crdv1alpha1.PacketSamplingPhase) (*crdv1alpha1.PacketSampling, error) { + var ps *crdv1alpha1.PacketSampling + var err error + timeout := 15 * time.Second + if err = wait.PollUntilContextTimeout(context.Background(), defaultInterval, timeout, true, func(ctx context.Context) (bool, error) { + ps, err = data.crdClient.CrdV1alpha1().PacketSamplings().Get(ctx, name, metav1.GetOptions{}) + if err != nil || ps.Status.Phase != phase { + return false, nil + } + return true, nil + }); err != nil { + if ps != nil { + t.Errorf("Latest PacketSampling status: %s %v", ps.Name, ps.Status) + } + return nil, err + } + return ps, nil +} diff --git a/test/integration/agent/openflow_test.go b/test/integration/agent/openflow_test.go index 57f48cf619b..45f7f3f8e53 100644 --- a/test/integration/agent/openflow_test.go +++ b/test/integration/agent/openflow_test.go @@ -120,7 +120,7 @@ func TestConnectivityFlows(t *testing.T) { antrearuntime.WindowsOS = runtime.GOOS } - c = ofClient.NewClient(br, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), true, false, false, true, true, false, false, false, false, false, false, false, false, groupIDAllocator, false, defaultPacketInRate) + c = ofClient.NewClient(br, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), true, false, false, true, true, false, false, false, false, false, false, false, false, groupIDAllocator, false, false, defaultPacketInRate) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge: %v", err)) defer func() { @@ -176,7 +176,7 @@ func TestAntreaFlexibleIPAMConnectivityFlows(t *testing.T) { legacyregistry.Reset() metrics.InitializeOVSMetrics() - c = ofClient.NewClient(br, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), true, false, false, true, true, false, false, false, true, false, false, false, false, groupIDAllocator, false, defaultPacketInRate) + c = ofClient.NewClient(br, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), true, false, false, true, true, false, false, false, true, false, false, false, false, groupIDAllocator, false, false, defaultPacketInRate) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge: %v", err)) defer func() { @@ -239,7 +239,7 @@ func TestReplayFlowsConnectivityFlows(t *testing.T) { legacyregistry.Reset() metrics.InitializeOVSMetrics() - c = ofClient.NewClient(br, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), true, false, false, true, true, false, false, false, false, false, false, false, false, groupIDAllocator, false, defaultPacketInRate) + c = ofClient.NewClient(br, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), true, false, false, true, true, false, false, false, false, false, false, false, false, groupIDAllocator, false, false, defaultPacketInRate) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge: %v", err)) @@ -281,7 +281,7 @@ func TestReplayFlowsNetworkPolicyFlows(t *testing.T) { legacyregistry.Reset() metrics.InitializeOVSMetrics() - c = ofClient.NewClient(br, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), true, false, false, false, false, false, false, false, false, false, false, false, false, groupIDAllocator, false, defaultPacketInRate) + c = ofClient.NewClient(br, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), true, false, false, false, false, false, false, false, false, false, false, false, false, groupIDAllocator, false, false, defaultPacketInRate) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge: %v", err)) @@ -466,7 +466,7 @@ func TestNetworkPolicyFlows(t *testing.T) { legacyregistry.Reset() metrics.InitializeOVSMetrics() - c = ofClient.NewClient(br, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), true, false, false, false, false, false, false, false, false, false, false, false, false, groupIDAllocator, false, defaultPacketInRate) + c = ofClient.NewClient(br, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), true, false, false, false, false, false, false, false, false, false, false, false, false, groupIDAllocator, false, false, defaultPacketInRate) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge %s", br)) @@ -580,7 +580,7 @@ func TestIPv6ConnectivityFlows(t *testing.T) { legacyregistry.Reset() metrics.InitializeOVSMetrics() - c = ofClient.NewClient(br, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), true, false, false, true, true, false, false, false, false, false, false, false, false, groupIDAllocator, false, defaultPacketInRate) + c = ofClient.NewClient(br, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), true, false, false, true, true, false, false, false, false, false, false, false, false, groupIDAllocator, false, false, defaultPacketInRate) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge: %v", err)) @@ -621,7 +621,7 @@ func TestProxyServiceFlowsAntreaPolicyDisabled(t *testing.T) { legacyregistry.Reset() metrics.InitializeOVSMetrics() - c = ofClient.NewClient(br, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), true, false, false, false, false, false, false, false, false, false, false, false, false, groupIDAllocator, false, defaultPacketInRate) + c = ofClient.NewClient(br, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), true, false, false, false, false, false, false, false, false, false, false, false, false, groupIDAllocator, false, false, defaultPacketInRate) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge %s", br)) @@ -711,7 +711,7 @@ func TestProxyServiceFlowsAntreaPoilcyEnabled(t *testing.T) { legacyregistry.Reset() metrics.InitializeOVSMetrics() - c = ofClient.NewClient(br, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), true, true, false, false, false, false, false, false, false, false, false, false, false, groupIDAllocator, false, defaultPacketInRate) + c = ofClient.NewClient(br, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), true, true, false, false, false, false, false, false, false, false, false, false, false, groupIDAllocator, false, false, defaultPacketInRate) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge %s", br)) @@ -1793,7 +1793,7 @@ func testEgressMarkFlows(t *testing.T, trafficShaping bool) { legacyregistry.Reset() metrics.InitializeOVSMetrics() - c = ofClient.NewClient(br, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), false, false, false, true, trafficShaping, false, false, false, false, false, false, false, false, groupIDAllocator, false, defaultPacketInRate) + c = ofClient.NewClient(br, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), false, false, false, true, trafficShaping, false, false, false, false, false, false, false, false, groupIDAllocator, false, false, defaultPacketInRate) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge %s", br)) @@ -1850,7 +1850,7 @@ func TestTrafficControlFlows(t *testing.T) { legacyregistry.Reset() metrics.InitializeOVSMetrics() - c = ofClient.NewClient(br, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), false, false, false, false, false, false, false, false, false, false, true, false, false, groupIDAllocator, false, defaultPacketInRate) + c = ofClient.NewClient(br, bridgeMgmtAddr, nodeiptest.NewFakeNodeIPChecker(), false, false, false, false, false, false, false, false, false, false, true, false, false, groupIDAllocator, false, false, defaultPacketInRate) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge %s", br))