From 06b9559f52aa0fb8078c988c6b0e0079dc3a9e7a Mon Sep 17 00:00:00 2001 From: Lukasz Dziedziak Date: Tue, 1 Oct 2024 06:29:35 -0500 Subject: [PATCH 1/2] revert(kuma-cp): do not use additional addresses (#11601) Signed-off-by: Lukasz Dziedziak --- .../envoy/listeners/listener_configurers.go | 6 - .../v3/additional_address_configuer.go | 42 ------- .../v3/additional_address_configurer_test.go | 84 -------------- pkg/xds/generator/outbound_proxy_generator.go | 19 +-- .../outbound-proxy/08.envoy.golden.yaml | 109 ++++++++++++++++-- .../outbound-proxy/10.envoy.golden.yaml | 61 +++++++--- 6 files changed, 159 insertions(+), 162 deletions(-) delete mode 100644 pkg/xds/envoy/listeners/v3/additional_address_configuer.go delete mode 100644 pkg/xds/envoy/listeners/v3/additional_address_configurer_test.go diff --git a/pkg/xds/envoy/listeners/listener_configurers.go b/pkg/xds/envoy/listeners/listener_configurers.go index 403245c9ba17..ca645da8b49a 100644 --- a/pkg/xds/envoy/listeners/listener_configurers.go +++ b/pkg/xds/envoy/listeners/listener_configurers.go @@ -97,9 +97,3 @@ func TagsMetadata(tags map[string]string) ListenerBuilderOpt { Tags: tags, }) } - -func AdditionalAddresses(addresses []mesh_proto.OutboundInterface) ListenerBuilderOpt { - return AddListenerConfigurer(&v3.AdditionalAddressConfigurer{ - Addresses: addresses, - }) -} diff --git a/pkg/xds/envoy/listeners/v3/additional_address_configuer.go b/pkg/xds/envoy/listeners/v3/additional_address_configuer.go deleted file mode 100644 index 5568d1b29bca..000000000000 --- a/pkg/xds/envoy/listeners/v3/additional_address_configuer.go +++ /dev/null @@ -1,42 +0,0 @@ -package v3 - -import ( - envoy_core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - listenerv3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" - - mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" -) - -type AdditionalAddressConfigurer struct { - Addresses []mesh_proto.OutboundInterface -} - -func (c *AdditionalAddressConfigurer) Configure(l *listenerv3.Listener) error { - if len(c.Addresses) < 1 || l.Address == nil { - return nil - } - - var addresses []*listenerv3.AdditionalAddress - for _, addr := range c.Addresses { - address := makeSocketAddress(addr.DataplaneIP, addr.DataplanePort, l.Address.GetSocketAddress().GetProtocol()) - addresses = append(addresses, address) - } - l.AdditionalAddresses = addresses - return nil -} - -func makeSocketAddress(addr string, port uint32, protocol envoy_core.SocketAddress_Protocol) *listenerv3.AdditionalAddress { - return &listenerv3.AdditionalAddress{ - Address: &envoy_core.Address{ - Address: &envoy_core.Address_SocketAddress{ - SocketAddress: &envoy_core.SocketAddress{ - Protocol: protocol, - Address: addr, - PortSpecifier: &envoy_core.SocketAddress_PortValue{ - PortValue: port, - }, - }, - }, - }, - } -} diff --git a/pkg/xds/envoy/listeners/v3/additional_address_configurer_test.go b/pkg/xds/envoy/listeners/v3/additional_address_configurer_test.go deleted file mode 100644 index 3c0a89c3c61a..000000000000 --- a/pkg/xds/envoy/listeners/v3/additional_address_configurer_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package v3_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" - "github.com/kumahq/kuma/pkg/core/xds" - plugins_xds "github.com/kumahq/kuma/pkg/plugins/policies/core/xds" - util_proto "github.com/kumahq/kuma/pkg/util/proto" - envoy_common "github.com/kumahq/kuma/pkg/xds/envoy" - . "github.com/kumahq/kuma/pkg/xds/envoy/listeners" -) - -var _ = Describe("AdditionalAddressConfigurer", func() { - type testCase struct { - listenerName string - listenerAddress string - listenerPort uint32 - - additionalAddress string - additionalAddressPort uint32 - - serviceName string - expected string - } - - DescribeTable("should generate proper Envoy config", - func(given testCase) { - tcpSplit := plugins_xds.NewSplitBuilder(). - WithClusterName(given.serviceName). - WithWeight(uint32(100)). - Build() - oface := []mesh_proto.OutboundInterface{ - { - DataplaneIP: given.additionalAddress, - DataplanePort: given.additionalAddressPort, - }, - } - // when - listener, err := NewOutboundListenerBuilder(envoy_common.APIV3, given.listenerAddress, given.listenerPort, xds.SocketAddressProtocolTCP). - Configure(FilterChain(NewFilterChainBuilder(envoy_common.APIV3, envoy_common.AnonymousResource).Configure(TCPProxy(given.serviceName, tcpSplit)))). - Configure(AdditionalAddresses(oface)). - Build() - - // then - Expect(err).ToNot(HaveOccurred()) - - // when - actual, err := util_proto.ToYAML(listener) - Expect(err).ToNot(HaveOccurred()) - // and - Expect(actual).To(MatchYAML(given.expected)) - }, - Entry("generate listener with additional addresses", testCase{ - listenerName: "outbound:192.168.24.58:8080", - listenerAddress: "192.168.24.58", - listenerPort: 8080, - additionalAddress: "240.0.0.1", - additionalAddressPort: 80, - serviceName: "httpbin_app-ns_svc_8080", - expected: ` - name: outbound:192.168.24.58:8080 - trafficDirection: OUTBOUND - address: - socketAddress: - address: 192.168.24.58 - portValue: 8080 - additionalAddresses: - - address: - socketAddress: - address: 240.0.0.1 - portValue: 80 - filterChains: - - filters: - - name: envoy.filters.network.tcp_proxy - typedConfig: - '@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy - cluster: httpbin_app-ns_svc_8080 - statPrefix: httpbin_app-ns_svc_8080 -`, - }), - ) -}) diff --git a/pkg/xds/generator/outbound_proxy_generator.go b/pkg/xds/generator/outbound_proxy_generator.go index 4f368733c52f..06e1015fff41 100644 --- a/pkg/xds/generator/outbound_proxy_generator.go +++ b/pkg/xds/generator/outbound_proxy_generator.go @@ -5,14 +5,16 @@ import ( "fmt" "github.com/pkg/errors" - "golang.org/x/exp/maps" mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" "github.com/kumahq/kuma/pkg/core" core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" "github.com/kumahq/kuma/pkg/core/user" model "github.com/kumahq/kuma/pkg/core/xds" +<<<<<<< HEAD util_maps "github.com/kumahq/kuma/pkg/util/maps" +======= +>>>>>>> 205997054 (revert(kuma-cp): do not use additional addresses (#11601)) util_protocol "github.com/kumahq/kuma/pkg/util/protocol" xds_context "github.com/kumahq/kuma/pkg/xds/context" envoy_common "github.com/kumahq/kuma/pkg/xds/envoy" @@ -49,11 +51,10 @@ func (g OutboundProxyGenerator) Generate(ctx context.Context, _ *model.ResourceS // If we have same split in many HTTP matches we can use the same cluster with different weight clusterCache := map[string]string{} - outboundsMultipleIPs := buildOutboundsWithMultipleIPs(proxy.Dataplane, outbounds, xdsCtx.Mesh.VIPDomains) - for _, outbound := range outboundsMultipleIPs { + for _, outbound := range outbounds { // Determine the list of destination subsets // For one outbound listener it may contain many subsets (ex. TrafficRoute to many destinations) - routes := g.determineRoutes(proxy, outbound.Addresses[0], clusterCache, xdsCtx.Mesh.Resource.ZoneEgressEnabled()) + routes := g.determineRoutes(proxy, proxy.Dataplane.Spec.Networking.ToOutboundInterface(outbound), clusterCache, xdsCtx.Mesh.Resource.ZoneEgressEnabled()) clusters := routes.Clusters() protocol := inferProtocol(xdsCtx.Mesh, clusters) @@ -89,8 +90,8 @@ func (g OutboundProxyGenerator) Generate(ctx context.Context, _ *model.ResourceS return resources, nil } -func (OutboundProxyGenerator) generateLDS(ctx xds_context.Context, proxy *model.Proxy, routes envoy_common.Routes, outbound OutboundWithMultipleIPs, protocol core_mesh.Protocol) (envoy_common.NamedResource, error) { - oface := outbound.Addresses[0] +func (OutboundProxyGenerator) generateLDS(ctx xds_context.Context, proxy *model.Proxy, routes envoy_common.Routes, outbound *mesh_proto.Dataplane_Networking_Outbound, protocol core_mesh.Protocol) (envoy_common.NamedResource, error) { + oface := proxy.Dataplane.Spec.Networking.ToOutboundInterface(outbound) rateLimits := []*core_mesh.RateLimitResource{} if rateLimit, exists := proxy.Policies.RateLimitsOutbound[oface]; exists { rateLimits = append(rateLimits, rateLimit) @@ -172,8 +173,7 @@ func (OutboundProxyGenerator) generateLDS(ctx xds_context.Context, proxy *model. listener, err := envoy_listeners.NewOutboundListenerBuilder(proxy.APIVersion, oface.DataplaneIP, oface.DataplanePort, model.SocketAddressProtocolTCP). Configure(envoy_listeners.FilterChain(filterChainBuilder)). Configure(envoy_listeners.TransparentProxying(proxy.Dataplane.Spec.Networking.GetTransparentProxying())). - Configure(envoy_listeners.TagsMetadata(envoy_tags.Tags(outbound.Tags).WithoutTags(mesh_proto.MeshTag))). - Configure(envoy_listeners.AdditionalAddresses(outbound.AdditionalAddresses())). + Configure(envoy_listeners.TagsMetadata(envoy_tags.Tags(outbound.GetTags()).WithoutTags(mesh_proto.MeshTag))). Build() if err != nil { return nil, errors.Wrapf(err, "could not generate listener %s for service %s", outboundListenerName, serviceName) @@ -439,6 +439,7 @@ func (OutboundProxyGenerator) determineRoutes( return routes } +<<<<<<< HEAD type OutboundWithMultipleIPs struct { Tags map[string]string @@ -482,3 +483,5 @@ func buildOutboundsWithMultipleIPs(dataplane *core_mesh.DataplaneResource, outbo } return result } +======= +>>>>>>> 205997054 (revert(kuma-cp): do not use additional addresses (#11601)) diff --git a/pkg/xds/generator/testdata/outbound-proxy/08.envoy.golden.yaml b/pkg/xds/generator/testdata/outbound-proxy/08.envoy.golden.yaml index 0ef58ae9c3b9..472ef9d5f550 100644 --- a/pkg/xds/generator/testdata/outbound-proxy/08.envoy.golden.yaml +++ b/pkg/xds/generator/testdata/outbound-proxy/08.envoy.golden.yaml @@ -57,18 +57,109 @@ resources: idleTimeout: 0s explicitHttpConfig: http2ProtocolOptions: {} +- name: outbound:240.0.0.0:80 + resource: + '@type': type.googleapis.com/envoy.config.listener.v3.Listener + address: + socketAddress: + address: 240.0.0.0 + portValue: 80 + bindToPort: false + filterChains: + - filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + commonHttpProtocolOptions: + idleTimeout: 0s + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + routeConfig: + name: outbound:es2 + requestHeadersToAdd: + - header: + key: x-kuma-tags + value: '&kuma.io/service=web&' + validateClusters: false + virtualHosts: + - domains: + - '*' + name: es2 + routes: + - match: + prefix: / + route: + autoHostRewrite: true + timeout: 0s + weightedClusters: + clusters: + - name: es2-b5516780eaf1ed13 + weight: 10 + - name: es2-d79214c8b3a5805b + weight: 90 + statPrefix: es2 + streamIdleTimeout: 0s + metadata: + filterMetadata: + io.kuma.tags: + kuma.io/service: es2 + name: outbound:240.0.0.0:80 + trafficDirection: OUTBOUND +- name: outbound:240.0.0.1:80 + resource: + '@type': type.googleapis.com/envoy.config.listener.v3.Listener + address: + socketAddress: + address: 240.0.0.1 + portValue: 80 + bindToPort: false + filterChains: + - filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + commonHttpProtocolOptions: + idleTimeout: 0s + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + routeConfig: + name: outbound:es2 + requestHeadersToAdd: + - header: + key: x-kuma-tags + value: '&kuma.io/service=web&' + validateClusters: false + virtualHosts: + - domains: + - '*' + name: es2 + routes: + - match: + prefix: / + route: + autoHostRewrite: true + timeout: 0s + weightedClusters: + clusters: + - name: es2-b5516780eaf1ed13 + weight: 10 + - name: es2-d79214c8b3a5805b + weight: 90 + statPrefix: es2 + streamIdleTimeout: 0s + metadata: + filterMetadata: + io.kuma.tags: + kuma.io/service: es2 + name: outbound:240.0.0.1:80 + trafficDirection: OUTBOUND - name: outbound:240.0.0.2:80 resource: '@type': type.googleapis.com/envoy.config.listener.v3.Listener - additionalAddresses: - - address: - socketAddress: - address: 240.0.0.1 - portValue: 80 - - address: - socketAddress: - address: 240.0.0.0 - portValue: 80 address: socketAddress: address: 240.0.0.2 diff --git a/pkg/xds/generator/testdata/outbound-proxy/10.envoy.golden.yaml b/pkg/xds/generator/testdata/outbound-proxy/10.envoy.golden.yaml index 014e88687512..0584248c4105 100644 --- a/pkg/xds/generator/testdata/outbound-proxy/10.envoy.golden.yaml +++ b/pkg/xds/generator/testdata/outbound-proxy/10.envoy.golden.yaml @@ -67,19 +67,6 @@ resources: - name: outbound:127.0.0.1:18080 resource: '@type': type.googleapis.com/envoy.config.listener.v3.Listener - additionalAddresses: - - address: - socketAddress: - address: 240.0.0.3 - portValue: 80 - - address: - socketAddress: - address: 240.0.0.4 - portValue: 80 - - address: - socketAddress: - address: 240.0.0.4 - portValue: 8080 address: socketAddress: address: 127.0.0.1 @@ -99,3 +86,51 @@ resources: kuma.io/service: backend name: outbound:127.0.0.1:18080 trafficDirection: OUTBOUND +- name: outbound:240.0.0.3:80 + resource: + '@type': type.googleapis.com/envoy.config.listener.v3.Listener + address: + socketAddress: + address: 240.0.0.3 + portValue: 80 + bindToPort: false + filterChains: + - {} + metadata: + filterMetadata: + io.kuma.tags: + kuma.io/service: backend + name: outbound:240.0.0.3:80 + trafficDirection: OUTBOUND +- name: outbound:240.0.0.4:80 + resource: + '@type': type.googleapis.com/envoy.config.listener.v3.Listener + address: + socketAddress: + address: 240.0.0.4 + portValue: 80 + bindToPort: false + filterChains: + - {} + metadata: + filterMetadata: + io.kuma.tags: + kuma.io/service: backend + name: outbound:240.0.0.4:80 + trafficDirection: OUTBOUND +- name: outbound:240.0.0.4:8080 + resource: + '@type': type.googleapis.com/envoy.config.listener.v3.Listener + address: + socketAddress: + address: 240.0.0.4 + portValue: 8080 + bindToPort: false + filterChains: + - {} + metadata: + filterMetadata: + io.kuma.tags: + kuma.io/service: backend + name: outbound:240.0.0.4:8080 + trafficDirection: OUTBOUND From 5dfd2600ce76720e7d0dfb1320bbb78e29daf9db Mon Sep 17 00:00:00 2001 From: Lukasz Dziedziak Date: Tue, 1 Oct 2024 06:41:28 -0500 Subject: [PATCH 2/2] resolve conflict Signed-off-by: Lukasz Dziedziak --- pkg/xds/generator/outbound_proxy_generator.go | 50 ------------------- 1 file changed, 50 deletions(-) diff --git a/pkg/xds/generator/outbound_proxy_generator.go b/pkg/xds/generator/outbound_proxy_generator.go index 06e1015fff41..d6412b783a71 100644 --- a/pkg/xds/generator/outbound_proxy_generator.go +++ b/pkg/xds/generator/outbound_proxy_generator.go @@ -11,10 +11,6 @@ import ( core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" "github.com/kumahq/kuma/pkg/core/user" model "github.com/kumahq/kuma/pkg/core/xds" -<<<<<<< HEAD - util_maps "github.com/kumahq/kuma/pkg/util/maps" -======= ->>>>>>> 205997054 (revert(kuma-cp): do not use additional addresses (#11601)) util_protocol "github.com/kumahq/kuma/pkg/util/protocol" xds_context "github.com/kumahq/kuma/pkg/xds/context" envoy_common "github.com/kumahq/kuma/pkg/xds/envoy" @@ -439,49 +435,3 @@ func (OutboundProxyGenerator) determineRoutes( return routes } -<<<<<<< HEAD - -type OutboundWithMultipleIPs struct { - Tags map[string]string - Addresses []mesh_proto.OutboundInterface -} - -func (o OutboundWithMultipleIPs) AdditionalAddresses() []mesh_proto.OutboundInterface { - if len(o.Addresses) > 1 { - return o.Addresses[1:] - } - return nil -} - -func buildOutboundsWithMultipleIPs(dataplane *core_mesh.DataplaneResource, outbounds []*mesh_proto.Dataplane_Networking_Outbound, meshVIPDomains []model.VIPDomains) []OutboundWithMultipleIPs { - kumaVIPs := map[string]bool{} - for _, vipDomain := range meshVIPDomains { - kumaVIPs[vipDomain.Address] = true - } - - tagsToOutbounds := map[string]OutboundWithMultipleIPs{} - for _, outbound := range outbounds { - tags := maps.Clone(outbound.GetTags()) - tags[mesh_proto.ServiceTag] = outbound.GetService() - tagsStr := mesh_proto.SingleValueTagSet(tags).String() - owmi := tagsToOutbounds[tagsStr] - owmi.Tags = tags - address := dataplane.Spec.Networking.ToOutboundInterface(outbound) - // add Kuma VIPs down the list, so if there is a non Kuma VIP (i.e. Kube Cluster IP), it goes as primary address. - if kumaVIPs[address.DataplaneIP] { - owmi.Addresses = append(owmi.Addresses, address) - } else { - owmi.Addresses = append([]mesh_proto.OutboundInterface{address}, owmi.Addresses...) - } - tagsToOutbounds[tagsStr] = owmi - } - - // return sorted outbounds for a stable XDS config - var result []OutboundWithMultipleIPs - for _, key := range util_maps.SortedKeys(tagsToOutbounds) { - result = append(result, tagsToOutbounds[key]) - } - return result -} -======= ->>>>>>> 205997054 (revert(kuma-cp): do not use additional addresses (#11601))