resolve proxy to inject using http package

NOTICE.txt

@@ -18275,6 +18275,43 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+--------------------------------------------------------------------------------
+Dependency : golang.org/x/net
+Version: v0.34.0
+Licence type (autodetected): BSD-3-Clause
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]/LICENSE:
+
+Copyright 2009 The Go Authors.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google LLC nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/sync
 Version: v0.10.0
@@ -101843,43 +101880,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
---------------------------------------------------------------------------------
-Dependency : golang.org/x/net
-Version: v0.34.0
-Licence type (autodetected): BSD-3-Clause
---------------------------------------------------------------------------------
-
-Contents of probable licence file $GOMODCACHE/golang.org/x/[email protected]/LICENSE:
-
-Copyright 2009 The Go Authors.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google LLC nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
 --------------------------------------------------------------------------------
 Dependency : golang.org/x/oauth2
 Version: v0.24.0

changelog/fragments/1738329703-environment-proxy-injection-fix.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: environment-proxy-injection-fix
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description: When injecting environment proxy into Elastic Defend config the Agent should use http library to resolve the proxy settings to ensure consistency across all components.
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/6675
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/6209

go.mod

@@ -79,6 +79,7 @@ require (
 	go.uber.org/zap v1.27.0
 	golang.org/x/crypto v0.32.0
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
+	golang.org/x/net v0.34.0
 	golang.org/x/sync v0.10.0
 	golang.org/x/sys v0.29.0
 	golang.org/x/term v0.28.0
@@ -607,7 +608,6 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/mod v0.21.0 // indirect
-	golang.org/x/net v0.34.0 // indirect
 	golang.org/x/oauth2 v0.24.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	gonum.org/v1/gonum v0.15.1 // indirect

internal/pkg/agent/application/inject_proxy_component_modifier.go

@@ -5,8 +5,9 @@
 package application
 
 import (
-	"os"
-	"strings"
+	"net/http"
+
+	"golang.org/x/net/http/httpproxy"
 
 	"github.com/elastic/elastic-agent-client/v7/pkg/client"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/coordinator"
@@ -59,9 +60,8 @@ func InjectProxyEndpointModifier() coordinator.ComponentsModifier {
 
 // injectProxyURL will inject the a proxy_url into the passed map if there is no existing key and there is an appropriate proxy through defined as an env var.
 //
-// The 1st item of the passed hosts list is checked to see if it starts with https or http and the corresponding proxy var is used.
-// Nothing is injected if the *_PROXY env var is empty, the map contains proxy_url: "", or the map has proxy_disable: true.
-// If no hosts are passed, then the HTTPS_PROXY value is used over the HTTP_PROXY value if it's defined.
+// Go http client is used to determine the proxy URL, to ensure consistent behavior across all components.
+// Traffic through proxy is preferred if the proxy is defined for any of the hosts.
 func injectProxyURL(m map[string]interface{}, hosts []string) {
 	if m == nil {
 		return
@@ -79,28 +79,20 @@ func injectProxyURL(m map[string]interface{}, hosts []string) {
 		}
 	}
 
-	var proxyURL string
-	matched := false
-	// If hosts are specified, check the 1st to see if HTTPS or HTTP is used to determine proxy
-	if len(hosts) > 0 {
-		if strings.HasPrefix(hosts[0], "https://") {
-			matched = true
-			proxyURL = os.Getenv("HTTPS_PROXY")
-		} else if strings.HasPrefix(hosts[0], "http://") {
-			matched = true
-			proxyURL = os.Getenv("HTTP_PROXY")
+	// Check if a proxy is defined for the hosts
+	for _, host := range hosts {
+		request, err := http.NewRequest("GET", host, nil)
+		if err != nil {
+			continue
 		}
-	}
-	// if no hosts are specified, or it could not match a host prefix prefer HTTPS_PROXY over HTTP_PROXY
-	if proxyURL == "" && !matched {
-		proxyURL = os.Getenv("HTTPS_PROXY")
-		if proxyURL == "" {
-			proxyURL = os.Getenv("HTTP_PROXY")
+		// not using http.ProxyFromEnvironment() to be able to change the environment in unit tests
+		proxyURL, err := httpproxy.FromEnvironment().ProxyFunc()(request.URL)
+		if err != nil {
+			continue
+		}
+		if proxyURL != nil && proxyURL.String() != "" {
+			m["proxy_url"] = proxyURL.String()
+			return
 		}
 	}
-	// No proxy defined
-	if proxyURL == "" {
-		return
-	}
-	m["proxy_url"] = proxyURL
 }

internal/pkg/agent/application/inject_proxy_component_modifier_test.go

@@ -43,7 +43,7 @@ func TestInjectProxyEndpointModifier(t *testing.T) {
 								Type: elasticsearch,
 								Source: func() *structpb.Struct {
 									var source structpb.Struct
-									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["https://localhost:9200"]}`))
+									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["https://example.output:9200"]}`))
 									require.NoError(t, err)
 									return &source
 								}(),
@@ -64,7 +64,7 @@ func TestInjectProxyEndpointModifier(t *testing.T) {
 								Type: elasticsearch,
 								Source: func() *structpb.Struct {
 									var source structpb.Struct
-									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["https://localhost:9200"]}`))
+									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["https://example.output:9200"]}`))
 									require.NoError(t, err)
 									return &source
 								}(),
@@ -88,7 +88,7 @@ func TestInjectProxyEndpointModifier(t *testing.T) {
 								Type: elasticsearch,
 								Source: func() *structpb.Struct {
 									var source structpb.Struct
-									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["https://localhost:9200"],"proxy_url":"https://proxy:8080"}`))
+									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["https://example.output:9200"],"proxy_url":"https://proxy:8080"}`))
 									require.NoError(t, err)
 									return &source
 								}(),
@@ -109,7 +109,7 @@ func TestInjectProxyEndpointModifier(t *testing.T) {
 								Type: elasticsearch,
 								Source: func() *structpb.Struct {
 									var source structpb.Struct
-									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["https://localhost:9200"],"proxy_url":"https://proxy:8080"}`))
+									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["https://example.output:9200"],"proxy_url":"https://proxy:8080"}`))
 									require.NoError(t, err)
 									return &source
 								}(),
@@ -133,7 +133,7 @@ func TestInjectProxyEndpointModifier(t *testing.T) {
 								Type: elasticsearch,
 								Source: func() *structpb.Struct {
 									var source structpb.Struct
-									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["https://localhost:9200"],"proxy_url":""}`))
+									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["https://example.output:9200"],"proxy_url":""}`))
 									require.NoError(t, err)
 									return &source
 								}(),
@@ -154,7 +154,7 @@ func TestInjectProxyEndpointModifier(t *testing.T) {
 								Type: elasticsearch,
 								Source: func() *structpb.Struct {
 									var source structpb.Struct
-									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["https://localhost:9200"],"proxy_url":""}`))
+									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["https://example.output:9200"],"proxy_url":""}`))
 									require.NoError(t, err)
 									return &source
 								}(),
@@ -178,7 +178,7 @@ func TestInjectProxyEndpointModifier(t *testing.T) {
 								Type: elasticsearch,
 								Source: func() *structpb.Struct {
 									var source structpb.Struct
-									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["https://localhost:9200"],"proxy_disable":true}`))
+									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["https://example.output:9200"],"proxy_disable":true}`))
 									require.NoError(t, err)
 									return &source
 								}(),
@@ -199,7 +199,7 @@ func TestInjectProxyEndpointModifier(t *testing.T) {
 								Type: elasticsearch,
 								Source: func() *structpb.Struct {
 									var source structpb.Struct
-									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["https://localhost:9200"],"proxy_disable":true}`))
+									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["https://example.output:9200"],"proxy_disable":true}`))
 									require.NoError(t, err)
 									return &source
 								}(),
@@ -223,7 +223,7 @@ func TestInjectProxyEndpointModifier(t *testing.T) {
 								Type: elasticsearch,
 								Source: func() *structpb.Struct {
 									var source structpb.Struct
-									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["https://localhost:9200"]}`))
+									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["https://example.output:9200"]}`))
 									require.NoError(t, err)
 									return &source
 								}(),
@@ -244,7 +244,7 @@ func TestInjectProxyEndpointModifier(t *testing.T) {
 								Type: elasticsearch,
 								Source: func() *structpb.Struct {
 									var source structpb.Struct
-									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["https://localhost:9200"],"proxy_url":"https://localhost:8080"}`))
+									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["https://example.output:9200"],"proxy_url":"https://localhost:8080"}`))
 									require.NoError(t, err)
 									return &source
 								}(),
@@ -268,7 +268,7 @@ func TestInjectProxyEndpointModifier(t *testing.T) {
 								Type: elasticsearch,
 								Source: func() *structpb.Struct {
 									var source structpb.Struct
-									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["http://localhost:9200"]}`))
+									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["http://example.output:9200"]}`))
 									require.NoError(t, err)
 									return &source
 								}(),
@@ -289,7 +289,7 @@ func TestInjectProxyEndpointModifier(t *testing.T) {
 								Type: elasticsearch,
 								Source: func() *structpb.Struct {
 									var source structpb.Struct
-									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["http://localhost:9200"],"proxy_url":"http://localhost:8081"}`))
+									err := source.UnmarshalJSON([]byte(`{"type":"elasticsearch","hosts":["http://example.output:9200"],"proxy_url":"http://localhost:8081"}`))
 									require.NoError(t, err)
 									return &source
 								}(),
@@ -320,6 +320,7 @@ func TestInjectProxyEndpointModifier(t *testing.T) {
 func Test_injectProxyURL(t *testing.T) {
 	t.Setenv("HTTPS_PROXY", "https://localhost:8080")
 	t.Setenv("HTTP_PROXY", "http://localhost:8081")
+	t.Setenv("NO_PROXY", "do.not.inject.proxy.for.me")
 
 	tests := []struct {
 		name   string
@@ -341,41 +342,26 @@ func Test_injectProxyURL(t *testing.T) {
 		hosts:  nil,
 		expect: map[string]interface{}{"key": "value", "proxy_disable": true},
 	}, {
-		name:   "no hosts uses HTTPS_PROXY",
+		name:   "http hosts uses HTTP_PROXY",
 		m:      map[string]interface{}{"key": "value"},
-		hosts:  nil,
-		expect: map[string]interface{}{"key": "value", "proxy_url": "https://localhost:8080"},
+		hosts:  []string{"http://example:80"},
+		expect: map[string]interface{}{"key": "value", "proxy_url": "http://localhost:8081"},
 	}, {
 		name:   "https hosts uses HTTPS_PROXY",
 		m:      map[string]interface{}{"key": "value"},
 		hosts:  []string{"https://example:443"},
 		expect: map[string]interface{}{"key": "value", "proxy_url": "https://localhost:8080"},
 	}, {
-		name:   "http host uses HTTP_PROXY",
+		name:   "host skipped by NO_PROXY",
 		m:      map[string]interface{}{"key": "value"},
-		hosts:  []string{"http://example:80"},
-		expect: map[string]interface{}{"key": "value", "proxy_url": "http://localhost:8081"},
+		hosts:  []string{"https://do.not.inject.proxy.for.me", "https://do.not.inject.proxy.for.me:8080", "really.do.not.inject.proxy.for.me"},
+		expect: map[string]interface{}{"key": "value"},
 	}}
+
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
 			injectProxyURL(tc.m, tc.hosts)
 			require.Equal(t, tc.expect, tc.m)
 		})
 	}
-
-	t.Run("no hosts or HTTPS_PROXY uses HTTP_PROXY", func(t *testing.T) {
-		t.Setenv("HTTPS_PROXY", "")
-		t.Setenv("HTTP_PROXY", "http://localhost:8081")
-
-		m := map[string]interface{}{"key": "value"}
-		injectProxyURL(m, nil)
-		require.Equal(t, map[string]interface{}{"key": "value", "proxy_url": "http://localhost:8081"}, m)
-	})
-	t.Run("no env vars", func(t *testing.T) {
-		t.Setenv("HTTPS_PROXY", "")
-		t.Setenv("HTTP_PROXY", "")
-		m := map[string]interface{}{"key": "value"}
-		injectProxyURL(m, nil)
-		require.Equal(t, map[string]interface{}{"key": "value"}, m)
-	})
 }