Merge pull request #60 from uswitch/SEC-516-add-sigsci-config

SEC-516 Support ExtAuthz and GRPC Logger configuration

README.md

@@ -160,25 +160,35 @@ The Yggdrasil-specific metrics which are available from the API are:
 
 ## Flags
 ```
---address string                           yggdrasil envoy control plane listen address (default "0.0.0.0:8080")
---ca string                                trustedCA
---cert string                              certfile
---config string                            config file
---debug                                    Log at debug level
---envoy-listener-ipv4-address string       IPv4 address by the envoy proxy to accept incoming connections (default 0.0.0.0)
---envoy-port uint32                        port by the envoy proxy to accept incoming connections (default 10000)
---health-address string                    yggdrasil health API listen address (default "0.0.0.0:8081")
--h, --help                                 help for yggdrasil
---host-selection-retry-attempts int        Number of host selection retry attempts. Set to value >=0 to enable (default -1)
---ingress-classes strings                  Ingress classes to watch
---key string                               keyfile
---kube-config stringArray                  Path to kube config
---max-ejection-percentage int32            maximal percentage of hosts ejected via outlier detection. Set to >=0 to activate outlier detection in envoy. (default -1)
---node-name string                         envoy node name
---upstream-healthcheck-healthy uint32      number of successful healthchecks before the backend is considered healthy (default 3)
---upstream-healthcheck-interval duration   duration of the upstream health check interval (default 10s)
---upstream-healthcheck-timeout duration    timeout of the upstream healthchecks (default 5s)
---upstream-healthcheck-unhealthy uint32    number of failed healthchecks before the backend is considered unhealthy (default 3)
---upstream-port uint32                     port used to connect to the upstream ingresses (default 443)
---use-remote-address                       populates the X-Forwarded-For header with the client address. Set to true when used as edge proxy - [documentation](https://www.envoyproxy.io/docs/envoy/v1.19.0/api-v3/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto#envoy-v3-api-field-extensions-filters-network-http-connection-manager-v3-httpconnectionmanager-use-remote-address) (default false)
+--address string                              yggdrasil envoy control plane listen address (default "0.0.0.0:8080")
+--ca string                                   trustedCA
+--cert string                                 certfile
+--config string                               config file
+--debug                                       Log at debug level
+--envoy-listener-ipv4-address string          IPv4 address by the envoy proxy to accept incoming connections (default "0.0.0.0")
+--envoy-port uint32                           port by the envoy proxy to accept incoming connections (default 10000)
+--health-address string                       yggdrasil health API listen address (default "0.0.0.0:8081")
+--help                                        help for yggdrasil
+--host-selection-retry-attempts int           Number of host selection retry attempts. Set to value >=0 to enable (default -1)
+--http-ext-authz-allow-partial-message        When this field is true, Envoy will buffer the message until max_request_bytes is reached (default true)
+--http-ext-authz-cluster string               The name of the upstream gRPC cluster
+--http-ext-authz-failure-mode-allow           Changes filters behaviour on errors (default true)
+--http-ext-authz-max-request-bytes uint32     Sets the maximum size of a message body that the filter will hold in memory (default 8192)
+--http-ext-authz-timeout duration             The timeout for the gRPC request. This is the timeout for a specific request. (default 200ms)
+--http-grpc-logger-cluster string             The name of the upstream gRPC cluster
+--http-grpc-logger-name string                Name of the access log
+--http-grpc-logger-request-headers strings    access logs request headers
+--http-grpc-logger-response-headers strings   access logs response headers
+--http-grpc-logger-timeout duration           The timeout for the gRPC request (default 200ms)
+--ingress-classes strings                     Ingress classes to watch
+--key string                                  keyfile
+--kube-config stringArray                     Path to kube config
+--max-ejection-percentage int32               maximal percentage of hosts ejected via outlier detection. Set to >=0 to activate outlier detection in envoy. (default -1)
+--node-name string                            envoy node name
+--upstream-healthcheck-healthy uint32         number of successful healthchecks before the backend is considered healthy (default 3)
+--upstream-healthcheck-interval duration      duration of the upstream health check interval (default 10s)
+--upstream-healthcheck-timeout duration       timeout of the upstream healthchecks (default 5s)
+--upstream-healthcheck-unhealthy uint32       number of failed healthchecks before the backend is considered unhealthy (default 3)
+--upstream-port uint32                        port used to connect to the upstream ingresses (default 443)
+--use-remote-address                          populates the X-Forwarded-For header with the client address. Set to true when used as edge proxy
 ```

cmd/root.go

@@ -44,6 +44,8 @@ type config struct {
 	HostSelectionRetryAttempts int64                     `json:"hostSelectionRetryAttempts"`
 	UpstreamHealthCheck        envoy.UpstreamHealthCheck `json:"upstreamHealthCheck"`
 	UseRemoteAddress           bool                      `json:"useRemoteAddress"`
+	HttpExtAuthz               envoy.HttpExtAuthz        `json:"httpExtAuthz"`
+	HttpGrpcLogger             envoy.HttpGrpcLogger      `json:"httpGrpcLogger"`
 }
 
 // Hasher returns node ID as an ID
@@ -57,7 +59,7 @@ var (
 
 var rootCmd = &cobra.Command{
 	Use:   "yggdrasil",
-	Short: "yggdrasil creates an envoy control plane that watches ingress objects",
+	Short: "yggdrasil creates an envoy control plane that watches ingress objects  ",
 	RunE:  main,
 }
 
@@ -91,6 +93,16 @@ func init() {
 	rootCmd.PersistentFlags().Uint32("upstream-healthcheck-healthy", 3, "number of successful healthchecks before the backend is considered healthy")
 	rootCmd.PersistentFlags().Uint32("upstream-healthcheck-unhealthy", 3, "number of failed healthchecks before the backend is considered unhealthy")
 	rootCmd.PersistentFlags().Bool("use-remote-address", false, "populates the X-Forwarded-For header with the client address. Set to true when used as edge proxy")
+	rootCmd.PersistentFlags().String("http-grpc-logger-name", "", "Name of the access log")
+	rootCmd.PersistentFlags().String("http-grpc-logger-cluster", "", "The name of the upstream gRPC cluster")
+	rootCmd.PersistentFlags().Duration("http-grpc-logger-timeout", 200*time.Millisecond, "The timeout for the gRPC request")
+	rootCmd.PersistentFlags().StringSlice("http-grpc-logger-request-headers", []string{}, "access logs request headers")
+	rootCmd.PersistentFlags().StringSlice("http-grpc-logger-response-headers", []string{}, "access logs response headers")
+	rootCmd.PersistentFlags().String("http-ext-authz-cluster", "", "The name of the upstream gRPC cluster")
+	rootCmd.PersistentFlags().Duration("http-ext-authz-timeout", 200*time.Millisecond, "The timeout for the gRPC request. This is the timeout for a specific request.")
+	rootCmd.PersistentFlags().Uint32("http-ext-authz-max-request-bytes", 8192, "Sets the maximum size of a message body that the filter will hold in memory")
+	rootCmd.PersistentFlags().Bool("http-ext-authz-allow-partial-message", true, "When this field is true, Envoy will buffer the message until max_request_bytes is reached")
+	rootCmd.PersistentFlags().Bool("http-ext-authz-failure-mode-allow", true, "Changes filters behaviour on errors")
 	viper.BindPFlag("debug", rootCmd.PersistentFlags().Lookup("debug"))
 	viper.BindPFlag("address", rootCmd.PersistentFlags().Lookup("address"))
 	viper.BindPFlag("healthAddress", rootCmd.PersistentFlags().Lookup("health-address"))
@@ -109,6 +121,16 @@ func init() {
 	viper.BindPFlag("upstreamHealthCheck.healthyThreshold", rootCmd.PersistentFlags().Lookup("upstream-healthcheck-healthy"))
 	viper.BindPFlag("upstreamHealthCheck.unhealthyThreshold", rootCmd.PersistentFlags().Lookup("upstream-healthcheck-unhealthy"))
 	viper.BindPFlag("useRemoteAddress", rootCmd.PersistentFlags().Lookup("use-remote-address"))
+	viper.BindPFlag("httpGrpcLogger.name", rootCmd.PersistentFlags().Lookup("http-grpc-logger-name"))
+	viper.BindPFlag("httpGrpcLogger.cluster", rootCmd.PersistentFlags().Lookup("http-grpc-logger-cluster"))
+	viper.BindPFlag("httpGrpcLogger.timeout", rootCmd.PersistentFlags().Lookup("http-grpc-logger-timeout"))
+	viper.BindPFlag("httpGrpcLogger.requestHeaders", rootCmd.PersistentFlags().Lookup("http-grpc-logger-request-headers"))
+	viper.BindPFlag("httpGrpcLogger.responseHeaders", rootCmd.PersistentFlags().Lookup("http-grpc-logger-response-headers"))
+	viper.BindPFlag("httpExtAuthz.cluster", rootCmd.PersistentFlags().Lookup("http-ext-authz-cluster"))
+	viper.BindPFlag("httpExtAuthz.timeout", rootCmd.PersistentFlags().Lookup("http-ext-authz-timeout"))
+	viper.BindPFlag("httpExtAuthz.maxRequestBytes", rootCmd.PersistentFlags().Lookup("http-ext-authz-max-request-bytes"))
+	viper.BindPFlag("httpExtAuthz.allowPartialMessage", rootCmd.PersistentFlags().Lookup("http-ext-authz-allow-partial-message"))
+	viper.BindPFlag("httpExtAuthz.FailureModeAllow", rootCmd.PersistentFlags().Lookup("http-ext-authz-failure-mode-allow"))
 }
 
 func initConfig() {
@@ -198,6 +220,8 @@ func main(*cobra.Command, []string) error {
 		envoy.WithHostSelectionRetryAttempts(viper.GetInt64("hostSelectionRetryAttempts")),
 		envoy.WithUpstreamHealthCheck(c.UpstreamHealthCheck),
 		envoy.WithUseRemoteAddress(c.UseRemoteAddress),
+		envoy.WithHttpExtAuthzCluster(c.HttpExtAuthz),
+		envoy.WithHttpGrpcLogger(c.HttpGrpcLogger),
 	)
 	snapshotter := envoy.NewSnapshotter(envoyCache, configurator, lister)
 

cmd/server.go

@@ -70,7 +70,7 @@ func runEnvoyServer(envoyServer server.Server, address string, healthAddress str
 	healthMux := http.NewServeMux()
 
 	healthServer := &http.Server{
-		Addr:    fmt.Sprintf(healthAddress),
+		Addr:    fmt.Sprint(healthAddress),
 		Handler: healthMux,
 	}
 

pkg/envoy/boilerplate.go

@@ -11,6 +11,8 @@ import (
 	listener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	route "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
 	eal "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/file/v3"
+	gal "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/grpc/v3"
+	eauthz "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_authz/v3"
 	hcfg "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/health_check/v3"
 	hcm "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
 	auth "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
@@ -19,6 +21,7 @@ import (
 	any "github.com/golang/protobuf/ptypes/any"
 	"github.com/golang/protobuf/ptypes/duration"
 	"github.com/golang/protobuf/ptypes/wrappers"
+	"google.golang.org/protobuf/types/known/durationpb"
 	"google.golang.org/protobuf/types/known/structpb"
 	"google.golang.org/protobuf/types/known/wrapperspb"
 )
@@ -109,7 +112,48 @@ func makeHealthConfig() *hcfg.HealthCheck {
 	}
 }
 
+func makeExtAuthzConfig(cfg HttpExtAuthz) *eauthz.ExtAuthz {
+	return &eauthz.ExtAuthz{
+		TransportApiVersion: core.ApiVersion_V3,
+		Services: &eauthz.ExtAuthz_GrpcService{
+			GrpcService: &core.GrpcService{
+				TargetSpecifier: &core.GrpcService_EnvoyGrpc_{
+					EnvoyGrpc: &core.GrpcService_EnvoyGrpc{
+						ClusterName: cfg.Cluster,
+					},
+				},
+				Timeout: durationpb.New(cfg.Timeout),
+			},
+		},
+		WithRequestBody: &eauthz.BufferSettings{
+			MaxRequestBytes:     cfg.MaxRequestBytes,
+			AllowPartialMessage: cfg.AllowPartialMessage,
+		},
+		FailureModeAllow: cfg.FailureModeAllow,
+	}
+}
+
+func makeGrpcLoggerConfig(cfg HttpGrpcLogger) *gal.HttpGrpcAccessLogConfig {
+	return &gal.HttpGrpcAccessLogConfig{
+		CommonConfig: &gal.CommonGrpcAccessLogConfig{
+			LogName: cfg.Name,
+			GrpcService: &core.GrpcService{
+				TargetSpecifier: &core.GrpcService_EnvoyGrpc_{
+					EnvoyGrpc: &core.GrpcService_EnvoyGrpc{
+						ClusterName: cfg.Cluster,
+					},
+				},
+				Timeout: durationpb.New(cfg.Timeout),
+			},
+			TransportApiVersion: core.ApiVersion_V3,
+		},
+		AdditionalRequestHeadersToLog:  cfg.AdditionalRequestHeaders,
+		AdditionalResponseHeadersToLog: cfg.AdditionalResponseHeaders,
+	}
+}
+
 func (c *KubernetesConfigurator) makeConnectionManager(virtualHosts []*route.VirtualHost) *hcm.HttpConnectionManager {
+	// Access Logs
 	accessLogConfig, err := util.MessageToStruct(
 		&eal.FileAccessLog{
 			Path: "/var/log/envoy/access.log",
@@ -129,6 +173,32 @@ func (c *KubernetesConfigurator) makeConnectionManager(virtualHosts []*route.Vir
 	if err != nil {
 		log.Fatalf("failed to marshal access log config struct to typed struct: %s", err)
 	}
+
+	accessLoggers := []*cal.AccessLog{
+		{
+			Name:       "envoy.access_loggers.file",
+			ConfigType: &cal.AccessLog_TypedConfig{TypedConfig: anyAccessLogConfig},
+		},
+	}
+
+	if c.httpGrpcLogger.Cluster != "" {
+		grpcLoggerConfig, err := util.MessageToStruct(makeGrpcLoggerConfig(c.httpGrpcLogger))
+		if err != nil {
+			log.Fatalf("failed to convert healthcheck proto message to struct: %s", err)
+		}
+		anyGrpcLoggerConfig, err := types.MarshalAny(grpcLoggerConfig)
+		if err != nil {
+			log.Fatalf("failed to marshal healthcheck config struct to typed struct: %s", err)
+		}
+		accessLoggers = append(accessLoggers, &cal.AccessLog{
+			Name:       "envoy.access_loggers.http_grpc",
+			ConfigType: &cal.AccessLog_TypedConfig{TypedConfig: anyGrpcLoggerConfig},
+		})
+	}
+
+	// HTTP Filters
+	filterBuilder := &httpFilterBuilder{}
+
 	healthConfig, err := util.MessageToStruct(makeHealthConfig())
 	if err != nil {
 		log.Fatalf("failed to convert healthcheck proto message to struct: %s", err)
@@ -137,18 +207,32 @@ func (c *KubernetesConfigurator) makeConnectionManager(virtualHosts []*route.Vir
 	if err != nil {
 		log.Fatalf("failed to marshal healthcheck config struct to typed struct: %s", err)
 	}
+
+	filterBuilder.Add(&hcm.HttpFilter{
+		Name:       "envoy.filters.http.health_check",
+		ConfigType: &hcm.HttpFilter_TypedConfig{TypedConfig: anyHealthConfig},
+	})
+
+	if c.httpExtAuthz.Cluster != "" {
+		extAuthzConfig, err := util.MessageToStruct(makeExtAuthzConfig(c.httpExtAuthz))
+		if err != nil {
+			log.Fatalf("failed to convert extAuthz proto message to struct: %s", err)
+		}
+		anyExtAuthzConfig, err := types.MarshalAny(extAuthzConfig)
+		if err != nil {
+			log.Fatalf("failed to marshal extAuthz config struct to typed struct: %s", err)
+		}
+
+		filterBuilder.Add(&hcm.HttpFilter{
+			Name:       "envoy.filters.http.ext_authz",
+			ConfigType: &hcm.HttpFilter_TypedConfig{TypedConfig: anyExtAuthzConfig},
+		})
+	}
+
 	return &hcm.HttpConnectionManager{
-		CodecType:  hcm.HttpConnectionManager_AUTO,
-		StatPrefix: "ingress_http",
-		HttpFilters: []*hcm.HttpFilter{
-			{
-				Name:       "envoy.filters.http.health_check",
-				ConfigType: &hcm.HttpFilter_TypedConfig{TypedConfig: anyHealthConfig},
-			},
-			{
-				Name: "envoy.filters.http.router",
-			},
-		},
+		CodecType:   hcm.HttpConnectionManager_AUTO,
+		StatPrefix:  "ingress_http",
+		HttpFilters: filterBuilder.Filters(),
 		UpgradeConfigs: []*hcm.HttpConnectionManager_UpgradeConfig{
 			{
 				UpgradeType: "websocket",
@@ -160,13 +244,8 @@ func (c *KubernetesConfigurator) makeConnectionManager(virtualHosts []*route.Vir
 				VirtualHosts: virtualHosts,
 			},
 		},
-		Tracing: &hcm.HttpConnectionManager_Tracing{},
-		AccessLog: []*cal.AccessLog{
-			{
-				Name:       "envoy.access_loggers.file",
-				ConfigType: &cal.AccessLog_TypedConfig{TypedConfig: anyAccessLogConfig},
-			},
-		},
+		Tracing:          &hcm.HttpConnectionManager_Tracing{},
+		AccessLog:        accessLoggers,
 		UseRemoteAddress: &wrapperspb.BoolValue{Value: c.useRemoteAddress},
 	}
 }
@@ -337,7 +416,7 @@ func makeCluster(c cluster, ca string, healthCfg UpstreamHealthCheck, outlierPer
 	cluster := &v3cluster.Cluster{
 		ClusterDiscoveryType: &v3cluster.Cluster_Type{Type: v3cluster.Cluster_STRICT_DNS},
 		Name:                 c.Name,
-		ConnectTimeout:       &duration.Duration{Seconds: int64(c.Timeout.Seconds())},
+		ConnectTimeout:       durationpb.New(c.Timeout),
 		LoadAssignment: &endpoint.ClusterLoadAssignment{
 			ClusterName: c.Name,
 			Endpoints: []*endpoint.LocalityLbEndpoints{

pkg/envoy/configurator.go

@@ -29,6 +29,22 @@ type UpstreamHealthCheck struct {
 	HealthyThreshold   uint32        `json:"healtyThreshold"`
 }
 
+type HttpExtAuthz struct {
+	Cluster             string        `json:"cluster"`
+	Timeout             time.Duration `json:"timeout"`
+	MaxRequestBytes     uint32        `json:"maxRequestBytes"`
+	AllowPartialMessage bool          `json:"allowPartialMessage"`
+	FailureModeAllow    bool          `json:"FailureModeAllow"`
+}
+
+type HttpGrpcLogger struct {
+	Name                      string        `json:"name"`
+	Cluster                   string        `json:"cluster"`
+	Timeout                   time.Duration `json:"timeout"`
+	AdditionalRequestHeaders  []string      `json:"additionalRequestHeaders"`
+	AdditionalResponseHeaders []string      `json:"additionalResponseHeaders"`
+}
+
 //KubernetesConfigurator takes a given Ingress Class and lister to find only ingresses of that class
 type KubernetesConfigurator struct {
 	ingressClasses             []string
@@ -42,6 +58,8 @@ type KubernetesConfigurator struct {
 	hostSelectionRetryAttempts int64
 	upstreamHealthCheck        UpstreamHealthCheck
 	useRemoteAddress           bool
+	httpExtAuthz               HttpExtAuthz
+	httpGrpcLogger             HttpGrpcLogger
 
 	previousConfig  *envoyConfiguration
 	listenerVersion string
@@ -100,7 +118,7 @@ func compareHosts(pattern, host string) bool {
 	hostParts := strings.Split(host, ".")
 
 	if len(patternParts) == len(hostParts) {
-		for i, _ := range patternParts {
+		for i := range patternParts {
 			if patternParts[i] != "*" && patternParts[i] != hostParts[i] {
 				return false
 			}

pkg/envoy/http_filters.go

@@ -0,0 +1,19 @@
+package envoy
+
+import (
+	hcm "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+)
+
+type httpFilterBuilder struct {
+	filters []*hcm.HttpFilter
+}
+
+func (b *httpFilterBuilder) Add(filter *hcm.HttpFilter) *httpFilterBuilder {
+	b.filters = append(b.filters, filter)
+	return b
+}
+
+func (b *httpFilterBuilder) Filters() []*hcm.HttpFilter {
+	b.Add(&hcm.HttpFilter{Name: "envoy.filters.http.router"})
+	return b.filters
+}

pkg/envoy/options.go

@@ -50,3 +50,17 @@ func WithUseRemoteAddress(useRemoteAddress bool) option {
 		c.useRemoteAddress = useRemoteAddress
 	}
 }
+
+// WithHttpExtAuthzCluster configures the options for the gRPC cluster
+func WithHttpExtAuthzCluster(httpExtAuthz HttpExtAuthz) option {
+	return func(c *KubernetesConfigurator) {
+		c.httpExtAuthz = httpExtAuthz
+	}
+}
+
+// WithHttpGrpcLogger configures the options for the gPRC access logger
+func WithHttpGrpcLogger(httpGrpcLogger HttpGrpcLogger) option {
+	return func(c *KubernetesConfigurator) {
+		c.httpGrpcLogger = httpGrpcLogger
+	}
+}