Skip to content

Commit

Permalink
feat: switch kubebuilder implementation to avoid using rbac proxy
Browse files Browse the repository at this point in the history 
kubernetes-sigs/kubebuilder#3907
  • Loading branch information
@ed382
ed382 committed Feb 10, 2025
1 parent e35b203 commit 755a905
Show file tree
Hide file tree
Showing 4 changed files with 426 additions and 246 deletions.
6 changes: 2 additions & 4 deletions api/testtriggers/v1/testtrigger_webhook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ package v1

import (
"context"

"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/webhook"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)

Expand All @@ -34,7 +34,7 @@ type TestTriggerValidator interface {
var ctx = context.Background()
var vldtr TestTriggerValidator

func (in *TestTrigger) ValidateCreate() (admission.Warnings, error) {
func (in *TestTrigger) ValidateCreate(ctx context.Context) (admission.Warnings, error) {
return nil, vldtr.ValidateCreate(ctx, in)
}

Expand All @@ -46,8 +46,6 @@ func (in *TestTrigger) ValidateDelete() (admission.Warnings, error) {
return nil, vldtr.ValidateDelete(ctx, in)
}

var _ webhook.Validator = &TestTrigger{}

func (in *TestTrigger) SetupWebhookWithManager(mgr ctrl.Manager, validator TestTriggerValidator) error {
vldtr = validator
return ctrl.NewWebhookManagedBy(mgr).
Expand Down
203 changes: 172 additions & 31 deletions cmd/main.go
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
Copyright 2021.
Copyright 2025 The Kubernetes authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
Expand All @@ -17,29 +17,32 @@ limitations under the License.
package main

import (
"crypto/tls"
"encoding/base64"
"flag"
"os"

metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"

testtriggersv1 "github.com/kubeshop/testkube-operator/api/testtriggers/v1"
testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1"
"path/filepath"

// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.

zapUber "go.uber.org/zap"
"go.uber.org/zap/zapcore"
_ "k8s.io/client-go/plugin/pkg/client/auth"

"github.com/kelseyhightower/envconfig"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/certwatcher"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
"sigs.k8s.io/controller-runtime/pkg/webhook"

"github.com/kelseyhightower/envconfig"
testtriggersv1 "github.com/kubeshop/testkube-operator/api/testtriggers/v1"
testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1"

executorv1 "github.com/kubeshop/testkube-operator/api/executor/v1"
testkubev1 "github.com/kubeshop/testkube-operator/api/script/v1"
Expand All @@ -66,24 +69,14 @@ import (
testworkflowexecutioncontrollers "github.com/kubeshop/testkube-operator/internal/controller/testworkflowexecution"
testworkflowscontrollers "github.com/kubeshop/testkube-operator/internal/controller/testworkflows"
"github.com/kubeshop/testkube-operator/pkg/cronjob"
//+kubebuilder:scaffold:imports
// +kubebuilder:scaffold:imports
)

var (
scheme = runtime.NewScheme()
setupLog = ctrl.Log.WithName("setup")
)

// config for HTTP server
type config struct {
Port int
Fullname string
TemplateCronjob string `split_words:"true"`
Registry string
UseArgocdSync bool `split_words:"true"`
PurgeExecutions bool `split_words:"true"`
}

func init() {
utilruntime.Must(clientgoscheme.AddToScheme(scheme))

Expand All @@ -105,48 +98,178 @@ func init() {
//+kubebuilder:scaffold:scheme
}

// HttpServerConfig for HTTP server
type HttpServerConfig struct {
Port int
Fullname string
TemplateCronjob string `split_words:"true"`
Registry string
UseArgocdSync bool `split_words:"true"`
PurgeExecutions bool `split_words:"true"`
}

// nolint:gocyclo
func main() {
var metricsAddr string
var metricsCertPath, metricsCertName, metricsCertKey string
var webhookCertPath, webhookCertName, webhookCertKey string
var enableLeaderElection bool
var probeAddr string
flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
var secureMetrics bool
var enableHTTP2 bool
var tlsOpts []func(*tls.Config)
flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
flag.BoolVar(&enableLeaderElection, "leader-elect", false,
"Enable leader election for controller manager. "+
"Enabling this will ensure there is only one active controller manager.")
flag.BoolVar(&secureMetrics, "metrics-secure", true,
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
flag.StringVar(&webhookCertPath, "webhook-cert-path", "", "The directory that contains the webhook certificate.")
flag.StringVar(&webhookCertName, "webhook-cert-name", "tls.crt", "The name of the webhook certificate file.")
flag.StringVar(&webhookCertKey, "webhook-cert-key", "tls.key", "The name of the webhook key file.")
flag.StringVar(&metricsCertPath, "metrics-cert-path", "",
"The directory that contains the metrics server certificate.")
flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.")
flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
opts := zap.Options{
Development: true,
}
opts.BindFlags(flag.CommandLine)
flag.Parse()

setLogger()

var httpConfig config
err := envconfig.Process("APISERVER", &httpConfig)
// TODO: Do we want to panic here or just ignore the error?
if err != nil {
panic(err)
// if the enable-http2 flag is false (the default), http/2 should be disabled
// due to its vulnerabilities. More specifically, disabling http/2 will
// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
// Rapid Reset CVEs. For more information see:
// - https://github.com/advisories/GHSA-qppj-fm5r-hxr3
// - https://github.com/advisories/GHSA-4374-p667-p6c8
disableHTTP2 := func(c *tls.Config) {
setupLog.Info("disabling http/2")
c.NextProtos = []string{"http/1.1"}
}

var templateCronjob string
if httpConfig.TemplateCronjob != "" {
data, err := base64.StdEncoding.DecodeString(httpConfig.TemplateCronjob)
if !enableHTTP2 {
tlsOpts = append(tlsOpts, disableHTTP2)
}

// Create watchers for metrics and webhooks certificates
var metricsCertWatcher, webhookCertWatcher *certwatcher.CertWatcher

// Initial webhook TLS options
webhookTLSOpts := tlsOpts

if len(webhookCertPath) > 0 {
setupLog.Info("Initializing webhook certificate watcher using provided certificates",
"webhook-cert-path", webhookCertPath, "webhook-cert-name", webhookCertName, "webhook-cert-key", webhookCertKey)

var err error
webhookCertWatcher, err = certwatcher.New(
filepath.Join(webhookCertPath, webhookCertName),
filepath.Join(webhookCertPath, webhookCertKey),
)
if err != nil {
panic(err)
setupLog.Error(err, "Failed to initialize webhook certificate watcher")
os.Exit(1)
}

templateCronjob = string(data)
webhookTLSOpts = append(webhookTLSOpts, func(config *tls.Config) {
config.GetCertificate = webhookCertWatcher.GetCertificate
})
}

webhookServer := webhook.NewServer(webhook.Options{
TLSOpts: webhookTLSOpts,
})

// Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server.
// More info:
// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/server
// - https://book.kubebuilder.io/reference/metrics.html
metricsServerOptions := metricsserver.Options{
BindAddress: metricsAddr,
SecureServing: secureMetrics,
TLSOpts: tlsOpts,
}

if secureMetrics {
// FilterProvider is used to protect the metrics endpoint with authn/authz.
// These configurations ensure that only authorized users and service accounts
// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
// https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/filters#WithAuthenticationAndAuthorization
metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
}

// If the certificate is not specified, controller-runtime will automatically
// generate self-signed certificates for the metrics server. While convenient for development and testing,
// this setup is not recommended for production.
//
// TODO(user): If you enable certManager, uncomment the following lines:
// - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates
// managed by cert-manager for the metrics server.
// - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification.
if len(metricsCertPath) > 0 {
setupLog.Info("Initializing metrics certificate watcher using provided certificates",
"metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey)

var err error
metricsCertWatcher, err = certwatcher.New(
filepath.Join(metricsCertPath, metricsCertName),
filepath.Join(metricsCertPath, metricsCertKey),
)
if err != nil {
setupLog.Error(err, "to initialize metrics certificate watcher", "error", err)
os.Exit(1)
}

metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
config.GetCertificate = metricsCertWatcher.GetCertificate
})
}

mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
Scheme: scheme,
Metrics: metricsserver.Options{BindAddress: metricsAddr},
Metrics: metricsServerOptions,
WebhookServer: webhookServer,
HealthProbeBindAddress: probeAddr,
LeaderElection: enableLeaderElection,
LeaderElectionID: "47f0dfc1.testkube.io",
// LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily
// when the Manager ends. This requires the binary to immediately end when the
// Manager is stopped, otherwise, this setting is unsafe. Setting this significantly
// speeds up voluntary leader transitions as the new leader don't have to wait
// LeaseDuration time first.
//
// In the default scaffold provided, the program ends immediately after
// the manager stops, so would be fine to enable this option. However,
// if you are doing or is intended to do any operation such as perform cleanups
// after the manager stops then its usage might be unsafe.
// LeaderElectionReleaseOnCancel: true,
})
if err != nil {
setupLog.Error(err, "unable to start manager")
os.Exit(1)
}

var httpConfig HttpServerConfig
err = envconfig.Process("APISERVER", &httpConfig)
// TODO: Do we want to panic here or just ignore the error?
if err != nil {
panic(err)
}
var templateCronjob string
if httpConfig.TemplateCronjob != "" {
data, err := base64.StdEncoding.DecodeString(httpConfig.TemplateCronjob)
if err != nil {
panic(err)
}
templateCronjob = string(data)
}
cronJobClient := cronjob.NewClient(mgr.GetClient(), httpConfig.Fullname, httpConfig.Port,
templateCronjob, httpConfig.Registry, httpConfig.UseArgocdSync)
if err = (&scriptcontrollers.ScriptReconciler{
Expand All @@ -156,6 +279,7 @@ func main() {
setupLog.Error(err, "unable to create controller", "controller", "Script")
os.Exit(1)
}

if err = (&executorcontrollers.ExecutorReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Expand Down Expand Up @@ -310,6 +434,23 @@ func main() {
} else {
setupLog.Info("Webhooks are disabled")
}
// +kubebuilder:scaffold:builder

if metricsCertWatcher != nil {
setupLog.Info("Adding metrics certificate watcher to manager")
if err := mgr.Add(metricsCertWatcher); err != nil {
setupLog.Error(err, "unable to add metrics certificate watcher to manager")
os.Exit(1)
}
}

if webhookCertWatcher != nil {
setupLog.Info("Adding webhook certificate watcher to manager")
if err := mgr.Add(webhookCertWatcher); err != nil {
setupLog.Error(err, "unable to add webhook certificate watcher to manager")
os.Exit(1)
}
}

if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
setupLog.Error(err, "unable to set up health check")
Expand Down
Loading

0 comments on commit 755a905

Please sign in to comment.