From ca6c1bfdeb8de01e5a26ec249ad4bb0c9e8ca363 Mon Sep 17 00:00:00 2001
From: Angelos Kolaitis <angelos.kolaitis@canonical.com>
Date: Fri, 26 Apr 2024 15:25:20 +0300
Subject: [PATCH 1/6] do not panic if reading invalid RSA public key (#378)

---
 src/k8s/pkg/k8sd/pki/load.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/k8s/pkg/k8sd/pki/load.go b/src/k8s/pkg/k8sd/pki/load.go
index c2bda4cab..19928e04e 100644
--- a/src/k8s/pkg/k8sd/pki/load.go
+++ b/src/k8s/pkg/k8sd/pki/load.go
@@ -61,6 +61,9 @@ func LoadRSAPrivateKey(keyPEM string) (*rsa.PrivateKey, error) {
 // LoadRSAPublicKey parses the specified PEM block and return the rsa.PublicKey.
 func LoadRSAPublicKey(keyPEM string) (*rsa.PublicKey, error) {
 	pb, _ := pem.Decode([]byte(keyPEM))
+	if pb == nil {
+		return nil, fmt.Errorf("failed to parse PEM block")
+	}
 	switch pb.Type {
 	case "PUBLIC KEY":
 		parsed, err := x509.ParsePKIXPublicKey(pb.Bytes)

From 057da8e0cbd9d0e23e7e1ab156a3a3c95bba15b4 Mon Sep 17 00:00:00 2001
From: Angelos Kolaitis <angelos.kolaitis@canonical.com>
Date: Fri, 26 Apr 2024 16:29:57 +0300
Subject: [PATCH 2/6] allow the apiserver to come up (#382)

---
 tests/integration/tests/test_etcd.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/tests/test_etcd.py b/tests/integration/tests/test_etcd.py
index b1b0dd776..994b5831c 100644
--- a/tests/integration/tests/test_etcd.py
+++ b/tests/integration/tests/test_etcd.py
@@ -72,7 +72,7 @@ def test_etcd(instances: List[harness.Instance], etcd_cluster: EtcdCluster):
     )
 
     # check that we can still connect to the kubernetes cluster
-    util.stubbornly(retries=3, delay_s=1).on(k8s_instance).exec(
+    util.stubbornly(retries=10, delay_s=2).on(k8s_instance).exec(
         ["k8s", "kubectl", "get", "pods", "-A"]
     )
 

From 571514141802864ffdd7bbdc2dd480e34bd2845b Mon Sep 17 00:00:00 2001
From: Angelos Kolaitis <angelos.kolaitis@canonical.com>
Date: Fri, 26 Apr 2024 16:35:45 +0300
Subject: [PATCH 3/6] Feature Controller (#375)

* Add stateUpgradeOnlyOrDeleted() helper, solves failure to disable network and dependent components

* Implement FeatureController

* Add FeatureController in microcluster app

* Adjust name of notify to better match controller names

* delegate feature management to the FeatureController

* use a struct for feature controller opts

* add helper utils.MaybeNotify

* retry in case of failure in FeatureController component reconcile

* separate reconcile loops for network components

* update notify conditions to not miss dns and kubelet updates

* start feature controller after node is ready

* properly wait for cilium in network tests
---
 src/k8s/pkg/k8sd/api/cluster_config.go        |  73 ++-------
 src/k8s/pkg/k8sd/api/provider.go              |   3 +-
 src/k8s/pkg/k8sd/app/app.go                   |  29 ++++
 src/k8s/pkg/k8sd/app/hooks_bootstrap.go       |  68 ++-------
 src/k8s/pkg/k8sd/app/hooks_start.go           |  30 ++++
 src/k8s/pkg/k8sd/app/provider.go              |  30 +++-
 src/k8s/pkg/k8sd/controllers/feature.go       | 140 ++++++++++++++++++
 src/k8s/pkg/k8sd/features/feature_dns.go      |  16 +-
 src/k8s/pkg/k8sd/features/feature_gateway.go  |  12 +-
 src/k8s/pkg/k8sd/features/feature_ingress.go  |  16 +-
 .../pkg/k8sd/features/feature_loadbalancer.go |  44 +++---
 .../k8sd/features/feature_local_storage.go    |   2 +-
 .../k8sd/features/feature_metrics_server.go   |   2 +-
 src/k8s/pkg/k8sd/features/state.go            |   9 +-
 src/k8s/pkg/utils/chan.go                     |   9 ++
 tests/integration/tests/test_gateway.py       |  32 ----
 tests/integration/tests/test_ingress.py       |  32 ----
 tests/integration/tests/test_loadbalancer.py  |  33 +----
 18 files changed, 313 insertions(+), 267 deletions(-)
 create mode 100644 src/k8s/pkg/k8sd/controllers/feature.go
 create mode 100644 src/k8s/pkg/utils/chan.go

diff --git a/src/k8s/pkg/k8sd/api/cluster_config.go b/src/k8s/pkg/k8sd/api/cluster_config.go
index fb94ba601..45c4e2265 100644
--- a/src/k8s/pkg/k8sd/api/cluster_config.go
+++ b/src/k8s/pkg/k8sd/api/cluster_config.go
@@ -9,7 +9,6 @@ import (
 	api "github.com/canonical/k8s/api/v1"
 	"github.com/canonical/k8s/pkg/k8sd/database"
 	databaseutil "github.com/canonical/k8s/pkg/k8sd/database/util"
-	"github.com/canonical/k8s/pkg/k8sd/features"
 	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/utils"
 	"github.com/canonical/lxd/lxd/response"
@@ -18,7 +17,6 @@ import (
 
 func (e *Endpoints) putClusterConfig(s *state.State, r *http.Request) response.Response {
 	var req api.UpdateClusterConfigRequest
-	snap := e.provider.Snap()
 
 	if err := utils.NewStrictJSONDecoder(r.Body).Decode(&req); err != nil {
 		return response.BadRequest(fmt.Errorf("failed to decode request: %w", err))
@@ -32,74 +30,25 @@ func (e *Endpoints) putClusterConfig(s *state.State, r *http.Request) response.R
 		return response.BadRequest(fmt.Errorf("failed to parse datastore config: %w", err))
 	}
 
-	var mergedConfig types.ClusterConfig
 	if err := s.Database.Transaction(r.Context(), func(ctx context.Context, tx *sql.Tx) error {
-		var err error
-		mergedConfig, err = database.SetClusterConfig(ctx, tx, requestedConfig)
-		if err != nil {
+		if _, err := database.SetClusterConfig(ctx, tx, requestedConfig); err != nil {
 			return fmt.Errorf("failed to update cluster configuration: %w", err)
 		}
-
 		return nil
 	}); err != nil {
 		return response.InternalError(fmt.Errorf("database transaction to update cluster configuration failed: %w", err))
 	}
 
-	if !requestedConfig.Network.Empty() {
-		if err := features.ApplyNetwork(s.Context, snap, mergedConfig.Network); err != nil {
-			return response.InternalError(fmt.Errorf("failed to apply network: %w", err))
-		}
-	}
-
-	if !requestedConfig.DNS.Empty() {
-		dnsIP, err := features.ApplyDNS(s.Context, snap, mergedConfig.DNS, mergedConfig.Kubelet)
-		if err != nil {
-			return response.InternalError(fmt.Errorf("failed to apply DNS: %w", err))
-		}
-
-		if dnsIP != "" {
-			if err := s.Database.Transaction(s.Context, func(ctx context.Context, tx *sql.Tx) error {
-				if mergedConfig, err = database.SetClusterConfig(ctx, tx, types.ClusterConfig{
-					Kubelet: types.Kubelet{
-						ClusterDNS: utils.Pointer(dnsIP),
-					},
-				}); err != nil {
-					return fmt.Errorf("failed to update cluster configuration for dns=%s: %w", dnsIP, err)
-				}
-				return nil
-			}); err != nil {
-				return response.InternalError(fmt.Errorf("database transaction to update cluster configuration failed: %w", err))
-			}
-		}
-	}
-
-	if !requestedConfig.LocalStorage.Empty() {
-		if err := features.ApplyLocalStorage(s.Context, snap, mergedConfig.LocalStorage); err != nil {
-			return response.InternalError(fmt.Errorf("failed to apply local-storage: %w", err))
-		}
-	}
-	if !requestedConfig.Gateway.Empty() {
-		if err := features.ApplyGateway(s.Context, snap, mergedConfig.Gateway); err != nil {
-			return response.InternalError(fmt.Errorf("failed to apply gateway: %w", err))
-		}
-	}
-	if !requestedConfig.Ingress.Empty() {
-		if err := features.ApplyIngress(s.Context, snap, mergedConfig.Ingress); err != nil {
-			return response.InternalError(fmt.Errorf("failed to apply ingress: %w", err))
-		}
-	}
-	if !requestedConfig.LoadBalancer.Empty() {
-		if err := features.ApplyLoadBalancer(s.Context, snap, mergedConfig.LoadBalancer); err != nil {
-			return response.InternalError(fmt.Errorf("failed to apply load-balancer: %w", err))
-		}
-	}
-	if !requestedConfig.MetricsServer.Empty() {
-		if err := features.ApplyMetricsServer(s.Context, snap, mergedConfig.MetricsServer); err != nil {
-			return response.InternalError(fmt.Errorf("failed to apply metrics-server: %w", err))
-		}
-	}
-
-	e.provider.NotifyNodeConfigController()
+	e.provider.NotifyUpdateNodeConfigController()
+	e.provider.NotifyFeatureController(
+		!requestedConfig.Network.Empty(),
+		!requestedConfig.Gateway.Empty(),
+		!requestedConfig.Ingress.Empty(),
+		!requestedConfig.LoadBalancer.Empty(),
+		!requestedConfig.LocalStorage.Empty(),
+		!requestedConfig.MetricsServer.Empty(),
+		!requestedConfig.DNS.Empty() || !requestedConfig.Kubelet.Empty(),
+	)
 
 	return response.SyncResponse(true, &api.UpdateClusterConfigResponse{})
 }
diff --git a/src/k8s/pkg/k8sd/api/provider.go b/src/k8s/pkg/k8sd/api/provider.go
index 803f74302..8e4172d21 100644
--- a/src/k8s/pkg/k8sd/api/provider.go
+++ b/src/k8s/pkg/k8sd/api/provider.go
@@ -9,5 +9,6 @@ import (
 type Provider interface {
 	MicroCluster() *microcluster.MicroCluster
 	Snap() snap.Snap
-	NotifyNodeConfigController()
+	NotifyUpdateNodeConfigController()
+	NotifyFeatureController(network, gateway, ingress, loadBalancer, localStorage, metricsServer, dns bool)
 }
diff --git a/src/k8s/pkg/k8sd/app/app.go b/src/k8s/pkg/k8sd/app/app.go
index cb2827a1b..cb2040d98 100644
--- a/src/k8s/pkg/k8sd/app/app.go
+++ b/src/k8s/pkg/k8sd/app/app.go
@@ -50,6 +50,16 @@ type App struct {
 	// updateNodeConfigController
 	triggerUpdateNodeConfigControllerCh chan struct{}
 	updateNodeConfigController          *controllers.UpdateNodeConfigurationController
+
+	// featureController
+	triggerFeatureControllerNetworkCh       chan struct{}
+	triggerFeatureControllerGatewayCh       chan struct{}
+	triggerFeatureControllerIngressCh       chan struct{}
+	triggerFeatureControllerLoadBalancerCh  chan struct{}
+	triggerFeatureControllerLocalStorageCh  chan struct{}
+	triggerFeatureControllerMetricsServerCh chan struct{}
+	triggerFeatureControllerDNSCh           chan struct{}
+	featureController                       *controllers.FeatureController
 }
 
 // New initializes a new microcluster instance from configuration.
@@ -98,6 +108,25 @@ func New(cfg Config) (*App, error) {
 		app.triggerUpdateNodeConfigControllerCh,
 	)
 
+	app.triggerFeatureControllerNetworkCh = make(chan struct{}, 1)
+	app.triggerFeatureControllerGatewayCh = make(chan struct{}, 1)
+	app.triggerFeatureControllerIngressCh = make(chan struct{}, 1)
+	app.triggerFeatureControllerLoadBalancerCh = make(chan struct{}, 1)
+	app.triggerFeatureControllerLocalStorageCh = make(chan struct{}, 1)
+	app.triggerFeatureControllerMetricsServerCh = make(chan struct{}, 1)
+	app.triggerFeatureControllerDNSCh = make(chan struct{}, 1)
+	app.featureController = controllers.NewFeatureController(controllers.FeatureControllerOpts{
+		Snap:                   cfg.Snap,
+		WaitReady:              app.readyWg.Wait,
+		TriggerNetworkCh:       app.triggerFeatureControllerNetworkCh,
+		TriggerGatewayCh:       app.triggerFeatureControllerGatewayCh,
+		TriggerIngressCh:       app.triggerFeatureControllerIngressCh,
+		TriggerLoadBalancerCh:  app.triggerFeatureControllerLoadBalancerCh,
+		TriggerDNSCh:           app.triggerFeatureControllerDNSCh,
+		TriggerLocalStorageCh:  app.triggerFeatureControllerLocalStorageCh,
+		TriggerMetricsServerCh: app.triggerFeatureControllerMetricsServerCh,
+	})
+
 	return app, nil
 }
 
diff --git a/src/k8s/pkg/k8sd/app/hooks_bootstrap.go b/src/k8s/pkg/k8sd/app/hooks_bootstrap.go
index 6e3e22387..001dc2af0 100644
--- a/src/k8s/pkg/k8sd/app/hooks_bootstrap.go
+++ b/src/k8s/pkg/k8sd/app/hooks_bootstrap.go
@@ -13,7 +13,6 @@ import (
 
 	apiv1 "github.com/canonical/k8s/api/v1"
 	"github.com/canonical/k8s/pkg/k8sd/database"
-	"github.com/canonical/k8s/pkg/k8sd/features"
 	"github.com/canonical/k8s/pkg/k8sd/pki"
 	"github.com/canonical/k8s/pkg/k8sd/setup"
 	"github.com/canonical/k8s/pkg/k8sd/types"
@@ -345,62 +344,15 @@ func (a *App) onBootstrapControlPlane(s *state.State, bootstrapConfig apiv1.Boot
 		return fmt.Errorf("kube-apiserver did not become ready in time: %w", err)
 	}
 
-	// TODO(neoaggelos): the work below should be a POST /cluster/config
-	if cfg.Network.GetEnabled() {
-		if err := features.ApplyNetwork(s.Context, snap, cfg.Network); err != nil {
-			return fmt.Errorf("failed to apply network: %w", err)
-		}
-	}
-
-	if cfg.DNS.GetEnabled() {
-		dnsIP, err := features.ApplyDNS(s.Context, snap, cfg.DNS, cfg.Kubelet)
-		if err != nil {
-			return fmt.Errorf("failed to apply DNS: %w", err)
-		}
-
-		if dnsIP != "" {
-			if err := s.Database.Transaction(s.Context, func(ctx context.Context, tx *sql.Tx) error {
-				if cfg, err = database.SetClusterConfig(ctx, tx, types.ClusterConfig{
-					Kubelet: types.Kubelet{
-						ClusterDNS: utils.Pointer(dnsIP),
-					},
-				}); err != nil {
-					return fmt.Errorf("failed to update cluster configuration for dns=%s: %w", dnsIP, err)
-				}
-				return nil
-			}); err != nil {
-				return fmt.Errorf("database transaction to update cluster configuration failed: %w", err)
-			}
-		}
-	}
-
-	if cfg.LocalStorage.GetEnabled() {
-		if err := features.ApplyLocalStorage(s.Context, snap, cfg.LocalStorage); err != nil {
-			return fmt.Errorf("failed to apply local-storage: %w", err)
-		}
-	}
-	if cfg.Gateway.GetEnabled() {
-		if err := features.ApplyGateway(s.Context, snap, cfg.Gateway); err != nil {
-			return fmt.Errorf("failed to apply gateway: %w", err)
-		}
-	}
-	if cfg.Ingress.GetEnabled() {
-		if err := features.ApplyIngress(s.Context, snap, cfg.Ingress); err != nil {
-			return fmt.Errorf("failed to apply ingress: %w", err)
-		}
-	}
-	if cfg.LoadBalancer.GetEnabled() {
-		if err := features.ApplyLoadBalancer(s.Context, snap, cfg.LoadBalancer); err != nil {
-			return fmt.Errorf("failed to apply load-balancer: %w", err)
-		}
-	}
-	if cfg.MetricsServer.GetEnabled() {
-		if err := features.ApplyMetricsServer(s.Context, snap, cfg.MetricsServer); err != nil {
-			return fmt.Errorf("failed to apply metrics-server: %w", err)
-		}
-	}
-
-	a.NotifyNodeConfigController()
-
+	a.NotifyFeatureController(
+		cfg.Network.GetEnabled(),
+		cfg.Gateway.GetEnabled(),
+		cfg.Ingress.GetEnabled(),
+		cfg.LoadBalancer.GetEnabled(),
+		cfg.LocalStorage.GetEnabled(),
+		cfg.MetricsServer.GetEnabled(),
+		cfg.DNS.GetEnabled(),
+	)
+	a.NotifyUpdateNodeConfigController()
 	return nil
 }
diff --git a/src/k8s/pkg/k8sd/app/hooks_start.go b/src/k8s/pkg/k8sd/app/hooks_start.go
index 4381a4493..610052180 100644
--- a/src/k8s/pkg/k8sd/app/hooks_start.go
+++ b/src/k8s/pkg/k8sd/app/hooks_start.go
@@ -3,11 +3,14 @@ package app
 import (
 	"context"
 	"crypto/rsa"
+	"database/sql"
 	"fmt"
 
+	"github.com/canonical/k8s/pkg/k8sd/database"
 	databaseutil "github.com/canonical/k8s/pkg/k8sd/database/util"
 	"github.com/canonical/k8s/pkg/k8sd/pki"
 	"github.com/canonical/k8s/pkg/k8sd/types"
+	"github.com/canonical/k8s/pkg/utils"
 	"github.com/canonical/microcluster/state"
 )
 
@@ -45,5 +48,32 @@ func (a *App) onStart(s *state.State) error {
 		})
 	}
 
+	// start feature controller
+	if a.featureController != nil {
+		go a.featureController.Run(
+			s.Context,
+			func(ctx context.Context) (types.ClusterConfig, error) {
+				return databaseutil.GetClusterConfig(ctx, s)
+			},
+			func(ctx context.Context, dnsIP string) error {
+				if err := s.Database.Transaction(s.Context, func(ctx context.Context, tx *sql.Tx) error {
+					if _, err := database.SetClusterConfig(ctx, tx, types.ClusterConfig{
+						Kubelet: types.Kubelet{ClusterDNS: utils.Pointer(dnsIP)},
+					}); err != nil {
+						return fmt.Errorf("failed to update cluster configuration for dns=%s: %w", dnsIP, err)
+					}
+					return nil
+				}); err != nil {
+					return fmt.Errorf("database transaction to update cluster configuration failed: %w", err)
+				}
+
+				// DNS IP has changed, notify node config controller
+				a.NotifyUpdateNodeConfigController()
+
+				return nil
+			},
+		)
+	}
+
 	return nil
 }
diff --git a/src/k8s/pkg/k8sd/app/provider.go b/src/k8s/pkg/k8sd/app/provider.go
index b3639f6ce..5e0e6a511 100644
--- a/src/k8s/pkg/k8sd/app/provider.go
+++ b/src/k8s/pkg/k8sd/app/provider.go
@@ -3,6 +3,7 @@ package app
 import (
 	"github.com/canonical/k8s/pkg/k8sd/api"
 	"github.com/canonical/k8s/pkg/snap"
+	"github.com/canonical/k8s/pkg/utils"
 	"github.com/canonical/microcluster/microcluster"
 )
 
@@ -14,10 +15,31 @@ func (a *App) Snap() snap.Snap {
 	return a.snap
 }
 
-func (a *App) NotifyNodeConfigController() {
-	select {
-	case a.triggerUpdateNodeConfigControllerCh <- struct{}{}:
-	default:
+func (a *App) NotifyUpdateNodeConfigController() {
+	utils.MaybeNotify(a.triggerUpdateNodeConfigControllerCh)
+}
+
+func (a *App) NotifyFeatureController(network, gateway, ingress, loadBalancer, localStorage, metricsServer, dns bool) {
+	if network {
+		utils.MaybeNotify(a.triggerFeatureControllerNetworkCh)
+	}
+	if gateway {
+		utils.MaybeNotify(a.triggerFeatureControllerGatewayCh)
+	}
+	if ingress {
+		utils.MaybeNotify(a.triggerFeatureControllerIngressCh)
+	}
+	if loadBalancer {
+		utils.MaybeNotify(a.triggerFeatureControllerLoadBalancerCh)
+	}
+	if localStorage {
+		utils.MaybeNotify(a.triggerFeatureControllerLocalStorageCh)
+	}
+	if metricsServer {
+		utils.MaybeNotify(a.triggerFeatureControllerMetricsServerCh)
+	}
+	if dns {
+		utils.MaybeNotify(a.triggerFeatureControllerDNSCh)
 	}
 }
 
diff --git a/src/k8s/pkg/k8sd/controllers/feature.go b/src/k8s/pkg/k8sd/controllers/feature.go
new file mode 100644
index 000000000..e5db2f239
--- /dev/null
+++ b/src/k8s/pkg/k8sd/controllers/feature.go
@@ -0,0 +1,140 @@
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/canonical/k8s/pkg/k8sd/features"
+	"github.com/canonical/k8s/pkg/k8sd/types"
+	"github.com/canonical/k8s/pkg/snap"
+	"github.com/canonical/k8s/pkg/utils"
+)
+
+// FeatureController manages the lifecycle of built-in Canonical Kubernetes features on a running cluster.
+// The controller has separate trigger channels for each feature.
+type FeatureController struct {
+	snap      snap.Snap
+	waitReady func()
+
+	triggerNetworkCh       chan struct{}
+	triggerGatewayCh       chan struct{}
+	triggerIngressCh       chan struct{}
+	triggerLoadBalancerCh  chan struct{}
+	triggerDNSCh           chan struct{}
+	triggerLocalStorageCh  chan struct{}
+	triggerMetricsServerCh chan struct{}
+
+	reconciledNetworkCh       chan struct{}
+	reconciledGatewayCh       chan struct{}
+	reconciledIngressCh       chan struct{}
+	reconciledLoadBalancerCh  chan struct{}
+	reconciledDNSCh           chan struct{}
+	reconciledLocalStorageCh  chan struct{}
+	reconciledMetricsServerCh chan struct{}
+}
+
+type FeatureControllerOpts struct {
+	Snap      snap.Snap
+	WaitReady func()
+
+	TriggerNetworkCh       chan struct{}
+	TriggerGatewayCh       chan struct{}
+	TriggerIngressCh       chan struct{}
+	TriggerLoadBalancerCh  chan struct{}
+	TriggerDNSCh           chan struct{}
+	TriggerLocalStorageCh  chan struct{}
+	TriggerMetricsServerCh chan struct{}
+}
+
+func NewFeatureController(opts FeatureControllerOpts) *FeatureController {
+	return &FeatureController{
+		snap:                      opts.Snap,
+		waitReady:                 opts.WaitReady,
+		triggerNetworkCh:          opts.TriggerNetworkCh,
+		triggerGatewayCh:          opts.TriggerGatewayCh,
+		triggerIngressCh:          opts.TriggerIngressCh,
+		triggerLoadBalancerCh:     opts.TriggerLoadBalancerCh,
+		triggerDNSCh:              opts.TriggerDNSCh,
+		triggerLocalStorageCh:     opts.TriggerLocalStorageCh,
+		triggerMetricsServerCh:    opts.TriggerMetricsServerCh,
+		reconciledNetworkCh:       make(chan struct{}, 1),
+		reconciledGatewayCh:       make(chan struct{}, 1),
+		reconciledIngressCh:       make(chan struct{}, 1),
+		reconciledLoadBalancerCh:  make(chan struct{}, 1),
+		reconciledDNSCh:           make(chan struct{}, 1),
+		reconciledLocalStorageCh:  make(chan struct{}, 1),
+		reconciledMetricsServerCh: make(chan struct{}, 1),
+	}
+}
+
+func (c *FeatureController) Run(ctx context.Context, getClusterConfig func(context.Context) (types.ClusterConfig, error), notifyDNSChangedIP func(ctx context.Context, dnsIP string) error) {
+	c.waitReady()
+
+	go c.reconcileLoop(ctx, getClusterConfig, "network", c.triggerNetworkCh, c.reconciledNetworkCh, func(cfg types.ClusterConfig) error {
+		return features.ApplyNetwork(ctx, c.snap, cfg.Network)
+	})
+
+	go c.reconcileLoop(ctx, getClusterConfig, "gateway", c.triggerGatewayCh, c.reconciledGatewayCh, func(cfg types.ClusterConfig) error {
+		return features.ApplyGateway(ctx, c.snap, cfg.Gateway, cfg.Network)
+	})
+
+	go c.reconcileLoop(ctx, getClusterConfig, "ingress", c.triggerIngressCh, c.reconciledIngressCh, func(cfg types.ClusterConfig) error {
+		return features.ApplyIngress(ctx, c.snap, cfg.Ingress, cfg.Network)
+	})
+
+	go c.reconcileLoop(ctx, getClusterConfig, "load balancer", c.triggerLoadBalancerCh, c.reconciledLoadBalancerCh, func(cfg types.ClusterConfig) error {
+		return features.ApplyLoadBalancer(ctx, c.snap, cfg.LoadBalancer, cfg.Network)
+	})
+
+	go c.reconcileLoop(ctx, getClusterConfig, "local storage", c.triggerLocalStorageCh, c.reconciledLocalStorageCh, func(cfg types.ClusterConfig) error {
+		return features.ApplyLocalStorage(ctx, c.snap, cfg.LocalStorage)
+	})
+
+	go c.reconcileLoop(ctx, getClusterConfig, "metrics server", c.triggerMetricsServerCh, c.reconciledMetricsServerCh, func(cfg types.ClusterConfig) error {
+		return features.ApplyMetricsServer(ctx, c.snap, cfg.MetricsServer)
+	})
+
+	go c.reconcileLoop(ctx, getClusterConfig, "DNS", c.triggerDNSCh, c.reconciledDNSCh, func(cfg types.ClusterConfig) error {
+		if dnsIP, err := features.ApplyDNS(ctx, c.snap, cfg.DNS, cfg.Kubelet); err != nil {
+			return fmt.Errorf("failed to apply DNS configuration: %w", err)
+		} else if dnsIP != "" {
+			if err := notifyDNSChangedIP(ctx, dnsIP); err != nil {
+				return fmt.Errorf("failed to update DNS IP address to %s: %w", dnsIP, err)
+			}
+		}
+		return nil
+	})
+}
+
+func (c *FeatureController) reconcile(ctx context.Context, getClusterConfig func(context.Context) (types.ClusterConfig, error), apply func(cfg types.ClusterConfig) error) error {
+	cfg, err := getClusterConfig(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to retrieve cluster configuration: %w", err)
+	}
+
+	if err := apply(cfg); err != nil {
+		return fmt.Errorf("failed to apply configuration: %w", err)
+	}
+	return nil
+}
+
+func (c *FeatureController) reconcileLoop(ctx context.Context, getClusterConfig func(context.Context) (types.ClusterConfig, error), componentName string, triggerCh chan struct{}, reconciledCh chan<- struct{}, apply func(cfg types.ClusterConfig) error) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-triggerCh:
+			if err := c.reconcile(ctx, getClusterConfig, apply); err != nil {
+				log.Printf("failed to reconcile %s configuration, will retry in 5 seconds: %v", componentName, err)
+
+				// notify triggerCh after 5 seconds to retry
+				time.AfterFunc(5*time.Second, func() { utils.MaybeNotify(triggerCh) })
+			} else {
+				utils.MaybeNotify(reconciledCh)
+			}
+
+		}
+	}
+}
diff --git a/src/k8s/pkg/k8sd/features/feature_dns.go b/src/k8s/pkg/k8sd/features/feature_dns.go
index 031735f76..262f5aa7b 100644
--- a/src/k8s/pkg/k8sd/features/feature_dns.go
+++ b/src/k8s/pkg/k8sd/features/feature_dns.go
@@ -11,15 +11,15 @@ import (
 )
 
 // ApplyDNS is used to configure the DNS feature on Canonical Kubernetes.
-// ApplyDNS manages the deployment of CoreDNS, with customization options from dnsConfig and kubeletConfig which are retrieved from the cluster configuration.
-// ApplyDNS will uninstall CoreDNS from the cluster if dnsConfig.Enabled is false.
-// ApplyDNS will install or refresh CoreDNS if dnsConfig.Enabled is true.
+// ApplyDNS manages the deployment of CoreDNS, with customization options from dns and kubelet, which are retrieved from the cluster configuration.
+// ApplyDNS will uninstall CoreDNS from the cluster if dns.Enabled is false.
+// ApplyDNS will install or refresh CoreDNS if dns.Enabled is true.
 // ApplyDNS will return the ClusterIP address of the coredns service, if successful.
 // ApplyDNS returns an error if anything fails.
-func ApplyDNS(ctx context.Context, snap snap.Snap, dnsConfig types.DNS, kubeletConfig types.Kubelet) (string, error) {
+func ApplyDNS(ctx context.Context, snap snap.Snap, dns types.DNS, kubelet types.Kubelet) (string, error) {
 	m := newHelm(snap)
 
-	if !dnsConfig.GetEnabled() {
+	if !dns.GetEnabled() {
 		if _, err := m.Apply(ctx, featureCoreDNS, stateDeleted, nil); err != nil {
 			return "", fmt.Errorf("failed to uninstall coredns: %w", err)
 		}
@@ -33,7 +33,7 @@ func ApplyDNS(ctx context.Context, snap snap.Snap, dnsConfig types.DNS, kubeletC
 		},
 		"service": map[string]any{
 			"name":      "coredns",
-			"clusterIP": kubeletConfig.GetClusterDNS(),
+			"clusterIP": kubelet.GetClusterDNS(),
 		},
 		"deployment": map[string]any{
 			"name": "coredns",
@@ -50,11 +50,11 @@ func ApplyDNS(ctx context.Context, snap snap.Snap, dnsConfig types.DNS, kubeletC
 					{"name": "ready"},
 					{
 						"name":        "kubernetes",
-						"parameters":  fmt.Sprintf("%s in-addr.arpa ip6.arpa", kubeletConfig.GetClusterDomain()),
+						"parameters":  fmt.Sprintf("%s in-addr.arpa ip6.arpa", kubelet.GetClusterDomain()),
 						"configBlock": "pods insecure\nfallthrough in-addr.arpa ip6.arpa\nttl 30",
 					},
 					{"name": "prometheus", "parameters": "0.0.0.0:9153"},
-					{"name": "forward", "parameters": fmt.Sprintf(". %s", strings.Join(dnsConfig.GetUpstreamNameservers(), " "))},
+					{"name": "forward", "parameters": fmt.Sprintf(". %s", strings.Join(dns.GetUpstreamNameservers(), " "))},
 					{"name": "cache", "parameters": "30"},
 					{"name": "loop"},
 					{"name": "reload"},
diff --git a/src/k8s/pkg/k8sd/features/feature_gateway.go b/src/k8s/pkg/k8sd/features/feature_gateway.go
index 1217d0007..6b4875293 100644
--- a/src/k8s/pkg/k8sd/features/feature_gateway.go
+++ b/src/k8s/pkg/k8sd/features/feature_gateway.go
@@ -10,23 +10,23 @@ import (
 
 // ApplyGateway is used to configure the gateway feature on Canonical Kubernetes.
 // ApplyGateway assumes that the managed Cilium CNI is already installed on the cluster. It will fail if that is not the case.
-// ApplyGateway will deploy the Gateway API CRDs on the cluster and enable the GatewayAPI controllers on Cilium, when cfg.Enabled is true.
-// ApplyGateway will remove the Gateway API CRDs from the cluster and disable the GatewayAPI controllers on Cilium, when cfg.Enabled is false.
+// ApplyGateway will deploy the Gateway API CRDs on the cluster and enable the GatewayAPI controllers on Cilium, when gateway.Enabled is true.
+// ApplyGateway will remove the Gateway API CRDs from the cluster and disable the GatewayAPI controllers on Cilium, when gateway.Enabled is false.
 // ApplyGateway will rollout restart the Cilium pods in case any Cilium configuration was changed.
 // ApplyGateway returns an error if anything fails.
-func ApplyGateway(ctx context.Context, snap snap.Snap, cfg types.Gateway) error {
+func ApplyGateway(ctx context.Context, snap snap.Snap, gateway types.Gateway, network types.Network) error {
 	m := newHelm(snap)
 
-	if _, err := m.Apply(ctx, featureCiliumGateway, stateFromBool(cfg.GetEnabled()), nil); err != nil {
+	if _, err := m.Apply(ctx, featureCiliumGateway, statePresentOrDeleted(gateway.GetEnabled()), nil); err != nil {
 		return fmt.Errorf("failed to install Gateway API CRDs: %w", err)
 	}
 
-	changed, err := m.Apply(ctx, featureCiliumCNI, stateUpgradeOnly, map[string]any{"gatewayAPI": map[string]any{"enabled": cfg.GetEnabled()}})
+	changed, err := m.Apply(ctx, featureCiliumCNI, stateUpgradeOnlyOrDeleted(network.GetEnabled()), map[string]any{"gatewayAPI": map[string]any{"enabled": gateway.GetEnabled()}})
 	if err != nil {
 		return fmt.Errorf("failed to apply Gateway API cilium configuration: %w", err)
 	}
 
-	if !changed || !cfg.GetEnabled() {
+	if !changed || !gateway.GetEnabled() {
 		return nil
 	}
 	if err := rolloutRestartCilium(ctx, snap, 3); err != nil {
diff --git a/src/k8s/pkg/k8sd/features/feature_ingress.go b/src/k8s/pkg/k8sd/features/feature_ingress.go
index 795f88789..2b590e46b 100644
--- a/src/k8s/pkg/k8sd/features/feature_ingress.go
+++ b/src/k8s/pkg/k8sd/features/feature_ingress.go
@@ -10,22 +10,22 @@ import (
 
 // ApplyIngress is used to configure the ingress controller feature on Canonical Kubernetes.
 // ApplyIngress assumes that the managed Cilium CNI is already installed on the cluster. It will fail if that is not the case.
-// ApplyIngress will enable Cilium's ingress controller when cfg.Enabled is true.
-// ApplyIngress will disable Cilium's ingress controller when cfg.Disabled is false.
+// ApplyIngress will enable Cilium's ingress controller when ingress.Enabled is true.
+// ApplyIngress will disable Cilium's ingress controller when ingress.Disabled is false.
 // ApplyIngress will rollout restart the Cilium pods in case any Cilium configuration was changed.
 // ApplyIngress returns an error if anything fails.
-func ApplyIngress(ctx context.Context, snap snap.Snap, cfg types.Ingress) error {
+func ApplyIngress(ctx context.Context, snap snap.Snap, ingress types.Ingress, network types.Network) error {
 	m := newHelm(snap)
 
 	var values map[string]any
-	if cfg.GetEnabled() {
+	if ingress.GetEnabled() {
 		values = map[string]any{
 			"ingressController": map[string]any{
 				"enabled":                true,
 				"loadbalancerMode":       "shared",
 				"defaultSecretNamespace": "kube-system",
-				"defaultTLSSecret":       cfg.GetDefaultTLSSecret(),
-				"enableProxyProtocol":    cfg.GetEnableProxyProtocol(),
+				"defaultTLSSecret":       ingress.GetDefaultTLSSecret(),
+				"enableProxyProtocol":    ingress.GetEnableProxyProtocol(),
 			},
 		}
 	} else {
@@ -39,11 +39,11 @@ func ApplyIngress(ctx context.Context, snap snap.Snap, cfg types.Ingress) error
 		}
 	}
 
-	changed, err := m.Apply(ctx, featureCiliumCNI, stateUpgradeOnly, values)
+	changed, err := m.Apply(ctx, featureCiliumCNI, stateUpgradeOnlyOrDeleted(network.GetEnabled()), values)
 	if err != nil {
 		return fmt.Errorf("failed to enable ingress: %w", err)
 	}
-	if !changed || !cfg.GetEnabled() {
+	if !changed || !ingress.GetEnabled() {
 		return nil
 	}
 
diff --git a/src/k8s/pkg/k8sd/features/feature_loadbalancer.go b/src/k8s/pkg/k8sd/features/feature_loadbalancer.go
index 5bdd41e15..cb7d912ef 100644
--- a/src/k8s/pkg/k8sd/features/feature_loadbalancer.go
+++ b/src/k8s/pkg/k8sd/features/feature_loadbalancer.go
@@ -12,25 +12,25 @@ import (
 
 // ApplyLoadBalancer is used to configure the load-balancer feature on Canonical Kubernetes.
 // ApplyLoadBalancer assumes that the managed Cilium CNI is already installed on the cluster. It will fail if that is not the case.
-// ApplyLoadBalancer will configure Cilium to enable L2 or BGP mode, and deploy necessary CRs for announcing the LoadBalancer external IPs when cfg.Enabled is true.
-// ApplyLoadBalancer will disable L2 and BGP on Cilium, and remove any previously created CRs when cfg.Enabled is false.
+// ApplyLoadBalancer will configure Cilium to enable L2 or BGP mode, and deploy necessary CRs for announcing the LoadBalancer external IPs when loadbalancer.Enabled is true.
+// ApplyLoadBalancer will disable L2 and BGP on Cilium, and remove any previously created CRs when loadbalancer.Enabled is false.
 // ApplyLoadBalancer will rollout restart the Cilium pods in case any Cilium configuration was changed.
 // ApplyLoadBalancer returns an error if anything fails.
-func ApplyLoadBalancer(ctx context.Context, snap snap.Snap, cfg types.LoadBalancer) error {
-	if !cfg.GetEnabled() {
-		if err := disableLoadBalancer(ctx, snap); err != nil {
+func ApplyLoadBalancer(ctx context.Context, snap snap.Snap, loadbalancer types.LoadBalancer, network types.Network) error {
+	if !loadbalancer.GetEnabled() {
+		if err := disableLoadBalancer(ctx, snap, network); err != nil {
 			return fmt.Errorf("failed to disable LoadBalancer: %w", err)
 		}
 		return nil
 	}
 
-	if err := enableLoadBalancer(ctx, snap, cfg); err != nil {
+	if err := enableLoadBalancer(ctx, snap, loadbalancer, network); err != nil {
 		return fmt.Errorf("failed to enable LoadBalancer: %w", err)
 	}
 	return nil
 }
 
-func disableLoadBalancer(ctx context.Context, snap snap.Snap) error {
+func disableLoadBalancer(ctx context.Context, snap snap.Snap, network types.Network) error {
 	m := newHelm(snap)
 
 	if _, err := m.Apply(ctx, featureCiliumLoadBalancer, stateDeleted, nil); err != nil {
@@ -55,21 +55,21 @@ func disableLoadBalancer(ctx context.Context, snap snap.Snap) error {
 		},
 	}
 
-	if _, err := m.Apply(ctx, featureCiliumCNI, stateUpgradeOnly, values); err != nil {
+	if _, err := m.Apply(ctx, featureCiliumCNI, stateUpgradeOnlyOrDeleted(network.GetEnabled()), values); err != nil {
 		return fmt.Errorf("failed to refresh network to apply LoadBalancer configuration: %w", err)
 	}
 	return nil
 }
 
-func enableLoadBalancer(ctx context.Context, snap snap.Snap, cfg types.LoadBalancer) error {
+func enableLoadBalancer(ctx context.Context, snap snap.Snap, loadbalancer types.LoadBalancer, network types.Network) error {
 	m := newHelm(snap)
 
 	networkValues := map[string]any{
 		"l2announcements": map[string]any{
-			"enabled": cfg.GetL2Mode(),
+			"enabled": loadbalancer.GetL2Mode(),
 		},
 		"bgpControlPlane": map[string]any{
-			"enabled": cfg.GetBGPMode(),
+			"enabled": loadbalancer.GetBGPMode(),
 		},
 		"externalIPs": map[string]any{
 			"enabled": true,
@@ -82,39 +82,39 @@ func enableLoadBalancer(ctx context.Context, snap snap.Snap, cfg types.LoadBalan
 		},
 	}
 
-	changed, err := m.Apply(ctx, featureCiliumCNI, stateUpgradeOnly, networkValues)
+	changed, err := m.Apply(ctx, featureCiliumCNI, stateUpgradeOnlyOrDeleted(network.GetEnabled()), networkValues)
 	if err != nil {
 		return fmt.Errorf("failed to update Cilium configuration for LoadBalancer: %w", err)
 	}
 
-	if err := waitForRequiredLoadBalancerCRDs(ctx, snap, cfg.GetBGPMode()); err != nil {
+	if err := waitForRequiredLoadBalancerCRDs(ctx, snap, loadbalancer.GetBGPMode()); err != nil {
 		return fmt.Errorf("failed to wait for required Cilium CRDs to be available: %w", err)
 	}
 
 	cidrs := []map[string]any{}
-	for _, cidr := range cfg.GetCIDRs() {
+	for _, cidr := range loadbalancer.GetCIDRs() {
 		cidrs = append(cidrs, map[string]any{"cidr": cidr})
 	}
-	for _, ipRange := range cfg.GetIPRanges() {
+	for _, ipRange := range loadbalancer.GetIPRanges() {
 		cidrs = append(cidrs, map[string]any{"start": ipRange.Start, "stop": ipRange.Stop})
 	}
 
 	values := map[string]any{
 		"l2": map[string]any{
-			"enabled":    cfg.GetL2Mode(),
-			"interfaces": cfg.GetL2Interfaces(),
+			"enabled":    loadbalancer.GetL2Mode(),
+			"interfaces": loadbalancer.GetL2Interfaces(),
 		},
 		"ipPool": map[string]any{
 			"cidrs": cidrs,
 		},
 		"bgp": map[string]any{
-			"enabled":  cfg.GetBGPMode(),
-			"localASN": cfg.GetBGPLocalASN(),
+			"enabled":  loadbalancer.GetBGPMode(),
+			"localASN": loadbalancer.GetBGPLocalASN(),
 			"neighbors": []map[string]any{
 				{
-					"peerAddress": cfg.GetBGPPeerAddress(),
-					"peerASN":     cfg.GetBGPPeerASN(),
-					"peerPort":    cfg.GetBGPPeerPort(),
+					"peerAddress": loadbalancer.GetBGPPeerAddress(),
+					"peerASN":     loadbalancer.GetBGPPeerASN(),
+					"peerPort":    loadbalancer.GetBGPPeerPort(),
 				},
 			},
 		},
diff --git a/src/k8s/pkg/k8sd/features/feature_local_storage.go b/src/k8s/pkg/k8sd/features/feature_local_storage.go
index 0a666f378..3815607fc 100644
--- a/src/k8s/pkg/k8sd/features/feature_local_storage.go
+++ b/src/k8s/pkg/k8sd/features/feature_local_storage.go
@@ -41,6 +41,6 @@ func ApplyLocalStorage(ctx context.Context, snap snap.Snap, cfg types.LocalStora
 		},
 	}
 
-	_, err := m.Apply(ctx, featureLocalStorage, stateFromBool(cfg.GetEnabled()), values)
+	_, err := m.Apply(ctx, featureLocalStorage, statePresentOrDeleted(cfg.GetEnabled()), values)
 	return err
 }
diff --git a/src/k8s/pkg/k8sd/features/feature_metrics_server.go b/src/k8s/pkg/k8sd/features/feature_metrics_server.go
index 71a31d279..83061cbf0 100644
--- a/src/k8s/pkg/k8sd/features/feature_metrics_server.go
+++ b/src/k8s/pkg/k8sd/features/feature_metrics_server.go
@@ -25,6 +25,6 @@ func ApplyMetricsServer(ctx context.Context, snap snap.Snap, cfg types.MetricsSe
 		},
 	}
 
-	_, err := m.Apply(ctx, featureMetricsServer, stateFromBool(cfg.GetEnabled()), values)
+	_, err := m.Apply(ctx, featureMetricsServer, statePresentOrDeleted(cfg.GetEnabled()), values)
 	return err
 }
diff --git a/src/k8s/pkg/k8sd/features/state.go b/src/k8s/pkg/k8sd/features/state.go
index 2f39afb2f..46f291855 100644
--- a/src/k8s/pkg/k8sd/features/state.go
+++ b/src/k8s/pkg/k8sd/features/state.go
@@ -14,9 +14,16 @@ const (
 	stateUpgradeOnly
 )
 
-func stateFromBool(enabled bool) state {
+func statePresentOrDeleted(enabled bool) state {
 	if enabled {
 		return statePresent
 	}
 	return stateDeleted
 }
+
+func stateUpgradeOnlyOrDeleted(enabled bool) state {
+	if enabled {
+		return stateUpgradeOnly
+	}
+	return stateDeleted
+}
diff --git a/src/k8s/pkg/utils/chan.go b/src/k8s/pkg/utils/chan.go
new file mode 100644
index 000000000..4920a1de1
--- /dev/null
+++ b/src/k8s/pkg/utils/chan.go
@@ -0,0 +1,9 @@
+package utils
+
+// MaybeNotify pushes an empty struct to a channel, but does not block if that fails.
+func MaybeNotify(ch chan<- struct{}) {
+	select {
+	case ch <- struct{}{}:
+	default:
+	}
+}
diff --git a/tests/integration/tests/test_gateway.py b/tests/integration/tests/test_gateway.py
index 93faa65d0..0d3fd68e9 100644
--- a/tests/integration/tests/test_gateway.py
+++ b/tests/integration/tests/test_gateway.py
@@ -16,38 +16,6 @@ def test_gateway(instances: List[harness.Instance]):
     util.wait_for_network(instance)
     util.wait_for_dns(instance)
 
-    util.stubbornly(retries=3, delay_s=1).on(instance).exec(
-        [
-            "k8s",
-            "kubectl",
-            "wait",
-            "--for=condition=ready",
-            "pod",
-            "-n",
-            "kube-system",
-            "-l",
-            "io.cilium/app=operator",
-            "--timeout",
-            "180s",
-        ]
-    )
-
-    util.stubbornly(retries=3, delay_s=1).on(instance).exec(
-        [
-            "k8s",
-            "kubectl",
-            "wait",
-            "--for=condition=ready",
-            "pod",
-            "-n",
-            "kube-system",
-            "-l",
-            "k8s-app=cilium",
-            "--timeout",
-            "180s",
-        ]
-    )
-
     manifest = MANIFESTS_DIR / "gateway-test.yaml"
     instance.exec(
         ["k8s", "kubectl", "apply", "-f", "-"],
diff --git a/tests/integration/tests/test_ingress.py b/tests/integration/tests/test_ingress.py
index bfa19bc8c..71c4801f3 100644
--- a/tests/integration/tests/test_ingress.py
+++ b/tests/integration/tests/test_ingress.py
@@ -37,38 +37,6 @@ def test_ingress(instances: List[harness.Instance]):
     )
     ingress_http_port = p.stdout.decode().replace("'", "")
 
-    util.stubbornly(retries=3, delay_s=1).on(instance).exec(
-        [
-            "k8s",
-            "kubectl",
-            "wait",
-            "--for=condition=ready",
-            "pod",
-            "-n",
-            "kube-system",
-            "-l",
-            "io.cilium/app=operator",
-            "--timeout",
-            "180s",
-        ]
-    )
-
-    util.stubbornly(retries=3, delay_s=1).on(instance).exec(
-        [
-            "k8s",
-            "kubectl",
-            "wait",
-            "--for=condition=ready",
-            "pod",
-            "-n",
-            "kube-system",
-            "-l",
-            "k8s-app=cilium",
-            "--timeout",
-            "180s",
-        ]
-    )
-
     manifest = MANIFESTS_DIR / "ingress-test.yaml"
     instance.exec(
         ["k8s", "kubectl", "apply", "-f", "-"],
diff --git a/tests/integration/tests/test_loadbalancer.py b/tests/integration/tests/test_loadbalancer.py
index 8e7901faf..9f882d7ed 100644
--- a/tests/integration/tests/test_loadbalancer.py
+++ b/tests/integration/tests/test_loadbalancer.py
@@ -57,37 +57,8 @@ def test_loadbalancer(instances: List[harness.Instance]):
     )
     instance.exec(["k8s", "enable", "load-balancer"])
 
-    util.stubbornly(retries=3, delay_s=1).on(instance).exec(
-        [
-            "k8s",
-            "kubectl",
-            "wait",
-            "--for=condition=ready",
-            "pod",
-            "-n",
-            "kube-system",
-            "-l",
-            "io.cilium/app=operator",
-            "--timeout",
-            "180s",
-        ]
-    )
-
-    util.stubbornly(retries=3, delay_s=1).on(instance).exec(
-        [
-            "k8s",
-            "kubectl",
-            "wait",
-            "--for=condition=ready",
-            "pod",
-            "-n",
-            "kube-system",
-            "-l",
-            "k8s-app=cilium",
-            "--timeout",
-            "180s",
-        ]
-    )
+    util.wait_for_network(instance)
+    util.wait_for_dns(instance)
 
     manifest = MANIFESTS_DIR / "loadbalancer-test.yaml"
     instance.exec(

From 069fac96b2f4885ac2047f02b03fee267e5b7066 Mon Sep 17 00:00:00 2001
From: Benjamin Schimke <benjamin.schimke@canonical.com>
Date: Fri, 26 Apr 2024 15:51:51 +0200
Subject: [PATCH 4/6] Fix branch name for k8s-dqlite in template (#381)

---
 .github/ISSUE_TEMPLATE/create_release_branch.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/create_release_branch.md b/.github/ISSUE_TEMPLATE/create_release_branch.md
index fe63cb483..05e1c4d5d 100644
--- a/.github/ISSUE_TEMPLATE/create_release_branch.md
+++ b/.github/ISSUE_TEMPLATE/create_release_branch.md
@@ -52,7 +52,7 @@ The steps are to be followed in-order, each task must be completed by the person
   - `git pull`
   - `git checkout -b release-1.xx`
   - `git push origin release-1.xx`
-- [ ] **Owner**: Create `1.xx` branch from latest `master` in k8s-dqlite
+- [ ] **Owner**: Create `release-1.xx` branch from latest `master` in k8s-dqlite
   - `git clone git@github.com:canonical/k8s-dqlite.git ~/tmp/release-1.xx`
   - `pushd ~/tmp/release-1.xx`
   - `git switch main`
@@ -101,7 +101,7 @@ The steps are to be followed in-order, each task must be completed by the person
 - [ ] **Reviewer**: Ensure `release-1.xx-strict` branch is based on latest changes on `autoupdate/strict` at the time of the release cut.
 - [ ] **Owner**: Create PR to initialize `release-1.xx` branch:
   - [ ] Update `KUBE_TRACK` to `1.xx` in [/build-scripts/components/kubernetes/version.sh][]
-  - [ ] Update `master` to `1.xx` in [/build-scripts/components/k8s-dqlite/version.sh][]
+  - [ ] Update `master` to `release-1.xx` in [/build-scripts/components/k8s-dqlite/version.sh][]
   - [ ] Update `"main"` to `"release-1.xx"` in [/build-scripts/hack/generate-sbom.py][]
   - [ ] `git commit -m 'Release 1.xx'`
   - [ ] Create PR with the changes and request review from **Reviewer**. Make sure to update the issue `Information` section with a link to the PR.

From ea6b8fc3bfe6d3c51b0b70bec735ae28feb73f14 Mon Sep 17 00:00:00 2001
From: Nick Veitch <nick.veitch@canonical.com>
Date: Fri, 26 Apr 2024 14:56:33 +0100
Subject: [PATCH 5/6] Add docs templates (#377)

* Add templates

---------

Co-authored-by: eaudetcobello <155978570+eaudetcobello@users.noreply.github.com>
---
 docs/src/_parts/template-explanation | 27 ++++++++
 docs/src/_parts/template-howto       | 62 ++++++++++++++++++
 docs/src/_parts/template-reference   | 84 ++++++++++++++++++++++++
 docs/src/_parts/template-tutorial    | 97 ++++++++++++++++++++++++++++
 docs/src/snap/howto/contribute.md    | 12 ++++
 5 files changed, 282 insertions(+)
 create mode 100644 docs/src/_parts/template-explanation
 create mode 100644 docs/src/_parts/template-howto
 create mode 100644 docs/src/_parts/template-reference
 create mode 100644 docs/src/_parts/template-tutorial

diff --git a/docs/src/_parts/template-explanation b/docs/src/_parts/template-explanation
new file mode 100644
index 000000000..905a30f07
--- /dev/null
+++ b/docs/src/_parts/template-explanation
@@ -0,0 +1,27 @@
+# Explanation docs template
+
+> Explanations are for learning and understanding
+
+There is no set format to an explanation document. Good examples consist of
+easy to understand text and are quite frequently accompanied by diagrams.
+
+## Diagrams
+
+You can include a diagram as an image using the usual Markdown format:
+
+![Illustration depicting working on components and clouds][logo]
+
+The documentation also supports various diagrams-as-code options. We
+prefer to use UML-style diagrams, but you can also use Mermaid or many
+other types.
+
+Diagrams like this are processed using the 'kroki' directive:
+
+```{kroki} ../../assets/ck-cluster.puml
+```
+
+## Links
+
+Explanations frequently include links to other documents. In particular, please
+consider adding a section titled 'Further Reading' at the end to collate
+related topics (internal or external).
diff --git a/docs/src/_parts/template-howto b/docs/src/_parts/template-howto
new file mode 100644
index 000000000..9afdcd691
--- /dev/null
+++ b/docs/src/_parts/template-howto
@@ -0,0 +1,62 @@
+# How to docs template
+
+> A How to guide consists of task-oriented, step-by-step instructions to
+> achieve a specific goal.
+> The title of the document should explain a clear objective.
+
+# How to use default DNS
+
+> Add an introductory paragraph explaining the objective and any background 
+> necessary.
+
+Canonical Kubernetes includes a default DNS (Domain Name System) which is
+essential for internal cluster communication. When enabled, the DNS facilitates
+service discovery by assigning each service a DNS name. When disabled, you can
+integrate a custom DNS solution into your cluster.
+
+
+> ALWAYS start with a list of requirements/assumptions. Link to other docs
+> if helpful
+
+## What you'll need
+
+This guide assumes the following:
+
+- You have root or sudo access to the machine.
+- You have a bootstrapped Canonical Kubernetes cluster (see the [Getting
+  Started][getting-started-guide] guide).
+
+> Proceed with clearly labelled steps. N.B. Use 'imperatives' rather than
+> gerunds ('Enable XXX' rather than 'Enabling XXX', 'Check' rather than
+> 'Checking')
+
+## Check DNS status
+
+> ALWAYS have some text directly after each heading
+
+Find out whether DNS is enabled or disabled with the following command:
+
+
+> Commands are identified by three backticks to start and end the sequence. No
+> need to specify a code type for normal bash commands. DO NOT include '#' or
+> '$' prompts
+
+> ALWAYS separate input and output
+
+```
+sudo k8s status
+```
+
+> Proceed with steps until the task is complete. Consider whether links to related
+> Guides or resources are appropriate
+
+## See also
+
+- [How to xxx with yyy][]
+- [Get more out of zzz][]
+
+> ALWAYS use links by reference for easier maintenance!
+
+<!-- LINKS -->
+
+[getting-started-guide]: /snap/tutorial/getting-started
diff --git a/docs/src/_parts/template-reference b/docs/src/_parts/template-reference
new file mode 100644
index 000000000..8386a91d5
--- /dev/null
+++ b/docs/src/_parts/template-reference
@@ -0,0 +1,84 @@
+# Reference docs template
+
+> Reference guides are technical descriptions of the machinery.
+
+The reference section is distinct from the How to section in that it provides
+information without demonstrating in any depth how that information is to be used.
+What's the point of that?
+
+Often users may need to check some information - "What release included support
+for xxxx?" or "What options can I supply to yyyy". 
+
+The reference section is also distinct from the explanation section in that it
+does not attempt to give any detailed explanation of why this information
+exists or how it is to be used.
+
+Consider an application which deals with many different image formats. A useful
+reference page would list the filetypes supported and maybe a list of common
+file extensions and perhaps a brief decscription:
+
+| FileType | Extensions | Notes                                              |
+|----------|------------|----------------------------------------------------|
+| TIFF     | .tif,.tiff | Tag Image File Format - lossless bitmaps           |
+| JPEG     | .jpg,.jpeg | Compressed photo-quality images                    |
+| PNG      | .png       | Portable Network Graphics - lossless bitmaps        |
+
+This table gives some reference information. It doesn't try to tell you which
+format to use. It doesn't explain in great detail the differences between them.
+It doesn't guide you on how to use them. It is simply a reference. You could use
+it to check whether the application can import or export files in a particular
+format.
+
+## Presentation
+
+As the reference guide is usually about collections of small piece of data,
+there are some specific ways of formatting this information which can be useful
+
+We have already seen the table style. In other situations it may be more
+relevant to use a simple bullet list:
+
+
+- TIFF (.tif,.tiff)
+- PNG (.png)
+- JPEG (.jpg, .jpeg)
+
+Or definition lists:
+
+
+TIFF
+: Tag Image File Format
+
+JPEG
+: Joint Photographic Experts Group
+
+## Automation
+
+For some types of information it may be more expedient to generate details or
+text automatically through a script. there are a few ways this can then be used
+in documentation.
+
+For example, in the command reference we use:
+
+```{include} ../../_parts/commands/k8s_config.md
+   :end-before: '### SEE ALSO'
+```
+which automatically includes the contents of the file (note you can also use a
+simple :start-after: and :end-before: option to filter out parts of the
+original)
+
+Pages can filter information from other sources to, for example:
+
+``` {csv-table} Canonical Kubernetes public roadmap
+   :file: ../../assets/roadmap.csv
+   :widths: 30, 30
+   :header-rows: 1
+```
+
+This defines an input from a CSV file.
+
+## Tips
+
+- Remember this is about presenting information.
+- People are unlikely to read the entire page in one go - use headings, lists,
+  tables etc to direct them to the information they are looking for.
+- Resist the temptation to over-explain, but do provide links to documents which do!
diff --git a/docs/src/_parts/template-tutorial b/docs/src/_parts/template-tutorial
new file mode 100644
index 000000000..039f0ee52
--- /dev/null
+++ b/docs/src/_parts/template-tutorial
@@ -0,0 +1,97 @@
+# Tutorial docs template
+
+> Tutorials are explained guides which fulfil the objectives of learning and
+understanding through example.
+
+>Unlike some of the other documentation types, tutorials are structured and easy
+to provide a definitive template.
+
+> Tutorials should start with a short introductory statement outlining the
+objective of the tutorial. For example:
+
+Installing Canonical Kubernetes should only take a few minutes. This tutorial
+explains how to install the snap package and some typical operations. 
+
+> For lengthier tutorials, it is worth considering adding a bullet list
+detailing this:
+
+## What you will learn
+
+- How to install the k8s snap
+- Configuring storage
+- Deploying an application to the cluster
+- Removing an application
+- Turning off the cluster
+
+> Next, ALL tutorials should include a section called 'What you will need'. This
+> is a list (with links if necessary) that covers the requirements to start the
+> tutorial. This could include things like system requirements, assumptions about
+> software already installed and even knowledge. This is essentially the list of
+> assumptions we are making
+
+## What you will need
+
+- Ubuntu 22.04 LTS or 20.04 LTS
+- At least 20G disk space and 4G of memory
+- The [Juju-client]
+
+> The tutorial should proceed with numbered steps if possible, showing the
+> sequence of the tutorial.
+
+### 1. Install Canonical Kubernetes
+
+Install the Canonical Kubernetes snap with:
+
+```
+sudo snap install k8s --edge --classic
+```
+
+> Remember this is a tutorial. It is fine to add words to explain what is going
+on, why we are doing particular things, and what results one might expect. In
+some cases you may even want to deliberately manufacture an error to better
+illustrate how things work and how to solve problems.
+
+> It is often a good idea to include a summarising statement at the end,
+detailing the things the reader should now know about. This isn't entirely a
+self-congratulatory exercise - it is an aid to learning an retention. It
+doesn't have to be long or detailed. For example:
+
+---
+
+Congratulations you have now completed the tutorial. We found out how to
+install the Kubernetes snap, how to change the configuration, deploy a
+workload and remove everything again!
+
+> Almost without exception you will then want to point the reader to other
+> tutorials or how to guides. Please include a "Next Steps" heading with a list
+> of links.
+
+## Next Steps
+
+- Keep mastering Canonical Kubernetes with kubectl: [How to use kubectl]
+- Explore Kubernetes commands with our [Command Reference Guide]
+- Learn how to set up a multi-node environment [Setting up a K8s cluster]
+- Configure storage options [Storage]
+- Master Kubernetes networking concepts: [Networking]
+- Discover how to enable and configure Ingress resources [Ingress]
+
+---
+
+> Note that the links throughout should always be by reference. Your document
+should end with a list of these links. This makes it 2000 times easier to
+maintain the document and update the links when necessary!
+
+Note the different type of links:
+
+- for pages in the same category, you can just use the name (not including the
+  file extension)
+- for pages in a different category, it is better to use the full path(relative
+  to the /docs/src/)
+- external links are straightforward
+
+<!-- LINKS -->
+
+[How to use kubectl]: kubectl
+[Command Reference Guide]: /snap/reference/commands
+[Ingress]: http://www.example.com
+
diff --git a/docs/src/snap/howto/contribute.md b/docs/src/snap/howto/contribute.md
index 5e3c729e3..3005d731d 100644
--- a/docs/src/snap/howto/contribute.md
+++ b/docs/src/snap/howto/contribute.md
@@ -97,6 +97,14 @@ Every page of documentation should fit into one of those categories. If it
 doesn't you may consider if it is actually two pages (e.g. a How to *and* an
 explanation).
 
+We have included some tips and outlines of the different types of docs we 
+create to help you get started:
+
+- [Tutorial template][]
+- [How to template][]
+- [Explanation template][]
+- [Reference template][]
+
 ### Small changes
 
 If you are simply correcting a typo or updating a link, you can follow the
@@ -140,3 +148,7 @@ press `F5` in your browser to reload the page without caching)!
 [Diátaxis website]: https://diataxis.fr/
 [_parts]: https://github.com/canonical/k8s-snap/blob/main/docs/src/_parts/doc-cheat-sheet-myst.md
 [community page]: ../reference/community
+[Tutorial template]: https://raw.githubusercontent.com/canonical/k8s-snap/main/docs/src/_parts/template-tutorial
+[How to template]: https://raw.githubusercontent.com/canonical/k8s-snap/main/docs/src/_parts/template-howto
+[Explanation template]: https://raw.githubusercontent.com/canonical/k8s-snap/main/docs/src/_parts/template-explanation
+[Reference template]: https://raw.githubusercontent.com/canonical/k8s-snap/main/docs/src/_parts/template-reference

From 7cd6466232c9b001edeaefb472d04e5da34237fd Mon Sep 17 00:00:00 2001
From: Angelos Kolaitis <angelos.kolaitis@canonical.com>
Date: Mon, 29 Apr 2024 15:37:10 +0300
Subject: [PATCH 6/6] Adjust snap.Interface to return Kubernetes clients (#384)

* move pkg/utils/k8s -> pkg/client/kubernetes

* adjust snap to return Kubernetes clients instead of config flags

* Move helm client logic to pkg/client/helm package

* adjust snap interface to return a HelmClient

* add unit test for features.ApplyMetricsServer
---
 src/k8s/pkg/client/helm/chart.go              | 14 ++++
 .../manager.go => client/helm/client.go}      | 71 +++++++++----------
 src/k8s/pkg/client/helm/interface.go          | 13 ++++
 src/k8s/pkg/client/helm/mock/mock.go          | 29 ++++++++
 src/k8s/pkg/client/helm/state.go              | 29 ++++++++
 .../k8s => client/kubernetes}/client.go       |  2 +-
 .../k8s => client/kubernetes}/configmap.go    |  2 +-
 .../kubernetes}/configmap_test.go             |  2 +-
 .../k8s => client/kubernetes}/endpoints.go    |  2 +-
 .../kubernetes}/endpoints_test.go             |  2 +-
 .../{utils/k8s => client/kubernetes}/node.go  |  2 +-
 .../k8s => client/kubernetes}/node_test.go    |  2 +-
 .../kubernetes}/restart_daemonset.go          |  2 +-
 .../kubernetes}/restart_daemonset_test.go     |  2 +-
 .../kubernetes}/restart_deployment.go         |  2 +-
 .../kubernetes}/restart_deployment_test.go    |  2 +-
 .../kubernetes}/server_groups.go              |  2 +-
 .../kubernetes}/server_groups_test.go         |  6 +-
 .../k8s => client/kubernetes}/services.go     |  2 +-
 .../kubernetes}/services_test.go              |  2 +-
 .../k8s => client/kubernetes}/status.go       |  2 +-
 .../k8s => client/kubernetes}/status_test.go  |  2 +-
 src/k8s/pkg/k8sd/api/cluster.go               |  6 +-
 src/k8s/pkg/k8sd/api/cluster_remove.go        |  3 +-
 src/k8s/pkg/k8sd/api/worker.go                |  3 +-
 src/k8s/pkg/k8sd/app/app.go                   |  7 --
 src/k8s/pkg/k8sd/app/cluster_util.go          |  6 +-
 src/k8s/pkg/k8sd/app/hooks_join.go            |  6 +-
 .../k8sd/controllers/node_configuration.go    | 18 +++--
 .../controllers/node_configuration_test.go    | 29 ++++----
 .../controllers/update_node_configuration.go  | 20 +++---
 .../update_node_configuration_test.go         | 26 +++----
 src/k8s/pkg/k8sd/features/feature_dns.go      | 10 +--
 src/k8s/pkg/k8sd/features/feature_gateway.go  |  7 +-
 src/k8s/pkg/k8sd/features/feature_ingress.go  |  5 +-
 .../pkg/k8sd/features/feature_loadbalancer.go | 16 ++---
 .../k8sd/features/feature_local_storage.go    |  5 +-
 .../k8sd/features/feature_metrics_server.go   |  5 +-
 .../features/feature_metrics_server_test.go   | 56 +++++++++++++++
 src/k8s/pkg/k8sd/features/feature_network.go  | 10 +--
 src/k8s/pkg/k8sd/features/features.go         | 66 +++++++++--------
 src/k8s/pkg/k8sd/features/interface.go        | 26 -------
 src/k8s/pkg/k8sd/features/state.go            | 29 --------
 src/k8s/pkg/snap/interface.go                 |  9 +--
 src/k8s/pkg/snap/mock/mock.go                 | 71 ++++++++++---------
 src/k8s/pkg/snap/snap.go                      | 33 +++++----
 46 files changed, 371 insertions(+), 295 deletions(-)
 create mode 100644 src/k8s/pkg/client/helm/chart.go
 rename src/k8s/pkg/{k8sd/features/manager.go => client/helm/client.go} (58%)
 create mode 100644 src/k8s/pkg/client/helm/interface.go
 create mode 100644 src/k8s/pkg/client/helm/mock/mock.go
 create mode 100644 src/k8s/pkg/client/helm/state.go
 rename src/k8s/pkg/{utils/k8s => client/kubernetes}/client.go (96%)
 rename src/k8s/pkg/{utils/k8s => client/kubernetes}/configmap.go (98%)
 rename src/k8s/pkg/{utils/k8s => client/kubernetes}/configmap_test.go (99%)
 rename src/k8s/pkg/{utils/k8s => client/kubernetes}/endpoints.go (98%)
 rename src/k8s/pkg/{utils/k8s => client/kubernetes}/endpoints_test.go (99%)
 rename src/k8s/pkg/{utils/k8s => client/kubernetes}/node.go (96%)
 rename src/k8s/pkg/{utils/k8s => client/kubernetes}/node_test.go (98%)
 rename src/k8s/pkg/{utils/k8s => client/kubernetes}/restart_daemonset.go (98%)
 rename src/k8s/pkg/{utils/k8s => client/kubernetes}/restart_daemonset_test.go (98%)
 rename src/k8s/pkg/{utils/k8s => client/kubernetes}/restart_deployment.go (98%)
 rename src/k8s/pkg/{utils/k8s => client/kubernetes}/restart_deployment_test.go (98%)
 rename src/k8s/pkg/{utils/k8s => client/kubernetes}/server_groups.go (96%)
 rename src/k8s/pkg/{utils/k8s => client/kubernetes}/server_groups_test.go (93%)
 rename src/k8s/pkg/{utils/k8s => client/kubernetes}/services.go (96%)
 rename src/k8s/pkg/{utils/k8s => client/kubernetes}/services_test.go (98%)
 rename src/k8s/pkg/{utils/k8s => client/kubernetes}/status.go (98%)
 rename src/k8s/pkg/{utils/k8s => client/kubernetes}/status_test.go (99%)
 create mode 100644 src/k8s/pkg/k8sd/features/feature_metrics_server_test.go
 delete mode 100644 src/k8s/pkg/k8sd/features/interface.go
 delete mode 100644 src/k8s/pkg/k8sd/features/state.go

diff --git a/src/k8s/pkg/client/helm/chart.go b/src/k8s/pkg/client/helm/chart.go
new file mode 100644
index 000000000..0357af3dc
--- /dev/null
+++ b/src/k8s/pkg/client/helm/chart.go
@@ -0,0 +1,14 @@
+package helm
+
+// InstallableChart describes a chart that can be deployed on a running cluster.
+type InstallableChart struct {
+	// Name is the install name of the chart.
+	Name string
+
+	// Namespace is the namespace to install the chart.
+	Namespace string
+
+	// ManifestPath is the path to the chart's manifest, typically relative to "$SNAP/k8s/manifests".
+	// TODO(neoaggelos): this should be a *chart.Chart, and we should use the "embed" package to load it when building k8sd.
+	ManifestPath string
+}
diff --git a/src/k8s/pkg/k8sd/features/manager.go b/src/k8s/pkg/client/helm/client.go
similarity index 58%
rename from src/k8s/pkg/k8sd/features/manager.go
rename to src/k8s/pkg/client/helm/client.go
index 2b70d9ae7..99e4d9465 100644
--- a/src/k8s/pkg/k8sd/features/manager.go
+++ b/src/k8s/pkg/client/helm/client.go
@@ -1,4 +1,4 @@
-package features
+package helm
 
 import (
 	"bytes"
@@ -8,31 +8,30 @@ import (
 	"log"
 	"path"
 
-	"github.com/canonical/k8s/pkg/snap"
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/chart/loader"
 	"helm.sh/helm/v3/pkg/storage/driver"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 )
 
-// helmManager implements Manager using Helm.
-type helmManager struct {
+// client implements Client using Helm.
+type client struct {
 	restClientGetter func(string) genericclioptions.RESTClientGetter
 	manifestsBaseDir string
 }
 
-// ensure *helmManager implements Manager.
-var _ Manager = &helmManager{}
+// ensure *client implements Client.
+var _ Client = &client{}
 
-// newHelm creates a new helmManager.
-func newHelm(snap snap.Snap) *helmManager {
-	return &helmManager{
-		restClientGetter: snap.KubernetesRESTClientGetter,
-		manifestsBaseDir: snap.ManifestsDir(),
+// NewClient creates a new client.
+func NewClient(manifestsBaseDir string, restClientGetter func(string) genericclioptions.RESTClientGetter) *client {
+	return &client{
+		restClientGetter: restClientGetter,
+		manifestsBaseDir: manifestsBaseDir,
 	}
 }
 
-func (h *helmManager) newActionConfiguration(namespace string) (*action.Configuration, error) {
+func (h *client) newActionConfiguration(namespace string) (*action.Configuration, error) {
 	actionConfig := new(action.Configuration)
 
 	if err := actionConfig.Init(h.restClientGetter(namespace), namespace, "", log.Printf); err != nil {
@@ -41,9 +40,9 @@ func (h *helmManager) newActionConfiguration(namespace string) (*action.Configur
 	return actionConfig, nil
 }
 
-// Apply implements the Manager interface.
-func (h *helmManager) Apply(ctx context.Context, f Feature, desired state, values map[string]any) (bool, error) {
-	cfg, err := h.newActionConfiguration(f.namespace)
+// Apply implements the Client interface.
+func (h *client) Apply(ctx context.Context, c InstallableChart, desired State, values map[string]any) (bool, error) {
+	cfg, err := h.newActionConfiguration(c.Namespace)
 	if err != nil {
 		return false, fmt.Errorf("failed to create action configuration: %w", err)
 	}
@@ -53,10 +52,10 @@ func (h *helmManager) Apply(ctx context.Context, f Feature, desired state, value
 
 	// get the latest Helm release with the specified name
 	get := action.NewGet(cfg)
-	release, err := get.Run(f.name)
+	release, err := get.Run(c.Name)
 	if err != nil {
 		if err != driver.ErrReleaseNotFound {
-			return false, fmt.Errorf("failed to get status of release %s: %w", f.name, err)
+			return false, fmt.Errorf("failed to get status of release %s: %w", c.Name, err)
 		}
 		isInstalled = false
 	} else {
@@ -65,50 +64,50 @@ func (h *helmManager) Apply(ctx context.Context, f Feature, desired state, value
 	}
 
 	switch {
-	case !isInstalled && desired == stateDeleted:
+	case !isInstalled && desired == StateDeleted:
 		// no-op
 		return false, nil
-	case !isInstalled && desired == stateUpgradeOnly:
+	case !isInstalled && desired == StateUpgradeOnly:
 		// there is no release installed, this is an error
-		return false, fmt.Errorf("cannot upgrade %s as it is not installed", f.name)
-	case !isInstalled && desired == statePresent:
+		return false, fmt.Errorf("cannot upgrade %s as it is not installed", c.Name)
+	case !isInstalled && desired == StatePresent:
 		// there is no release installed, so we must run an install action
 		install := action.NewInstall(cfg)
-		install.ReleaseName = f.name
-		install.Namespace = f.namespace
+		install.ReleaseName = c.Name
+		install.Namespace = c.Namespace
 
-		chart, err := loader.Load(path.Join(h.manifestsBaseDir, f.manifestPath))
+		chart, err := loader.Load(path.Join(h.manifestsBaseDir, c.ManifestPath))
 		if err != nil {
-			return false, fmt.Errorf("failed to load manifest for %s: %w", f.name, err)
+			return false, fmt.Errorf("failed to load manifest for %s: %w", c.Name, err)
 		}
 
 		if _, err := install.RunWithContext(ctx, chart, values); err != nil {
-			return false, fmt.Errorf("failed to install %s: %w", f.name, err)
+			return false, fmt.Errorf("failed to install %s: %w", c.Name, err)
 		}
 		return true, nil
-	case isInstalled && desired != stateDeleted:
-		// there is already a release installed, so we must run an install action
+	case isInstalled && desired != StateDeleted:
+		// there is already a release installed, so we must run an upgrade action
 		upgrade := action.NewUpgrade(cfg)
-		upgrade.Namespace = f.namespace
+		upgrade.Namespace = c.Namespace
 		upgrade.ReuseValues = true
 
-		chart, err := loader.Load(path.Join(h.manifestsBaseDir, f.manifestPath))
+		chart, err := loader.Load(path.Join(h.manifestsBaseDir, c.ManifestPath))
 		if err != nil {
-			return false, fmt.Errorf("failed to load manifest for %s: %w", f.name, err)
+			return false, fmt.Errorf("failed to load manifest for %s: %w", c.Name, err)
 		}
 
-		release, err := upgrade.RunWithContext(ctx, f.name, chart, values)
+		release, err := upgrade.RunWithContext(ctx, c.Name, chart, values)
 		if err != nil {
-			return false, fmt.Errorf("failed to upgrade %s: %w", f.name, err)
+			return false, fmt.Errorf("failed to upgrade %s: %w", c.Name, err)
 		}
 
 		// oldConfig and release.Config are the previous and current values. they are compared by checking their respective JSON, as that is good enough for our needs of comparing unstructured map[string]any data.
 		return !jsonEqual(oldConfig, release.Config), nil
-	case isInstalled && desired == stateDeleted:
+	case isInstalled && desired == StateDeleted:
 		// run an uninstall action
 		uninstall := action.NewUninstall(cfg)
-		if _, err := uninstall.Run(f.name); err != nil {
-			return false, fmt.Errorf("failed to uninstall %s: %w", f.name, err)
+		if _, err := uninstall.Run(c.Name); err != nil {
+			return false, fmt.Errorf("failed to uninstall %s: %w", c.Name, err)
 		}
 
 		return true, nil
diff --git a/src/k8s/pkg/client/helm/interface.go b/src/k8s/pkg/client/helm/interface.go
new file mode 100644
index 000000000..f9e35a446
--- /dev/null
+++ b/src/k8s/pkg/client/helm/interface.go
@@ -0,0 +1,13 @@
+package helm
+
+import "context"
+
+// Client handles the lifecycle of charts (manifests + config) on the cluster.
+type Client interface {
+	// Apply ensures the state of a InstallableChart on the cluster.
+	// When state is StatePresent, Apply will install or upgrade the chart using the specified values as configuration. Apply returns true if the chart was not installed, or any values were changed.
+	// When state is StateUpgradeOnly, Apply will upgrade the chart using the specified values as configuration. Apply returns true if the chart was not installed, or any values were changed. An error is returned if the chart is not already installed.
+	// When state is StateDeleted, Apply will ensure that the chart is removed. If the chart is not installed, this is a no-op. Apply returns true if the chart was previously installed.
+	// Apply returns an error in case of failure.
+	Apply(ctx context.Context, f InstallableChart, desired State, values map[string]any) (bool, error)
+}
diff --git a/src/k8s/pkg/client/helm/mock/mock.go b/src/k8s/pkg/client/helm/mock/mock.go
new file mode 100644
index 000000000..6eea2d797
--- /dev/null
+++ b/src/k8s/pkg/client/helm/mock/mock.go
@@ -0,0 +1,29 @@
+package mock
+
+import (
+	"context"
+
+	"github.com/canonical/k8s/pkg/client/helm"
+)
+
+type MockApplyArguments struct {
+	Context context.Context
+	Chart   helm.InstallableChart
+	State   helm.State
+	Values  map[string]any
+}
+
+// Mock is a mock implementation of helm.Client
+type Mock struct {
+	ApplyCalledWith []MockApplyArguments
+	ApplyChanged    bool
+	ApplyErr        error
+}
+
+// Apply implements helm.Client
+func (m *Mock) Apply(ctx context.Context, c helm.InstallableChart, desired helm.State, values map[string]any) (bool, error) {
+	m.ApplyCalledWith = append(m.ApplyCalledWith, MockApplyArguments{Context: ctx, Chart: c, State: desired, Values: values})
+	return m.ApplyChanged, m.ApplyErr
+}
+
+var _ helm.Client = &Mock{}
diff --git a/src/k8s/pkg/client/helm/state.go b/src/k8s/pkg/client/helm/state.go
new file mode 100644
index 000000000..7a52ff243
--- /dev/null
+++ b/src/k8s/pkg/client/helm/state.go
@@ -0,0 +1,29 @@
+package helm
+
+// State is used to define how Client.Apply() handles install, upgrade or delete operations.
+type State int
+
+const (
+	// StateDeleted means that the chart should not be installed.
+	StateDeleted State = iota
+
+	// StatePresent means that the chart must be present. If it already exists, it is upgraded with the new configuration, otherwise it is installed.
+	StatePresent
+
+	// StateUpgradeOnly means that the chart will be refreshed if installed, fail otherwise.
+	StateUpgradeOnly
+)
+
+func StatePresentOrDeleted(enabled bool) State {
+	if enabled {
+		return StatePresent
+	}
+	return StateDeleted
+}
+
+func StateUpgradeOnlyOrDeleted(enabled bool) State {
+	if enabled {
+		return StateUpgradeOnly
+	}
+	return StateDeleted
+}
diff --git a/src/k8s/pkg/utils/k8s/client.go b/src/k8s/pkg/client/kubernetes/client.go
similarity index 96%
rename from src/k8s/pkg/utils/k8s/client.go
rename to src/k8s/pkg/client/kubernetes/client.go
index 5e761cb4b..890718edc 100644
--- a/src/k8s/pkg/utils/k8s/client.go
+++ b/src/k8s/pkg/client/kubernetes/client.go
@@ -1,4 +1,4 @@
-package k8s
+package kubernetes
 
 import (
 	"fmt"
diff --git a/src/k8s/pkg/utils/k8s/configmap.go b/src/k8s/pkg/client/kubernetes/configmap.go
similarity index 98%
rename from src/k8s/pkg/utils/k8s/configmap.go
rename to src/k8s/pkg/client/kubernetes/configmap.go
index 5afe85d4b..13770ba69 100644
--- a/src/k8s/pkg/utils/k8s/configmap.go
+++ b/src/k8s/pkg/client/kubernetes/configmap.go
@@ -1,4 +1,4 @@
-package k8s
+package kubernetes
 
 import (
 	"context"
diff --git a/src/k8s/pkg/utils/k8s/configmap_test.go b/src/k8s/pkg/client/kubernetes/configmap_test.go
similarity index 99%
rename from src/k8s/pkg/utils/k8s/configmap_test.go
rename to src/k8s/pkg/client/kubernetes/configmap_test.go
index dcc11d515..ea11d6aba 100644
--- a/src/k8s/pkg/utils/k8s/configmap_test.go
+++ b/src/k8s/pkg/client/kubernetes/configmap_test.go
@@ -1,4 +1,4 @@
-package k8s
+package kubernetes
 
 import (
 	"context"
diff --git a/src/k8s/pkg/utils/k8s/endpoints.go b/src/k8s/pkg/client/kubernetes/endpoints.go
similarity index 98%
rename from src/k8s/pkg/utils/k8s/endpoints.go
rename to src/k8s/pkg/client/kubernetes/endpoints.go
index 8277426b9..3d6709800 100644
--- a/src/k8s/pkg/utils/k8s/endpoints.go
+++ b/src/k8s/pkg/client/kubernetes/endpoints.go
@@ -1,4 +1,4 @@
-package k8s
+package kubernetes
 
 import (
 	"context"
diff --git a/src/k8s/pkg/utils/k8s/endpoints_test.go b/src/k8s/pkg/client/kubernetes/endpoints_test.go
similarity index 99%
rename from src/k8s/pkg/utils/k8s/endpoints_test.go
rename to src/k8s/pkg/client/kubernetes/endpoints_test.go
index 771e33107..238886aa6 100644
--- a/src/k8s/pkg/utils/k8s/endpoints_test.go
+++ b/src/k8s/pkg/client/kubernetes/endpoints_test.go
@@ -1,4 +1,4 @@
-package k8s
+package kubernetes
 
 import (
 	"context"
diff --git a/src/k8s/pkg/utils/k8s/node.go b/src/k8s/pkg/client/kubernetes/node.go
similarity index 96%
rename from src/k8s/pkg/utils/k8s/node.go
rename to src/k8s/pkg/client/kubernetes/node.go
index 05ca6155e..14d0457a8 100644
--- a/src/k8s/pkg/utils/k8s/node.go
+++ b/src/k8s/pkg/client/kubernetes/node.go
@@ -1,4 +1,4 @@
-package k8s
+package kubernetes
 
 import (
 	"context"
diff --git a/src/k8s/pkg/utils/k8s/node_test.go b/src/k8s/pkg/client/kubernetes/node_test.go
similarity index 98%
rename from src/k8s/pkg/utils/k8s/node_test.go
rename to src/k8s/pkg/client/kubernetes/node_test.go
index 3baef9b82..1a3f2c1e6 100644
--- a/src/k8s/pkg/utils/k8s/node_test.go
+++ b/src/k8s/pkg/client/kubernetes/node_test.go
@@ -1,4 +1,4 @@
-package k8s
+package kubernetes
 
 import (
 	"context"
diff --git a/src/k8s/pkg/utils/k8s/restart_daemonset.go b/src/k8s/pkg/client/kubernetes/restart_daemonset.go
similarity index 98%
rename from src/k8s/pkg/utils/k8s/restart_daemonset.go
rename to src/k8s/pkg/client/kubernetes/restart_daemonset.go
index 669ebbc24..a66425371 100644
--- a/src/k8s/pkg/utils/k8s/restart_daemonset.go
+++ b/src/k8s/pkg/client/kubernetes/restart_daemonset.go
@@ -1,4 +1,4 @@
-package k8s
+package kubernetes
 
 import (
 	"context"
diff --git a/src/k8s/pkg/utils/k8s/restart_daemonset_test.go b/src/k8s/pkg/client/kubernetes/restart_daemonset_test.go
similarity index 98%
rename from src/k8s/pkg/utils/k8s/restart_daemonset_test.go
rename to src/k8s/pkg/client/kubernetes/restart_daemonset_test.go
index 6b7b6887a..55ecc8061 100644
--- a/src/k8s/pkg/utils/k8s/restart_daemonset_test.go
+++ b/src/k8s/pkg/client/kubernetes/restart_daemonset_test.go
@@ -1,4 +1,4 @@
-package k8s
+package kubernetes
 
 import (
 	"context"
diff --git a/src/k8s/pkg/utils/k8s/restart_deployment.go b/src/k8s/pkg/client/kubernetes/restart_deployment.go
similarity index 98%
rename from src/k8s/pkg/utils/k8s/restart_deployment.go
rename to src/k8s/pkg/client/kubernetes/restart_deployment.go
index cc172ae51..0dac4c44b 100644
--- a/src/k8s/pkg/utils/k8s/restart_deployment.go
+++ b/src/k8s/pkg/client/kubernetes/restart_deployment.go
@@ -1,4 +1,4 @@
-package k8s
+package kubernetes
 
 import (
 	"context"
diff --git a/src/k8s/pkg/utils/k8s/restart_deployment_test.go b/src/k8s/pkg/client/kubernetes/restart_deployment_test.go
similarity index 98%
rename from src/k8s/pkg/utils/k8s/restart_deployment_test.go
rename to src/k8s/pkg/client/kubernetes/restart_deployment_test.go
index f403714cb..2bc7aa9d7 100644
--- a/src/k8s/pkg/utils/k8s/restart_deployment_test.go
+++ b/src/k8s/pkg/client/kubernetes/restart_deployment_test.go
@@ -1,4 +1,4 @@
-package k8s
+package kubernetes
 
 import (
 	"context"
diff --git a/src/k8s/pkg/utils/k8s/server_groups.go b/src/k8s/pkg/client/kubernetes/server_groups.go
similarity index 96%
rename from src/k8s/pkg/utils/k8s/server_groups.go
rename to src/k8s/pkg/client/kubernetes/server_groups.go
index cfe2a8b67..cb58b9d0c 100644
--- a/src/k8s/pkg/utils/k8s/server_groups.go
+++ b/src/k8s/pkg/client/kubernetes/server_groups.go
@@ -1,4 +1,4 @@
-package k8s
+package kubernetes
 
 import (
 	"fmt"
diff --git a/src/k8s/pkg/utils/k8s/server_groups_test.go b/src/k8s/pkg/client/kubernetes/server_groups_test.go
similarity index 93%
rename from src/k8s/pkg/utils/k8s/server_groups_test.go
rename to src/k8s/pkg/client/kubernetes/server_groups_test.go
index d8e0d057f..3126442a5 100644
--- a/src/k8s/pkg/utils/k8s/server_groups_test.go
+++ b/src/k8s/pkg/client/kubernetes/server_groups_test.go
@@ -1,4 +1,4 @@
-package k8s_test
+package kubernetes_test
 
 import (
 	"testing"
@@ -6,7 +6,7 @@ import (
 	fakediscovery "k8s.io/client-go/discovery/fake"
 	fakeclientset "k8s.io/client-go/kubernetes/fake"
 
-	"github.com/canonical/k8s/pkg/utils/k8s"
+	"github.com/canonical/k8s/pkg/client/kubernetes"
 	. "github.com/onsi/gomega"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -49,7 +49,7 @@ func TestListResourcesForGroupVersion(t *testing.T) {
 			}
 
 			// Create a new k8s client with the fake discovery client
-			client := &k8s.Client{
+			client := &kubernetes.Client{
 				Interface: clientset,
 			}
 
diff --git a/src/k8s/pkg/utils/k8s/services.go b/src/k8s/pkg/client/kubernetes/services.go
similarity index 96%
rename from src/k8s/pkg/utils/k8s/services.go
rename to src/k8s/pkg/client/kubernetes/services.go
index 21cfc136b..68e1f5e09 100644
--- a/src/k8s/pkg/utils/k8s/services.go
+++ b/src/k8s/pkg/client/kubernetes/services.go
@@ -1,4 +1,4 @@
-package k8s
+package kubernetes
 
 import (
 	"context"
diff --git a/src/k8s/pkg/utils/k8s/services_test.go b/src/k8s/pkg/client/kubernetes/services_test.go
similarity index 98%
rename from src/k8s/pkg/utils/k8s/services_test.go
rename to src/k8s/pkg/client/kubernetes/services_test.go
index d0dfbe74c..6d5f1aa3f 100644
--- a/src/k8s/pkg/utils/k8s/services_test.go
+++ b/src/k8s/pkg/client/kubernetes/services_test.go
@@ -1,4 +1,4 @@
-package k8s
+package kubernetes
 
 import (
 	"context"
diff --git a/src/k8s/pkg/utils/k8s/status.go b/src/k8s/pkg/client/kubernetes/status.go
similarity index 98%
rename from src/k8s/pkg/utils/k8s/status.go
rename to src/k8s/pkg/client/kubernetes/status.go
index 74249d737..1c478217d 100644
--- a/src/k8s/pkg/utils/k8s/status.go
+++ b/src/k8s/pkg/client/kubernetes/status.go
@@ -1,4 +1,4 @@
-package k8s
+package kubernetes
 
 import (
 	"context"
diff --git a/src/k8s/pkg/utils/k8s/status_test.go b/src/k8s/pkg/client/kubernetes/status_test.go
similarity index 99%
rename from src/k8s/pkg/utils/k8s/status_test.go
rename to src/k8s/pkg/client/kubernetes/status_test.go
index f5b829ecb..7dae3e115 100644
--- a/src/k8s/pkg/utils/k8s/status_test.go
+++ b/src/k8s/pkg/client/kubernetes/status_test.go
@@ -1,4 +1,4 @@
-package k8s
+package kubernetes
 
 import (
 	"context"
diff --git a/src/k8s/pkg/k8sd/api/cluster.go b/src/k8s/pkg/k8sd/api/cluster.go
index 0733025e7..5f16accfa 100644
--- a/src/k8s/pkg/k8sd/api/cluster.go
+++ b/src/k8s/pkg/k8sd/api/cluster.go
@@ -2,12 +2,11 @@ package api
 
 import (
 	"fmt"
-	databaseutil "github.com/canonical/k8s/pkg/k8sd/database/util"
 	"net/http"
 
 	apiv1 "github.com/canonical/k8s/api/v1"
 	"github.com/canonical/k8s/pkg/k8sd/api/impl"
-	"github.com/canonical/k8s/pkg/utils/k8s"
+	databaseutil "github.com/canonical/k8s/pkg/k8sd/database/util"
 	"github.com/canonical/lxd/lxd/response"
 	"github.com/canonical/microcluster/state"
 )
@@ -27,8 +26,7 @@ func (e *Endpoints) getClusterStatus(s *state.State, r *http.Request) response.R
 		return response.InternalError(fmt.Errorf("failed to get cluster config: %w", err))
 	}
 
-	snap := e.provider.Snap()
-	client, err := k8s.NewClient(snap.KubernetesRESTClientGetter(""))
+	client, err := e.provider.Snap().KubernetesClient("")
 	if err != nil {
 		return response.InternalError(fmt.Errorf("failed to create k8s client: %w", err))
 	}
diff --git a/src/k8s/pkg/k8sd/api/cluster_remove.go b/src/k8s/pkg/k8sd/api/cluster_remove.go
index c8235b832..f3f24bec5 100644
--- a/src/k8s/pkg/k8sd/api/cluster_remove.go
+++ b/src/k8s/pkg/k8sd/api/cluster_remove.go
@@ -7,7 +7,6 @@ import (
 	apiv1 "github.com/canonical/k8s/api/v1"
 	databaseutil "github.com/canonical/k8s/pkg/k8sd/database/util"
 	"github.com/canonical/k8s/pkg/utils"
-	"github.com/canonical/k8s/pkg/utils/k8s"
 	nodeutil "github.com/canonical/k8s/pkg/utils/node"
 	"github.com/canonical/lxd/lxd/response"
 	"github.com/canonical/microcluster/state"
@@ -43,7 +42,7 @@ func (e *Endpoints) postClusterRemove(s *state.State, r *http.Request) response.
 	}
 	if isWorker {
 		// For worker nodes, we need to manually clean up the kubernetes node and db entry.
-		c, err := k8s.NewClient(snap.KubernetesRESTClientGetter(""))
+		c, err := snap.KubernetesClient("")
 		if err != nil {
 			return response.InternalError(fmt.Errorf("failed to create k8s client: %w", err))
 		}
diff --git a/src/k8s/pkg/k8sd/api/worker.go b/src/k8s/pkg/k8sd/api/worker.go
index 716aae508..424e15f45 100644
--- a/src/k8s/pkg/k8sd/api/worker.go
+++ b/src/k8s/pkg/k8sd/api/worker.go
@@ -12,7 +12,6 @@ import (
 	databaseutil "github.com/canonical/k8s/pkg/k8sd/database/util"
 	"github.com/canonical/k8s/pkg/k8sd/pki"
 	"github.com/canonical/k8s/pkg/utils"
-	"github.com/canonical/k8s/pkg/utils/k8s"
 	"github.com/canonical/lxd/lxd/response"
 	"github.com/canonical/microcluster/state"
 )
@@ -45,7 +44,7 @@ func (e *Endpoints) postWorkerInfo(s *state.State, r *http.Request) response.Res
 		return response.InternalError(fmt.Errorf("failed to generate worker PKI: %w", err))
 	}
 
-	client, err := k8s.NewClient(snap.KubernetesRESTClientGetter(""))
+	client, err := snap.KubernetesClient("")
 	if err != nil {
 		return response.InternalError(fmt.Errorf("failed to create kubernetes client: %w", err))
 	}
diff --git a/src/k8s/pkg/k8sd/app/app.go b/src/k8s/pkg/k8sd/app/app.go
index cb2040d98..51c529859 100644
--- a/src/k8s/pkg/k8sd/app/app.go
+++ b/src/k8s/pkg/k8sd/app/app.go
@@ -13,7 +13,6 @@ import (
 	"github.com/canonical/k8s/pkg/k8sd/controllers"
 	"github.com/canonical/k8s/pkg/k8sd/database"
 	"github.com/canonical/k8s/pkg/snap"
-	"github.com/canonical/k8s/pkg/utils/k8s"
 	"github.com/canonical/microcluster/config"
 	"github.com/canonical/microcluster/microcluster"
 	"github.com/canonical/microcluster/state"
@@ -87,9 +86,6 @@ func New(cfg Config) (*App, error) {
 	app.nodeConfigController = controllers.NewNodeConfigurationController(
 		cfg.Snap,
 		app.readyWg.Wait,
-		func() (*k8s.Client, error) {
-			return k8s.NewClient(cfg.Snap.KubernetesNodeRESTClientGetter("kube-system"))
-		},
 	)
 
 	app.controlPlaneConfigController = controllers.NewControlPlaneConfigurationController(
@@ -102,9 +98,6 @@ func New(cfg Config) (*App, error) {
 	app.updateNodeConfigController = controllers.NewUpdateNodeConfigurationController(
 		cfg.Snap,
 		app.readyWg.Wait,
-		func() (*k8s.Client, error) {
-			return k8s.NewClient(cfg.Snap.KubernetesRESTClientGetter("kube-system"))
-		},
 		app.triggerUpdateNodeConfigControllerCh,
 	)
 
diff --git a/src/k8s/pkg/k8sd/app/cluster_util.go b/src/k8s/pkg/k8sd/app/cluster_util.go
index 5e1aaa607..f27a129da 100644
--- a/src/k8s/pkg/k8sd/app/cluster_util.go
+++ b/src/k8s/pkg/k8sd/app/cluster_util.go
@@ -3,15 +3,14 @@ package app
 import (
 	"context"
 	"fmt"
-	databaseutil "github.com/canonical/k8s/pkg/k8sd/database/util"
 	"net"
 	"path"
 
+	databaseutil "github.com/canonical/k8s/pkg/k8sd/database/util"
 	"github.com/canonical/k8s/pkg/k8sd/setup"
 	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/snap"
 	snaputil "github.com/canonical/k8s/pkg/snap/util"
-	"github.com/canonical/k8s/pkg/utils/k8s"
 	"github.com/canonical/microcluster/state"
 )
 
@@ -82,9 +81,8 @@ func startControlPlaneServices(ctx context.Context, snap snap.Snap, datastore st
 }
 
 func waitApiServerReady(ctx context.Context, snap snap.Snap) error {
-
 	// Wait for API server to come up
-	client, err := k8s.NewClient(snap.KubernetesRESTClientGetter(""))
+	client, err := snap.KubernetesClient("")
 	if err != nil {
 		return fmt.Errorf("failed to create Kubernetes client: %w", err)
 	}
diff --git a/src/k8s/pkg/k8sd/app/hooks_join.go b/src/k8s/pkg/k8sd/app/hooks_join.go
index cfb2c71c1..430a10d91 100644
--- a/src/k8s/pkg/k8sd/app/hooks_join.go
+++ b/src/k8s/pkg/k8sd/app/hooks_join.go
@@ -2,15 +2,13 @@ package app
 
 import (
 	"fmt"
-	databaseutil "github.com/canonical/k8s/pkg/k8sd/database/util"
 	"net"
 
 	apiv1 "github.com/canonical/k8s/api/v1"
-
+	databaseutil "github.com/canonical/k8s/pkg/k8sd/database/util"
 	"github.com/canonical/k8s/pkg/k8sd/pki"
 	"github.com/canonical/k8s/pkg/k8sd/setup"
 	"github.com/canonical/k8s/pkg/utils"
-	"github.com/canonical/k8s/pkg/utils/k8s"
 	"github.com/canonical/microcluster/state"
 )
 
@@ -183,7 +181,7 @@ func (a *App) onPreRemove(s *state.State, force bool) error {
 	default:
 	}
 
-	c, err := k8s.NewClient(snap.KubernetesRESTClientGetter(""))
+	c, err := snap.KubernetesClient("")
 	if err != nil {
 		return fmt.Errorf("failed to create Kubernetes client: %w", err)
 	}
diff --git a/src/k8s/pkg/k8sd/controllers/node_configuration.go b/src/k8s/pkg/k8sd/controllers/node_configuration.go
index da645d44a..b398b41ab 100644
--- a/src/k8s/pkg/k8sd/controllers/node_configuration.go
+++ b/src/k8s/pkg/k8sd/controllers/node_configuration.go
@@ -7,30 +7,28 @@ import (
 	"log"
 	"time"
 
+	"github.com/canonical/k8s/pkg/client/kubernetes"
 	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/snap"
 	snaputil "github.com/canonical/k8s/pkg/snap/util"
-	"github.com/canonical/k8s/pkg/utils/k8s"
 	v1 "k8s.io/api/core/v1"
 )
 
 type NodeConfigurationController struct {
-	snap         snap.Snap
-	waitReady    func()
-	newK8sClient func() (*k8s.Client, error)
+	snap      snap.Snap
+	waitReady func()
 }
 
-func NewNodeConfigurationController(snap snap.Snap, waitReady func(), newK8sClient func() (*k8s.Client, error)) *NodeConfigurationController {
+func NewNodeConfigurationController(snap snap.Snap, waitReady func()) *NodeConfigurationController {
 	return &NodeConfigurationController{
-		snap:         snap,
-		waitReady:    waitReady,
-		newK8sClient: newK8sClient,
+		snap:      snap,
+		waitReady: waitReady,
 	}
 }
 
-func (c *NodeConfigurationController) retryNewK8sClient(ctx context.Context) (*k8s.Client, error) {
+func (c *NodeConfigurationController) retryNewK8sClient(ctx context.Context) (*kubernetes.Client, error) {
 	for {
-		client, err := c.newK8sClient()
+		client, err := c.snap.KubernetesNodeClient("kube-system")
 		if err == nil {
 			return client, nil
 		}
diff --git a/src/k8s/pkg/k8sd/controllers/node_configuration_test.go b/src/k8s/pkg/k8sd/controllers/node_configuration_test.go
index 9a7c17753..69ebcd660 100644
--- a/src/k8s/pkg/k8sd/controllers/node_configuration_test.go
+++ b/src/k8s/pkg/k8sd/controllers/node_configuration_test.go
@@ -8,10 +8,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/canonical/k8s/pkg/client/kubernetes"
 	"github.com/canonical/k8s/pkg/k8sd/setup"
 	"github.com/canonical/k8s/pkg/snap/mock"
 	snaputil "github.com/canonical/k8s/pkg/snap/util"
-	"github.com/canonical/k8s/pkg/utils/k8s"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -26,16 +26,6 @@ func TestConfigPropagation(t *testing.T) {
 
 	g := NewWithT(t)
 
-	s := &mock.Snap{
-		Mock: mock.Mock{
-			ServiceArgumentsDir: path.Join(t.TempDir(), "args"),
-			UID:                 os.Getuid(),
-			GID:                 os.Getgid(),
-		},
-	}
-
-	g.Expect(setup.EnsureAllDirectories(s)).To(Succeed())
-
 	tests := []struct {
 		name          string
 		configmap     *corev1.ConfigMap
@@ -126,12 +116,21 @@ func TestConfigPropagation(t *testing.T) {
 	watcher := watch.NewFake()
 	clientset.PrependWatchReactor("configmaps", k8stesting.DefaultWatchReactor(watcher, nil))
 
-	configController := NewNodeConfigurationController(s, func() {}, func() (*k8s.Client, error) {
-		return &k8s.Client{Interface: clientset}, nil
-	})
+	s := &mock.Snap{
+		Mock: mock.Mock{
+			ServiceArgumentsDir:  path.Join(t.TempDir(), "args"),
+			UID:                  os.Getuid(),
+			GID:                  os.Getgid(),
+			KubernetesNodeClient: &kubernetes.Client{Interface: clientset},
+		},
+	}
+
+	g.Expect(setup.EnsureAllDirectories(s)).To(Succeed())
+
+	ctrl := NewNodeConfigurationController(s, func() {})
 
 	// TODO: add test with signing key
-	go configController.Run(ctx, func(ctx context.Context) (*rsa.PublicKey, error) { return nil, nil })
+	go ctrl.Run(ctx, func(ctx context.Context) (*rsa.PublicKey, error) { return nil, nil })
 	defer watcher.Stop()
 
 	for _, tc := range tests {
diff --git a/src/k8s/pkg/k8sd/controllers/update_node_configuration.go b/src/k8s/pkg/k8sd/controllers/update_node_configuration.go
index d104736f4..12b3eee05 100644
--- a/src/k8s/pkg/k8sd/controllers/update_node_configuration.go
+++ b/src/k8s/pkg/k8sd/controllers/update_node_configuration.go
@@ -6,19 +6,18 @@ import (
 	"log"
 	"time"
 
+	"github.com/canonical/k8s/pkg/client/kubernetes"
 	"github.com/canonical/k8s/pkg/k8sd/pki"
 	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/snap"
 	snaputil "github.com/canonical/k8s/pkg/snap/util"
-	"github.com/canonical/k8s/pkg/utils/k8s"
 )
 
 // UpdateNodeConfigurationController asynchronously performs updates of the cluster config.
 // A new reconcile loop is triggered by pushing to the triggerCh channel.
 type UpdateNodeConfigurationController struct {
-	snap         snap.Snap
-	waitReady    func()
-	newK8sClient func() (*k8s.Client, error)
+	snap      snap.Snap
+	waitReady func()
 
 	// triggerCh is used to trigger config updates on the controller.
 	triggerCh <-chan struct{}
@@ -27,20 +26,19 @@ type UpdateNodeConfigurationController struct {
 }
 
 // NewUpdateNodeConfigurationController creates a new controller.
-func NewUpdateNodeConfigurationController(snap snap.Snap, waitReady func(), newK8sClient func() (*k8s.Client, error), triggerCh <-chan struct{}) *UpdateNodeConfigurationController {
+func NewUpdateNodeConfigurationController(snap snap.Snap, waitReady func(), triggerCh <-chan struct{}) *UpdateNodeConfigurationController {
 	return &UpdateNodeConfigurationController{
-		snap:         snap,
-		waitReady:    waitReady,
-		newK8sClient: newK8sClient,
+		snap:      snap,
+		waitReady: waitReady,
 
 		triggerCh:    triggerCh,
 		reconciledCh: make(chan struct{}, 1),
 	}
 }
 
-func (c *UpdateNodeConfigurationController) retryNewK8sClient(ctx context.Context) (*k8s.Client, error) {
+func (c *UpdateNodeConfigurationController) retryNewK8sClient(ctx context.Context) (*kubernetes.Client, error) {
 	for {
-		client, err := c.newK8sClient()
+		client, err := c.snap.KubernetesClient("kube-system")
 		if err == nil {
 			return client, nil
 		}
@@ -98,7 +96,7 @@ func (c *UpdateNodeConfigurationController) Run(ctx context.Context, getClusterC
 	}
 }
 
-func (c *UpdateNodeConfigurationController) reconcile(ctx context.Context, client *k8s.Client, config types.ClusterConfig) error {
+func (c *UpdateNodeConfigurationController) reconcile(ctx context.Context, client *kubernetes.Client, config types.ClusterConfig) error {
 	keyPEM := config.Certificates.GetK8sdPrivateKey()
 	key, err := pki.LoadRSAPrivateKey(keyPEM)
 	if err != nil && keyPEM != "" {
diff --git a/src/k8s/pkg/k8sd/controllers/update_node_configuration_test.go b/src/k8s/pkg/k8sd/controllers/update_node_configuration_test.go
index 87bbacbb2..16efd2bcb 100644
--- a/src/k8s/pkg/k8sd/controllers/update_node_configuration_test.go
+++ b/src/k8s/pkg/k8sd/controllers/update_node_configuration_test.go
@@ -7,15 +7,14 @@ import (
 	"testing"
 	"time"
 
+	"github.com/canonical/k8s/pkg/client/kubernetes"
 	"github.com/canonical/k8s/pkg/k8sd/controllers"
 	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/snap/mock"
 	"github.com/canonical/k8s/pkg/utils"
-	"github.com/canonical/k8s/pkg/utils/k8s"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"k8s.io/client-go/kubernetes/fake"
 )
 
@@ -48,16 +47,6 @@ func TestUpdateNodeConfigurationController(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			dir := t.TempDir()
 
-			s := &mock.Snap{
-				Mock: mock.Mock{
-					EtcdPKIDir:                 path.Join(dir, "etcd-pki"),
-					ServiceArgumentsDir:        path.Join(dir, "args"),
-					UID:                        os.Getuid(),
-					GID:                        os.Getgid(),
-					KubernetesRESTClientGetter: genericclioptions.NewTestConfigFlags(),
-				},
-			}
-
 			g := NewWithT(t)
 			ctx, cancel := context.WithCancel(context.Background())
 			defer cancel()
@@ -76,12 +65,19 @@ func TestUpdateNodeConfigurationController(t *testing.T) {
 			}
 			clientset := fake.NewSimpleClientset(configMap)
 
+			s := &mock.Snap{
+				Mock: mock.Mock{
+					EtcdPKIDir:          path.Join(dir, "etcd-pki"),
+					ServiceArgumentsDir: path.Join(dir, "args"),
+					UID:                 os.Getuid(),
+					GID:                 os.Getgid(),
+					KubernetesClient:    &kubernetes.Client{Interface: clientset},
+				},
+			}
 			triggerCh := make(chan struct{})
 			defer close(triggerCh)
 
-			ctrl := controllers.NewUpdateNodeConfigurationController(s, func() {}, func() (*k8s.Client, error) {
-				return &k8s.Client{Interface: clientset}, nil
-			}, triggerCh)
+			ctrl := controllers.NewUpdateNodeConfigurationController(s, func() {}, triggerCh)
 			go ctrl.Run(ctx, configProvider.getConfig)
 
 			select {
diff --git a/src/k8s/pkg/k8sd/features/feature_dns.go b/src/k8s/pkg/k8sd/features/feature_dns.go
index 262f5aa7b..50a7d7a4b 100644
--- a/src/k8s/pkg/k8sd/features/feature_dns.go
+++ b/src/k8s/pkg/k8sd/features/feature_dns.go
@@ -5,9 +5,9 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/canonical/k8s/pkg/client/helm"
 	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/snap"
-	"github.com/canonical/k8s/pkg/utils/k8s"
 )
 
 // ApplyDNS is used to configure the DNS feature on Canonical Kubernetes.
@@ -17,10 +17,10 @@ import (
 // ApplyDNS will return the ClusterIP address of the coredns service, if successful.
 // ApplyDNS returns an error if anything fails.
 func ApplyDNS(ctx context.Context, snap snap.Snap, dns types.DNS, kubelet types.Kubelet) (string, error) {
-	m := newHelm(snap)
+	m := snap.HelmClient()
 
 	if !dns.GetEnabled() {
-		if _, err := m.Apply(ctx, featureCoreDNS, stateDeleted, nil); err != nil {
+		if _, err := m.Apply(ctx, chartCoreDNS, helm.StateDeleted, nil); err != nil {
 			return "", fmt.Errorf("failed to uninstall coredns: %w", err)
 		}
 		return "", nil
@@ -64,11 +64,11 @@ func ApplyDNS(ctx context.Context, snap snap.Snap, dns types.DNS, kubelet types.
 		},
 	}
 
-	if _, err := m.Apply(ctx, featureCoreDNS, statePresent, values); err != nil {
+	if _, err := m.Apply(ctx, chartCoreDNS, helm.StatePresent, values); err != nil {
 		return "", fmt.Errorf("failed to apply coredns: %w", err)
 	}
 
-	client, err := k8s.NewClient(snap.KubernetesRESTClientGetter(""))
+	client, err := snap.KubernetesClient("")
 	if err != nil {
 		return "", fmt.Errorf("failed to create kubernetes client: %w", err)
 	}
diff --git a/src/k8s/pkg/k8sd/features/feature_gateway.go b/src/k8s/pkg/k8sd/features/feature_gateway.go
index 6b4875293..9414e2c26 100644
--- a/src/k8s/pkg/k8sd/features/feature_gateway.go
+++ b/src/k8s/pkg/k8sd/features/feature_gateway.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/canonical/k8s/pkg/client/helm"
 	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/snap"
 )
@@ -15,13 +16,13 @@ import (
 // ApplyGateway will rollout restart the Cilium pods in case any Cilium configuration was changed.
 // ApplyGateway returns an error if anything fails.
 func ApplyGateway(ctx context.Context, snap snap.Snap, gateway types.Gateway, network types.Network) error {
-	m := newHelm(snap)
+	m := snap.HelmClient()
 
-	if _, err := m.Apply(ctx, featureCiliumGateway, statePresentOrDeleted(gateway.GetEnabled()), nil); err != nil {
+	if _, err := m.Apply(ctx, chartCiliumGateway, helm.StatePresentOrDeleted(gateway.GetEnabled()), nil); err != nil {
 		return fmt.Errorf("failed to install Gateway API CRDs: %w", err)
 	}
 
-	changed, err := m.Apply(ctx, featureCiliumCNI, stateUpgradeOnlyOrDeleted(network.GetEnabled()), map[string]any{"gatewayAPI": map[string]any{"enabled": gateway.GetEnabled()}})
+	changed, err := m.Apply(ctx, chartCilium, helm.StateUpgradeOnlyOrDeleted(network.GetEnabled()), map[string]any{"gatewayAPI": map[string]any{"enabled": gateway.GetEnabled()}})
 	if err != nil {
 		return fmt.Errorf("failed to apply Gateway API cilium configuration: %w", err)
 	}
diff --git a/src/k8s/pkg/k8sd/features/feature_ingress.go b/src/k8s/pkg/k8sd/features/feature_ingress.go
index 2b590e46b..48d1a9c2a 100644
--- a/src/k8s/pkg/k8sd/features/feature_ingress.go
+++ b/src/k8s/pkg/k8sd/features/feature_ingress.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/canonical/k8s/pkg/client/helm"
 	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/snap"
 )
@@ -15,7 +16,7 @@ import (
 // ApplyIngress will rollout restart the Cilium pods in case any Cilium configuration was changed.
 // ApplyIngress returns an error if anything fails.
 func ApplyIngress(ctx context.Context, snap snap.Snap, ingress types.Ingress, network types.Network) error {
-	m := newHelm(snap)
+	m := snap.HelmClient()
 
 	var values map[string]any
 	if ingress.GetEnabled() {
@@ -39,7 +40,7 @@ func ApplyIngress(ctx context.Context, snap snap.Snap, ingress types.Ingress, ne
 		}
 	}
 
-	changed, err := m.Apply(ctx, featureCiliumCNI, stateUpgradeOnlyOrDeleted(network.GetEnabled()), values)
+	changed, err := m.Apply(ctx, chartCilium, helm.StateUpgradeOnlyOrDeleted(network.GetEnabled()), values)
 	if err != nil {
 		return fmt.Errorf("failed to enable ingress: %w", err)
 	}
diff --git a/src/k8s/pkg/k8sd/features/feature_loadbalancer.go b/src/k8s/pkg/k8sd/features/feature_loadbalancer.go
index cb7d912ef..3595fd01b 100644
--- a/src/k8s/pkg/k8sd/features/feature_loadbalancer.go
+++ b/src/k8s/pkg/k8sd/features/feature_loadbalancer.go
@@ -4,10 +4,10 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/canonical/k8s/pkg/client/helm"
 	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/snap"
 	"github.com/canonical/k8s/pkg/utils/control"
-	"github.com/canonical/k8s/pkg/utils/k8s"
 )
 
 // ApplyLoadBalancer is used to configure the load-balancer feature on Canonical Kubernetes.
@@ -31,9 +31,9 @@ func ApplyLoadBalancer(ctx context.Context, snap snap.Snap, loadbalancer types.L
 }
 
 func disableLoadBalancer(ctx context.Context, snap snap.Snap, network types.Network) error {
-	m := newHelm(snap)
+	m := snap.HelmClient()
 
-	if _, err := m.Apply(ctx, featureCiliumLoadBalancer, stateDeleted, nil); err != nil {
+	if _, err := m.Apply(ctx, chartCiliumLoadBalancer, helm.StateDeleted, nil); err != nil {
 		return fmt.Errorf("failed to uninstall LoadBalancer manifests: %w", err)
 	}
 
@@ -55,14 +55,14 @@ func disableLoadBalancer(ctx context.Context, snap snap.Snap, network types.Netw
 		},
 	}
 
-	if _, err := m.Apply(ctx, featureCiliumCNI, stateUpgradeOnlyOrDeleted(network.GetEnabled()), values); err != nil {
+	if _, err := m.Apply(ctx, chartCilium, helm.StateUpgradeOnlyOrDeleted(network.GetEnabled()), values); err != nil {
 		return fmt.Errorf("failed to refresh network to apply LoadBalancer configuration: %w", err)
 	}
 	return nil
 }
 
 func enableLoadBalancer(ctx context.Context, snap snap.Snap, loadbalancer types.LoadBalancer, network types.Network) error {
-	m := newHelm(snap)
+	m := snap.HelmClient()
 
 	networkValues := map[string]any{
 		"l2announcements": map[string]any{
@@ -82,7 +82,7 @@ func enableLoadBalancer(ctx context.Context, snap snap.Snap, loadbalancer types.
 		},
 	}
 
-	changed, err := m.Apply(ctx, featureCiliumCNI, stateUpgradeOnlyOrDeleted(network.GetEnabled()), networkValues)
+	changed, err := m.Apply(ctx, chartCilium, helm.StateUpgradeOnlyOrDeleted(network.GetEnabled()), networkValues)
 	if err != nil {
 		return fmt.Errorf("failed to update Cilium configuration for LoadBalancer: %w", err)
 	}
@@ -119,7 +119,7 @@ func enableLoadBalancer(ctx context.Context, snap snap.Snap, loadbalancer types.
 			},
 		},
 	}
-	if _, err := m.Apply(ctx, featureCiliumLoadBalancer, statePresent, values); err != nil {
+	if _, err := m.Apply(ctx, chartCiliumLoadBalancer, helm.StatePresent, values); err != nil {
 		return fmt.Errorf("failed to apply LoadBalancer configuration: %w", err)
 	}
 
@@ -134,7 +134,7 @@ func enableLoadBalancer(ctx context.Context, snap snap.Snap, loadbalancer types.
 }
 
 func waitForRequiredLoadBalancerCRDs(ctx context.Context, snap snap.Snap, bgpMode bool) error {
-	client, err := k8s.NewClient(snap.KubernetesRESTClientGetter(""))
+	client, err := snap.KubernetesClient("")
 	if err != nil {
 		return fmt.Errorf("failed to create Kubernetes client: %w", err)
 	}
diff --git a/src/k8s/pkg/k8sd/features/feature_local_storage.go b/src/k8s/pkg/k8sd/features/feature_local_storage.go
index 3815607fc..940370381 100644
--- a/src/k8s/pkg/k8sd/features/feature_local_storage.go
+++ b/src/k8s/pkg/k8sd/features/feature_local_storage.go
@@ -3,6 +3,7 @@ package features
 import (
 	"context"
 
+	"github.com/canonical/k8s/pkg/client/helm"
 	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/snap"
 )
@@ -12,7 +13,7 @@ import (
 // ApplyLocalStorage removes the rawfile-localpv when cfg.Enabled is false.
 // ApplyLocalStorage returns an error if anything fails.
 func ApplyLocalStorage(ctx context.Context, snap snap.Snap, cfg types.LocalStorage) error {
-	m := newHelm(snap)
+	m := snap.HelmClient()
 
 	values := map[string]any{
 		"storageClass": map[string]any{
@@ -41,6 +42,6 @@ func ApplyLocalStorage(ctx context.Context, snap snap.Snap, cfg types.LocalStora
 		},
 	}
 
-	_, err := m.Apply(ctx, featureLocalStorage, statePresentOrDeleted(cfg.GetEnabled()), values)
+	_, err := m.Apply(ctx, chartLocalStorage, helm.StatePresentOrDeleted(cfg.GetEnabled()), values)
 	return err
 }
diff --git a/src/k8s/pkg/k8sd/features/feature_metrics_server.go b/src/k8s/pkg/k8sd/features/feature_metrics_server.go
index 83061cbf0..41bb376c8 100644
--- a/src/k8s/pkg/k8sd/features/feature_metrics_server.go
+++ b/src/k8s/pkg/k8sd/features/feature_metrics_server.go
@@ -3,6 +3,7 @@ package features
 import (
 	"context"
 
+	"github.com/canonical/k8s/pkg/client/helm"
 	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/snap"
 )
@@ -12,7 +13,7 @@ import (
 // ApplyMetricsServer removes metrics-server when cfg.Enabled is false.
 // ApplyMetricsServer returns an error if anything fails.
 func ApplyMetricsServer(ctx context.Context, snap snap.Snap, cfg types.MetricsServer) error {
-	m := newHelm(snap)
+	m := snap.HelmClient()
 
 	values := map[string]any{
 		"image": map[string]any{
@@ -25,6 +26,6 @@ func ApplyMetricsServer(ctx context.Context, snap snap.Snap, cfg types.MetricsSe
 		},
 	}
 
-	_, err := m.Apply(ctx, featureMetricsServer, statePresentOrDeleted(cfg.GetEnabled()), values)
+	_, err := m.Apply(ctx, chartMetricsServer, helm.StatePresentOrDeleted(cfg.GetEnabled()), values)
 	return err
 }
diff --git a/src/k8s/pkg/k8sd/features/feature_metrics_server_test.go b/src/k8s/pkg/k8sd/features/feature_metrics_server_test.go
new file mode 100644
index 000000000..45f32be6b
--- /dev/null
+++ b/src/k8s/pkg/k8sd/features/feature_metrics_server_test.go
@@ -0,0 +1,56 @@
+package features_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/canonical/k8s/pkg/client/helm"
+	helmmock "github.com/canonical/k8s/pkg/client/helm/mock"
+	"github.com/canonical/k8s/pkg/k8sd/features"
+	"github.com/canonical/k8s/pkg/k8sd/types"
+	snapmock "github.com/canonical/k8s/pkg/snap/mock"
+	"github.com/canonical/k8s/pkg/utils"
+	. "github.com/onsi/gomega"
+)
+
+func TestApplyMetricsServer(t *testing.T) {
+	for _, tc := range []struct {
+		name        string
+		config      types.MetricsServer
+		expectState helm.State
+	}{
+		{
+			name: "Enable",
+			config: types.MetricsServer{
+				Enabled: utils.Pointer(true),
+			},
+			expectState: helm.StatePresent,
+		},
+		{
+			name: "Disable",
+			config: types.MetricsServer{
+				Enabled: utils.Pointer(false),
+			},
+			expectState: helm.StateDeleted,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			g := NewWithT(t)
+			h := &helmmock.Mock{}
+			s := &snapmock.Snap{
+				Mock: snapmock.Mock{
+					HelmClient: h,
+				},
+			}
+
+			err := features.ApplyMetricsServer(context.Background(), s, tc.config)
+			g.Expect(err).ToNot(HaveOccurred())
+
+			g.Expect(h.ApplyCalledWith).To(ConsistOf(SatisfyAll(
+				HaveField("Chart.Name", Equal("metrics-server")),
+				HaveField("Chart.Namespace", Equal("kube-system")),
+				HaveField("State", Equal(tc.expectState)),
+			)))
+		})
+	}
+}
diff --git a/src/k8s/pkg/k8sd/features/feature_network.go b/src/k8s/pkg/k8sd/features/feature_network.go
index 0022bf966..5cc08b3ed 100644
--- a/src/k8s/pkg/k8sd/features/feature_network.go
+++ b/src/k8s/pkg/k8sd/features/feature_network.go
@@ -7,11 +7,11 @@ import (
 	"net"
 	"strings"
 
+	"github.com/canonical/k8s/pkg/client/helm"
 	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/snap"
 	"github.com/canonical/k8s/pkg/utils"
 	"github.com/canonical/k8s/pkg/utils/control"
-	"github.com/canonical/k8s/pkg/utils/k8s"
 )
 
 // ApplyNetwork is used to configure the CNI feature on Canonical Kubernetes.
@@ -21,10 +21,10 @@ import (
 // ApplyNetwork requires that `/sys` is mounted as a shared mount when running under classic snap confinement. This is to ensure that Cilium will be able to automatically mount bpf and cgroups2 on the pods.
 // ApplyNetwork returns an error if anything fails.
 func ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network) error {
-	m := newHelm(snap)
+	m := snap.HelmClient()
 
 	if !cfg.GetEnabled() {
-		if _, err := m.Apply(ctx, featureCiliumCNI, stateDeleted, nil); err != nil {
+		if _, err := m.Apply(ctx, chartCilium, helm.StateDeleted, nil); err != nil {
 			return fmt.Errorf("failed to uninstall network: %w", err)
 		}
 		return nil
@@ -124,7 +124,7 @@ func ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network) error
 		}
 	}
 
-	if _, err := m.Apply(ctx, featureCiliumCNI, statePresent, values); err != nil {
+	if _, err := m.Apply(ctx, chartCilium, helm.StatePresent, values); err != nil {
 		return fmt.Errorf("failed to enable network: %w", err)
 	}
 
@@ -132,7 +132,7 @@ func ApplyNetwork(ctx context.Context, snap snap.Snap, cfg types.Network) error
 }
 
 func rolloutRestartCilium(ctx context.Context, snap snap.Snap, attempts int) error {
-	client, err := k8s.NewClient(snap.KubernetesRESTClientGetter(""))
+	client, err := snap.KubernetesClient("")
 	if err != nil {
 		return fmt.Errorf("failed to create kubernetes client: %w", err)
 	}
diff --git a/src/k8s/pkg/k8sd/features/features.go b/src/k8s/pkg/k8sd/features/features.go
index e1edc8249..cde037e21 100644
--- a/src/k8s/pkg/k8sd/features/features.go
+++ b/src/k8s/pkg/k8sd/features/features.go
@@ -1,47 +1,51 @@
 package features
 
-import "path"
+import (
+	"path"
+
+	"github.com/canonical/k8s/pkg/client/helm"
+)
 
 var (
-	// featureCoreDNS is manifests for the built-in DNS feature, powered by CoreDNS.
-	featureCoreDNS = Feature{
-		name:         "ck-dns",
-		namespace:    "kube-system",
-		manifestPath: path.Join("charts", "coredns-1.29.0.tgz"),
+	// chartCoreDNS is manifests for the built-in DNS feature, powered by CoreDNS.
+	chartCoreDNS = helm.InstallableChart{
+		Name:         "ck-dns",
+		Namespace:    "kube-system",
+		ManifestPath: path.Join("charts", "coredns-1.29.0.tgz"),
 	}
 
-	// featureCiliumCNI is manifests for the built-in CNI feature, powered by Cilium.
-	featureCiliumCNI = Feature{
-		name:         "ck-network",
-		namespace:    "kube-system",
-		manifestPath: path.Join("charts", "cilium-1.15.2.tgz"),
+	// chartCilium is manifests for the built-in CNI feature, powered by Cilium.
+	chartCilium = helm.InstallableChart{
+		Name:         "ck-network",
+		Namespace:    "kube-system",
+		ManifestPath: path.Join("charts", "cilium-1.15.2.tgz"),
 	}
 
-	// featureCiliumLoadBalancer is manifests for the built-in load-balancer feature, powered by Cilium.
-	featureCiliumLoadBalancer = Feature{
-		name:         "ck-loadbalancer",
-		namespace:    "kube-system",
-		manifestPath: path.Join("charts", "ck-loadbalancer"),
+	// chartCiliumLoadBalancer is manifests for the built-in load-balancer feature, powered by Cilium.
+	chartCiliumLoadBalancer = helm.InstallableChart{
+		Name:         "ck-loadbalancer",
+		Namespace:    "kube-system",
+		ManifestPath: path.Join("charts", "ck-loadbalancer"),
 	}
 
-	// featureCiliumGateway is manifests for the built-in gateway feature, powered by Cilium.
-	featureCiliumGateway = Feature{
-		name:         "ck-gateway",
-		namespace:    "kube-system",
-		manifestPath: path.Join("charts", "gateway-api-1.0.0.tgz"),
+	// chartCiliumGateway is manifests for the built-in gateway feature, powered by Cilium.
+	chartCiliumGateway = helm.InstallableChart{
+		Name:         "ck-gateway",
+		Namespace:    "kube-system",
+		ManifestPath: path.Join("charts", "gateway-api-1.0.0.tgz"),
 	}
 
-	// featureLocalStorage is manifests for the built-in local storage feature, powered by Rawfile LocalPV CSI.
-	featureLocalStorage = Feature{
-		name:         "ck-storage",
-		namespace:    "kube-system",
-		manifestPath: path.Join("charts", "rawfile-csi-0.8.0.tgz"),
+	// chartLocalStorage is manifests for the built-in local storage feature, powered by Rawfile LocalPV CSI.
+	chartLocalStorage = helm.InstallableChart{
+		Name:         "ck-storage",
+		Namespace:    "kube-system",
+		ManifestPath: path.Join("charts", "rawfile-csi-0.8.0.tgz"),
 	}
 
-	// featureMetricsServer is manifests for the built-in metrics-server feature, powered by the upstream metrics-server.
-	featureMetricsServer = Feature{
-		name:         "metrics-server",
-		namespace:    "kube-system",
-		manifestPath: path.Join("charts", "metrics-server-3.12.0.tgz"),
+	// chartMetricsServer is manifests for the built-in metrics-server feature, powered by the upstream metrics-server.
+	chartMetricsServer = helm.InstallableChart{
+		Name:         "metrics-server",
+		Namespace:    "kube-system",
+		ManifestPath: path.Join("charts", "metrics-server-3.12.0.tgz"),
 	}
 )
diff --git a/src/k8s/pkg/k8sd/features/interface.go b/src/k8s/pkg/k8sd/features/interface.go
deleted file mode 100644
index 91ac2a177..000000000
--- a/src/k8s/pkg/k8sd/features/interface.go
+++ /dev/null
@@ -1,26 +0,0 @@
-package features
-
-import "context"
-
-// Manager handles the lifecycle of features (manifests + config) on the cluster.
-type Manager interface {
-	// Apply ensures the state of a Feature on the cluster.
-	// When state is statePresent, Apply will install or upgrade the feature using the specified values as configuration. Apply returns true if the feature was not installed, or any values were changed.
-	// When state is stateUpgradeOnly, Apply will upgrade the feature using the specified values as configuration. Apply returns true if the feature was not installed, or any values were changed. An error is returned if the feature is not already installed.
-	// When state is stateDeleted, Apply will ensure that the feature is removed. If the feature is not installed, this is a no-op. Apply returns true if the feature was previously installed.
-	// Apply returns an error in case of failure.
-	Apply(ctx context.Context, f Feature, desired state, values map[string]any) (bool, error)
-}
-
-// feature describes a feature that can be deployed on a running cluster.
-type Feature struct {
-	// name is the install name of the feature.
-	name string
-
-	// namespace is the namespace to install the feature.
-	namespace string
-
-	// manifestPath is the path to the feature's manifest, relative to the Snap.ManifestsDir(), typically "$SNAP/k8s/manifests".
-	// TODO(neoaggelos): this should be a *chart.Chart, and we should use the "embed" package to load it during build.
-	manifestPath string
-}
diff --git a/src/k8s/pkg/k8sd/features/state.go b/src/k8s/pkg/k8sd/features/state.go
deleted file mode 100644
index 46f291855..000000000
--- a/src/k8s/pkg/k8sd/features/state.go
+++ /dev/null
@@ -1,29 +0,0 @@
-package features
-
-// state is used to define how Manager.Apply() handles install, upgrade or delete operations.
-type state int
-
-const (
-	// stateDeleted means that the feature should not be installed.
-	stateDeleted state = iota
-
-	// statePresent means that the feature must be present. If it already exists, it is upgraded with the new configuration, otherwise it is installed.
-	statePresent
-
-	// stateUpgradeOnly means that the feature will be refreshed if installed, fail otherwise.
-	stateUpgradeOnly
-)
-
-func statePresentOrDeleted(enabled bool) state {
-	if enabled {
-		return statePresent
-	}
-	return stateDeleted
-}
-
-func stateUpgradeOnlyOrDeleted(enabled bool) state {
-	if enabled {
-		return stateUpgradeOnly
-	}
-	return stateDeleted
-}
diff --git a/src/k8s/pkg/snap/interface.go b/src/k8s/pkg/snap/interface.go
index 120fc5e99..0fa9fb610 100644
--- a/src/k8s/pkg/snap/interface.go
+++ b/src/k8s/pkg/snap/interface.go
@@ -4,7 +4,8 @@ import (
 	"context"
 
 	"github.com/canonical/k8s/pkg/client/dqlite"
-	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"github.com/canonical/k8s/pkg/client/helm"
+	"github.com/canonical/k8s/pkg/client/kubernetes"
 )
 
 // Snap abstracts file system paths and interacting with the k8s services.
@@ -44,10 +45,10 @@ type Snap interface {
 
 	LockFilesDir() string // /var/snap/k8s/common/lock
 
-	ManifestsDir() string // /snap/k8s/current/k8s/manifests
+	KubernetesClient(namespace string) (*kubernetes.Client, error)     // admin kubernetes client
+	KubernetesNodeClient(namespace string) (*kubernetes.Client, error) // node kubernetes client
 
-	KubernetesRESTClientGetter(namespace string) genericclioptions.RESTClientGetter     // admin kubernetes client
-	KubernetesNodeRESTClientGetter(namespace string) genericclioptions.RESTClientGetter // node kubernetes client
+	HelmClient() helm.Client // admin helm client
 
 	K8sDqliteClient(ctx context.Context) (*dqlite.Client, error) // go-dqlite client for k8s-dqlite
 }
diff --git a/src/k8s/pkg/snap/mock/mock.go b/src/k8s/pkg/snap/mock/mock.go
index 8af0035fb..0cd9e13e8 100644
--- a/src/k8s/pkg/snap/mock/mock.go
+++ b/src/k8s/pkg/snap/mock/mock.go
@@ -4,39 +4,40 @@ import (
 	"context"
 
 	"github.com/canonical/k8s/pkg/client/dqlite"
+	"github.com/canonical/k8s/pkg/client/helm"
+	"github.com/canonical/k8s/pkg/client/kubernetes"
 	"github.com/canonical/k8s/pkg/snap"
-	"k8s.io/cli-runtime/pkg/genericclioptions"
 )
 
 type Mock struct {
-	Strict                         bool
-	OnLXD                          bool
-	OnLXDErr                       error
-	UID                            int
-	GID                            int
-	KubernetesConfigDir            string
-	KubernetesPKIDir               string
-	EtcdPKIDir                     string
-	KubeletRootDir                 string
-	CNIConfDir                     string
-	CNIBinDir                      string
-	CNIPlugins                     []string
-	CNIPluginsBinary               string
-	ContainerdConfigDir            string
-	ContainerdExtraConfigDir       string
-	ContainerdRegistryConfigDir    string
-	ContainerdRootDir              string
-	ContainerdSocketDir            string
-	ContainerdStateDir             string
-	K8sdStateDir                   string
-	K8sDqliteStateDir              string
-	ServiceArgumentsDir            string
-	ServiceExtraConfigDir          string
-	LockFilesDir                   string
-	ManifestsDir                   string
-	KubernetesRESTClientGetter     genericclioptions.RESTClientGetter
-	KubernetesNodeRESTClientGetter genericclioptions.RESTClientGetter
-	K8sDqliteClient                *dqlite.Client
+	Strict                      bool
+	OnLXD                       bool
+	OnLXDErr                    error
+	UID                         int
+	GID                         int
+	KubernetesConfigDir         string
+	KubernetesPKIDir            string
+	EtcdPKIDir                  string
+	KubeletRootDir              string
+	CNIConfDir                  string
+	CNIBinDir                   string
+	CNIPlugins                  []string
+	CNIPluginsBinary            string
+	ContainerdConfigDir         string
+	ContainerdExtraConfigDir    string
+	ContainerdRegistryConfigDir string
+	ContainerdRootDir           string
+	ContainerdSocketDir         string
+	ContainerdStateDir          string
+	K8sdStateDir                string
+	K8sDqliteStateDir           string
+	ServiceArgumentsDir         string
+	ServiceExtraConfigDir       string
+	LockFilesDir                string
+	KubernetesClient            *kubernetes.Client
+	KubernetesNodeClient        *kubernetes.Client
+	HelmClient                  helm.Client
+	K8sDqliteClient             *dqlite.Client
 }
 
 // Snap is a mock implementation for snap.Snap.
@@ -145,14 +146,14 @@ func (s *Snap) ServiceExtraConfigDir() string {
 func (s *Snap) LockFilesDir() string {
 	return s.Mock.LockFilesDir
 }
-func (s *Snap) ManifestsDir() string {
-	return s.Mock.ManifestsDir
+func (s *Snap) KubernetesClient(namespace string) (*kubernetes.Client, error) {
+	return s.Mock.KubernetesClient, nil
 }
-func (s *Snap) KubernetesRESTClientGetter(namespace string) genericclioptions.RESTClientGetter {
-	return s.Mock.KubernetesRESTClientGetter
+func (s *Snap) KubernetesNodeClient(namespace string) (*kubernetes.Client, error) {
+	return s.Mock.KubernetesNodeClient, nil
 }
-func (s *Snap) KubernetesNodeRESTClientGetter(namespace string) genericclioptions.RESTClientGetter {
-	return s.Mock.KubernetesNodeRESTClientGetter
+func (s *Snap) HelmClient() helm.Client {
+	return s.Mock.HelmClient
 }
 func (s *Snap) K8sDqliteClient(context.Context) (*dqlite.Client, error) {
 	return s.Mock.K8sDqliteClient, nil
diff --git a/src/k8s/pkg/snap/snap.go b/src/k8s/pkg/snap/snap.go
index 33204f91e..6bdbb51c4 100644
--- a/src/k8s/pkg/snap/snap.go
+++ b/src/k8s/pkg/snap/snap.go
@@ -8,6 +8,8 @@ import (
 	"strings"
 
 	"github.com/canonical/k8s/pkg/client/dqlite"
+	"github.com/canonical/k8s/pkg/client/helm"
+	"github.com/canonical/k8s/pkg/client/kubernetes"
 	"github.com/canonical/k8s/pkg/utils"
 	"github.com/moby/sys/mountinfo"
 	"gopkg.in/yaml.v2"
@@ -190,13 +192,9 @@ func (s *snap) ContainerdRegistryConfigDir() string {
 	return path.Join(s.snapCommonDir, "etc", "containerd", "hosts.d")
 }
 
-func (s *snap) ManifestsDir() string {
-	return path.Join(s.snapDir, "k8s", "manifests")
-}
-
-func (s *snap) KubernetesRESTClientGetter(namespace string) genericclioptions.RESTClientGetter {
+func (s *snap) restClientGetter(path string, namespace string) genericclioptions.RESTClientGetter {
 	flags := &genericclioptions.ConfigFlags{
-		KubeConfig: &[]string{"/etc/kubernetes/admin.conf"}[0],
+		KubeConfig: utils.Pointer(path),
 	}
 	if namespace != "" {
 		flags.Namespace = &namespace
@@ -204,14 +202,21 @@ func (s *snap) KubernetesRESTClientGetter(namespace string) genericclioptions.RE
 	return flags
 }
 
-func (s *snap) KubernetesNodeRESTClientGetter(namespace string) genericclioptions.RESTClientGetter {
-	flags := &genericclioptions.ConfigFlags{
-		KubeConfig: utils.Pointer(path.Join(s.KubernetesConfigDir(), "kubelet.conf")),
-	}
-	if namespace != "" {
-		flags.Namespace = &namespace
-	}
-	return flags
+func (s *snap) KubernetesClient(namespace string) (*kubernetes.Client, error) {
+	return kubernetes.NewClient(s.restClientGetter(path.Join(s.KubernetesConfigDir(), "admin.conf"), namespace))
+}
+
+func (s *snap) KubernetesNodeClient(namespace string) (*kubernetes.Client, error) {
+	return kubernetes.NewClient(s.restClientGetter(path.Join(s.KubernetesConfigDir(), "kubelet.conf"), namespace))
+}
+
+func (s *snap) HelmClient() helm.Client {
+	return helm.NewClient(
+		path.Join(s.snapDir, "k8s", "manifests"),
+		func(namespace string) genericclioptions.RESTClientGetter {
+			return s.restClientGetter(path.Join(s.KubernetesConfigDir(), "admin.conf"), namespace)
+		},
+	)
 }
 
 func (s *snap) K8sDqliteClient(ctx context.Context) (*dqlite.Client, error) {