Cleans up all containerd-related resources on snap removal (#827)

Currently, when removing the snap, the /var/run/containerd folder is not
properly removed, as it is a folder. This fixes this issue.

Additionally removes other containerd-related folders: /etc/containerd
and /var/lib/containerd.

We're also removing /opt/cni/bin on snap removal, which is created when
bootstrapping the node. As we're removing the k8s snap, we no longer
need this folder either.

k8s/lib.sh

@@ -99,9 +99,11 @@ k8s::remove::containerd() {
 
   # only remove containerd if the snap was already bootstrapped.
   # this is to prevent removing containerd when it is not installed by the snap.
-  if [ -f "$SNAP_COMMON/lock/containerd-socket-path" ]; then
-     rm -f $(cat "$SNAP_COMMON/lock/containerd-socket-path")
-  fi
+  for file in "containerd-socket-path" "containerd-config-dir" "containerd-root-dir" "containerd-cni-bin-dir"; do
+    if [ -f "$SNAP_COMMON/lock/$file" ]; then
+      rm -rf $(cat "$SNAP_COMMON/lock/$file")
+    fi
+  done
 }
 
 # Run a ctr command against the local containerd socket

src/k8s/pkg/k8sd/setup/containerd.go

@@ -159,14 +159,30 @@ func Containerd(snap snap.Snap, extraContainerdConfig map[string]any, extraArgs
 		}
 	}
 
-	// Write the containerd socket path to a file to properly clean-up on removal.
-	if err := utils.WriteFile(filepath.Join(snap.LockFilesDir(), "containerd-socket-path"), []byte(snap.ContainerdSocketDir()), 0o600); err != nil {
-		return fmt.Errorf("failed to write containerd-socket-path: %w", err)
+	if err := saveSnapContainerdPaths(snap); err != nil {
+		return err
 	}
 
 	return nil
 }
 
+func saveSnapContainerdPaths(s snap.Snap) error {
+	// Write the containerd-related paths to files to properly clean-up on removal.
+	m := map[string]string{
+		"containerd-socket-path": s.ContainerdSocketDir(),
+		"containerd-config-dir":  s.ContainerdConfigDir(),
+		"containerd-root-dir":    s.ContainerdRootDir(),
+		"containerd-cni-bin-dir": s.CNIBinDir(),
+	}
+
+	for filename, content := range m {
+		if err := utils.WriteFile(filepath.Join(s.LockFilesDir(), filename), []byte(content), 0o600); err != nil {
+			return fmt.Errorf("failed to write %s: %w", filename, err)
+		}
+	}
+	return nil
+}
+
 func init() {
 	images.Register(defaultPauseImage)
 }

src/k8s/pkg/k8sd/setup/containerd_test.go

@@ -121,10 +121,19 @@ func TestContainerd(t *testing.T) {
 		})
 	})
 
-	t.Run("Lockfile", func(t *testing.T) {
+	t.Run("Lockfiles", func(t *testing.T) {
 		g := NewWithT(t)
-		b, err := os.ReadFile(filepath.Join(s.LockFilesDir(), "containerd-socket-path"))
-		g.Expect(err).To(Not(HaveOccurred()))
-		g.Expect(string(b)).To(Equal(s.ContainerdSocketDir()))
+		m := map[string]string{
+			"containerd-socket-path": s.ContainerdSocketDir(),
+			"containerd-config-dir":  s.ContainerdConfigDir(),
+			"containerd-root-dir":    s.ContainerdRootDir(),
+			"containerd-cni-bin-dir": s.CNIBinDir(),
+		}
+		for filename, content := range m {
+
+			b, err := os.ReadFile(filepath.Join(s.LockFilesDir(), filename))
+			g.Expect(err).To(Not(HaveOccurred()))
+			g.Expect(string(b)).To(Equal(content))
+		}
 	})
 }

tests/integration/tests/test_cleanup.py

@@ -9,6 +9,13 @@
 
 LOG = logging.getLogger(__name__)
 
+CONTAINERD_PATHS = [
+    "/etc/containerd",
+    "/opt/cni/bin",
+    "/run/containerd",
+    "/var/lib/containerd",
+]
+
 
 @pytest.mark.node_count(1)
 def test_node_cleanup(instances: List[harness.Instance]):
@@ -17,3 +24,10 @@ def test_node_cleanup(instances: List[harness.Instance]):
     util.wait_for_network(instance)
 
     util.remove_k8s_snap(instance)
+
+    # Check that the containerd-related folders are removed on snap removal.
+    process = instance.exec(
+        ["ls", *CONTAINERD_PATHS], capture_output=True, text=True, check=False
+    )
+    for path in CONTAINERD_PATHS:
+        assert f"cannot access '{path}': No such file or directory" in process.stderr