From 7cfe0c9249fc25ffc9ac246bfa9a04a6716d2d19 Mon Sep 17 00:00:00 2001 From: Atanas Dinov Date: Wed, 17 Jan 2024 17:35:00 +0200 Subject: [PATCH] Add Kubernetes component (#110) * Bootstrap kubernetes component Signed-off-by: Atanas Dinov * Configure RKE2 Signed-off-by: Atanas Dinov * Remove KUBECONFIG and PATH env variables from the installer script Signed-off-by: Atanas Dinov --------- Signed-off-by: Atanas Dinov --- pkg/combustion/combustion.go | 4 + pkg/combustion/kubernetes.go | 149 +++++++++++ pkg/combustion/kubernetes_test.go | 251 ++++++++++++++++++ .../templates/15-rke2-installer.sh.tpl | 23 ++ 4 files changed, 427 insertions(+) create mode 100644 pkg/combustion/kubernetes.go create mode 100644 pkg/combustion/kubernetes_test.go create mode 100644 pkg/combustion/templates/15-rke2-installer.sh.tpl diff --git a/pkg/combustion/combustion.go b/pkg/combustion/combustion.go index 134a2fed..3bd985a9 100644 --- a/pkg/combustion/combustion.go +++ b/pkg/combustion/combustion.go @@ -84,6 +84,10 @@ func Configure(ctx *image.Context) error { name: keymapComponentName, runnable: configureKeymap, }, + { + name: k8sComponentName, + runnable: configureKubernetes, + }, } for _, component := range combustionComponents { diff --git a/pkg/combustion/kubernetes.go b/pkg/combustion/kubernetes.go new file mode 100644 index 00000000..e89f09a0 --- /dev/null +++ b/pkg/combustion/kubernetes.go @@ -0,0 +1,149 @@ +package combustion + +import ( + _ "embed" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/suse-edge/edge-image-builder/pkg/fileio" + "github.com/suse-edge/edge-image-builder/pkg/image" + "github.com/suse-edge/edge-image-builder/pkg/log" + "github.com/suse-edge/edge-image-builder/pkg/template" + "go.uber.org/zap" +) + +const ( + k8sComponentName = "kubernetes" + k8sConfigDir = "kubernetes" + k8sConfigFile = "config.yaml" + rke2InstallScript = "15-rke2-install.sh" +) + +var ( + //go:embed templates/15-rke2-installer.sh.tpl + rke2InstallerScript string +) + +func configureKubernetes(ctx *image.Context) ([]string, error) { + version := ctx.ImageDefinition.Kubernetes.Version + + if version == "" { + log.AuditComponentSkipped(k8sComponentName) + return nil, nil + } + + // Show a message to the user to indicate that the Kubernetes component + // is usually taking longer to complete due to downloading files + log.Audit("Configuring Kubernetes component...") + + configureFunc := kubernetesConfigurator(version) + if configureFunc == nil { + log.AuditComponentFailed(k8sComponentName) + return nil, fmt.Errorf("cannot configure kubernetes version: %s", version) + } + + script, err := configureFunc(ctx) + if err != nil { + log.AuditComponentFailed(k8sComponentName) + return nil, fmt.Errorf("configuring kubernetes components: %w", err) + } + + log.AuditComponentSuccessful(k8sComponentName) + return []string{script}, nil +} + +func kubernetesConfigurator(version string) func(*image.Context) (string, error) { + switch { + case strings.Contains(version, image.KubernetesDistroRKE2): + return configureRKE2 + case strings.Contains(version, image.KubernetesDistroK3S): + return configureK3S + default: + return nil + } +} + +func installKubernetesScript(ctx *image.Context, distribution string) error { + sourcePath := "/" // root level of the container image + destPath := ctx.CombustionDir + + return ctx.KubernetesScriptInstaller.InstallScript(distribution, sourcePath, destPath) +} + +func configureK3S(_ *image.Context) (string, error) { + return "", fmt.Errorf("not implemented yet") +} + +func configureRKE2(ctx *image.Context) (string, error) { + if err := installKubernetesScript(ctx, image.KubernetesDistroRKE2); err != nil { + return "", fmt.Errorf("copying RKE2 installer script: %w", err) + } + + configFile, err := copyKubernetesConfig(ctx, image.KubernetesDistroRKE2) + if err != nil { + return "", fmt.Errorf("copying RKE2 config: %w", err) + } + + installPath, imagesPath, err := ctx.KubernetesArtefactDownloader.DownloadArtefacts( + ctx.ImageDefinition.Kubernetes, + ctx.ImageDefinition.Image.Arch, + ctx.CombustionDir, + ) + if err != nil { + return "", fmt.Errorf("downloading RKE2 artefacts: %w", err) + } + + rke2 := struct { + image.Kubernetes + ConfigFile string + InstallPath string + ImagesPath string + }{ + Kubernetes: ctx.ImageDefinition.Kubernetes, + ConfigFile: configFile, + InstallPath: installPath, + ImagesPath: imagesPath, + } + + data, err := template.Parse(rke2InstallScript, rke2InstallerScript, &rke2) + if err != nil { + return "", fmt.Errorf("parsing RKE2 install template: %w", err) + } + + installScript := filepath.Join(ctx.CombustionDir, rke2InstallScript) + if err = os.WriteFile(installScript, []byte(data), fileio.ExecutablePerms); err != nil { + return "", fmt.Errorf("writing RKE2 install script: %w", err) + } + + return rke2InstallScript, nil +} + +func copyKubernetesConfig(ctx *image.Context, distro string) (string, error) { + if !isComponentConfigured(ctx, k8sConfigDir) { + zap.S().Info("Kubernetes config file not provided") + return "", nil + } + + configDir := generateComponentPath(ctx, k8sConfigDir) + configFile := filepath.Join(configDir, k8sConfigFile) + + _, err := os.Stat(configFile) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return "", fmt.Errorf("kubernetes component directory exists but does not contain config.yaml") + } + return "", fmt.Errorf("error checking kubernetes config file: %w", err) + } + + destFile := fmt.Sprintf("%s_config.yaml", distro) + + if err = fileio.CopyFile(configFile, filepath.Join(ctx.CombustionDir, destFile), fileio.NonExecutablePerms); err != nil { + return "", fmt.Errorf("copying kubernetes config file: %w", err) + } + + return destFile, nil +} diff --git a/pkg/combustion/kubernetes_test.go b/pkg/combustion/kubernetes_test.go new file mode 100644 index 00000000..730ed848 --- /dev/null +++ b/pkg/combustion/kubernetes_test.go @@ -0,0 +1,251 @@ +package combustion + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/suse-edge/edge-image-builder/pkg/fileio" + "github.com/suse-edge/edge-image-builder/pkg/image" +) + +type mockKubernetesScriptInstaller struct { + installScript func(distribution, sourcePath, destPath string) error +} + +func (m mockKubernetesScriptInstaller) InstallScript(distribution, sourcePath, destPath string) error { + if m.installScript != nil { + return m.installScript(distribution, sourcePath, destPath) + } + + panic("not implemented") +} + +type mockKubernetesArtefactDownloader struct { + downloadArtefacts func(kubernetes image.Kubernetes, arch image.Arch, destPath string) (string, string, error) +} + +func (m mockKubernetesArtefactDownloader) DownloadArtefacts(kubernetes image.Kubernetes, arch image.Arch, destPath string) (installPath, imagesPath string, err error) { + if m.downloadArtefacts != nil { + return m.downloadArtefacts(kubernetes, arch, destPath) + } + + panic("not implemented") +} + +func TestConfigureKubernetes_Skipped(t *testing.T) { + ctx := &image.Context{ + ImageDefinition: &image.Definition{}, + } + + scripts, err := configureKubernetes(ctx) + require.NoError(t, err) + assert.Nil(t, scripts) +} + +func TestConfigureKubernetes_UnsupportedVersion(t *testing.T) { + ctx := &image.Context{ + ImageDefinition: &image.Definition{ + Kubernetes: image.Kubernetes{ + Version: "v1.29.0", + }, + }, + } + + scripts, err := configureKubernetes(ctx) + require.Error(t, err) + assert.EqualError(t, err, "cannot configure kubernetes version: v1.29.0") + assert.Nil(t, scripts) +} + +func TestConfigureKubernetes_UnimplementedK3S(t *testing.T) { + ctx := &image.Context{ + ImageDefinition: &image.Definition{ + Kubernetes: image.Kubernetes{ + Version: "v1.29.0+k3s1", + }, + }, + } + + scripts, err := configureKubernetes(ctx) + require.Error(t, err) + assert.EqualError(t, err, "configuring kubernetes components: not implemented yet") + assert.Nil(t, scripts) +} + +func TestConfigureKubernetes_ScriptInstallerErrorRKE2(t *testing.T) { + ctx := &image.Context{ + ImageDefinition: &image.Definition{ + Kubernetes: image.Kubernetes{ + Version: "v1.29.0+rke2r1", + }, + }, + KubernetesScriptInstaller: mockKubernetesScriptInstaller{ + installScript: func(distribution, sourcePath, destPath string) error { + return fmt.Errorf("some error") + }, + }, + } + + scripts, err := configureKubernetes(ctx) + require.Error(t, err) + assert.EqualError(t, err, "configuring kubernetes components: copying RKE2 installer script: some error") + assert.Nil(t, scripts) +} + +func TestConfigureKubernetes_ArtefactDownloaderErrorRKE2(t *testing.T) { + ctx := &image.Context{ + ImageDefinition: &image.Definition{ + Kubernetes: image.Kubernetes{ + Version: "v1.29.0+rke2r1", + }, + }, + KubernetesScriptInstaller: mockKubernetesScriptInstaller{ + installScript: func(distribution, sourcePath, destPath string) error { + return nil + }, + }, + KubernetesArtefactDownloader: mockKubernetesArtefactDownloader{ + downloadArtefacts: func(kubernetes image.Kubernetes, arch image.Arch, destPath string) (string, string, error) { + return "", "", fmt.Errorf("some error") + }, + }, + } + + scripts, err := configureKubernetes(ctx) + require.Error(t, err) + assert.EqualError(t, err, "configuring kubernetes components: downloading RKE2 artefacts: some error") + assert.Nil(t, scripts) +} + +func TestConfigureKubernetes_ConfigFileMissingErrorRKE2(t *testing.T) { + ctx, teardown := setupContext(t) + defer teardown() + + ctx.ImageDefinition.Kubernetes = image.Kubernetes{ + Version: "v1.29.0+rke2r1", + } + ctx.KubernetesScriptInstaller = mockKubernetesScriptInstaller{ + installScript: func(distribution, sourcePath, destPath string) error { + return nil + }, + } + ctx.KubernetesArtefactDownloader = mockKubernetesArtefactDownloader{ + downloadArtefacts: func(kubernetes image.Kubernetes, arch image.Arch, destPath string) (string, string, error) { + return "", "", nil + }, + } + + require.NoError(t, os.Mkdir(filepath.Join(ctx.ImageConfigDir, k8sConfigDir), os.ModePerm)) + + scripts, err := configureKubernetes(ctx) + require.Error(t, err) + assert.EqualError(t, err, "configuring kubernetes components: copying RKE2 config: "+ + "kubernetes component directory exists but does not contain config.yaml") + assert.Nil(t, scripts) +} + +func TestConfigureKubernetes_SuccessfulRKE2Server(t *testing.T) { + ctx, teardown := setupContext(t) + defer teardown() + + ctx.ImageDefinition.Kubernetes = image.Kubernetes{ + Version: "v1.29.0+rke2r1", + } + ctx.KubernetesScriptInstaller = mockKubernetesScriptInstaller{ + installScript: func(distribution, sourcePath, destPath string) error { + return nil + }, + } + ctx.KubernetesArtefactDownloader = mockKubernetesArtefactDownloader{ + downloadArtefacts: func(kubernetes image.Kubernetes, arch image.Arch, destPath string) (string, string, error) { + return "server-installer", "server-images", nil + }, + } + + configDir := filepath.Join(ctx.ImageConfigDir, k8sConfigDir) + require.NoError(t, os.Mkdir(configDir, os.ModePerm)) + configFile := filepath.Join(configDir, k8sConfigFile) + require.NoError(t, os.WriteFile(configFile, []byte("some-config-data"), os.ModePerm)) + + scripts, err := configureKubernetes(ctx) + require.NoError(t, err) + require.Len(t, scripts, 1) + + // Script file assertions + scriptPath := filepath.Join(ctx.CombustionDir, scripts[0]) + + info, err := os.Stat(scriptPath) + require.NoError(t, err) + + assert.Equal(t, fileio.ExecutablePerms, info.Mode()) + + b, err := os.ReadFile(scriptPath) + require.NoError(t, err) + + contents := string(b) + assert.NotContains(t, contents, "export INSTALL_RKE2_TYPE=server", + "INSTALL_RKE2_TYPE is set when the definition file does not explicitly set it") + assert.Contains(t, contents, "cp server-images/* /var/lib/rancher/rke2/agent/images/") + assert.Contains(t, contents, "cp rke2_config.yaml /etc/rancher/rke2/config.yaml") + assert.Contains(t, contents, "export INSTALL_RKE2_ARTIFACT_PATH=server-installer") + assert.Contains(t, contents, "systemctl enable rke2-server.service") + + // Config file assertions + configPath := filepath.Join(ctx.CombustionDir, "rke2_config.yaml") + + info, err = os.Stat(configPath) + require.NoError(t, err) + + assert.Equal(t, fileio.NonExecutablePerms, info.Mode()) + + b, err = os.ReadFile(configPath) + require.NoError(t, err) + + contents = string(b) + assert.Equal(t, "some-config-data", contents) +} + +func TestConfigureKubernetes_SuccessfulRKE2Agent(t *testing.T) { + ctx, teardown := setupContext(t) + defer teardown() + + ctx.ImageDefinition.Kubernetes = image.Kubernetes{ + Version: "v1.29.0+rke2r1", + NodeType: "agent", + } + ctx.KubernetesScriptInstaller = mockKubernetesScriptInstaller{ + installScript: func(distribution, sourcePath, destPath string) error { + return nil + }, + } + ctx.KubernetesArtefactDownloader = mockKubernetesArtefactDownloader{ + downloadArtefacts: func(kubernetes image.Kubernetes, arch image.Arch, destPath string) (string, string, error) { + return "agent-installer", "agent-images", nil + }, + } + + scripts, err := configureKubernetes(ctx) + require.NoError(t, err) + require.Len(t, scripts, 1) + + scriptPath := filepath.Join(ctx.CombustionDir, scripts[0]) + + info, err := os.Stat(scriptPath) + require.NoError(t, err) + + assert.Equal(t, fileio.ExecutablePerms, info.Mode()) + + b, err := os.ReadFile(scriptPath) + require.NoError(t, err) + + contents := string(b) + assert.Contains(t, contents, "cp agent-images/* /var/lib/rancher/rke2/agent/images/") + assert.Contains(t, contents, "export INSTALL_RKE2_TYPE=agent") + assert.NotContains(t, contents, "cp rke2_config.yaml /etc/rancher/rke2/config.yaml") + assert.Contains(t, contents, "export INSTALL_RKE2_ARTIFACT_PATH=agent-installer") + assert.Contains(t, contents, "systemctl enable rke2-agent.service") +} diff --git a/pkg/combustion/templates/15-rke2-installer.sh.tpl b/pkg/combustion/templates/15-rke2-installer.sh.tpl new file mode 100644 index 00000000..116c9ed6 --- /dev/null +++ b/pkg/combustion/templates/15-rke2-installer.sh.tpl @@ -0,0 +1,23 @@ +#!/bin/bash +set -euo pipefail + +mount /var +mkdir -p /var/lib/rancher/rke2/agent/images/ +cp {{ .ImagesPath }}/* /var/lib/rancher/rke2/agent/images/ +umount /var + +{{- if .ConfigFile }} +mkdir -p /etc/rancher/rke2/ +cp {{ .ConfigFile }} /etc/rancher/rke2/config.yaml +{{- end }} + +{{- if .NodeType }} +export INSTALL_RKE2_TYPE={{ .NodeType }} +{{- end }} + +export INSTALL_RKE2_TAR_PREFIX=/opt/rke2 +export INSTALL_RKE2_ARTIFACT_PATH={{ .InstallPath }} + +./rke2_installer.sh + +systemctl enable rke2-{{ or .NodeType "server" }}.service