diff --git a/node-bootstrapper/build.sh b/node-bootstrapper/build.sh index 3e0388a5de7..01d8b5c4006 100755 --- a/node-bootstrapper/build.sh +++ b/node-bootstrapper/build.sh @@ -1,3 +1,4 @@ +go test ./... GOOS=linux GOARCH=amd64 go build -o ./dist/node-bootstrapper-linux-amd64 GOOS=linux GOARCH=arm64 go build -o ./dist/node-bootstrapper-linux-arm64 GOOS=windows GOARCH=amd64 go build -o ./dist/node-bootstrapper-windows-amd64.exe diff --git a/node-bootstrapper/custom_data_test.go b/node-bootstrapper/custom_data_test.go new file mode 100644 index 00000000000..29493295208 --- /dev/null +++ b/node-bootstrapper/custom_data_test.go @@ -0,0 +1,161 @@ +package main + +import ( + "io/fs" + "testing" + + "github.com/Azure/agentbaker/pkg/agent/datamodel" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCustomData(t *testing.T) { + getFile := func(t *testing.T, nbc *datamodel.NodeBootstrappingConfiguration, path string, expectedMode fs.FileMode) string { + t.Helper() + files, err := customData(nbc) + require.NoError(t, err) + require.Contains(t, files, path) + actual := files[path] + assert.Equal(t, expectedMode, actual.Mode) + return actual.Content + } + + t.Run("kubeconfig", func(t *testing.T) { + nbc := validNBC() + actual := getFile(t, nbc, "/var/lib/kubelet/kubeconfig", 0644) + expected := ` +apiVersion: v1 +kind: Config +clusters: +- name: localcluster + cluster: + certificate-authority: /etc/kubernetes/certs/ca.crt + server: https://:443 +users: +- name: client + user: + client-certificate: /etc/kubernetes/certs/client.crt + client-key: /etc/kubernetes/certs/client.key +contexts: +- context: + cluster: localcluster + user: client + name: localclustercontext +current-context: localclustercontext +` + assert.YAMLEq(t, expected, actual) + }) + + t.Run("ca.crt", func(t *testing.T) { + nbc := validNBC() + actual := getFile(t, nbc, "/etc/kubernetes/certs/ca.crt", 0600) + expected := "test-ca-cert" + assert.Equal(t, expected, actual) + }) + + t.Run("bootstrap-kubeconfig", func(t *testing.T) { + nbc := validNBC() + nbc.KubeletClientTLSBootstrapToken = Ptr("test-token") + actual := getFile(t, nbc, "/var/lib/kubelet/bootstrap-kubeconfig", 0644) + expected := `apiVersion: v1 +clusters: + - cluster: + certificate-authority: /etc/kubernetes/certs/ca.crt + server: https://:443 + name: localcluster +contexts: + - context: + cluster: localcluster + user: kubelet-bootstrap + name: bootstrap-context +current-context: bootstrap-context +kind: Config +users: + - name: kubelet-bootstrap + user: + token: test-token +` + assert.YAMLEq(t, expected, actual) + }) + + t.Run("exec_start.conf", func(t *testing.T) { + nbc := validNBC() + actual := getFile(t, nbc, "/etc/systemd/system/docker.service.d/exec_start.conf", 0644) + nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.DockerBridgeSubnet = "1.1.1.1" + expected := `[Service] +ExecStart= +ExecStart=/usr/bin/dockerd -H fd:// --storage-driver=overlay2 --bip=1.1.1.1 +ExecStartPost=/sbin/iptables -P FORWARD ACCEPT +#EOF` + assert.Equal(t, expected, actual) + }) + + t.Run("docker-daemon.json", func(t *testing.T) { + nbc := validNBC() + actual := getFile(t, nbc, "/etc/docker/daemon.json", 0644) + expected := ` +{ + "data-root":"/mnt/aks/containers", + "live-restore":true, + "log-driver":"json-file", + "log-opts": { + "max-file":"5", + "max-size":"50m" + } +} +` + assert.JSONEq(t, expected, actual) + }) + t.Run("kubelet", func(t *testing.T) { + nbc := validNBC() + actual := getFile(t, nbc, "/etc/default/kubelet", 0644) + expected := `KUBELET_FLAGS= +KUBELET_REGISTER_SCHEDULABLE=true +NETWORK_POLICY= +KUBELET_NODE_LABELS=agentpool=,kubernetes.azure.com/agentpool= +` + assert.Equal(t, expected, actual) + }) + + t.Run("containerDMCRHosts", func(t *testing.T) { + nbc := validNBC() + nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{ + PrivateEgress: &datamodel.PrivateEgress{ + Enabled: true, + ContainerRegistryServer: "test-registry", + }, + } + actual := getFile(t, nbc, "/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml", 0644) + expected := `[host."https://test-registry"] +capabilities = ["pull", "resolve"] +` + assert.Equal(t, expected, actual) + }) +} + +func validNBC() *datamodel.NodeBootstrappingConfiguration { + return &datamodel.NodeBootstrappingConfiguration{ + ContainerService: &datamodel.ContainerService{ + Properties: &datamodel.Properties{ + CertificateProfile: &datamodel.CertificateProfile{ + CaCertificate: "test-ca-cert", + }, + OrchestratorProfile: &datamodel.OrchestratorProfile{ + OrchestratorType: datamodel.Kubernetes, + OrchestratorVersion: "1.31.0", + KubernetesConfig: &datamodel.KubernetesConfig{ + DockerBridgeSubnet: "1.1.1.1", + }, + }, + }, + }, + CustomSecureTLSBootstrapAADServerAppID: "test-app-id", + AgentPoolProfile: &datamodel.AgentPoolProfile{ + KubeletDiskType: datamodel.TempDisk, + }, + } +} + +func Ptr[T any](input T) *T { + return &input +} diff --git a/node-bootstrapper/go.mod b/node-bootstrapper/go.mod index e6b8830c543..75ba118b85d 100644 --- a/node-bootstrapper/go.mod +++ b/node-bootstrapper/go.mod @@ -4,13 +4,20 @@ go 1.23.0 replace github.com/Azure/agentbaker => ../ -require github.com/Azure/agentbaker v1.0.1238 +require ( + github.com/Azure/agentbaker v1.0.1238 + github.com/stretchr/testify v1.9.0 + sigs.k8s.io/yaml v1.4.0 +) require ( github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apimachinery v0.28.5 // indirect ) diff --git a/node-bootstrapper/go.sum b/node-bootstrapper/go.sum index 1fdf831c8c0..ae0976f3afd 100644 --- a/node-bootstrapper/go.sum +++ b/node-bootstrapper/go.sum @@ -12,8 +12,13 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.12.2 h1:Ke9m3h2Hu0wsZ45yewCqhYr3Z+emcNTuLY2nMWCkrSI= @@ -24,6 +29,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= @@ -32,9 +39,14 @@ golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/apimachinery v0.28.5 h1:EEj2q1qdTcv2p5wl88KavAn3VlFRjREgRu8Sm/EuMPY= k8s.io/apimachinery v0.28.5/go.mod h1:wI37ncBvfAoswfq626yPTe6Bz1c22L7uaJ8dho83mgg= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/node-bootstrapper/main.go b/node-bootstrapper/main.go index b9b394eac2e..6cfbe7ffd17 100644 --- a/node-bootstrapper/main.go +++ b/node-bootstrapper/main.go @@ -8,9 +8,11 @@ import ( "log/slog" "os" "os/exec" + "strings" "github.com/Azure/agentbaker/pkg/agent" "github.com/Azure/agentbaker/pkg/agent/datamodel" + yaml "sigs.k8s.io/yaml/goyaml.v3" // TODO: should we use JSON instead of YAML to avoid 3rd party dependencies? ) func main() { @@ -33,7 +35,9 @@ func run(ctx context.Context) error { return err } - // TODO: apply UserData from NodeBootstrappingConfiguration or delete it completely + if err := writeCustomData(config); err != nil { + return fmt.Errorf("write custom data: %w", err) + } if err := provisionStart(ctx, config); err != nil { return fmt.Errorf("provision start: %w", err) @@ -84,3 +88,235 @@ func CSEScript(ctx context.Context, config *datamodel.NodeBootstrappingConfigura } return nodeBootstrapping.CSE, nil } + +func writeCustomData(config *datamodel.NodeBootstrappingConfiguration) error { + files, err := customData(config) + if err != nil { + return err + } + for path, file := range files { + if err := os.WriteFile(path, []byte(file.Content), file.Mode); err != nil { + return fmt.Errorf("write file %s: %w", path, err) + } + } + return nil +} + +type File struct { + Content string + Mode os.FileMode +} + +func customData(config *datamodel.NodeBootstrappingConfiguration) (map[string]File, error) { + bootstrapKubeconfig, err := contentBootstrapKubeconfig(config) + if err != nil { + return nil, fmt.Errorf("content bootstrap kubeconfig: %w", err) + } + contentDockerDaemon, err := contentDockerDaemonJSON(config) + if err != nil { + return nil, fmt.Errorf("content docker daemon json: %w", err) + } + + files := map[string]File{ + "/etc/kubernetes/certs/ca.crt": { + Content: config.ContainerService.Properties.CertificateProfile.CaCertificate, + Mode: 0600, + }, + "/etc/systemd/system/docker.service.d/exec_start.conf": { + Content: contentDockerExecStart(config), + Mode: 0644, + }, + "/etc/docker/daemon.json": { + Content: contentDockerDaemon, + Mode: 0644, + }, + "/etc/default/kubelet": { + Content: contentKubelet(config), + Mode: 0644, + }, + } + + if config.ContainerService.Properties.SecurityProfile.GetPrivateEgressContainerRegistryServer() != "" { + files["/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"] = File{ + Content: containerDMCRHosts(config), + Mode: 0644, + } + } + + if config.EnableSecureTLSBootstrapping || agent.IsTLSBootstrappingEnabledWithHardCodedToken(config.KubeletClientTLSBootstrapToken) { + files["/var/lib/kubelet/bootstrap-kubeconfig"] = File{ + Content: bootstrapKubeconfig, + Mode: 0644, + } + } else { + files["/var/lib/kubelet/kubeconfig"] = File{ + Content: contentKubeconfig(config), + Mode: 0644, + } + } + + for path, file := range files { + file.Content = strings.TrimLeft(file.Content, "\n") + files[path] = file + } + + return files, nil +} + +func contentKubeconfig(config *datamodel.NodeBootstrappingConfiguration) string { + return fmt.Sprintf(` +apiVersion: v1 +kind: Config +clusters: +- name: localcluster + cluster: + certificate-authority: /etc/kubernetes/certs/ca.crt + server: https://%s:443 +users: +- name: client + user: + client-certificate: /etc/kubernetes/certs/client.crt + client-key: /etc/kubernetes/certs/client.key +contexts: +- context: + cluster: localcluster + user: client + name: localclustercontext +current-context: localclustercontext +`, agent.GetKubernetesEndpoint(config.ContainerService)) +} + +func contentBootstrapKubeconfig(config *datamodel.NodeBootstrappingConfiguration) (string, error) { + data := map[string]any{ + "apiVersion": "v1", + "kind": "Config", + "clusters": []map[string]any{ + { + "name": "localcluster", + "cluster": map[string]any{ + "certificate-authority": "/etc/kubernetes/certs/ca.crt", + "server": "https://" + agent.GetKubernetesEndpoint(config.ContainerService) + ":443", + }, + }, + }, + "users": []map[string]any{ + { + "name": "kubelet-bootstrap", + "user": func() map[string]any { + if config.EnableSecureTLSBootstrapping { + appID := config.CustomSecureTLSBootstrapAADServerAppID + if appID == "" { + appID = "6dae42f8-4368-4678-94ff-3960e28e3630" + } + return map[string]any{ + "exec": map[string]any{ + "apiVersion": "client.authentication.k8s.io/v1", + "command": "/opt/azure/tlsbootstrap/tls-bootstrap-client", + "args": []string{ + "bootstrap", + "--next-proto=aks-tls-bootstrap", + "--aad-resource=" + appID}, + "interactiveMode": "Never", + "provideClusterInfo": true, + }, + } + } + return map[string]any{ + "token": agent.GetTLSBootstrapTokenForKubeConfig(config.KubeletClientTLSBootstrapToken), + } + }(), + }, + }, + "contexts": []map[string]any{ + { + "context": map[string]any{ + "cluster": "localcluster", + "user": "kubelet-bootstrap", + }, + "name": "bootstrap-context", + }, + }, + "current-context": "bootstrap-context", + } + dataYAML, err := yaml.Marshal(data) + if err != nil { + return "", err + } + return string(dataYAML), nil +} + +func contentDockerExecStart(config *datamodel.NodeBootstrappingConfiguration) string { + return fmt.Sprintf(` +[Service] +ExecStart= +ExecStart=/usr/bin/dockerd -H fd:// --storage-driver=overlay2 --bip=%s +ExecStartPost=/sbin/iptables -P FORWARD ACCEPT +#EOF`, config.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.DockerBridgeSubnet) +} + +func contentDockerDaemonJSON(config *datamodel.NodeBootstrappingConfiguration) (string, error) { + data := map[string]any{ + "live-restore": true, + "log-driver": "json-file", + "log-opts": map[string]string{ + "max-size": "50m", + "max-file": "5", + }, + } + if config.EnableNvidia { + data["default-runtime"] = "nvidia" + data["runtimes"] = map[string]any{ + "nvidia": map[string]any{ + "path": "/usr/bin/nvidia-container-runtime", + "runtimeArgs": []string{}, + }, + } + } + if agent.HasDataDir(config) { + data["data-root"] = agent.GetDataDir(config) + } + dataJSON, err := json.Marshal(data) + if err != nil { + return "", err + } + return string(dataJSON), nil +} + +func contentKubelet(config *datamodel.NodeBootstrappingConfiguration) string { + data := make([][2]string, 0) + data = append(data, [2]string{"KUBELET_FLAGS", agent.GetOrderedKubeletConfigFlagString(config)}) + data = append(data, [2]string{"KUBELET_REGISTER_SCHEDULABLE", "true"}) + data = append(data, [2]string{"NETWORK_POLICY", config.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.NetworkPolicy}) + IsKubernetesVersionGe := func(version string) bool { + return config.ContainerService.Properties.OrchestratorProfile.IsKubernetes() && agent.IsKubernetesVersionGe(config.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion, version) + } + + if !IsKubernetesVersionGe("1.17.0") { + data = append(data, [2]string{"KUBELET_IMAGE", config.K8sComponents.HyperkubeImageURL}) + } + + labels := func() string { + if IsKubernetesVersionGe("1.16.0") { + return agent.GetAgentKubernetesLabels(config.AgentPoolProfile, config) + } + return config.AgentPoolProfile.GetKubernetesLabels() + } + + data = append(data, [2]string{"KUBELET_NODE_LABELS", labels()}) + if config.ContainerService.IsAKSCustomCloud() { + data = append(data, [2]string{"AZURE_ENVIRONMENT_FILEPATH", "/etc/kubernetes/" + config.ContainerService.Properties.CustomCloudEnv.Name + ".json"}) + + } + result := "" + for _, d := range data { + result += fmt.Sprintf("%s=%s\n", d[0], d[1]) + } + return result +} + +func containerDMCRHosts(config *datamodel.NodeBootstrappingConfiguration) string { + return fmt.Sprintf(` +[host."https://%s"] +capabilities = ["pull", "resolve"] +`, config.ContainerService.Properties.SecurityProfile.GetPrivateEgressContainerRegistryServer()) +} diff --git a/parts/linux/cloud-init/nodecustomdata.yml b/parts/linux/cloud-init/nodecustomdata.yml index 9417b251318..89e8b36f58c 100644 --- a/parts/linux/cloud-init/nodecustomdata.yml +++ b/parts/linux/cloud-init/nodecustomdata.yml @@ -424,4 +424,4 @@ write_files: content: | [host."https://{{GetBootstrapProfileContainerRegistryServer}}"] capabilities = ["pull", "resolve"] -{{- end}} \ No newline at end of file +{{- end}} diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index f8a14198e8d..8e510fad618 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -373,7 +373,7 @@ func getContainerServiceFuncMap(config *datamodel.NodeBootstrappingConfiguration return cs.Properties.OrchestratorProfile.IsKubernetes() && IsKubernetesVersionGe(cs.Properties.OrchestratorProfile.OrchestratorVersion, version) }, "GetAgentKubernetesLabels": func(profile *datamodel.AgentPoolProfile) string { - return getAgentKubernetesLabels(profile, config) + return GetAgentKubernetesLabels(profile, config) }, "GetAgentKubernetesLabelsDeprecated": func(profile *datamodel.AgentPoolProfile) string { return profile.GetKubernetesLabels() @@ -411,7 +411,7 @@ func getContainerServiceFuncMap(config *datamodel.NodeBootstrappingConfiguration return IsKubeletServingCertificateRotationEnabled(config) }, "GetKubeletConfigKeyVals": func() string { - return GetOrderedKubeletConfigFlagString(config.KubeletConfig, cs, profile, config.EnableKubeletConfigFile) + return GetOrderedKubeletConfigFlagString(config) }, "GetKubeletConfigKeyValsPsh": func() string { return config.GetOrderedKubeletConfigStringForPowershell(profile.CustomKubeletConfig) @@ -484,13 +484,7 @@ func getContainerServiceFuncMap(config *datamodel.NodeBootstrappingConfiguration return cs.Properties.OrchestratorProfile.IsKubernetes() }, "GetKubernetesEndpoint": func() string { - if cs.Properties.HostedMasterProfile == nil { - return "" - } - if cs.Properties.HostedMasterProfile.IPAddress != "" { - return cs.Properties.HostedMasterProfile.IPAddress - } - return cs.Properties.HostedMasterProfile.FQDN + return GetKubernetesEndpoint(cs) }, "IsAzureCNI": func() bool { return cs.Properties.OrchestratorProfile.IsAzureCNI() @@ -662,26 +656,10 @@ func getContainerServiceFuncMap(config *datamodel.NodeBootstrappingConfiguration return cs.Properties.OrchestratorProfile.KubernetesConfig.RequiresDocker() }, "HasDataDir": func() bool { - if profile != nil && profile.KubernetesConfig != nil && profile.KubernetesConfig.ContainerRuntimeConfig != nil && - profile.KubernetesConfig.ContainerRuntimeConfig[datamodel.ContainerDataDirKey] != "" { - return true - } - if profile.KubeletDiskType == datamodel.TempDisk { - return true - } - return cs.Properties.OrchestratorProfile.KubernetesConfig.ContainerRuntimeConfig != nil && - cs.Properties.OrchestratorProfile.KubernetesConfig.ContainerRuntimeConfig[datamodel.ContainerDataDirKey] != "" + return HasDataDir(config) }, "GetDataDir": func() string { - if profile != nil && profile.KubernetesConfig != nil && - profile.KubernetesConfig.ContainerRuntimeConfig != nil && - profile.KubernetesConfig.ContainerRuntimeConfig[datamodel.ContainerDataDirKey] != "" { - return profile.KubernetesConfig.ContainerRuntimeConfig[datamodel.ContainerDataDirKey] - } - if profile.KubeletDiskType == datamodel.TempDisk { - return datamodel.TempDiskContainerDataDir - } - return cs.Properties.OrchestratorProfile.KubernetesConfig.ContainerRuntimeConfig[datamodel.ContainerDataDirKey] + return GetDataDir(config) }, "HasKubeletDiskType": func() bool { return profile != nil && profile.KubeletDiskType != "" && profile.KubeletDiskType != datamodel.OSDisk @@ -1001,6 +979,44 @@ func getContainerServiceFuncMap(config *datamodel.NodeBootstrappingConfiguration } } +func GetDataDir(config *datamodel.NodeBootstrappingConfiguration) string { + cs := config.ContainerService + profile := config.AgentPoolProfile + if profile != nil && profile.KubernetesConfig != nil && + profile.KubernetesConfig.ContainerRuntimeConfig != nil && + profile.KubernetesConfig.ContainerRuntimeConfig[datamodel.ContainerDataDirKey] != "" { + return profile.KubernetesConfig.ContainerRuntimeConfig[datamodel.ContainerDataDirKey] + } + if profile.KubeletDiskType == datamodel.TempDisk { + return datamodel.TempDiskContainerDataDir + } + return cs.Properties.OrchestratorProfile.KubernetesConfig.ContainerRuntimeConfig[datamodel.ContainerDataDirKey] +} + +func HasDataDir(config *datamodel.NodeBootstrappingConfiguration) bool { + cs := config.ContainerService + profile := config.AgentPoolProfile + if profile != nil && profile.KubernetesConfig != nil && profile.KubernetesConfig.ContainerRuntimeConfig != nil && + profile.KubernetesConfig.ContainerRuntimeConfig[datamodel.ContainerDataDirKey] != "" { + return true + } + if profile.KubeletDiskType == datamodel.TempDisk { + return true + } + return cs.Properties.OrchestratorProfile.KubernetesConfig.ContainerRuntimeConfig != nil && + cs.Properties.OrchestratorProfile.KubernetesConfig.ContainerRuntimeConfig[datamodel.ContainerDataDirKey] != "" +} + +func GetKubernetesEndpoint(cs *datamodel.ContainerService) string { + if cs.Properties.HostedMasterProfile == nil { + return "" + } + if cs.Properties.HostedMasterProfile.IPAddress != "" { + return cs.Properties.HostedMasterProfile.IPAddress + } + return cs.Properties.HostedMasterProfile.FQDN +} + func getPortRangeEndValue(portRange string) int { arr := strings.Split(portRange, " ") num, err := strconv.Atoi(arr[1]) diff --git a/pkg/agent/utils.go b/pkg/agent/utils.go index 3afe447127e..007c63be08b 100644 --- a/pkg/agent/utils.go +++ b/pkg/agent/utils.go @@ -323,8 +323,11 @@ func getCustomDataFromJSON(jsonStr string) string { // GetOrderedKubeletConfigFlagString returns an ordered string of key/val pairs. // copied from AKS-Engine and filter out flags that already translated to config file. -func GetOrderedKubeletConfigFlagString(k map[string]string, cs *datamodel.ContainerService, profile *datamodel.AgentPoolProfile, - kubeletConfigFileToggleEnabled bool) string { +func GetOrderedKubeletConfigFlagString(config *datamodel.NodeBootstrappingConfiguration) string { + k := config.KubeletConfig + cs := config.ContainerService + profile := config.AgentPoolProfile + kubeletConfigFileToggleEnabled := config.EnableKubeletConfigFile /* NOTE(mainred): kubeConfigFile now relies on CustomKubeletConfig, while custom configuration is not compatible with CustomKubeletConfig. When custom configuration is set we want to override every configuration with the customized one. */ @@ -432,7 +435,7 @@ func IsKubeletServingCertificateRotationEnabled(config *datamodel.NodeBootstrapp return config.KubeletConfig["--rotate-server-certificates"] == "true" } -func getAgentKubernetesLabels(profile *datamodel.AgentPoolProfile, config *datamodel.NodeBootstrappingConfiguration) string { +func GetAgentKubernetesLabels(profile *datamodel.AgentPoolProfile, config *datamodel.NodeBootstrappingConfiguration) string { var labels string if profile != nil { labels = profile.GetKubernetesLabels() diff --git a/pkg/agent/utils_test.go b/pkg/agent/utils_test.go index 364895effc7..06c8a2ed6c0 100644 --- a/pkg/agent/utils_test.go +++ b/pkg/agent/utils_test.go @@ -542,7 +542,7 @@ func TestGetAgentKubernetesLabels(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { - actual := getAgentKubernetesLabels(c.profile, c.config) + actual := GetAgentKubernetesLabels(c.profile, c.config) assert.Equal(t, c.expected, actual) }) }