diff --git a/node-bootstrapper/app.go b/node-bootstrapper/app.go index 19d60e0b42e..5fbf5ac6b6e 100644 --- a/node-bootstrapper/app.go +++ b/node-bootstrapper/app.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "encoding/json" "errors" "flag" "fmt" @@ -12,7 +13,7 @@ import ( "os/exec" "github.com/Azure/agentbaker/node-bootstrapper/parser" - "github.com/Azure/agentbaker/node-bootstrapper/utils" + nbcontractv1 "github.com/Azure/agentbaker/pkg/proto/nbcontract/v1" ) type App struct { @@ -67,30 +68,23 @@ func (a *App) Provision(ctx context.Context, flags ProvisionFlags) error { return fmt.Errorf("open proision file %s: %w", flags.ProvisionConfig, err) } - cseCmd, err := parser.Parse(inputJSON) + config := &nbcontractv1.Configuration{} + err = json.Unmarshal(inputJSON, config) if err != nil { - return fmt.Errorf("parse config: %w", err) + return fmt.Errorf("unmarshal provision config: %w", err) } - - if err := a.provisionStart(ctx, cseCmd); err != nil { - return fmt.Errorf("provision start: %w", err) + if config.Version != "v0" { + return fmt.Errorf("unsupported version: %s", config.Version) } - return nil -} -func (a *App) provisionStart(ctx context.Context, cse utils.SensitiveString) error { - // CSEScript can't be logged because it contains sensitive information. - slog.Info("Running CSE script") - //nolint:gosec // we generate the script, so it's safe to execute - cmd := exec.CommandContext(ctx, "/bin/bash", "-c", cse.UnsafeValue()) - cmd.Dir = "/" + cmd, err := parser.BuildCSECmd(ctx, config) + if err != nil { + return fmt.Errorf("build CSE command: %w", err) + } var stdoutBuf, stderrBuf bytes.Buffer - // We want to preserve the original stdout and stderr to avoid any issues during migration to the "scriptless" approach - // RP may rely on stdout and stderr for error handling - // it's also nice to have a single log file for all the important information, so write to both places cmd.Stdout = io.MultiWriter(os.Stdout, &stdoutBuf) cmd.Stderr = io.MultiWriter(os.Stderr, &stderrBuf) - err := a.cmdRunner(cmd) + err = a.cmdRunner(cmd) exitCode := -1 if cmd.ProcessState != nil { exitCode = cmd.ProcessState.ExitCode() diff --git a/node-bootstrapper/parser/helper.go b/node-bootstrapper/parser/helper.go index c280e49f1c7..b8852ad8970 100644 --- a/node-bootstrapper/parser/helper.go +++ b/node-bootstrapper/parser/helper.go @@ -51,59 +51,8 @@ var ( func getFuncMap() template.FuncMap { return template.FuncMap{ - "derefString": deref[string], - "derefBool": deref[bool], - "getStringFromVMType": getStringFromVMType, - "getStringFromNetworkPluginType": getStringFromNetworkPluginType, - "getStringFromNetworkPolicyType": getStringFromNetworkPolicyType, - "getStringFromLoadBalancerSkuType": getStringFromLoadBalancerSkuType, - "getKubenetTemplate": getKubenetTemplate, - "getSysctlContent": getSysctlContent, - "getUlimitContent": getUlimitContent, - "getContainerdConfig": getContainerdConfig, - "getStringifiedStringArray": getStringifiedStringArray, - "getIsMIGNode": getIsMIGNode, - "getCustomCACertsStatus": getCustomCACertsStatus, - "getEnableTLSBootstrap": getEnableTLSBootstrap, - "getEnableSecureTLSBootstrap": getEnableSecureTLSBootstrap, - "getTLSBootstrapToken": getTLSBootstrapToken, - "getCustomSecureTLSBootstrapAADServerAppID": getCustomSecureTLSBootstrapAADServerAppID, - "getIsKrustlet": getIsKrustlet, - "getEnsureNoDupePromiscuousBridge": getEnsureNoDupePromiscuousBridge, - "getHasSearchDomain": getHasSearchDomain, - "getCSEHelpersFilepath": getCSEHelpersFilepath, - "getCSEDistroHelpersFilepath": getCSEDistroHelpersFilepath, - "getCSEInstallFilepath": getCSEInstallFilepath, - "getCSEDistroInstallFilepath": getCSEDistroInstallFilepath, - "getCSEConfigFilepath": getCSEConfigFilepath, - "getCustomSearchDomainFilepath": getCustomSearchDomainFilepath, - "getDHCPV6ConfigFilepath": getDHCPV6ConfigFilepath, - "getDHCPV6ServiceFilepath": getDHCPV6ServiceFilepath, - "getShouldConfigContainerdUlimits": getShouldConfigContainerdUlimits, - "createSortedKeyValueStringPairs": createSortedKeyValuePairs[string], - "createSortedKeyValueInt32Pairs": createSortedKeyValuePairs[int32], - "getExcludeMasterFromStandardLB": getExcludeMasterFromStandardLB, - "getMaxLBRuleCount": getMaxLBRuleCount, - "getGpuImageSha": getGpuImageSha, - "getGpuDriverVersion": getGpuDriverVersion, - "getIsSgxEnabledSKU": getIsSgxEnabledSKU, - "getShouldConfigureHTTPProxy": getShouldConfigureHTTPProxy, - "getShouldConfigureHTTPProxyCA": getShouldConfigureHTTPProxyCA, - "getAzureEnvironmentFilepath": getAzureEnvironmentFilepath, - "getLinuxAdminUsername": getLinuxAdminUsername, - "getTargetEnvironment": getTargetEnvironment, - "getIsVHD": getIsVHD, - "getDisableSSH": getDisableSSH, - "getServicePrincipalFileContent": getServicePrincipalFileContent, - "getEnableSwapConfig": getEnableSwapConfig, - "getShouldConfigTransparentHugePage": getShouldConfigTransparentHugePage, - "getProxyVariables": getProxyVariables, - "getHasKubeletDiskType": getHasKubeletDiskType, - "getInitAKSCustomCloudFilepath": getInitAKSCustomCloudFilepath, - "getTargetCloud": getTargetCloud, - "getIsAksCustomCloud": getIsAksCustomCloud, - "getGPUNeedsFabricManager": getGPUNeedsFabricManager, - "getEnableNvidia": getEnableNvidia, + "getInitAKSCustomCloudFilepath": getInitAKSCustomCloudFilepath, + "getIsAksCustomCloud": getIsAksCustomCloud, } } diff --git a/node-bootstrapper/parser/parser.go b/node-bootstrapper/parser/parser.go index 3bcb13b3bdf..d9c104b9936 100644 --- a/node-bootstrapper/parser/parser.go +++ b/node-bootstrapper/parser/parser.go @@ -2,13 +2,14 @@ package parser import ( "bytes" + "context" _ "embed" - "encoding/json" "fmt" + "os/exec" + "sort" "strings" "text/template" - "github.com/Azure/agentbaker/node-bootstrapper/utils" nbcontractv1 "github.com/Azure/agentbaker/pkg/proto/nbcontract/v1" ) @@ -27,25 +28,163 @@ func executeBootstrapTemplate(inputContract *nbcontractv1.Configuration) (string return buffer.String(), nil } -// this function will eventually take a pointer to the bootstrap contract struct. -// it will then template out the variables into the final bootstrap trigger script. -func Parse(inputJSON []byte) (utils.SensitiveString, error) { - // Parse the JSON into a nbcontractv1.Configuration struct - var nbc nbcontractv1.Configuration - err := json.Unmarshal(inputJSON, &nbc) - if err != nil { - return "", fmt.Errorf("failed to unmarshal the json to nbcontractv1: %w", err) - } +func getCSEEnv(config *nbcontractv1.Configuration) map[string]string { + env := make(map[string]string) - if nbc.Version != "v0" { - return "", fmt.Errorf("unsupported version: %s", nbc.Version) + env["PROVISION_OUTPUT"] = "/var/log/azure/cluster-provision.log;" + env["MOBY_VERSION"] = "" + env["CLOUDPROVIDER_BACKOFF"] = "true" + env["CLOUDPROVIDER_BACKOFF_MODE"] = "v2" + env["CLOUDPROVIDER_BACKOFF_RETRIES"] = "6" + env["CLOUDPROVIDER_BACKOFF_EXPONENT"] = "0" + env["CLOUDPROVIDER_BACKOFF_DURATION"] = "5" + env["CLOUDPROVIDER_BACKOFF_JITTER"] = "0" + env["CLOUDPROVIDER_RATELIMIT"] = "true" + env["CLOUDPROVIDER_RATELIMIT_QPS"] = "10" + env["CLOUDPROVIDER_RATELIMIT_QPS_WRITE"] = "10" + env["CLOUDPROVIDER_RATELIMIT_BUCKET"] = "100" + env["CLOUDPROVIDER_RATELIMIT_BUCKET_WRITE"] = "100" + env["CONTAINER_RUNTIME"] = " containerd" + env["CLI_TOOL"] = "ctr" + env["NETWORK_MODE"] = "transparent" + env["NEEDS_CONTAINERD"] = "true" + env["NEEDS_DOCKER_LOGIN"] = "false" + + env["ADMINUSER"] = getLinuxAdminUsername(config.GetLinuxAdminUsername()) + env["TENANT_ID"] = config.AuthConfig.GetTenantId() + env["KUBERNETES_VERSION"] = config.GetKubernetesVersion() + env["KUBE_BINARY_URL"] = config.KubeBinaryConfig.GetKubeBinaryUrl() + env["CUSTOM_KUBE_BINARY_URL"] = config.KubeBinaryConfig.GetCustomKubeBinaryUrl() + env["PRIVATE_KUBE_BINARY_URL"] = config.KubeBinaryConfig.GetPrivateKubeBinaryUrl() + env["KUBEPROXY_URL"] = config.GetKubeProxyUrl() + env["APISERVER_PUBLIC_KEY"] = config.ApiServerConfig.GetApiServerPublicKey() + env["SUBSCRIPTION_ID"] = config.AuthConfig.GetSubscriptionId() + env["RESOURCE_GROUP"] = config.ClusterConfig.GetResourceGroup() + env["LOCATION"] = config.ClusterConfig.GetLocation() + env["VM_TYPE"] = getStringFromVMType(config.ClusterConfig.GetVmType()) + env["SUBNET"] = config.ClusterConfig.GetClusterNetworkConfig().GetSubnet() + env["NETWORK_SECURITY_GROUP"] = config.ClusterConfig.GetClusterNetworkConfig().GetSecurityGroupName() + env["VIRTUAL_NETWORK"] = config.ClusterConfig.GetClusterNetworkConfig().GetVnetName() + env["VIRTUAL_NETWORK_RESOURCE_GROUP"] = config.ClusterConfig.GetClusterNetworkConfig().GetVnetResourceGroup() + env["ROUTE_TABLE"] = config.ClusterConfig.GetClusterNetworkConfig().GetRouteTable() + env["PRIMARY_AVAILABILITY_SET"] = config.ClusterConfig.GetPrimaryAvailabilitySet() + env["PRIMARY_SCALE_SET"] = config.ClusterConfig.GetPrimaryScaleSet() + env["SERVICE_PRINCIPAL_CLIENT_ID"] = config.AuthConfig.GetServicePrincipalId() + env["NETWORK_PLUGIN"] = getStringFromNetworkPluginType(config.GetNetworkConfig().GetNetworkPlugin()) + env["NETWORK_POLICY"] = getStringFromNetworkPolicyType(config.GetNetworkConfig().GetNetworkPolicy()) + env["VNET_CNI_PLUGINS_URL"] = config.GetNetworkConfig().GetVnetCniPluginsUrl() + env["LOAD_BALANCER_DISABLE_OUTBOUND_SNAT"] = fmt.Sprintf("%v", config.ClusterConfig.GetLoadBalancerConfig().GetDisableOutboundSnat()) + env["USE_MANAGED_IDENTITY_EXTENSION"] = fmt.Sprintf("%v", config.AuthConfig.GetUseManagedIdentityExtension()) + env["USE_INSTANCE_METADATA"] = fmt.Sprintf("%v", config.ClusterConfig.GetUseInstanceMetadata()) + env["LOAD_BALANCER_SKU"] = getStringFromLoadBalancerSkuType(config.ClusterConfig.GetLoadBalancerConfig().GetLoadBalancerSku()) + env["EXCLUDE_MASTER_FROM_STANDARD_LB"] = fmt.Sprintf("%v", getExcludeMasterFromStandardLB(config.ClusterConfig.GetLoadBalancerConfig())) + env["MAXIMUM_LOADBALANCER_RULE_COUNT"] = fmt.Sprintf("%v", getMaxLBRuleCount(config.ClusterConfig.GetLoadBalancerConfig())) + env["CONTAINERD_DOWNLOAD_URL_BASE"] = config.ContainerdConfig.GetContainerdDownloadUrlBase() + env["USER_ASSIGNED_IDENTITY_ID"] = config.AuthConfig.GetAssignedIdentityId() + env["API_SERVER_NAME"] = config.ApiServerConfig.GetApiServerName() + env["IS_VHD"] = fmt.Sprintf("%v", getIsVHD(config.IsVhd)) + env["GPU_NODE"] = fmt.Sprintf("%v", getEnableNvidia(config)) + env["SGX_NODE"] = fmt.Sprintf("%v", getIsSgxEnabledSKU(config.VmSize)) + env["MIG_NODE"] = fmt.Sprintf("%v", getIsMIGNode(config.GpuConfig.GetGpuInstanceProfile())) + env["CONFIG_GPU_DRIVER_IF_NEEDED"] = fmt.Sprintf("%v", config.GpuConfig.GetConfigGpuDriver()) + env["ENABLE_GPU_DEVICE_PLUGIN_IF_NEEDED"] = fmt.Sprintf("%v", config.GpuConfig.GetGpuDevicePlugin()) + env["TELEPORTD_PLUGIN_DOWNLOAD_URL"] = config.TeleportConfig.GetTeleportdPluginDownloadUrl() + env["CREDENTIAL_PROVIDER_DOWNLOAD_URL"] = config.KubeBinaryConfig.GetLinuxCredentialProviderUrl() + env["CONTAINERD_VERSION"] = config.ContainerdConfig.GetContainerdVersion() + env["CONTAINERD_PACKAGE_URL"] = config.ContainerdConfig.GetContainerdPackageUrl() + env["RUNC_VERSION"] = config.RuncConfig.GetRuncVersion() + env["RUNC_PACKAGE_URL"] = config.RuncConfig.GetRuncPackageUrl() + env["ENABLE_HOSTS_CONFIG_AGENT"] = fmt.Sprintf("%v", config.GetEnableHostsConfigAgent()) + env["DISABLE_SSH"] = fmt.Sprintf("%v", getDisableSSH(config)) + env["TELEPORT_ENABLED"] = fmt.Sprintf("%v", config.TeleportConfig.GetStatus()) + env["SHOULD_CONFIGURE_HTTP_PROXY"] = fmt.Sprintf("%v", getShouldConfigureHTTPProxy(config.HttpProxyConfig)) + env["SHOULD_CONFIGURE_HTTP_PROXY_CA"] = fmt.Sprintf("%v", getShouldConfigureHTTPProxyCA(config.HttpProxyConfig)) + env["HTTP_PROXY_TRUSTED_CA"] = config.HttpProxyConfig.GetProxyTrustedCa() + env["SHOULD_CONFIGURE_CUSTOM_CA_TRUST"] = fmt.Sprintf("%v", getCustomCACertsStatus(config.GetCustomCaCerts())) + env["CUSTOM_CA_TRUST_COUNT"] = fmt.Sprintf("%v", len(config.GetCustomCaCerts())) + for i, cert := range config.CustomCaCerts { + env[fmt.Sprintf("CUSTOM_CA_CERT_%d", i)] = cert } + env["IS_KRUSTLET"] = fmt.Sprintf("%v", getIsKrustlet(config.GetWorkloadRuntime())) + env["GPU_NEEDS_FABRIC_MANAGER"] = fmt.Sprintf("%v", getGPUNeedsFabricManager(config.VmSize)) + env["IPV6_DUAL_STACK_ENABLED"] = fmt.Sprintf("%v", config.GetIpv6DualStackEnabled()) + env["OUTBOUND_COMMAND"] = config.GetOutboundCommand() + env["ENABLE_UNATTENDED_UPGRADES"] = fmt.Sprintf("%v", config.GetEnableUnattendedUpgrade()) + env["ENSURE_NO_DUPE_PROMISCUOUS_BRIDGE"] = fmt.Sprintf("%v", getEnsureNoDupePromiscuousBridge(config.GetNetworkConfig())) + env["SHOULD_CONFIG_SWAP_FILE"] = fmt.Sprintf("%v", getEnableSwapConfig(config.CustomLinuxOsConfig)) + env["SHOULD_CONFIG_TRANSPARENT_HUGE_PAGE"] = fmt.Sprintf("%v", getShouldConfigTransparentHugePage(config.CustomLinuxOsConfig)) + env["SHOULD_CONFIG_CONTAINERD_ULIMITS"] = fmt.Sprintf("%v", getShouldConfigContainerdUlimits(config.CustomLinuxOsConfig.GetUlimitConfig())) + env["CONTAINERD_ULIMITS"] = getUlimitContent(config.CustomLinuxOsConfig.GetUlimitConfig()) + env["TARGET_CLOUD"] = getTargetCloud(config) + env["TARGET_ENVIRONMENT"] = getTargetEnvironment(config) + env["CUSTOM_ENV_JSON"] = config.CustomCloudConfig.GetCustomEnvJsonContent() + env["IS_CUSTOM_CLOUD"] = fmt.Sprintf("%v", getIsAksCustomCloud(config.CustomCloudConfig)) + env["AKS_CUSTOM_CLOUD_CONTAINER_REGISTRY_DNS_SUFFIX"] = config.CustomCloudConfig.GetContainerRegistryDnsSuffix() + env["CSE_HELPERS_FILEPATH"] = getCSEHelpersFilepath() + env["CSE_DISTRO_HELPERS_FILEPATH"] = getCSEDistroHelpersFilepath() + env["CSE_INSTALL_FILEPATH"] = getCSEInstallFilepath() + env["CSE_DISTRO_INSTALL_FILEPATH"] = getCSEDistroInstallFilepath() + env["CSE_CONFIG_FILEPATH"] = getCSEConfigFilepath() + env["AZURE_PRIVATE_REGISTRY_SERVER"] = config.GetAzurePrivateRegistryServer() + env["HAS_CUSTOM_SEARCH_DOMAIN"] = fmt.Sprintf("%v", getHasSearchDomain(config.GetCustomSearchDomainConfig())) + env["CUSTOM_SEARCH_DOMAIN_FILEPATH"] = getCustomSearchDomainFilepath() + env["HTTP_PROXY_URLS"] = config.HttpProxyConfig.GetHttpProxy() + env["HTTPS_PROXY_URLS"] = config.HttpProxyConfig.GetHttpsProxy() + env["NO_PROXY_URLS"] = getStringifiedStringArray(config.HttpProxyConfig.GetNoProxyEntries(), ",") + env["PROXY_VARS"] = getProxyVariables(config.HttpProxyConfig) + env["ENABLE_TLS_BOOTSTRAPPING"] = fmt.Sprintf("%v", getEnableTLSBootstrap(config.TlsBootstrappingConfig)) + env["ENABLE_SECURE_TLS_BOOTSTRAPPING"] = fmt.Sprintf("%v", getEnableSecureTLSBootstrap(config.TlsBootstrappingConfig)) + env["CUSTOM_SECURE_TLS_BOOTSTRAP_AAD_SERVER_APP_ID"] = getCustomSecureTLSBootstrapAADServerAppID(config.TlsBootstrappingConfig) + env["DHCPV6_SERVICE_FILEPATH"] = getDHCPV6ServiceFilepath() + env["DHCPV6_CONFIG_FILEPATH"] = getDHCPV6ConfigFilepath() + env["THP_ENABLED"] = config.CustomLinuxOsConfig.GetTransparentHugepageSupport() + env["THP_DEFRAG"] = config.CustomLinuxOsConfig.GetTransparentDefrag() + env["SERVICE_PRINCIPAL_FILE_CONTENT"] = getServicePrincipalFileContent(config.AuthConfig) + env["KUBELET_CLIENT_CONTENT"] = config.KubeletConfig.GetKubeletClientKey() + env["KUBELET_CLIENT_CONTENT"] = config.KubeletConfig.GetKubeletClientKey() + env["KUBELET_CLIENT_CERT_CONTENT"] = config.KubeletConfig.GetKubeletClientCertContent() + env["KUBELET_CONFIG_FILE_ENABLED"] = fmt.Sprintf("%v", config.KubeletConfig.GetEnableKubeletConfigFile()) + env["KUBELET_CONFIG_FILE_CONTENT"] = config.KubeletConfig.GetKubeletConfigFileContent() + env["SWAP_FILE_SIZE_MB"] = fmt.Sprintf("%v", config.CustomLinuxOsConfig.GetSwapFileSize()) + env["GPU_DRIVER_VERSION"] = getGpuDriverVersion(config.VmSize) + env["GPU_IMAGE_SHA"] = getGpuImageSha(config.VmSize) + env["GPU_INSTANCE_PROFILE"] = config.GpuConfig.GetGpuInstanceProfile() + env["CUSTOM_SEARCH_DOMAIN_NAME"] = config.CustomSearchDomainConfig.GetDomainName() + env["CUSTOM_SEARCH_REALM_USER"] = config.CustomSearchDomainConfig.GetRealmUser() + env["CUSTOM_SEARCH_REALM_PASSWORD"] = config.CustomSearchDomainConfig.GetRealmPassword() + env["MESSAGE_OF_THE_DAY"] = config.GetMessageOfTheDay() + env["HAS_KUBELET_DISK_TYPE"] = fmt.Sprintf("%v", getHasKubeletDiskType(config.KubeletConfig)) + env["NEEDS_CGROUPV2"] = fmt.Sprintf("%v", config.GetNeedsCgroupv2()) + env["TLS_BOOTSTRAP_TOKEN"] = getTLSBootstrapToken(config.TlsBootstrappingConfig) + env["KUBELET_FLAGS"] = createSortedKeyValuePairs(config.KubeletConfig.GetKubeletFlags(), " ") + env["NETWORK_POLICY"] = getStringFromNetworkPolicyType(config.NetworkConfig.GetNetworkPolicy()) + env["KUBELET_NODE_LABELS"] = createSortedKeyValuePairs(config.KubeletConfig.GetKubeletNodeLabels(), ",") + env["AZURE_ENVIRONMENT_FILEPATH"] = getAzureEnvironmentFilepath(config) + env["KUBE_CA_CRT"] = config.GetKubernetesCaCert() + env["KUBENET_TEMPLATE"] = getKubenetTemplate() + env["CONTAINERD_CONFIG_CONTENT"] = getContainerdConfig(config) + env["IS_KATA"] = fmt.Sprintf("%v", config.GetIsKata()) + env["ARTIFACT_STREAMING_ENABLED"] = fmt.Sprintf("%v", config.GetEnableArtifactStreaming()) + env["SYSCTL_CONTENT"] = getSysctlContent(config.CustomLinuxOsConfig.GetSysctlConfig()) + env["PRIVATE_EGRESS_PROXY_ADDRESS"] = config.GetPrivateEgressProxyAddress() + env["BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER"] = config.GetBootstrapProfileContainerRegistryServer() + env["ENABLE_IMDS_RESTRICTION"] = fmt.Sprintf("%v", config.ImdsRestrictionConfig.GetEnableImdsRestriction()) + env["INSERT_IMDS_RESTRICTION_RULE_TO_MANGLE_TABLE"] = fmt.Sprintf("%v", config.ImdsRestrictionConfig.GetInsertImdsRestrictionRuleToMangleTable()) - triggerBootstrapScript, err := executeBootstrapTemplate(&nbc) + return env +} + +func BuildCSECmd(ctx context.Context, config *nbcontractv1.Configuration) (*exec.Cmd, error) { + triggerBootstrapScript, err := executeBootstrapTemplate(config) if err != nil { - return "", fmt.Errorf("failed to execute the template: %w", err) + return nil, fmt.Errorf("failed to execute the template: %w", err) } - // Convert to one-liner - return utils.SensitiveString(strings.ReplaceAll(triggerBootstrapScript, "\n", " ")), nil + triggerBootstrapScript = strings.ReplaceAll(triggerBootstrapScript, "\n", " ") + cmd := exec.CommandContext(ctx, "/bin/bash", "-c", triggerBootstrapScript) + for k, v := range getCSEEnv(config) { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) + } + sort.Strings(cmd.Env) // produce deterministic output + return cmd, nil } diff --git a/node-bootstrapper/parser/parser_test.go b/node-bootstrapper/parser/parser_test.go index 44bc67101a3..23a2e64381d 100644 --- a/node-bootstrapper/parser/parser_test.go +++ b/node-bootstrapper/parser/parser_test.go @@ -1,12 +1,14 @@ package parser_test import ( + "context" "encoding/base64" "encoding/json" "fmt" "log" "os" - "regexp" + "os/exec" + "strings" "testing" "github.com/Azure/agentbaker/node-bootstrapper/parser" @@ -17,30 +19,16 @@ import ( "github.com/stretchr/testify/require" ) -type nodeBootstrappingOutput struct { - cseCmd string - vars map[string]string -} - -type outputValidator func(*nodeBootstrappingOutput) - -// this regex looks for groups of the following forms, returning KEY and VALUE as submatches. -/* - KEY=VALUE -- KEY="VALUE" -- KEY= -- KEY="VALUE WITH WHITSPACE". */ -const cseRegexString = `([^=\s]+)=(\"[^\"]*\"|[^\s]*)` - // test certificate. const encodedTestCert = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUgvVENDQmVXZ0F3SUJBZ0lRYUJZRTMvTTA4WEhZQ25OVm1jRkJjakFOQmdrcWhraUc5dzBCQVFzRkFEQnkKTVFzd0NRWURWUVFHRXdKVlV6RU9NQXdHQTFVRUNBd0ZWR1Y0WVhNeEVEQU9CZ05WQkFjTUIwaHZkWE4wYjI0eApFVEFQQmdOVkJBb01DRk5UVENCRGIzSndNUzR3TEFZRFZRUUREQ1ZUVTB3dVkyOXRJRVZXSUZOVFRDQkpiblJsCmNtMWxaR2xoZEdVZ1EwRWdVbE5CSUZJek1CNFhEVEl3TURRd01UQXdOVGd6TTFvWERUSXhNRGN4TmpBd05UZ3oKTTFvd2diMHhDekFKQmdOVkJBWVRBbFZUTVE0d0RBWURWUVFJREFWVVpYaGhjekVRTUE0R0ExVUVCd3dIU0c5MQpjM1J2YmpFUk1BOEdBMVVFQ2d3SVUxTk1JRU52Y25BeEZqQVVCZ05WQkFVVERVNVdNakF3T0RFMk1UUXlORE14CkZEQVNCZ05WQkFNTUMzZDNkeTV6YzJ3dVkyOXRNUjB3R3dZRFZRUVBEQlJRY21sMllYUmxJRTl5WjJGdWFYcGgKZEdsdmJqRVhNQlVHQ3lzR0FRUUJnamM4QWdFQ0RBWk9aWFpoWkdFeEV6QVJCZ3NyQmdFRUFZSTNQQUlCQXhNQwpWVk13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRREhoZVJrYmIxRkNjN3hSS3N0CndLMEpJR2FLWTh0N0piUzJiUTJiNllJSkRnbkh1SVlIcUJyQ1VWNzlvZWxpa2tva1JrRnZjdnBhS2luRkhEUUgKVXBXRUk2UlVFUlltU0NnM084V2k0MnVPY1YyQjVaYWJtWENrd2R4WTVFY2w1MUJiTThVbkdkb0FHYmRObWlSbQpTbVRqY3MrbGhNeGc0ZkZZNmxCcGlFVkZpR1VqR1JSKzYxUjY3THo2VTRLSmVMTmNDbTA3UXdGWUtCbXBpMDhnCmR5Z1N2UmRVdzU1Sm9wcmVkaitWR3RqVWtCNGhGVDRHUVgvZ2h0NjlSbHF6Lys4dTBkRVFraHVVdXVjcnFhbG0KU0d5NDNIUndCZkRLRndZZVdNN0NQTWQ1ZS9kTyt0MDh0OFBianpWVFR2NWhRRENzRVlJVjJUN0FGSTlTY054TQpraDcvQWdNQkFBR2pnZ05CTUlJRFBUQWZCZ05WSFNNRUdEQVdnQlMvd1ZxSC95ajZRVDM5dDAva0hhK2dZVmdwCnZUQi9CZ2dyQmdFRkJRY0JBUVJ6TUhFd1RRWUlLd1lCQlFVSE1BS0dRV2gwZEhBNkx5OTNkM2N1YzNOc0xtTnYKYlM5eVpYQnZjMmwwYjNKNUwxTlRUR052YlMxVGRXSkRRUzFGVmkxVFUwd3RVbE5CTFRRd09UWXRVak11WTNKMApNQ0FHQ0NzR0FRVUZCekFCaGhSb2RIUndPaTh2YjJOemNITXVjM05zTG1OdmJUQWZCZ05WSFJFRUdEQVdnZ3QzCmQzY3VjM05zTG1OdmJZSUhjM05zTG1OdmJUQmZCZ05WSFNBRVdEQldNQWNHQldlQkRBRUJNQTBHQ3lxRWFBR0cKOW5jQ0JRRUJNRHdHRENzR0FRUUJncWt3QVFNQkJEQXNNQ29HQ0NzR0FRVUZCd0lCRmg1b2RIUndjem92TDNkMwpkeTV6YzJ3dVkyOXRMM0psY0c5emFYUnZjbmt3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdJR0NDc0dBUVVGCkJ3TUJNRWdHQTFVZEh3UkJNRDh3UGFBN29EbUdOMmgwZEhBNkx5OWpjbXh6TG5OemJDNWpiMjB2VTFOTVkyOXQKTFZOMVlrTkJMVVZXTFZOVFRDMVNVMEV0TkRBNU5pMVNNeTVqY213d0hRWURWUjBPQkJZRUZBREFGVUlhenc1cgpaSUhhcG5SeElVbnB3K0dMTUE0R0ExVWREd0VCL3dRRUF3SUZvRENDQVgwR0Npc0dBUVFCMW5rQ0JBSUVnZ0Z0CkJJSUJhUUZuQUhjQTlseVVMOUYzTUNJVVZCZ0lNSlJXanVOTkV4a3p2OThNTHlBTHpFN3haT01BQUFGeE0waG8KYndBQUJBTUFTREJHQWlFQTZ4ZWxpTlI4R2svNjNwWWRuUy92T3gvQ2pwdEVNRXY4OVdXaDEvdXJXSUVDSVFEeQpCcmVIVTI1RHp3dWtRYVJRandXNjU1WkxrcUNueGJ4UVdSaU9lbWo5SkFCMUFKUWd2QjZPMVkxc2lITWZnb3NpCkxBM1IyazFlYkUrVVBXSGJUaTlZVGFMQ0FBQUJjVE5JYU53QUFBUURBRVl3UkFJZ0dSRTR3emFiTlJkRDhrcS8KdkZQM3RRZTJobTB4NW5YdWxvd2g0SWJ3M2xrQ0lGWWIvM2xTRHBsUzdBY1I0citYcFd0RUtTVEZXSm1OQ1JiYwpYSnVyMlJHQkFIVUE3c0NWN28xeVpBK1M0OE81RzhjU28ybHFDWHRMYWhvVU9PWkhzc3Z0eGZrQUFBRnhNMGhvCjh3QUFCQU1BUmpCRUFpQjZJdmJvV3NzM1I0SXRWd2plYmw3RDN5b0ZhWDBORGgyZFdoaGd3Q3hySHdJZ0NmcTcKb2NNQzV0KzFqaTVNNXhhTG1QQzRJK1dYM0kvQVJrV1N5aU83SVFjd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dJQgpBQ2V1dXI0UW51anFtZ3VTckhVM21oZitjSm9kelRRTnFvNHRkZStQRDEvZUZkWUFFTHU4eEYrMEF0N3hKaVBZCmk1Ukt3aWx5UDU2diszaVkyVDlsdzdTOFRKMDQxVkxoYUlLcDE0TXpTVXpSeWVvT0FzSjdRQURNQ2xIS1VEbEgKVVUycE51bzg4WTZpZ292VDNic253Sk5pRVFOcXltU1NZaGt0dzB0YWR1b3FqcVhuMDZnc1Zpb1dUVkRYeXNkNQpxRXg0dDZzSWdJY01tMjZZSDF2SnBDUUVoS3BjMnkwN2dSa2tsQlpSdE1qVGh2NGNYeXlNWDd1VGNkVDdBSkJQCnVlaWZDb1YyNUp4WHVvOGQ1MTM5Z3dQMUJBZTdJQlZQeDJ1N0tOL1V5T1hkWm13TWYvVG1GR3dEZENmc3lIZi8KWnNCMndMSG96VFlvQVZtUTlGb1UxSkxnY1ZpdnFKK3ZObEJoSFhobHhNZE4wajgwUjlOejZFSWdsUWplSzNPOApJL2NGR20vQjgrNDJoT2xDSWQ5WmR0bmRKY1JKVmppMHdEMHF3ZXZDYWZBOWpKbEh2L2pzRStJOVV6NmNwQ3loCnN3K2xyRmR4VWdxVTU4YXhxZUs4OUZSK05vNHEwSUlPK0ppMXJKS3I5bmtTQjBCcVhvelZuRTFZQi9LTHZkSXMKdVlaSnVxYjJwS2t1K3p6VDZnVXdIVVRadkJpTk90WEw0Tnh3Yy9LVDdXek9TZDJ3UDEwUUk4REtnNHZmaU5EcwpIV21CMWM0S2ppNmdPZ0E1dVNVemFHbXEvdjRWbmNLNVVyK245TGJmbmZMYzI4SjVmdC9Hb3Rpbk15RGszaWFyCkYxMFlscWNPbWVYMXVGbUtiZGkvWG9yR2xrQ29NRjNURHg4cm1wOURCaUIvCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0=" //nolint:lll -func TestCustomDataAndCSE(t *testing.T) { +func TestBuildCSECmd(t *testing.T) { tests := []struct { name string folder string k8sVersion string nbcUpdator func(*nbcontractv1.Configuration) - validator outputValidator + validator func(cmd *exec.Cmd) }{ { name: "AKSUbuntu2204 containerd with multi-instance GPU", @@ -52,14 +40,15 @@ func TestCustomDataAndCSE(t *testing.T) { nbc.GpuConfig.EnableNvidia = to.BoolPtr(false) nbc.VmSize = "Standard_ND96asr_v4" }, - validator: func(o *nodeBootstrappingOutput) { - assert.Equal(t, "false", o.vars["GPU_NODE"]) - assert.NotEmpty(t, o.vars["CONTAINERD_CONFIG_CONTENT"]) + validator: func(cmd *exec.Cmd) { + vars := environToMap(cmd.Env) + assert.Equal(t, "false", vars["GPU_NODE"]) + assert.NotEmpty(t, vars["CONTAINERD_CONFIG_CONTENT"]) // Ensure the containerd config does not use the // nvidia container runtime when skipping the // GPU driver install, since it will fail to run even non-GPU // pods, as it will not be installed. - containerdConfigFileContent, err := getBase64DecodedValue([]byte(o.vars["CONTAINERD_CONFIG_CONTENT"])) + containerdConfigFileContent, err := getBase64DecodedValue([]byte(vars["CONTAINERD_CONFIG_CONTENT"])) require.NoError(t, err) expectedShimConfig := `version = 2 oom_score = 0 @@ -90,8 +79,9 @@ oom_score = 0 nbcUpdator: func(nbc *nbcontractv1.Configuration) { nbc.EnableSsh = to.BoolPtr(true) }, - validator: func(o *nodeBootstrappingOutput) { - assert.Equal(t, "false", o.vars["DISABLE_SSH"]) + validator: func(cmd *exec.Cmd) { + vars := environToMap(cmd.Env) + assert.Equal(t, "false", vars["DISABLE_SSH"]) }, }, { @@ -102,10 +92,11 @@ oom_score = 0 nbc.ClusterConfig.Location = "chinaeast2" nbc.CustomCloudConfig.CustomCloudEnvName = "AzureChinaCloud" }, - validator: func(o *nodeBootstrappingOutput) { - assert.Equal(t, "AzureChinaCloud", o.vars["TARGET_ENVIRONMENT"]) - assert.Equal(t, "AzureChinaCloud", o.vars["TARGET_CLOUD"]) - assert.Equal(t, "false", o.vars["IS_CUSTOM_CLOUD"]) + validator: func(cmd *exec.Cmd) { + vars := environToMap(cmd.Env) + assert.Equal(t, "AzureChinaCloud", vars["TARGET_ENVIRONMENT"]) + assert.Equal(t, "AzureChinaCloud", vars["TARGET_CLOUD"]) + assert.Equal(t, "false", vars["IS_CUSTOM_CLOUD"]) }, }, { @@ -115,10 +106,11 @@ oom_score = 0 nbcUpdator: func(nbc *nbcontractv1.Configuration) { nbc.CustomCloudConfig.CustomCloudEnvName = nbcontractv1.AksCustomCloudName }, - validator: func(o *nodeBootstrappingOutput) { - assert.Equal(t, nbcontractv1.AksCustomCloudName, o.vars["TARGET_ENVIRONMENT"]) - assert.Equal(t, nbcontractv1.AzureStackCloud, o.vars["TARGET_CLOUD"]) - assert.Equal(t, "true", o.vars["IS_CUSTOM_CLOUD"]) + validator: func(cmd *exec.Cmd) { + vars := environToMap(cmd.Env) + assert.Equal(t, nbcontractv1.AksCustomCloudName, vars["TARGET_ENVIRONMENT"]) + assert.Equal(t, nbcontractv1.AzureStackCloud, vars["TARGET_CLOUD"]) + assert.Equal(t, "true", vars["IS_CUSTOM_CLOUD"]) }, }, { @@ -142,8 +134,9 @@ oom_score = 0 }, } }, - validator: func(o *nodeBootstrappingOutput) { - sysctlContent, err := getBase64DecodedValue([]byte(o.vars["SYSCTL_CONTENT"])) + validator: func(cmd *exec.Cmd) { + vars := environToMap(cmd.Env) + sysctlContent, err := getBase64DecodedValue([]byte(vars["SYSCTL_CONTENT"])) require.NoError(t, err) assert.Contains(t, sysctlContent, "net.core.somaxconn=1638499") assert.Contains(t, sysctlContent, "net.ipv4.tcp_max_syn_backlog=1638498") @@ -162,10 +155,11 @@ oom_score = 0 nbc.EnableUnattendedUpgrade = true nbc.NeedsCgroupv2 = to.BoolPtr(true) }, - validator: func(o *nodeBootstrappingOutput) { - assert.Equal(t, "true", o.vars["IS_KATA"]) - assert.Equal(t, "true", o.vars["ENABLE_UNATTENDED_UPGRADES"]) - assert.Equal(t, "true", o.vars["NEEDS_CGROUPV2"]) + validator: func(cmd *exec.Cmd) { + vars := environToMap(cmd.Env) + assert.Equal(t, "true", vars["IS_KATA"]) + assert.Equal(t, "true", vars["ENABLE_UNATTENDED_UPGRADES"]) + assert.Equal(t, "true", vars["NEEDS_CGROUPV2"]) }, }, { @@ -175,9 +169,10 @@ oom_score = 0 nbcUpdator: func(nbc *nbcontractv1.Configuration) { nbc.NetworkConfig.NetworkPlugin = nbcontractv1.GetNetworkPluginType(nbcontractv1.NetworkPluginKubenet) }, - validator: func(o *nodeBootstrappingOutput) { - assert.NotEmpty(t, o.vars["CONTAINERD_CONFIG_CONTENT"]) - assert.Equal(t, "kubenet", o.vars["NETWORK_PLUGIN"]) + validator: func(cmd *exec.Cmd) { + vars := environToMap(cmd.Env) + assert.NotEmpty(t, vars["CONTAINERD_CONFIG_CONTENT"]) + assert.Equal(t, "kubenet", vars["NETWORK_PLUGIN"]) }, }, { @@ -195,9 +190,10 @@ oom_score = 0 ProxyTrustedCa: encodedTestCert, } }, - validator: func(o *nodeBootstrappingOutput) { + validator: func(cmd *exec.Cmd) { httpProxyStr := "export http_proxy=\"http://myproxy.server.com:8080/\"" - assert.Contains(t, o.cseCmd, httpProxyStr) + vars := environToMap(cmd.Env) + assert.Contains(t, vars["PROXY_VARS"], httpProxyStr) }, }, { @@ -207,10 +203,11 @@ oom_score = 0 nbcUpdator: func(nbc *nbcontractv1.Configuration) { nbc.CustomCaCerts = []string{encodedTestCert, encodedTestCert, encodedTestCert} }, - validator: func(o *nodeBootstrappingOutput) { - assert.Equal(t, "3", o.vars["CUSTOM_CA_TRUST_COUNT"]) - assert.Equal(t, "true", o.vars["SHOULD_CONFIGURE_CUSTOM_CA_TRUST"]) - assert.Equal(t, encodedTestCert, o.vars["CUSTOM_CA_CERT_0"]) + validator: func(cmd *exec.Cmd) { + vars := environToMap(cmd.Env) + assert.Equal(t, "3", vars["CUSTOM_CA_TRUST_COUNT"]) + assert.Equal(t, "true", vars["SHOULD_CONFIGURE_CUSTOM_CA_TRUST"]) + assert.Equal(t, encodedTestCert, vars["CUSTOM_CA_CERT_0"]) }, }, } @@ -341,23 +338,13 @@ oom_score = 0 tt.nbcUpdator(nbc) } - inputJSON, err := json.Marshal(nbc) + cseCMD, err := parser.BuildCSECmd(context.TODO(), nBCB.GetNodeBootstrapConfig()) require.NoError(t, err) - cseCmd, err := parser.Parse(inputJSON) - assert.NoError(t, err) - generateTestDataIfRequested(t, tt.folder, cseCmd.UnsafeValue()) - - vars, err := getDecodedVarsFromCseCmd([]byte(cseCmd)) - assert.NoError(t, err) - - result := &nodeBootstrappingOutput{ - cseCmd: cseCmd.UnsafeValue(), - vars: vars, - } + generateTestDataIfRequested(t, tt.folder, cseCMD) if tt.validator != nil { - tt.validator(result) + tt.validator(cseCMD) } }) } @@ -367,13 +354,14 @@ func TestNBContractCompatibilityFromJsonToCSECommand(t *testing.T) { tests := []struct { name string folder string - validator outputValidator + validator func(cmd *exec.Cmd) }{ { name: "with empty config. Parser Should provide default values to unset fields.", folder: "Compatibility+EmptyConfig", - validator: func(o *nodeBootstrappingOutput) { - sysctlContent, err := getBase64DecodedValue([]byte(o.vars["SYSCTL_CONTENT"])) + validator: func(cmd *exec.Cmd) { + vars := environToMap(cmd.Env) + sysctlContent, err := getBase64DecodedValue([]byte(vars["SYSCTL_CONTENT"])) require.NoError(t, err) assert.Contains(t, sysctlContent, fmt.Sprintf("net.ipv4.tcp_retries2=%v", 8)) assert.Contains(t, sysctlContent, fmt.Sprintf("net.core.message_burst=%v", 80)) @@ -383,33 +371,33 @@ func TestNBContractCompatibilityFromJsonToCSECommand(t *testing.T) { assert.Contains(t, sysctlContent, fmt.Sprintf("net.ipv4.neigh.default.gc_thresh1=%v", 4096)) assert.Contains(t, sysctlContent, fmt.Sprintf("net.ipv4.neigh.default.gc_thresh2=%v", 8192)) assert.Contains(t, sysctlContent, fmt.Sprintf("net.ipv4.neigh.default.gc_thresh3=%v", 16384)) - assert.Equal(t, "false", o.vars["IS_KATA"]) - assert.Equal(t, "false", o.vars["ENABLE_UNATTENDED_UPGRADES"]) - assert.Equal(t, "false", o.vars["NEEDS_CGROUPV2"]) - assert.Equal(t, "azureuser", o.vars["ADMINUSER"]) - assert.Equal(t, "0", o.vars["SWAP_FILE_SIZE_MB"]) - assert.Equal(t, "false", o.vars["SHOULD_CONFIG_TRANSPARENT_HUGE_PAGE"]) - assert.Equal(t, "", o.vars["THP_ENABLED"]) - assert.Equal(t, "", o.vars["THP_DEFRAG"]) - assert.Equal(t, "false", o.vars["DISABLE_SSH"]) - assert.Equal(t, "true", o.vars["IS_VHD"]) - assert.Equal(t, "false", o.vars["NEEDS_DOCKER_LOGIN"]) - assert.Equal(t, "", o.vars["MOBY_VERSION"]) - assert.Equal(t, "", o.vars["LOAD_BALANCER_SKU"]) - assert.Equal(t, "", o.vars["NETWORK_POLICY"]) - assert.Equal(t, "", o.vars["NETWORK_PLUGIN"]) - assert.Equal(t, "", o.vars["VNET_CNI_PLUGINS_URL"]) - assert.Equal(t, "false", o.vars["GPU_NODE"]) - assert.Equal(t, "", o.vars["GPU_INSTANCE_PROFILE"]) - assert.Equal(t, "0", o.vars["CUSTOM_CA_TRUST_COUNT"]) - assert.Equal(t, "false", o.vars["SHOULD_CONFIGURE_CUSTOM_CA_TRUST"]) - assert.Equal(t, "", o.vars["KUBELET_FLAGS"]) - assert.Equal(t, "", o.vars["KUBELET_NODE_LABELS"]) - assert.Equal(t, "", o.vars["HTTP_PROXY"]) - assert.Equal(t, "", o.vars["HTTPS_PROXY"]) - assert.Equal(t, "", o.vars["NO_PROXY"]) - assert.Equal(t, "", o.vars["PROXY_TRUSTED_CA"]) - assert.Equal(t, nbcontractv1.DefaultCloudName, o.vars["TARGET_ENVIRONMENT"]) + assert.Equal(t, "false", vars["IS_KATA"]) + assert.Equal(t, "false", vars["ENABLE_UNATTENDED_UPGRADES"]) + assert.Equal(t, "false", vars["NEEDS_CGROUPV2"]) + assert.Equal(t, "azureuser", vars["ADMINUSER"]) + assert.Equal(t, "0", vars["SWAP_FILE_SIZE_MB"]) + assert.Equal(t, "false", vars["SHOULD_CONFIG_TRANSPARENT_HUGE_PAGE"]) + assert.Equal(t, "", vars["THP_ENABLED"]) + assert.Equal(t, "", vars["THP_DEFRAG"]) + assert.Equal(t, "false", vars["DISABLE_SSH"]) + assert.Equal(t, "true", vars["IS_VHD"]) + assert.Equal(t, "false", vars["NEEDS_DOCKER_LOGIN"]) + assert.Equal(t, "", vars["MOBY_VERSION"]) + assert.Equal(t, "", vars["LOAD_BALANCER_SKU"]) + assert.Equal(t, "", vars["NETWORK_POLICY"]) + assert.Equal(t, "", vars["NETWORK_PLUGIN"]) + assert.Equal(t, "", vars["VNET_CNI_PLUGINS_URL"]) + assert.Equal(t, "false", vars["GPU_NODE"]) + assert.Equal(t, "", vars["GPU_INSTANCE_PROFILE"]) + assert.Equal(t, "0", vars["CUSTOM_CA_TRUST_COUNT"]) + assert.Equal(t, "false", vars["SHOULD_CONFIGURE_CUSTOM_CA_TRUST"]) + assert.Equal(t, "", vars["KUBELET_FLAGS"]) + assert.Equal(t, "", vars["KUBELET_NODE_LABELS"]) + assert.Equal(t, "", vars["HTTP_PROXY"]) + assert.Equal(t, "", vars["HTTPS_PROXY"]) + assert.Equal(t, "", vars["NO_PROXY"]) + assert.Equal(t, "", vars["PROXY_TRUSTED_CA"]) + assert.Equal(t, nbcontractv1.DefaultCloudName, vars["TARGET_ENVIRONMENT"]) }, }, } @@ -419,28 +407,29 @@ func TestNBContractCompatibilityFromJsonToCSECommand(t *testing.T) { nBCB := nbcontractv1.NewNBContractBuilder() nBCB.ApplyConfiguration(&nbcontractv1.Configuration{}) - inputJSON, err := json.Marshal(nBCB.GetNodeBootstrapConfig()) + cseCMD, err := parser.BuildCSECmd(context.TODO(), nBCB.GetNodeBootstrapConfig()) require.NoError(t, err) - cseCmd, err := parser.Parse(inputJSON) - require.NoError(t, err) - - generateTestDataIfRequested(t, tt.folder, cseCmd.UnsafeValue()) - vars, err := getDecodedVarsFromCseCmd([]byte(cseCmd)) - require.NoError(t, err) - - result := &nodeBootstrappingOutput{ - cseCmd: cseCmd.UnsafeValue(), - vars: vars, - } + generateTestDataIfRequested(t, tt.folder, cseCMD) if tt.validator != nil { - tt.validator(result) + tt.validator(cseCMD) } }) } } +func environToMap(env []string) map[string]string { + envMap := make(map[string]string) + for _, e := range env { + kv := strings.SplitN(e, "=", 2) + if len(kv) == 2 { + envMap[kv[0]] = kv[1] + } + } + return envMap +} + func TestContractCompatibilityHandledByProtobuf(t *testing.T) { tests := []struct { name string @@ -489,32 +478,6 @@ func TestContractCompatibilityHandledByProtobuf(t *testing.T) { } } -func getDecodedVarsFromCseCmd(data []byte) (map[string]string, error) { - cseRegex := regexp.MustCompile(cseRegexString) - cseVariableList := cseRegex.FindAllStringSubmatch(string(data), -1) - vars := make(map[string]string) - - for _, cseVar := range cseVariableList { - if len(cseVar) < 3 { - return nil, fmt.Errorf("expected 3 results (match, key, value) from regex, found %d, result %q", len(cseVar), cseVar) - } - - key := cseVar[1] - val := getValueWithoutQuotes(cseVar[2]) - - vars[key] = val - } - - return vars, nil -} - -func getValueWithoutQuotes(value string) string { - if len(value) > 1 && value[0] == '"' && value[len(value)-1] == '"' { - return value[1 : len(value)-1] - } - return value -} - func getBase64DecodedValue(data []byte) (string, error) { decoded, err := base64.StdEncoding.DecodeString(string(data)) if err != nil { @@ -538,13 +501,13 @@ func getNBCInstance(jsonFilePath string) *nbcontractv1.Configuration { return nBCB.GetNodeBootstrapConfig() } -func generateTestDataIfRequested(t *testing.T, folder, cseCmd string) { +func generateTestDataIfRequested(t *testing.T, folder string, cmd *exec.Cmd) { if os.Getenv("GENERATE_TEST_DATA") == "true" { if _, err := os.Stat(fmt.Sprintf("./testdata/%s", folder)); os.IsNotExist(err) { e := os.MkdirAll(fmt.Sprintf("./testdata/%s", folder), 0755) assert.NoError(t, e) } - err := os.WriteFile(fmt.Sprintf("./testdata/%s/generatedCSECommand", folder), []byte(cseCmd), 0644) + err := os.WriteFile(fmt.Sprintf("./testdata/%s/generatedCSECommand", folder), []byte(cmd.String()), 0644) assert.NoError(t, err) } } diff --git a/node-bootstrapper/parser/templates/cse_cmd.sh.gtpl b/node-bootstrapper/parser/templates/cse_cmd.sh.gtpl index c4ab500e337..f1fab3aae73 100644 --- a/node-bootstrapper/parser/templates/cse_cmd.sh.gtpl +++ b/node-bootstrapper/parser/templates/cse_cmd.sh.gtpl @@ -1,4 +1,3 @@ -PROVISION_OUTPUT="/var/log/azure/cluster-provision-cse-output.log"; echo $(date),$(hostname) > ${PROVISION_OUTPUT}; {{if not .GetDisableCustomData}} cloud-init status --wait > /dev/null 2>&1; @@ -9,146 +8,4 @@ echo "cloud-init succeeded" >> ${PROVISION_OUTPUT}; REPO_DEPOT_ENDPOINT="{{.CustomCloudConfig.RepoDepotEndpoint}}" {{getInitAKSCustomCloudFilepath}} >> /var/log/azure/cluster-provision.log 2>&1; {{end}} -ADMINUSER={{getLinuxAdminUsername .GetLinuxAdminUsername}} -MOBY_VERSION= -TENANT_ID={{.AuthConfig.GetTenantId}} -KUBERNETES_VERSION={{.GetKubernetesVersion}} -KUBE_BINARY_URL={{.KubeBinaryConfig.GetKubeBinaryUrl}} -CUSTOM_KUBE_BINARY_URL={{.KubeBinaryConfig.GetCustomKubeBinaryUrl}} -PRIVATE_KUBE_BINARY_URL="{{.KubeBinaryConfig.GetPrivateKubeBinaryUrl}}" -KUBEPROXY_URL={{.GetKubeProxyUrl}} -APISERVER_PUBLIC_KEY={{.ApiServerConfig.GetApiServerPublicKey}} -SUBSCRIPTION_ID={{.AuthConfig.GetSubscriptionId}} -RESOURCE_GROUP={{.ClusterConfig.GetResourceGroup}} -LOCATION={{.ClusterConfig.GetLocation}} -VM_TYPE={{getStringFromVMType .ClusterConfig.GetVmType}} -SUBNET={{.ClusterConfig.GetClusterNetworkConfig.GetSubnet}} -NETWORK_SECURITY_GROUP={{.ClusterConfig.GetClusterNetworkConfig.GetSecurityGroupName}} -VIRTUAL_NETWORK={{.ClusterConfig.GetClusterNetworkConfig.GetVnetName}} -VIRTUAL_NETWORK_RESOURCE_GROUP={{.ClusterConfig.GetClusterNetworkConfig.GetVnetResourceGroup}} -ROUTE_TABLE={{.ClusterConfig.GetClusterNetworkConfig.GetRouteTable}} -PRIMARY_AVAILABILITY_SET={{.ClusterConfig.GetPrimaryAvailabilitySet}} -PRIMARY_SCALE_SET={{.ClusterConfig.GetPrimaryScaleSet}} -SERVICE_PRINCIPAL_CLIENT_ID={{.AuthConfig.GetServicePrincipalId}} -NETWORK_PLUGIN={{getStringFromNetworkPluginType .GetNetworkConfig.GetNetworkPlugin}} -NETWORK_POLICY="{{getStringFromNetworkPolicyType .GetNetworkConfig.GetNetworkPolicy}}" -VNET_CNI_PLUGINS_URL={{.GetNetworkConfig.GetVnetCniPluginsUrl}} -CLOUDPROVIDER_BACKOFF=true -CLOUDPROVIDER_BACKOFF_MODE=v2 -CLOUDPROVIDER_BACKOFF_RETRIES=6 -CLOUDPROVIDER_BACKOFF_EXPONENT=0 -CLOUDPROVIDER_BACKOFF_DURATION=5 -CLOUDPROVIDER_BACKOFF_JITTER=0 -CLOUDPROVIDER_RATELIMIT=true -CLOUDPROVIDER_RATELIMIT_QPS=10 -CLOUDPROVIDER_RATELIMIT_QPS_WRITE=10 -CLOUDPROVIDER_RATELIMIT_BUCKET=100 -CLOUDPROVIDER_RATELIMIT_BUCKET_WRITE=100 -LOAD_BALANCER_DISABLE_OUTBOUND_SNAT={{.ClusterConfig.GetLoadBalancerConfig.GetDisableOutboundSnat}} -USE_MANAGED_IDENTITY_EXTENSION={{.AuthConfig.GetUseManagedIdentityExtension}} -USE_INSTANCE_METADATA={{.ClusterConfig.GetUseInstanceMetadata}} -LOAD_BALANCER_SKU={{getStringFromLoadBalancerSkuType .ClusterConfig.GetLoadBalancerConfig.GetLoadBalancerSku}} -EXCLUDE_MASTER_FROM_STANDARD_LB={{getExcludeMasterFromStandardLB .ClusterConfig.GetLoadBalancerConfig}} -MAXIMUM_LOADBALANCER_RULE_COUNT={{getMaxLBRuleCount .ClusterConfig.GetLoadBalancerConfig}} -CONTAINER_RUNTIME=containerd -CLI_TOOL=ctr -CONTAINERD_DOWNLOAD_URL_BASE={{.ContainerdConfig.GetContainerdDownloadUrlBase}} -NETWORK_MODE="transparent" -KUBE_BINARY_URL={{.KubeBinaryConfig.GetKubeBinaryUrl}} -USER_ASSIGNED_IDENTITY_ID={{.AuthConfig.GetAssignedIdentityId}} -API_SERVER_NAME={{.ApiServerConfig.GetApiServerName}} -IS_VHD={{getIsVHD .IsVhd}} -GPU_NODE={{getEnableNvidia .}} -SGX_NODE={{getIsSgxEnabledSKU .VmSize}} -MIG_NODE={{getIsMIGNode .GpuConfig.GetGpuInstanceProfile}} -CONFIG_GPU_DRIVER_IF_NEEDED={{.GpuConfig.GetConfigGpuDriver}} -ENABLE_GPU_DEVICE_PLUGIN_IF_NEEDED={{.GpuConfig.GetGpuDevicePlugin}} -TELEPORTD_PLUGIN_DOWNLOAD_URL={{.TeleportConfig.GetTeleportdPluginDownloadUrl}} -CREDENTIAL_PROVIDER_DOWNLOAD_URL={{.KubeBinaryConfig.GetLinuxCredentialProviderUrl}} -CONTAINERD_VERSION={{.ContainerdConfig.GetContainerdVersion}} -CONTAINERD_PACKAGE_URL={{.ContainerdConfig.GetContainerdPackageUrl}} -RUNC_VERSION={{.RuncConfig.GetRuncVersion}} -RUNC_PACKAGE_URL={{.RuncConfig.GetRuncPackageUrl}} -ENABLE_HOSTS_CONFIG_AGENT="{{.GetEnableHostsConfigAgent}}" -DISABLE_SSH="{{getDisableSSH .}}" -NEEDS_CONTAINERD="true" -TELEPORT_ENABLED="{{.TeleportConfig.GetStatus}}" -SHOULD_CONFIGURE_HTTP_PROXY="{{getShouldConfigureHTTPProxy .HttpProxyConfig}}" -SHOULD_CONFIGURE_HTTP_PROXY_CA="{{getShouldConfigureHTTPProxyCA .HttpProxyConfig}}" -HTTP_PROXY_TRUSTED_CA="{{.HttpProxyConfig.GetProxyTrustedCa}}" -SHOULD_CONFIGURE_CUSTOM_CA_TRUST="{{getCustomCACertsStatus .GetCustomCaCerts}}" -CUSTOM_CA_TRUST_COUNT="{{len .GetCustomCaCerts}}" -{{range $i, $cert := .CustomCaCerts}} -CUSTOM_CA_CERT_{{$i}}="{{$cert}}" -{{end}} -IS_KRUSTLET="{{getIsKrustlet .GetWorkloadRuntime}}" -GPU_NEEDS_FABRIC_MANAGER="{{getGPUNeedsFabricManager .VmSize}}" -NEEDS_DOCKER_LOGIN="false" -IPV6_DUAL_STACK_ENABLED="{{.GetIpv6DualStackEnabled}}" -OUTBOUND_COMMAND="{{.GetOutboundCommand}}" -ENABLE_UNATTENDED_UPGRADES="{{.GetEnableUnattendedUpgrade}}" -ENSURE_NO_DUPE_PROMISCUOUS_BRIDGE="{{getEnsureNoDupePromiscuousBridge .GetNetworkConfig}}" -SHOULD_CONFIG_SWAP_FILE="{{getEnableSwapConfig .CustomLinuxOsConfig}}" -SHOULD_CONFIG_TRANSPARENT_HUGE_PAGE="{{getShouldConfigTransparentHugePage .CustomLinuxOsConfig}}" -SHOULD_CONFIG_CONTAINERD_ULIMITS="{{getShouldConfigContainerdUlimits .CustomLinuxOsConfig.GetUlimitConfig}}" -CONTAINERD_ULIMITS="{{getUlimitContent .CustomLinuxOsConfig.GetUlimitConfig}}" -{{/* both CLOUD and ENVIRONMENT have special values when IsAKSCustomCloud == true */}} -{{/* CLOUD uses AzureStackCloud and seems to be used by kubelet, k8s cloud provider */}} -{{/* target environment seems to go to ARM SDK config */}} -{{/* not sure why separate/inconsistent? */}} -{{/* see GetCustomEnvironmentJSON for more weirdness. */}} -TARGET_CLOUD="{{getTargetCloud .}}" -TARGET_ENVIRONMENT="{{getTargetEnvironment .}}" -CUSTOM_ENV_JSON="{{.CustomCloudConfig.GetCustomEnvJsonContent}}" -IS_CUSTOM_CLOUD="{{getIsAksCustomCloud .CustomCloudConfig}}" -AKS_CUSTOM_CLOUD_CONTAINER_REGISTRY_DNS_SUFFIX="{{- if getIsAksCustomCloud .CustomCloudConfig}}{{.CustomCloudConfig.GetContainerRegistryDnsSuffix}}{{end}}" -CSE_HELPERS_FILEPATH="{{getCSEHelpersFilepath}}" -CSE_DISTRO_HELPERS_FILEPATH="{{getCSEDistroHelpersFilepath}}" -CSE_INSTALL_FILEPATH="{{getCSEInstallFilepath}}" -CSE_DISTRO_INSTALL_FILEPATH="{{getCSEDistroInstallFilepath}}" -CSE_CONFIG_FILEPATH="{{getCSEConfigFilepath}}" -AZURE_PRIVATE_REGISTRY_SERVER="{{.GetAzurePrivateRegistryServer}}" -HAS_CUSTOM_SEARCH_DOMAIN="{{getHasSearchDomain .GetCustomSearchDomainConfig}}" -CUSTOM_SEARCH_DOMAIN_FILEPATH="{{getCustomSearchDomainFilepath}}" -HTTP_PROXY_URLS="{{.HttpProxyConfig.GetHttpProxy}}" -HTTPS_PROXY_URLS="{{.HttpProxyConfig.GetHttpsProxy}}" -NO_PROXY_URLS="{{getStringifiedStringArray .HttpProxyConfig.GetNoProxyEntries ","}}" -PROXY_VARS="{{getProxyVariables .HttpProxyConfig}}" -ENABLE_TLS_BOOTSTRAPPING="{{getEnableTLSBootstrap .TlsBootstrappingConfig}}" -ENABLE_SECURE_TLS_BOOTSTRAPPING="{{getEnableSecureTLSBootstrap .TlsBootstrappingConfig}}" -CUSTOM_SECURE_TLS_BOOTSTRAP_AAD_SERVER_APP_ID="{{getCustomSecureTLSBootstrapAADServerAppID .TlsBootstrappingConfig}}" -DHCPV6_SERVICE_FILEPATH="{{getDHCPV6ServiceFilepath}}" -DHCPV6_CONFIG_FILEPATH="{{getDHCPV6ConfigFilepath}}" -THP_ENABLED="{{.CustomLinuxOsConfig.GetTransparentHugepageSupport}}" -THP_DEFRAG="{{.CustomLinuxOsConfig.GetTransparentDefrag}}" -SERVICE_PRINCIPAL_FILE_CONTENT="{{getServicePrincipalFileContent .AuthConfig}}" -KUBELET_CLIENT_CONTENT="{{.KubeletConfig.GetKubeletClientKey}}" -KUBELET_CLIENT_CERT_CONTENT="{{.KubeletConfig.GetKubeletClientCertContent}}" -KUBELET_CONFIG_FILE_ENABLED="{{.KubeletConfig.GetEnableKubeletConfigFile}}" -KUBELET_CONFIG_FILE_CONTENT="{{.KubeletConfig.GetKubeletConfigFileContent}}" -SWAP_FILE_SIZE_MB="{{.CustomLinuxOsConfig.GetSwapFileSize}}" -GPU_DRIVER_VERSION="{{getGpuDriverVersion .VmSize}}" -GPU_IMAGE_SHA="{{getGpuImageSha .VmSize}}" -GPU_INSTANCE_PROFILE="{{.GpuConfig.GetGpuInstanceProfile}}" -CUSTOM_SEARCH_DOMAIN_NAME="{{.CustomSearchDomainConfig.GetDomainName}}" -CUSTOM_SEARCH_REALM_USER="{{.CustomSearchDomainConfig.GetRealmUser}}" -CUSTOM_SEARCH_REALM_PASSWORD="{{.CustomSearchDomainConfig.GetRealmPassword}}" -MESSAGE_OF_THE_DAY="{{.GetMessageOfTheDay}}" -HAS_KUBELET_DISK_TYPE="{{getHasKubeletDiskType .KubeletConfig}}" -NEEDS_CGROUPV2="{{.GetNeedsCgroupv2}}" -TLS_BOOTSTRAP_TOKEN="{{getTLSBootstrapToken .TlsBootstrappingConfig}}" -KUBELET_FLAGS="{{createSortedKeyValueStringPairs .KubeletConfig.GetKubeletFlags " "}}" -NETWORK_POLICY="{{getStringFromNetworkPolicyType .NetworkConfig.GetNetworkPolicy}}" -KUBELET_NODE_LABELS="{{createSortedKeyValueStringPairs .KubeletConfig.GetKubeletNodeLabels ","}}" -AZURE_ENVIRONMENT_FILEPATH="{{getAzureEnvironmentFilepath .}}" -KUBE_CA_CRT="{{.GetKubernetesCaCert}}" -KUBENET_TEMPLATE="{{getKubenetTemplate}}" -CONTAINERD_CONFIG_CONTENT="{{getContainerdConfig .}}" -IS_KATA="{{.GetIsKata}}" -ARTIFACT_STREAMING_ENABLED="{{.GetEnableArtifactStreaming}}" -SYSCTL_CONTENT="{{getSysctlContent .CustomLinuxOsConfig.GetSysctlConfig}}" -PRIVATE_EGRESS_PROXY_ADDRESS="{{.GetPrivateEgressProxyAddress}}" -BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER="{{.GetBootstrapProfileContainerRegistryServer}}" -ENABLE_IMDS_RESTRICTION={{.ImdsRestrictionConfig.GetEnableImdsRestriction}} -INSERT_IMDS_RESTRICTION_RULE_TO_MANGLE_TABLE={{.ImdsRestrictionConfig.GetInsertImdsRestrictionRuleToMangleTable}} /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision_start.sh" \ No newline at end of file