Skip to content

Commit

Permalink
Merge pull request kubernetes#11719 from ritazh/feat-test-ccm
Browse files Browse the repository at this point in the history 
Add testCcm for Azure deployer
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Mar 13, 2019
2 parents 66b5153 + f2b5422 commit b752bcf
Show file tree
Hide file tree
Showing 2 changed files with 94 additions and 13 deletions.
67 changes: 62 additions & 5 deletions kubetest/azure.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ import (
"time"

"github.com/pelletier/go-toml"
"k8s.io/test-infra/kubetest/e2e"
"k8s.io/test-infra/kubetest/process"
"k8s.io/test-infra/kubetest/util"

"github.com/Azure/azure-storage-blob-go/2016-05-31/azblob"
Expand Down Expand Up @@ -63,6 +65,7 @@ var (
acsOrchestratorRelease = flag.String("acsengine-orchestratorRelease", "", "Orchestrator Profile for acs-engine")
acsWinZipBuildScript = flag.String("acsengine-winZipBuildScript", "https://raw.githubusercontent.com/Azure/acs-engine/master/scripts/build-windows-k8s.sh", "Build script to create custom zip containing win binaries for acs-engine")
acsNetworkPlugin = flag.String("acsengine-networkPlugin", "azure", "Network pluging to use with acs-engine")
testCcm = flag.Bool("test-ccm", false, "Set to True if you want kubetest to run e2e tests for ccm")
)

type Creds struct {
Expand Down Expand Up @@ -141,7 +144,7 @@ func checkParams() error {
*acsResourceName = "kubetest-" + uuid.NewV1().String()
}
if *acsResourceGroupName == "" {
*acsResourceGroupName = *acsResourceName + "-rg"
*acsResourceGroupName = *acsResourceName
}
if *acsDnsPrefix == "" {
*acsDnsPrefix = *acsResourceName
Expand Down Expand Up @@ -236,7 +239,11 @@ func (c *Cluster) populateApiModelTemplate() error {
if v.Properties.OrchestratorProfile.KubernetesConfig == nil {
v.Properties.OrchestratorProfile.KubernetesConfig = &KubernetesConfig{}
}
v.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = c.networkPlugin // default Azure
// to support aks-engine validation logic `networkPolicy 'none' is not supported with networkPlugin 'azure'`
if v.Properties.OrchestratorProfile.KubernetesConfig.NetworkPolicy != "none" && v.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin == "" {
// default NetworkPlugin to Azure if not provided
v.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = c.networkPlugin
}
if c.dnsPrefix != "" {
v.Properties.MasterProfile.DNSPrefix = c.dnsPrefix
}
Expand Down Expand Up @@ -451,7 +458,7 @@ func dockerLogout() error {

func (c *Cluster) buildCcm() error {

image := fmt.Sprintf("%v/azure-cloud-controller-manager:%v", os.Getenv("REGISTRY"), os.Getenv("BUILD_ID"))
image := fmt.Sprintf("%v/azure-cloud-controller-manager:%v-%v", os.Getenv("REGISTRY"), os.Getenv("BUILD_ID"), uuid.NewV1().String()[:8])
if err := c.dockerLogin(); err != nil {
return err
}
Expand Down Expand Up @@ -480,7 +487,7 @@ func (c *Cluster) buildCcm() error {

func (c *Cluster) buildHyperKube() error {

os.Setenv("VERSION", fmt.Sprintf("azure-e2e-%v", os.Getenv("BUILD_ID")))
os.Setenv("VERSION", fmt.Sprintf("azure-e2e-%v-%v", os.Getenv("BUILD_ID"), uuid.NewV1().String()[:8]))
if err := c.dockerLogin(); err != nil {
return err
}
Expand Down Expand Up @@ -579,7 +586,7 @@ func getZipBuildScript(buildScriptURL string, retry int) (string, error) {

func (c *Cluster) buildWinZip() error {

zipName := fmt.Sprintf("%s.zip", os.Getenv("BUILD_ID"))
zipName := fmt.Sprintf("%s%s.zip", os.Getenv("BUILD_ID"), uuid.NewV1().String()[:8])
buildFolder := path.Join(os.Getenv("HOME"), "winbuild")
zipPath := path.Join(os.Getenv("HOME"), zipName)
log.Printf("Building %s", zipName)
Expand Down Expand Up @@ -670,6 +677,25 @@ func (c Cluster) GetClusterCreated(clusterName string) (time.Time, error) {

func (c Cluster) TestSetup() error {

// set env vars required by the ccm e2e tests
if *testCcm == true {
if err := os.Setenv("K8S_AZURE_TENANTID", c.credentials.TenantID); err != nil {
return err
}
if err := os.Setenv("K8S_AZURE_SUBSID", c.credentials.SubscriptionID); err != nil {
return err
}
if err := os.Setenv("K8S_AZURE_SPID", c.credentials.ClientID); err != nil {
return err
}
if err := os.Setenv("K8S_AZURE_SPSEC", c.credentials.ClientSecret); err != nil {
return err
}
if err := os.Setenv("K8S_AZURE_LOCATION", c.location); err != nil {
return err
}
}

// Download repo-list that defines repositories for Windows test images.

downloadUrl, ok := os.LookupEnv("KUBE_TEST_REPO_LIST_DOWNLOAD_LOCATION")
Expand Down Expand Up @@ -704,3 +730,34 @@ func (c Cluster) IsUp() error {
}

func (_ Cluster) KubectlCommand() (*exec.Cmd, error) { return nil, nil }

// BuildTester returns a standard ginkgo-script tester or a custom one if testCcm is enabled
func (c *Cluster) BuildTester(o *e2e.BuildTesterOptions) (e2e.Tester, error) {
if *testCcm != true {
return &GinkgoScriptTester{}, nil
}
log.Printf("running go tests directly")
return &GinkgoCustomTester{}, nil
}

// GinkgoCustomTester implements Tester by calling a custom ginkgo script
type GinkgoCustomTester struct {
}

// Run executes custom ginkgo script
func (t *GinkgoCustomTester) Run(control *process.Control, testArgs []string) error {
artifactsDir, ok := os.LookupEnv("ARTIFACTS")
if !ok {
artifactsDir = filepath.Join(os.Getenv("WORKSPACE"), "_artifacts")
}
log.Printf("artifactsDir %v", artifactsDir)
// set CCM_JUNIT_REPORT_DIR for ccm e2e test to use the same dir
if err := os.Setenv("CCM_JUNIT_REPORT_DIR", artifactsDir); err != nil {
return err
}
cmd := exec.Command("make", "test-ccm-e2e")
projectPath := util.K8s("cloud-provider-azure")
cmd.Dir = projectPath
testErr := control.FinishRunning(cmd)
return testErr
}
40 changes: 32 additions & 8 deletions kubetest/azure_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,15 +76,39 @@ type WindowsProfile struct {
SSHEnabled bool `json:"sshEnabled,omitempty"`
}

// KubernetesContainerSpec defines configuration for a container spec
type KubernetesContainerSpec struct {
Name string `json:"name,omitempty"`
Image string `json:"image,omitempty"`
CPURequests string `json:"cpuRequests,omitempty"`
MemoryRequests string `json:"memoryRequests,omitempty"`
CPULimits string `json:"cpuLimits,omitempty"`
MemoryLimits string `json:"memoryLimits,omitempty"`
}

// KubernetesAddon defines a list of addons w/ configuration to include with the cluster deployment
type KubernetesAddon struct {
Name string `json:"name,omitempty"`
Enabled *bool `json:"enabled,omitempty"`
Containers []KubernetesContainerSpec `json:"containers,omitempty"`
Config map[string]string `json:"config,omitempty"`
Data string `json:"data,omitempty"`
}

type KubernetesConfig struct {
CustomWindowsPackageURL string `json:"customWindowsPackageURL,omitempty"`
CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"`
CustomCcmImage string `json:"customCcmImage,omitempty"` // Image for cloud-controller-manager
UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"`
NetworkPlugin string `json:"networkPlugin,omitempty"`
PrivateAzureRegistryServer string `json:"privateAzureRegistryServer,omitempty"`
AzureCNIURLLinux string `json:"azureCNIURLLinux,omitempty"`
AzureCNIURLWindows string `json:"azureCNIURLWindows,omitempty"`
CustomWindowsPackageURL string `json:"customWindowsPackageURL,omitempty"`
CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"`
CustomCcmImage string `json:"customCcmImage,omitempty"` // Image for cloud-controller-manager
UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"`
NetworkPlugin string `json:"networkPlugin,omitempty"`
PrivateAzureRegistryServer string `json:"privateAzureRegistryServer,omitempty"`
AzureCNIURLLinux string `json:"azureCNIURLLinux,omitempty"`
AzureCNIURLWindows string `json:"azureCNIURLWindows,omitempty"`
Addons []KubernetesAddon `json:"addons,omitempty"`
NetworkPolicy string `json:"networkPolicy,omitempty"`
CloudProviderRateLimitQPS float64 `json:"cloudProviderRateLimitQPS,omitempty"`
CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket,omitempty"`
APIServerConfig map[string]string `json:"apiServerConfig,omitempty"`
}
type OrchestratorProfile struct {
OrchestratorType string `json:"orchestratorType"`
Expand Down

0 comments on commit b752bcf

Please sign in to comment.