Merge pull request #14 from baidu/master

update

Makefile

@@ -1,11 +1,14 @@
-VERSION ?= $(shell git describe --always --dirty)
+include hack/Makefile.buildinfo
+
 GOOS ?= linux
 ARCH ?= amd64
 REGISTRY := hub.baidubce.com/jpaas-public
 BIN := cce-cloud-controller-manager
 IMAGE := $(REGISTRY)/$(BIN)
 SRC_DIRS := cmd pkg # directories which hold app source (not vendored)
 
+LDFLAGS=$(VERSION_LDFLAGS)
+
 .PHONY: all
 all: build
 
@@ -20,6 +23,7 @@ build: build-output
 	    -o output/${BIN}  \
 	    -installsuffix "static"               \
 	    -ldflags "-X main.version=${VERSION}" \
+	    -ldflags "${LDFLAGS}" \
 	    ./cmd/cce-cloud-controller-manager
 
 .PHONY: local-build
@@ -29,6 +33,7 @@ local-build: build-output
 	    -o output/${BIN}  \
 	    -installsuffix "static"               \
 	    -ldflags "-X main.version=${VERSION}" \
+	    -ldflags "${LDFLAGS}" \
 	    ./cmd/cce-cloud-controller-manager
 
 .PHONY: image-build
@@ -50,4 +55,4 @@ deploy:
 
 .PHONY: version
 version:
-	@echo ${VERSION}
+	@echo ${VERSION}

hack/Makefile.buildinfo

@@ -0,0 +1,56 @@
+# adapted from the hack scripts in kubernetes/kubernetes
+
+GIT_COMMIT:=$(shell git rev-parse "HEAD^{commit}" 2>/dev/null)
+
+# the raw git version from `git describe` -- our starting point
+GIT_VERSION_RAW:=$(shell git describe --tags --abbrev=14 "$(GIT_COMMIT)^{commit}" 2>/dev/null)
+
+# use the number of dashes in the raw version to figure out what kind of
+# version this is, and turn it into a semver-compatible version
+DASHES_IN_VERSION:=$(shell echo "$(GIT_VERSION_RAW)" | sed "s/[^-]//g")
+
+# just use the raw version by default
+GIT_VERSION:=$(GIT_VERSION_RAW)
+
+ifeq ($(DASHES_IN_VERSION), ---)
+# we have a distance to a subversion (v1.1.0-subversion-1-gCommitHash)
+GIT_VERSION:=$(shell echo "$(GIT_VERSION_RAW)" | sed "s/-\([0-9]\{1,\}\)-g\([0-9a-f]\{14\}\)$$/.\1\+\2/")
+endif
+ifeq ($(DASHES_IN_VERSION), --)
+# we have distance to base tag (v1.1.0-1-gCommitHash)
+GIT_VERSION:=$(shell echo "$(GIT_VERSION_RAW)" | sed "s/-g\([0-9a-f]\{14\}\)$$/+\1/")
+endif
+
+# figure out if we have new or changed files
+ifeq ($(shell git status --porcelain 2>/dev/null), "")
+GIT_TREE_STATE:=clean
+else
+# append the -dirty manually, since `git describe --dirty` only considers
+# changes to existing files
+GIT_TREE_STATE:=dirty
+GIT_VERSION:=$(GIT_VERSION)-dirty
+endif
+
+# construct a "shorter" version without the commit info, etc for use as container image tag, etc
+VERSION?=$(shell echo "$(GIT_VERSION)" | grep -E -o '^v[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+(-(alpha|beta)\.[[:digit:]]+)?')
+
+# construct the build date, taking into account SOURCE_DATE_EPOCH, which is
+# used for the purpose of reproducible builds
+ifdef SOURCE_DATE_EPOCH
+BUILD_DATE:=$(shell date --date=@${SOURCE_DATE_EPOCH} -u +'%Y-%m-%dT%H:%M:%SZ')
+else
+BUILD_DATE:=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ')
+endif
+
+# set the build information version ldflags (but not other ldflags)
+VERSION_LDFLAGS:=-X k8s.io/cloud-provider-baiducloud/vendor/k8s.io/kubernetes/pkg/version.gitVersion=$(GIT_VERSION) -X k8s.io/cloud-provider-baiducloud/vendor/k8s.io/kubernetes/pkg/version.gitCommit=$(GIT_COMMIT) -X k8s.io/cloud-provider-baiducloud/vendor/k8s.io/kubernetes/pkg/version.gitTreeState=$(GIT_TREE_STATE) -X k8s.io/cloud-provider-baiducloud/vendor/k8s.io/kubernetes/pkg/version.buildDate=$(BUILD_DATE)
+
+export VERSION
+export VERSION_LDFLAGS
+
+# print out a summary of the current version info
+version-info:
+	@echo "Version: $(GIT_VERSION) ($(VERSION))"
+	@echo "    built from $(GIT_COMMIT) ($(GIT_TREE_STATE))"
+	@echo "    built on $(BUILD_DATE)"
+.PHONY: version-info

pkg/cloud-provider/instances.go

@@ -138,6 +138,10 @@ func (bc *Baiducloud) CurrentNodeName(ctx context.Context, hostname string) (typ
 // InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running.
 // If false is returned with no error, the instance will be immediately deleted by the cloud controller manager.
 func (bc *Baiducloud) InstanceExistsByProviderID(ctx context.Context, providerID string) (bool, error) {
+	// when node.spec.providerID is not set, providerID is only instanceID, not start with cce://
+	if !strings.HasPrefix(providerID, bc.ProviderName() + "://") {
+		providerID = bc.ProviderName() + "://" + providerID
+	}
 	splitted := strings.Split(providerID, "//")
 	if len(splitted) != 2 {
 		return false, fmt.Errorf("parse ProviderID failed: %v", providerID)

pkg/cloud-provider/load_balancer.go

@@ -18,7 +18,6 @@ package cloud_provider
 
 import (
 	"context"
-	"fmt"
 	"strings"
 
 	"github.com/golang/glog"
@@ -101,7 +100,7 @@ func (bc *Baiducloud) EnsureLoadBalancer(ctx context.Context, clusterName string
 	if err != nil {
 		return nil, err
 	}
-
+	service.Annotations[ServiceAnnotationCceAutoAddEip] = pubIP
 	glog.V(3).Infof("[%v %v] EnsureLoadBalancer: EXTERNAL-IP is %s", service.Namespace, service.Name, pubIP)
 	return &v1.LoadBalancerStatus{Ingress: []v1.LoadBalancerIngress{{IP: pubIP}}}, nil
 }
@@ -150,6 +149,10 @@ func (bc *Baiducloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName
 	}
 	if !existsLb {
 		glog.V(3).Infof("[%v %v] EnsureLoadBalancerDeleted: target blb not exist", serviceName, clusterName)
+		err = bc.DeleteEipFinally(service, result, lb, serviceName, clusterName)
+		if err != nil {
+			return err
+		}
 		return nil
 	}
 
@@ -273,42 +276,10 @@ func (bc *Baiducloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName
 		}
 		return nil
 	}
-
-	// delete EIP
-	if result.LoadBalancerInternalVpc == "true" { //do not assign the eip
-		if service.Annotations != nil {
-			delete(service.Annotations, ServiceAnnotationCceAutoAddLoadBalancerId)
-		}
-		glog.V(3).Infof("[%v %v] EnsureLoadBalancerDeleted: use LoadBalancerInternalVpc, no EIP to delete", service.Namespace, service.Name)
-		glog.V(2).Infof("[%v %v] EnsureLoadBalancerDeleted: delete %v FINISH", serviceName, clusterName, serviceName)
-		return nil
-	}
-	if len(service.Spec.LoadBalancerIP) != 0 { //use user’s eip, do not delete
-		if service.Annotations != nil {
-			delete(service.Annotations, ServiceAnnotationCceAutoAddLoadBalancerId)
-		}
-		glog.V(3).Infof("[%v %v] EnsureLoadBalancerDeleted: LoadBalancerIP is set, not delete EIP.", serviceName, clusterName)
-		glog.V(2).Infof("[%v %v] EnsureLoadBalancerDeleted: delete %v FINISH", serviceName, clusterName, serviceName)
-		return nil
-	}
-	glog.V(3).Infof("[%v %v] EnsureLoadBalancerDeleted: Start delete EIP: %s", serviceName, clusterName, lb.PublicIp)
-	var targetEip string
-	if len(service.Status.LoadBalancer.Ingress) != 0 { // P0: use service EXTERNAL_IP
-		targetEip = service.Status.LoadBalancer.Ingress[0].IP
-	}
-	if len(targetEip) == 0 { // P1: use BLB public ip
-		targetEip = lb.PublicIp
-	}
-	if len(targetEip) == 0 { // get none EIP
-		return fmt.Errorf("EnsureLoadBalancerDeleted failed: can not get a EIP to delete")
-	}
-	err = bc.deleteEIP(targetEip)
+	// does not assign blb, delete EIP
+	err = bc.DeleteEipFinally(service, result, lb, serviceName, clusterName)
 	if err != nil {
 		return err
 	}
-	if service.Annotations != nil {
-		delete(service.Annotations, ServiceAnnotationCceAutoAddLoadBalancerId)
-	}
-	glog.V(2).Infof("[%v %v] EnsureLoadBalancerDeleted: delete %v FINISH", serviceName, clusterName, serviceName)
 	return nil
 }

pkg/cloud-provider/load_balancer_blb_backend_server.go

@@ -48,7 +48,7 @@ func (bc *Baiducloud) reconcileBackendServers(service *v1.Service, nodes []*v1.N
 	var candidateBackends []blb.BackendServer
 	for _, node := range nodes {
 		splitted := strings.Split(node.Spec.ProviderID, "//")
-		if len(splitted) < 1 {
+		if len(splitted) != 2 {
 			glog.Warningf("node %s has no spec.providerId", node.Name)
 			continue
 		}

pkg/cloud-provider/load_balancer_eip.go

@@ -440,3 +440,48 @@ func (bc *Baiducloud) resizeEip(service *v1.Service, serviceAnnotation *ServiceA
 		Ip:              targetEip,
 	})
 }
+
+func (bc *Baiducloud) DeleteEipFinally(service *v1.Service, result *ServiceAnnotation, lb *blb.LoadBalancer, serviceName, clusterName string) error {
+	// delete EIP
+	if result.LoadBalancerInternalVpc == "true" { //do not assign the eip
+		if service.Annotations != nil {
+			delete(service.Annotations, ServiceAnnotationCceAutoAddLoadBalancerId)
+		}
+		glog.V(3).Infof("[%v %v] EnsureLoadBalancerDeleted: use LoadBalancerInternalVpc, no EIP to delete", service.Namespace, service.Name)
+		glog.V(2).Infof("[%v %v] EnsureLoadBalancerDeleted: delete %v FINISH", serviceName, clusterName, serviceName)
+		return nil
+	}
+	if len(service.Spec.LoadBalancerIP) != 0 { //use user’s eip, do not delete
+		if service.Annotations != nil {
+			delete(service.Annotations, ServiceAnnotationCceAutoAddLoadBalancerId)
+		}
+		glog.V(3).Infof("[%v %v] EnsureLoadBalancerDeleted: LoadBalancerIP is set, not delete EIP.", serviceName, clusterName)
+		glog.V(2).Infof("[%v %v] EnsureLoadBalancerDeleted: delete %v FINISH", serviceName, clusterName, serviceName)
+		return nil
+	}
+	glog.V(3).Infof("[%v %v] EnsureLoadBalancerDeleted: Start delete EIP: %s", serviceName, clusterName, lb.PublicIp)
+	var targetEip string
+	if len(service.Status.LoadBalancer.Ingress) != 0 { // P0: use service EXTERNAL_IP
+		targetEip = service.Status.LoadBalancer.Ingress[0].IP
+	}
+	if len(targetEip) == 0 { // P1: use BLB public ip
+		targetEip = lb.PublicIp
+	}
+	if len(targetEip) == 0 {
+		targetEip = result.CceAutoAddEip
+	}
+	if len(targetEip) == 0 { // get none EIP
+		glog.V(3).Infof("Eip does not exist, Delete completed ")
+		return nil
+	}
+	err := bc.deleteEIP(targetEip)
+	if err != nil {
+		return err
+	}
+	if service.Annotations != nil {
+		delete(service.Annotations, ServiceAnnotationCceAutoAddLoadBalancerId)
+		delete(service.Annotations, ServiceAnnotationCceAutoAddEip)
+	}
+	glog.V(2).Infof("[%v %v] EnsureLoadBalancerDeleted: delete %v FINISH", serviceName, clusterName, serviceName)
+	return nil
+}

pkg/cloud-provider/options.go

@@ -29,6 +29,8 @@ const (
 	ServiceAnnotationLoadBalancerPrefix = "service.beta.kubernetes.io/cce-load-balancer-"
 	// CceAutoAddLoadBalancerId is the annotation of CCE adding LoadBalancerId
 	ServiceAnnotationCceAutoAddLoadBalancerId = ServiceAnnotationLoadBalancerPrefix + "cce-add-id"
+	// CceAutoAddEip is the annotation of CCE adding Eip
+	ServiceAnnotationCceAutoAddEip = ServiceAnnotationLoadBalancerPrefix + "cce-add-eip"
 
 	ServiceAnnotationLoadBalancerExistId = ServiceAnnotationLoadBalancerPrefix + "exist-id"
 	// ServiceAnnotationLoadBalancerInternalVpc is the annotation of LoadBalancerInternalVpc
@@ -85,18 +87,21 @@ const (
 
 	// NodeAnnotationCCMVersion is the version of CCM
 	NodeAnnotationCCMVersion = NodeAnnotationPrefix + "ccm-version"
+
+	// NodeAnnotationAdvertiseRoute indicates whether to advertise route to vpc route table
+	NodeAnnotationAdvertiseRoute = NodeAnnotationPrefix + "advertise-route"
 )
 
 // ServiceAnnotation contains annotations from service
 type ServiceAnnotation struct {
 	/* BLB */
-	CceAutoAddLoadBalancerId               string
-	LoadBalancerExistId                    string
-	LoadBalancerInternalVpc                string
-	LoadBalancerAllocateVip                string
-	LoadBalancerSubnetId                   string
-	LoadBalancerScheduler                  string
-	LoadBalancerRsMaxNum                   int
+	CceAutoAddLoadBalancerId string
+	LoadBalancerExistId      string
+	LoadBalancerInternalVpc  string
+	LoadBalancerAllocateVip  string
+	LoadBalancerSubnetId     string
+	LoadBalancerScheduler    string
+	LoadBalancerRsMaxNum     int
 
 	LoadBalancerHealthCheckTimeoutInSecond int
 	LoadBalancerHealthCheckInterval        int
@@ -118,6 +123,7 @@ type NodeAnnotation struct {
 	VpcRouteTableId string
 	VpcRouteRuleId  string
 	CCMVersion      string
+	AdvertiseRoute  bool
 }
 
 // ExtractServiceAnnotation extract annotations from service
@@ -133,7 +139,11 @@ func ExtractServiceAnnotation(service *v1.Service) (*ServiceAnnotation, error) {
 	if exist {
 		result.CceAutoAddLoadBalancerId = loadBalancerId
 	}
+	cceAddEip, exist := annotation[ServiceAnnotationCceAutoAddEip]
+	if exist {
+		result.CceAutoAddEip = cceAddEip
 
+	}
 	LoadBalancerExistId, exist := annotation[ServiceAnnotationLoadBalancerExistId]
 	if exist {
 		result.LoadBalancerExistId = LoadBalancerExistId
@@ -283,5 +293,16 @@ func ExtractNodeAnnotation(node *v1.Node) (*NodeAnnotation, error) {
 		result.CCMVersion = ccmVersion
 	}
 
+	advertiseRoute, ok := annotation[NodeAnnotationAdvertiseRoute]
+	if ok {
+		advertise, err := strconv.ParseBool(advertiseRoute)
+		if err != nil {
+			return nil, fmt.Errorf("NodeAnnotationAdvertiseRoute syntex error: %v", err)
+		}
+		result.AdvertiseRoute = advertise
+	} else {
+		result.AdvertiseRoute = true
+	}
+
 	return result, nil
 }

pkg/cloud-provider/routes.go

@@ -79,6 +79,16 @@ func (bc *Baiducloud) ListRoutes(ctx context.Context, clusterName string) (route
 			DestinationCIDR: r.DestinationAddress,
 			TargetNode:      types.NodeName(insName),
 		}
+
+		advertiseRoute, err := bc.advertiseRoute(insName)
+		if err != nil {
+			continue
+		}
+		// use route.Blackhole to mark this route to be deleted
+		if !advertiseRoute {
+			route.Blackhole = true
+		}
+
 		vpcId, err := bc.getVpcID()
 		if err != nil {
 			return nil, err
@@ -104,6 +114,17 @@ func (bc *Baiducloud) CreateRoute(ctx context.Context, clusterName string, nameH
 	if len(vpcRoutes) < 1 {
 		return fmt.Errorf("VPC route length error: length is : %d", len(vpcRoutes))
 	}
+
+	advertiseRoute, err := bc.advertiseRoute(string(kubeRoute.TargetNode))
+	if err != nil {
+		return err
+	}
+
+	if !advertiseRoute {
+		glog.V(3).Infof("Node %s has annotation not to advertise route", string(kubeRoute.TargetNode))
+		return nil
+	}
+
 	var insID string
 	inss, err := bc.clientSet.Cce().ListInstances(bc.ClusterID)
 	if err != nil {
@@ -141,6 +162,11 @@ func (bc *Baiducloud) CreateRoute(ctx context.Context, clusterName string, nameH
 		}
 	}
 
+	if insID == "" {
+		glog.Errorf("InstanceId not found for k8s node %s, not create route", string(kubeRoute.TargetNode))
+		return fmt.Errorf("InstanceId not found for k8s node %s, create route failed", string(kubeRoute.TargetNode))
+	}
+
 	args := vpc.CreateRouteRuleArgs{
 		RouteTableID:       vpcRoutes[0].RouteTableID,
 		NexthopType:        "custom",
@@ -228,7 +254,7 @@ func (bc *Baiducloud) ensureRouteInfoToNode(nodeName, vpcId, vpcRouteTableId, vp
 	if err != nil {
 		return err
 	}
-  
+
 	isChanged := false
 	if nodeAnnotation.VpcId != vpcId {
 		curNode.Annotations[NodeAnnotationVpcId] = vpcId
@@ -342,3 +368,23 @@ func (bc *Baiducloud) isConflict(otherRR vpc.RouteRule, cceRR vpc.RouteRule) boo
 
 	return false
 }
+
+func (bc *Baiducloud) advertiseRoute(nodename string) (bool, error) {
+
+	// check node resource in k8s has advertise route annotation, if is false, not create route
+	curNode, err := bc.kubeClient.CoreV1().Nodes().Get(nodename, metav1.GetOptions{})
+	if err != nil {
+		if !strings.Contains(err.Error(), "not found") {
+			return true, err
+		}
+	}
+
+	if curNode.Annotations == nil {
+		curNode.Annotations = make(map[string]string)
+	}
+	nodeAnnotation, err := ExtractNodeAnnotation(curNode)
+	if err != nil {
+		return true, err
+	}
+	return nodeAnnotation.AdvertiseRoute, nil
+}