Merge pull request #120 from anarkiwi/reconcile

Background consolidation task

lib_test.sh

@@ -1,17 +1,23 @@
 #!/bin/bash
 
-restart_wait_dovesnap ()
+restart_dovesnap ()
 {
-	echo waiting for FAUCET config to have testnet
+        echo restarting dovesnap
 	DOVESNAPID="$(docker ps -q --filter name=dovesnap_plugin)"
+        docker logs $DOVESNAPID
+        docker restart $DOVESNAPID
+        docker logs $DOVESNAPID
+}
+
+restart_wait_dovesnap ()
+{
+	echo waiting for FAUCET config to have testnet mirror port
         TESTNETCOUNT=0
         while [ "$TESTNETCOUNT" != "1" ] ; do
-                TESTNETCOUNT=$(sudo grep -c testnet: $FAUCET_CONFIG)
+                TESTNETCOUNT=$(sudo grep -c 99: $FAUCET_CONFIG)
                 sleep 1
         done
-	echo restarting dovesnap
-	docker restart $DOVESNAPID
-	docker logs $DOVESNAPID
+	restart_dovesnap
 }
 
 init_dirs()

ovs/driver.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"reflect"
 	"strconv"
 	"strings"
 	"time"
@@ -481,6 +482,7 @@ func mustHandleCreate(d *Driver, confclient faucetconfserver.FaucetConfServerCli
 	}
 	configYaml := mergeInterfacesYaml(ns.NetworkName, ns.BridgeDpidInt, ns.BridgeName, add_interfaces)
 	if usingMirrorBridge(d) {
+		log.Debugf("configuring mirror bridge port for %s", ns.BridgeName)
 		stackMirrorConfig := d.stackMirrorConfigs[mapMsg.NetworkID]
 		ofportNum, mirrorOfportNum, err := d.addPatchPort(ns.BridgeName, mirrorBridgeName, uint(stackMirrorConfig.LbPort), 0)
 		if err != nil {
@@ -673,20 +675,60 @@ func mustHandleRm(d *Driver, confclient faucetconfserver.FaucetConfServerClient,
 	delete(*OFPorts, mapMsg.EndpointID)
 }
 
+func reconcileOvs(d *Driver, allPortDesc *map[string]map[uint]string) {
+	for id, ns := range d.networks {
+		stackMirrorConfig := d.stackMirrorConfigs[id]
+		newPortDesc := make(map[uint]string)
+		err := scrapePortDesc(ns.BridgeName, &newPortDesc)
+		if err != nil {
+			continue
+		}
+		portDesc, have_port_desc := (*allPortDesc)[id]
+		if have_port_desc {
+			if reflect.DeepEqual(newPortDesc, portDesc) {
+				continue
+			}
+			log.Debugf("portDesc for %s updated", ns.BridgeName)
+		} else {
+			log.Debugf("new portDesc for %s", ns.BridgeName)
+		}
+
+		for ofport, desc := range newPortDesc {
+			if uint32(ofport) == ofPortLocal {
+				continue
+			}
+			if uint32(ofport) == stackMirrorConfig.LbPort {
+				continue
+			}
+			if strings.HasPrefix(desc, ovsPortPrefix) {
+				continue
+			}
+			log.Debugf("non container port: %s %s %d %s", id, ns.BridgeName, ofport, desc)
+		}
+		(*allPortDesc)[id] = newPortDesc
+	}
+}
+
 func consolidateDockerInfo(d *Driver, confclient faucetconfserver.FaucetConfServerClient) {
 	OFPorts := make(map[string]OFPortContainer)
+	AllPortDesc := make(map[string]map[uint]string)
 
 	for {
-		mapMsg := <-d.ofportmapChan
-		switch mapMsg.Operation {
-		case "create":
-			mustHandleCreate(d, confclient, mapMsg)
-		case "add":
-			mustHandleAdd(d, confclient, mapMsg, &OFPorts)
-		case "rm":
-			mustHandleRm(d, confclient, mapMsg, &OFPorts)
+		select {
+		case mapMsg := <-d.ofportmapChan:
+			switch mapMsg.Operation {
+			case "create":
+				mustHandleCreate(d, confclient, mapMsg)
+			case "add":
+				mustHandleAdd(d, confclient, mapMsg, &OFPorts)
+			case "rm":
+				mustHandleRm(d, confclient, mapMsg, &OFPorts)
+			default:
+				log.Errorf("Unknown consolidation message: %s", mapMsg)
+			}
 		default:
-			log.Errorf("Unknown consolidation message: %s", mapMsg)
+			reconcileOvs(d, &AllPortDesc)
+			time.Sleep(3 * time.Second)
 		}
 	}
 }

ovs/ovs_config.go

@@ -43,16 +43,16 @@ const (
 	modeFlat = "flat"
 	modeNAT  = "nat"
 
-	bridgePrefix            = "ovsbr-"
-	containerEthName        = "eth"
-	mirrorBridgeName        = "mirrorbr"
-	netNsPath               = "/var/run/netns"
-	ofPortLocal       int64 = 4294967294
-	ovsPortPrefix           = "ovs-veth0-"
-	peerOvsPortPrefix       = "ethc"
-	stackDpidPrefix         = "0x0E0F00"
-	ovsStartupRetries       = 5
-	dockerRetries           = 3
+	bridgePrefix             = "ovsbr-"
+	containerEthName         = "eth"
+	mirrorBridgeName         = "mirrorbr"
+	netNsPath                = "/var/run/netns"
+	ofPortLocal       uint32 = 4294967294
+	ovsPortPrefix            = "ovs-veth0-"
+	peerOvsPortPrefix        = "ethc"
+	stackDpidPrefix          = "0x0E0F00"
+	ovsStartupRetries        = 5
+	dockerRetries            = 3
 )
 
 var (

ovs/ovs_port.go

@@ -12,20 +12,32 @@ import (
 	"github.com/vishvananda/netlink"
 )
 
-func (ovsdber *ovsdber) lowestFreePortOnBridge(bridgeName string) (lowestFreePort uint, err error) {
+func scrapePortDesc(bridgeName string, portDesc *map[uint]string) error {
 	output, err := OfCtl("dump-ports-desc", bridgeName)
 	if err != nil {
-		return 0, err
+		return err
 	}
-	var ofportNumberDump = regexp.MustCompile(`^\s*(\d+)\(\S+\).+$`)
-	existingOfPorts := []int{}
+	ofportNumberDump := regexp.MustCompile(`^\s*(\d+)\((\S+)\).+$`)
 	for _, line := range strings.Split(string(output), "\n") {
 		match := ofportNumberDump.FindAllStringSubmatch(line, -1)
 		if len(match) > 0 {
 			ofport, _ := strconv.Atoi(match[0][1])
-			existingOfPorts = append(existingOfPorts, ofport)
+			(*portDesc)[uint(ofport)] = match[0][2]
 		}
 	}
+	return nil
+}
+
+func (ovsdber *ovsdber) lowestFreePortOnBridge(bridgeName string) (lowestFreePort uint, err error) {
+	portDesc := make(map[uint]string)
+	err = scrapePortDesc(bridgeName, &portDesc)
+	if err != nil {
+		return 0, err
+	}
+	existingOfPorts := []int{}
+	for ofport, _ := range portDesc {
+		existingOfPorts = append(existingOfPorts, int(ofport))
+	}
 	sort.Ints(existingOfPorts)
 	intLowestFreePort := 1
 	for _, existingPort := range existingOfPorts {

test_dovesnap_standalone.sh

@@ -16,7 +16,7 @@ docker ps -a
 echo creating testnet
 docker network create testnet -d ovs --internal -o ovs.bridge.mode=nat -o ovs.bridge.dpid=0x1 -o ovs.bridge.controller=tcp:127.0.0.1:6653,tcp:127.0.0.1:6654 || exit 1
 docker network ls
-restart_wait_dovesnap
+restart_dovesnap
 echo creating testcon
 # github test runner can't use ping.
 docker pull busybox