Merge pull request #121 from anarkiwi/reconcile2

Detect non dovesnap ports in OVS, and add/remove them from FAUCET.

Gopkg.toml

@@ -4,7 +4,7 @@
 
 [[constraint]]
   name = "github.com/iqtlabs/faucetconfrpc"
-  version = "0.17"
+  version = "0.20"
 
 [prune]
   go-tests = true

lib_test.sh

@@ -161,7 +161,7 @@ wait_for_pcap_match ()
 {
 	i=0
 	OUT=""
-	while [ "$OUT" == "" ] && [ i != 30 ] ; do
+	while [ "$OUT" == "" ] && [ "$i" != 30 ] ; do
 		echo waiting for pcap match $PCAPMATCH: $i
 		OUT=$(sudo tcpdump -n -r $MIRROR_PCAP -v | grep $PCAPMATCH)
 		((i=i+1))

ovs/driver.go

@@ -69,6 +69,7 @@ type NetworkState struct {
 	BridgeVLAN        int
 	MTU               int
 	Mode              string
+	AddPorts          string
 	Gateway           string
 	GatewayMask       string
 	FlatBindInterface string
@@ -224,6 +225,7 @@ func (d *Driver) ReOrCreateNetwork(r *networkplugin.CreateNetworkRequest, operat
 		BridgeVLAN:        vlan,
 		MTU:               mtu,
 		Mode:              mode,
+		AddPorts:          add_ports,
 		Gateway:           gateway,
 		GatewayMask:       mask,
 		FlatBindInterface: bindInterface,
@@ -618,6 +620,28 @@ func mustHandleAdd(d *Driver, confclient faucetconfserver.FaucetConfServerClient
 	(*OFPorts)[mapMsg.EndpointID] = containerMap
 }
 
+func deleteDpInterface(confclient faucetconfserver.FaucetConfServerClient, dpName string, ofport uint32) {
+	interfaces := &faucetconfserver.InterfaceInfo{
+		PortNo: uint32(ofport),
+	}
+	interfacesConf := []*faucetconfserver.DpInfo{
+		{
+			Name:       dpName,
+			Interfaces: []*faucetconfserver.InterfaceInfo{interfaces},
+		},
+	}
+
+	req := &faucetconfserver.DelDpInterfacesRequest{
+		InterfacesConfig: interfacesConf,
+		DeleteEmptyDp:    true,
+	}
+
+	_, err := confclient.DelDpInterfaces(context.Background(), req)
+	if err != nil {
+		log.Errorf("Error while calling DelDpInterfaces RPC %s: %v", req, err)
+	}
+}
+
 func mustHandleRm(d *Driver, confclient faucetconfserver.FaucetConfServerClient, mapMsg OFPortMap, OFPorts *map[string]OFPortContainer) {
 	defer func() {
 		if rerr := recover(); rerr != nil {
@@ -632,9 +656,11 @@ func mustHandleRm(d *Driver, confclient faucetconfserver.FaucetConfServerClient,
 		udhcpcCmd.Process.Kill()
 		udhcpcCmd.Wait()
 	}
-	ns := d.networks[mapMsg.NetworkID]
-	interfaces := &faucetconfserver.InterfaceInfo{
-		PortNo: int32(mapMsg.OFPort),
+	ns, have_network := d.networks[mapMsg.NetworkID]
+	if !have_network {
+		log.Debugf("Network %s already gone", mapMsg.NetworkID)
+		delete(*OFPorts, mapMsg.EndpointID)
+		return
 	}
 
 	log.Debugf("Removing port %d on %s from Faucet config", mapMsg.OFPort, ns.NetworkName)
@@ -655,21 +681,7 @@ func mustHandleRm(d *Driver, confclient faucetconfserver.FaucetConfServerClient,
 		}
 	}
 
-	interfacesConf := []*faucetconfserver.DpInfo{
-		{
-			Name:       ns.NetworkName,
-			Interfaces: []*faucetconfserver.InterfaceInfo{interfaces},
-		},
-	}
-
-	req := &faucetconfserver.DelDpInterfacesRequest{
-		InterfacesConfig: interfacesConf,
-		DeleteEmptyDp:    true,
-	}
-	_, err := confclient.DelDpInterfaces(context.Background(), req)
-	if err != nil {
-		log.Errorf("Error while calling DelDpInterfaces RPC %s: %v", req, err)
-	}
+	deleteDpInterface(confclient, ns.NetworkName, uint32(mapMsg.OFPort))
 
 	// The container will be gone by the time we query docker.
 	delete(*OFPorts, mapMsg.EndpointID)
@@ -683,28 +695,57 @@ func reconcileOvs(d *Driver, allPortDesc *map[string]map[uint]string) {
 		if err != nil {
 			continue
 		}
+		addPorts := make(map[string]uint32)
+		d.ovsdber.parseAddPorts(ns.AddPorts, &addPorts)
+
 		portDesc, have_port_desc := (*allPortDesc)[id]
 		if have_port_desc {
 			if reflect.DeepEqual(newPortDesc, portDesc) {
 				continue
 			}
 			log.Debugf("portDesc for %s updated", ns.BridgeName)
+
+			for ofport, desc := range portDesc {
+				_, have_new_port_desc := newPortDesc[ofport]
+				if have_new_port_desc {
+					continue
+				}
+				// Ignore container ports
+				if strings.HasPrefix(desc, ovsPortPrefix) {
+					continue
+				}
+				log.Infof("removing non dovesnap port: %s %s %d %s", id, ns.BridgeName, ofport, desc)
+				deleteDpInterface(d.faucetclient, ns.NetworkName, uint32(ofport))
+			}
 		} else {
 			log.Debugf("new portDesc for %s", ns.BridgeName)
 		}
 
+		add_interfaces := ""
+
 		for ofport, desc := range newPortDesc {
-			if uint32(ofport) == ofPortLocal {
+			// Ignore NAT and mirror port
+			if uint32(ofport) == ofPortLocal || uint32(ofport) == stackMirrorConfig.LbPort {
 				continue
 			}
-			if uint32(ofport) == stackMirrorConfig.LbPort {
+			// Ignore container and patch ports.
+			if strings.HasPrefix(desc, ovsPortPrefix) || strings.HasPrefix(desc, patchPrefix) {
 				continue
 			}
-			if strings.HasPrefix(desc, ovsPortPrefix) {
+			// Skip ports that were added at creation time.
+			_, have_add_port := addPorts[desc]
+			if have_add_port {
 				continue
 			}
-			log.Debugf("non container port: %s %s %d %s", id, ns.BridgeName, ofport, desc)
+			log.Infof("adding non dovesnap port: %s %s %d %s", id, ns.BridgeName, ofport, desc)
+			add_interfaces += fmt.Sprintf("%d: {description: %s, native_vlan: %d},", ofport, "Physical interface "+desc, ns.BridgeVLAN)
 		}
+
+		if add_interfaces != "" {
+			configYaml := mergeInterfacesYaml(ns.NetworkName, ns.BridgeDpidInt, ns.BridgeName, add_interfaces)
+			setFaucetConfigFile(d.faucetclient, configYaml)
+		}
+
 		(*allPortDesc)[id] = newPortDesc
 	}
 }

ovs/ovs_bridge.go

@@ -2,6 +2,7 @@ package ovs
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 	"time"
 
@@ -69,6 +70,26 @@ func (ovsdber *ovsdber) makeLoopbackBridge(bridgeName string) (err error) {
 	return err
 }
 
+func (ovsdber *ovsdber) parseAddPorts(add_ports string, addPorts *map[string]uint32) {
+	if add_ports == "" {
+		return
+	}
+	for _, add_port_number_str := range strings.Split(add_ports, ",") {
+		add_port_number := strings.Split(add_port_number_str, "/")
+		add_port := add_port_number[0]
+		if len(add_port_number) == 2 {
+			number, err := strconv.ParseUint(add_port_number[1], 10, 32)
+			if err != nil {
+				panic(err)
+			}
+			(*addPorts)[add_port] = uint32(number)
+		} else {
+			(*addPorts)[add_port] = 0
+		}
+	}
+	return
+}
+
 func (ovsdber *ovsdber) createBridge(bridgeName string, controller string, dpid string, add_ports string, exists bool) error {
 	if exists {
 		if _, err := ovsdber.addBridgeExists(bridgeName); err != nil {
@@ -94,12 +115,11 @@ func (ovsdber *ovsdber) createBridge(bridgeName string, controller string, dpid
 	}
 
 	if add_ports != "" {
-		for _, add_port_number_str := range strings.Split(add_ports, ",") {
-			add_port_number := strings.Split(add_port_number_str, "/")
-			add_port := add_port_number[0]
-			if len(add_port_number) == 2 {
-				number := add_port_number[1]
-				ovsConfigCmds = append(ovsConfigCmds, []string{"add-port", bridgeName, add_port, "--", "set", "Interface", add_port, fmt.Sprintf("ofport_request=%s", number)})
+		addPorts := make(map[string]uint32)
+		ovsdber.parseAddPorts(add_ports, &addPorts)
+		for add_port, number := range addPorts {
+			if number > 0 {
+				ovsConfigCmds = append(ovsConfigCmds, []string{"add-port", bridgeName, add_port, "--", "set", "Interface", add_port, fmt.Sprintf("ofport_request=%d", number)})
 			} else {
 				ovsConfigCmds = append(ovsConfigCmds, []string{"add-port", bridgeName, add_port})
 			}

ovs/ovs_config.go

@@ -49,6 +49,7 @@ const (
 	netNsPath                = "/var/run/netns"
 	ofPortLocal       uint32 = 4294967294
 	ovsPortPrefix            = "ovs-veth0-"
+	patchPrefix              = "ovp"
 	peerOvsPortPrefix        = "ethc"
 	stackDpidPrefix          = "0x0E0F00"
 	ovsStartupRetries        = 5
@@ -400,6 +401,7 @@ func getNetworkStateFromResource(r *types.NetworkResource) (ns NetworkState, err
 		MTU:               getIntOptionFromResource(r, mtuOption, defaultMTU),
 		Mode:              getStrOptionFromResource(r, modeOption, defaultMode),
 		FlatBindInterface: getStrOptionFromResource(r, bindInterfaceOption, ""),
+		AddPorts:          getStrOptionFromResource(r, bridgeAddPorts, ""),
 		UseDHCP:           parseBool(getStrOptionFromResource(r, dhcpOption, "")),
 		Gateway:           gateway,
 		GatewayMask:       mask,

ovs/ovs_port.go

@@ -8,6 +8,7 @@ import (
 	"strconv"
 	"strings"
 
+	base62 "github.com/kare/base62"
 	log "github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
 )
@@ -39,6 +40,7 @@ func (ovsdber *ovsdber) lowestFreePortOnBridge(bridgeName string) (lowestFreePor
 		existingOfPorts = append(existingOfPorts, int(ofport))
 	}
 	sort.Ints(existingOfPorts)
+	log.Debugf("existing ports on %s: %+v", bridgeName, existingOfPorts)
 	intLowestFreePort := 1
 	for _, existingPort := range existingOfPorts {
 		if existingPort != intLowestFreePort {
@@ -63,11 +65,15 @@ func (ovsdber *ovsdber) addInternalPort(bridgeName string, portName string, tag
 }
 
 func patchStr(a string) string {
-	return strconv.FormatUint(uint64(crc32.ChecksumIEEE([]byte(a))), 36)
+	return base62.Encode(int64(crc32.ChecksumIEEE([]byte(a))))
 }
 
 func patchName(a string, b string) string {
-	return patchStr(a) + patchStr(b)
+	name := patchPrefix + patchStr(a) + patchStr(b)
+	if len(name) > 15 {
+		panic(fmt.Errorf("%s too long for ifName", name))
+	}
+	return name
 }
 
 func (ovsdber *ovsdber) addPatchPort(bridgeName string, bridgeNamePeer string, port uint, portPeer uint) (uint, uint, error) {

test_dovesnap_mirrorbridge.sh

@@ -37,6 +37,7 @@ wait_mirror
 sudo grep -q "description: /testcon" $FAUCET_CONFIG || exit 1
 echo verifying networking
 sudo timeout 30s tcpdump -n -c 1 -U -i mirroro -w $MIRROR_PCAP tcp &
+sleep 3
 docker exec -t testcon wget -q -O- bing.com || exit 1
 PCAPMATCH=TCP
 wait_for_pcap_match