From 061f0073148c662a793ae5bfc2ffdab957639e3a Mon Sep 17 00:00:00 2001 From: Josh Bailey Date: Wed, 26 Aug 2020 13:23:13 +1200 Subject: [PATCH 1/2] Background consolidation task (logs only) --- lib_test.sh | 18 +++++++++----- ovs/driver.go | 47 ++++++++++++++++++++++++++++++------- ovs/ovs_config.go | 20 ++++++++-------- ovs/ovs_port.go | 22 +++++++++++++---- test_dovesnap_standalone.sh | 2 +- 5 files changed, 78 insertions(+), 31 deletions(-) diff --git a/lib_test.sh b/lib_test.sh index 2382e643..f8fea72d 100644 --- a/lib_test.sh +++ b/lib_test.sh @@ -1,17 +1,23 @@ #!/bin/bash -restart_wait_dovesnap () +restart_dovesnap () { - echo waiting for FAUCET config to have testnet + echo restarting dovesnap DOVESNAPID="$(docker ps -q --filter name=dovesnap_plugin)" + docker logs $DOVESNAPID + docker restart $DOVESNAPID + docker logs $DOVESNAPID +} + +restart_wait_dovesnap () +{ + echo waiting for FAUCET config to have testnet mirror port TESTNETCOUNT=0 while [ "$TESTNETCOUNT" != "1" ] ; do - TESTNETCOUNT=$(sudo grep -c testnet: $FAUCET_CONFIG) + TESTNETCOUNT=$(sudo grep -c 99: $FAUCET_CONFIG) sleep 1 done - echo restarting dovesnap - docker restart $DOVESNAPID - docker logs $DOVESNAPID + restart_dovesnap } init_dirs() diff --git a/ovs/driver.go b/ovs/driver.go index cf18cd2b..b093153e 100755 --- a/ovs/driver.go +++ b/ovs/driver.go @@ -481,6 +481,7 @@ func mustHandleCreate(d *Driver, confclient faucetconfserver.FaucetConfServerCli } configYaml := mergeInterfacesYaml(ns.NetworkName, ns.BridgeDpidInt, ns.BridgeName, add_interfaces) if usingMirrorBridge(d) { + log.Debugf("configuring mirror bridge port for %s", ns.BridgeName) stackMirrorConfig := d.stackMirrorConfigs[mapMsg.NetworkID] ofportNum, mirrorOfportNum, err := d.addPatchPort(ns.BridgeName, mirrorBridgeName, uint(stackMirrorConfig.LbPort), 0) if err != nil { @@ -673,20 +674,48 @@ func mustHandleRm(d *Driver, confclient faucetconfserver.FaucetConfServerClient, delete(*OFPorts, mapMsg.EndpointID) } +func reconcileOvs(d *Driver) { + for id, ns := range d.networks { + stackMirrorConfig := d.stackMirrorConfigs[id] + portDesc := make(map[uint]string) + err := scrapePortDesc(ns.BridgeName, &portDesc) + if err != nil { + continue + } + for ofport, desc := range portDesc { + if uint32(ofport) == ofPortLocal { + continue + } + if uint32(ofport) == stackMirrorConfig.LbPort { + continue + } + if strings.HasPrefix(desc, ovsPortPrefix) { + continue + } + log.Debugf("non container port: %s %s %d %s", id, ns.BridgeName, ofport, desc) + } + } +} + func consolidateDockerInfo(d *Driver, confclient faucetconfserver.FaucetConfServerClient) { OFPorts := make(map[string]OFPortContainer) for { - mapMsg := <-d.ofportmapChan - switch mapMsg.Operation { - case "create": - mustHandleCreate(d, confclient, mapMsg) - case "add": - mustHandleAdd(d, confclient, mapMsg, &OFPorts) - case "rm": - mustHandleRm(d, confclient, mapMsg, &OFPorts) + select { + case mapMsg := <-d.ofportmapChan: + switch mapMsg.Operation { + case "create": + mustHandleCreate(d, confclient, mapMsg) + case "add": + mustHandleAdd(d, confclient, mapMsg, &OFPorts) + case "rm": + mustHandleRm(d, confclient, mapMsg, &OFPorts) + default: + log.Errorf("Unknown consolidation message: %s", mapMsg) + } default: - log.Errorf("Unknown consolidation message: %s", mapMsg) + reconcileOvs(d) + time.Sleep(3 * time.Second) } } } diff --git a/ovs/ovs_config.go b/ovs/ovs_config.go index 5958326c..0e46fb84 100644 --- a/ovs/ovs_config.go +++ b/ovs/ovs_config.go @@ -43,16 +43,16 @@ const ( modeFlat = "flat" modeNAT = "nat" - bridgePrefix = "ovsbr-" - containerEthName = "eth" - mirrorBridgeName = "mirrorbr" - netNsPath = "/var/run/netns" - ofPortLocal int64 = 4294967294 - ovsPortPrefix = "ovs-veth0-" - peerOvsPortPrefix = "ethc" - stackDpidPrefix = "0x0E0F00" - ovsStartupRetries = 5 - dockerRetries = 3 + bridgePrefix = "ovsbr-" + containerEthName = "eth" + mirrorBridgeName = "mirrorbr" + netNsPath = "/var/run/netns" + ofPortLocal uint32 = 4294967294 + ovsPortPrefix = "ovs-veth0-" + peerOvsPortPrefix = "ethc" + stackDpidPrefix = "0x0E0F00" + ovsStartupRetries = 5 + dockerRetries = 3 ) var ( diff --git a/ovs/ovs_port.go b/ovs/ovs_port.go index 8746ccc4..380c38b3 100644 --- a/ovs/ovs_port.go +++ b/ovs/ovs_port.go @@ -12,20 +12,32 @@ import ( "github.com/vishvananda/netlink" ) -func (ovsdber *ovsdber) lowestFreePortOnBridge(bridgeName string) (lowestFreePort uint, err error) { +func scrapePortDesc(bridgeName string, portDesc *map[uint]string) error { output, err := OfCtl("dump-ports-desc", bridgeName) if err != nil { - return 0, err + return err } - var ofportNumberDump = regexp.MustCompile(`^\s*(\d+)\(\S+\).+$`) - existingOfPorts := []int{} + ofportNumberDump := regexp.MustCompile(`^\s*(\d+)\((\S+)\).+$`) for _, line := range strings.Split(string(output), "\n") { match := ofportNumberDump.FindAllStringSubmatch(line, -1) if len(match) > 0 { ofport, _ := strconv.Atoi(match[0][1]) - existingOfPorts = append(existingOfPorts, ofport) + (*portDesc)[uint(ofport)] = match[0][2] } } + return nil +} + +func (ovsdber *ovsdber) lowestFreePortOnBridge(bridgeName string) (lowestFreePort uint, err error) { + portDesc := make(map[uint]string) + err = scrapePortDesc(bridgeName, &portDesc) + if err != nil { + return 0, err + } + existingOfPorts := []int{} + for ofport, _ := range portDesc { + existingOfPorts = append(existingOfPorts, int(ofport)) + } sort.Ints(existingOfPorts) intLowestFreePort := 1 for _, existingPort := range existingOfPorts { diff --git a/test_dovesnap_standalone.sh b/test_dovesnap_standalone.sh index 829fac5d..1a939c6b 100755 --- a/test_dovesnap_standalone.sh +++ b/test_dovesnap_standalone.sh @@ -16,7 +16,7 @@ docker ps -a echo creating testnet docker network create testnet -d ovs --internal -o ovs.bridge.mode=nat -o ovs.bridge.dpid=0x1 -o ovs.bridge.controller=tcp:127.0.0.1:6653,tcp:127.0.0.1:6654 || exit 1 docker network ls -restart_wait_dovesnap +restart_dovesnap echo creating testcon # github test runner can't use ping. docker pull busybox From 0a2a71b6a7f2acd60a2f576b932e1a907ed786d1 Mon Sep 17 00:00:00 2001 From: Josh Bailey Date: Wed, 26 Aug 2020 16:39:48 +1200 Subject: [PATCH 2/2] Detect when port map changed. --- ovs/driver.go | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/ovs/driver.go b/ovs/driver.go index b093153e..3c3274fd 100755 --- a/ovs/driver.go +++ b/ovs/driver.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "os" "os/exec" + "reflect" "strconv" "strings" "time" @@ -674,15 +675,25 @@ func mustHandleRm(d *Driver, confclient faucetconfserver.FaucetConfServerClient, delete(*OFPorts, mapMsg.EndpointID) } -func reconcileOvs(d *Driver) { +func reconcileOvs(d *Driver, allPortDesc *map[string]map[uint]string) { for id, ns := range d.networks { stackMirrorConfig := d.stackMirrorConfigs[id] - portDesc := make(map[uint]string) - err := scrapePortDesc(ns.BridgeName, &portDesc) + newPortDesc := make(map[uint]string) + err := scrapePortDesc(ns.BridgeName, &newPortDesc) if err != nil { continue } - for ofport, desc := range portDesc { + portDesc, have_port_desc := (*allPortDesc)[id] + if have_port_desc { + if reflect.DeepEqual(newPortDesc, portDesc) { + continue + } + log.Debugf("portDesc for %s updated", ns.BridgeName) + } else { + log.Debugf("new portDesc for %s", ns.BridgeName) + } + + for ofport, desc := range newPortDesc { if uint32(ofport) == ofPortLocal { continue } @@ -694,11 +705,13 @@ func reconcileOvs(d *Driver) { } log.Debugf("non container port: %s %s %d %s", id, ns.BridgeName, ofport, desc) } + (*allPortDesc)[id] = newPortDesc } } func consolidateDockerInfo(d *Driver, confclient faucetconfserver.FaucetConfServerClient) { OFPorts := make(map[string]OFPortContainer) + AllPortDesc := make(map[string]map[uint]string) for { select { @@ -714,7 +727,7 @@ func consolidateDockerInfo(d *Driver, confclient faucetconfserver.FaucetConfServ log.Errorf("Unknown consolidation message: %s", mapMsg) } default: - reconcileOvs(d) + reconcileOvs(d, &AllPortDesc) time.Sleep(3 * time.Second) } }