Containerized testing of DTS (#323)

.github/workflows/main.yaml

@@ -63,7 +63,9 @@ jobs:
           bin/setup_remote faucet
           bin/setup_remote daq
       - name: build docker
-        run: bin/retry_cmd bin/build_docker controller
+        run: |
+          bin/retry_cmd bin/build_docker controller
+          bin/build_dts
       - name: run python tests
         run: testing/python_test test_fot
       - name: setup fot stack
@@ -74,13 +76,17 @@ jobs:
         run: bin/setup_stack devices 5 skip-conn-check dhcp fot vxlan
       - name: run vxlan integration tests
         run: timeout 20m bin/run_test_set fot
+      - name: setup dts
+        run: bin/setup_stack devices 5 skip-conn-check dhcp fot dts localhost
+      - name: run dts integration tests
+        run: timeout 20m bin/run_test_set fot
       - name: post-run logs
         if: ${{ always() }}
         run: |
           echo %%%%%%%%%%%%% Controller startup logs
           docker logs forch-controller-1
           echo %%%%%%%%%%%%% DAQ startup logs
-          cat inst/forch-controller-1/daq/inst/cmdrun.log
+          cat inst/forch-controller-1/daq/inst/cmdrun.log || cat inst/forch-dts/cmdrun.log
           echo %%%%%%%%%%%%% Forch logs
           cat inst/forch-controller-1/forch.log
           echo %%%%%%%%%%%%% Endpoint logs

bin/build_dts

@@ -0,0 +1,15 @@
+#!/bin/bash -e
+
+echo Building daq dts container.
+
+cd daq
+
+echo Building daqf/dts...
+
+dts/build_dts
+
+docker tag daqf/dts forch/dts
+
+docker images | fgrep /dts
+
+echo Done with dts build.

bin/net_clean

@@ -5,7 +5,7 @@ source bin/stack_functions
 kill_forch
 
 echo Cleaning docker images...
-for image in forch/faucet forch/faux forch/gauge forch/controller; do
+for image in forch/faucet forch/faux forch/gauge forch/controller forch/dts; do
     echo Cleaning docker image $image...
     images=$(docker ps --filter ancestor=$image -q)
     if [ -n "$images" ]; then

bin/run_controller

@@ -39,8 +39,9 @@ if [ "$1" == "no-test" ]; then
 fi
 
 if [ "$1" == "vxlan" ]; then
-    vxlan=vxlan
-    shift
+    # Argument is target server for vxlan gRPC
+    vxlan=$2
+    shift 2
 fi
 
 if [ -n "$1" ]; then
@@ -119,7 +120,7 @@ docker_envs="--env RUN_MODE=$run_mode"
 
 if [[ -n $vxlan ]]; then
     docker_ip=$(ip addr show docker0 | sed -nr 's~.*inet ([0-9.]+)/.*~\1~p')
-    docker_envs+=" --env VXLAN_IP=$docker_ip"
+    docker_envs+=" --env VXLAN_IP=$docker_ip --env DTS_IP=$vxlan"
 fi
 
 docker_volumes+="\

bin/setup_remote

@@ -17,7 +17,7 @@ else
     echo Using base GIT_URL from environment.
 fi
 
-GIT_PEER=`echo $GIT_URL | sed s/forch/$TARGET/`
+GIT_PEER=`echo $GIT_URL | sed -E "s/(daq|forch)/$TARGET/"`
 echo Using $TARGET target git repo $GIT_PEER
 
 TARGET_REPO=$GIT_PEER
@@ -42,6 +42,9 @@ else
 fi
 
 echo Switching $TARGET to version $TARGET_VERSION
-(cd $TARGET; git checkout $TARGET_VERSION)
+(cd $TARGET
+ git checkout --detach --no-guess $TARGET_VERSION -- || git checkout --detach --no-guess origin/$TARGET_VERSION --
+)
+
 echo -n "Last $TARGET commit: "
 (cd $TARGET; git log -n 1 --pretty=format:"%h - %an, %ar : %s" || true)

bin/setup_stack

@@ -1,6 +1,6 @@
 #!/bin/bash -e
 
-ROOT=$(dirname $0)/..
+ROOT=$(realpath $(dirname $0)/..)
 cd $ROOT
 local=
 devices=
@@ -9,6 +9,7 @@ skip_conn_check=
 no_clean=
 faux_dhcp=
 fot=
+dts=
 vxlan=
 no_test=
 remote=
@@ -64,10 +65,16 @@ if [ "$1" == "fot" ]; then
 fi
 
 if [ "$1" == "vxlan" ]; then
-    vxlan=vxlan
+    vxlan="vxlan localhost"
     shift
 fi
 
+if [ "$1" == "dts" ]; then
+    # Argument is target address of dts server
+    dts="$2"
+    shift 2
+fi
+
 if [ "$1" == "dumptcp" ]; then
     dumptcp=y
     shift
@@ -135,6 +142,20 @@ docker kill forch-faucet-corp || true
 sudo modprobe bonding
 bin/run_faucet $local corp 6683
 
+if [[ -n $dts ]]; then
+    if [[ $dts == localhost ]]; then
+        docker rm -f forch-dts || true
+        echo Running local DTS container...
+        DTS_CID=$(docker run -d --privileged --name forch-dts -v $ROOT/inst/forch-dts:/root/inst forch/dts)
+        DTS_IP=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $DTS_CID)
+        vxlan="vxlan $DTS_IP"
+        sudo cp -a controller/site_config inst/forch-dts/
+    else
+        vxlan="vxlan $dts"
+    fi
+    echo Targeting DTS container at $vxlan
+fi
+
 finst=1
 if [ -z "$remote" ]; then
     if [ -n "$fot" ]; then
@@ -169,6 +190,10 @@ add_faux corp 1 0 dnsmasq
 if [ -n "$vxlan" ]; then
     echo Configuring for vxlan and endpoint handler server
     add_vxlan t1sw1 4
+    if [[ -z $skip_conn_check ]]; then
+        echo Connection check incompatible with vxlan: will wait forever.
+        false
+    fi
     bin/endpoint_server 2> inst/endpoint.log &
 elif [ -z "$remote" ]; then
     echo Enabling tap interface cntrl_tap_1

bin/stack_functions

@@ -535,7 +535,7 @@ function monitor_log {
 
 function monitor_daq_log {
     expected=$1
-    daqlog=$DAQ_BASE/inst/cmdrun.log
+    daqlog=$DAQ_BASE/cmdrun.log
     monitor_log "$expected" "$daqlog"
 }
 

controller/bin/controller_go

@@ -69,10 +69,11 @@ sudo /usr/share/openvswitch/scripts/ovs-ctl start
 
 if [[ -n $VXLAN_IP ]]; then
     # VXLAN_IP is where the switching fabric lives (outside of controller), including endpoint handler.
+    # DTS_IP is the address of the testing server (or localhost if to run locally).
     # docker_ip is the IP address of this container, reachable from the outside address.
-    echo Configuring for vxlan ip $VXLAN_IP, server at $docker_ip
+    echo Configuring for vxlan ip $VXLAN_IP, dts ${DTS_IP?}, endpoint $docker_ip
     yq -i -y .orchestration.sequester_config.tunnel_ip=\"$VXLAN_IP\" $FORCH_CONFIG
-    yq -i -y .orchestration.sequester_config.service_address=\"$docker_ip\" $FORCH_CONFIG
+    yq -i -y .orchestration.sequester_config.service_address=\"$DTS_IP\" $FORCH_CONFIG
     yq -i -y .switch_setup.endpoint.ip=\"$docker_ip\" $DAQ_CONFIG
 else
     echo Waiting for data0...
@@ -88,12 +89,14 @@ cat $FORCH_CONFIG
 /usr/bin/gauge_go &
 forch/bin/run_forch &
 
-echo Starting daq, output in daq/inst/cmdrun.log
-sudo \
-    PYTHONPATH=$PYTHONPATH \
-    PATH=$DAQ_ROOT/mininet:$PATH \
-    FAUCET_EVENT_SOCK=$DAQ_EVENT_SOCK \
-    daq/cmd/start ../$DAQ_BASE ../$DAQ_CONFIG -k $no_test > daq/inst/cmdrun.log 2>&1 &
+if [[ -z $DTS_IP || $DTS_IP == localhost ]]; then
+    echo Starting daq, output in daq/inst/cmdrun.log
+    sudo \
+        PYTHONPATH=$PYTHONPATH \
+        PATH=$DAQ_ROOT/mininet:$PATH \
+        FAUCET_EVENT_SOCK=$DAQ_EVENT_SOCK \
+        daq/cmd/start ../$DAQ_BASE ../$DAQ_CONFIG -k $no_test > daq/inst/cmdrun.log 2>&1 &
+fi
 
 echo Blocking forever...
 tail -f /dev/null

etc/DAQ_VERSION

@@ -1 +1 @@
-1.10.18
+1.10.20

forch/faucetizer.py

@@ -183,6 +183,9 @@ def _augment_include_file_name(self, file_name):
         return base_file_name + INCLUDE_FILE_SUFFIX + ext
 
     def _validate_and_initialize_config(self):
+        if self._config.unauthenticated_vlan:
+            self._logger.info('Unauthenticated vlan is %s', self._config.unauthenticated_vlan)
+
         if self._sequester_segment:
             starting_vlan = self._config.sequester_config.vlan_start
             ending_vlan = self._config.sequester_config.vlan_end

forch/port_state_manager.py

@@ -232,6 +232,7 @@ def _handle_learned_device(self, mac, device_placement, static=False):
         self._process_device_placement(mac, device_placement, static=static)
 
         if mac not in self._state_machines:
+            self._logger.info('Adding state machine %s', mac)
             self._state_machines[mac] = PortStateMachine(
                 mac, PortStateMachine.UNAUTHENTICATED, self._state_callbacks,
                 state_overwrites=self._state_overwrites)
@@ -253,6 +254,7 @@ def _handle_disconnected_device(self, device_placement):
 
         self._process_device_placement(eth_src, device_placement, static=False)
         if eth_src in self._state_machines:
+            self._logger.info('Removing state machine %s', eth_src)
             self._state_machines.pop(eth_src)
 
         self._update_device_state_varz(eth_src, DVAState.initial)

testing/test_fot

@@ -12,8 +12,16 @@ function report_vlans {
     $YQ '.dps."nz-kiwi-t2sw3".interfaces."1".native_vlan' $BEHAVIORAL_CONFIG | tee -a $TEST_RESULTS
 }
 
+if [ -f inst/forch-controller-1/daq/inst/cmdrun.log ]; then
+    DAQ_BASE=inst/forch-controller-1/daq/inst
+elif [ -f inst/forch-dts/cmdrun.log ]; then
+    DAQ_BASE=inst/forch-dts/
+else
+    echo Could not find cmdrun.log file.
+    false
+fi
+
 INST_BASE=inst/forch-controller-1
-DAQ_BASE=$INST_BASE/daq
 FAUCET_BASE=$INST_BASE/faucet
 BEHAVIORAL_CONFIG=$FAUCET_BASE/faucet.yaml
 YQ=venv/bin/yq
@@ -32,24 +40,24 @@ docker exec forch-faux-4 ping -c 10 192.168.1.0
 docker exec forch-faux-5 ping -c 1000 192.168.1.0 > /dev/null &
 
 echo Tail of DAQ log
-tail $DAQ_BASE/inst/cmdrun.log || true
+tail $DAQ_BASE/cmdrun.log || true
 
 # Basic process startup.
 echo Check basic process startup
-fgrep "Loading base module config" $DAQ_BASE/inst/cmdrun.log
+fgrep "Loading base module config" $DAQ_BASE/cmdrun.log
 
 # Testing that python path is good.
-ls -l $DAQ_BASE/inst/daq.pid
+ls -l $DAQ_BASE/daq.pid
 
 # Requires valid config available.
-cat $DAQ_BASE/inst/daq_run_id.txt
+cat $DAQ_BASE/daq_run_id.txt
 
 # Requiring mininet path.
-ls -l $DAQ_BASE/inst/dp_port_acls.yaml
+ls -l $DAQ_BASE/dp_port_acls.yaml
 
 # Check that OVS switch connects properly to managed faucet.
 echo Check for managed faucet
-fgrep "System port 1 on dpid 1 is active True" $DAQ_BASE/inst/cmdrun.log
+fgrep "System port 1 on dpid 1 is active True" $DAQ_BASE/cmdrun.log
 
 echo Done with daq startup tests.
 
@@ -68,28 +76,28 @@ vlan=$($YQ '.dps."nz-kiwi-t2sw1".interfaces."1".native_vlan' $BEHAVIORAL_CONFIG)
 echo $(( (vlan > 271) + (vlan > 276) )) | tee -a $TEST_RESULTS
 echo $(( (vlan > 271) + (vlan > 276) )) | tee -a $TEST_RESULTS
 
-monitor_daq_log "Sending device result for device 9a:02:57:1e:8f:01: passed"
-monitor_daq_log "Sending device result for device 9a:02:57:1e:8f:02: failed"
+monitor_daq_log "Send device result 9a:02:57:1e:8f:01 passed"
+monitor_daq_log "Send device result 9a:02:57:1e:8f:02 failed"
 monitor_daq_log "Target device 9a02571e8f03 test hold running"
 monitor_daq_log "Target device 9a02571e8f05 test hold running"
 
 echo `timestamp` stopping forch-faux-3
 docker stop forch-faux-3 # Remove device 3 to generate port-down event.
 
-log_count=$(fgrep "9a:02:57:1e:8f:04" $DAQ_BASE/inst/cmdrun.log | wc -l)
+log_count=$(fgrep "9a:02:57:1e:8f:04" $DAQ_BASE/cmdrun.log | wc -l)
 echo Device 04 log count $log_count | tee -a $TEST_RESULTS
 
 report_vlans
 
 monitor_forch_log "Device 9a:02:57:1e:8f:04 is entering operational state from unauthenticated state"
 
 # Check if it's running in client/server or server/client mode and adjust expectations accordingly.
-sserv_mode=$(fgrep ssserv $DAQ_BASE/inst/cmdrun.log | wc -l)
+sserv_mode=$(fgrep ssserv $DAQ_BASE/cmdrun.log | wc -l)
 if [[ $sserv_mode == 0 ]]; then
     monitor_daq_log "Remaining target sets: \[9a02571e8f05\]"
     expected_timeouts=0
     # Device 8f:05 doesn't timeout in this mode, so add artificial result for later on.
-    echo "9a02571e8f05: ['9a02571e8f05:hold:DaqException']" | sudo tee -a $DAQ_BASE/inst/result.log
+    echo "9a02571e8f05: ['9a02571e8f05:hold:DaqException']" | sudo tee -a $DAQ_BASE/result.log
 else
     monitor_forch_log "Device 9a:02:57:1e:8f:03 disconnected"
     monitor_daq_log "Remaining target sets: \[\]"
@@ -100,7 +108,7 @@ sequester_timeout=$(egrep "Handle device .* sequester timeout" $INST_BASE/forch.
 echo `timestamp` Found $sequester_timeouts and expected $expected_timeouts
 [[ $sequester_timeout == $expected_timeouts ]]
 
-cat $DAQ_BASE/inst/result.log | sort | tee -a $TEST_RESULTS
+cat $DAQ_BASE/result.log | sort | tee -a $TEST_RESULTS
 docker exec forch-faux-1 ping -c 5 192.168.1.0  > /dev/null # Make tests less flaky
 docker exec forch-faux-1 ping -c 5 192.168.1.0 | fgrep "packet loss" | redact | tee -a $TEST_RESULTS
 docker exec forch-faux-2 ping -c 5 192.168.1.0 | fgrep "packet loss" | redact | tee -a $TEST_RESULTS

topo/fot/forch/faucet.yaml

@@ -121,9 +121,6 @@ dps:
 include: [uniform.yaml]
 version: 2
 vlans:
-  100:
-    acls_in: [uniform_100]
-    description: unauthenticated
   500:
     acls_in: [uniform_500]
     description: INFRA