FOT test improvements (#343)

faucetsdn · Sep 21, 2021 · dc7b161 · dc7b161
1 parent 6e39bee
commit dc7b161
Show file tree

Hide file tree

Showing 8 changed files with 77 additions and 25 deletions.
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -66,45 +66,71 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: 3.8
-      - name: setup base
-        run: bin/setup_base
-      - name: setup faucet and daq
+      - name: setup forch
         env:
           GIT_URL: ${{ secrets.GIT_URL }}
         run: |
+          bin/setup_base
           bin/setup_remote faucet
           bin/setup_remote daq
-      - name: build docker
+          bin/retry_cmd bin/build_docker controller
+      - name: run python tests
+        run: testing/python_test test_fot
+      - name: run fot integration tests
+        run: bin/retry_cmd bin/run_fot_test
+      - name: post-run logs
+        if: ${{ always() }}
+        run: |
+          echo %%%%%%%%%%%%% Controller log
+          docker logs forch-controller-1
+          echo %%%%%%%%%%%%% DAQ log
+          cat inst/forch-controller-1/daq/inst/cmdrun.log || cat inst/forch-dts/cmdrun.log
+          echo %%%%%%%%%%%%% Forch log
+          cat inst/forch-controller-1/forch.log
+          echo %%%%%%%%%%%%% Faucet log
+          cat inst/forch-controller-1/faucet.log
+          echo %%%%%%%%%%%%% Endpoint log
+          cat inst/endpoint.log || true
+          echo %%%%%%%%%%%%% Log lengths
+          find inst -name \*.log | fgrep -v nodes/ | xargs wc -l
+
+  test_vxlan:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: setup forch
         env:
           GIT_URL: ${{ secrets.GIT_URL }}
         run: |
+          bin/setup_base
+          bin/setup_remote faucet
+          bin/setup_remote daq
           bin/retry_cmd bin/build_docker controller
           bin/build_dts
-      - name: run python tests
-        run: testing/python_test test_fot
-      - name: setup fot stack
-        run: bin/setup_stack devices 5 faux-mac-prefix "9a:99:57:1e" skip-conn-check dhcp fot
-      - name: run fot integration tests
-        run: timeout 20m bin/run_test_set fot
-      - name: setup vxlan stack
-        run: bin/setup_stack devices 5 faux-mac-prefix "9a:99:57:1e" skip-conn-check dhcp fot vxlan
       - name: run vxlan integration tests
-        run: timeout 20m bin/run_test_set fot
-      - name: setup dts
-        run: bin/setup_stack devices 5 faux-mac-prefix "9a:99:57:1e" skip-conn-check dhcp fot dts localhost
+        run: bin/retry_cmd bin/run_fot_test vxlan
       - name: run dts integration tests
-        run: timeout 20m bin/run_test_set fot
+        run: bin/retry_cmd bin/run_fot_test dts localhost
       - name: post-run logs
         if: ${{ always() }}
         run: |
-          echo %%%%%%%%%%%%% Controller startup logs
+          echo %%%%%%%%%%%%% Controller log
           docker logs forch-controller-1
-          echo %%%%%%%%%%%%% DAQ startup logs
+          echo %%%%%%%%%%%%% DAQ log
           cat inst/forch-controller-1/daq/inst/cmdrun.log || cat inst/forch-dts/cmdrun.log
-          echo %%%%%%%%%%%%% Forch logs
+          echo %%%%%%%%%%%%% Forch log
           cat inst/forch-controller-1/forch.log
-          echo %%%%%%%%%%%%% Endpoint logs
+          echo %%%%%%%%%%%%% Faucet log
+          cat inst/forch-controller-1/faucet.log
+          echo %%%%%%%%%%%%% Endpoint log
           cat inst/endpoint.log || true
+          echo %%%%%%%%%%%%% Log lengths
+          find inst -name \*.log | fgrep -v nodes/ | xargs wc -l
 
   test_reconfig:
     runs-on: ubuntu-latest

diff --git a/bin/run_fot_test b/bin/run_fot_test
@@ -0,0 +1,9 @@
+#!/bin/bash -e
+
+echo Starting fot test run $@
+
+bin/setup_stack devices 5 faux-mac-prefix "9a:99:57:1e" skip-conn-check dhcp fot $@
+
+timeout 15m bin/run_test_set fot
+
+echo Done with successful fot test run.
diff --git a/controller/bin/daq_config.yaml b/controller/bin/daq_config.yaml
@@ -4,6 +4,8 @@ include: ${DAQ_LIB}/config/system/default.yaml
 
 base_conf: ${DAQ_LIB}/resources/setups/orchestration/base_config.json
 
+host_tests: ${DAQ_LIB}/config/modules/dts.conf
+
 switch_setup:
   lo_port: 3984
   varz_port: 5678

diff --git a/etc/DAQ_VERSION b/etc/DAQ_VERSION
@@ -1 +1 @@
-1.10.29
+1.10.30
diff --git a/etc/Dockerfile.controller b/etc/Dockerfile.controller
@@ -31,7 +31,7 @@ RUN bin/install_faucet && rm -rf faucet/.git
 
 COPY controller/bin/install_forch bin/
 COPY . forch/
-RUN bin/install_forch && rm -rf forch/.git
+RUN rm -rf forch/venv && bin/install_forch && rm -rf forch/.git
 
 COPY daq/ daq/
 RUN daq/bin/setup_dev simple && rm -rf daq/.git

diff --git a/etc/test_fot.out b/etc/test_fot.out
@@ -14,3 +14,5 @@ Starting Forchestrator
 9a99571e8f05: ['9a99571e8f05:hold:DaqException']
 5 packets transmitted, 5 received, 0% packet loss, XXX
 5 packets transmitted, 0 received, XX errors, 100% packet loss, XXX
+X    Configured topology with 1 interfaces: [1]
+Log line lengths: 1 1 1
diff --git a/testing/test_fot b/testing/test_fot
@@ -35,8 +35,8 @@ echo `timestamp` DAQ spot checks
 
 # Test that no-sequester works
 docker exec forch-faux-4 ping -c 10 192.168.1.0
-# Schedule testing for faux-4 in 20s
-timestamp=`python3 -c "import datetime; print((datetime.datetime.now() + datetime.timedelta(seconds=20)).isoformat())"`
+# Test scheduled testing for faux-4
+timestamp=`python3 -c "import datetime; print((datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=20)).isoformat())"`
 docker exec forch-controller-1 bash -c "sed -i \"s/REPLACE_WITH_TIMESTAMP/$timestamp/\" /etc/forch/behaviors.yaml && touch /etc/forch/behaviors.yaml"
 
 # Ongoing activity faux-5 to keep it learned
@@ -113,4 +113,17 @@ docker exec forch-faux-1 ping -c 5 192.168.1.0  > /dev/null # Make tests less fl
 docker exec forch-faux-1 ping -c 5 192.168.1.0 | fgrep "packet loss" | redact | tee -a $TEST_RESULTS
 docker exec forch-faux-2 ping -c 5 192.168.1.0 | fgrep "packet loss" | redact | tee -a $TEST_RESULTS
 
+# Check to make sure allocated interfaces are cleaned up properly.
+fgrep "Configured topology with" $DAQ_BASE/cmdrun.log | tail -n 1 | sed 's/.*INFO/X/' | tee -a $TEST_RESULTS
+
+echo Remaining vxlan interfaces:
+docker exec forch-controller-1 ip link | fgrep vxlan | tee -a $TEST_RESULT
+
+faucet_lc=$(cat $INST_BASE/faucet.log | wc -l)
+forch_lc=$(cat $INST_BASE/forch.log | wc -l)
+daq_lc=$(cat $DAQ_BASE/cmdrun.log | wc -l)
+echo Log line lengths: faucet $faucet_lc forch $forch_lc daq $daq_lc
+# TODO: Tighten these bounds so they effectivly catch log spamming due to (e.g.) learning loops.
+echo Log line lengths: $((faucet_lc < 8000)) $((forch_lc < 4000)) $((daq_lc < 2000)) | tee -a $TEST_RESULTS
+
 echo `timestamp` Done with test_fot
diff --git a/topo/fot/forch/forch.yaml b/topo/fot/forch/forch.yaml
@@ -18,7 +18,7 @@ orchestration:
     vlan_start: 272
     vlan_end: 276
     port_description: TAP
-    sequester_timeout_sec: 360
+    sequester_timeout_sec: 420
     auto_sequestering: disabled
     service_address: 127.0.0.1
     test_result_device_states: