Merge branch 'config_autoreload'

.gitignore

@@ -17,5 +17,4 @@ testhosts
 **/debian-binary
 packaging/dist
 itest/dist
-itest/config.yml
 .coverage

Makefile

@@ -45,15 +45,16 @@ docker-run-testhosts: testhosts
 	make EXTRA_DOCKER_ARGS="-v $(CURDIR)/testhosts:/etc/hosts:ro" docker-run
 
 itest: clean-cache docker-base-$(HOST_OS_RELEASE)
-	./itest/itest.sh docker
+	./itest/itest_generic.sh docker
 	./itest/itest_sourceipmap.sh
+	./itest/itest_autoreload.sh
 
 docker-base-%: Dockerfile.base
 	$(eval dollar_star := $(subst ubuntu_,,$*))
 	docker build -t pidtree-docker-base-$(dollar_star) --build-arg BASE_IMAGE=$(subst OS_RELEASE_PH,$(dollar_star),$(DOCKER_BASE_IMAGE_TMPL)) -f Dockerfile.base .
 
 itest_%: clean-cache docker-base-%
-	./itest/itest.sh $*
+	./itest/itest_generic.sh $*
 
 test: clean-cache
 	tox
@@ -74,8 +75,7 @@ clean-cache:
 
 clean: clean-cache
 	rm -Rf packaging/dist itest/dist
-	rm -f itest/itest_output_* itest/itest_server_*
-	rm -Rf itest/itest-sourceip-* itest/tmp
+	rm -Rf itest/tmp
 	rm -Rf .tox venv
 
 release:

itest/Dockerfile.ubuntu

@@ -28,6 +28,6 @@ COPY    --from=builder /bcc/*.deb /bcc/
 RUN     apt-get update && apt-get install -y /bcc/libbcc_*.deb /bcc/python3-bcc*.deb
 
 # Test package install
-ADD     deb_package_itest.sh /work/deb_package_itest.sh
+ADD     entrypoint_deb_package.sh /work/entrypoint_deb_package.sh
 ADD     dist/ubuntu_${OS_RELEASE}/ /work/dist/
-RUN     /work/deb_package_itest.sh setup ${HOSTRELEASE}
+RUN     /work/entrypoint_deb_package.sh setup ${HOSTRELEASE}

itest/config_autoreload.yml

@@ -0,0 +1,45 @@
+---
+_net_filters: &net_filters
+  - subnet_name: 0_0_0_0__2
+    network: 0.0.0.0
+    network_mask: 192.0.0.0
+    description: "Non-loopback subnet section"
+  - subnet_name: 64_0_0_0__3
+    network: 64.0.0.0
+    network_mask: 224.0.0.0
+    description: "Non-loopback subnet section"
+  - subnet_name: 96_0_0_0__4
+    network: 96.0.0.0
+    network_mask: 240.0.0.0
+    description: "Non-loopback subnet section"
+  - subnet_name: 112_0_0_0__5
+    network: 112.0.0.0
+    network_mask: 248.0.0.0
+    description: "Non-loopback subnet section"
+  - subnet_name: 120_0_0_0__6
+    network: 120.0.0.0
+    network_mask: 252.0.0.0
+    description: "Non-loopback subnet section"
+  - subnet_name: 124_0_0_0__7
+    network: 124.0.0.0
+    network_mask: 254.0.0.0
+    description: "Non-loopback subnet section"
+  - subnet_name: 126_0_0_0__8
+    network: 126.0.0.0
+    network_mask: 255.0.0.0
+    description: "Non-loopback subnet section"
+  - subnet_name: 128_0_0_0__1
+    network: 128.0.0.0
+    network_mask: 128.0.0.0
+    description: "Non-loopback subnet section"
+  - subnet_name: 127_0_0_0__16
+    network: 127.0.0.0
+    network_mask: 255.255.0.0
+    description: "127.0/16 to get rid of the noise"
+  - subnet_name: testcase
+    network: <net_address>
+    network_mask: 255.255.255.255
+    description: "filter for autoreload testcase"
+
+tcp_connect:
+  filters: *net_filters

itest/itest_autoreload.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+
+export TOPLEVEL=$(git rev-parse --show-toplevel)
+export CONTAINER_NAME=pidtree-autoreload-itest-$$
+export OUTPUT_NAME=itest/tmp/itest-autoreload-$$
+export DADDR='127.1.33.7'
+
+mkdir -p $TOPLEVEL/itest/tmp/autoreload
+
+function cleanup {
+    set +e
+    docker kill $CONTAINER_NAME
+    rm -f $TOPLEVEL/$OUTPUT_NAME
+}
+
+function create_connect_event {
+    echo "Creating connection event"
+    nc -w 2 -l -p 41337 -s $1 &
+    listener_pid=$!
+    sleep 1
+    nc -w 1 $1 41337
+    wait $listener_pid
+    echo "Connection event completed"
+}
+
+function test_output {
+    echo "Waiting for pidtree-bcc output, looking for $DADDR"
+    create_connect_event $DADDR &
+    connect_pid=$!
+    tail -n0 -f $OUTPUT_NAME | while read line; do
+        if echo "$line" | grep "$DADDR"; then
+            echo "Caught test traffic"
+            pkill -x --parent $$ tail
+            break
+        fi
+    done
+    wait $connect_pid
+    exit 0
+}
+
+function write_config {
+    sed "s/<net_address>/$1/g" $TOPLEVEL/itest/config_autoreload.yml > $TOPLEVEL/itest/tmp/autoreload/config.yml
+}
+
+trap cleanup INT EXIT
+
+touch $TOPLEVEL/$OUTPUT_NAME
+
+if [ -f /etc/lsb-release ]; then
+    source /etc/lsb-release
+else
+    echo "WARNING: Could not source /etc/lsb-release, tentatively creating bionic docker image"
+    DISTRIB_CODENAME=bionic
+fi
+docker build -t pidtree-itest-base --build-arg OS_RELEASE=$DISTRIB_CODENAME .
+docker build -t pidtree-itest itest
+
+echo "Creating background pidtree-bcc container to catch traffic"
+write_config $DADDR
+docker run --name $CONTAINER_NAME --rm -d \
+    --rm --privileged --cap-add sys_admin --pid host \
+    -v $TOPLEVEL/itest/tmp/autoreload:/work/config \
+    -v $TOPLEVEL/$OUTPUT_NAME:/work/output \
+    pidtree-itest -c /work/config/config.yml -f /work/output -w --health-check-period 1
+
+echo "Waiting a bit to let pidtree bootstrap"
+sleep 15
+
+export -f test_output
+export -f create_connect_event
+
+timeout 10s bash -c test_output
+if [ $? -eq 0 ]; then
+    echo "ERRROR: first connection even should have been filtered"
+    exit 1
+fi
+
+echo "Changing configuration values and waiting for hot-swap"
+write_config 1.1.1.1
+sleep 5
+
+timeout 20s bash -c test_output
+if [ $? -eq 0 ]; then
+    echo "SUCCESS!"
+    exit 0
+else
+    echo "FAILED! (timeout)"
+    exit 1
+fi

itest/itest.sh → itest/itest_generic.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -eE
 
-export OUTPUT_NAME=itest/itest_output_$$
+export OUTPUT_NAME=itest/tmp/itest_output_$$
 export TEST_PORT_1=${TEST_CONNECT_PORT:-31337}
 export TEST_PORT_2=${TEST_LISTEN_PORT:-41337}
 export TEST_LISTEN_TIMEOUT=${TEST_LISTEN_TIMEOUT:-2}
@@ -65,12 +65,12 @@ function cleanup {
   echo "CLEANUP: Killing container"
   docker kill $CONTAINER_NAME
   echo "CLEANUP: Removing FIFO"
-  rm -f $OUTPUT_NAME $TOPLEVEL/itest/config.yml
+  rm -f $TOPLEVEL/$OUTPUT_NAME $TOPLEVEL/itest/config.yml
 }
 
 function wait_for_tame_output {
   echo "Tailing output $OUTPUT_NAME to catch test traffic '$1'"
-  tail -n0 -f $OUTPUT_NAME | while read line; do
+  tail -n0 -f $TOPLEVEL/$OUTPUT_NAME | while read line; do
     if echo "$line" | grep "$1"; then
       echo "Caught test traffic matching '$1'"
       pkill -x --parent $$ tail
@@ -83,11 +83,12 @@ function wait_for_tame_output {
 
 function main {
   trap cleanup EXIT
+  mkdir -p $TOPLEVEL/itest/tmp
   is_port_used $TEST_PORT_1
   is_port_used $TEST_PORT_2
-  sed "s/<port1>/$TEST_PORT_1/g; s/<port2>/$TEST_PORT_2/g" $TOPLEVEL/itest/itest_config.yml > $TOPLEVEL/itest/config.yml
+  sed "s/<port1>/$TEST_PORT_1/g; s/<port2>/$TEST_PORT_2/g" $TOPLEVEL/itest/config_generic.yml > $TOPLEVEL/itest/tmp/config.yml
   if [ "$DEBUG" = "true" ]; then set -x; fi
-  touch $OUTPUT_NAME
+  touch $TOPLEVEL/$OUTPUT_NAME
   if [[ "$1" = "docker" ]]; then
     echo "Building itest image"
     # Build the base image
@@ -103,7 +104,7 @@ function main {
     echo "Launching itest-container $CONTAINER_NAME"
     docker run --name $CONTAINER_NAME -d\
         --rm --privileged --cap-add sys_admin --pid host \
-        -v $TOPLEVEL/itest/config.yml:/work/config.yml \
+        -v $TOPLEVEL/itest/tmp/config.yml:/work/config.yml \
         -v $TOPLEVEL/$OUTPUT_NAME:/work/outfile \
         pidtree-itest -c /work/config.yml -f /work/outfile
   elif [[ "$1" =~ ^ubuntu_[a-z]+$ ]]; then
@@ -119,11 +120,10 @@ function main {
         --build-arg OS_RELEASE=${1/ubuntu_/} --build-arg HOSTRELEASE=$DISTRIB_CODENAME itest/
     docker run --name $CONTAINER_NAME -d \
         --rm --privileged --cap-add sys_admin --pid host \
-        -v $TOPLEVEL/itest/config.yml:/work/config.yml \
+        -v $TOPLEVEL/itest/tmp/config.yml:/work/config.yml \
         -v $TOPLEVEL/$OUTPUT_NAME:/work/outfile \
         -v $TOPLEVEL/itest/dist/$1/:/work/dist \
-        -v $TOPLEVEL/itest/deb_package_itest.sh:/work/deb_package_itest.sh \
-        pidtree-itest-$1 /work/deb_package_itest.sh run -c /work/config.yml -f /work/outfile
+        pidtree-itest-$1 /work/entrypoint_deb_package.sh run -c /work/config.yml -f /work/outfile
   else
     echo "ERROR: '$@' is not a supported argument (see 'itest/itest.sh' for options)" >&2
     exit 1

itest/itest_sourceipmap.sh

@@ -1,30 +1,33 @@
 #!/bin/bash -e
 
+export TOPLEVEL=$(git rev-parse --show-toplevel)
 export CONTAINER_NAME=pidtree-mapping-itest-$$
 export TAME_CONTAINER_NAME=hello-$$
-export FIFONAME=itest/itest-sourceip-$$
+export FIFONAME=itest/tmp/itest-sourceip-$$
 
-itest/gen-ip-mapping.sh itest/tmp/ipmapping.txt 2 &
+mkdir -p $TOPLEVEL/itest/tmp
+
+$TOPLEVEL/itest/gen-ip-mapping.sh $TOPLEVEL/itest/tmp/maps/ipmapping.txt 2 &
 MAPGEN_PID=$!
 
 function cleanup {
     set +e
     kill $MAPGEN_PID
     docker kill $CONTAINER_NAME
-    rm -f $FIFONAME
+    rm -f $TOPLEVEL/$FIFONAME
 }
 
 function test_output {
     echo "Waiting for pidtree-bcc output corresponding to container $TAME_CONTAINER_NAME"
     while read line; do
         echo $line | grep $TAME_CONTAINER_NAME
         [ $? -eq 0 ] && return 0
-    done <$FIFONAME
+    done < $TOPLEVEL/$FIFONAME
 }
 
 trap cleanup INT EXIT
 
-mkfifo $FIFONAME
+mkfifo $TOPLEVEL/$FIFONAME
 
 if [ -f /etc/lsb-release ]; then
     source /etc/lsb-release
@@ -38,9 +41,9 @@ docker pull ubuntu:latest
 echo "Creating background pidtree-bcc container to catch traffic"
 docker run --name $CONTAINER_NAME --rm -d\
     --rm --privileged --cap-add sys_admin --pid host \
-    -v $(git rev-parse --show-toplevel)/itest/mapping_config.yml:/work/config.yml \
-    -v $(git rev-parse --show-toplevel)/itest/tmp:/maps/ \
-    -v $(git rev-parse --show-toplevel)/$FIFONAME:/work/output \
+    -v $TOPLEVEL/itest/config_mapping.yml:/work/config.yml \
+    -v $TOPLEVEL/itest/tmp/maps:/maps/ \
+    -v $TOPLEVEL/$FIFONAME:/work/output \
     pidtree-itest -c /work/config.yml -f /work/output
 
 echo "Creating background container $TAME_CONTAINER_NAME to send traffic"