Add service to tcpdump during run

This adds a service to run a tcpdump during the run.  This can be
useful to capture various network traffic for post analysis.

There didn't seem to quite be an appropriate place to document it, so
a new debugging file is started, with some terse explaination of our
various system-wide debugging services.

Change-Id: I09aaa57611c5047d09a9bce7932d34e9d50b30e6

.zuul.yaml

@@ -232,6 +232,7 @@
         '{{ devstack_log_dir }}/dstat-csv.log': logs
         '{{ devstack_log_dir }}/devstacklog.txt': logs
         '{{ devstack_log_dir }}/devstacklog.txt.summary': logs
+        '{{ devstack_log_dir }}/tcpdump.pcap': logs
         '{{ devstack_full_log}}': logs
         '{{ stage_dir }}/verify_tempest_conf.log': logs
         '{{ stage_dir }}/apache': logs

doc/source/debugging.rst

@@ -0,0 +1,46 @@
+=====================
+System-wide debugging
+=====================
+
+A lot can go wrong during a devstack run, and there are a few inbuilt
+tools to help you.
+
+dstat
+-----
+
+Enable the ``dstat`` service to produce performance logs during the
+devstack run.  These will be logged to the journal and also as a CSV
+file.
+
+memory_tracker
+--------------
+
+The ``memory_tracker`` service periodically monitors RAM usage and
+provides consumption output when available memory is seen to be
+falling (i.e. processes are consuming memory).  It also provides
+output showing locked (unswappable) memory.
+
+tcpdump
+-------
+
+Enable the ``tcpdump`` service to run a background tcpdump.  You must
+set the ``TCPDUMP_ARGS`` variable to something suitable (there is no
+default).  For example, to trace iSCSI communication during a job in
+the OpenStack gate and copy the result into the log output, you might
+use:
+
+.. code-block:: yaml
+
+   job:
+     name: devstack-job
+     parent: devstack
+     vars:
+       devstack_services:
+         tcpdump: true
+       devstack_localrc:
+         TCPDUMP_ARGS: "-i any tcp port 3260"
+       zuul_copy_output:
+         '{{ devstack_log_dir }}/tcpdump.pcap': logs
+
+
+

lib/tcpdump

@@ -0,0 +1,43 @@
+#!/bin/bash
+#
+# lib/tcpdump
+# Functions to start and stop a tcpdump
+
+# Dependencies:
+#
+# - ``functions`` file
+
+# ``stack.sh`` calls the entry points in this order:
+#
+# - start_tcpdump
+# - stop_tcpdump
+
+# Save trace setting
+_XTRACE_TCPDUMP=$(set +o | grep xtrace)
+set +o xtrace
+
+TCPDUMP_OUTPUT=${TCPDUMP_OUTPUT:-$LOGDIR/tcpdump.pcap}
+
+# e.g. for iscsi
+#  "-i any tcp port 3260"
+TCPDUMP_ARGS=${TCPDUMP_ARGS:-""}
+
+# start_tcpdump() - Start running processes
+function start_tcpdump {
+    # Run a tcpdump with given arguments and save the packet capture
+    if is_service_enabled tcpdump; then
+        if [[ -z "${TCPDUMP_ARGS}" ]]; then
+            die $LINENO "The tcpdump service requires TCPDUMP_ARGS to be set"
+        fi
+        touch ${TCPDUMP_OUTPUT}
+        run_process tcpdump "/usr/sbin/tcpdump -w $TCPDUMP_OUTPUT $TCPDUMP_ARGS" root root
+    fi
+}
+
+# stop_tcpdump() stop tcpdump process
+function stop_tcpdump {
+    stop_process tcpdump
+}
+
+# Restore xtrace
+$_XTRACE_TCPDUMP

stack.sh

@@ -614,6 +614,7 @@ source $TOP_DIR/lib/swift
 source $TOP_DIR/lib/neutron
 source $TOP_DIR/lib/ldap
 source $TOP_DIR/lib/dstat
+source $TOP_DIR/lib/tcpdump
 source $TOP_DIR/lib/etcd3
 
 # Extras Source
@@ -1053,6 +1054,12 @@ fi
 # A better kind of sysstat, with the top process per time slice
 start_dstat
 
+# Run a background tcpdump for debugging
+# Note: must set TCPDUMP_ARGS with the enabled service
+if is_service_enabled tcpdump; then
+    start_tcpdump
+fi
+
 # Etcd
 # -----