fullstack: Don't let dhcp agents failover

It turned out dhcp tests work only because agents are considered dead
after 10 seconds while they report to server every 60 seconds. This led
to calling network resync after agent revival and hiding the fact dhcp
agent is not capable of receiving any amqp messages.

This patch sets the report interval of agents to the half of
agent_down_time on server side and uses eventlet dhcp agent in order to
trigger eventlet monkey patching code.

Eventlet was behind the failure with messages not getting processed. As
[1] notes: "Note: If the “eventlet” executor is used, the threading and
time library need to be monkeypatched."

Because each port calls dhclient to obtain IP address and each dhclient
instance overwrites /etc/resolv.conf there was added a script that
generates fullstack-dhclient-script from an existing dhclient-script
before starting fulltstack tests. This generated script is passed to
each dhclient process running in fake fullstack machine using -sf
parameter.

[1] https://docs.openstack.org/developer/oslo.messaging/server.html

Related-bug: 1453350

Change-Id: I0336176b9c364fe3a95be5cef9e7a3af1ef9d7e9

neutron/tests/fullstack/base.py

@@ -26,6 +26,7 @@
 # This is the directory from which infra fetches log files for fullstack tests
 DEFAULT_LOG_DIR = os.path.join(helpers.get_test_log_path(),
                                'dsvm-fullstack-logs')
+ROOTDIR = os.path.dirname(__file__)
 
 
 class BaseFullStackTestCase(testlib_api.MySQLTestCaseMixin,

neutron/tests/common/agents/fullstack_dhcp_agent.py → neutron/tests/fullstack/cmd/dhcp_agent.py

@@ -20,8 +20,8 @@
 from oslo_config import cfg
 from oslo_utils import uuidutils
 
-from neutron.agent import dhcp_agent
 from neutron.agent.linux import dhcp as linux_dhcp
+from neutron.cmd.eventlet.agents import dhcp as dhcp_agent
 
 
 OPTS = [

neutron/tests/fullstack/resources/config.py

@@ -87,6 +87,9 @@ def __init__(self, env_desc, host_desc, temp_dir,
             'oslo_policy': {
                 'policy_file': self._generate_policy_json(),
             },
+            'agent': {
+                'report_interval': env_desc.agent_down_time / 2.0
+            },
         })
 
     def _setUp(self):

neutron/tests/fullstack/resources/machine.py

@@ -12,6 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from distutils import spawn
 import itertools
 
 import netaddr
@@ -25,6 +26,8 @@
 from neutron.tests.common import machine_fixtures
 from neutron.tests.common import net_helpers
 
+FULLSTACK_DHCLIENT_SCRIPT = 'fullstack-dhclient-script'
+
 
 class FakeFullstackMachinesList(list):
     """A list of items implementing the FakeFullstackMachine interface."""
@@ -42,6 +45,8 @@ def ping_all(self):
 
 
 class FakeFullstackMachine(machine_fixtures.FakeMachineBase):
+    NO_RESOLV_CONF_DHCLIENT_SCRIPT_PATH = (
+        spawn.find_executable(FULLSTACK_DHCLIENT_SCRIPT))
 
     def __init__(self, host, network_id, tenant_id, safe_client,
                  neutron_port=None, bridge_name=None, use_dhcp=False):
@@ -129,7 +134,8 @@ def _configure_ipaddress_via_dhcp(self):
         self.addCleanup(self._stop_async_dhclient)
 
     def _start_async_dhclient(self):
-        cmd = ["dhclient", '--no-pid', '-d', self.port.name]
+        cmd = ["dhclient", '-sf', self.NO_RESOLV_CONF_DHCLIENT_SCRIPT_PATH,
+               '--no-pid', '-d', self.port.name]
         self.dhclient_async = async_process.AsyncProcess(
             cmd, run_as_root=True, namespace=self.namespace)
         self.dhclient_async.start()

neutron/tests/fullstack/resources/process.py

@@ -301,8 +301,9 @@ def _setUp(self):
                 test_name=self.test_name,
                 process_name=self.NEUTRON_DHCP_AGENT,
                 exec_name=spawn.find_executable(
-                    'fullstack_dhcp_agent.py',
-                    path=os.path.join(base.ROOTDIR, 'common', 'agents')),
+                    'dhcp_agent.py',
+                    path=os.path.join(
+                        fullstack_base.ROOTDIR, 'cmd')),
                 config_filenames=config_filenames,
                 namespace=self.namespace
             )

neutron/tests/fullstack/test_dhcp_agent.py

@@ -46,7 +46,7 @@ def setUp(self):
             environment.EnvironmentDescription(
                 l2_pop=False,
                 arp_responder=False,
-                agent_down_time=10),
+                agent_down_time=self.agent_down_time),
             host_descriptions)
 
         super(BaseDhcpAgentTest, self).setUp(env)
@@ -88,6 +88,7 @@ def _agent_down():
 class TestDhcpAgentNoHA(BaseDhcpAgentTest):
 
     number_of_hosts = 1
+    agent_down_time = 60
 
     def test_dhcp_assignment(self):
         # First check if network was scheduled to one DHCP agent
@@ -102,6 +103,7 @@ def test_dhcp_assignment(self):
 class TestDhcpAgentHA(BaseDhcpAgentTest):
 
     number_of_hosts = 2
+    agent_down_time = 10
 
     def _wait_until_network_rescheduled(self, old_agent):
         def _agent_rescheduled():

tools/generate_dhclient_script_for_fullstack.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+MAKE_RESOLV_CONF_FUNCTION=make_resolv_conf
+
+USAGE="$0 <path to virtual environment to place executable>
+The script takes existing dhclient-script and makes $MAKE_RESOLV_CONF_FUNCTION function a noop function.
+"
+
+if [ $# -lt 1 ]; then
+    echo "Path to virtual environment directory is a required parameter."
+    echo $USAGE
+    exit 2
+fi
+
+VENV_DIR=$1
+DHCLIENT_SCRIPT_NAME=dhclient-script
+DHCLIENT_PATH=$(which $DHCLIENT_SCRIPT_NAME)
+FULLSTACK_DHCLIENT_SCRIPT=$VENV_DIR/bin/fullstack-dhclient-script
+
+if [ -n "$DHCLIENT_PATH" ]; then
+    # Return from make_resolv_conf function immediately. This will cause
+    # that /etc/resolv.conf will not be updated by fake fullstack machines.
+    sed "/^$MAKE_RESOLV_CONF_FUNCTION()/a\    return" $DHCLIENT_PATH > $FULLSTACK_DHCLIENT_SCRIPT
+    chmod +x $FULLSTACK_DHCLIENT_SCRIPT
+else
+    echo "$DHCLIENT_SCRIPT_NAME not found."
+    exit 1
+fi

tox.ini

@@ -76,6 +76,9 @@ setenv = {[testenv]setenv}
          OS_TEST_PATH=./neutron/tests/fullstack
 deps =
   {[testenv:functional]deps}
+commands =
+  {toxinidir}/tools/generate_dhclient_script_for_fullstack.sh {envdir}
+  {[testenv]commands}
 
 [testenv:releasenotes]
 commands = sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html