Rip out the testsuite and replace it with a much better one

- No more hpilo_cli _test, but python -munittest discover
- Tests the xml generating/parsing without needing an iLO
- Can test parsing XML generated by many iLOs
- Can still test its actions on iLOs if you want it to
- Can test many ilos in one run
- Can actually be maintained...

.gitignore

@@ -8,3 +8,6 @@ firmware
 *.bin
 *.scexe
 .*.sw?
+*.request.tmp
+hpilo_test_debug_output
+.firmware_version_cache

hpilo_cli

@@ -63,7 +63,6 @@ def main():
     p.add_option('-v', '--version', action="callback", callback=hpilo_version)
 
     opts, args = p.parse_args()
-    opts.do_write_tests = False
 
     if opts.format == 'json':
         import json
@@ -74,10 +73,7 @@ def main():
     if len(args) < 2:
         p.error("Not enough arguments")
 
-    if args[1] == '_test_writes':
-        args[1] = '_test'
-        opts.do_write_tests = True
-    if args[1] not in ilo_methods and args[1] != '_test' and args[0] != 'download_rib_firmware':
+    if args[1] not in ilo_methods and args[0] != 'download_rib_firmware':
         p.error("Unknown method: %s" % args[1])
 
     config = ConfigParser.ConfigParser()
@@ -97,10 +93,6 @@ def main():
     for args in args_:
         method = args.pop(0)
 
-        if method == '_test':
-            calls.append(('_test', {'opts': opts, 'tests': args}))
-            continue
-
         # Arguments must be passed as param=value pairs that are valid arguments to the methods
         if sys.version_info[0] >= 3:
             func = getattr(hpilo.Ilo, method)#.__func__

tests/README

@@ -0,0 +1,63 @@
+Testsuite for python-hpilo
+--------------------------
+
+This testsuite consists of two parts:
+- Tests for the XML generator and parser
+- Tests that actually contact iLO's
+
+The former use a corpus of known data, they're safe to run and run pretty
+quickly. The latter will change data on iLO's (and do their best to restore
+the data to previous values), will reset iLO's and take a long time to run.
+
+Test contributions wanted!
+--------------------------
+For the corpus of test data, we need more people to submit data from their
+machines. This process is safe (no iLO data is changed at all), though can
+leak information about your systems if you are not careful. If you want to
+contribute, please do the following:
+
+$ cd tests/xml
+$ ./fetch_responses hostname-of-your-ilo-here
+
+This creates a new directory named after the hardware model, ilo model and ilo
+firmware of your machine. In it, you find raw and parsed responses of all
+python-hpilo queries. You should inspect this data and see if it's correct.
+You should also redact information you do not wish to share, but please don't
+go overboard.
+
+- Redacting iLO license keys makes perfect sense.
+- Redacting hostnames and IP addresses is acceptable, but please do not redact
+  anything else
+- When redacting, replace each letter of the redacted string with an x. Don't
+  add or remove characters
+- The .raw files may contain http chunked data, be very careful when editing
+  this and don't break the format. Preferably use a hex editor, or an editor
+  that does not mess with newlines (vim will do)
+
+You can either send me your datafiles via e-mail, or send a pull request on
+GitHub. If you do not know how to redact the data, or want me to do it for
+you, please send the data by e-mail and I will redact hostnames and iLO
+license keys.
+
+Running the tests
+-----------------
+Running the safe tests is as simple as running:
+
+$ python -munittest tests.test_requests
+$ python -munittest tests.test_responses
+
+You can run any of the tests this way, just replace test_requests with the
+name of the test. To run all tests, you can use:
+
+$ python -munittest discover
+
+However, if you run this without configuring the tests, you'll notice that
+nothing actually attempts to contact an iLO. You will need to tell it which
+iLOs to test. You do this in ~/.ilo.conf by adding test sections like the
+following:
+
+[test test-server]
+hostname = test-server.ilo.example.com
+
+Please don't run these tests on mission-critical servers. While they try very
+hard to restore the settings it changes, bugs and errors happen.

tests/test_boot.py

@@ -0,0 +1,30 @@
+#!/usr/bin/python
+
+from utils import *
+import random
+
+class BootTests(IloTestCase):
+    def test_persistent_boot(self, ilo):
+        old = ilo.get_persistent_boot()
+        new = old[:]
+        random.shuffle(new)
+        try:
+            ilo.set_persistent_boot(new)
+            self.assertEqual(new, ilo.get_persistent_boot())
+        finally:
+            ilo.set_persistent_boot(old)
+
+    def test_one_time_boot(self, ilo):
+        old = ilo.get_one_time_boot()
+        all = ilo.get_persistent_boot()
+        if old in all:
+            all.remove(old)
+        new = random.choice(all)
+        try:
+            ilo.set_one_time_boot(new)
+            self.assertEqual(new, ilo.get_one_time_boot())
+        finally:
+            ilo.set_one_time_boot(old)
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_delayed.py

@@ -0,0 +1,23 @@
+#!/usr/bin/python
+
+from utils import *
+
+class DelayedTests(IloTestCase):
+    def test_delayed_calls(self, ilo):
+        uid = {'ON': 'Yes', 'OFF': 'No'}[ilo.get_uid_status()]
+        non_delayed = [
+            ilo.get_all_users(),
+            ilo.get_global_settings(),
+        ]
+        try:
+            ilo.delayed = True
+            ilo.get_all_users()
+            ilo.uid_control(uid=uid)
+            ilo.get_global_settings()
+            delayed = ilo.call_delayed()
+        finally:
+            ilo.delayed = False
+        self.assertEquals(non_delayed, delayed)
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_global_settings.py

@@ -0,0 +1,23 @@
+#!/usr/bin/python
+
+from utils import *
+
+class GlobalSettingsTests(IloTestCase):
+    def test_mod_global_settings(self, ilo):
+        old = ilo.get_global_settings()
+        try:
+            ilo.mod_global_settings(
+                f8_login_required=True,
+                min_password=3
+            )
+            new = ilo.get_global_settings()
+            self.assertEqual(new['f8_login_required'], True)
+            self.assertEqual(new['min_password'], 3)
+        finally:
+            ilo.mod_global_settings(
+                f8_login_required=old['f8_login_required'],
+                min_password=old['min_password']
+            )
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_languages.py

@@ -0,0 +1,10 @@
+#!/usr/bin/python
+
+from utils import *
+
+class LanguageTests(IloTestCase):
+    def test_languages(self, ilo):
+        ilo.set_language(ilo.get_language()['lang_id'])
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_logs.py

@@ -0,0 +1,15 @@
+#!/usr/bin/python
+
+from utils import *
+import time
+
+class LogTests(IloTestCase):
+    def test_ilo_event_log(self, ilo):
+        ilo.clear_ilo_event_log()
+        time.sleep(2)
+        log = ilo.get_ilo_event_log()
+        self.assertTrue(type(log) != dict)
+        self.assertTrue(len(log) <= 3)
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_network_settings.py

@@ -0,0 +1,21 @@
+#!/usr/bin/python
+
+from utils import *
+
+class NetworkTests(IloTestCase):
+    def test_mod_network_settings(self, ilo):
+        old = ilo.get_network_settings()
+        try:
+            ilo.mod_network_settings(ter_dns_server='10.1.1.1')
+            self.reset_delay(ilo)
+            new = ilo.get_network_settings()
+            self.assertEquals(new['ter_dns_server'], '10.1.1.1')
+        finally:
+            ilo.mod_network_settings(ter_dns_server='' if old['ter_dns_server'] == '0.0.0.0' else old['ter_dns_server'])
+            self.reset_delay(ilo)
+
+    def test_ipv6_routes(self, ilo):
+        self.require_ilo(ilo, 'ilo3:1.50', 'ilo4:1.20')
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_requests.py

@@ -0,0 +1,60 @@
+#!/usr/bin/python
+
+from utils import *
+
+class RequestsTestMeta(type):
+    def __new__(cls, name, parents, attrs):
+        root = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'xml')
+        for machine in os.listdir(root):
+            mdir = os.path.join(root, machine)
+            if not os.path.isdir(mdir):
+                continue
+
+            files = os.listdir(mdir)
+            for argsfile in files:
+                if not argsfile.endswith('.args'):
+                    continue
+                method = argsfile[:argsfile.find('-')]
+                reqfile = argsfile.replace('.args', '.request')
+                reqtmpfile = argsfile.replace('.args', '.request.tmp')
+                argspath = os.path.join(mdir, argsfile)
+                reqpath = os.path.join(mdir, reqfile)
+                reqtmppath = os.path.join(mdir, reqtmpfile)
+                fname = 'test_%s_%s' % (machine, argsfile[:-5])
+                fname = re.sub('[^a-zA-Z_0-9]', '_', fname).lower()
+                attrs[fname] = eval("lambda self: self._test_request('%s', '%s', '%s', '%s', '%s')" % 
+                                    (machine, method, argspath, reqpath, reqtmppath))
+        return super(RequestsTestMeta, cls).__new__(cls, name, parents, attrs)
+
+class RequestsTest(unittest.TestCase):
+    __metaclass__ = RequestsTestMeta
+    maxDiff = None
+
+    def _test_request(self, machine, method, argsfile, reqfile, reqtmpfile):
+        ilo = hpilo.Ilo('nonexistent-machine','Administrator','TestPassword')
+        if 'ilo3' in machine.lower() or 'ilo4' in machine.lower():
+            ilo.protocol = hpilo.ILO_HTTP
+        else:
+            ilo.protocol = hpilo.ILO_RAW
+        if os.path.exists(reqtmpfile):
+            os.unlink(reqtmpfile)
+        ilo.save_request = reqtmpfile
+        with open(argsfile) as fd:
+            args, kwargs = eval(fd.read()), {}
+        if isinstance(args, dict):
+            args, kwargs = [], args
+
+        if not os.path.exists(reqfile):
+            self.assertRaises(ValueError, getattr(ilo, method), *args, **kwargs)
+            return
+
+        getattr(ilo, method)(*args, **kwargs)
+        with open(reqfile) as fd:
+            old = fd.read()
+        with open(reqtmpfile) as fd:
+            new = fd.read()
+        self.assertMultiLineEqual(old, new)
+        os.unlink(reqtmpfile)
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_responses.py

@@ -0,0 +1,54 @@
+#!/usr/bin/python
+
+from utils import *
+import pprint
+
+class ResponsesTestMeta(type):
+    def __new__(cls, name, parents, attrs):
+        root = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'xml')
+        for machine in os.listdir(root):
+            mdir = os.path.join(root, machine)
+            if not os.path.isdir(mdir):
+                continue
+
+            files = os.listdir(mdir)
+            for rawfile in files:
+                if not rawfile.endswith('.raw'):
+                    continue
+                method = rawfile.replace('.raw', '')
+                parsedfile = rawfile.replace('.raw', '.parsed')
+                rawpath = os.path.join(mdir, rawfile)
+                parsedpath = os.path.join(mdir, parsedfile)
+                fname = 'test_%s_%s' % (machine, method)
+                fname = re.sub('[^a-zA-Z_0-9]', '_', fname).lower()
+                attrs[fname] = eval("lambda self: self._test_response('%s', '%s', '%s', '%s')" % 
+                                    (machine, method, rawpath, parsedpath))
+        return super(ResponsesTestMeta, cls).__new__(cls, name, parents, attrs)
+
+class ResponsesTest(unittest.TestCase):
+    __metaclass__ = ResponsesTestMeta
+    maxDiff = None
+    method_args = {
+        'get_user': ['Administrator'],
+        'get_federation_group': ['slartibartfast'],
+    }
+    def _test_response(self, machine, method, rawfile, parsedfile):
+        ilo = hpilo.Ilo('nonexistent-machine','Administrator','TestPassword')
+        if 'ilo3' in machine.lower() or 'ilo4' in machine.lower():
+            ilo.protocol = hpilo.ILO_HTTP
+        else:
+            ilo.protocol = hpilo.ILO_RAW
+        ilo.read_response = rawfile
+
+        args = self.method_args.get(method, [])
+        if not os.path.exists(parsedfile):
+            self.assertRaises(hpilo.IloError, getattr(ilo, method), *args)
+            return
+        response = getattr(ilo, method)(*args)
+        new = pprint.pformat(response) + '\n'
+        with open(parsedfile) as fd:
+            old = fd.read()
+        self.assertMultiLineEqual(old, new)
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_snmp.py

@@ -0,0 +1,20 @@
+#!/usr/bin/python
+
+from utils import *
+
+class SnmpTests(IloTestCase):
+    def test_mod_snmp_im_settings(self, ilo):
+        old = ilo.get_snmp_im_settings()
+        try:
+            ilo.mod_snmp_im_settings(snmp_address_3='10.42.42.42', rib_traps=True)
+            new = ilo.get_snmp_im_settings()
+            self.assertEqual(new['snmp_address_3'], '10.42.42.42')
+            self.assertEqual(new['rib_traps'], True)
+        finally:
+            ilo.mod_snmp_im_settings(snmp_address_3=old['snmp_address_3'], rib_traps=old['rib_traps'])
+
+    def test_snmp_user_profiles(self, ilo):
+        self.require_ilo(ilo, 'ilo4')
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_uid.py

@@ -0,0 +1,17 @@
+#!/usr/bin/python
+
+from utils import *
+
+class UidTests(IloTestCase):
+    def test_uid(self, ilo):
+        old = ilo.get_uid_status()
+        new = {'ON': 'No', 'OFF': 'Yes'}[old]
+        new2 = {'ON': 'OFF', 'OFF': 'ON'}[old]
+        try:
+            ilo.uid_control(uid=new)
+            self.assertEqual(new2, ilo.get_uid_status())
+        finally:
+            ilo.uid_control(uid=old)
+
+if __name__ == '__main__':
+    unittest.main()