Add experimental --parallel option to invoke unittest (nautobot#5130

)

* Add --parallel option to invoke unittest, not fully validated yet.

* Ruff

* Change fragment

* Pylint

* Add tblib as a testing dependency as it's needed when running in parallel

* Fixes for parallel test execution

* Alternate safer approach to parallel graphql test

* Add comments

---------

Co-authored-by: Hanlin Miao <[email protected]>

changes/5130.housekeeping

@@ -0,0 +1 @@
+Added experimental `--parallel` option to `invoke unittest`.

nautobot/core/management/commands/generate_test_data.py

@@ -254,7 +254,7 @@ def handle(self, *args, **options):
 
         if options["cache_test_fixtures"] and os.path.exists(options["fixture_file"]):
             self.stdout.write(self.style.WARNING(f"Loading factory data from file {options['fixture_file']}"))
-            call_command("loaddata", options["fixture_file"])
+            call_command("loaddata", "--database", options["database"], options["fixture_file"])
         else:
             self._generate_factory_data(options["seed"], options["database"])
 

nautobot/core/tests/runner.py

@@ -1,6 +1,8 @@
 from django.conf import settings
 from django.core.management import call_command
+from django.db import connections
 from django.test.runner import DiscoverRunner
+from django.test.utils import get_unique_databases_and_mirrors, NullTimeKeeper
 import yaml
 
 from nautobot.core.celery import app, setup_nautobot_job_logging
@@ -55,31 +57,90 @@ def setup_test_environment(self, **kwargs):
             setup_nautobot_job_logging(None, None, app.conf)
 
     def setup_databases(self, **kwargs):
-        result = super().setup_databases(**kwargs)
-
-        if settings.TEST_USE_FACTORIES and result:
-            command = ["generate_test_data", "--flush", "--no-input"]
-            if settings.TEST_FACTORY_SEED is not None:
-                command += ["--seed", settings.TEST_FACTORY_SEED]
-            if self.cache_test_fixtures:
-                command += ["--cache-test-fixtures"]
-            for connection in result:
-                db_name = connection[0].alias
-                print(f'Pre-populating test database "{db_name}" with factory data...')
-                db_command = [*command, "--database", db_name]
-                call_command(*db_command)
-
-        return result
+        # Adapted from Django 3.2 django.test.utils.setup_databases
+        time_keeper = self.time_keeper
+        if time_keeper is None:
+            time_keeper = NullTimeKeeper()
+
+        test_databases, mirrored_aliases = get_unique_databases_and_mirrors(kwargs.get("aliases", None))
+
+        old_names = []
+
+        for db_name, aliases in test_databases.values():
+            first_alias = None
+            for alias in aliases:
+                connection = connections[alias]
+                old_names.append((connection, db_name, first_alias is None))
+
+                # Actually create the database for the first connection
+                if first_alias is None:
+                    first_alias = alias
+                    with time_keeper.timed(f"  Creating '{alias}'"):
+                        connection.creation.create_test_db(
+                            verbosity=self.verbosity,
+                            autoclobber=not self.interactive,
+                            keepdb=self.keepdb,
+                            serialize=connection.settings_dict["TEST"].get("SERIALIZE", True),
+                        )
+
+                    # Extra block added for Nautobot
+                    if settings.TEST_USE_FACTORIES:
+                        command = ["generate_test_data", "--flush", "--no-input"]
+                        if settings.TEST_FACTORY_SEED is not None:
+                            command += ["--seed", settings.TEST_FACTORY_SEED]
+                        if self.cache_test_fixtures:
+                            command += ["--cache-test-fixtures"]
+                        with time_keeper.timed(f'  Pre-populating test database "{alias}" with factory data...'):
+                            db_command = [*command, "--database", alias]
+                            call_command(*db_command)
+
+                    if self.parallel > 1:
+                        for index in range(self.parallel):
+                            with time_keeper.timed(f"  Cloning '{alias}'"):
+                                connection.creation.clone_test_db(
+                                    suffix=str(index + 1),
+                                    verbosity=self.verbosity,
+                                    keepdb=self.keepdb
+                                    # Extra check added for Nautobot:
+                                    and not settings.TEST_USE_FACTORIES,
+                                )
+
+                # Configure all other connections as mirrors of the first one
+                else:
+                    connection.creation.set_as_test_mirror(connections[first_alias].settings_dict)
+
+        # Configure the test mirrors
+        for alias, mirror_alias in mirrored_aliases.items():
+            connections[alias].creation.set_as_test_mirror(connections[mirror_alias].settings_dict)
+
+        if self.debug_sql:
+            for alias in connections:
+                connections[alias].force_debug_cursor = True
+
+        return old_names
 
     def teardown_databases(self, old_config, **kwargs):
-        if settings.TEST_USE_FACTORIES and old_config:
-            for connection in old_config:
-                db_name = connection[0].alias
-                print(f'Emptying test database "{db_name}"...')
-                call_command("flush", "--no-input", "--database", db_name)
-                print(f"Database {db_name} emptied!")
-
-        super().teardown_databases(old_config, **kwargs)
+        # Adapted from Django 3.2 django.test.utils.teardown_databases
+        for connection, old_name, destroy in old_config:
+            if destroy:
+                if self.parallel > 1:
+                    for index in range(self.parallel):
+                        connection.creation.destroy_test_db(
+                            suffix=str(index + 1),
+                            verbosity=self.verbosity,
+                            keepdb=self.keepdb
+                            # Extra check added for Nautobot
+                            and not settings.TEST_USE_FACTORIES,
+                        )
+
+                # Extra block added for Nautobot
+                if settings.TEST_USE_FACTORIES:
+                    db_name = connection.alias
+                    print(f'Emptying test database "{db_name}"...')
+                    call_command("flush", "--no-input", "--database", db_name)
+                    print(f"Database {db_name} emptied!")
+
+                connection.creation.destroy_test_db(old_name, self.verbosity, self.keepdb)
 
 
 # Use django_slowtests only when GENERATE_PERFORMANCE_REPORT flag is set to true

nautobot/core/tests/test_graphql.py

@@ -70,13 +70,13 @@
     Webhook,
 )
 from nautobot.extras.registry import registry
-from nautobot.ipam.factory import VLANGroupFactory
 from nautobot.ipam.models import (
     IPAddress,
     IPAddressToInterface,
     Namespace,
     Prefix,
     VLAN,
+    VLANGroup,
     VRF,
     VRFDeviceAssignment,
     VRFPrefixAssignment,
@@ -90,7 +90,15 @@
 User = get_user_model()
 
 
-class GraphQLTestCase(TestCase):
+class GraphQLTestCaseBase(TestCase):
+    @classmethod
+    def setUpTestData(cls):
+        # TODO: the below *shouldn't* be needed, but without it, when run with --parallel test flag,
+        # these tests consistently fail with schema construction errors
+        cls.SCHEMA = graphene_settings.SCHEMA  # not a no-op; this causes the schema to be built
+
+
+class GraphQLTestCase(GraphQLTestCaseBase):
     def setUp(self):
         self.user = create_test_user("graphql_testuser")
         GraphQLQuery.objects.create(name="GQL 1", query="{ query: locations {name} }")
@@ -157,13 +165,13 @@ def test_graphql_types_registry(self):
     @override_settings(EXEMPT_VIEW_PERMISSIONS=["*"])
     def test_graphql_url_field(self):
         """Test the url field for all graphql types."""
-        schema = graphene_settings.SCHEMA.introspect()
+        schema = self.SCHEMA.introspect()
         graphql_fields = schema["__schema"]["types"][0]["fields"]
         for graphql_field in graphql_fields:
             if graphql_field["type"]["kind"] == "LIST" or graphql_field["name"] == "content_type":
                 continue
             with self.subTest(f"Testing graphql url field for {graphql_field['name']}"):
-                graphene_object_type_definition = graphene_settings.SCHEMA.get_type(graphql_field["type"]["name"])
+                graphene_object_type_definition = self.SCHEMA.get_type(graphql_field["type"]["name"])
 
                 # simple check for url field in type definition
                 self.assertIn(
@@ -189,14 +197,14 @@ def test_graphql_url_field(self):
                     )
 
 
-class GraphQLUtilsTestCase(TestCase):
+class GraphQLUtilsTestCase(GraphQLTestCaseBase):
     def test_str_to_var_name(self):
         self.assertEqual(str_to_var_name("IP Addresses"), "ip_addresses")
         self.assertEqual(str_to_var_name("My New VAR"), "my_new_var")
         self.assertEqual(str_to_var_name("My-VAR"), "my_var")
 
 
-class GraphQLGenerateSchemaTypeTestCase(TestCase):
+class GraphQLGenerateSchemaTypeTestCase(GraphQLTestCaseBase):
     def test_model_w_filterset(self):
         schema = generate_schema_type(app_name="dcim", model=Device)
         self.assertEqual(schema.__bases__[0], OptimizedNautobotObjectType)
@@ -210,7 +218,7 @@ def test_model_wo_filterset(self):
         self.assertIsNone(schema._meta.filterset_class)
 
 
-class GraphQLExtendSchemaType(TestCase):
+class GraphQLExtendSchemaType(GraphQLTestCaseBase):
     def setUp(self):
         self.datas = (
             {"field_name": "my_text", "field_type": CustomFieldTypeChoices.TYPE_TEXT},
@@ -301,7 +309,7 @@ def test_extend_schema_null_field_choices(self):
         self.assertIsInstance(getattr(schema, "resolve_mode"), types.FunctionType)
 
 
-class GraphQLExtendSchemaRelationship(TestCase):
+class GraphQLExtendSchemaRelationship(GraphQLTestCaseBase):
     def setUp(self):
         location_ct = ContentType.objects.get_for_model(Location)
         rack_ct = ContentType.objects.get_for_model(Rack)
@@ -427,7 +435,7 @@ def test_extend_relationship_w_prefix(self):
             self.assertNotIn(field_name, schema._meta.fields.keys())
 
 
-class GraphQLSearchParameters(TestCase):
+class GraphQLSearchParameters(GraphQLTestCaseBase):
     def setUp(self):
         self.schema = generate_schema_type(app_name="dcim", model=Location)
 
@@ -444,12 +452,14 @@ def test_search_parameters(self):
                 self.assertNotIn(field, params.keys())
 
 
-class GraphQLAPIPermissionTest(TestCase):
+class GraphQLAPIPermissionTest(GraphQLTestCaseBase):
     client_class = NautobotTestClient
 
     @classmethod
     def setUpTestData(cls):
         """Initialize the Database with some datas and multiple users associated with different permissions."""
+        super().setUpTestData()
+
         cls.groups = (
             Group.objects.create(name="Group 1"),
             Group.objects.create(name="Group 2"),
@@ -683,7 +693,7 @@ def test_graphql_query_format(self):
         self.assertEqual(location_names, location_list)
 
 
-class GraphQLQueryTest(TestCase):
+class GraphQLQueryTest(GraphQLTestCaseBase):
     """Execute various GraphQL queries and verify their correct responses."""
 
     @classmethod
@@ -726,8 +736,8 @@ def setUpTestData(cls):
 
         vlan_statuses = Status.objects.get_for_model(VLAN)
         vlan_groups = (
-            VLANGroupFactory.create(location=cls.location1),
-            VLANGroupFactory.create(location=cls.location2),
+            VLANGroup.objects.create(name="VLANGroup 1", location=cls.location1),
+            VLANGroup.objects.create(name="VLANGroup 2", location=cls.location2),
         )
         cls.vlan1 = VLAN.objects.create(
             name="VLAN 1", vid=100, location=cls.location1, status=vlan_statuses[0], vlan_group=vlan_groups[0]
@@ -1076,10 +1086,9 @@ def setUpTestData(cls):
         cls.rm2ms_assoc_3.validated_save()
 
         cls.backend = get_default_backend()
-        cls.schema = graphene_settings.SCHEMA
 
     def execute_query(self, query, variables=None):
-        document = self.backend.document_from_string(self.schema, query)
+        document = self.backend.document_from_string(self.SCHEMA, query)
         if variables:
             return document.execute(context_value=self.request, variable_values=variables)
         else:

nautobot/ipam/factory.py

@@ -130,9 +130,6 @@ class Meta:
     class Params:
         unique_name = UniqueFaker("word", part_of_speech="noun")
 
-    # TODO: name is not globally unique, but (location, name) tuple must be.
-    # The likelihood of collision with random names is pretty low, but non-zero.
-    # We might want to consider *intentionally* using non-globally-unique names for testing purposes?
     name = factory.LazyAttribute(lambda o: o.unique_name.upper())
 
     has_description = NautobotBoolIterator()

pyproject.toml

@@ -211,6 +211,8 @@ requests = ">=2.28.0,<2.32.0"
 selenium = "~4.9.1"
 # Abstraction layer for working with Selenium
 splinter = "~0.18.1"
+# Serialization of tracebacks - used when running unittest with --parallel flag
+tblib = "~3.0.0"
 
 [tool.poetry.scripts]
 nautobot-server = "nautobot.core.cli:main"

tasks.py

@@ -699,6 +699,7 @@ def check_schema(context, api_version=None):
         "exclude_tag": "Do not run tests with the specified tag. Can be used multiple times.",
         "verbose": "Enable verbose test output.",
         "append": "Append coverage data to .coverage, otherwise it starts clean each time.",
+        "parallel": "Run tests in parallel.",
         "skip_docs_build": "Skip (re)build of documentation before running the test.",
         "performance_report": "Generate Performance Testing report in the terminal. Has to set GENERATE_PERFORMANCE_REPORT=True in settings.py",
         "performance_snapshot": "Generate a new performance testing report to report.yml. Has to set GENERATE_PERFORMANCE_REPORT=True in settings.py",
@@ -716,6 +717,7 @@ def unittest(
     tag=None,
     verbose=False,
     append=False,
+    parallel=False,
     skip_docs_build=False,
     performance_report=False,
     performance_snapshot=False,
@@ -739,6 +741,8 @@ def unittest(
         command += " --buffer"
     if verbose:
         command += " --verbosity 2"
+    if parallel:
+        command += " --parallel"
     if performance_report or (tag and "performance" in tag):
         command += " --slowreport"
     if performance_snapshot: