Merge pull request #9225 from mshriver/polarion-custscenario

[1LP][RFR] Add customer scenario marker/filter, move tier and requirement markers

cfme/markers/__init__.py

@@ -3,13 +3,14 @@
     "cfme.markers.crud",
     "cfme.markers.fixtureconf",
     "cfme.markers.meta",
-    "cfme.markers.stream_excluder",
+    "cfme.markers.polarion",
     "cfme.markers.requires",
     "cfme.markers.rhv",
     "cfme.markers.sauce",
     "cfme.markers.skipper",
+    "cfme.markers.smoke",
+    "cfme.markers.stream_excluder",
+    "cfme.markers.marker_filters",
     "cfme.markers.uses",
     "cfme.markers.uncollect",
-    "cfme.markers.smoke",
-    "cfme.markers.polarion",
 ]

cfme/markers/manual.py

@@ -1,4 +1,4 @@
-"""manual: Marker for marking tests asmanual tests."""
+"""manual: Marker for marking tests as manual tests."""
 import pytest
 
 from cfme.fixtures.pytest_store import store

cfme/markers/marker_filters.py

@@ -0,0 +1,78 @@
+"""Marker definitions for tier and requirement, and meta filter plugin
+
+"""
+import re
+
+from cfme.fixtures.pytest_store import store
+
+
+TEST_PARAM_FILTER = re.compile(r"\[.*\]")
+
+
+# Markers
+def pytest_configure(config):
+    if config.getoption('--help'):
+        return
+    markers_to_add = [
+        'tier: mark a test case with a tier',
+        'requirement: mark a test case with a requirement',
+        'customer_scenario: mark a test case as a customer story',
+    ]
+    for marker in markers_to_add:
+        config.addinivalue_line('markers', marker)
+
+
+# Filtering options
+def pytest_addoption(parser):
+    group = parser.getgroup('cfme')
+    group.addoption('--tier',
+                    type=int,
+                    action='append',
+                    help='only run tests of the given tier levels')
+    group.addoption('--requirement',
+                    type=str,
+                    action='append',
+                    help='only run tests with given requirement markers')
+    group.addoption('--customer-scenario',
+                    action='store_true',
+                    default=False,
+                    help='only run tests marked with customer_scenario')
+
+
+def pytest_collection_modifyitems(session, config, items):
+    """Provide filtering of test case collection based on the CLI options"""
+    tiers = config.getoption('tier')
+    requirements = config.getoption('requirement')
+    customer = config.getoption('customer_scenario')
+    if not tiers and not requirements and not customer:
+        return
+    # TODO(rpfannsc) trim after pytest #1373 is done
+    keep, discard_tier, discard_requirement, discard_customer = [], [], [], []
+
+    for item in items:
+        # for each filter, check if its active and that the item has the marker
+        # Then check if the marker content matches the passed filter
+        # Discard items without the matching value
+        if (tiers and
+                not getattr(item.get_marker('tier'), 'args', [False])[0] in tiers):
+            discard_tier.append(item)
+            continue
+        if (requirements and
+                not getattr(item.get_marker('requirement'), 'args', [False])[0] in requirements):
+            discard_requirement.append(item)
+            continue
+        if customer and item.get_marker('customer_scenario') is None:
+            discard_customer.append(item)
+            continue
+        keep.append(item)
+
+    items[:] = keep
+    # TODO(rpfannsc) add a reason after pytest #1372 is fixed
+    discarded = discard_tier + discard_requirement + discard_customer
+    config.hook.pytest_deselected(items=discarded)
+    if tiers:
+        store.uncollection_stats['tier mark'] = len(discard_tier)
+    if requirements:
+        store.uncollection_stats['requirement mark'] = len(discard_requirement)
+    if customer:
+        store.uncollection_stats['customer_scenario mark'] = len(discard_customer)

cfme/test_framework/pytest_plugin.py

@@ -70,7 +70,6 @@ def pytest_collection_finish(session):
     'cfme.fixtures.maximized',
     'cfme.fixtures.merkyl',
     'cfme.fixtures.nelson',
-    'cfme.fixtures.node_annotate',
     'cfme.fixtures.page_screenshots',
     'cfme.fixtures.perf',
     'cfme.fixtures.provider',

cfme/test_requirements.py

@@ -108,18 +108,6 @@ def test_quota_alert():
     assignee_id='nachandr',
 )
 
-custom_button = pytest.mark.requirement(
-    "Custom Buttons",
-    description='Custom Buttons and Custom Groups',
-    assignee_id='ndhandre',
-)
-
-customer_stories = pytest.mark.requirement(
-    "Customer Stories",
-    description='Integration of multiple FAs, Customer Day2 Scenarios',
-    assignee_id='ndhandre',
-)
-
 chargeback = pytest.mark.requirement(
     "Chargeback",
     description='Chargeback rates, calculations, and reports',
@@ -150,6 +138,17 @@ def test_quota_alert():
     assignee_id='juwatts',
 )
 
+custom_button = pytest.mark.requirement(
+    "Custom Buttons",
+    description='Custom Buttons and Custom Groups',
+    assignee_id='ndhandre',
+)
+
+customer_stories = pytest.mark.requirement(
+    "Customer Stories",
+    description='Integration of multiple FAs, Day 2 Operations type tests',
+    assignee_id='ndhandre',)
+
 dashboard = pytest.mark.requirement(
     "Dashboard",
     description='MIQ/CFME Dashboards creation, usability',

cfme/tests/automate/custom_button/test_service_vm_custom_button.py

@@ -134,6 +134,7 @@ def test_custom_button_display_service_vm(request, appliance, service_vm, button
             assert custom_button_group.has_item(button.text)
 
 
+@pytest.mark.customer_scenario
 @test_requirements.customer_stories
 @pytest.mark.tier(1)
 @pytest.mark.meta(automates=[1687061])

cfme/tests/v2v/test_csv_import.py

@@ -19,6 +19,7 @@
 
 pytestmark = [
     test_requirements.v2v,
+    pytest.mark.customer_scenario,
     pytest.mark.provider(
         classes=[RHEVMProvider, OpenStackProvider],
         selector=ONE_PER_VERSION,
@@ -32,7 +33,7 @@
         required_flags=["v2v"],
         scope="module",
     ),
-    pytest.mark.usefixtures("v2v_provider_setup")
+    pytest.mark.usefixtures("v2v_provider_setup"),
 ]
 
 
@@ -137,7 +138,6 @@ def test_non_csv(appliance, infra_map):
         caseposneg: negative
         startsin: 5.10
         casecomponent: V2V
-        customerscenario: true
         initialEstimate: 1/8h
     """
     error_text = "Invalid file extension. Only .csv files are accepted."
@@ -152,7 +152,6 @@ def test_blank_csv(appliance, infra_map):
         caseposneg: negative
         startsin: 5.10
         casecomponent: V2V
-        customerscenario: true
         initialEstimate: 1/8h
     """
     error_msg = "Error: Possibly a blank .CSV file"
@@ -167,7 +166,6 @@ def test_column_headers(appliance, infra_map):
         caseposneg: positive
         startsin: 5.10
         casecomponent: V2V
-        customerscenario: true
         initialEstimate: 1/8h
     """
     content = fauxfactory.gen_alpha(10)
@@ -183,7 +181,6 @@ def test_inconsistent_columns(appliance, infra_map):
         caseposneg: negative
         startsin: 5.10
         casecomponent: V2V
-        customerscenario: true
         initialEstimate: 1/8h
     """
     content = "Name\n{}, {}".format(fauxfactory.gen_alpha(10), fauxfactory.gen_alpha(10))
@@ -199,7 +196,6 @@ def test_csv_empty_vm(appliance, infra_map):
         caseposneg: positive
         startsin: 5.10
         casecomponent: V2V
-        customerscenario: true
         initialEstimate: 1/8h
     """
     content = "Name\n\n"
@@ -216,7 +212,6 @@ def test_csv_invalid_vm(appliance, infra_map):
         caseposneg: negative
         startsin: 5.10
         casecomponent: V2V
-        customerscenario: true
         initialEstimate: 1/8h
     """
     content = "Name\n{}".format(fauxfactory.gen_alpha(10))
@@ -233,7 +228,6 @@ def test_csv_valid_vm(appliance, infra_map, valid_vm):
         caseposneg: positive
         startsin: 5.10
         casecomponent: V2V
-        customerscenario: true
         initialEstimate: 1/8h
     """
     content = "Name\n{}".format(valid_vm)
@@ -250,7 +244,6 @@ def test_csv_duplicate_vm(appliance, infra_map, valid_vm):
         caseposneg: positive
         startsin: 5.10
         casecomponent: V2V
-        customerscenario: true
         initialEstimate: 1/8h
     """
     content = "Name\n{}\n{}".format(valid_vm, valid_vm)
@@ -267,7 +260,6 @@ def test_csv_archived_vm(appliance, infra_map, archived_vm):
         caseposneg: positive
         startsin: 5.10
         casecomponent: V2V
-        customerscenario: true
         initialEstimate: 1/8h
     """
     content = "Name\n{}".format(archived_vm)
@@ -285,7 +277,6 @@ def test_csv_security_group_flavor(appliance, infra_map, valid_vm, provider):
         caseposneg: positive
         startsin: 5.10
         casecomponent: V2V
-        customerscenario: true
         initialEstimate: 1/4h
     """
     try: