avoid clobbering all indexes by running tests

- rename the `es` pytest marker to `es_metrics` (for clarity)
- update the effect of that marker:
    - patch a prefix to each metric class's index and template names
    - instead of deleting ALL indexes and index templates, delete only
      those with the patched prefix
- update `api_tests.metrics.test_raw_metrics` to clean up after itself
  and stop depending on the clobbering

api_tests/institutions/views/test_institution_department_list.py

@@ -10,7 +10,7 @@
 from osf.metrics import UserInstitutionProjectCounts
 
 
-@pytest.mark.es
+@pytest.mark.es_metrics
 @pytest.mark.django_db
 class TestInstitutionDepartmentList:
 

api_tests/institutions/views/test_institution_summary_metrics.py

@@ -9,7 +9,7 @@
 from osf.metrics import InstitutionProjectCounts
 
 
-@pytest.mark.es
+@pytest.mark.es_metrics
 @pytest.mark.django_db
 class TestInstitutionSummaryMetrics:
 

api_tests/institutions/views/test_institution_user_metric_list.py

@@ -17,7 +17,7 @@
 from osf.metrics import UserInstitutionProjectCounts
 from osf.metrics.reports import InstitutionalUserReport
 
-@pytest.mark.es
+@pytest.mark.es_metrics
 @pytest.mark.django_db
 class TestOldInstitutionUserMetricList:
 
@@ -266,6 +266,7 @@ def test_filter_and_sort(self, app, url, user, user2, user3, admin, user4, popul
         assert data[2]['attributes']['department'] == 'Psychology dept'
 
 
+@pytest.mark.es_metrics
 @pytest.mark.django_db
 class TestNewInstitutionUserMetricList:
     @pytest.fixture(autouse=True)
@@ -336,21 +337,18 @@ def test_rando(self, app, url, rando):
         _resp = app.get(url, auth=rando.auth, expect_errors=True)
         assert _resp.status_code == 403
 
-    @pytest.mark.es
     def test_get_empty(self, app, url, institutional_admin):
         _resp = app.get(url, auth=institutional_admin.auth)
         assert _resp.status_code == 200
         assert _resp.json['data'] == []
 
-    @pytest.mark.es
     def test_get_reports(self, app, url, institutional_admin, institution, reports, unshown_reports):
         _resp = app.get(url, auth=institutional_admin.auth)
         assert _resp.status_code == 200
         assert len(_resp.json['data']) == len(reports)
         _expected_user_ids = {_report.user_id for _report in reports}
         assert set(_user_ids(_resp)) == _expected_user_ids
 
-    @pytest.mark.es
     def test_filter_reports(self, app, url, institutional_admin, institution, reports, unshown_reports):
         for _query, _expected_user_ids in (
             ({'filter[department]': 'nunavum'}, set()),
@@ -386,7 +384,6 @@ def test_filter_reports(self, app, url, institutional_admin, institution, report
             assert _resp.status_code == 200
             assert set(_user_ids(_resp)) == _expected_user_ids
 
-    @pytest.mark.es
     def test_sort_reports(self, app, url, institutional_admin, institution, reports, unshown_reports):
         for _query, _expected_user_id_list in (
             ({'sort': 'storage_byte_count'}, ['u_sparse', 'u_orc', 'u_blargl', 'u_orcomma']),
@@ -396,7 +393,6 @@ def test_sort_reports(self, app, url, institutional_admin, institution, reports,
             assert _resp.status_code == 200
             assert list(_user_ids(_resp)) == _expected_user_id_list
 
-    @pytest.mark.es
     def test_paginate_reports(self, app, url, institutional_admin, institution, reports, unshown_reports):
         for _query, _expected_user_id_list in (
             ({'sort': 'storage_byte_count', 'page[size]': 2}, ['u_sparse', 'u_orc']),

api_tests/metrics/test_composite_query.py

@@ -29,7 +29,7 @@ def base_url():
     return f'/{API_BASE}metrics/preprints/'
 
 
-@pytest.mark.es
+@pytest.mark.es_metrics
 @pytest.mark.django_db
 class TestElasticSearch:
 

api_tests/metrics/test_preprint_metrics.py

@@ -116,7 +116,7 @@ def test_custom_metric_malformed_query(self, mock_execute, app, user, base_url):
         assert res.status_code == 400
         assert res.json['errors'][0]['detail'] == 'Malformed elasticsearch query.'
 
-    @pytest.mark.es
+    @pytest.mark.es_metrics
     def test_agg_query(self, app, user, base_url):
 
         post_url = f'{base_url}downloads/'

api_tests/metrics/test_raw_metrics.py

@@ -14,14 +14,20 @@
 pytestmark = pytest.mark.django_db
 
 
-@pytest.mark.es
+@pytest.mark.es_metrics
 class TestRawMetrics:
 
     @pytest.fixture(autouse=True)
     def enable_elasticsearch_metrics(self):
         with override_switch(features.ENABLE_RAW_METRICS, active=True):
             yield
 
+    @pytest.fixture(autouse=True)
+    def teardown_customer_index(self, es6_client):
+        es6_client.indices.delete(index='customer', ignore_unavailable=True)
+        yield
+        es6_client.indices.delete(index='customer', ignore_unavailable=True)
+
     @pytest.fixture
     def user(self):
         user = AuthUserFactory()
@@ -132,7 +138,7 @@ def test_post_and_get(self, app, user, base_url):
 
         time.sleep(3)
 
-        get_url = f'{base_url}_search?q=*'
+        get_url = f'{base_url}customer/_search?q=*'
         res = app.get(get_url, auth=user.auth)
 
         assert res.json['hits']['total'] == 1

api_tests/metrics/test_registries_moderation_metrics.py

@@ -22,7 +22,7 @@ def enable_elasticsearch_metrics(self):
         with override_switch(features.ELASTICSEARCH_METRICS, active=True):
             yield
 
-    @pytest.mark.es
+    @pytest.mark.es_metrics
     def test_record_transitions(self, registration):
         registration._write_registration_action(
             RegistrationModerationStates.INITIAL,
@@ -70,7 +70,7 @@ def other_user(self):
     def base_url(self):
         return '/_/metrics/registries_moderation/transitions/'
 
-    @pytest.mark.es
+    @pytest.mark.es_metrics
     def test_registries_moderation_view(self, app, user, base_url, registration):
         registration._write_registration_action(
             RegistrationModerationStates.INITIAL,

conftest.py

@@ -1,11 +1,14 @@
+import contextlib
 from unittest import mock
 import logging
 import os
 import re
 
 from django.core.management import call_command
 from django.db import transaction
+from elasticsearch import exceptions as es_exceptions
 from elasticsearch_dsl.connections import connections
+from elasticsearch_metrics.registry import registry as es_metrics_registry
 from faker import Factory
 import pytest
 import responses
@@ -133,22 +136,44 @@ def es6_client(setup_connections):
 
 
 @pytest.fixture(scope='function', autouse=True)
-def _es_marker(request):
+def _es_metrics_marker(request):
     """Clear out all indices and index templates before and after
-    tests marked with ``es``.
+    tests marked with `es_metrics`.
     """
-    marker = request.node.get_closest_marker('es')
+    marker = request.node.get_closest_marker('es_metrics')
     if marker:
         es6_client = request.getfixturevalue('es6_client')
-
-        def teardown_es():
-            es6_client.indices.delete(index='*')
-            es6_client.indices.delete_template('*')
-
-        teardown_es()
-        call_command('sync_metrics')
-        yield
-        teardown_es()
+        _temp_prefix = 'temp_metrics_'
+        _temp_wildcard = f'{_temp_prefix}*'
+
+        def _teardown_es_temps():
+            es6_client.indices.delete(index=_temp_wildcard)
+            try:
+                es6_client.indices.delete_template(_temp_wildcard)
+            except es_exceptions.NotFoundError:
+                pass
+
+        @contextlib.contextmanager
+        def _mock_metric_names():
+            with contextlib.ExitStack() as _exit:
+                for _metric_class in es_metrics_registry.get_metrics():
+                    _exit.enter_context(mock.patch.object(
+                        _metric_class,
+                        '_template_name',  # also used to construct index names
+                        f'{_temp_prefix}{_metric_class._template_name}',
+                    ))
+                    _exit.enter_context(mock.patch.object(
+                        _metric_class,
+                        '_template',  # a wildcard string for indexes and templates
+                        f'{_temp_prefix}{_metric_class._template}',
+                    ))
+                yield
+
+        _teardown_es_temps()
+        with _mock_metric_names():
+            call_command('sync_metrics')
+            yield
+        _teardown_es_temps()
     else:
         yield
 

osf_tests/management_commands/test_reindex_es6.py

@@ -45,7 +45,7 @@ def admin(self):
     def url(self):
         return f'{settings.API_DOMAIN}_/metrics/preprints/downloads/'
 
-    @pytest.mark.es
+    @pytest.mark.es_metrics
     @pytest.mark.skipif(django_settings.TRAVIS_ENV, reason='Non-deterministic fails on travis')
     def test_reindexing(self, app, url, preprint, user, admin, es6_client):
         preprint_download = PreprintDownload.record_for_preprint(

osf_tests/test_management_commands.py

@@ -265,7 +265,7 @@ def test_data_storage_usage_command(self):
                 assert (key, expected_summary_data[key]) == (key, actual_summary_data[key])
 
 
-@pytest.mark.es
+@pytest.mark.es_metrics
 @pytest.mark.django_db
 class TestInstitutionMetricsUpdate: