"OPEN-3086: Finishing touches on Data Strategy Tracker

search/management/commands/combine_messages.py

@@ -7,7 +7,6 @@
 class Command(BaseCommand):
     help = 'Combines .po files to the django.po file for use with builtin gettext support.'
 
-    requires_system_checks = False
     BASE_DIR = Path(__file__).resolve().parent.parent.parent.parent
     program = 'msgcat'
     french_locale_dir = path.join(BASE_DIR, 'locale', 'fr', 'LC_MESSAGES')

search/management/commands/search_run_tests.py

@@ -0,0 +1,76 @@
+import csv
+from django.core.management.base import BaseCommand
+from django.conf import settings
+import json
+import logging
+from os import path
+from search.models import Search, Field, Code, Setting, SearchLog
+from search.query import get_query_fields
+import traceback
+import SolrClient
+
+
+class Command(BaseCommand):
+    help = 'Do a search quality evaluatoin test run'
+
+    def __init__(self, *args, **kwargs):
+        super().__init__()
+        self.logger = logging.getLogger(__name__)
+        # class variables that hold the search models
+        self.search_target = None
+        self.solr_core = None
+        self.search_fields = None
+        self.field_codes = {}
+        self.solr = SolrClient(settings.SOLR_SERVER_URL)
+
+    def add_arguments(self, parser):
+        parser.add_argument('--search', type=str, help='The Search ID that is being evaluated', required=True)
+        parser.add_argument('--csv', type=str, help='CSV file with search tests', required=True)
+        parser.add_argument('--run_label', type=str, help='Test Run label. Example: "ATI with P4"', required=True)
+
+    def handle(self, *args, **options):
+        try:
+            self.search_target = Search.objects.get(search_id=options['search'])
+            self.solr_core = self.search_target.solr_core_name
+
+            self.search_fields = Field.objects.filter(search_id=self.search_target,
+                alt_format='ALL') | Field.objects.filter(
+                search_id=self.search_target, alt_format='')
+
+# @NOTE Not sure this is needed
+            for search_field in self.search_fields:
+
+                codes = Code.objects.filter(field_fid=search_field)
+                # Most csv_fields will not  have codes, so the queryset will be zero length
+                if len(codes) > 0:
+                    code_dict = {}
+                    for code in codes:
+                        code_dict[code.code_id.lower()] = code
+                self.field_codes[search_field.field_id] = code_dict
+
+        except Search.DoesNotExist as x:
+            self.logger.error('Search not found: "{0}"'.format(x))
+            exit(-1)
+        except Field.DoesNotExist as x1:
+            self.logger.error('Fields not found for search: "{0}"'.format(x1))
+
+        # Verify the CSV test file exists
+
+        if not path.exists(options['csv']):
+            self.logger.error(f'File not found: {options["csv"]}')
+            exit(-1)
+
+        # Process the records in the CSV file one at a time
+        with open(options['csv'], 'r', encoding='utf-8-sig', errors="xmlcharrefreplace") as csv_file:
+            csv_reader = csv.DictReader(csv_file, dialect='excel')
+            for row_num, csv_record in enumerate(csv_reader):
+                try:
+                    # Query Solr with the indicated text and evaluate its results
+                    solr_query = {'q': csv_record['text_en'], 'defType': 'edismax', 'sow': True, 'sort': 'score desc'}
+                    solr_query['q.op'] = self.search_target.solr_default_op
+                    solr_query['qf'] = get_query_fields('en', self.search_fields)
+
+                except Exception as x:
+                    self.logger.error('Unexpected Error "{0}" while processing row {1}'.format(x, row_num + 1))
+                    if settings.DEBUG:
+                        traceback.print_exception(type(x), x, x.__traceback__)

search/query.py

@@ -108,21 +108,21 @@ def get_search_terms(search_text: str):
     return solr_search_terms
 
 
-def get_query_fields(request: HttpRequest, fields: dict):
+def get_query_fields(query_lang: str, fields: dict):
     qf = ['id']
     for f in fields:
-        if fields[f].solr_field_lang in [request.LANGUAGE_CODE, 'bi']:
+        if fields[f].solr_field_lang in [query_lang, 'bi']:
             qf.append(f)
             if fields[f].solr_extra_fields:
-                if fields[f].solr_field_lang == request.LANGUAGE_CODE:
+                if fields[f].solr_field_lang == query_lang:
                     qf.extend(fields[f].solr_extra_fields.split(","))
                 else:
                     copy_fields = fields[f].solr_extra_fields.split(",")
                     for copy_field in copy_fields:
-                        if copy_field.endswith("_" + request.LANGUAGE_CODE):
+                        if copy_field.endswith("_" + query_lang):
                             qf.append(copy_field)
             if fields[f].solr_field_is_coded:
-                code_value_field = "{0}_{1}".format(f, request.LANGUAGE_CODE)
+                code_value_field = "{0}_{1}".format(f, query_lang)
                 if code_value_field not in qf:
                     qf.append(code_value_field)
     return qf
@@ -204,7 +204,7 @@ def create_solr_query(request: HttpRequest, search: Search, fields: dict, Codes:
     solr_query['q.op'] = search.solr_default_op
 
     # Create a Solr query field list based on the Fields Model. Expand the field list where needed
-    solr_query['qf'] = get_query_fields(request, fields)
+    solr_query['qf'] = get_query_fields(request.LANGUAGE_CODE, fields)
 
     if export:
         ef = ['id']
@@ -287,7 +287,7 @@ def create_solr_mlt_query(request: HttpRequest, search: Search, fields: dict, st
         'mlt.boost': True,
         'start': start_row,
         'rows': search.mlt_items,
-        'fl':  get_query_fields(request, fields),
+        'fl':  get_query_fields(request.LANGUAGE_CODE, fields),
         'mlt.fl': get_mlt_fields(request, fields),
         'mlt.mintf': 1,
         'mlt.minwl': 3,

search/templates/snippets/default_breadcrumb.html

@@ -25,9 +25,6 @@
             },{
                 "title": "{% translate 'Open Government' %}",
                 "href": "{% if not url_uses_path %}{% if LANGUAGE_CODE == 'fr' %}{{ site_host_fr }}{% else %}{{ site_host_en }}{% endif %}{% endif %}/{{ LANGUAGE_CODE }}",
-            },{
-                "title": "{% translate 'Proactive Disclosure' %}",
-                "href": "{% if not url_uses_path %}{% if LANGUAGE_CODE == 'fr' %}{{ site_host_fr }}{% else %}{{ site_host_en }}{% endif %}{% endif %}{% translate '/en/proactive-disclosure' %}",
             }]
         });
     </script>