OPEN-3260: Add search term logging

search/management/commands/import_data_csv.py

@@ -9,7 +9,7 @@
 import importlib
 import pkgutil
 import re
-from search.models import Search, Field, Code
+from search.models import Search, Field, Code, Event
 import search.plugins
 from SolrClient import SolrClient
 from SolrClient.exceptions import ConnectionError
@@ -435,6 +435,9 @@ def handle(self, *args, **options):
                         if len(solr_items) > 0:
                             solr.index(self.solr_core, solr_items)
                             commit_count += len(solr_items)
+                            Event.objects.create(search_id=options['search'], component_id="import_data_csv",
+                                                 title=f"Total rows processed: {total}, committed to Solr: {commit_count}",
+                                                 category='success', message=f"For Search {options['search']}, {total} records loaded from {options['csv']}")
                     except ConnectionError as cex:
                         self.logger.warning(
                             f"Unexpected error encountered while indexing starting on row {total}. Row data has {len(solr_items)} items")
@@ -456,6 +459,8 @@ def handle(self, *args, **options):
                         self.logger.info(
                             "\nTotal rows processed: {0}, committed to Solr: {1}".format(total, commit_count))
 
+
+
         except Exception as x:
             self.logger.error('Unexpected Error "{0}"'.format(x.args))
         finally:

search/tasks.py

@@ -81,7 +81,7 @@ def save_search_logs_to_file():
     # Use a CSV writer with forced quoting for the body of the file
     with open(settings.SEARCH_LOGGING_ARCHIVE_FILE, 'a', newline='', encoding='utf8') as csv_file:
         log_writer = csv.writer(csv_file, dialect='excel', quoting=csv.QUOTE_ALL)
-        older_logs = Event.objects.order_by('log_timestamp').filter(log_timestamp__lte=one_week_ago)
+        older_logs = Event.objects.order_by('event_timestamp').filter(log_timestamp__lte=one_week_ago)
         for log in older_logs:
             log_entry = [log.id, log.search_id, log.component_id, log.title, log.event_timestamp, log.category, log.message]
             log_writer.writerow(log_entry)

search/views.py

@@ -23,6 +23,7 @@
 from SolrClient.exceptions import ConnectionError, SolrError
 from search.tasks import export_search_results_csv
 from unidecode import unidecode
+from urllib import parse
 
 
 def iter_namespace(ns_pkg):
@@ -33,6 +34,31 @@ def iter_namespace(ns_pkg):
     return pkgutil.iter_modules(ns_pkg.__path__, ns_pkg.__name__ + ".")
 
 
+def log_search_results(request: HttpRequest, search_logger: logging.Logger, doc_count: int = 0):
+    qurl = parse.urlparse(request.get_full_path())
+
+    query_values = parse.parse_qs(request.META["QUERY_STRING"])
+    page = '1'
+    sort = 'default'
+    search_text = ""
+    facets = ''
+    search_format = 'default'
+    if 'page' in query_values:
+        page = query_values['page'][0]
+    if 'sort' in query_values:
+        sort = query_values['sort'][0]
+    if 'search_text' in query_values:
+        search_text = parse.quote_plus(query_values['search_text'][0])
+    if 'search_format' in query_values:
+        search_format = query_values['search_format'][0]
+    for q in query_values:
+        if q not in ['page', 'sort', 'search_text', 'search_format']:
+            facets += f"{q}:{parse.quote_plus(query_values[q][0])},"
+    search_type = qurl.path.rstrip('/').split('/')[-1]
+    log_message = f'{qurl.hostname},{search_type},{request.session.session_key},{page},{sort},"{search_text}","{facets}",{search_format},{doc_count}'
+    search_logger.info(log_message)
+
+
 def get_error_context(search_type: str, lang: str, error_msg=""):
     return {
         "language": lang,
@@ -96,6 +122,7 @@ class SearchView(View):
     discovered_plugins = {}
 
     logger = logging.getLogger(__name__)
+    search_logger = logging.getLogger("search_term_logger")
 
     def __init__(self):
         super().__init__()
@@ -514,6 +541,7 @@ def get(self, request: HttpRequest, lang='en', search_type=''):
                     else:
                         json_link = json_link + "&search_format=json"
                     context["json_format_url"] = json_link
+                    log_search_results(request, self.search_logger, doc_count=solr_response.num_found)
                     return render(request, self.searches[search_type].page_template, context)
             except (ConnectionError, SolrError) as ce:
                 return render(request, 'error.html', get_error_context(search_type, lang, ce.args[0]))