OPEN 2790, 2902: Remove temporary CSV loading fix and add support for…

… NLTK feature

.gitignore

@@ -161,5 +161,9 @@ cython_debug/
 !/search/management/commands/contracts_*.py
 !/search/management/commands/nap5_*.py
 !/search/management/commands/qpnotes_*.py
+/nltk/corpora
+/nltk/tokenizers
+/nltk/*.model
+!/
 # Don't include CSV files from the cache
 !/cache/*.csv

nltk/NLTK.md

@@ -0,0 +1 @@
+Install the NLTK Stopworks and Punkt tokenizer here

oc_search/settings-sample.py

@@ -300,3 +300,4 @@
 IMPORT_DATA_CSV_SOLR_INDEX_GROUP_SIZE = 10
 IMPORT_DATA_CSV_BAD_DATA_DIR = os.path.join(BASE_DIR, 'bad_data')
 
+NLTK_DATADIR = os.path.join(BASE_DIR, 'nltk')

requirements.txt

@@ -17,10 +17,11 @@ inflection==0.5.1
 markdown2==2.4.10
 nltk==3.8.1
 pandas==1.4.4
-psycopg2==2.9.7
+psycopg2==2.9.9
 python-dateutil==2.8.2
 PyYAML==6.0.1
-Unidecode==1.3.6
+scikit-learn==1.3.2
+Unidecode==1.3.7
 uWSGI==2.0.19.1; sys_platform == 'linux'
 #git+https://github.com/thriuin/SolrClient.git@master
 

search/management/commands/import_data_csv.py

@@ -376,11 +376,6 @@ def handle(self, *args, **options):
                                 bd_writer.writeheader()
                                 bd_file.flush()
 
-                            # Temporary fix for connection blocking issue OPEN 2902
-                            for solr_field in solr_record:
-                                if type(solr_record[solr_field]) == str:
-                                    solr_record[solr_field] = str(solr_record[solr_field]).replace(" echo ", " ẹcho ")
-
                             solr_items.append(solr_record)
                             total += 1
 

search/management/commands/nltk_download.py

@@ -0,0 +1,18 @@
+from django.core.management.base import BaseCommand
+from django.conf import settings
+import logging
+import nltk
+
+
+class Command(BaseCommand):
+    help = 'Download the NLTK Stopwords and punkt tokenizer to a project folder'
+
+    logger = logging.getLogger(__name__)
+
+    def handle(self, *args, **options):
+
+        try:
+            nltk.download('stopwords',download_dir=settings.NLTK_DATADIR)
+            nltk.download('punkt', download_dir=settings.NLTK_DATADIR)
+        except Exception as x:
+            self.logger.error(x)