[ZEP-13] corrige le comportement d'Elasticsearch

templates/searchv2/includes/publishedcontent.part.html

@@ -30,6 +30,11 @@ <h3 class="content-title" itemprop="itemListElement">
                 {% trans "Article publié" %}
             {% elif search_result.content_type == 'TUTORIAL' %}
                 {% trans "Tutoriel publié" %}
+            {% elif search_result.content_type == 'OPINION' %}
+                {% trans "Opinion" %}
+                {% if search_result.picked %}
+                    {% trans "mise en avant" %}
+                {% endif %}
             {% else %}
                 {% trans "Contenu publié" %}
             {% endif %}

update.md

@@ -1010,10 +1010,16 @@ Une fois que tout est indexé,
 Tribunes
 --------
 
-Ajouter à la fin de `/etc/munin/plugin-conf.d/zds.conf`
++ Ajouter à la fin de `/etc/munin/plugin-conf.d/zds.conf`
 
 ```
     [zds_total_tribunes]
     env.url http://www.zestedesavoir.com/munin/total_tribunes/
     env.graph_category zds
 ```
+
++ Réindexer les données (un champ a été rajouté):
+
+    ```
+    python manage.py es_manager index_all
+    ```

zds/searchv2/models.py

@@ -234,15 +234,21 @@ def delete_document_in_elasticsearch(instance):
     """
 
     index_manager = ESIndexManager(**settings.ES_SEARCH_INDEX)
-    index_manager.delete_document(instance)
-    index_manager.refresh_index()
+
+    if index_manager.index_exists:
+        index_manager.delete_document(instance)
+        index_manager.refresh_index()
 
 
 def get_django_indexable_objects():
     """Return all indexable objects registered in Django"""
     return [model for model in apps.get_models() if issubclass(model, AbstractESDjangoIndexable)]
 
 
+class NeedIndex(Exception):
+    pass
+
+
 class ESIndexManager(object):
     """Manage a given index with different taylor-made functions"""
 
@@ -260,6 +266,7 @@ def __init__(self, name, shards=5, replicas=0, connection_alias='default'):
         """
 
         self.index = name
+        self.index_exists = False
 
         self.number_of_shards = shards
         self.number_of_replicas = replicas
@@ -282,6 +289,9 @@ def __init__(self, name, shards=5, replicas=0, connection_alias='default'):
             else:
                 self.logger.info('connected to ES cluster')
 
+            if self.connected_to_es:
+                self.index_exists = self.es.indices.exists(self.index)
+
     def clear_es_index(self):
         """Clear index
         """
@@ -293,6 +303,8 @@ def clear_es_index(self):
             self.es.indices.delete(self.index)
             self.logger.info('index cleared')
 
+            self.index_exists = False
+
     def reset_es_index(self, models):
         """Delete old index and create an new one (with the same name). Setup the number of shards and replicas.
         Then, set mappings for the different models.
@@ -327,6 +339,8 @@ def reset_es_index(self, models):
             }
         )
 
+        self.index_exists = True
+
         self.logger.info('index created')
 
     def setup_custom_analyzer(self):
@@ -351,6 +365,9 @@ def setup_custom_analyzer(self):
         if not self.connected_to_es:
             return
 
+        if not self.index_exists:
+            raise NeedIndex()
+
         self.es.indices.close(self.index)
 
         document = {
@@ -422,9 +439,6 @@ def clear_indexing_of_model(self, model):
         :type model: class
         """
 
-        if not self.connected_to_es:
-            return
-
         if issubclass(model, AbstractESDjangoIndexable):  # use a global update with Django
             objs = model.get_es_django_indexable(force_reindexing=True)
             objs.update(es_flagged=True, es_already_indexed=False)
@@ -456,6 +470,9 @@ def es_bulk_indexing_of_model(self, model, force_reindexing=False):
         if not self.connected_to_es:
             return
 
+        if not self.index_exists:
+            raise NeedIndex()
+
         # better safe than sorry
         if model.__name__ == 'FakeChapter':
             self.logger.warn('Cannot index FakeChapter model. Please index its parent model.')
@@ -568,6 +585,9 @@ def refresh_index(self):
         if not self.connected_to_es:
             return
 
+        if not self.index_exists:
+            raise NeedIndex()
+
         self.es.indices.refresh(self.index)
 
     def update_single_document(self, document, doc):
@@ -584,6 +604,9 @@ def update_single_document(self, document, doc):
         if not self.connected_to_es:
             return
 
+        if not self.index_exists:
+            raise NeedIndex()
+
         arguments = {'index': self.index, 'doc_type': document.get_es_document_type(), 'id': document.es_id}
         if self.es.exists(**arguments):
             self.es.update(body={'doc': doc}, **arguments)
@@ -599,6 +622,9 @@ def delete_document(self, document):
         if not self.connected_to_es:
             return
 
+        if not self.index_exists:
+            raise NeedIndex()
+
         arguments = {'index': self.index, 'doc_type': document.get_es_document_type(), 'id': document.es_id}
         if self.es.exists(**arguments):
             self.es.delete(**arguments)
@@ -621,6 +647,9 @@ def delete_by_query(self, doc_type='', query=MatchAll()):
         if not self.connected_to_es:
             return
 
+        if not self.index_exists:
+            raise NeedIndex()
+
         response = self.es.delete_by_query(index=self.index, doc_type=doc_type, body={'query': query})
 
         self.logger.info('delete_by_query {}s ({})'.format(doc_type, response['deleted']))
@@ -641,6 +670,9 @@ def analyze_sentence(self, request):
         if not self.connected_to_es:
             return
 
+        if not self.index_exists:
+            raise NeedIndex()
+
         document = {'text': request}
         tokens = []
         for token in self.es.indices.analyze(index=self.index, body=document)['tokens']:
@@ -660,4 +692,7 @@ def setup_search(self, request):
         if not self.connected_to_es:
             return
 
+        if not self.index_exists:
+            raise NeedIndex()
+
         return request.index(self.index).using(self.es)

zds/searchv2/tests/tests_utils.py

@@ -91,6 +91,8 @@ def test_es_manager(self):
 
         # 1. test "index-all"
         call_command('es_manager', 'index_all')
+        self.assertTrue(self.index_manager.es.indices.exists(self.index_manager.index))
+        self.index_manager.index_exists = True
 
         topic = Topic.objects.get(pk=topic.pk)
         post = Post.objects.get(pk=post.pk)
@@ -126,7 +128,10 @@ def test_es_manager(self):
 
         # 2. test "clear"
         self.assertTrue(self.index_manager.index in self.index_manager.es.cat.indices())  # index in
+
         call_command('es_manager', 'clear')
+        self.assertFalse(self.index_manager.es.indices.exists(self.index_manager.index))
+        self.index_manager.index_exists = False
 
         # must reset every object
         topic = Topic.objects.get(pk=topic.pk)
@@ -145,6 +150,9 @@ def test_es_manager(self):
 
         # 3. test "setup"
         call_command('es_manager', 'setup')
+        self.assertTrue(self.index_manager.es.indices.exists(self.index_manager.index))
+        self.index_manager.index_exists = True
+
         self.assertTrue(self.index_manager.index in self.index_manager.es.cat.indices())  # index back in ...
 
         s = Search()

zds/searchv2/tests/tests_views.py

@@ -2,6 +2,7 @@
 
 import os
 import shutil
+import datetime
 
 from elasticsearch_dsl import Search
 from elasticsearch_dsl.query import MatchAll
@@ -284,13 +285,23 @@ def test_boosts(self):
         article = PublishedContentFactory(type='ARTICLE', title=text)
         published_article = PublishedContent.objects.get(content_pk=article.pk)
 
+        opinion_not_picked = PublishedContentFactory(type='OPINION', title=text)
+        published_opinion_not_picked = PublishedContent.objects.get(content_pk=opinion_not_picked.pk)
+
+        opinion_picked = PublishedContentFactory(type='OPINION', title=text)
+        opinion_picked.sha_picked = opinion_picked.sha_draft
+        opinion_picked.date_picked = datetime.datetime.now()
+        opinion_picked.save()
+
+        published_opinion_picked = PublishedContent.objects.get(content_pk=opinion_picked.pk)
+
         for model in self.indexable:
             if model is FakeChapter:
                 continue
             self.manager.es_bulk_indexing_of_model(model)
         self.manager.refresh_index()
 
-        self.assertEqual(len(self.manager.setup_search(Search().query(MatchAll())).execute()), 8)
+        self.assertEqual(len(self.manager.setup_search(Search().query(MatchAll())).execute()), 10)
 
         # 2. Reset all boosts to 1
         for doc_type in settings.ZDS_APP['search']['boosts']:
@@ -418,10 +429,14 @@ def test_boosts(self):
 
         self.assertEqual(result.status_code, 200)
         response = result.context['object_list'].execute()
-        self.assertEquals(response.hits.total, 3)
+        self.assertEquals(response.hits.total, 5)
 
         # score are equals without boost:
-        self.assertTrue(response[0].meta.score == response[1].meta.score)
+        self.assertTrue(response[0].meta.score ==
+                        response[1].meta.score ==
+                        response[2].meta.score ==
+                        response[3].meta.score ==
+                        response[4].meta.score)
 
         settings.ZDS_APP['search']['boosts']['publishedcontent']['if_article'] = 2.0
 
@@ -430,7 +445,7 @@ def test_boosts(self):
 
         self.assertEqual(result.status_code, 200)
         response = result.context['object_list'].execute()
-        self.assertEqual(response.hits.total, 3)
+        self.assertEqual(response.hits.total, 5)
 
         self.assertTrue(response[0].meta.score > response[1].meta.score)
         self.assertEquals(response[0].meta.id, str(published_article.pk))  # obvious
@@ -443,12 +458,42 @@ def test_boosts(self):
 
         self.assertEqual(result.status_code, 200)
         response = result.context['object_list'].execute()
-        self.assertEqual(response.hits.total, 3)
+        self.assertEqual(response.hits.total, 5)
 
         self.assertTrue(response[0].meta.score > response[1].meta.score)
         self.assertEquals(response[0].meta.id, str(published_tuto.pk))  # obvious
 
         settings.ZDS_APP['search']['boosts']['publishedcontent']['if_tutorial'] = 1.0
+        settings.ZDS_APP['search']['boosts']['publishedcontent']['if_opinion'] = 2.0
+        settings.ZDS_APP['search']['boosts']['publishedcontent']['if_opinion_not_picked'] = 4.0
+        # Note: in "real life", unpicked opinion would get a boost < 1.
+
+        result = self.client.get(
+            reverse('search:query') + '?q=' + text + '&models=content', follow=False)
+
+        self.assertEqual(result.status_code, 200)
+        response = result.context['object_list'].execute()
+        self.assertEqual(response.hits.total, 5)
+
+        self.assertTrue(response[0].meta.score > response[1].meta.score > response[2].meta.score)
+        self.assertEquals(response[0].meta.id, str(published_opinion_not_picked.pk))  # unpicked opinion got first
+        self.assertEquals(response[1].meta.id, str(published_opinion_picked.pk))
+
+        settings.ZDS_APP['search']['boosts']['publishedcontent']['if_opinion'] = 1.0
+        settings.ZDS_APP['search']['boosts']['publishedcontent']['if_opinion_not_picked'] = 1.0
+        settings.ZDS_APP['search']['boosts']['publishedcontent']['if_medium_or_big_tutorial'] = 2.0
+
+        result = self.client.get(
+            reverse('search:query') + '?q=' + text + '&models=content', follow=False)
+
+        self.assertEqual(result.status_code, 200)
+        response = result.context['object_list'].execute()
+        self.assertEqual(response.hits.total, 5)
+
+        self.assertTrue(response[0].meta.score > response[1].meta.score)
+        self.assertEquals(response[0].meta.id, str(published_tuto.pk))  # obvious
+
+        settings.ZDS_APP['search']['boosts']['publishedcontent']['if_medium_or_big_tutorial'] = 1.0
 
         # 6. Test global boosts
         # NOTE: score are NOT the same for all documents, no matter how hard it tries to, small differences exists
@@ -463,7 +508,7 @@ def test_boosts(self):
 
             self.assertEqual(result.status_code, 200)
             response = result.context['object_list'].execute()
-            self.assertEqual(response.hits.total, 8)
+            self.assertEqual(response.hits.total, 10)
 
             self.assertEqual(response[0].meta.doc_type, model.get_es_document_type())  # obvious
 

zds/searchv2/views.py

@@ -129,10 +129,22 @@ def get_queryset_publishedcontents(self):
                 'filter': Match(content_type='TUTORIAL'),
                 'weight': settings.ZDS_APP['search']['boosts']['publishedcontent']['if_tutorial']
             },
+            {
+                'filter': Match(content_type='TUTORIAL') & Match(has_chapters=True),
+                'weight': settings.ZDS_APP['search']['boosts']['publishedcontent']['if_medium_or_big_tutorial']
+            },
             {
                 'filter': Match(content_type='ARTICLE'),
                 'weight': settings.ZDS_APP['search']['boosts']['publishedcontent']['if_article']
             },
+            {
+                'filter': Match(content_type='OPINION'),
+                'weight': settings.ZDS_APP['search']['boosts']['publishedcontent']['if_opinion']
+            },
+            {
+                'filter': Match(content_type='OPINION') & Match(picked=False),
+                'weight': settings.ZDS_APP['search']['boosts']['publishedcontent']['if_opinion_not_picked']
+            },
         ]
 
         scored_query = FunctionScore(query=query, boost_mode='multiply', functions=functions_score)

zds/settings.py

@@ -557,6 +557,9 @@
                 'global': 3.0,
                 'if_article': 1.0,
                 'if_tutorial': 1.0,
+                'if_medium_or_big_tutorial': 1.5,
+                'if_opinion': 0.66,
+                'if_opinion_not_picked': 0.5
             },
             'topic': {
                 'global': 2.0,