From c8c9ec4c089e7a4762c349c66e0366dcf27fa37c Mon Sep 17 00:00:00 2001 From: pierre-24 Date: Sun, 2 Apr 2017 15:55:30 +0200 Subject: [PATCH 1/6] fix on ES side --- zds/searchv2/models.py | 45 ++++++++++++++++++--- zds/searchv2/tests/tests_utils.py | 8 ++++ zds/tutorialv2/models/models_database.py | 4 +- zds/tutorialv2/tests/tests_opinion_views.py | 2 + 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/zds/searchv2/models.py b/zds/searchv2/models.py index 458899821a..5750000bc2 100644 --- a/zds/searchv2/models.py +++ b/zds/searchv2/models.py @@ -234,8 +234,10 @@ def delete_document_in_elasticsearch(instance): """ index_manager = ESIndexManager(**settings.ES_SEARCH_INDEX) - index_manager.delete_document(instance) - index_manager.refresh_index() + + if index_manager.index_exists: + index_manager.delete_document(instance) + index_manager.refresh_index() def get_django_indexable_objects(): @@ -243,6 +245,10 @@ def get_django_indexable_objects(): return [model for model in apps.get_models() if issubclass(model, AbstractESDjangoIndexable)] +class NeedIndex(Exception): + pass + + class ESIndexManager(object): """Manage a given index with different taylor-made functions""" @@ -260,6 +266,7 @@ def __init__(self, name, shards=5, replicas=0, connection_alias='default'): """ self.index = name + self.index_exists = False self.number_of_shards = shards self.number_of_replicas = replicas @@ -282,6 +289,9 @@ def __init__(self, name, shards=5, replicas=0, connection_alias='default'): else: self.logger.info('connected to ES cluster') + if self.connected_to_es: + self.index_exists = self.es.indices.exists(self.index) + def clear_es_index(self): """Clear index """ @@ -293,6 +303,8 @@ def clear_es_index(self): self.es.indices.delete(self.index) self.logger.info('index cleared') + self.index_exists = False + def reset_es_index(self, models): """Delete old index and create an new one (with the same name). Setup the number of shards and replicas. Then, set mappings for the different models. @@ -327,6 +339,8 @@ def reset_es_index(self, models): } ) + self.index_exists = True + self.logger.info('index created') def setup_custom_analyzer(self): @@ -351,6 +365,9 @@ def setup_custom_analyzer(self): if not self.connected_to_es: return + if not self.index_exists: + raise NeedIndex() + self.es.indices.close(self.index) document = { @@ -422,9 +439,6 @@ def clear_indexing_of_model(self, model): :type model: class """ - if not self.connected_to_es: - return - if issubclass(model, AbstractESDjangoIndexable): # use a global update with Django objs = model.get_es_django_indexable(force_reindexing=True) objs.update(es_flagged=True, es_already_indexed=False) @@ -456,6 +470,9 @@ def es_bulk_indexing_of_model(self, model, force_reindexing=False): if not self.connected_to_es: return + if not self.index_exists: + raise NeedIndex() + # better safe than sorry if model.__name__ == 'FakeChapter': self.logger.warn('Cannot index FakeChapter model. Please index its parent model.') @@ -568,6 +585,9 @@ def refresh_index(self): if not self.connected_to_es: return + if not self.index_exists: + raise NeedIndex() + self.es.indices.refresh(self.index) def update_single_document(self, document, doc): @@ -584,6 +604,9 @@ def update_single_document(self, document, doc): if not self.connected_to_es: return + if not self.index_exists: + raise NeedIndex() + arguments = {'index': self.index, 'doc_type': document.get_es_document_type(), 'id': document.es_id} if self.es.exists(**arguments): self.es.update(body={'doc': doc}, **arguments) @@ -599,6 +622,9 @@ def delete_document(self, document): if not self.connected_to_es: return + if not self.index_exists: + raise NeedIndex() + arguments = {'index': self.index, 'doc_type': document.get_es_document_type(), 'id': document.es_id} if self.es.exists(**arguments): self.es.delete(**arguments) @@ -621,6 +647,9 @@ def delete_by_query(self, doc_type='', query=MatchAll()): if not self.connected_to_es: return + if not self.index_exists: + raise NeedIndex() + response = self.es.delete_by_query(index=self.index, doc_type=doc_type, body={'query': query}) self.logger.info('delete_by_query {}s ({})'.format(doc_type, response['deleted'])) @@ -641,6 +670,9 @@ def analyze_sentence(self, request): if not self.connected_to_es: return + if not self.index_exists: + raise NeedIndex() + document = {'text': request} tokens = [] for token in self.es.indices.analyze(index=self.index, body=document)['tokens']: @@ -660,4 +692,7 @@ def setup_search(self, request): if not self.connected_to_es: return + if not self.index_exists: + raise NeedIndex() + return request.index(self.index).using(self.es) diff --git a/zds/searchv2/tests/tests_utils.py b/zds/searchv2/tests/tests_utils.py index 15200295a5..0718ad90a4 100644 --- a/zds/searchv2/tests/tests_utils.py +++ b/zds/searchv2/tests/tests_utils.py @@ -91,6 +91,8 @@ def test_es_manager(self): # 1. test "index-all" call_command('es_manager', 'index_all') + self.assertTrue(self.index_manager.es.indices.exists(self.index_manager.index)) + self.index_manager.index_exists = True topic = Topic.objects.get(pk=topic.pk) post = Post.objects.get(pk=post.pk) @@ -126,7 +128,10 @@ def test_es_manager(self): # 2. test "clear" self.assertTrue(self.index_manager.index in self.index_manager.es.cat.indices()) # index in + call_command('es_manager', 'clear') + self.assertFalse(self.index_manager.es.indices.exists(self.index_manager.index)) + self.index_manager.index_exists = False # must reset every object topic = Topic.objects.get(pk=topic.pk) @@ -145,6 +150,9 @@ def test_es_manager(self): # 3. test "setup" call_command('es_manager', 'setup') + self.assertTrue(self.index_manager.es.indices.exists(self.index_manager.index)) + self.index_manager.index_exists = True + self.assertTrue(self.index_manager.index in self.index_manager.es.cat.indices()) # index back in ... s = Search() diff --git a/zds/tutorialv2/models/models_database.py b/zds/tutorialv2/models/models_database.py index 0078d60f3f..b5829aed33 100644 --- a/zds/tutorialv2/models/models_database.py +++ b/zds/tutorialv2/models/models_database.py @@ -945,7 +945,9 @@ def delete_published_content_in_elasticsearch(sender, instance, **kwargs): """ index_manager = ESIndexManager(**settings.ES_SEARCH_INDEX) - index_manager.delete_by_query(FakeChapter.get_es_document_type(), ES_Q('match', _routing=instance.es_id)) + + if index_manager.index_exists: + index_manager.delete_by_query(FakeChapter.get_es_document_type(), ES_Q('match', _routing=instance.es_id)) return delete_document_in_elasticsearch(instance) diff --git a/zds/tutorialv2/tests/tests_opinion_views.py b/zds/tutorialv2/tests/tests_opinion_views.py index a54596b25c..0bc8ed15fe 100644 --- a/zds/tutorialv2/tests/tests_opinion_views.py +++ b/zds/tutorialv2/tests/tests_opinion_views.py @@ -20,7 +20,9 @@ overrided_zds_app['content']['extra_content_generation_policy'] = 'NONE' +@override_settings(MEDIA_ROOT=os.path.join(BASE_DIR, 'media-test')) @override_settings(ZDS_APP=overrided_zds_app) +@override_settings(ES_ENABLED=False) class PublishedContentTests(TestCase): def setUp(self): overrided_zds_app['member']['bot_account'] = ProfileFactory().user.username From 21cf7efbdd262cde069bb82a3e7741bdae3f46a8 Mon Sep 17 00:00:00 2001 From: pierre-24 Date: Sun, 2 Apr 2017 16:11:32 +0200 Subject: [PATCH 2/6] fix tests as well! --- zds/tutorialv2/tests/tests_opinion_views.py | 92 +++++++++++++++++---- zds/tutorialv2/views/views_validations.py | 3 + 2 files changed, 80 insertions(+), 15 deletions(-) diff --git a/zds/tutorialv2/tests/tests_opinion_views.py b/zds/tutorialv2/tests/tests_opinion_views.py index 0bc8ed15fe..452bb77e8a 100644 --- a/zds/tutorialv2/tests/tests_opinion_views.py +++ b/zds/tutorialv2/tests/tests_opinion_views.py @@ -11,7 +11,7 @@ from zds.member.factories import ProfileFactory, StaffProfileFactory from zds.settings import BASE_DIR from zds.tutorialv2.factories import PublishableContentFactory, ExtractFactory, LicenceFactory, PublishedContentFactory -from zds.tutorialv2.models.models_database import PublishableContent +from zds.tutorialv2.models.models_database import PublishableContent, PublishedContent from zds.utils.models import Alert overrided_zds_app = settings.ZDS_APP @@ -61,11 +61,17 @@ def test_opinion_publication_author(self): { 'text': text_publication, 'source': '', - 'version': opinion.load_version().current_version + 'version': opinion_draft.current_version }, follow=False) self.assertEqual(result.status_code, 302) + self.assertEqual(PublishedContent.objects.count(), 1) + + opinion = PublishableContent.objects.get(pk=opinion.pk) + self.assertIsNotNone(opinion.public_version) + self.assertEqual(opinion.public_version.sha_public, opinion_draft.current_version) + def test_accessible_ui_for_author(self): opinion = PublishedContentFactory(author_list=[self.user_author], type='OPINION') self.assertEqual( @@ -124,11 +130,17 @@ def test_opinion_publication_staff(self): { 'text': text_publication, 'source': '', - 'version': opinion.load_version().current_version + 'version': opinion_draft.current_version }, follow=False) self.assertEqual(result.status_code, 302) + self.assertEqual(PublishedContent.objects.count(), 1) + + opinion = PublishableContent.objects.get(pk=opinion.pk) + self.assertIsNotNone(opinion.public_version) + self.assertEqual(opinion.public_version.sha_public, opinion_draft.current_version) + def test_opinion_publication_guest(self): """ Test the publication of PublishableContent where type is OPINION (with guest => 403). @@ -158,11 +170,13 @@ def test_opinion_publication_guest(self): { 'text': text_publication, 'source': '', - 'version': opinion.load_version().current_version + 'version': opinion_draft.current_version }, follow=False) self.assertEqual(result.status_code, 403) + self.assertEqual(PublishedContent.objects.count(), 0) + def test_opinion_unpublication(self): """ Test the unpublication of PublishableContent where type is OPINION (with author). @@ -196,22 +210,33 @@ def test_opinion_unpublication(self): { 'text': text_publication, 'source': '', - 'version': opinion.load_version().current_version + 'version': opinion_draft.current_version }, follow=False) self.assertEqual(result.status_code, 302) + self.assertEqual(PublishedContent.objects.count(), 1) + + opinion = PublishableContent.objects.get(pk=opinion.pk) + self.assertIsNotNone(opinion.public_version) + self.assertEqual(opinion.public_version.sha_public, opinion_draft.current_version) + # unpublish result = self.client.post( reverse('validation:unpublish-opinion', kwargs={'pk': opinion.pk, 'slug': opinion.slug}), { 'text': text_unpublication, 'source': '', - 'version': opinion.load_version().current_version + 'version': opinion_draft.current_version }, follow=False) self.assertEqual(result.status_code, 302) + self.assertEqual(PublishedContent.objects.count(), 0) + + opinion = PublishableContent.objects.get(pk=opinion.pk) + self.assertIsNone(opinion.public_version) + # staff self.assertEqual( @@ -226,22 +251,33 @@ def test_opinion_unpublication(self): { 'text': text_publication, 'source': '', - 'version': opinion.load_version().current_version + 'version': opinion_draft.current_version }, follow=False) self.assertEqual(result.status_code, 302) + self.assertEqual(PublishedContent.objects.count(), 1) + + opinion = PublishableContent.objects.get(pk=opinion.pk) + self.assertIsNotNone(opinion.public_version) + self.assertEqual(opinion.public_version.sha_public, opinion_draft.current_version) + # unpublish result = self.client.post( reverse('validation:unpublish-opinion', kwargs={'pk': opinion.pk, 'slug': opinion.slug}), { 'text': text_unpublication, 'source': '', - 'version': opinion.load_version().current_version + 'version': opinion_draft.current_version }, follow=False) self.assertEqual(result.status_code, 302) + self.assertEqual(PublishedContent.objects.count(), 0) + + opinion = PublishableContent.objects.get(pk=opinion.pk) + self.assertIsNone(opinion.public_version) + # guest => 403 self.assertEqual( @@ -256,11 +292,17 @@ def test_opinion_unpublication(self): { 'text': text_publication, 'source': '', - 'version': opinion.load_version().current_version + 'version': opinion_draft.current_version }, follow=False) self.assertEqual(result.status_code, 302) + self.assertEqual(PublishedContent.objects.count(), 1) + + opinion = PublishableContent.objects.get(pk=opinion.pk) + self.assertIsNotNone(opinion.public_version) + self.assertEqual(opinion.public_version.sha_public, opinion_draft.current_version) + self.assertEqual( self.client.login( username=self.user_guest.username, @@ -273,11 +315,13 @@ def test_opinion_unpublication(self): { 'text': text_unpublication, 'source': '', - 'version': opinion.load_version().current_version + 'version': opinion_draft.current_version }, follow=False) self.assertEqual(result.status_code, 403) + self.assertEqual(PublishedContent.objects.count(), 1) + def test_opinion_validation(self): """ Test the validation of PublishableContent where type is OPINION. @@ -308,11 +352,17 @@ def test_opinion_validation(self): { 'text': text_publication, 'source': '', - 'version': opinion.load_version().current_version + 'version': opinion_draft.current_version }, follow=False) self.assertEqual(result.status_code, 302) + self.assertEqual(PublishedContent.objects.count(), 1) + + opinion = PublishableContent.objects.get(pk=opinion.pk) + self.assertIsNotNone(opinion.public_version) + self.assertEqual(opinion.public_version.sha_public, opinion_draft.current_version) + # valid with author => 403 opinion = PublishableContent.objects.get(pk=opinion.pk) opinion_draft = opinion.load_version() @@ -429,17 +479,23 @@ def test_opinion_conversion(self): { 'text': text_publication, 'source': '', - 'version': opinion.load_version().current_version + 'version': opinion_draft.current_version }, follow=False) self.assertEqual(result.status_code, 302) + self.assertEqual(PublishedContent.objects.count(), 1) + + opinion = PublishableContent.objects.get(pk=opinion.pk) + self.assertIsNotNone(opinion.public_version) + self.assertEqual(opinion.public_version.sha_public, opinion_draft.current_version) + # valid with author => 403 result = self.client.post( reverse('validation:promote-opinion', kwargs={'pk': opinion.pk, 'slug': opinion.slug}), { 'source': '', - 'version': opinion.load_version().current_version + 'version': opinion_draft.current_version }, follow=False) self.assertEqual(result.status_code, 403) @@ -455,7 +511,7 @@ def test_opinion_conversion(self): reverse('validation:promote-opinion', kwargs={'pk': opinion.pk, 'slug': opinion.slug}), { 'source': '', - 'version': opinion.load_version().current_version + 'version': opinion_draft.current_version }, follow=False) self.assertEqual(result.status_code, 302) @@ -487,11 +543,17 @@ def test_opinion_alert(self): { 'text': text_publication, 'source': '', - 'version': opinion.load_version().current_version + 'version': opinion_draft.current_version }, follow=False) self.assertEqual(result.status_code, 302) + self.assertEqual(PublishedContent.objects.count(), 1) + + opinion = PublishableContent.objects.get(pk=opinion.pk) + self.assertIsNotNone(opinion.public_version) + self.assertEqual(opinion.public_version.sha_public, opinion_draft.current_version) + # Alert content random_user = ProfileFactory().user diff --git a/zds/tutorialv2/views/views_validations.py b/zds/tutorialv2/views/views_validations.py index 4ef3b736dd..0d945776b9 100644 --- a/zds/tutorialv2/views/views_validations.py +++ b/zds/tutorialv2/views/views_validations.py @@ -607,6 +607,9 @@ def form_valid(self, form): if user not in versioned.authors.all() and not user.has_perm('tutorialv2.change_validation'): raise PermissionDenied + if form.cleaned_data['version'] != self.object.sha_public: + raise PermissionDenied + unpublish_content(self.object) self.object.sha_public = None From 3b8cb7269fe710b7ebc18c263519aa13a3969b06 Mon Sep 17 00:00:00 2001 From: pierre-24 Date: Sun, 2 Apr 2017 17:48:34 +0200 Subject: [PATCH 3/6] setup search for opinions (see #4133) --- update.md | 8 ++++- zds/searchv2/tests/tests_views.py | 44 ++++++++++++++++++++---- zds/searchv2/views.py | 8 +++++ zds/settings.py | 2 ++ zds/tutorialv2/models/models_database.py | 10 ++++-- 5 files changed, 63 insertions(+), 9 deletions(-) diff --git a/update.md b/update.md index f4e10bcb3c..66c95623c3 100644 --- a/update.md +++ b/update.md @@ -1010,10 +1010,16 @@ Une fois que tout est indexé, Tribunes -------- -Ajouter à la fin de `/etc/munin/plugin-conf.d/zds.conf` ++ Ajouter à la fin de `/etc/munin/plugin-conf.d/zds.conf` ``` [zds_total_tribunes] env.url http://www.zestedesavoir.com/munin/total_tribunes/ env.graph_category zds ``` + ++ Réindexer les données (un champ a été rajouté): + + ``` + python manage.py es_manager index_all + ``` diff --git a/zds/searchv2/tests/tests_views.py b/zds/searchv2/tests/tests_views.py index 1d0e202f97..fc686a528f 100644 --- a/zds/searchv2/tests/tests_views.py +++ b/zds/searchv2/tests/tests_views.py @@ -2,6 +2,7 @@ import os import shutil +import datetime from elasticsearch_dsl import Search from elasticsearch_dsl.query import MatchAll @@ -284,13 +285,23 @@ def test_boosts(self): article = PublishedContentFactory(type='ARTICLE', title=text) published_article = PublishedContent.objects.get(content_pk=article.pk) + opinion_not_picked = PublishedContentFactory(type='OPINION', title=text) + published_opinion_not_picked = PublishedContent.objects.get(content_pk=opinion_not_picked.pk) + + opinion_picked = PublishedContentFactory(type='OPINION', title=text) + opinion_picked.sha_picked = opinion_picked.sha_draft + opinion_picked.date_picked = datetime.datetime.now() + opinion_picked.save() + + published_opinion_picked = PublishedContent.objects.get(content_pk=opinion_picked.pk) + for model in self.indexable: if model is FakeChapter: continue self.manager.es_bulk_indexing_of_model(model) self.manager.refresh_index() - self.assertEqual(len(self.manager.setup_search(Search().query(MatchAll())).execute()), 8) + self.assertEqual(len(self.manager.setup_search(Search().query(MatchAll())).execute()), 10) # 2. Reset all boosts to 1 for doc_type in settings.ZDS_APP['search']['boosts']: @@ -418,10 +429,14 @@ def test_boosts(self): self.assertEqual(result.status_code, 200) response = result.context['object_list'].execute() - self.assertEquals(response.hits.total, 3) + self.assertEquals(response.hits.total, 5) # score are equals without boost: - self.assertTrue(response[0].meta.score == response[1].meta.score) + self.assertTrue(response[0].meta.score == + response[1].meta.score == + response[2].meta.score == + response[3].meta.score == + response[4].meta.score) settings.ZDS_APP['search']['boosts']['publishedcontent']['if_article'] = 2.0 @@ -430,7 +445,7 @@ def test_boosts(self): self.assertEqual(result.status_code, 200) response = result.context['object_list'].execute() - self.assertEqual(response.hits.total, 3) + self.assertEqual(response.hits.total, 5) self.assertTrue(response[0].meta.score > response[1].meta.score) self.assertEquals(response[0].meta.id, str(published_article.pk)) # obvious @@ -443,12 +458,29 @@ def test_boosts(self): self.assertEqual(result.status_code, 200) response = result.context['object_list'].execute() - self.assertEqual(response.hits.total, 3) + self.assertEqual(response.hits.total, 5) self.assertTrue(response[0].meta.score > response[1].meta.score) self.assertEquals(response[0].meta.id, str(published_tuto.pk)) # obvious settings.ZDS_APP['search']['boosts']['publishedcontent']['if_tutorial'] = 1.0 + settings.ZDS_APP['search']['boosts']['publishedcontent']['if_opinion'] = 2.0 + settings.ZDS_APP['search']['boosts']['publishedcontent']['if_opinion_not_picked'] = 4.0 + # Note: in "real life", unpicked opinion would get a boost < 1. + + result = self.client.get( + reverse('search:query') + '?q=' + text + '&models=content', follow=False) + + self.assertEqual(result.status_code, 200) + response = result.context['object_list'].execute() + self.assertEqual(response.hits.total, 5) + + self.assertTrue(response[0].meta.score > response[1].meta.score > response[2].meta.score) + self.assertEquals(response[0].meta.id, str(published_opinion_not_picked.pk)) # unpicked opinion got first + self.assertEquals(response[1].meta.id, str(published_opinion_picked.pk)) + + settings.ZDS_APP['search']['boosts']['publishedcontent']['if_opinion'] = 1.0 + settings.ZDS_APP['search']['boosts']['publishedcontent']['if_opinion_not_picked'] = 1.0 # 6. Test global boosts # NOTE: score are NOT the same for all documents, no matter how hard it tries to, small differences exists @@ -463,7 +495,7 @@ def test_boosts(self): self.assertEqual(result.status_code, 200) response = result.context['object_list'].execute() - self.assertEqual(response.hits.total, 8) + self.assertEqual(response.hits.total, 10) self.assertEqual(response[0].meta.doc_type, model.get_es_document_type()) # obvious diff --git a/zds/searchv2/views.py b/zds/searchv2/views.py index ef6dc203fb..2b7be44ce2 100644 --- a/zds/searchv2/views.py +++ b/zds/searchv2/views.py @@ -133,6 +133,14 @@ def get_queryset_publishedcontents(self): 'filter': Match(content_type='ARTICLE'), 'weight': settings.ZDS_APP['search']['boosts']['publishedcontent']['if_article'] }, + { + 'filter': Match(content_type='OPINION'), + 'weight': settings.ZDS_APP['search']['boosts']['publishedcontent']['if_opinion'] + }, + { + 'filter': Match(content_type='OPINION') & Match(picked=False), + 'weight': settings.ZDS_APP['search']['boosts']['publishedcontent']['if_opinion_not_picked'] + }, ] scored_query = FunctionScore(query=query, boost_mode='multiply', functions=functions_score) diff --git a/zds/settings.py b/zds/settings.py index 2708e6e876..08d1c3b507 100644 --- a/zds/settings.py +++ b/zds/settings.py @@ -557,6 +557,8 @@ 'global': 3.0, 'if_article': 1.0, 'if_tutorial': 1.0, + 'if_opinion': 0.66, + 'if_opinion_not_picked': 0.5 }, 'topic': { 'global': 2.0, diff --git a/zds/tutorialv2/models/models_database.py b/zds/tutorialv2/models/models_database.py index b5829aed33..d6eb1a6b8c 100644 --- a/zds/tutorialv2/models/models_database.py +++ b/zds/tutorialv2/models/models_database.py @@ -32,7 +32,7 @@ from uuslug import uuslug from elasticsearch_dsl import Mapping, Q as ES_Q -from elasticsearch_dsl.field import Text, Keyword, Date +from elasticsearch_dsl.field import Text, Keyword, Date, Boolean from zds.forum.models import Topic from zds.gallery.models import Image, Gallery, UserGallery @@ -843,6 +843,7 @@ def get_es_mapping(cls): mapping.field('tags', Text(boost=2.0)) mapping.field('categories', Text(boost=2.25)) mapping.field('text', Text()) # for article and mini-tuto, text is directly included into the main object + mapping.field('picked', Boolean()) # not indexed: mapping.field('get_absolute_url_online', Keyword(index=False)) @@ -910,7 +911,7 @@ def get_es_document_source(self, excluded_fields=None): """ excluded_fields = excluded_fields or [] - excluded_fields.extend(['title', 'description', 'tags', 'categories', 'text', 'thumbnail']) + excluded_fields.extend(['title', 'description', 'tags', 'categories', 'text', 'thumbnail', 'picked']) data = super(PublishedContent, self).get_es_document_source(excluded_fields=excluded_fields) @@ -935,6 +936,11 @@ def get_es_document_source(self, excluded_fields=None): if versioned.has_extracts(): data['text'] = versioned.get_content_online() + data['picked'] = False + + if self.content_type == 'OPINION' and self.content.sha_picked is not None: + data['picked'] = True + return data From 22b918fb4117367e5a79455324663aca21020d96 Mon Sep 17 00:00:00 2001 From: pierre-24 Date: Sun, 2 Apr 2017 17:51:27 +0200 Subject: [PATCH 4/6] boost medium and big tutorials over the rest --- zds/searchv2/tests/tests_views.py | 13 +++++++++++++ zds/searchv2/views.py | 4 ++++ zds/settings.py | 1 + zds/tutorialv2/models/models_database.py | 4 ++++ 4 files changed, 22 insertions(+) diff --git a/zds/searchv2/tests/tests_views.py b/zds/searchv2/tests/tests_views.py index fc686a528f..ae2bd07076 100644 --- a/zds/searchv2/tests/tests_views.py +++ b/zds/searchv2/tests/tests_views.py @@ -481,6 +481,19 @@ def test_boosts(self): settings.ZDS_APP['search']['boosts']['publishedcontent']['if_opinion'] = 1.0 settings.ZDS_APP['search']['boosts']['publishedcontent']['if_opinion_not_picked'] = 1.0 + settings.ZDS_APP['search']['boosts']['publishedcontent']['if_medium_or_big_tutorial'] = 2.0 + + result = self.client.get( + reverse('search:query') + '?q=' + text + '&models=content', follow=False) + + self.assertEqual(result.status_code, 200) + response = result.context['object_list'].execute() + self.assertEqual(response.hits.total, 5) + + self.assertTrue(response[0].meta.score > response[1].meta.score) + self.assertEquals(response[0].meta.id, str(published_tuto.pk)) # obvious + + settings.ZDS_APP['search']['boosts']['publishedcontent']['if_medium_or_big_tutorial'] = 1.0 # 6. Test global boosts # NOTE: score are NOT the same for all documents, no matter how hard it tries to, small differences exists diff --git a/zds/searchv2/views.py b/zds/searchv2/views.py index 2b7be44ce2..887f82f628 100644 --- a/zds/searchv2/views.py +++ b/zds/searchv2/views.py @@ -129,6 +129,10 @@ def get_queryset_publishedcontents(self): 'filter': Match(content_type='TUTORIAL'), 'weight': settings.ZDS_APP['search']['boosts']['publishedcontent']['if_tutorial'] }, + { + 'filter': Match(content_type='TUTORIAL') & Match(has_chapters=True), + 'weight': settings.ZDS_APP['search']['boosts']['publishedcontent']['if_medium_or_big_tutorial'] + }, { 'filter': Match(content_type='ARTICLE'), 'weight': settings.ZDS_APP['search']['boosts']['publishedcontent']['if_article'] diff --git a/zds/settings.py b/zds/settings.py index 08d1c3b507..abec394114 100644 --- a/zds/settings.py +++ b/zds/settings.py @@ -557,6 +557,7 @@ 'global': 3.0, 'if_article': 1.0, 'if_tutorial': 1.0, + 'if_medium_or_big_tutorial': 1.5, 'if_opinion': 0.66, 'if_opinion_not_picked': 0.5 }, diff --git a/zds/tutorialv2/models/models_database.py b/zds/tutorialv2/models/models_database.py index d6eb1a6b8c..fc5f4d271b 100644 --- a/zds/tutorialv2/models/models_database.py +++ b/zds/tutorialv2/models/models_database.py @@ -843,6 +843,7 @@ def get_es_mapping(cls): mapping.field('tags', Text(boost=2.0)) mapping.field('categories', Text(boost=2.25)) mapping.field('text', Text()) # for article and mini-tuto, text is directly included into the main object + mapping.field('has_chapters', Boolean()) # ... otherwise, it is written mapping.field('picked', Boolean()) # not indexed: @@ -935,6 +936,9 @@ def get_es_document_source(self, excluded_fields=None): if versioned.has_extracts(): data['text'] = versioned.get_content_online() + data['has_chapters'] = False + else: + data['has_chapters'] = True data['picked'] = False From 83bf3b2b2a8581dd6311d27aad9145d4f23b8a7b Mon Sep 17 00:00:00 2001 From: pierre-24 Date: Sun, 2 Apr 2017 18:17:20 +0200 Subject: [PATCH 5/6] Setup the last details --- templates/searchv2/includes/publishedcontent.part.html | 5 +++++ zds/tutorialv2/views/views_validations.py | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/templates/searchv2/includes/publishedcontent.part.html b/templates/searchv2/includes/publishedcontent.part.html index bf1178df97..0bf51ff353 100644 --- a/templates/searchv2/includes/publishedcontent.part.html +++ b/templates/searchv2/includes/publishedcontent.part.html @@ -30,6 +30,11 @@

{% trans "Article publié" %} {% elif search_result.content_type == 'TUTORIAL' %} {% trans "Tutoriel publié" %} + {% elif search_result.content_type == 'OPINION' %} + {% trans "Opinion" %} + {% if search_result.picked %} + {% trans "mise en avant" %} + {% endif %} {% else %} {% trans "Contenu publié" %} {% endif %} diff --git a/zds/tutorialv2/views/views_validations.py b/zds/tutorialv2/views/views_validations.py index 0d945776b9..45cafb36f0 100644 --- a/zds/tutorialv2/views/views_validations.py +++ b/zds/tutorialv2/views/views_validations.py @@ -671,6 +671,10 @@ def form_valid(self, form): db_object.picked_date = datetime.now() db_object.save() + # mark to reindex to boost correctly in the search + self.public_content_object.es_flagged = True + self.public_content_object.save() + msg = render_to_string( 'tutorialv2/messages/validation_opinion.md', { @@ -726,6 +730,10 @@ def form_valid(self, form): db_object.sha_picked = None db_object.save() + # mark to reindex to boost correctly in the search + self.public_content_object.es_flagged = True + self.public_content_object.save() + msg = render_to_string( 'tutorialv2/messages/validation_invalid_opinion.md', { From 11828c2fa64be87cb760f867d4d6a658fbc05cc2 Mon Sep 17 00:00:00 2001 From: pierre-24 Date: Sun, 2 Apr 2017 20:28:56 +0200 Subject: [PATCH 6/6] comments of @artragis --- templates/searchv2/includes/publishedcontent.part.html | 4 ++-- zds/searchv2/models.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/templates/searchv2/includes/publishedcontent.part.html b/templates/searchv2/includes/publishedcontent.part.html index 0bf51ff353..6a93b0582b 100644 --- a/templates/searchv2/includes/publishedcontent.part.html +++ b/templates/searchv2/includes/publishedcontent.part.html @@ -31,9 +31,9 @@

{% elif search_result.content_type == 'TUTORIAL' %} {% trans "Tutoriel publié" %} {% elif search_result.content_type == 'OPINION' %} - {% trans "Opinion" %} + {% trans "Billet" %} {% if search_result.picked %} - {% trans "mise en avant" %} + {% trans "mis en avant" %} {% endif %} {% else %} {% trans "Contenu publié" %} diff --git a/zds/searchv2/models.py b/zds/searchv2/models.py index 5750000bc2..acfbf12de0 100644 --- a/zds/searchv2/models.py +++ b/zds/searchv2/models.py @@ -246,6 +246,7 @@ def get_django_indexable_objects(): class NeedIndex(Exception): + """Raised when an action requires an index, but it is not created (yet).""" pass