From c7af417e0fd9fe3abdddc699e2c5a758876ddd4e Mon Sep 17 00:00:00 2001 From: Philippe MILINK Date: Sun, 19 Jan 2025 20:50:04 +0100 Subject: [PATCH] =?UTF-8?q?Permet=20de=20chercher=20les=20contenus=20dans?= =?UTF-8?q?=20les=20(sous-)cat=C3=A9gories?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ajoute notamment les (sous-)catégories dans les éléments de la collection "chapter" de Typesense --- zds/search/forms.py | 13 +++- zds/search/tests/tests_views.py | 103 ++++++++++++++++++++++-------- zds/search/views.py | 10 ++- zds/settings/abstract_base/zds.py | 2 + zds/tutorialv2/models/database.py | 48 +++++++++++--- 5 files changed, 138 insertions(+), 38 deletions(-) diff --git a/zds/search/forms.py b/zds/search/forms.py index 1efc2bdc69..691923efa8 100644 --- a/zds/search/forms.py +++ b/zds/search/forms.py @@ -2,12 +2,14 @@ from django import forms from django.conf import settings +from django.urls import reverse from django.utils.translation import gettext_lazy as _ from crispy_forms.bootstrap import StrictButton from crispy_forms.helper import FormHelper from crispy_forms.layout import Layout, Field -from django.urls import reverse + +from zds.utils.models import Category, SubCategory class SearchForm(forms.Form): @@ -28,6 +30,15 @@ class SearchForm(forms.Form): choices=model_choices, ) + category = forms.CharField( # actually the slug of the category + max_length=Category._meta.get_field("slug").max_length, + required=False, + ) + subcategory = forms.CharField( # actually the slug of the subcategory + max_length=SubCategory._meta.get_field("slug").max_length, + required=False, + ) + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/zds/search/tests/tests_views.py b/zds/search/tests/tests_views.py index 982be4de63..2186cfcb0c 100644 --- a/zds/search/tests/tests_views.py +++ b/zds/search/tests/tests_views.py @@ -23,6 +23,7 @@ ) from zds.tutorialv2.models.database import PublishedContent, FakeChapter, PublishableContent from zds.tutorialv2.tests import TutorialTestMixin, override_for_contents +from zds.utils.tests.factories import CategoryFactory overridden_zds_app = deepcopy(settings.ZDS_APP) @@ -41,6 +42,8 @@ def setUp(self): self.category, self.forum = create_category_and_forum() + self.tag = TagFactory(title="Clémentine à pépins") # with accents to make a different slug + self.user = ProfileFactory().user self.staff = StaffProfileFactory().user @@ -56,36 +59,19 @@ def _index_everything(self): continue self.manager.indexing_of_model(model, force_reindexing=True, verbose=False) - def test_basic_search(self): - """Basic search and filtering""" - - if not self.manager.connected: - return - - tag = TagFactory(title="Clémentine à pépins") # with accents to make a different slug - - # 1. Index and test search: - text = "test" - - topic_1 = TopicFactory(forum=self.forum, author=self.user, title=text) - topic_1.tags.add(tag) - post_1 = PostFactory(topic=topic_1, author=self.user, position=1) - post_1.text = post_1.text_html = text - post_1.save() - - # create a middle-size content and publish it + def _create_tutorial(self, text): tuto = PublishableContentFactory(type="TUTORIAL") tuto_draft = tuto.load_version() - tuto.tags.add(tag) + tuto.tags.add(self.tag) tuto.title = text tuto.authors.add(self.user) tuto.save() tuto_draft.repo_update_top_container(text, tuto.slug, text, text) # change title to be sure it will match - chapter1 = ContainerFactory(parent=tuto_draft, db_object=tuto) - extract = ExtractFactory(container=chapter1, db_object=tuto) + chapter = ContainerFactory(parent=tuto_draft, db_object=tuto) + extract = ExtractFactory(container=chapter, db_object=tuto) extract.repo_update(text, text) published = publish_content(tuto, tuto_draft, is_major_update=True) @@ -95,6 +81,25 @@ def test_basic_search(self): tuto.public_version = published tuto.save() + return tuto, chapter + + def test_basic_search(self): + """Basic search and filtering""" + + if not self.manager.connected: + return + + text = "test" + + topic_1 = TopicFactory(forum=self.forum, author=self.user, title=text) + topic_1.tags.add(self.tag) + post_1 = PostFactory(topic=topic_1, author=self.user, position=1) + post_1.text = post_1.text_html = text + post_1.save() + + # create a middle-size content and publish it + tuto1, chapter1 = self._create_tutorial(text) + # nothing has been indexed yet: results = self.manager.search("*") number_of_results = sum(result["found"] for result in results) @@ -116,13 +121,13 @@ def test_basic_search(self): # may contain or not these tags. content_search_results = result.content.decode()[result.content.decode().find("search-results") :] # The tag appears 2 times: in two search results - self.assertEqual(content_search_results.count(tag.title), 2) - self.assertEqual(content_search_results.count(tag.slug), 2) + self.assertEqual(content_search_results.count(self.tag.title), 2) + self.assertEqual(content_search_results.count(self.tag.slug), 2) - # 2. Test filtering: + # Test filtering: topic_1 = Topic.objects.get(pk=topic_1.pk) post_1 = Post.objects.get(pk=post_1.pk) - published = PublishedContent.objects.get(pk=published.pk) + published = PublishedContent.objects.get(pk=tuto1.public_version.pk) ids = { "topic": [topic_1.search_engine_id], @@ -146,7 +151,53 @@ def test_basic_search(self): result = self.client.get(reverse("search:query") + "?q=-c", follow=False) self.assertEqual(result.status_code, 200) - def test_search_many_pages(self): + def test_search_category_filter(self): + """Search in published contents of a specific category (form on the page of a category of contents)""" + if not self.manager.connected: + return + + text = "test" + + cat1 = CategoryFactory() + subcat11 = SubCategoryFactory(category=cat1) + subcat12 = SubCategoryFactory(category=cat1) + cat2 = CategoryFactory() + subcat21 = SubCategoryFactory(category=cat2) + + tuto1, _ = self._create_tutorial(text) + tuto1.subcategory.add(subcat11) + tuto1.save() + + tuto2, _ = self._create_tutorial(text) + tuto2.subcategory.add(subcat12) + tuto2.save() + + tuto3, _ = self._create_tutorial(text) + tuto3.subcategory.add(subcat21) + tuto3.save() + + # index + self._index_everything() + + # no filter on (sub)categories + result = self.client.get(reverse("search:query") + "?q=" + text, follow=False) + self.assertEqual(result.status_code, 200) + response = result.context["object_list"] + self.assertEqual(len(response), 6) # get 6 results (3 tutorials and each tutorial has one chapter) + + # filter on categories + result = self.client.get(reverse("search:query") + "?q=" + text + "&category=" + cat1.slug, follow=False) + self.assertEqual(result.status_code, 200) + response = result.context["object_list"] + self.assertEqual(len(response), 4) # get 4 results (2 tutorials and each tutorial has one chapter) + + # filter on subcategories + result = self.client.get(reverse("search:query") + "?q=" + text + "&subcategory=" + subcat11.slug, follow=False) + self.assertEqual(result.status_code, 200) + response = result.context["object_list"] + self.assertEqual(len(response), 2) # get 2 results (1 tutorial and with one chapter) + + def test_search_pagination_of_results(self): if not self.manager.connected: return diff --git a/zds/search/views.py b/zds/search/views.py index 9337759aab..cf5a954242 100644 --- a/zds/search/views.py +++ b/zds/search/views.py @@ -155,8 +155,14 @@ def get_queryset(self): search_collections = self.search_form.cleaned_data["search_collections"] searches = { - "publishedcontent": PublishedContent.get_search_query(), - "chapter": FakeChapter.get_search_query(), + "publishedcontent": PublishedContent.get_search_query( + category_slug=self.search_form.cleaned_data["category"], + subcategory_slug=self.search_form.cleaned_data["subcategory"], + ), + "chapter": FakeChapter.get_search_query( + category_slug=self.search_form.cleaned_data["category"], + subcategory_slug=self.search_form.cleaned_data["subcategory"], + ), "topic": Topic.get_search_query(self.request.user), "post": Post.get_search_query(self.request.user), } diff --git a/zds/settings/abstract_base/zds.py b/zds/settings/abstract_base/zds.py index cc9bcbbfdb..ae64c39e5f 100644 --- a/zds/settings/abstract_base/zds.py +++ b/zds/settings/abstract_base/zds.py @@ -267,6 +267,8 @@ "chapter": { "global": global_weight_chapter, "title": global_weight_chapter * 3, + "categories": global_weight_chapter * 1, + "subcategories": global_weight_chapter * 1, "text": global_weight_chapter * 2, }, "topic": { diff --git a/zds/tutorialv2/models/database.py b/zds/tutorialv2/models/database.py index f5355af41c..41e215947b 100644 --- a/zds/tutorialv2/models/database.py +++ b/zds/tutorialv2/models/database.py @@ -1037,8 +1037,8 @@ def get_search_document_schema(cls): {"name": "publication_date", "type": "int64", "index": False}, {"name": "tags", "type": "string[]", "facet": True, "optional": True}, # we search on it {"name": "tag_slugs", "type": "string[]", "index": False, "optional": True}, - {"name": "subcategories", "type": "string[]", "facet": True, "optional": True}, # we search on it - {"name": "categories", "type": "string[]", "facet": True, "optional": True}, # we search on it + {"name": "subcategories", "type": "string[]", "facet": True, "optional": True}, # slugs; we search on it + {"name": "categories", "type": "string[]", "facet": True, "optional": True}, # slugs; we search on it {"name": "text", "type": "string", "facet": False, "optional": True}, # we search on it {"name": "description", "type": "string", "facet": False, "optional": True}, # we search on it {"name": "get_absolute_url_online", "type": "string", "index": False}, @@ -1103,7 +1103,9 @@ def get_indexable(cls, force_reindexing=False): def get_document_source(self, excluded_fields=[]): """Overridden to handle the fact that most information are versioned""" - excluded_fields.extend(["title", "description", "tags", "categories", "text", "thumbnail", "publication_date"]) + excluded_fields.extend( + ["title", "description", "tags", "categories", "subcategories", "text", "thumbnail", "publication_date"] + ) data = super().get_document_source(excluded_fields=excluded_fields) @@ -1161,8 +1163,8 @@ def _get_search_weight(self, is_multipage: bool): return weights["if_opinion_not_picked"] @classmethod - def get_search_query(cls): - return { + def get_search_query(cls, category_slug=None, subcategory_slug=None): + ret = { "query_by": "title,description,categories,subcategories,tags,text", "query_by_weights": "{},{},{},{},{},{}".format( settings.ZDS_APP["search"]["boosts"]["publishedcontent"]["title"], @@ -1175,6 +1177,18 @@ def get_search_query(cls): "sort_by": "weight:desc", } + filter_by = SearchFilter() + + if category_slug is not None and len(category_slug.strip()) != 0: + filter_by.add_exact_filter("categories", [category_slug]) + if subcategory_slug is not None and len(subcategory_slug.strip()) != 0: + filter_by.add_exact_filter("subcategories", [subcategory_slug]) + + if str(filter_by) != "": + ret["filter_by"] = str(filter_by) + + return ret + @receiver(pre_delete, sender=PublishedContent) def delete_published_content_in_search_engine(sender, instance, **kwargs): @@ -1268,6 +1282,8 @@ def get_search_document_schema(self): {"name": "parent_get_absolute_url_online", "type": "string", "index": False}, {"name": "thumbnail", "type": "string", "index": False}, {"name": "weight", "type": "float", "facet": False}, # we sort on it + {"name": "subcategories", "type": "string[]", "facet": True, "optional": True}, # slugs; we search on it + {"name": "categories", "type": "string[]", "facet": True, "optional": True}, # slugs; we search on it ] return search_engine_schema @@ -1285,16 +1301,30 @@ def get_document_source(self, excluded_fields=[]): return data @classmethod - def get_search_query(cls): - return { - "query_by": "title,text", - "query_by_weights": "{},{}".format( + def get_search_query(cls, category_slug=None, subcategory_slug=None): + ret = { + "query_by": "title,categories,subcategories,text", + "query_by_weights": "{},{},{},{}".format( settings.ZDS_APP["search"]["boosts"]["chapter"]["title"], + settings.ZDS_APP["search"]["boosts"]["chapter"]["categories"], + settings.ZDS_APP["search"]["boosts"]["chapter"]["subcategories"], settings.ZDS_APP["search"]["boosts"]["chapter"]["text"], ), "sort_by": "weight:desc", } + filter_by = SearchFilter() + + if category_slug is not None and len(category_slug.strip()) != 0: + filter_by.add_exact_filter("categories", [category_slug]) + if subcategory_slug is not None and len(subcategory_slug.strip()) != 0: + filter_by.add_exact_filter("subcategories", [subcategory_slug]) + + if str(filter_by) != "": + ret["filter_by"] = str(filter_by) + + return ret + @classmethod def remove_from_search_engine(cls, search_engine_manager: SearchIndexManager, parent_search_engine_id: int): filter_by = SearchFilter()