From 8eda7d10e183381806a48b11b75ba3f495fc8b0b Mon Sep 17 00:00:00 2001 From: gjuro87 Date: Tue, 25 May 2021 15:50:26 +0200 Subject: [PATCH] feat: zoo search (#392) * feat: zoo search * refactor: CR fixes * refactor: rename meilisearch to globalsearch app Co-authored-by: Jurica Grgicevic --- .../{meilisearch => globalsearch}/__init__.py | 0 .../test_indexer.py | 2 +- .../test_search.py | 2 +- .../test_utils.py | 0 zoo/api/query.py | 87 +++++++++++++++ zoo/api/types.py | 16 +++ zoo/base/apps.py | 2 +- zoo/base/assets/js/header_search.js | 32 ++++++ zoo/base/assets/js/project_list.js | 21 ---- zoo/base/settings.py | 2 +- zoo/base/templates/base.html | 23 ++++ zoo/base/templates/shared/base_list.html | 98 +++++++++++++++++ zoo/base/templates/shared/project_list.html | 101 +----------------- zoo/base/urls.py | 2 +- zoo/{meilisearch => globalsearch}/__init__.py | 0 zoo/globalsearch/apps.py | 5 + zoo/{meilisearch => globalsearch}/indexer.py | 19 +++- .../management/__init__.py | 0 .../management/commands/__init__.py | 0 .../commands/create_initial_indexes.py | 72 +++++++++++++ .../commands/index_db_model_instances.py | 10 ++ .../commands/index_openapi_definitions.py | 10 ++ zoo/globalsearch/meili_client.py | 4 + .../migrations/__init__.py | 0 zoo/{meilisearch => globalsearch}/tasks.py | 2 +- zoo/globalsearch/templates/dependency.html | 41 +++++++ .../templates/search_overview.html | 92 ++++++++++++++++ zoo/globalsearch/urls.py | 5 + zoo/globalsearch/views.py | 98 +++++++++++++++++ .../templates/libraries/library_list.html | 8 +- zoo/libraries/views.py | 6 ++ zoo/meilisearch/apps.py | 5 - .../commands/create_initial_indexes.py | 19 ---- .../templates/search_overview.html | 16 --- zoo/meilisearch/urls.py | 5 - zoo/meilisearch/views.py | 48 --------- .../templates/dependency_overview.html | 3 +- .../templates/services/service_list.html | 10 +- zoo/services/views.py | 12 +++ 39 files changed, 640 insertions(+), 238 deletions(-) rename test/{meilisearch => globalsearch}/__init__.py (100%) rename test/{meilisearch => globalsearch}/test_indexer.py (94%) rename test/{meilisearch => globalsearch}/test_search.py (91%) rename test/{meilisearch => globalsearch}/test_utils.py (100%) create mode 100644 zoo/base/assets/js/header_search.js create mode 100644 zoo/base/templates/shared/base_list.html rename zoo/{meilisearch => globalsearch}/__init__.py (100%) create mode 100644 zoo/globalsearch/apps.py rename zoo/{meilisearch => globalsearch}/indexer.py (78%) rename zoo/{meilisearch => globalsearch}/management/__init__.py (100%) rename zoo/{meilisearch => globalsearch}/management/commands/__init__.py (100%) create mode 100644 zoo/globalsearch/management/commands/create_initial_indexes.py create mode 100644 zoo/globalsearch/management/commands/index_db_model_instances.py create mode 100644 zoo/globalsearch/management/commands/index_openapi_definitions.py create mode 100644 zoo/globalsearch/meili_client.py rename zoo/{meilisearch => globalsearch}/migrations/__init__.py (100%) rename zoo/{meilisearch => globalsearch}/tasks.py (84%) create mode 100644 zoo/globalsearch/templates/dependency.html create mode 100644 zoo/globalsearch/templates/search_overview.html create mode 100644 zoo/globalsearch/urls.py create mode 100644 zoo/globalsearch/views.py delete mode 100644 zoo/meilisearch/apps.py delete mode 100644 zoo/meilisearch/management/commands/create_initial_indexes.py delete mode 100644 zoo/meilisearch/templates/search_overview.html delete mode 100644 zoo/meilisearch/urls.py delete mode 100644 zoo/meilisearch/views.py diff --git a/test/meilisearch/__init__.py b/test/globalsearch/__init__.py similarity index 100% rename from test/meilisearch/__init__.py rename to test/globalsearch/__init__.py diff --git a/test/meilisearch/test_indexer.py b/test/globalsearch/test_indexer.py similarity index 94% rename from test/meilisearch/test_indexer.py rename to test/globalsearch/test_indexer.py index 567fba16..0f4e5034 100644 --- a/test/meilisearch/test_indexer.py +++ b/test/globalsearch/test_indexer.py @@ -3,7 +3,7 @@ import pytest from zoo.analytics.models import Dependency, DependencyType -from zoo.meilisearch.indexer import Indexer as uut +from zoo.globalsearch.indexer import Indexer as uut from zoo.services.models import Service pytestmark = pytest.mark.django_db diff --git a/test/meilisearch/test_search.py b/test/globalsearch/test_search.py similarity index 91% rename from test/meilisearch/test_search.py rename to test/globalsearch/test_search.py index 68bdf401..2351d5b5 100644 --- a/test/meilisearch/test_search.py +++ b/test/globalsearch/test_search.py @@ -25,6 +25,6 @@ def test_search_result(): "Service": [service], "Schema": [], } - with patch("zoo.meilisearch.views.MeiliSearchView") as mock_view: + with patch("zoo.globalsearch.views.GlobalSearchView") as mock_view: mock_view.get_context_data.return_value = expected_result assert expected_result == mock_view.get_context_data() diff --git a/test/meilisearch/test_utils.py b/test/globalsearch/test_utils.py similarity index 100% rename from test/meilisearch/test_utils.py rename to test/globalsearch/test_utils.py diff --git a/zoo/api/query.py b/zoo/api/query.py index b37ba48d..32b734f0 100644 --- a/zoo/api/query.py +++ b/zoo/api/query.py @@ -1,14 +1,20 @@ +from math import ceil + import graphene +from django.apps import apps from graphene import relay from ..analytics.models import Dependency, DependencyType, DependencyUsage from ..auditing.models import Issue +from ..globalsearch.indexer import IndexType +from ..globalsearch.meili_client import meili_client from ..repos.models import Repository from ..services.models import Service from . import types from .paginator import Paginator DependencyTypeEnum = graphene.Enum.from_enum(DependencyType) +SearchTypeEnum = graphene.Enum.from_enum(IndexType) class Query(graphene.ObjectType): @@ -36,6 +42,13 @@ class Query(graphene.ObjectType): description="List of dependency usages. Returns first 10 nodes if pagination is not specified.", ) + all_search_results = relay.ConnectionField( + types.SearchResultsConnection, + description="List of search results. Returns first 10 nodes if pagination is not specified.", + search_query=graphene.String(), + search_type=SearchTypeEnum(), + ) + def resolve_all_issues(self, info, **kwargs): paginator = Paginator(**kwargs) total = Issue.objects.all().count() @@ -142,3 +155,77 @@ def resolve_all_dependency_usages(self, info, **kwargs): return types.DependencyUsageConnection( page_info=page_info, edges=edges, total_count=total ) + + def resolve_all_search_results(self, info, **kwargs): + paginator = Paginator(**kwargs) + query_param = kwargs.get("search_query", "") + search_type = kwargs.get("search_type", IndexType.Service.value) + + indexes = meili_client.get_indexes() + # get all total counts for each meili index + search_index = None + total_count = {} + for index in indexes: + total_count[index["uid"]] = meili_client.get_index(index["uid"]).search( + query=query_param, opt_params={"offset": 0, "limit": 1} + )["nbHits"] + + if index["uid"] == search_type: + search_index = index + + # get all results for requested index + search_results = Query._get_all_for_index( + query_param, search_index, total_count.get(search_index["uid"]) + ) + edges = [] + for i, search_result in enumerate( + search_results[paginator.slice_from : paginator.slice_to] + ): + cursor = paginator.get_edge_cursor(i + 1) + edges.append( + types.SearchResultsConnection.Edge(node=search_result, cursor=cursor) + ) + + return types.SearchResultsConnection( + page_info=paginator.get_page_info(total_count[search_type]), + edges=edges, + total_services_count=total_count[IndexType.Service.value], + total_analytics_count=total_count[IndexType.Dependency.value], + ) + + @staticmethod + def _get_all_for_index(query_param, search_index, total_count): + limit, search_results = 100, [] + + if not any([total_count, search_index]): + return search_results + + pages = ceil(total_count / limit) + for page in range(pages): + results = meili_client.get_index(search_index["uid"]).search( + query=query_param, opt_params={"offset": page * limit, "limit": limit} + ) + search_results.extend( + Query._objects_from_result(results["hits"], search_index) + ) + + return search_results + + @staticmethod + def _objects_from_result(search_results, index): + result_objects = [] + try: + model = apps.get_model(index["uid"], index["name"]) + for i, result in enumerate(search_results): + model_object = model.objects.get(pk=result["id"]) + if index["name"] == IndexType.Dependency.name: + dependency = types.Dependency.from_db(model_object) + node = types.SearchResult(id=i, dependency=dependency) + if index["name"] == IndexType.Service.name: + service = types.Service.from_db(model_object) + node = types.SearchResult(id=i, service=service) + result_objects.append(node) + except LookupError: + pass + + return result_objects diff --git a/zoo/api/types.py b/zoo/api/types.py index 48a487ad..4b31e3ab 100644 --- a/zoo/api/types.py +++ b/zoo/api/types.py @@ -430,3 +430,19 @@ def resolve_title(self, info): def resolve_description(self, info): return self.kind.format_description(self.details) + + +class SearchResult(graphene.ObjectType): + dependency = graphene.Field(Dependency) + service = graphene.Field(Service) + + class Meta: + interfaces = (relay.Node,) + + +class SearchResultsConnection(relay.Connection): + total_analytics_count = graphene.Int() + total_services_count = graphene.Int() + + class Meta: + node = SearchResult diff --git a/zoo/base/apps.py b/zoo/base/apps.py index 36f0a243..68dff2b3 100644 --- a/zoo/base/apps.py +++ b/zoo/base/apps.py @@ -15,7 +15,7 @@ def ready(self): from ..analytics import tasks as analytics_tasks from ..auditing import tasks as auditing_tasks from ..datacenters import tasks as datacenters_tasks - from ..meilisearch import tasks as meilisearch_tasks + from ..globalsearch import tasks as meilisearch_tasks from ..objectives import tasks as objective_tasks from ..repos import tasks as repos_tasks from ..services import tasks as service_tasks diff --git a/zoo/base/assets/js/header_search.js b/zoo/base/assets/js/header_search.js new file mode 100644 index 00000000..03dec552 --- /dev/null +++ b/zoo/base/assets/js/header_search.js @@ -0,0 +1,32 @@ +$(document).ready(function() { + const URI = require('urijs') + const searchInput = $('.ui.search input') + const enterKeyCode = 13 + const slashKeyCode = 47 + const currentUrl = URI(window.location.href) + const currentQuery = URI.parseQuery(currentUrl.query()).q + + $(document).keypress(function(event) { + if(event.which == slashKeyCode) { + event.preventDefault(); + searchInput.focus() + searchInput.selectionStart = searchInput.selectionEnd = searchInput.value.length + } + }); + + searchInput.keypress((event) => { + if(event.which === enterKeyCode) { + let newUrl = currentUrl.pathname('search') + newUrl.search("") + + if(searchInput.val()) + newUrl.addSearch(URI.parseQuery('?q=' + searchInput.val())) + window.location.href = newUrl.toString() + } + }) + + if(currentQuery) { + searchInput.val(currentQuery) + } + +}); diff --git a/zoo/base/assets/js/project_list.js b/zoo/base/assets/js/project_list.js index 1d26d839..5ad70a50 100644 --- a/zoo/base/assets/js/project_list.js +++ b/zoo/base/assets/js/project_list.js @@ -1,25 +1,4 @@ import '../style/project_list.less' -const URI = require('urijs') -const searchInput = $('.ui.search input') -const enterKeyCode = 13 - -const currentUrl = URI(window.location.href) -const currentQuery = URI.parseQuery(currentUrl.query()).q - -searchInput.keypress((event) => { - if(event.which === enterKeyCode) { - let newUrl = URI(currentUrl.path()) - - if(searchInput.val()) - newUrl.addSearch(URI.parseQuery('?q=' + searchInput.val())) - - window.location.href = newUrl.toString() - } -}) - -if(currentQuery) { - searchInput.val(currentQuery) -} $('.project-filter .ui.basic.label').popup({ transition: 'fade down', diff --git a/zoo/base/settings.py b/zoo/base/settings.py index 132feaaf..f19318d4 100644 --- a/zoo/base/settings.py +++ b/zoo/base/settings.py @@ -107,7 +107,7 @@ "zoo.datacenters.apps.DatacentersConfig", "zoo.instance.apps.InstanceConfig", "zoo.libraries.apps.LibrariesConfig", - "zoo.meilisearch.apps.MeiliSearchConfig", + "zoo.globalsearch.apps.GlobalSearchConfig", "zoo.objectives.apps.ObjectivesConfig", "zoo.pagerduty.apps.PagerdutyConfig", "zoo.repos.apps.ReposConfig", diff --git a/zoo/base/templates/base.html b/zoo/base/templates/base.html index e6b02299..424710cc 100644 --- a/zoo/base/templates/base.html +++ b/zoo/base/templates/base.html @@ -1,4 +1,5 @@ {% extends 'skeleton.html' %} +{% load static %} {% block body %}
@@ -26,6 +27,24 @@ Objectives + {% if selected == 'search_overview' %} + + + Search Results + + {% endif %} + {% block search_form %} +
+
+
+ + +
+
+
+
+
+ {% endblock %} {% endblock %} + +{% block scripts %} + +{% endblock %} diff --git a/zoo/base/templates/shared/base_list.html b/zoo/base/templates/shared/base_list.html new file mode 100644 index 00000000..50400215 --- /dev/null +++ b/zoo/base/templates/shared/base_list.html @@ -0,0 +1,98 @@ +{% load utils %} + + {% for object in object_list %} +
+

+ + {{ object.name }} + + · + owned by + {% if object.owner_url %} + {{ object.owner }} + {% else %} + {{ object.owner }} + {% endif %} + + + + {% spaceless %} +
+ {% if object.status %} + status:{{ object.status }} + {% endif %} + {% if object.impact %} + impact:{{ object.impact }} + {% endif %} +
+ {% endspaceless %} + +

+
+ {% if 'Support' in project_links %} + {% project_link 'Support' object.slack_url icon='comment alternate' %} + {% endif %} + {% if 'Repository' in project_links %} + {% project_link 'Repository' object.gitlab_url icon='code' %} + {% endif %} + {% if 'Dashboard' in project_links %} + {% project_link 'Dashboard' object.dashboard_url icon='chart pie' %} + {% endif %} + {% if 'Alerts' in project_links %} + {% project_link 'Alerts' object.pagerduty_url icon='warning' %} + {% endif %} + {% if 'Documentation' in project_links %} + {% project_link 'Documentation' object.docs_url icon='graduation cap' %} + {% endif %} +
+
+ {% empty %} +
+
+
+ +
+
+ Oops! We couldn't find your thing... +
+ Make sure the query is correct or add the project if needed +
+
+
+
+ Add project +
+
+ {% endfor %} + {% if is_paginated %} +
+ +
+ {% endif %} +{% if project_create_view %} + + + +{% endif %} diff --git a/zoo/base/templates/shared/project_list.html b/zoo/base/templates/shared/project_list.html index 6795b2d5..cec609e0 100644 --- a/zoo/base/templates/shared/project_list.html +++ b/zoo/base/templates/shared/project_list.html @@ -2,109 +2,18 @@ {% load static %} {% load utils %} -{% block navtitle %} - {{ block.super }} -
-
-
- -
- - -
-
-
-{% endblock %} {% block stylesheets %} {% endblock %} {% block content %} -{% for object in object_list %} -
-

- - {{ object.name }} - - · - owned by - {% if object.owner_url %} - {{ object.owner }} - {% else %} - {{ object.owner }} - {% endif %} - - - - {% spaceless %} -
- {% if object.status %} - status:{{ object.status }} - {% endif %} - {% if object.impact %} - impact:{{ object.impact }} - {% endif %} -
- {% endspaceless %} - -

-
- {% block project_links %} - {% endblock %} -
-
-{% empty %} -
-
-
- -
-
- Oops! We couldn't find your thing... -
- Make sure the query is correct or add the project if needed -
-
-
-
- Add project -
-
-{% endfor %} -{% if is_paginated %} -
- -
-{% endif %} - - - - + {% with object_list=object_list project_create_view=project_create_view is_paginated=is_paginated page_obj=page_obj request=request project_links=project_links %} + {% include 'shared/base_list.html' %} + {% endwith %} {% endblock %} {% block scripts %} - + {{ block.super }} + {% endblock %} diff --git a/zoo/base/urls.py b/zoo/base/urls.py index 39706edf..1abb8556 100644 --- a/zoo/base/urls.py +++ b/zoo/base/urls.py @@ -30,7 +30,7 @@ path("pagerduty/", include("zoo.pagerduty.urls")), path("", include("zoo.auditing.urls")), path("", include("zoo.checklists.urls")), - path("", include("zoo.meilisearch.urls")), + path("", include("zoo.globalsearch.urls")), path("objectives/", include("zoo.objectives.urls")), path("analytics/", include("zoo.analytics.urls")), path("resources/", include("zoo.resources.urls")), diff --git a/zoo/meilisearch/__init__.py b/zoo/globalsearch/__init__.py similarity index 100% rename from zoo/meilisearch/__init__.py rename to zoo/globalsearch/__init__.py diff --git a/zoo/globalsearch/apps.py b/zoo/globalsearch/apps.py new file mode 100644 index 00000000..b1ba4f88 --- /dev/null +++ b/zoo/globalsearch/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class GlobalSearchConfig(AppConfig): + name = "zoo.globalsearch" diff --git a/zoo/meilisearch/indexer.py b/zoo/globalsearch/indexer.py similarity index 78% rename from zoo/meilisearch/indexer.py rename to zoo/globalsearch/indexer.py index f4d6fac8..8771dc86 100644 --- a/zoo/meilisearch/indexer.py +++ b/zoo/globalsearch/indexer.py @@ -1,4 +1,5 @@ import json +from enum import Enum import meilisearch import structlog @@ -12,12 +13,20 @@ log = structlog.get_logger() +class IndexType(Enum): + Service = "services" + Dependency = "analytics" + + class Indexer: def __init__(self): self.meiliclient = meilisearch.Client( settings.MEILI_HOST, settings.MEILI_MASTER_KEY ) - self.models_to_index = [(Service, "services"), (Dependency, "analytics")] + self.models_to_index = [ + (Service, IndexType.Service.value), + (Dependency, IndexType.Dependency.value), + ] def index_specified_models(self): for model, index_name in self.models_to_index: @@ -27,9 +36,9 @@ def index_specified_models(self): serialized_model_instance["fields"][ "id" ] = serialized_model_instance["pk"] - self.meiliclient.get_index(index_name).update_documents( - [serialized_model_instance["fields"]] - ) + self.meiliclient.get_or_create_index( + index_name, {"name": model.__name__} + ).update_documents([serialized_model_instance["fields"]]) # deepcode ignore W0703: Multiple Possible Exceptions except Exception as err: log.info( @@ -47,7 +56,7 @@ def index_openapi(self): json_definition = json.loads(definition) try: json_definition["id"] = key - self.meiliclient.get_index("open-api").update_documents( + self.meiliclient.get_or_create_index("open-api").update_documents( [{json_definition}] ) # deepcode ignore W0703: Multiple Possible Exceptions diff --git a/zoo/meilisearch/management/__init__.py b/zoo/globalsearch/management/__init__.py similarity index 100% rename from zoo/meilisearch/management/__init__.py rename to zoo/globalsearch/management/__init__.py diff --git a/zoo/meilisearch/management/commands/__init__.py b/zoo/globalsearch/management/commands/__init__.py similarity index 100% rename from zoo/meilisearch/management/commands/__init__.py rename to zoo/globalsearch/management/commands/__init__.py diff --git a/zoo/globalsearch/management/commands/create_initial_indexes.py b/zoo/globalsearch/management/commands/create_initial_indexes.py new file mode 100644 index 00000000..dfb4eea3 --- /dev/null +++ b/zoo/globalsearch/management/commands/create_initial_indexes.py @@ -0,0 +1,72 @@ +import meilisearch +import structlog +from django.conf import settings +from django.core.management.base import BaseCommand + +log = structlog.get_logger() + + +class Command(BaseCommand): + help = "Create initial MeiliSearch indexes" + + def add_arguments(self, parser): + # Optional argument + parser.add_argument( + "-d", "--delete", action="store_true", help="Delete indexes" + ) + parser.add_argument( + "-r", "--recreate", action="store_true", help="Recreate indexes" + ) + + def handle(self, *args, **options): + delete = options["delete"] + recreate = options["recreate"] + + client = meilisearch.Client(settings.MEILI_HOST, settings.MEILI_MASTER_KEY) + + if delete: + Command._delete_indexes(client) + return + + if recreate: + Command._delete_indexes(client) + Command._create_indexes(client) + return + + Command._create_indexes(client) + + @staticmethod + def _create_indexes(client): + try: + client.create_index( + uid="services", options={"name": "Service", "primaryKey": "id"} + ) + log.info("globalsearch.commands.create_indexes.services.success") + client.create_index( + uid="analytics", options={"name": "Dependency", "primaryKey": "id"} + ) + log.info("globalsearch.commands.create_indexes.analytics.success") + client.create_index( + uid="open-api", options={"name": "Schema", "primaryKey": "id"} + ) + log.info("globalsearch.commands.create_indexes.open-api.success") + except meilisearch.errors.MeiliSearchApiError as e: + log.error("globalsearch.commands.create_indexes.error") + log.error(e) + return + log.info("globalsearch.commands.create_indexes.success") + + @staticmethod + def _delete_indexes(client): + try: + client.index("services").delete() + log.info("globalsearch.commands.delete_indexes.services.success") + client.index("analytics").delete() + log.info("globalsearch.commands.delete_indexes.analytics.success") + client.index("open-api").delete() + log.info("globalsearch.commands.delete_indexes.open-api.success") + except meilisearch.errors.MeiliSearchApiError as e: + log.error("globalsearch.commands.delete_indexes.success.error") + log.error(e) + return + log.info("globalsearch.commands.delete_indexes.success") diff --git a/zoo/globalsearch/management/commands/index_db_model_instances.py b/zoo/globalsearch/management/commands/index_db_model_instances.py new file mode 100644 index 00000000..67de891e --- /dev/null +++ b/zoo/globalsearch/management/commands/index_db_model_instances.py @@ -0,0 +1,10 @@ +from django.core.management.base import BaseCommand + +from zoo.globalsearch.tasks import index_db_model_instances + + +class Command(BaseCommand): + help = "Indexes database model instances" + + def handle(self, *args, **options): + index_db_model_instances() diff --git a/zoo/globalsearch/management/commands/index_openapi_definitions.py b/zoo/globalsearch/management/commands/index_openapi_definitions.py new file mode 100644 index 00000000..01325749 --- /dev/null +++ b/zoo/globalsearch/management/commands/index_openapi_definitions.py @@ -0,0 +1,10 @@ +from django.core.management.base import BaseCommand + +from zoo.globalsearch.tasks import index_openapi_definitions + + +class Command(BaseCommand): + help = "Indexes open api defintions" + + def handle(self, *args, **options): + index_openapi_definitions() diff --git a/zoo/globalsearch/meili_client.py b/zoo/globalsearch/meili_client.py new file mode 100644 index 00000000..56875bcf --- /dev/null +++ b/zoo/globalsearch/meili_client.py @@ -0,0 +1,4 @@ +import meilisearch +from django.conf import settings + +meili_client = meilisearch.Client(settings.MEILI_HOST, settings.MEILI_MASTER_KEY) diff --git a/zoo/meilisearch/migrations/__init__.py b/zoo/globalsearch/migrations/__init__.py similarity index 100% rename from zoo/meilisearch/migrations/__init__.py rename to zoo/globalsearch/migrations/__init__.py diff --git a/zoo/meilisearch/tasks.py b/zoo/globalsearch/tasks.py similarity index 84% rename from zoo/meilisearch/tasks.py rename to zoo/globalsearch/tasks.py index c29a2fa1..3ee963f0 100644 --- a/zoo/meilisearch/tasks.py +++ b/zoo/globalsearch/tasks.py @@ -1,6 +1,6 @@ from celery import shared_task -from zoo.meilisearch.indexer import Indexer +from zoo.globalsearch.indexer import Indexer @shared_task diff --git a/zoo/globalsearch/templates/dependency.html b/zoo/globalsearch/templates/dependency.html new file mode 100644 index 00000000..0e0b868a --- /dev/null +++ b/zoo/globalsearch/templates/dependency.html @@ -0,0 +1,41 @@ +{% load utils %} + + + + + + + + + + + + + {% for object in dependencies %} + {% url 'dependency_detail' object.id as detail_url %} + + + + {% with object.depusage|dependency_versions:1 as version %} + + {% endwith %} + + + + {% endfor %} + +
UsageNameLatest versionKindType
{{ object.depusage.all.count }}{{ object.name }}{% if version.0.0 %}{{ version.0.0 }}{% endif %} + {% if "Library" in object.type %} +
+ {% if "kiwi-" in object.name %} + Internal + {% else %} + External + {% endif %} +
+ {% endif %} + 		+
+ {{ object.type }} +
+
diff --git a/zoo/globalsearch/templates/search_overview.html b/zoo/globalsearch/templates/search_overview.html new file mode 100644 index 00000000..b4bb1440 --- /dev/null +++ b/zoo/globalsearch/templates/search_overview.html @@ -0,0 +1,92 @@ +{% extends 'base.html' %} +{% load static %} +{% load utils %} + + +{% block stylesheets %} + +{% endblock %} + +{% block menu %} + {% with selected='search_overview' %} + {{ block.super }} + {% endwith %} +{% endblock %} + +{% block navtitle %} + {% with icon='search' header='Search Results' %} + {{ block.super }} + {% endwith %} +{% endblock %} + +{% block content %} + {% if not total_service and not total_dependency %} +
+
+
+ +
+
+ Oops! We couldn't find your thing... +
+ Make sure the query is correct +
+
+
+
+
+
+ {% else %} + + + {% if service %} + {% with object_list=service project_links=project_links %} + {% include 'shared/base_list.html' %} + {% endwith %} + {% endif %} + + {% if dependency %} + {% include 'dependency.html' with dependencies=dependency %} + {% endif %} + +
+ +
+ {% endif %} +{% endblock %} + +{% block scripts %} + {{ block.super }} + +{% endblock %} diff --git a/zoo/globalsearch/urls.py b/zoo/globalsearch/urls.py new file mode 100644 index 00000000..d583711c --- /dev/null +++ b/zoo/globalsearch/urls.py @@ -0,0 +1,5 @@ +from django.urls import path + +from . import views + +urlpatterns = [path("search", views.GlobalSearchView.as_view(), name="search_overview")] diff --git a/zoo/globalsearch/views.py b/zoo/globalsearch/views.py new file mode 100644 index 00000000..965947a8 --- /dev/null +++ b/zoo/globalsearch/views.py @@ -0,0 +1,98 @@ +from math import ceil + +import structlog +from django.apps import apps +from django.views.generic import TemplateView + +from .meili_client import meili_client + +log = structlog.get_logger() + + +class GlobalSearchView(TemplateView): + template_name = "search_overview.html" + context_object_name = "context" + meili_limit = 20 + + @staticmethod + def _objects_from_result(search_results, index, result_objects=None): + try: + model = apps.get_model(index["uid"], index["name"]) + for result in search_results: + key = index["name"].lower() + if key not in result_objects.keys(): + result_objects[key] = [] + result_objects[key].append(model.objects.get(pk=result["id"])) + except LookupError: + for result in search_results: + result_objects[index["name"].lower()].append(result["id"]) + + return result_objects + + def _search(self, search_query, index_type, offset=0, limit=meili_limit): + objects_to_return = {} + new_offset, total_hits = 0, 0 + indexes = meili_client.get_indexes() + for index in indexes: + results = meili_client.get_index(index["uid"]).search( + query=search_query, opt_params={"offset": offset, "limit": limit} + ) + objects_to_return[f"total_{index['name'].lower()}"] = results["nbHits"] + + if index_type == index["uid"]: + new_offset = results["offset"] + total_hits = results["nbHits"] + objects_for_index = self._objects_from_result( + results["hits"], index, objects_to_return + ) + objects_to_return.update(objects_for_index) + + return objects_to_return, new_offset, total_hits + + def convert_meili_to_pages(self, total_hits, offset, limit=meili_limit): + if total_hits < limit: + return { + "total_pages": 1, + "current_page": 1, + "next_page": None, + "previous_page": None, + } + + total_pages = ceil(total_hits / limit) + current_page = ceil((offset + limit) / limit) + next_page = None if total_pages == current_page else current_page + 1 + previous_page = None if next_page == 1 else current_page - 1 + + return { + "total_pages": total_pages, + "current_page": current_page, + "next_page": next_page, + "previous_page": previous_page, + } + + def convert_page_to_offset(self, page, limit=meili_limit): + return int((page * limit) - limit) + + def get_context_data(self, **kwargs): + context_data = super().get_context_data(**kwargs) + if "q" in self.request.GET: + results, offset, total_hits = self._search( + search_query=self.request.GET["q"], + index_type=self.request.GET.get("t", "services"), + offset=self.convert_page_to_offset( + int(self.request.GET.get("page", 1)) + ), + ) + context_data.update(results) + context_data["search_query"] = self.request.GET["q"] + context_data["search_type"] = self.request.GET.get("t", "services") + context_data["pagination"] = self.convert_meili_to_pages(total_hits, offset) + + context_data["project_links"] = [ + "Support", + "Repository", + "Dashboard", + "Alerts", + "Documentation", + ] + return context_data diff --git a/zoo/libraries/templates/libraries/library_list.html b/zoo/libraries/templates/libraries/library_list.html index 1aa75a73..9a237797 100644 --- a/zoo/libraries/templates/libraries/library_list.html +++ b/zoo/libraries/templates/libraries/library_list.html @@ -15,13 +15,7 @@ {% endblock %} {% block content %} - {% with project_type='libraries' project_create_view='library_create' %} + {% with project_type='libraries' project_create_view='library_create' project_links=project_links is_paginated=is_paginated page_obj=page_obj request=request %} {{ block.super }} {% endwith %} {% endblock %} - -{% block project_links %} - {% project_link 'Support' object.slack_url icon='comment alternate' %} - {% project_link 'Repository' object.gitlab_url icon='code' %} - {% project_link 'Documentation' object.docs_url icon='graduation cap' %} -{% endblock %} diff --git a/zoo/libraries/views.py b/zoo/libraries/views.py index b13a909b..e51e66e1 100644 --- a/zoo/libraries/views.py +++ b/zoo/libraries/views.py @@ -131,6 +131,12 @@ def get_queryset(self): return queryset.order_by("name") + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + + context["project_links"] = ["Support", "Repository", "Documentation"] + return context + class LibraryUpdate(LibraryMixin, generic_views.UpdateView): form_class = forms.LibraryForm diff --git a/zoo/meilisearch/apps.py b/zoo/meilisearch/apps.py deleted file mode 100644 index d39ec96f..00000000 --- a/zoo/meilisearch/apps.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.apps import AppConfig - - -class MeiliSearchConfig(AppConfig): - name = "zoo.meilisearch" diff --git a/zoo/meilisearch/management/commands/create_initial_indexes.py b/zoo/meilisearch/management/commands/create_initial_indexes.py deleted file mode 100644 index d9eaad5f..00000000 --- a/zoo/meilisearch/management/commands/create_initial_indexes.py +++ /dev/null @@ -1,19 +0,0 @@ -import meilisearch -from django.conf import settings -from django.core.management.base import BaseCommand - - -class Command(BaseCommand): - help = "Create initial MeiliSearch indexes" - - def handle(self, *args, **options): - client = meilisearch.Client(settings.MEILI_HOST, settings.MEILI_MASTER_KEY) - client.create_index( - uid="services", options={"name": "Service", "primaryKey": "id"} - ) - client.create_index( - uid="analytics", options={"name": "Dependency", "primaryKey": "id"} - ) - client.create_index( - uid="open-api", options={"name": "Schema", "primaryKey": "id"} - ) diff --git a/zoo/meilisearch/templates/search_overview.html b/zoo/meilisearch/templates/search_overview.html deleted file mode 100644 index 3d340e77..00000000 --- a/zoo/meilisearch/templates/search_overview.html +++ /dev/null @@ -1,16 +0,0 @@ -Services
-{% for s in services %} - {{ s }}
-{% endfor %} -
- -Service Detail Urls
-{% for ss in service_detail_urls %} - {{ ss }}
-{% endfor %} - -
-Dependencies
-{% for dep in dependecies %} - {{ dep }}
-{% endfor %} diff --git a/zoo/meilisearch/urls.py b/zoo/meilisearch/urls.py deleted file mode 100644 index 0c1ae3da..00000000 --- a/zoo/meilisearch/urls.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.urls import path - -from . import views - -urlpatterns = [path("search", views.MeiliSearchView.as_view(), name="search_overview")] diff --git a/zoo/meilisearch/views.py b/zoo/meilisearch/views.py deleted file mode 100644 index 2060ea1d..00000000 --- a/zoo/meilisearch/views.py +++ /dev/null @@ -1,48 +0,0 @@ -import meilisearch -import structlog -from django.apps import apps -from django.conf import settings -from django.views.generic import TemplateView - -log = structlog.get_logger() - - -class MeiliSearchView(TemplateView): - template_name = "search_overview.html" - context_object_name = "context" - per_page = 25 - - @staticmethod - def _objects_from_result(search_results, index, result_objects=None): - if index["name"] not in result_objects.keys(): - result_objects[index["name"]] = [] - try: - model = apps.get_model(index["uid"], index["name"]) - for result in search_results: - result_objects[index["name"]].append(model.objects.get(pk=result["id"])) - except LookupError: - for result in search_results: - result_objects[index["name"]].append(result["id"]) - - return result_objects - - def _search(self, query_param): - meili_client = meilisearch.Client( - settings.MEILI_HOST, settings.MEILI_MASTER_KEY - ) - indexes = meili_client.get_indexes() - objects_to_return = {} - for index in indexes: - results = meili_client.get_index(index["uid"]).search(query_param)["hits"] - objects_for_index = self._objects_from_result( - results, index, objects_to_return - ) - objects_to_return.update(objects_for_index) - return objects_to_return - - def get_context_data(self, **kwargs): - context_data = super().get_context_data(**kwargs) - if "q" in self.request.GET: - results = self._search(self.request.GET["q"]) - context_data.update(results) - return context_data diff --git a/zoo/resources/templates/dependency_overview.html b/zoo/resources/templates/dependency_overview.html index 74402a73..b88e2b24 100644 --- a/zoo/resources/templates/dependency_overview.html +++ b/zoo/resources/templates/dependency_overview.html @@ -86,5 +86,6 @@ {% endblock %} {% block scripts %} - + {{ block.super }} + {% endblock scripts %} diff --git a/zoo/services/templates/services/service_list.html b/zoo/services/templates/services/service_list.html index bf4938e9..6ded4c67 100644 --- a/zoo/services/templates/services/service_list.html +++ b/zoo/services/templates/services/service_list.html @@ -15,15 +15,7 @@ {% endblock %} {% block content %} - {% with project_type='services' project_create_view='service_create' %} + {% with project_type='services' project_create_view='service_create' project_links=project_links is_paginated=is_paginated page_obj=page_obj request=request %} {{ block.super }} {% endwith %} {% endblock %} - -{% block project_links %} - {% project_link 'Support' object.slack_url icon='comment alternate' %} - {% project_link 'Repository' object.gitlab_url icon='code' %} - {% project_link 'Dashboard' object.dashboard_url icon='chart pie' %} - {% project_link 'Alerts' object.pagerduty_url icon='warning' %} - {% project_link 'Documentation' object.docs_url icon='graduation cap' %} -{% endblock %} diff --git a/zoo/services/views.py b/zoo/services/views.py index 94ad1bd1..30eca805 100644 --- a/zoo/services/views.py +++ b/zoo/services/views.py @@ -205,6 +205,18 @@ def get_queryset(self): return queryset.order_by("name") + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + + context["project_links"] = [ + "Support", + "Repository", + "Dashboard", + "Alerts", + "Documentation", + ] + return context + class ServiceUpdate( ServiceEnvironmentMixin, ServiceLinkMixin, ServiceMixin, generic_views.UpdateView