diff --git a/galaxy_ng/app/api/ui/serializers/search.py b/galaxy_ng/app/api/ui/serializers/search.py index 7bc1f0e590..02a8a18d06 100644 --- a/galaxy_ng/app/api/ui/serializers/search.py +++ b/galaxy_ng/app/api/ui/serializers/search.py @@ -15,4 +15,4 @@ class SearchResultsSerializer(serializers.Serializer): tags = serializers.JSONField(source="tag_names") platforms = serializers.JSONField(source="platform_names") relevance = serializers.FloatField() - search_vector = serializers.CharField() + search = serializers.CharField() diff --git a/galaxy_ng/app/api/ui/views/search.py b/galaxy_ng/app/api/ui/views/search.py index 20ef1dc975..83f6bc6bca 100644 --- a/galaxy_ng/app/api/ui/views/search.py +++ b/galaxy_ng/app/api/ui/views/search.py @@ -1,6 +1,5 @@ -from django.db import connection from django.contrib.postgres.aggregates import JSONBAgg -from django.contrib.postgres.search import SearchQuery, SearchVector +from django.contrib.postgres.search import SearchQuery from django.db.models import ( Exists, F, @@ -43,8 +42,8 @@ SORT_PARAM = "order_by" SORTABLE_FIELDS = ["name", "namespace_name", "download_count", "last_updated", "relevance"] SORTABLE_FIELDS += [f"-{item}" for item in SORTABLE_FIELDS] -DEFAULT_SORT = "-download_count" -DEFAULT_SEARCH_TYPE = "sql" # websearch,sql +DEFAULT_SORT = "-download_count,-relevance" +DEFAULT_SEARCH_TYPE = "websearch" # websearch,sql QUERYSET_VALUES = [ "namespace_avatar", "content_list", @@ -58,15 +57,15 @@ "tag_names", "content_type", "latest_version", - "search_vector", + "search", "relevance", ] RANK_NORMALIZATION = 32 -EMPTY_QUERY = SearchQuery(Value(None)) class SearchListView(api_base.GenericViewSet, mixins.ListModelMixin): """Search collections and roles""" + permission_classes = [AllowAny] serializer_class = SearchResultsSerializer @@ -99,7 +98,7 @@ def list(self, *args, **kwargs): - **search_type:** ["sql", "websearch"] - **keywords:** string - queried against name,namespace,description,tags,platform - - when search_type is websearch allows operators e.g: "this OR that AND (A OR B) -notthis" + - when search_type is websearch allows operators e.g: "this OR that AND (A OR B) -C" - when search_type is sql performs a SQL ilike on the same fields - **type:** ["collection", "role"] - **deprecated:** boolean @@ -185,11 +184,12 @@ def get_filter_params(self, request): def get_sorting_param(self, request): """Validates the sorting parameter is valid.""" - sort = request.query_params.get(SORT_PARAM, DEFAULT_SORT) - if sort not in SORTABLE_FIELDS: - raise ValidationError(f"{SORT_PARAM} requires one of {SORTABLE_FIELDS}") - search_type = request.query_params.get("search_type", "sql") - if "relevance" in sort and search_type != "websearch": + sort = request.query_params.get(SORT_PARAM, DEFAULT_SORT).split(",") + for item in sort: + if item not in SORTABLE_FIELDS: + raise ValidationError(f"{SORT_PARAM} requires one of {SORTABLE_FIELDS}") + search_type = request.query_params.get("search_type", DEFAULT_SEARCH_TYPE) + if ("relevance" in sort or "-relevance" in sort) and search_type != "websearch": raise ValidationError("'order_by=relevance' works only with 'search_type=websearch'") return sort @@ -229,7 +229,7 @@ def get_collection_queryset(self, query=None): latest_version=F("version"), content_list=F("contents"), namespace_avatar=Subquery(namespace_qs.values("_avatar_url")), - # search_vector=F("search_vector"), + search=F("search_vector"), relevance=relevance, ) .values(*QUERYSET_VALUES) @@ -239,20 +239,10 @@ def get_collection_queryset(self, query=None): def get_role_queryset(self, query=None): """Build the LegacyRole queryset from annotations.""" - vector = Value("") relevance = Value(0) if query: - # TODO: Build search_vector field in the LegacyRole model and update via trigger or - # hook during import. - vector = ( - SearchVector("namespace__name", weight="A") - + SearchVector("name", weight="A") - + SearchVector("full_metadata__tags", weight="B") - + SearchVector("full_metadata__platforms", weight="C") - + SearchVector(KT("full_metadata__description"), weight="D") - ) relevance = Func( - F("search_vector"), + F("search"), query, RANK_NORMALIZATION, function="ts_rank", @@ -270,12 +260,10 @@ def get_role_queryset(self, query=None): download_count=Coalesce(F("legacyroledownloadcount__count"), Value(0)), latest_version=KT("full_metadata__versions__-1__version"), content_list=Value([], JSONField()), # There is no contents for roles - namespace_avatar=F("namespace__avatar_url"), - search_vector=vector, + namespace_avatar=F("namespace__namespace___avatar_url"), # v3 namespace._avatar_url + search=F("legacyrolesearchvector__search_vector"), relevance=relevance, ).values(*QUERYSET_VALUES) - print(qs.all()) - print(connection.queries) return qs def filter_and_sort(self, collections, roles, filter_params, sort, type="", query=None): @@ -305,8 +293,8 @@ def filter_and_sort(self, collections, roles, filter_params, sort, type="", quer collections = collections.filter(platform_names=platform) # never match but required if query: - collections = collections.filter(search_vector=query) - roles = roles.filter(search_vector=query) + collections = collections.filter(search=query) + roles = roles.filter(search=query) elif keywords := filter_params.get("keywords"): # search_type=sql query = ( Q(name__icontains=keywords) @@ -319,11 +307,11 @@ def filter_and_sort(self, collections, roles, filter_params, sort, type="", quer roles = roles.filter(query) if type.lower() == "role": - qs = roles.order_by(sort) + qs = roles.order_by(*sort) elif type.lower() == "collection": - qs = collections.order_by(sort) + qs = collections.order_by(*sort) else: - qs = collections.union(roles, all=True).order_by(sort) + qs = collections.union(roles, all=True).order_by(*sort) return qs @@ -335,7 +323,7 @@ def test(): print(f"{' START ':#^40}") s = SearchListView() data = s.get_search_results( - {"type": "", "search_type": "websearch", "keywords": "java web"}, sort="-relevance" + {"type": "", "keywords": "java web"}, sort="-relevance" ) print(f"{' SQLQUERY ':#^40}") print(data._query) diff --git a/galaxy_ng/app/migrations/0047_update_role_search_vector_trigger.py b/galaxy_ng/app/migrations/0047_update_role_search_vector_trigger.py index de551cebad..c66e09711b 100755 --- a/galaxy_ng/app/migrations/0047_update_role_search_vector_trigger.py +++ b/galaxy_ng/app/migrations/0047_update_role_search_vector_trigger.py @@ -29,6 +29,7 @@ ON CONFLICT (role_id) DO UPDATE SET search_vector = _search_vector, modified = current_timestamp; + RETURN NEW; END; $$ LANGUAGE plpgsql; DROP TRIGGER IF EXISTS update_ts_vector ON galaxy_legacyrole; @@ -50,4 +51,13 @@ class Migration(migrations.Migration): ("galaxy", "0046_legacyrolesearchvector"), ] - operations = [] + operations = [ + migrations.RunSQL( + sql=CREATE_ROLE_TS_VECTOR_TRIGGER, + reverse_sql=DROP_ROLE_TS_VECTOR_TRIGGER, + ), + migrations.RunSQL( + sql=REBUILD_ROLES_TS_VECTOR, + reverse_sql=migrations.RunSQL.noop, + ) + ] diff --git a/galaxy_ng/tests/integration/community/test_search.py b/galaxy_ng/tests/integration/community/test_search.py index 818dbaace3..d4cf4c460b 100644 --- a/galaxy_ng/tests/integration/community/test_search.py +++ b/galaxy_ng/tests/integration/community/test_search.py @@ -119,7 +119,9 @@ def test_namespace_with_sql_search(admin_client): @pytest.mark.deployment_community def test_name_with_sql_search(admin_client): """Test search.""" - name = admin_client(f"/api/_ui/v1/search/?namespace=ansible&name={COLLECTION_NAME}") + name = admin_client( + f"/api/_ui/v1/search/?search_type=sql&namespace=ansible&name={COLLECTION_NAME}" + ) assert name["meta"]["count"] == 1 assert name["data"][0]["name"] == COLLECTION_NAME assert name["data"][0]["namespace"] == NAMESPACE_NAME @@ -131,7 +133,7 @@ def test_name_with_sql_search(admin_client): @pytest.mark.deployment_community def test_tags_with_sql_search(admin_client): """Test search.""" - tag_url = "/api/_ui/v1/search/?namespace=ansible" + tag_url = "/api/_ui/v1/search/?search_type=sql&namespace=ansible" for tag in COLLECTION_TAGS: tag_url += f"&tags={tag}" tags = admin_client(tag_url) @@ -146,7 +148,7 @@ def test_tags_with_sql_search(admin_client): @pytest.mark.deployment_community def test_type_with_sql_search(admin_client): """Test search.""" - content_type = admin_client("/api/_ui/v1/search/?namespace=ansible&type=role") + content_type = admin_client("/api/_ui/v1/search/?search_type=sql&namespace=ansible&type=role") assert content_type["meta"]["count"] == 1 assert content_type["data"][0]["name"] == ROLE_NAME assert content_type["data"][0]["namespace"] == NAMESPACE_NAME @@ -158,7 +160,7 @@ def test_type_with_sql_search(admin_client): @pytest.mark.deployment_community def test_platform_with_sql_search(admin_client): """Test search.""" - platform = admin_client("/api/_ui/v1/search/?namespace=ansible&platform=fedora") + platform = admin_client("/api/_ui/v1/search/?search_type=sql&namespace=ansible&platform=fedora") assert platform["meta"]["count"] == 1 assert platform["data"][0]["name"] == ROLE_NAME assert platform["data"][0]["namespace"] == NAMESPACE_NAME @@ -170,14 +172,18 @@ def test_platform_with_sql_search(admin_client): @pytest.mark.deployment_community def test_deprecated_with_sql_search(admin_client): """Test search.""" - deprecated = admin_client("/api/_ui/v1/search/?namespace=ansible&deprecated=true") + deprecated = admin_client( + "/api/_ui/v1/search/?search_type=sql&namespace=ansible&deprecated=true" + ) assert deprecated["meta"]["count"] == 0 @pytest.mark.deployment_community def test_keywords_with_sql_search(admin_client): """Test search.""" - keywords = admin_client("/api/_ui/v1/search/?namespace=ansible&keywords=infinidash") + keywords = admin_client( + "/api/_ui/v1/search/?search_type=sql&namespace=ansible&keywords=infinidash" + ) assert keywords["meta"]["count"] == 1 assert keywords["data"][0]["name"] == COLLECTION_NAME assert keywords["data"][0]["namespace"] == NAMESPACE_NAME @@ -189,7 +195,9 @@ def test_keywords_with_sql_search(admin_client): @pytest.mark.deployment_community def test_sorting_with_sql_search(admin_client): """Test search.""" - sorting = admin_client("/api/_ui/v1/search/?namespace=ansible&order_by=-last_updated") + sorting = admin_client( + "/api/_ui/v1/search/?search_type=sql&namespace=ansible&order_by=-last_updated" + ) assert sorting["meta"]["count"] == 2 assert sorting["data"][0]["type"] == "role" assert sorting["data"][1]["type"] == "collection" @@ -198,7 +206,7 @@ def test_sorting_with_sql_search(admin_client): @pytest.mark.deployment_community def test_facets_with_web_search(admin_client): """Search using vector websearch""" - namespace = admin_client("/api/_ui/v1/search/?namespace=ansible&search_type=websearch") + namespace = admin_client("/api/_ui/v1/search/?namespace=ansible") assert namespace["meta"]["count"] == 2 # IMPORTANT: Keep filtering by namespace to avoid including content from other tests @@ -207,9 +215,7 @@ def test_facets_with_web_search(admin_client): @pytest.mark.deployment_community def test_name_with_web_search(admin_client): """Search using vector websearch""" - name = admin_client( - f"/api/_ui/v1/search/?namespace=ansible&search_type=websearch&name={COLLECTION_NAME}" - ) + name = admin_client(f"/api/_ui/v1/search/?namespace=ansible&name={COLLECTION_NAME}") assert name["meta"]["count"] == 1 assert name["data"][0]["name"] == COLLECTION_NAME assert name["data"][0]["namespace"] == NAMESPACE_NAME @@ -221,7 +227,7 @@ def test_name_with_web_search(admin_client): @pytest.mark.deployment_community def test_tags_with_web_search(admin_client): """Search using vector websearch""" - tag_url = "/api/_ui/v1/search/?namespace=ansible&search_type=websearch" + tag_url = "/api/_ui/v1/search/?namespace=ansible" for tag in COLLECTION_TAGS: tag_url += f"&tags={tag}" tags = admin_client(tag_url) @@ -236,9 +242,7 @@ def test_tags_with_web_search(admin_client): @pytest.mark.deployment_community def test_type_with_web_search(admin_client): """Search using vector websearch""" - content_type = admin_client( - "/api/_ui/v1/search/?namespace=ansible&search_type=websearch&type=role" - ) + content_type = admin_client("/api/_ui/v1/search/?namespace=ansible&type=role") assert content_type["meta"]["count"] == 1 assert content_type["data"][0]["name"] == ROLE_NAME assert content_type["data"][0]["namespace"] == NAMESPACE_NAME @@ -250,9 +254,7 @@ def test_type_with_web_search(admin_client): @pytest.mark.deployment_community def test_platform_with_web_search(admin_client): """Search using vector websearch""" - platform = admin_client( - "/api/_ui/v1/search/?namespace=ansible&search_type=websearch&platform=fedora" - ) + platform = admin_client("/api/_ui/v1/search/?namespace=ansible&platform=fedora") assert platform["meta"]["count"] == 1 assert platform["data"][0]["name"] == ROLE_NAME assert platform["data"][0]["namespace"] == NAMESPACE_NAME @@ -264,18 +266,14 @@ def test_platform_with_web_search(admin_client): @pytest.mark.deployment_community def test_deprecated_with_web_search(admin_client): """Search using vector websearch""" - deprecated = admin_client( - "/api/_ui/v1/search/?namespace=ansible&search_type=websearch&deprecated=true" - ) + deprecated = admin_client("/api/_ui/v1/search/?namespace=ansible&deprecated=true") assert deprecated["meta"]["count"] == 0 @pytest.mark.deployment_community def test_keywords_with_web_search(admin_client): """Search using vector websearch""" - keywords = admin_client( - "/api/_ui/v1/search/?namespace=ansible&search_type=websearch&keywords=infinidash" - ) + keywords = admin_client("/api/_ui/v1/search/?namespace=ansible&keywords=infinidash") assert keywords["meta"]["count"] == 1 assert keywords["data"][0]["name"] == COLLECTION_NAME assert keywords["data"][0]["namespace"] == NAMESPACE_NAME @@ -287,9 +285,7 @@ def test_keywords_with_web_search(admin_client): @pytest.mark.deployment_community def test_sorting_with_web_search(admin_client): """Search using vector websearch""" - sorting = admin_client( - "/api/_ui/v1/search/?namespace=ansible&search_type=websearch&order_by=-last_updated" - ) + sorting = admin_client("/api/_ui/v1/search/?namespace=ansible&order_by=-last_updated") assert sorting["meta"]["count"] == 2 assert sorting["data"][0]["type"] == "role" assert sorting["data"][1]["type"] == "collection" @@ -304,9 +300,7 @@ def test_compound_query_with_web_search(admin_client): "infinidash%20OR%20java", "api%20-kubernetes", ]: - websearch = admin_client( - f"/api/_ui/v1/search/?namespace=ansible&search_type=websearch&keywords={term}" - ) + websearch = admin_client(f"/api/_ui/v1/search/?namespace=ansible&keywords={term}") assert websearch["meta"]["count"] == 1 assert websearch["data"][0]["name"] == COLLECTION_NAME assert websearch["data"][0]["namespace"] == NAMESPACE_NAME @@ -320,9 +314,7 @@ def test_compound_query_with_web_search(admin_client): "kubernetes%20OR%20java", "api%20-infinidash", ]: - websearch = admin_client( - f"/api/_ui/v1/search/?namespace=ansible&search_type=websearch&keywords={term}" - ) + websearch = admin_client(f"/api/_ui/v1/search/?namespace=ansible&keywords={term}") assert websearch["meta"]["count"] == 1 assert websearch["data"][0]["name"] == ROLE_NAME assert websearch["data"][0]["namespace"] == NAMESPACE_NAME @@ -336,8 +328,7 @@ def test_relevance_with_web_search(admin_client): """Search using vector websearch""" # Both has api tag and fedora term as a platform for role and description for collection keywords = admin_client( - "/api/_ui/v1/search/?namespace=ansible&search_type=websearch" - "&keywords=api%20AND%20fedora&order_by=-relevance" + "/api/_ui/v1/search/?namespace=ansible" "&keywords=api%20AND%20fedora&order_by=-relevance" ) assert keywords["meta"]["count"] == 2 assert keywords["data"][0]["name"] == ROLE_NAME