Merge pull request #90 from oskardydo/Q3-initiative

[TASK] Add suggestion endpoint

.ddev/config.yaml

@@ -5,7 +5,7 @@ php_version: "8.2"
 webserver_type: nginx-fpm
 router_http_port: "80"
 router_https_port: "443"
-xdebug_enabled: true
+xdebug_enabled: trueent: [ ]
 additional_hostnames: []
 additional_fqdns: []
 use_dns_when_possible: true
@@ -156,7 +156,7 @@ omit_containers: [db]
 # In this case the user must provide all such settings.
 
 # You can inject environment variables into the web container with:
-# web_environment: 
+# web_environment:
 # - SOMEENV=somevalue
 # - SOMEOTHERENV=someothervalue
 
@@ -167,7 +167,7 @@ omit_containers: [db]
 # For advanced users only!
 
 # provider: default # Currently "default", "pantheon", "ddev-live"
-# 
+#
 # Many ddev commands can be extended to run tasks before or after the
 # ddev command is executed, for example "post-start", "post-import-db",
 # "pre-composer", "post-composer"

.ddev/docker-compose.environment.yaml

@@ -4,6 +4,6 @@ services:
     environment:
       - APP_ENV=dev
       - APP_SECRET=ddf011a53a72d4f85e6fcfd4d1a72737
-      - DOCS_ROOT_PATH=../docs_server/docs.typo3.org/Web
+      - DOCS_ROOT_PATH=/var/www/html/docs_server/docs.typo3.org/Web
       - ELASTICA_HOST=elasticsearch
       - PHP_IDE_CONFIG=serverName=t3docs-search-indexer.ddev.site

.gitignore

@@ -1,4 +1,5 @@
 /_docs
+/docs_server*
 
 ###> symfony/framework-bundle ###
 .env

config/Elasticorn/docsearch/IndexConfiguration.yaml

@@ -1,6 +1,17 @@
 number_of_shards: 4
 number_of_replicas: 1
 analysis:
+  tokenizer:
+    autocomplete_small_tokenizer:
+      type: ngram
+      min_gram: 1
+      max_gram: 2
+      token_chars: ["letter", "digit"]
+    autocomplete_large_tokenizer:
+      type: ngram
+      min_gram: 3
+      max_gram: 4
+      token_chars: ["letter", "digit"]
   filter:
     typo3_stemmer:
       type: stemmer
@@ -17,4 +28,16 @@ analysis:
         - asciifolding
         - typo3_filter
         - typo3_stemmer
+    typo3_autocomplete_small:
+      type: custom
+      tokenizer: autocomplete_small_tokenizer
+      filter:
+        - lowercase
+        - asciifolding
+    typo3_autocomplete_large:
+      type: custom
+      tokenizer: autocomplete_large_tokenizer
+      filter:
+        - lowercase
+        - asciifolding
 

config/Elasticorn/docsearch/Mapping.yaml

@@ -10,12 +10,57 @@ manual_type:
   type: keyword
 manual_version:
   type: keyword
+  fields:
+    small_suggest:
+      type: search_as_you_type
+      analyzer: typo3_autocomplete_small
+    large_suggest:
+      type: search_as_you_type
+      analyzer: typo3_autocomplete_large
 manual_language:
   type: keyword
 manual_slug:
   type: keyword
 manual_keywords:
   type: keyword
+manual_vendor:
+  type: keyword
+  fields:
+    small_suggest:
+      type: search_as_you_type
+      analyzer: typo3_autocomplete_small
+    large_suggest:
+      type: search_as_you_type
+      analyzer: typo3_autocomplete_large
+manual_package:
+  type: keyword
+  fields:
+    small_suggest:
+      type: search_as_you_type
+      analyzer: typo3_autocomplete_small
+    large_suggest:
+      type: search_as_you_type
+      analyzer: typo3_autocomplete_large
+manual_extension:
+  type: keyword
+  fields:
+    small_suggest:
+      type: search_as_you_type
+      analyzer: typo3_autocomplete_small
+    large_suggest:
+      type: search_as_you_type
+      analyzer: typo3_autocomplete_large
+option:
+  type: keyword
+  fields:
+    small_suggest:
+      type: search_as_you_type
+      analyzer: typo3_autocomplete_small
+    large_suggest:
+      type: search_as_you_type
+      analyzer: typo3_autocomplete_large
+option_keywords:
+  type: keyword
 fragment:
   type: keyword
 page_title:
@@ -34,4 +79,6 @@ snippet_content:
 content_hash:
   type: keyword
 major_versions:
-  type: keyword
+  type: keyword
+is_core:
+  type: boolean

src/Config/Labels.php

@@ -0,0 +1,29 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Config;
+
+class Labels
+{
+    public const MAP = [
+        'manual_vendor' => 'Vendor',
+        'manual_package' => 'Package',
+        'manual_version' => 'Version',
+        'is_core' => 'Core?',
+        'manual_type' => 'Document Type',
+        'major_versions' => 'Major Version',
+        'manual_language' => 'Language',
+        'option' => 'Option',
+        'optionaggs' => 'Option',
+    ];
+
+    public static function getLabelForEsColumn(string $filter, string $default = ''): string
+    {
+        if ($default !== '') {
+            return self::MAP[$filter] ?? $default;
+        }
+
+        return self::MAP[$filter] ?? $filter;
+    }
+}

src/Config/ManualType.php

@@ -37,4 +37,4 @@ public static function getMap(): array
             'typo3cms' => self::ExceptionReference->value,
         ];
     }
-}
+}

src/Controller/SearchController.php

@@ -5,8 +5,8 @@
 use App\Dto\SearchDemand;
 use App\Repository\ElasticRepository;
 use Elastica\Exception\InvalidException;
-use JsonException;
 use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
+use Symfony\Component\HttpFoundation\JsonResponse;
 use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\HttpFoundation\Response;
 use Symfony\Component\Routing\Annotation\Route;
@@ -36,6 +36,7 @@ public function search(Request $request): Response
         if ($request->query->get('q', '') === '') {
             return $this->redirectToRoute('index');
         }
+
         $searchDemand = SearchDemand::createFromRequest($request);
 
         return $this->render('search/search.html.twig', [
@@ -46,31 +47,49 @@ public function search(Request $request): Response
         ]);
     }
 
-    /**
-     * @return Response
-     * @throws InvalidException|JsonException
-     */
     #[Route(path: '/suggest', name: 'suggest')]
     public function suggest(Request $request): Response
     {
         $searchDemand = SearchDemand::createFromRequest($request);
+        $jsonData = [
+            'demand' => $searchDemand->toArray(),
+            'suggest' => $this->elasticRepository->suggestScopes($searchDemand)
+        ];
 
-        $results = $this->elasticRepository->suggest($searchDemand);
-        $suggestions = [];
-        foreach ($results['results'] as $result) {
-            $hit = $result->getData();
-            $suggestions[] = [
-                'label' => $hit['snippet_title'],
-                'value' => $hit['snippet_title'],
-                'url' => 'https://docs.typo3.org/' . $hit['manual_slug'] . '/' . $hit['relative_url'] . '#' . $hit['fragment'],
-                'group' => $hit['manual_title'],
-                'content' => \mb_substr((string)$hit['snippet_content'], 0, 100)
-            ];
-        }
-        $jsonBody =  \json_encode($suggestions, JSON_THROW_ON_ERROR);
+        $searchResults = $this->elasticRepository->searchDocumentsForSuggest($searchDemand);
+        $jsonData['time'] = $searchResults['time'];
+
+        $jsonData['results'] = array_map(static function ($result) {
+            return $result->getData();
+        }, $searchResults['results']);
+
+        return new JsonResponse($jsonData);
+    }
+
+    #[Route(path: '/suggest/list', name: 'suggest-list')]
+    public function suggestList(Request $request): Response
+    {
+        $searchDemand = SearchDemand::createFromRequest($request);
+        $jsonData = [
+            'demand' => $searchDemand->toArray(),
+            'suggest' => $this->elasticRepository->suggestScopes($searchDemand)
+        ];
+
+        return new JsonResponse($jsonData);
+    }
+
+    #[Route(path: '/suggest/results', name: 'suggest-results')]
+    public function suggestResults(Request $request): Response
+    {
+        $searchDemand = SearchDemand::createFromRequest($request);
+
+        $searchResults = $this->elasticRepository->searchDocumentsForSuggest($searchDemand);
+        $jsonData['time'] = $searchResults['time'];
+
+        $jsonData['results'] = array_map(static function ($result) {
+            return $result->getData();
+        }, $searchResults['results']);
 
-        $response = new Response();
-        $response->setContent($jsonBody);
-        return $response;
+        return new JsonResponse($jsonData);
     }
 }

src/Dto/Manual.php

@@ -9,12 +9,14 @@ class Manual
 {
     public function __construct(
         private readonly string $absolutePath,
-        private readonly string $title,
+        private readonly string $name,
         private readonly string $type,
         private readonly string $version,
         private readonly string $language,
         private readonly string $slug,
-        private readonly array $keywords
+        private readonly array $keywords,
+        private readonly string $vendor = '',
+        private readonly bool $isCore = false,
     ) {
     }
 
@@ -43,19 +45,23 @@ public static function createFromFolder(\SplFileInfo $folder, $changelog = false
 
         $map = ManualType::getMap();
         $type = $map[$type] ?? $type;
+        $isCore = in_array($type, [ManualType::SystemExtension->value, ManualType::Typo3Manual->value, ManualType::CoreChangelog->value], true);
 
         $keywords = [];
         if ($type === ManualType::SystemExtension->value || $type === ManualType::CommunityExtension->value) {
             $keywords[] = $name;
         }
+
         return new Manual(
             $folder,
-            implode('/', [$vendor, $name]),
+            $name,
             $type,
             $version,
             $language,
             implode('/', $values),
-            $keywords
+            $keywords,
+            $vendor,
+            $isCore,
         );
     }
 
@@ -98,6 +104,7 @@ public function getSubManuals(): array
             ->depth(0);
 
         $subManuals = [];
+
         foreach ($finder as $changelogFolder) {
             $subManuals[] = self::createFromFolder($changelogFolder, true);
         }
@@ -111,7 +118,32 @@ public function getAbsolutePath(): string
 
     public function getTitle(): string
     {
-        return $this->title;
+        $titleParts = [];
+
+        if ($this->vendor !== '') {
+            $titleParts[] = $this->vendor;
+        }
+
+        if ($this->name !== '') {
+            $titleParts[] = $this->name;
+        }
+
+        return implode('/', $titleParts);
+    }
+
+    public function getName(): string
+    {
+        return $this->name;
+    }
+
+    public function getVendor(): string
+    {
+        return $this->vendor;
+    }
+
+    public function isCore(): bool
+    {
+        return $this->isCore;
     }
 
     public function getType(): string