fix; KTL-1180: Exclude hidden pages from search

JetBrains · Oct 9, 2023 · 8b70a93 · 8b70a93
1 parent 67e00ae
commit 8b70a93
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/src/dist.py b/src/dist.py
@@ -52,6 +52,9 @@ def get_dist_page_type(url):
         if parsed.find("meta", {"http-equiv": "refresh"}):
             page_type = 'Redirect'
 
+        if parsed.find("meta", {"name": "robots", "content": "noindex"}):
+            page_type = 'Hidden'
+
     if url.endswith('pdf'):
         page_type = 'File_Pdf'