feat: added ahrefsbot in allow list

pricefx · Jul 12, 2024 · e44d2a4 · e44d2a4
1 parent ce67e41
commit e44d2a4
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/robots.txt b/robots.txt
@@ -12,10 +12,17 @@ Disallow: /blocks/
 Disallow: /*?*
 
 User-agent: GPTBot
+Disallow: /
+
 User-agent: CCbot
+Disallow: /
+
 User-agent: ChatGPT-User
 Disallow: /
 
+User-agent: AhrefsBot
+Allow: /
+
 Sitemap: https://www.pricefx.com/sitemap.xml
 
 Crawl-delay: 5