From fa499fb22327237341f7bbfa18c26e7d435a0d2e Mon Sep 17 00:00:00 2001 From: Rory McNicholl Date: Tue, 5 Nov 2024 11:51:08 +0000 Subject: [PATCH] stop facet crawling and add sitemaps --- public/robots.txt | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/public/robots.txt b/public/robots.txt index 3c9c7c01..7f56cdbd 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,5 +1,12 @@ -# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file -# -# To ban all spiders from the entire site uncomment the next two lines: -# User-agent: * -# Disallow: / +# Stop crawlers running searches (via facet links) +User-agent: * +Disallow: /catalog +# but do give them some lovely resource-sync sitemaps +Sitemap: https://bl.iro.bl.uk/resourcelist +Sitemap: https://mola.iro.bl.uk/resourcelist +Sitemap: https://nls.iro.bl.uk/resourcelist +Sitemap: https://nms.iro.bl.uk/resourcelist +Sitemap: https://nt.iro.bl.uk/resourcelist +Sitemap: https://kew.iro.bl.uk/resourcelist +Sitemap: https://sciencemuseumgroup.iro.bl.uk/resourcelist +Sitemap: https://britishmuseum.iro.bl.uk/resourcelist