diff --git a/client/public/robots.txt b/client/public/robots.txt new file mode 100644 index 0000000..e75060c --- /dev/null +++ b/client/public/robots.txt @@ -0,0 +1,54 @@ +User-agent: * +Disallow: /admin +Allow: / + +User-agent: Amazonbot +Disallow: / +User-agent: anthropic-ai +Disallow: / +User-agent: Applebot-Extended +Disallow: / +User-agent: AwarioRssBot +Disallow: / +User-agent: AwarioSmartBot +Disallow: / +User-agent: Bytespider +Disallow: / +User-agent: CCBot +Disallow: / +User-agent: ClaudeBot +Disallow: / +User-agent: Claude-Web +Disallow: / +User-agent: cohere-ai +Disallow: / +User-agent: DataForSeoBot +Disallow: / +User-agent: Diffbot +Disallow: / +User-agent: FacebookBot +Disallow: / +User-agent: Google-Extended +Disallow: / +User-agent: GPTBot +Disallow: / +User-agent: magpie-crawler +Disallow: / +User-agent: NewsNow +Disallow: / +User-agent: news-please +Disallow: / +User-agent: omgili +Disallow: / +User-agent: omgilibot +Disallow: / +User-agent: peer39_crawler +Disallow: / +User-agent: peer39_crawler/1.0 +Disallow: / +User-agent: PerplexityBot +Disallow: / +User-agent: Scrapy +Disallow: / +User-agent: TurnitinBot +Disallow: /