User-agent: *
Disallow: /api/
Disallow: /nutra/

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Amzn-SearchBot
Disallow: /

User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
Crawl-delay: 30
Disallow: /api/
Disallow: /nutra/

User-agent: AhrefsBot
Crawl-delay: 10
Disallow: /en/ingredients/
Disallow: /es/ingredients/
Disallow: /es-419/ingredients/
Disallow: /fr/ingredients/
Disallow: /de/ingredients/
Disallow: /pt/ingredients/
Disallow: /it/ingredients/
Disallow: /ja/ingredients/
Disallow: /zh-CN/ingredients/
Disallow: /th/ingredients/
Disallow: /id/ingredients/

User-agent: meta-externalagent
Disallow: /ingredients/
Disallow: /en/ingredients/
Disallow: /es/ingredients/
Disallow: /es-419/ingredients/
Disallow: /fr/ingredients/
Disallow: /de/ingredients/
Disallow: /pt/ingredients/
Disallow: /it/ingredients/
Disallow: /ja/ingredients/
Disallow: /zh-CN/ingredients/
Disallow: /th/ingredients/
Disallow: /id/ingredients/

# Crawlers that take the pages for model training. The answer engines that send
# readers back (OAI-SearchBot, ChatGPT-User, Claude-User, PerplexityBot) are
# deliberately absent, as are Googlebot and Bingbot. Google-Extended and
# Applebot-Extended refuse training without affecting search ranking.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Diffbot
User-agent: Omgilibot
User-agent: ImagesiftBot
User-agent: cohere-ai
User-agent: Timpibot
Disallow: /

Sitemap: https://www.prospre.io/sitemap.xml