# Railway Herald

# Aggressive SEO/marketing crawlers are not permitted.
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# All other crawlers: keep out of account, admin, checkout, auth, and the
# expensive search endpoints. Public content sections remain crawlable.
User-agent: *
Crawl-delay: 10
Disallow: /my-account/
Disallow: /admin/
Disallow: /basket
Disallow: /checkout
Disallow: /login
Disallow: /register
Disallow: /password-reset
Disallow: /imaging-centre/search