# robots.txt - Blocking known troublesome bots while allowing legitimate ones
# Last updated: March 16, 2025

# Default rule - allow good bots
User-agent: *
Allow: /

# Known aggressive crawlers and bandwidth consumers
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: Sogou inst spider
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Qwantify
Disallow: /

User-agent: PetalBot
Disallow: /

# Aggressive marketing/SEO bots
User-agent: rogerbot
Disallow: /

# Bots that often ignore crawl-delay and rate limits
User-agent: Yandex
Crawl-delay: 30

User-agent: Baiduspider
Crawl-delay: 30

# Rate-limiting for all bots
# Some bots honor this, though it's not universal
Crawl-delay: 10

# Explicitly allow major search engines (recommended)
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurpbot
Allow: /

User-agent: DuckDuckBot
Allow: /

Sitemap: https://www.fsquared.co.uk/sitemap-index.xml