# Allow all major bots full access
User-agent: *
Disallow:

# Explicit allow for Twitterbot
User-agent: Twitterbot
Allow: /

# Explicit allow for Facebook crawler
User-agent: facebookexternalhit
Allow: /

# Block common non-content areas (optional, for SEO hygiene)
Disallow: /login/
Disallow: /register/
Disallow: /join/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/

# Crawl delay for aggressive bots (optional)
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

# Sitemap