# Best of Money - Enhanced Robots.txt
# Complete robots.txt file with comprehensive SEO directives

# Allow all crawlers by default
User-agent: *
Allow: /

# Main website structure - Enhanced with more specific paths
Allow: /categories
Allow: /category/
Allow: /ai-assistant
Allow: /emi-calculator
Allow: /gold-price
Allow: /about
Allow: /contact
Allow: /blog
Allow: /faq
Allow: /calculators
Allow: /credit-cards
Allow: /personal-loans
Allow: /insurance
Allow: /investments
Allow: /mortgages
Allow: /banking

# Disallow admin and private areas
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /backend/
Disallow: */draft/*
Disallow: */preview/*
Disallow: */debug/*
Disallow: /_next/
Disallow: /static/

# Disallow specific URL parameters that create duplicate content
Disallow: *?source=
Disallow: *?utm_
Disallow: *?ref=
Disallow: *?fbclid=
Disallow: *?gclid=
Disallow: *?msclkid=

# Specific crawler directives with optimized settings
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

# Set crawl rate limits for aggressive bots
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: BLEXBot
Crawl-delay: 20

User-agent: DotBot
Crawl-delay: 10

User-agent: YandexBot
Crawl-delay: 5

# Block known bad bots
User-agent: Baiduspider
Disallow: /

User-agent: PetalBot
Disallow: /

# Sitemap locations - Updated to correct domain with all sitemaps
Sitemap: https://bestofmoney.app/sitemap.xml    
Sitemap: https://bestofmoney.app/sitemap-main.xml
Sitemap: https://bestofmoney.app/sitemap-calculators.xml
Sitemap: https://bestofmoney.app/sitemap-gold-price.xml
Sitemap: https://bestofmoney.app/sitemap-credit-cards.xml
Sitemap: https://bestofmoney.app/sitemap-bankings.xml
Sitemap: https://bestofmoney.app/sitemap-health-insurances.xml
Sitemap: https://bestofmoney.app/sitemap-investments.xml
Sitemap: https://bestofmoney.app/sitemap-mortgages.xml
Sitemap: https://bestofmoney.app/sitemap-personal-loans.xml
Sitemap: https://bestofmoney.app/sitemap-student-loans.xml
Sitemap: https://bestofmoney.app/sitemap-term-insurances.xml
Sitemap: https://bestofmoney.app/sitemap-business-loans.xml