# Explicitly allow AdSense and ads quality crawlers
User-agent: Mediapartners-Google
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: AdsBot-Google-Mobile
Allow: /
# Robots.txt for FrenzyCap.com Financial Analysis Platform
# Last updated: 2025-01-17

# Allow all well-behaved crawlers to index public content
User-agent: *
Allow: /
Allow: /sitemap.xml
Allow: /ads.txt

# Block access to sensitive API endpoints and admin areas
Disallow: /api/
Disallow: /admin/
Disallow: /login/
Disallow: /auth/
Disallow: /private/
Disallow: /tmp/
Disallow: /var/
Disallow: /bin/
Disallow: /config/
Disallow: /logs/
Disallow: /wiki/

# Block access to file extensions that shouldn't be indexed
Disallow: /*.json$
Disallow: /*.xml$
Disallow: /*.log$
Disallow: /*.py$
Disallow: /*.php$
Disallow: /*.sql$
Disallow: /*.bak$
Disallow: /*.old$

# Block search and filter pages with query parameters to avoid duplicate content
Disallow: /search?*
Disallow: /filter?*

# Allow specific crawlers better access to financial content
User-agent: Googlebot
Allow: /
Allow: /quote*
Allow: /analysis*
Allow: /market*
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Allow: /quote*
Allow: /analysis*
Allow: /market*
Crawl-delay: 2

# Financial data crawlers (if you want them)
User-agent: YahooSeeker
Allow: /quote*
Allow: /market*
Crawl-delay: 5

# Block aggressive crawlers that might overload your servers
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

# Block problematic bots
User-agent: SiteBot
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: ia_archiver
Disallow: /

# Expanded community bad bot blocklist (based on nginx-ultimate-bad-bot-blocker and other sources)
User-agent: Baiduspider
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Screaming Frog SEO Spider
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: Ahoy!Bot
Disallow: /

User-agent: spbot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: SEOkicks-Robot
Disallow: /

User-agent: proximic
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: trendictionbot
Disallow: /

User-agent: adscanner
Disallow: /

User-agent: Qwantify
Disallow: /

User-agent: UptimeRobot
Disallow: /

User-agent: TickertapeBot
Disallow: /

# Specify sitemap location (create this file)
Sitemap: https://www.frenzycap.com/sitemap.xml

# Cache directive for better performance
# Note: This is not standard robots.txt but some crawlers respect it
# Cache-delay: 1