# RedeProveedores — Robots.txt
# https://redeproveedores.com

# === AI Crawlers (allowed on public content) ===
User-agent: GPTBot
Allow: /
Disallow: /mi-negocio
Disallow: /dashboard
Disallow: /admin*
Disallow: /perfil
Disallow: /api-keys
Disallow: /login
Disallow: /registro
Disallow: /reset-password

User-agent: ClaudeBot
Allow: /
Disallow: /mi-negocio
Disallow: /dashboard
Disallow: /admin*
Disallow: /perfil
Disallow: /api-keys
Disallow: /login
Disallow: /registro
Disallow: /reset-password

User-agent: Google-Extended
Allow: /
Disallow: /mi-negocio
Disallow: /dashboard
Disallow: /admin*
Disallow: /perfil
Disallow: /api-keys

User-agent: PerplexityBot
Allow: /
Disallow: /mi-negocio
Disallow: /dashboard
Disallow: /admin*
Disallow: /perfil
Disallow: /api-keys

User-agent: Bytespider
Allow: /
Disallow: /mi-negocio
Disallow: /dashboard
Disallow: /admin*
Disallow: /perfil
Disallow: /api-keys

User-agent: CCBot
Allow: /
Disallow: /mi-negocio
Disallow: /dashboard
Disallow: /admin*
Disallow: /perfil
Disallow: /api-keys

# === Search Engines ===
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# === Social Media Bots ===
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

# === Default ===
User-agent: *
Allow: /
Disallow: /mi-negocio
Disallow: /dashboard
Disallow: /admin*
Disallow: /perfil
Disallow: /api-keys
Disallow: /login
Disallow: /registro
Disallow: /reset-password
Disallow: /trap-bot
Crawl-delay: 10

# === Blocked: SEO scraper bots that don't help our SEO in MX ===
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# Sitemaps
# - sitemap.xml: sitemapindex generated at build time (scripts/generate-sitemap.ts)
# - news-sitemap.xml: served fresh via Cloudflare Worker proxying the edge function
Sitemap: https://redeproveedores.com/sitemap.xml
Sitemap: https://redeproveedores.com/news-sitemap.xml

# AI Agent Instructions
# See https://redeproveedores.com/llms.txt for detailed AI agent guidelines
# See https://redeproveedores.com/llms-full.txt for complete site context
# See https://redeproveedores.com/.well-known/ai-plugin.json for capabilities manifest