# Finnextho - politica de descoberta publica e IA # # ARQUIVO GERADO. Nao edite a mao. # fonte: frontend/config/crawler-policy.mjs # gerador: frontend/scripts/generate-robots.mjs # gate: npm run robots:validate (roda no prebuild e no CI) # # Politica: paginas publicas liberadas para busca, citacao e treinamento. # Areas autenticadas, APIs e portais bloqueados em todos os grupos. # # Cada grupo abaixo repete a lista completa de bloqueios de proposito. # Pela RFC 9309 um grupo termina no proximo User-agent: regras colocadas # apos o ultimo grupo pertencem SOMENTE a ele. Repetir e o unico jeito # correto de aplicar a mesma politica a todos os agentes. # # robots.txt NAO e controle de acesso. As areas privadas continuam # protegidas por autenticacao, autorizacao e X-Robots-Tag: noindex. # # Agentes acionados por usuario (ChatGPT-User, Google-Agent, Google-GeminiNotebook, Perplexity-User) nao tem grupo proprio: # caem no grupo "*", que ja e a politica correta. Para estes o controle # efetivo e o WAF/CDN, que nao deve responder 403 nem CAPTCHA. User-agent: * Disallow: /admin Disallow: /api Disallow: /__/auth Disallow: /sistema Disallow: /dashboard Disallow: /app Disallow: /finn Disallow: /personal Disallow: /profile Disallow: /configuracoes Disallow: /notificacoes Disallow: /metas Disallow: /transacoes Disallow: /investimentos Disallow: /mercados Disallow: /cartoes Disallow: /milhas Disallow: /relatorios Disallow: /agendamentos Disallow: /arquivos Disallow: /ebook Disallow: /assinaturas Disallow: /billing Disallow: /consumo Disallow: /payment Disallow: /trial Disallow: /business Disallow: /empresa/dashboard Disallow: /empresa/assinaturas Disallow: /portal Disallow: /test- Disallow: /auth/reset-password Disallow: /auth/forgot-password Disallow: /auth/verify-email Disallow: /auth/change-password Disallow: /auth/complete-registration Allow: / # OpenAI / ChatGPT - Busca e citacoes no ChatGPT User-agent: OAI-SearchBot Disallow: /admin Disallow: /api Disallow: /__/auth Disallow: /sistema Disallow: /dashboard Disallow: /app Disallow: /finn Disallow: /personal Disallow: /profile Disallow: /configuracoes Disallow: /notificacoes Disallow: /metas Disallow: /transacoes Disallow: /investimentos Disallow: /mercados Disallow: /cartoes Disallow: /milhas Disallow: /relatorios Disallow: /agendamentos Disallow: /arquivos Disallow: /ebook Disallow: /assinaturas Disallow: /billing Disallow: /consumo Disallow: /payment Disallow: /trial Disallow: /business Disallow: /empresa/dashboard Disallow: /empresa/assinaturas Disallow: /portal Disallow: /test- Disallow: /auth/reset-password Disallow: /auth/forgot-password Disallow: /auth/verify-email Disallow: /auth/change-password Disallow: /auth/complete-registration Allow: / # OpenAI - Treinamento de modelos fundacionais User-agent: GPTBot Disallow: /admin Disallow: /api Disallow: /__/auth Disallow: /sistema Disallow: /dashboard Disallow: /app Disallow: /finn Disallow: /personal Disallow: /profile Disallow: /configuracoes Disallow: /notificacoes Disallow: /metas Disallow: /transacoes Disallow: /investimentos Disallow: /mercados Disallow: /cartoes Disallow: /milhas Disallow: /relatorios Disallow: /agendamentos Disallow: /arquivos Disallow: /ebook Disallow: /assinaturas Disallow: /billing Disallow: /consumo Disallow: /payment Disallow: /trial Disallow: /business Disallow: /empresa/dashboard Disallow: /empresa/assinaturas Disallow: /portal Disallow: /test- Disallow: /auth/reset-password Disallow: /auth/forgot-password Disallow: /auth/verify-email Disallow: /auth/change-password Disallow: /auth/complete-registration Allow: / # OpenAI - Validacao de landing pages de anuncios User-agent: OAI-AdsBot Disallow: /admin Disallow: /api Disallow: /__/auth Disallow: /sistema Disallow: /dashboard Disallow: /app Disallow: /finn Disallow: /personal Disallow: /profile Disallow: /configuracoes Disallow: /notificacoes Disallow: /metas Disallow: /transacoes Disallow: /investimentos Disallow: /mercados Disallow: /cartoes Disallow: /milhas Disallow: /relatorios Disallow: /agendamentos Disallow: /arquivos Disallow: /ebook Disallow: /assinaturas Disallow: /billing Disallow: /consumo Disallow: /payment Disallow: /trial Disallow: /business Disallow: /empresa/dashboard Disallow: /empresa/assinaturas Disallow: /portal Disallow: /test- Disallow: /auth/reset-password Disallow: /auth/forgot-password Disallow: /auth/verify-email Disallow: /auth/change-password Disallow: /auth/complete-registration Allow: / # Google - Search, Discover, Imagens, Video, News User-agent: Googlebot Disallow: /admin Disallow: /api Disallow: /__/auth Disallow: /sistema Disallow: /dashboard Disallow: /app Disallow: /finn Disallow: /personal Disallow: /profile Disallow: /configuracoes Disallow: /notificacoes Disallow: /metas Disallow: /transacoes Disallow: /investimentos Disallow: /mercados Disallow: /cartoes Disallow: /milhas Disallow: /relatorios Disallow: /agendamentos Disallow: /arquivos Disallow: /ebook Disallow: /assinaturas Disallow: /billing Disallow: /consumo Disallow: /payment Disallow: /trial Disallow: /business Disallow: /empresa/dashboard Disallow: /empresa/assinaturas Disallow: /portal Disallow: /test- Disallow: /auth/reset-password Disallow: /auth/forgot-password Disallow: /auth/verify-email Disallow: /auth/change-password Disallow: /auth/complete-registration Allow: / # Google / Gemini - Treinamento Gemini e grounding Vertex AI User-agent: Google-Extended Disallow: /admin Disallow: /api Disallow: /__/auth Disallow: /sistema Disallow: /dashboard Disallow: /app Disallow: /finn Disallow: /personal Disallow: /profile Disallow: /configuracoes Disallow: /notificacoes Disallow: /metas Disallow: /transacoes Disallow: /investimentos Disallow: /mercados Disallow: /cartoes Disallow: /milhas Disallow: /relatorios Disallow: /agendamentos Disallow: /arquivos Disallow: /ebook Disallow: /assinaturas Disallow: /billing Disallow: /consumo Disallow: /payment Disallow: /trial Disallow: /business Disallow: /empresa/dashboard Disallow: /empresa/assinaturas Disallow: /portal Disallow: /test- Disallow: /auth/reset-password Disallow: /auth/forgot-password Disallow: /auth/verify-email Disallow: /auth/change-password Disallow: /auth/complete-registration Allow: / # Microsoft / Bing / Copilot - Indice do Bing e experiencias generativas User-agent: bingbot Disallow: /admin Disallow: /api Disallow: /__/auth Disallow: /sistema Disallow: /dashboard Disallow: /app Disallow: /finn Disallow: /personal Disallow: /profile Disallow: /configuracoes Disallow: /notificacoes Disallow: /metas Disallow: /transacoes Disallow: /investimentos Disallow: /mercados Disallow: /cartoes Disallow: /milhas Disallow: /relatorios Disallow: /agendamentos Disallow: /arquivos Disallow: /ebook Disallow: /assinaturas Disallow: /billing Disallow: /consumo Disallow: /payment Disallow: /trial Disallow: /business Disallow: /empresa/dashboard Disallow: /empresa/assinaturas Disallow: /portal Disallow: /test- Disallow: /auth/reset-password Disallow: /auth/forgot-password Disallow: /auth/verify-email Disallow: /auth/change-password Disallow: /auth/complete-registration Allow: / # Microsoft - Geracao de previews User-agent: MicrosoftPreview Disallow: /admin Disallow: /api Disallow: /__/auth Disallow: /sistema Disallow: /dashboard Disallow: /app Disallow: /finn Disallow: /personal Disallow: /profile Disallow: /configuracoes Disallow: /notificacoes Disallow: /metas Disallow: /transacoes Disallow: /investimentos Disallow: /mercados Disallow: /cartoes Disallow: /milhas Disallow: /relatorios Disallow: /agendamentos Disallow: /arquivos Disallow: /ebook Disallow: /assinaturas Disallow: /billing Disallow: /consumo Disallow: /payment Disallow: /trial Disallow: /business Disallow: /empresa/dashboard Disallow: /empresa/assinaturas Disallow: /portal Disallow: /test- Disallow: /auth/reset-password Disallow: /auth/forgot-password Disallow: /auth/verify-email Disallow: /auth/change-password Disallow: /auth/complete-registration Allow: / # Anthropic / Claude - Indexacao para busca do Claude User-agent: Claude-SearchBot Disallow: /admin Disallow: /api Disallow: /__/auth Disallow: /sistema Disallow: /dashboard Disallow: /app Disallow: /finn Disallow: /personal Disallow: /profile Disallow: /configuracoes Disallow: /notificacoes Disallow: /metas Disallow: /transacoes Disallow: /investimentos Disallow: /mercados Disallow: /cartoes Disallow: /milhas Disallow: /relatorios Disallow: /agendamentos Disallow: /arquivos Disallow: /ebook Disallow: /assinaturas Disallow: /billing Disallow: /consumo Disallow: /payment Disallow: /trial Disallow: /business Disallow: /empresa/dashboard Disallow: /empresa/assinaturas Disallow: /portal Disallow: /test- Disallow: /auth/reset-password Disallow: /auth/forgot-password Disallow: /auth/verify-email Disallow: /auth/change-password Disallow: /auth/complete-registration Allow: / # Anthropic / Claude - Acesso em resposta a pedido do usuario User-agent: Claude-User Disallow: /admin Disallow: /api Disallow: /__/auth Disallow: /sistema Disallow: /dashboard Disallow: /app Disallow: /finn Disallow: /personal Disallow: /profile Disallow: /configuracoes Disallow: /notificacoes Disallow: /metas Disallow: /transacoes Disallow: /investimentos Disallow: /mercados Disallow: /cartoes Disallow: /milhas Disallow: /relatorios Disallow: /agendamentos Disallow: /arquivos Disallow: /ebook Disallow: /assinaturas Disallow: /billing Disallow: /consumo Disallow: /payment Disallow: /trial Disallow: /business Disallow: /empresa/dashboard Disallow: /empresa/assinaturas Disallow: /portal Disallow: /test- Disallow: /auth/reset-password Disallow: /auth/forgot-password Disallow: /auth/verify-email Disallow: /auth/change-password Disallow: /auth/complete-registration Allow: / # Anthropic - Coleta publica que pode alimentar treinamento User-agent: ClaudeBot Disallow: /admin Disallow: /api Disallow: /__/auth Disallow: /sistema Disallow: /dashboard Disallow: /app Disallow: /finn Disallow: /personal Disallow: /profile Disallow: /configuracoes Disallow: /notificacoes Disallow: /metas Disallow: /transacoes Disallow: /investimentos Disallow: /mercados Disallow: /cartoes Disallow: /milhas Disallow: /relatorios Disallow: /agendamentos Disallow: /arquivos Disallow: /ebook Disallow: /assinaturas Disallow: /billing Disallow: /consumo Disallow: /payment Disallow: /trial Disallow: /business Disallow: /empresa/dashboard Disallow: /empresa/assinaturas Disallow: /portal Disallow: /test- Disallow: /auth/reset-password Disallow: /auth/forgot-password Disallow: /auth/verify-email Disallow: /auth/change-password Disallow: /auth/complete-registration Allow: / # Perplexity - Indexacao e citacao no Perplexity User-agent: PerplexityBot Disallow: /admin Disallow: /api Disallow: /__/auth Disallow: /sistema Disallow: /dashboard Disallow: /app Disallow: /finn Disallow: /personal Disallow: /profile Disallow: /configuracoes Disallow: /notificacoes Disallow: /metas Disallow: /transacoes Disallow: /investimentos Disallow: /mercados Disallow: /cartoes Disallow: /milhas Disallow: /relatorios Disallow: /agendamentos Disallow: /arquivos Disallow: /ebook Disallow: /assinaturas Disallow: /billing Disallow: /consumo Disallow: /payment Disallow: /trial Disallow: /business Disallow: /empresa/dashboard Disallow: /empresa/assinaturas Disallow: /portal Disallow: /test- Disallow: /auth/reset-password Disallow: /auth/forgot-password Disallow: /auth/verify-email Disallow: /auth/change-password Disallow: /auth/complete-registration Allow: / # Apple - Spotlight, Siri e Safari User-agent: Applebot Disallow: /admin Disallow: /api Disallow: /__/auth Disallow: /sistema Disallow: /dashboard Disallow: /app Disallow: /finn Disallow: /personal Disallow: /profile Disallow: /configuracoes Disallow: /notificacoes Disallow: /metas Disallow: /transacoes Disallow: /investimentos Disallow: /mercados Disallow: /cartoes Disallow: /milhas Disallow: /relatorios Disallow: /agendamentos Disallow: /arquivos Disallow: /ebook Disallow: /assinaturas Disallow: /billing Disallow: /consumo Disallow: /payment Disallow: /trial Disallow: /business Disallow: /empresa/dashboard Disallow: /empresa/assinaturas Disallow: /portal Disallow: /test- Disallow: /auth/reset-password Disallow: /auth/forgot-password Disallow: /auth/verify-email Disallow: /auth/change-password Disallow: /auth/complete-registration Allow: / # Apple - Treinamento de modelos fundacionais Apple User-agent: Applebot-Extended Disallow: /admin Disallow: /api Disallow: /__/auth Disallow: /sistema Disallow: /dashboard Disallow: /app Disallow: /finn Disallow: /personal Disallow: /profile Disallow: /configuracoes Disallow: /notificacoes Disallow: /metas Disallow: /transacoes Disallow: /investimentos Disallow: /mercados Disallow: /cartoes Disallow: /milhas Disallow: /relatorios Disallow: /agendamentos Disallow: /arquivos Disallow: /ebook Disallow: /assinaturas Disallow: /billing Disallow: /consumo Disallow: /payment Disallow: /trial Disallow: /business Disallow: /empresa/dashboard Disallow: /empresa/assinaturas Disallow: /portal Disallow: /test- Disallow: /auth/reset-password Disallow: /auth/forgot-password Disallow: /auth/verify-email Disallow: /auth/change-password Disallow: /auth/complete-registration Allow: / # DuckDuckGo - Indice de busca do DuckDuckGo User-agent: DuckDuckBot Disallow: /admin Disallow: /api Disallow: /__/auth Disallow: /sistema Disallow: /dashboard Disallow: /app Disallow: /finn Disallow: /personal Disallow: /profile Disallow: /configuracoes Disallow: /notificacoes Disallow: /metas Disallow: /transacoes Disallow: /investimentos Disallow: /mercados Disallow: /cartoes Disallow: /milhas Disallow: /relatorios Disallow: /agendamentos Disallow: /arquivos Disallow: /ebook Disallow: /assinaturas Disallow: /billing Disallow: /consumo Disallow: /payment Disallow: /trial Disallow: /business Disallow: /empresa/dashboard Disallow: /empresa/assinaturas Disallow: /portal Disallow: /test- Disallow: /auth/reset-password Disallow: /auth/forgot-password Disallow: /auth/verify-email Disallow: /auth/change-password Disallow: /auth/complete-registration Allow: / # DuckDuckGo - Respostas assistidas por IA com fonte User-agent: DuckAssistBot Disallow: /admin Disallow: /api Disallow: /__/auth Disallow: /sistema Disallow: /dashboard Disallow: /app Disallow: /finn Disallow: /personal Disallow: /profile Disallow: /configuracoes Disallow: /notificacoes Disallow: /metas Disallow: /transacoes Disallow: /investimentos Disallow: /mercados Disallow: /cartoes Disallow: /milhas Disallow: /relatorios Disallow: /agendamentos Disallow: /arquivos Disallow: /ebook Disallow: /assinaturas Disallow: /billing Disallow: /consumo Disallow: /payment Disallow: /trial Disallow: /business Disallow: /empresa/dashboard Disallow: /empresa/assinaturas Disallow: /portal Disallow: /test- Disallow: /auth/reset-password Disallow: /auth/forgot-password Disallow: /auth/verify-email Disallow: /auth/change-password Disallow: /auth/complete-registration Allow: / # ByteDance / TikTok - bloqueado: Crawler agressivo, sem retorno de citacao User-agent: Bytespider Disallow: / Sitemap: https://www.finnextho.com/sitemap.xml