#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# https://www.robotstxt.org/robotstxt.html
# robots.txt

# 1. Autorisation specifique pour les moteurs de recherche et IA Search
User-agent: Bingbot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-Web
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: OAI-SearchBot
User-agent: Perplexity-User
Allow: /
Disallow: /recherche*
Disallow: /*/recherche*
Disallow: /*/recherche/*
Disallow: /*/resultats-de-recherche-couverture-tnt*
Disallow: /en/recherche*

# 2. Blocage explicite des robots d'entrainement LLM (Scrapers/Trainers)
User-agent: Amazonbot
User-agent: anthropic-ai
User-agent: Applebot-Extended
User-agent: AwarioRssBot
User-agent: AwarioSmartBot
User-agent: ByteDance
User-agent: Bytespider
User-agent: CCBot
User-agent: Cision
User-agent: ClaudeBot
User-agent: Cohere-ai
User-agent: DataForSeoBot
User-agent: Diffbot
User-agent: Digimind
User-agent: FacebookBot
User-agent: Google-CloudVertexBot
User-agent: Google-Extended
User-agent: GPTBot
User-agent: ia_archiver
User-agent: ia_archiver-web.archive.org
User-agent: ImagesiftBot
User-agent: linkfluence
User-agent: Magpie-crawler
User-agent: Meltwater
User-agent: mention
User-agent: Meta-ExternalAgent
User-agent: Midjourney
User-agent: MistralAI-User
User-agent: Peer39_crawler
User-agent: PerplexityBot
User-agent: TikTokSpider
User-agent: trendybuzz
User-agent: xAI
User-agent: YouBot
Disallow: /

# 3. Regles generales pour tous les autres robots (par defaut)
User-agent: *

# Autoriser les ressources necessaires au rendu (Drupal /core et /profiles)
Disallow: /core/
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg

Disallow: /profiles/
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
Allow: /sites/default/files/

# Interdictions generales (Sécurité & Drupal)
Disallow: /README.txt
Disallow: /web.config

# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /node*
Disallow: /*/node/*
Disallow: /search/
Disallow: /user/register
Disallow: /user/password
Disallow: /user/login
Disallow: /user/logout
Disallow: /media/oembed
Disallow: /*/media/oembed

# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password
Disallow: /index.php/user/register
Disallow: /index.php/user/login
Disallow: /index.php/user/logout
Disallow: /index.php/media/oembed
Disallow: /index.php/*/media/oembed

# Extensions et motifs spécifiques bloqués
Disallow: /*.pdf
Disallow: /*.doc
Disallow: /*.xls
Disallow: /*.odt
Disallow: /*.csv
Disallow: /*.epub
Disallow: /*/etape*
Disallow: /*field_*
Disallow: /*_value_*
Disallow: /archive-liste
Disallow: /*tarteaucitron.js*
Disallow: /archives/
Disallow: /*/emetteur*
Disallow: /*/radios-detail*
Disallow: /offres-emplois/*
Disallow: /taxonomy/term/*
Disallow: /sites/tpee/files/
Disallow: /temps-parole/election/releves/
Disallow: /temps-parole/election/releves-candidats/
Disallow: /repertoire-medias*
Disallow: /*?form_build_id*
Disallow: /recherche*
Disallow: /*/recherche*
Disallow: /*/recherche/*
Disallow: /*/resultats-de-recherche-couverture-tnt*
Disallow: /en/recherche*