# robots.txt — Cosmic Scribe
# https://cosmoscribe.ru/robots.txt

# Default policy: index public marketing surface, block auth/API/admin.
User-agent: *
Allow: /
Allow: /llms.txt
Allow: /humans.txt
Allow: /.well-known/security.txt
Disallow: /share/
Disallow: /reset-password
Disallow: /admin
Disallow: /api/
Disallow: /broadcasting/
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /forgot-password
Disallow: /email/
Disallow: /storage/

# AI search crawlers — explicitly opted in.
# Each group repeats the disallows because robots.txt does NOT inherit rules
# from the `*` group when a specific User-agent group exists.

User-agent: GPTBot
Allow: /
Disallow: /share/
Disallow: /reset-password
Disallow: /admin
Disallow: /api/
Disallow: /broadcasting/
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /forgot-password
Disallow: /email/
Disallow: /storage/

User-agent: ChatGPT-User
Allow: /
Disallow: /share/
Disallow: /reset-password
Disallow: /admin
Disallow: /api/
Disallow: /broadcasting/
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /forgot-password
Disallow: /email/
Disallow: /storage/

User-agent: OAI-SearchBot
Allow: /
Disallow: /share/
Disallow: /reset-password
Disallow: /admin
Disallow: /api/
Disallow: /broadcasting/
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /forgot-password
Disallow: /email/
Disallow: /storage/

User-agent: ClaudeBot
Allow: /
Disallow: /share/
Disallow: /reset-password
Disallow: /admin
Disallow: /api/
Disallow: /broadcasting/
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /forgot-password
Disallow: /email/
Disallow: /storage/

User-agent: Claude-Web
Allow: /
Disallow: /share/
Disallow: /reset-password
Disallow: /admin
Disallow: /api/
Disallow: /broadcasting/
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /forgot-password
Disallow: /email/
Disallow: /storage/

User-agent: PerplexityBot
Allow: /
Disallow: /share/
Disallow: /reset-password
Disallow: /admin
Disallow: /api/
Disallow: /broadcasting/
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /forgot-password
Disallow: /email/
Disallow: /storage/

User-agent: Perplexity-User
Allow: /
Disallow: /share/
Disallow: /reset-password
Disallow: /admin
Disallow: /api/
Disallow: /broadcasting/
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /forgot-password
Disallow: /email/
Disallow: /storage/

User-agent: Google-Extended
Allow: /
Disallow: /share/
Disallow: /reset-password
Disallow: /admin
Disallow: /api/
Disallow: /broadcasting/
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /forgot-password
Disallow: /email/
Disallow: /storage/

User-agent: GoogleOther
Allow: /
Disallow: /share/
Disallow: /reset-password
Disallow: /admin
Disallow: /api/
Disallow: /broadcasting/
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /forgot-password
Disallow: /email/
Disallow: /storage/

User-agent: Applebot-Extended
Allow: /
Disallow: /share/
Disallow: /reset-password
Disallow: /admin
Disallow: /api/
Disallow: /broadcasting/
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /forgot-password
Disallow: /email/
Disallow: /storage/

User-agent: Bytespider
Allow: /
Disallow: /share/
Disallow: /reset-password
Disallow: /admin
Disallow: /api/
Disallow: /broadcasting/
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /forgot-password
Disallow: /email/
Disallow: /storage/

User-agent: YandexBot
Allow: /
Disallow: /share/
Disallow: /reset-password
Disallow: /admin
Disallow: /api/
Disallow: /broadcasting/
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /forgot-password
Disallow: /email/
Disallow: /storage/
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Disallow: /share/
Disallow: /reset-password
Disallow: /admin
Disallow: /api/
Disallow: /broadcasting/
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /forgot-password
Disallow: /email/
Disallow: /storage/

# Hostile / abusive crawlers known to ignore robots.txt or scrape aggressively.
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

Sitemap: https://cosmoscribe.ru/sitemap.xml