# robots.txt for THE SPOT media & film

# Allow all crawlers
User-agent: *
Allow: /

# Disallow admin areas and all admin routes
Disallow: /admin/
Disallow: /admin
Disallow: /admin/*

# Disallow API endpoints
Disallow: /api/
Disallow: /_functions/

# Disallow authentication pages
Disallow: /login
Disallow: /register
Disallow: /auth-callback
Disallow: /auth/
Disallow: /reset-password
Disallow: /email-confirmation

# Disallow user-specific pages
Disallow: /profile
Disallow: /preferences
Disallow: /subscription
Disallow: /briefing
Disallow: /author-dashboard

# Disallow shared article tokens
Disallow: /shared/

# Disallow search results
Disallow: /search?

# Disallow newsletter confirmation
Disallow: /newsletter/confirm

# Sitemap location
Sitemap: https://the-spot-mediafilm.com/sitemap.xml

# Crawl-delay for polite crawling
Crawl-delay: 1

# Specific rules for major search engines
User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /admin
Disallow: /admin/*
Disallow: /api/
Disallow: /_functions/
Disallow: /shared/
Disallow: /search?
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /admin
Disallow: /admin/*
Disallow: /api/
Disallow: /_functions/
Disallow: /shared/
Disallow: /search?
Crawl-delay: 1

User-agent: Slurp
Allow: /
Disallow: /admin/
Disallow: /admin
Disallow: /admin/*
Disallow: /api/
Disallow: /_functions/
Disallow: /shared/
Disallow: /search?
Crawl-delay: 1