# SnailSleep - Optimized Robots.txt for Search Engines
# Version: 2.0 - Simplified and Standardized

# ===================================
# Global Search Engines
# ===================================
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# ===================================
# Asian Market Search Engines
# ===================================
User-agent: Yeti
User-agent: Baiduspider
User-agent: Sogou web spider
User-agent: 360Spider
User-agent: Daumoa
User-agent: YandexBot
Allow: /

# ===================================
# Privacy-Focused Search Engines
# ===================================
User-agent: DuckDuckBot
User-agent: StartpageBot
User-agent: EcosiaBot
User-agent: Qwantify
Allow: /

# ===================================
# AI-Powered Search & Assistants
# ===================================
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: Claude-Web
User-agent: Gemini
User-agent: PerplexityBot
User-agent: YouBot
User-agent: DeepSeekBot
User-agent: LlamaBot
User-agent: GrokBot
Allow: /

# ===================================
# Mobile and Voice Search
# ===================================
User-agent: Applebot
User-agent: AlexaBot
Allow: /

# ===================================
# Social Media Crawlers
# ===================================
User-agent: facebookexternalhit
User-agent: Facebot
User-agent: Twitterbot
User-agent: LinkedInBot
Allow: /

# ===================================
# SEO Analysis Tools (Allowed for monitoring)
# ===================================
User-agent: AhrefsBot
User-agent: SemrushBot
Allow: /

# ===================================
# Block Known Spam Crawlers
# ===================================
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
Disallow: /

# ===================================
# Default Rules for All Other Crawlers
# ===================================
User-agent: *

# Allow critical Next.js static resources
Allow: /_next/static/
Allow: /_next/image

# Allow public assets
Allow: /favicon.ico
Allow: /logo.png
Allow: /images/
Allow: /optimized/images/
Allow: /wellness/

# Allow static file types
Allow: /*.css$
Allow: /*.js$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.webp$
Allow: /*.svg$
Allow: /*.avif$

# Disallow admin and build directories
Disallow: /admin/
Disallow: /private/
Disallow: /api/
Disallow: /_next/
Disallow: /.next/
Disallow: /node_modules/

# Block spam referrer URLs
Disallow: /*?ref=live-164.com
Disallow: /*?utm_source=live-164.com
Disallow: /*?referrer=live-164.com

# ===================================
# Sitemap Location
# ===================================
Sitemap: https://www.snailsleep.com/sitemap.xml