# Robots.txt for DermaVision - AI-Powered Skin Analysis
# Updated: 2025-01-15 with latest SEO best practices

# Google and major search engines
User-agent: Googlebot
Allow: /
Allow: /landing
Allow: /privacy
Allow: /terms
Allow: /articles/
Allow: /auth/sign-in
Allow: /auth/sign-up
Allow: /about
Allow: /contact
Allow: /how-to-check-skin-moles
Allow: /ai-skin-cancer-detection-guide
Allow: /what-is-melanoma
Allow: /skin-cancer-prevention-tips
Allow: /how-accurate-is-ai-skin-analysis
Allow: /when-to-see-dermatologist
Allow: /abcde-melanoma-rule
Allow: /skin-mole-analysis-app

# Allow assets for proper indexing
Allow: /assets/
Allow: /_expo/static/
Allow: /favicon.ico
Allow: /manifest.json
Allow: /.well-known/

# Disallow private/authenticated areas
Disallow: /scan
Disallow: /history
Disallow: /profile
Disallow: /(tabs)/
Disallow: /(auth)/reset-password
Disallow: /(auth)/verify-email
Disallow: /(auth)/new-password
Disallow: /admin/
Disallow: /api/
Disallow: /debug

# Bing crawler
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# All other bots
User-agent: *
Allow: /

# Allow crawling of public educational pages
Allow: /landing
Allow: /privacy
Allow: /terms
Allow: /articles/
Allow: /auth/sign-in
Allow: /about
Allow: /contact

# 2025 Voice Search & AI-Optimized Pages
Allow: /how-to-check-skin-moles
Allow: /ai-skin-cancer-detection-guide
Allow: /what-is-melanoma
Allow: /skin-cancer-prevention-tips
Allow: /how-accurate-is-ai-skin-analysis
Allow: /when-to-see-dermatologist
Allow: /abcde-melanoma-rule

# Disallow private/authenticated areas
Disallow: /scan
Disallow: /history
Disallow: /profile
Disallow: /(tabs)/
Disallow: /(auth)/
Disallow: /admin/
Disallow: /api/
Disallow: /debug
Disallow: /+not-found

# Allow common assets for proper page rendering
Allow: /assets/
Allow: /_expo/static/
Allow: /favicon.ico
Allow: /manifest.json
Allow: /.well-known/

# AI Training Bots - Control content usage (2025 update)
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

# Allow Google Extended for Bard/Gemini (we control this)
User-agent: Google-Extended
Allow: /articles/
Allow: /how-to-check-skin-moles
Allow: /ai-skin-cancer-detection-guide
Allow: /what-is-melanoma
Allow: /skin-cancer-prevention-tips
Disallow: /scan
Disallow: /api/
Disallow: /admin/

# Sitemap locations
Sitemap: https://derma-vision.com/sitemap.xml

# General crawl delay (respectful crawling)
Crawl-delay: 1

# Host directive
Host: https://derma-vision.com