# Spoki Website - robots.txt
# Updated: 2026-02-08 - v1.0.384: Comprehensive indexing fix from Google Search Console audit

User-agent: *
Allow: /

# Explicitly allow favicon and icon files for search engines
Allow: /favicon.ico
Allow: /icon.png
Allow: /icon-192.png
Allow: /apple-icon.png

# Block Next.js static assets with deployment parameters (prevent duplicate indexing)
# These files change hash with every deployment and shouldn't be indexed
Disallow: /_next/static/css/*?dpl=*
Disallow: /_next/static/media/*?dpl=*
Disallow: /_next/static/chunks/*?dpl=*

# Block all query string variations of static assets
Disallow: /_next/static/*?*

# Block font files from indexing (not content, served as static assets)
Disallow: /_next/static/media/*.woff2
Disallow: /_next/static/media/*.woff
Disallow: /_next/static/media/*.ttf

# Block web manifest (not content for search)
Disallow: /site.webmanifest

# Block invalid URL paths discovered in GSC crawl
Disallow: /$
Disallow: /&

# Block API endpoints that shouldn't be indexed (except markdown API)
Disallow: /api/
Allow: /api/markdown/

# Block internal/testing pages
Disallow: /*/internal/

# Block feeds and RSS
Disallow: /*feed/
Disallow: /*feed$
Disallow: /*/feed/
Disallow: /feed/

# Block search result pages with placeholders
Disallow: /*search_term_string*

# Block WordPress JSON API endpoints (from blogapi subdomain references)
Disallow: /wp-json/
Disallow: /*/wp-json/

# Block URLs with tracking parameters that create duplicate content
Disallow: /*?wordfence_lh=*
Disallow: /*?spoki_visitor_id=*
Disallow: /*?spk_mk_id=*
Disallow: /*?post_type=*
Disallow: /*?action=*

# Block WordPress admin/login pages if crawled cross-domain
Disallow: /wp-login.php
Disallow: /wp-admin/

# Yandex-specific: strip UTM and tracking parameters to avoid duplicate crawling
Clean-param: utm_source&utm_medium&utm_campaign&utm_id&utm_content&utm_term
Clean-param: wordfence_lh&hid&spoki_visitor_id&spk_mk_id

# AI Crawlers - Full access to LLM-friendly content
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Anthropic-AI
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

# Search Engine Bots - Full access without crawl-delay
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

# SEO Tool Bots - Full access (removed crawl-delay)
User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: MJ12bot
Allow: /

User-agent: DotBot
Allow: /

User-agent: BLEXBot
Allow: /

User-agent: Screaming Frog SEO Spider
Allow: /

# International Search Engines - Full access (removed crawl-delay)
User-agent: PetalBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# Primary Sitemap Index (recommended for crawlers)
Sitemap: https://spoki.com/sitemap_index.xml

# Legacy sitemaps (kept for backward compatibility)
Sitemap: https://spoki.com/sitemap.xml
Sitemap: https://spoki.com/image-sitemap.xml
Sitemap: https://spoki.com/video-sitemap.xml