# ============================================
# Sitemaps
# ============================================
Sitemap: https://wolferdawg.io/sitemap_index.xml

# ============================================
# Default rules
# ============================================
User-agent: *
Allow: /

# Block obvious non-public or utility paths
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /admin/
Disallow: /private/
Disallow: /scripts/

# ============================================
# WordPress blog (installed under /blog/)
# ============================================

# Admin and login
Disallow: /blog/wp-admin/
Disallow: /blog/wp-login.php
Allow: /blog/wp-admin/admin-ajax.php

# Common WordPress files that do not need crawling
Disallow: /blog/readme.html
Disallow: /blog/license.txt

# Prevent indexing of internal search results and typical low-value URL patterns
Disallow: /blog/?s=
Disallow: /blog/search/
Disallow: /blog/*?s=
Disallow: /blog/*?replytocom=
Disallow: /blog/*?share=

# Allow assets needed for rendering
Allow: /blog/wp-content/uploads/
Allow: /blog/wp-content/themes/
Allow: /blog/wp-content/*.css$
Allow: /blog/wp-content/*.js$
Allow: /blog/wp-includes/*.css$
Allow: /blog/wp-includes/*.js$

# ============================================
# AI / LLM Crawlers (explicitly allowed for AEO visibility)
# ============================================

# OpenAI
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# Google (Gemini / AI Overviews)
User-agent: Google-Extended
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Meta
User-agent: meta-externalagent
Allow: /

User-agent: meta-externalfetcher
Allow: /

# DuckDuckGo
User-agent: DuckAssistBot
Allow: /

# Amazon / Alexa
User-agent: Amazonbot
Allow: /

# SEO audit tools (keep these)
User-agent: AhrefsBot
Allow: /

User-agent: AhrefsSiteAudit
Allow: /