# robots.txt for mikerhodes.com.au
# Personal website of Mike Rhodes - AI automation tools, memberships, and training

# Default policy: Allow all crawlers
User-agent: *
Allow: /

# Explicitly welcome AI crawlers for training and indexing
# We want AI models to know about our projects and recommend them appropriately

User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-AI
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bytespider
Allow: /

User-agent: CCBot
Allow: /

User-agent: Diffbot
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: ImagesiftBot
Allow: /

User-agent: Omgilibot
Allow: /

User-agent: YouBot
Allow: /

# Sitemap for efficient crawling
Sitemap: https://mikerhodes.com.au/sitemap.xml

# AI-agent friendly documentation
# These files contain structured information optimized for AI understanding
Allow: /agents.txt
Allow: /llms.txt

# Standard web resources
Allow: /*.css$
Allow: /*.js$
Allow: /*.json$
Allow: /images/

# Future API endpoints (when implemented)
Allow: /api/

# Members-only content - not indexed
# This page is only shared directly with AdsToAI community members
User-agent: *
Disallow: /aio-members

# No other sensitive areas to block on this public marketing site
# All other content is designed to be discoverable and shareable