# ===========================================
# ROBOTS.TXT FOR HUSTLEMARKETERS.COM
# Fully Optimized for AI Visibility + Search Engines
# Last Updated: March 2026
# ===========================================

# -----------------------------------------
# DEFAULT RULES FOR ALL CRAWLERS
# -----------------------------------------

User-agent: *
Allow: /

# Block WordPress admin & system files
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-config.php
Disallow: /readme.html
Disallow: /license.txt
Disallow: /xmlrpc.php
Disallow: /trackback/
Disallow: /cgi-bin/

# Block duplicate content generators
Disallow: /?s=
Disallow: /search/
Disallow: /*?replytocom=
Disallow: /*?preview=true
Disallow: /comments/feed/
Disallow: /*/comments/feed/
Disallow: /author/

# -----------------------------------------
# SEARCH ENGINE CRAWLERS (Full Access)
# -----------------------------------------

User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Yandex
Allow: /

User-agent: Baiduspider
Allow: /

# -----------------------------------------
# AI SEARCH & ASSISTANT CRAWLERS (Full Access)
# Maximum visibility in AI-powered search results
# -----------------------------------------

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic / Claude
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# Google AI (Gemini, AI Overviews)
User-agent: Google-Extended
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Microsoft Copilot / Bing AI
User-agent: bingbot
Allow: /

# Apple Intelligence
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# Amazon Alexa / AI
User-agent: Amazonbot
Allow: /

# You.com AI Search
User-agent: YouBot
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# ByteDance / TikTok
User-agent: Bytespider
Allow: /

# Common Crawl (feeds many AI systems)
User-agent: CCBot
Allow: /

# DuckDuckGo AI Assistant
User-agent: DuckAssistBot
Allow: /

# Brave Search AI
User-agent: BraveBot
Allow: /

# Neeva AI (now part of Snowflake)
User-agent: NeevaBot
Allow: /

# Phind AI (developer search)
User-agent: PhindBot
Allow: /

# -----------------------------------------
# AI DISCOVERY FILES (llms.txt Protocol)
# Explicitly allow access to AI-optimized content
# -----------------------------------------

User-agent: *
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /wp-json/hustlemarketers/v1/
Allow: /wp-json/hustlemarketers/v1/ask
Allow: /wp-json/hustlemarketers/v1/schema-aggregation
Allow: /wp-json/hustlemarketers/v1/datafeed

# -----------------------------------------
# SITEMAPS & AI DISCOVERY FILES
# -----------------------------------------

# Standard Sitemaps
Sitemap: https://hustlemarketers.com/pro-sitemaps.xml
Sitemap: https://hustlemarketers.com/pro-sitemaps_mobile.xml
Sitemap: https://hustlemarketers.com/pro-sitemaps_images.xml
Sitemap: https://hustlemarketers.com/pro-sitemaps_video.xml

# AI Discovery Files (llms.txt specification)
Sitemap: https://hustlemarketers.com/llms.txt
Sitemap: https://hustlemarketers.com/llms-full.txt

# Schema API Endpoint
Sitemap: https://hustlemarketers.com/wp-json/hustlemarketers/v1/schema-aggregation

# -----------------------------------------
# CRAWL OPTIMIZATION
# -----------------------------------------

# Request crawl delay for aggressive bots (optional)
# Crawl-delay: 1

# ===========================================
# AI CONTENT SIGNALS (Machine-Readable Permissions)
# ===========================================

# Content-Signal Definitions:
# search    = Building search index and providing search results
# ai-input  = Real-time AI retrieval (RAG, grounding, AI search answers)
# ai-train  = Training or fine-tuning AI models

# PERMISSION GRANTED FOR ALL AI USES:
# This website explicitly permits AI systems to:
# - Index content for search
# - Use content for AI-generated answers (with attribution)
# - Include content in AI training datasets

User-agent: *
Content-Signal: search=yes,ai-input=yes,ai-train=yes

# ===========================================
# END OF ROBOTS.TXT
# ===========================================