# CrunchyTech.net robots.txt
# Allow good crawlers (Google, Bing, etc.) but block AI training bots & junk pages

User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /cgi-bin/
Disallow: /?s=          # Blocks search results pages
Disallow: /search/      # Alternate search URL pattern
Disallow: /?attachment_id=
Disallow: /*?replytocom=
Disallow: /feed/
Disallow: /comments/
Allow: /wp-admin/admin-ajax.php

# Block AI training crawlers
User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

# Sitemap location (Rank Math generates these automatically)
Sitemap: https://crunchytech.net/sitemap_index.xml