# robots.txt for https://codedistrict.com/
# Purpose: Control crawler access for SEO, performance & security
# Platform: WordPress (Rank Math / Yoast compatible)
# Maintained by: Marketing + Dev Teams | Last updated: 04-06-2026

User-agent: *

# Allow key assets required for proper page rendering
Allow: /blog/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/

# Allow specific REST API endpoints
Allow: /wp-json/wp/v2/

# Block sensitive or system directories
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/cache/
Disallow: /wp-json/

# Block sensitive WordPress files
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /license.txt

# Block parameterized & duplicate URLs
Disallow: /*?s=
Disallow: /search/
Disallow: /*replytocom=
Disallow: /*attachment_id=
Disallow: /*preview=true
Disallow: /*feed=
Disallow: /*?rest_route=

# Block low-value pages
Disallow: /tag/
Disallow: /apply-job/
Disallow: /job-details/

# Block feeds
Disallow: /feed/
Disallow: /comments/feed/

# Sitemap
Sitemap: https://codedistrict.com/sitemap_index.xml

User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /