# robots.txt for tgardsolutions.com
# Last updated: 2026-02-26

# -----------------------------------------------
# Allow all major search engine crawlers
# -----------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

# -----------------------------------------------
# Allow major AI crawlers (GEO optimization)
# -----------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: anthropic-ai
Allow: /

# -----------------------------------------------
# Global rules for all other crawlers
# -----------------------------------------------
User-agent: *

# Protect WordPress admin (allow ajax for functionality)
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Block WordPress core files and system paths
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-json/

# Block login, register, and user-facing WordPress URLs
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php

# Block search results (duplicate/thin content)
Disallow: /?s=
Disallow: /search/

# Block feeds and trackbacks (not useful for SEO)
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /trackback/
Disallow: /*/trackback/
Disallow: /*/feed/

# Block tag and date archive pages (thin/duplicate content)
Disallow: /tag/
Disallow: /date/
Disallow: /author/

# Block utility and legal pages from wasting crawl budget
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/

# -----------------------------------------------
# Sitemaps
# -----------------------------------------------
Sitemap: https://tgardsolutions.com/sitemap.xml
Sitemap: https://tgardsolutions.com/sitemap-llms.xml