# robots.txt for zydcoerp.com
# Last updated: June 2026

# =============================================
# ALLOW ALL STANDARD CRAWLERS
# =============================================

User-agent: *
Allow: /

# =============================================
# BLOCK WORDPRESS BACKEND & SECURITY PATHS
# =============================================

User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-json/
Disallow: /xmlrpc.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /cgi-bin/
Disallow: /tmp/

# =============================================
# BLOCK WORDPRESS JUNK PAGES
# =============================================

User-agent: *
Disallow: /feed/
Disallow: /trackback/
Disallow: /comments/feed/
Disallow: /tag/
Disallow: /author/
Disallow: /*/embed/
Disallow: /*/print/
Disallow: /?s=
Disallow: /search/

# =============================================
# BLOCK E-COMMERCE & FORM JUNK PAGES
# =============================================

User-agent: *
Disallow: /checkout/
Disallow: /cart/
Disallow: /my-account/
Disallow: /thank-you/
Disallow: /order-confirmation/
Disallow: /wishlist/

# =============================================
# BLOCK DUPLICATE URL PARAMETERS
# =============================================

User-agent: *
Disallow: /*?s=
Disallow: /*?filter*
Disallow: /*?sort*
Disallow: /*?ref=*
Disallow: /*?utm_*
Disallow: /*&utm_*
Disallow: /*?replytocom=*

# =============================================
# AI BOTS — ALLOW CITATION, BLOCK SCRAPING
# =============================================

# OpenAI — ChatGPT Search & Browsing (Allow)
# GPTBot is training scraper — Block
User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Disallow: /

# Google — Gemini & SGE Citation (Allow)
# Google-Extended is training scraper — Block
User-agent: Googlebot
Allow: /

User-agent: Google-Extended
Disallow: /

# Microsoft — Bing Copilot Citation (Allow)
User-agent: Bingbot
Allow: /

# Perplexity — Citation & Answer Engine (Allow)
User-agent: PerplexityBot
Allow: /

# Anthropic — Claude Citation (Allow)
# ClaudeBot is training scraper — Block
User-agent: Claude-User
Allow: /

User-agent: ClaudeBot
Disallow: /

# Meta — AI Citation (Allow)
# Meta training scraper — Block
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Disallow: /

# Apple — Siri & Apple Intelligence (Allow)
User-agent: Applebot
Allow: /

# You.com AI Search (Allow)
User-agent: YouBot
Allow: /

# Common Crawl — Pure Training Dataset (Block)
User-agent: CCBot
Disallow: /

# Bytedance — Pure Scraper (Block)
User-agent: Bytespider
Disallow: /

# =============================================
# CRAWL DELAY FOR AGGRESSIVE SEO TOOL BOTS
# =============================================

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: DotBot
Crawl-delay: 10

User-agent: BLEXBot
Crawl-delay: 10

# =============================================
# SITEMAP
# =============================================

Sitemap: https://zydcoerp.com/sitemap.xml
Sitemap: https://zydcoerp.com/sitemap_index.xml