# =======================================================
# robots.txt — Pooja Electronics (poojaelectronics.in)
# Updated: 2026-08-18
# Changes this revision:
#   - add_to_wishlist blocked (was unblocked; 1,239 crawl-issue URLs, each a
#     full ~518KB uncached render — the main source of 500/502/503)
#   - add-to-cart patterns widened: /*?add-to-cart= only matched the parameter
#     in first position, so /?x=1&add-to-cart=2 was never blocked
#   - brand/price facets blocked (filter_*, pa_brands, min_price, max_price) —
#     these currently return 404
#   - wc-ajax, remove_item, num-items, attachment_id, req_id blocked
#   - Sitemap: directive repointed to sitemaps.xml (sitemap_index.xml 301s)
# Strategy: Full SEO crawl + AI Search visibility +
#            Server protection + Bot abuse prevention
# =======================================================


# -------------------------------------------------------
# GOOGLEBOT — No crawl delay. Let Google index freely.
# Only block non-indexable WooCommerce junk URLs.
# -------------------------------------------------------
User-agent: Googlebot
Disallow: /wp-admin/
Allow:     /wp-admin/admin-ajax.php
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /order-received/
Disallow: /wc-api/
Disallow: /?add-to-cart=
Disallow: /*?add-to-cart=
Disallow: /*add-to-cart=
Disallow: /*add_to_wishlist=
Disallow: /*remove_item=
Disallow: /*wc-ajax=
Disallow: /*filter_
Disallow: /*pa_brands=
Disallow: /*min_price=
Disallow: /*max_price=
Disallow: /*num-items=
Disallow: /*attachment_id=
Disallow: /*req_id=
Disallow: /*&orderby=
Disallow: /?orderby=
Disallow: /*?orderby=
Disallow: /*?currency=
Disallow: /?s=
Disallow: /search/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /author/
Disallow: /tag/
Disallow: /*?replytocom=
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-includes/


# -------------------------------------------------------
# GOOGLEBOT-IMAGE — Allow all product images
# -------------------------------------------------------
User-agent: Googlebot-Image
Allow: /wp-content/uploads/
Disallow: /wp-admin/

# -------------------------------------------------------
# Block Bad Bot Plugin Settings
# -------------------------------------------------------

# (blackhole rules merged into the ALL OTHER BOTS group below — a second
#  "User-agent: *" group was shadowing it for every parser that stops at the
#  first match, so the junk-path rules were not being applied.)


# -------------------------------------------------------
# BINGBOT — Crawl delay of 10s. Aggressive by default.
# -------------------------------------------------------
User-agent: Bingbot
Crawl-delay: 10
Disallow: /wp-admin/
Allow:     /wp-admin/admin-ajax.php
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /order-received/
Disallow: /wc-api/
Disallow: /?add-to-cart=
Disallow: /*?add-to-cart=
Disallow: /*add-to-cart=
Disallow: /*add_to_wishlist=
Disallow: /*remove_item=
Disallow: /*wc-ajax=
Disallow: /*filter_
Disallow: /*pa_brands=
Disallow: /*min_price=
Disallow: /*max_price=
Disallow: /*num-items=
Disallow: /*attachment_id=
Disallow: /*req_id=
Disallow: /*&orderby=
Disallow: /?orderby=
Disallow: /*?orderby=
Disallow: /*?currency=
Disallow: /?s=
Disallow: /search/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /feed/
Disallow: /author/
Disallow: /tag/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-includes/


# -------------------------------------------------------
# AI SEARCH BOTS — ALLOW (drive real traffic & citations)
# These power AI Overviews, Perplexity answers, ChatGPT
# Search, Claude citations — real visibility & referrals.
# -------------------------------------------------------

# Perplexity.ai — high-quality referral traffic
User-agent: PerplexityBot
Crawl-delay: 5
Disallow: /wp-admin/
Allow:     /wp-admin/admin-ajax.php
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /order-received/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /?add-to-cart=
Disallow: /*?add-to-cart=
Disallow: /*add-to-cart=
Disallow: /*add_to_wishlist=
Disallow: /*remove_item=
Disallow: /*wc-ajax=
Disallow: /*filter_
Disallow: /*pa_brands=
Disallow: /*min_price=
Disallow: /*max_price=
Disallow: /*num-items=
Disallow: /*attachment_id=
Disallow: /*req_id=
Disallow: /*&orderby=
Disallow: /?s=
Disallow: /search/

# ChatGPT browsing / ChatGPT Search
User-agent: ChatGPT-User
Crawl-delay: 5
Disallow: /wp-admin/
Allow:     /wp-admin/admin-ajax.php
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /order-received/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /?add-to-cart=
Disallow: /*?add-to-cart=
Disallow: /*add-to-cart=
Disallow: /*add_to_wishlist=
Disallow: /*remove_item=
Disallow: /*wc-ajax=
Disallow: /*filter_
Disallow: /*pa_brands=
Disallow: /*min_price=
Disallow: /*max_price=
Disallow: /*num-items=
Disallow: /*attachment_id=
Disallow: /*req_id=
Disallow: /*&orderby=
Disallow: /?s=

# OpenAI SearchGPT
User-agent: OAI-SearchBot
Crawl-delay: 5
Disallow: /wp-admin/
Allow:     /wp-admin/admin-ajax.php
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /order-received/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /?add-to-cart=
Disallow: /*?add-to-cart=
Disallow: /*add-to-cart=
Disallow: /*add_to_wishlist=
Disallow: /*remove_item=
Disallow: /*wc-ajax=
Disallow: /*filter_
Disallow: /*pa_brands=
Disallow: /*min_price=
Disallow: /*max_price=
Disallow: /*num-items=
Disallow: /*attachment_id=
Disallow: /*req_id=
Disallow: /*&orderby=

# Google-Extended — powers Google AI Overviews & Gemini
# Blocking this removes you from Google's AI answers
User-agent: Google-Extended
Crawl-delay: 5
Disallow: /wp-admin/
Allow:     /wp-admin/admin-ajax.php
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /order-received/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /?add-to-cart=
Disallow: /*?add-to-cart=
Disallow: /*add-to-cart=
Disallow: /*add_to_wishlist=
Disallow: /*remove_item=
Disallow: /*wc-ajax=
Disallow: /*filter_
Disallow: /*pa_brands=
Disallow: /*min_price=
Disallow: /*max_price=
Disallow: /*num-items=
Disallow: /*attachment_id=
Disallow: /*req_id=
Disallow: /*&orderby=
Disallow: /?s=

# anthropic-ai — powers Claude.ai citations & answers
User-agent: anthropic-ai
Crawl-delay: 5
Disallow: /wp-admin/
Allow:     /wp-admin/admin-ajax.php
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /order-received/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /?add-to-cart=
Disallow: /*?add-to-cart=
Disallow: /*add-to-cart=
Disallow: /*add_to_wishlist=
Disallow: /*remove_item=
Disallow: /*wc-ajax=
Disallow: /*filter_
Disallow: /*pa_brands=
Disallow: /*min_price=
Disallow: /*max_price=
Disallow: /*num-items=
Disallow: /*attachment_id=
Disallow: /*req_id=
Disallow: /*&orderby=
Disallow: /?s=

# You.com AI search
User-agent: YouBot
Crawl-delay: 5
Disallow: /wp-admin/
Allow:     /wp-admin/admin-ajax.php
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /order-received/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /?add-to-cart=
Disallow: /*?add-to-cart=
Disallow: /*add-to-cart=
Disallow: /*add_to_wishlist=
Disallow: /*remove_item=
Disallow: /*wc-ajax=
Disallow: /*filter_
Disallow: /*pa_brands=
Disallow: /*min_price=
Disallow: /*max_price=
Disallow: /*num-items=
Disallow: /*attachment_id=
Disallow: /*req_id=
Disallow: /*&orderby=

# DuckDuckGo AI answers
User-agent: DuckAssistBot
Crawl-delay: 5
Disallow: /wp-admin/
Allow:     /wp-admin/admin-ajax.php
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /order-received/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /?add-to-cart=
Disallow: /*?add-to-cart=
Disallow: /*add-to-cart=
Disallow: /*add_to_wishlist=
Disallow: /*remove_item=
Disallow: /*wc-ajax=
Disallow: /*filter_
Disallow: /*pa_brands=
Disallow: /*min_price=
Disallow: /*max_price=
Disallow: /*num-items=
Disallow: /*attachment_id=
Disallow: /*req_id=
Disallow: /*&orderby=

# Meta AI search (WhatsApp AI, Instagram AI, Meta AI)
User-agent: Meta-ExternalAgent
Crawl-delay: 8
Disallow: /wp-admin/
Allow:     /wp-admin/admin-ajax.php
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /order-received/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /?add-to-cart=
Disallow: /*?add-to-cart=
Disallow: /*add-to-cart=
Disallow: /*add_to_wishlist=
Disallow: /*remove_item=
Disallow: /*wc-ajax=
Disallow: /*filter_
Disallow: /*pa_brands=
Disallow: /*min_price=
Disallow: /*max_price=
Disallow: /*num-items=
Disallow: /*attachment_id=
Disallow: /*req_id=
Disallow: /*&orderby=


# -------------------------------------------------------
# AI TRAINING BOTS — PARTIAL ACCESS
# Allow product pages so your brand appears in AI answers.
# Block /shop/ pricing grid to protect competitive data.
# -------------------------------------------------------

# GPTBot — OpenAI training (also powers search answers)
User-agent: GPTBot
Crawl-delay: 10
Allow:    /product/
Allow:    /product-category/
Allow:    /blog/
Allow:    /about/
Allow:    /contact/
Disallow: /shop/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /order-received/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /?add-to-cart=
Disallow: /*?add-to-cart=
Disallow: /*add-to-cart=
Disallow: /*add_to_wishlist=
Disallow: /*remove_item=
Disallow: /*wc-ajax=
Disallow: /*filter_
Disallow: /*pa_brands=
Disallow: /*min_price=
Disallow: /*max_price=
Disallow: /*num-items=
Disallow: /*attachment_id=
Disallow: /*req_id=
Disallow: /*&orderby=
Disallow: /?orderby=
Disallow: /*?orderby=
Disallow: /?s=
Disallow: /search/

# ClaudeBot — Anthropic training (also powers Claude answers)
User-agent: ClaudeBot
Crawl-delay: 10
Allow:    /product/
Allow:    /product-category/
Allow:    /blog/
Allow:    /about/
Allow:    /contact/
Disallow: /shop/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /order-received/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /?add-to-cart=
Disallow: /*?add-to-cart=
Disallow: /*add-to-cart=
Disallow: /*add_to_wishlist=
Disallow: /*remove_item=
Disallow: /*wc-ajax=
Disallow: /*filter_
Disallow: /*pa_brands=
Disallow: /*min_price=
Disallow: /*max_price=
Disallow: /*num-items=
Disallow: /*attachment_id=
Disallow: /*req_id=
Disallow: /*&orderby=
Disallow: /?orderby=
Disallow: /*?orderby=
Disallow: /?s=
Disallow: /search/


# -------------------------------------------------------
# AGGRESSIVE SEO TOOLS — BLOCK
# Respect robots.txt. Saves bandwidth. Hides your data.
# -------------------------------------------------------
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SiteAuditBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /


# -------------------------------------------------------
# PURE SCRAPERS & DATA HARVESTERS — BLOCK
# No traffic return. Only take your data.
# -------------------------------------------------------

# CommonCrawl — feeds many AI/research datasets, no traffic
User-agent: CCBot
Disallow: /

# TikTok/ByteDance — extremely aggressive, zero referrals
User-agent: Bytespider
Disallow: /

# Data harvester — sells scraped data commercially
User-agent: Omgilibot
Disallow: /

# Diffbot — sells structured data scraped from your site
User-agent: Diffbot
Disallow: /

# Huawei — zero referral traffic to non-Chinese sites
User-agent: PetalBot
Disallow: /

# Screaming Frog cloud crawler (block cloud, allow desktop)
User-agent: ScreamingFrogSEOSpider
Disallow: /

# Archive.org — optional, remove Disallow if you want archiving
User-agent: ia_archiver
Disallow: /


# -------------------------------------------------------
# ALL OTHER BOTS — Crawl delay + block junk paths
# -------------------------------------------------------
User-agent: *
Crawl-delay: 5
Disallow: /*blackhole
Disallow: /?blackhole
Disallow: /wp-admin/
Allow:     /wp-admin/admin-ajax.php
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /order-received/
Disallow: /wc-api/
Disallow: /?add-to-cart=
Disallow: /*?add-to-cart=
Disallow: /*add-to-cart=
Disallow: /*add_to_wishlist=
Disallow: /*remove_item=
Disallow: /*wc-ajax=
Disallow: /*filter_
Disallow: /*pa_brands=
Disallow: /*min_price=
Disallow: /*max_price=
Disallow: /*num-items=
Disallow: /*attachment_id=
Disallow: /*req_id=
Disallow: /*&orderby=
Disallow: /?orderby=
Disallow: /*?orderby=
Disallow: /*?currency=
Disallow: /?s=
Disallow: /search/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /author/
Disallow: /tag/
Disallow: /*?replytocom=
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-includes/
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /wp-toolkit/


# -------------------------------------------------------
# SITEMAP — Critical for Google to find all products
# -------------------------------------------------------
Sitemap: https://poojaelectronics.in/sitemaps.xml