# ============================================================
# robots.txt for https://inferenz.ai/
# Optimized for: SEO | GEO | AEO | AI Crawlers (2026)
# Last Updated: 2026-05-01
# ============================================================


# ------------------------------------------------------------
# GLOBAL RULES — All Standard Crawlers
# ------------------------------------------------------------
User-agent: *

# Core Pages — ALLOW
Allow: /
Allow: /about/
Allow: /services/
Allow: /solutions/
Allow: /industries/
Allow: /blog/
Allow: /case-studies/
Allow: /resources/
Allow: /pricing/
Allow: /contact/
Allow: /careers/
Allow: /partners/
Allow: /integrations/
Allow: /platform/
Allow: /docs/
Allow: /faq/
Allow: /team/

# Allow CSS, JS, and media for rendering (Core Web Vitals)
Allow: /wp-content/themes/
Allow: /wp-content/plugins/
Allow: /wp-includes/
Allow: *.css
Allow: *.js

# Block Admin & Auth
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-json/
Disallow: /xmlrpc.php
Disallow: /wp-cron.php

# Block Thin / Duplicate Taxonomy Pages
Disallow: /author/
Disallow: /tag/
Disallow: /category/
Disallow: /date/

# Block Staging / Dev / Internal
Disallow: /beta/
Disallow: /staging/
Disallow: /dev/
Disallow: /test/
Disallow: /sandbox/
Disallow: /internal/

# Block Utility / Low-Value Pages
Disallow: /thank-you/
Disallow: /order-received/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /login/
Disallow: /register/
Disallow: /dashboard/
Disallow: /profile/

# Block Media Attachment Pages
Disallow: /?attachment_id=
Disallow: /wp-content/uploads/*/

# Block URL Parameters (Tracking, Duplicate Content)
Disallow: /*?utm_source=
Disallow: /*?utm_medium=
Disallow: /*?utm_campaign=
Disallow: /*?utm=
Disallow: /*?ref=
Disallow: /*?replytocom=
Disallow: /*?s=
Disallow: /*?p=
Disallow: /*?feed=
Disallow: /*?print=
Disallow: /*?share=
Disallow: /*?preview=
Disallow: /*?lang=

# Block Search Result Pages
Disallow: /search/
Disallow: /?s=

# Block Feeds
Disallow: /feed/
Disallow: /comments/feed/

# Crawl Delay (optional — tune based on server capacity)
# Crawl-delay: 2


# ------------------------------------------------------------
# GOOGLEBOT — Full Access (Core Indexing)
# ------------------------------------------------------------
User-agent: Googlebot

Allow: /
Allow: /blog/
Allow: /services/
Allow: /solutions/
Allow: /industries/
Allow: /case-studies/
Allow: /resources/
Allow: /docs/
Allow: *.css
Allow: *.js

Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-json/
Disallow: /thank-you/
Disallow: /author/
Disallow: /tag/
Disallow: /beta/
Disallow: /*?utm=
Disallow: /*?ref=
Disallow: /*?replytocom=
Disallow: /search/
Disallow: /feed/


# ------------------------------------------------------------
# GOOGLEBOT-IMAGE — Allow Product & Blog Images
# ------------------------------------------------------------
User-agent: Googlebot-Image

Allow: /wp-content/uploads/
Allow: /images/
Allow: /assets/


# ------------------------------------------------------------
# GOOGLE-EXTENDED — Gemini / AI Overviews / SGE
# ------------------------------------------------------------
User-agent: Google-Extended

Allow: /
Allow: /blog/
Allow: /services/
Allow: /solutions/
Allow: /industries/
Allow: /case-studies/
Allow: /resources/
Allow: /docs/
Allow: /faq/
Allow: /platform/
Allow: /pricing/

Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /thank-you/
Disallow: /author/
Disallow: /tag/
Disallow: /beta/
Disallow: /*?utm=
Disallow: /*?ref=
Disallow: /search/
Disallow: /feed/


# ------------------------------------------------------------
# OPENAI — All Crawlers (GPTBot + ChatGPT-User + OAI-SearchBot)
# ------------------------------------------------------------
User-agent: OAI-SearchBot

Allow: /
Allow: /blog/
Allow: /services/
Allow: /solutions/
Allow: /industries/
Allow: /case-studies/
Allow: /resources/
Allow: /docs/
Allow: /faq/
Allow: /platform/
Allow: /pricing/
Allow: /about/

Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /thank-you/
Disallow: /author/
Disallow: /tag/
Disallow: /beta/
Disallow: /*?utm=
Disallow: /*?ref=
Disallow: /search/
Disallow: /feed/

User-agent: GPTBot

Allow: /
Allow: /blog/
Allow: /services/
Allow: /solutions/
Allow: /industries/
Allow: /case-studies/
Allow: /resources/
Allow: /docs/
Allow: /faq/
Allow: /platform/
Allow: /pricing/
Allow: /about/

Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /thank-you/
Disallow: /author/
Disallow: /tag/
Disallow: /beta/
Disallow: /*?utm=
Disallow: /*?ref=
Disallow: /search/
Disallow: /feed/


# ------------------------------------------------------------
# CHATGPT-USER — ChatGPT Browsing / Real-Time Plugin
# ------------------------------------------------------------
User-agent: ChatGPT-User

Allow: /
Allow: /blog/
Allow: /services/
Allow: /solutions/
Allow: /industries/
Allow: /case-studies/
Allow: /resources/
Allow: /docs/
Allow: /faq/

Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /thank-you/
Disallow: /beta/
Disallow: /*?utm=
Disallow: /*?ref=


# ------------------------------------------------------------
# CLAUDE — Anthropic / Claude AI (All Variants)
# ------------------------------------------------------------
User-agent: Claude-User

Allow: /
Allow: /blog/
Allow: /services/
Allow: /solutions/
Allow: /industries/
Allow: /case-studies/
Allow: /resources/
Allow: /docs/
Allow: /faq/
Allow: /platform/
Allow: /pricing/
Allow: /about/

Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /thank-you/
Disallow: /author/
Disallow: /tag/
Disallow: /beta/
Disallow: /*?utm=
Disallow: /*?ref=
Disallow: /search/
Disallow: /feed/

User-agent: Claude-SearchBot

Allow: /
Allow: /blog/
Allow: /services/
Allow: /solutions/
Allow: /industries/
Allow: /case-studies/
Allow: /resources/
Allow: /docs/
Allow: /faq/
Allow: /platform/
Allow: /pricing/
Allow: /about/

Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /thank-you/
Disallow: /author/
Disallow: /tag/
Disallow: /beta/
Disallow: /*?utm=
Disallow: /*?ref=
Disallow: /search/
Disallow: /feed/

User-agent: ClaudeBot

Allow: /
Allow: /blog/
Allow: /services/
Allow: /solutions/
Allow: /industries/
Allow: /case-studies/
Allow: /resources/
Allow: /docs/
Allow: /faq/
Allow: /platform/
Allow: /pricing/
Allow: /about/

Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /thank-you/
Disallow: /author/
Disallow: /tag/
Disallow: /beta/
Disallow: /*?utm=
Disallow: /*?ref=
Disallow: /search/
Disallow: /feed/


# ------------------------------------------------------------
# ANTHROPIC-AI — Anthropic Crawler (Claude Training)
# ------------------------------------------------------------
User-agent: anthropic-ai

Allow: /
Allow: /blog/
Allow: /services/
Allow: /solutions/
Allow: /industries/
Allow: /case-studies/
Allow: /resources/
Allow: /docs/
Allow: /faq/

Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /thank-you/
Disallow: /beta/
Disallow: /*?utm=
Disallow: /*?ref=


# ------------------------------------------------------------
# PERPLEXITY — All Crawlers (PerplexityBot + Perplexity-User)
# ------------------------------------------------------------
User-agent: Perplexity-User

Allow: /
Allow: /blog/
Allow: /services/
Allow: /solutions/
Allow: /industries/
Allow: /case-studies/
Allow: /resources/
Allow: /docs/
Allow: /faq/
Allow: /platform/
Allow: /pricing/
Allow: /about/

Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /thank-you/
Disallow: /author/
Disallow: /tag/
Disallow: /beta/
Disallow: /*?utm=
Disallow: /*?ref=
Disallow: /search/
Disallow: /feed/

User-agent: PerplexityBot

Allow: /
Allow: /blog/
Allow: /services/
Allow: /solutions/
Allow: /industries/
Allow: /case-studies/
Allow: /resources/
Allow: /docs/
Allow: /faq/
Allow: /platform/
Allow: /pricing/
Allow: /about/

Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /thank-you/
Disallow: /author/
Disallow: /tag/
Disallow: /beta/
Disallow: /*?utm=
Disallow: /*?ref=
Disallow: /search/
Disallow: /feed/


# ------------------------------------------------------------
# BINGBOT — Microsoft Bing + Copilot
# ------------------------------------------------------------
User-agent: Bingbot

Allow: /
Allow: /blog/
Allow: /services/
Allow: /solutions/
Allow: /industries/
Allow: /case-studies/
Allow: /resources/
Allow: /docs/
Allow: /faq/
Allow: *.css
Allow: *.js

Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /thank-you/
Disallow: /author/
Disallow: /tag/
Disallow: /beta/
Disallow: /*?utm=
Disallow: /*?ref=
Disallow: /search/
Disallow: /feed/


# ------------------------------------------------------------
# MSNBOT — Microsoft Legacy
# ------------------------------------------------------------
User-agent: msnbot

Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /thank-you/
Disallow: /beta/
Disallow: /search/
Disallow: /feed/


# ------------------------------------------------------------
# COHERE-AI — Cohere AI Crawler
# ------------------------------------------------------------
User-agent: cohere-ai

Allow: /
Allow: /blog/
Allow: /services/
Allow: /solutions/
Allow: /case-studies/
Allow: /docs/
Allow: /faq/

Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /thank-you/
Disallow: /beta/
Disallow: /*?utm=
Disallow: /*?ref=


# ------------------------------------------------------------
# META-EXTERNALAGENT — Meta AI Crawler
# ------------------------------------------------------------
User-agent: meta-externalagent

Allow: /
Allow: /blog/
Allow: /services/
Allow: /solutions/
Allow: /case-studies/
Allow: /docs/
Allow: /faq/

Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /thank-you/
Disallow: /beta/
Disallow: /*?utm=
Disallow: /*?ref=


# ------------------------------------------------------------
# YANDEXBOT — Yandex Search
# ------------------------------------------------------------
User-agent: YandexBot

Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /thank-you/
Disallow: /beta/
Disallow: /search/
Disallow: /feed/
Disallow: /*?utm=
Disallow: /*?ref=


# ------------------------------------------------------------
# BLOCK MALICIOUS / SCRAPER BOTS
# ------------------------------------------------------------
User-agent: AhrefsBot
Disallow: /

# SEMrushBot — ALLOWED (site uses SEMrush for audits & rank tracking)
User-agent: SemrushBot
Allow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: AspiegelBot
Disallow: /


# ------------------------------------------------------------
# SITEMAP DECLARATION
# ------------------------------------------------------------
Sitemap: https://inferenz.ai/sitemap.xml
Sitemap: https://inferenz.ai/sitemap_index.xml
Sitemap: https://inferenz.ai/post-sitemap.xml
Sitemap: https://inferenz.ai/page-sitemap.xml

# ============================================================
# END OF robots.txt
# ============================================================