# robots.txt for cnicsiminfo.pk
# Optimized for AIO / AEO (AI Optimization / Answer Engine Optimization)
# All AI crawlers and bots ALLOWED for full crawling, indexing, and training

# ---- Default rule for all bots ----
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# ---- OpenAI (ChatGPT, GPTBot) ----
User-agent: GPTBot
Allow: /

# ---- Google AI (Gemini, Bard, Vertex AI) ----
User-agent: Google-Extended
Allow: /

# ---- Anthropic (Claude) ----
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# ---- Perplexity AI ----
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-ai
Allow: /

# ---- Meta AI (Llama) ----
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# ---- Amazon (Alexa, Bedrock, Titan) ----
User-agent: Amazonbot
Allow: /

# ---- Apple (Apple Intelligence, Applebot) ----
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# ---- Microsoft (Copilot, Bing AI) ----
User-agent: Bingbot
Allow: /

# ---- ByteDance (Doubao) ----
User-agent: Bytespider
Allow: /

# ---- Cohere ----
User-agent: cohere-ai
Allow: /

# ---- Common Crawl (open dataset used by many AI models) ----
User-agent: CCBot
Allow: /

# ---- Diffbot ----
User-agent: Diffbot
Allow: /

# ---- Facebook / Meta external ----
User-agent: FacebookBot
Allow: /

# ---- Imagesift ----
User-agent: ImagesiftBot
Allow: /

# ---- Omgili / Omgilibot ----
User-agent: Omgilibot
Allow: /

User-agent: Omgili
Allow: /

# ---- You.com ----
User-agent: YouBot
Allow: /

# ---- Sitemap ----
Sitemap: https://cnicsiminfo.pk/sitemap_index.xml