# ================================================================
# robots.txt — mulhim.sa
# Last updated: 2026-05-03
# Strategy: Maximum SEO/GEO/AEO visibility + strict private LMS block
# ================================================================

Sitemap: https://mulhim.sa/sitemap.xml
Sitemap: https://mulhim.sa/sitemap-ar.xml

# ================================================================
# DEFAULT — All crawlers
# ================================================================
User-agent: *

# ── Public pages (always allow) ─────────────────────────────────
Allow: /en/home
Allow: /en/about
Allow: /en/vision-and-mission
Allow: /en/courses
Allow: /en/courses/
Allow: /en/bundles
Allow: /en/bundles/
Allow: /en/services
Allow: /en/contact-us
Allow: /en/register
Allow: /en/supervisory-staff-for-the-training
Allow: /en/instructional_guides
Allow: /en/e-learning-systems
Allow: /en/evaluation-of-courses-satisfaction
Allow: /en/academic-integrity-policy
Allow: /en/technical-support
Allow: /en/privacy-policy
Allow: /en/terms-and-conditions
Allow: /en/return-and-refund-policy
Allow: /en/shipping-policy
Allow: /en/marketing

# Arabic equivalents
Allow: /ar/home
Allow: /ar/about
Allow: /ar/vision-and-mission
Allow: /ar/courses
Allow: /ar/courses/
Allow: /ar/bundles
Allow: /ar/bundles/
Allow: /ar/services
Allow: /ar/contact-us
Allow: /ar/register

# ── Static assets ────────────────────────────────────────────────
Allow: /images/
Allow: /svg/
Allow: /css/
Allow: /js/
Allow: /fonts/

# ── Private LMS areas (hard block) ──────────────────────────────
Disallow: /en/dashboard/
Disallow: /ar/dashboard/
Disallow: /en/admin/
Disallow: /ar/admin/
Disallow: /en/login
Disallow: /ar/login
Disallow: /en/logout
Disallow: /ar/logout
Disallow: /en/profile/
Disallow: /ar/profile/
Disallow: /en/user/
Disallow: /ar/user/
Disallow: /en/student/
Disallow: /ar/student/
Disallow: /en/instructor/
Disallow: /ar/instructor/
Disallow: /en/cart/
Disallow: /ar/cart/
Disallow: /en/checkout/
Disallow: /ar/checkout/
Disallow: /en/orders/
Disallow: /ar/orders/
Disallow: /en/password
Disallow: /ar/password

# ── System / backend paths ───────────────────────────────────────
Disallow: /admin/
Disallow: /api/
Disallow: /ajax/
Disallow: /storage/private/
Disallow: /uploads/private/
Disallow: /cdn-cgi/
Disallow: /.env
Disallow: /.git/

# ── Tracking & session parameters (correct syntax) ───────────────
Disallow: /*?utm_*
Disallow: /*?token=*
Disallow: /*?session=*
Disallow: /*?preview=*

# ── Legacy / low-value paths ─────────────────────────────────────
Disallow: /pages/
Disallow: /tags/
Disallow: /categories/
Disallow: /users/profile
Disallow: /support
Disallow: /blog/3-Laws-to-Become-a-Straight-A-Student


# ================================================================
# GOOGLE — Full SEO authority
# ================================================================
User-agent: Googlebot
Allow: /en/
Allow: /ar/
Disallow: /admin/
Disallow: /api/
Disallow: /cdn-cgi/
Disallow: /en/dashboard/
Disallow: /ar/dashboard/
Disallow: /en/login
Disallow: /ar/login
Disallow: /en/logout
Disallow: /ar/logout
Disallow: /en/profile/
Disallow: /ar/profile/
Disallow: /en/cart/
Disallow: /ar/cart/
Disallow: /en/checkout/
Disallow: /ar/checkout/
Disallow: /*?utm_*
Disallow: /*?token=*
Disallow: /*?session=*

User-agent: Googlebot-Image
Allow: /images/
Allow: /svg/

User-agent: Googlebot-Video
Allow: /

# ================================================================
# BING / MICROSOFT
# ================================================================
User-agent: Bingbot
Crawl-delay: 2
Allow: /en/
Allow: /ar/
Disallow: /admin/
Disallow: /api/
Disallow: /cdn-cgi/
Disallow: /en/dashboard/
Disallow: /ar/dashboard/
Disallow: /en/login
Disallow: /ar/login
Disallow: /en/cart/
Disallow: /ar/cart/
Disallow: /en/checkout/
Disallow: /ar/checkout/
Disallow: /*?utm_*
Disallow: /*?token=*

# ================================================================
# OTHER MAJOR SEARCH ENGINES
# ================================================================
User-agent: Applebot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: PetalBot
Allow: /
Crawl-delay: 2

User-agent: Yandex
Allow: /en/
Allow: /ar/
Disallow: /admin/
Disallow: /api/
Disallow: /en/dashboard/
Disallow: /ar/dashboard/
Disallow: /en/login
Disallow: /ar/login

# ================================================================
# SOCIAL PREVIEW BOTS — Always allow (OGP / rich previews)
# ================================================================
User-agent: LinkedInBot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: Facebot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Telegrambot
Allow: /

# ================================================================
# SEO AUDIT TOOLS — Allowed for internal SEO team use
# These bots respect robots.txt and are used by professionals.
# Re-enable Disallow: / below each if your team stops using them.
# ================================================================
User-agent: AhrefsBot
Allow: /en/
Allow: /ar/
Disallow: /admin/
Disallow: /api/
Disallow: /en/dashboard/
Disallow: /ar/dashboard/
Disallow: /en/login
Disallow: /ar/login

User-agent: SemrushBot
Allow: /en/
Allow: /ar/
Disallow: /admin/
Disallow: /api/
Disallow: /en/dashboard/
Disallow: /ar/dashboard/
Disallow: /en/login
Disallow: /ar/login

User-agent: MJ12bot
Allow: /en/
Allow: /ar/
Disallow: /admin/
Disallow: /api/

User-agent: DotBot
Allow: /en/
Allow: /ar/
Disallow: /admin/
Disallow: /api/

# ================================================================
# AI / LLM CRAWLERS — Full access (AEO priority)
# ================================================================

# OpenAI
User-agent: GPTBot
Allow: /en/
Allow: /ar/
Disallow: /admin/
Disallow: /api/
Disallow: /cdn-cgi/
Disallow: /en/dashboard/
Disallow: /ar/dashboard/
Disallow: /en/login
Disallow: /ar/login
Disallow: /en/profile/
Disallow: /ar/profile/
Disallow: /en/cart/
Disallow: /ar/cart/
Disallow: /en/checkout/
Disallow: /ar/checkout/

User-agent: ChatGPT-User
Allow: /en/
Allow: /ar/
Disallow: /admin/
Disallow: /api/
Disallow: /en/dashboard/
Disallow: /ar/dashboard/
Disallow: /en/login
Disallow: /ar/login
Disallow: /en/cart/
Disallow: /en/checkout/

User-agent: OAI-SearchBot
Allow: /

# Anthropic / Claude
User-agent: ClaudeBot
Allow: /en/
Allow: /ar/
Disallow: /admin/
Disallow: /api/
Disallow: /cdn-cgi/
Disallow: /en/dashboard/
Disallow: /ar/dashboard/
Disallow: /en/login
Disallow: /ar/login
Disallow: /en/cart/
Disallow: /ar/cart/
Disallow: /en/checkout/
Disallow: /ar/checkout/

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google AI (Gemini / AI Overviews)
User-agent: Google-Extended
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /en/dashboard/
Disallow: /ar/dashboard/

# Mistral
User-agent: MistralAI-User
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Amazon Alexa
User-agent: Amazonbot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# ================================================================
# BLOCKED — Exploitative or low-quality crawlers only
# ================================================================

# Mass AI training scrapers (not citation or search bots)
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

# Google Cloud AI training (distinct from Googlebot search)
User-agent: Google-CloudVertexBot
Disallow: /

# Abusive / data harvesting crawlers
User-agent: BLEXBot
Disallow: /

User-agent: SiteAuditBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# ================================================================
# END OF FILE
# ================================================================